1/*
2 * Copyright (c) 2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include "ecmascript/mem/heap-inl.h"
17#include "ecmascript/mem/jit_fort.h"
18#include "ecmascript/jit/jit.h"
19#if defined(JIT_ENABLE_CODE_SIGN) && !defined(JIT_FORT_DISABLE)
20#include <sys/ioctl.h>
21#include <sys/prctl.h>
22
23#define XPM_JITFORT_ENABLE_OPCODE 3
24#define XPM_MAGIC 'x'
25#define XPM_SET_JITFORT_ENABLE _IOW(XPM_MAGIC, XPM_JITFORT_ENABLE_OPCODE, unsigned long)
26#endif
27
28namespace panda::ecmascript {
29
30template <>
31FreeListAllocator<MemDesc>::FreeListAllocator(BaseHeap *heap, MemDescPool *pool, JitFort *fort)
32    : memDescPool_(pool), heap_(heap)
33{
34    freeList_ = std::make_unique<FreeObjectList<MemDesc>>(fort);
35}
36
37JitFort::JitFort()
38{
39    jitFortMem_ = PageMap(JIT_FORT_REG_SPACE_MAX,
40                          PageProtectProt(Jit::GetInstance()->IsDisableCodeSign() || !IsResourceAvailable()),
41                          DEFAULT_REGION_SIZE, nullptr, MAP_JITFORT);
42    jitFortBegin_ = reinterpret_cast<uintptr_t>(jitFortMem_.GetMem());
43    jitFortSize_ = JIT_FORT_REG_SPACE_MAX;
44    memDescPool_ = new MemDescPool(jitFortBegin_, jitFortSize_);
45    allocator_ = new FreeListAllocator<MemDesc>(nullptr, memDescPool_, this);
46    InitRegions();
47    LOG_JIT(DEBUG) << "JitFort Begin " << (void *)JitFortBegin() << " end " << (void *)(JitFortBegin() + JitFortSize());
48}
49
50JitFort::~JitFort()
51{
52    constexpr size_t numRegions = JIT_FORT_REG_SPACE_MAX / DEFAULT_REGION_SIZE;
53    for (size_t i = 0; i < numRegions; i++) {
54        regions_[i]->DestroyFreeObjectSets();
55        delete regions_[i];
56    }
57    delete allocator_;
58    delete memDescPool_;
59    PageUnmap(jitFortMem_);
60}
61
62void JitFort::InitRegions()
63{
64    constexpr size_t numRegions = JIT_FORT_REG_SPACE_MAX / DEFAULT_REGION_SIZE;
65    for (size_t i = 0; i < numRegions; i++) {
66        uintptr_t mem = reinterpret_cast<uintptr_t>(jitFortMem_.GetMem()) + i*DEFAULT_REGION_SIZE;
67        uintptr_t end = mem + DEFAULT_REGION_SIZE;
68        JitFortRegion *region = new JitFortRegion(nullptr, mem, end, RegionSpaceFlag::IN_MACHINE_CODE_SPACE,
69            memDescPool_);
70        regions_[i] = region;
71    }
72    AddRegion();
73}
74
75bool JitFort::AddRegion()
76{
77    if (nextFreeRegionIdx_ < MAX_JIT_FORT_REGIONS) {
78        allocator_->AddFree(regions_[nextFreeRegionIdx_]);
79        regionList_.AddNode(regions_[nextFreeRegionIdx_]);
80        nextFreeRegionIdx_++;
81        return true;
82    }
83    return false;
84}
85
86// Fort buf allocation is in multiples of FORT_BUF_ALIGN
87size_t JitFort::FortAllocSize(size_t instrSize)
88{
89    return(AlignUp(instrSize, FORT_BUF_ALIGN));
90}
91
92uintptr_t JitFort::Allocate(MachineCodeDesc *desc)
93{
94    LockHolder lock(mutex_);
95
96    size_t size = FortAllocSize(desc->instructionsSize);
97    auto ret = allocator_->Allocate(size);
98    if (ret == ToUintPtr(nullptr)) {
99        if (AddRegion()) {
100            LOG_JIT(DEBUG) << "JitFort: Allocate - AddRegion";
101            ret = allocator_->Allocate(size);
102        }
103    }
104    if (ret == ToUintPtr(nullptr)) {
105        LOG_JIT(DEBUG) << "JitFort:: Allocate return nullptr for size " << size;
106        return ret;
107    }
108    // Record allocation to keep it from being collected by the next
109    // JitFort::UpdateFreeSpace in case corresponding Machine code object is not
110    // marked for sweep yet by then.
111    ASSERT((ret & FORT_BUF_ADDR_MASK) == 0);
112    MarkJitFortMemAwaitInstall(ret, size);
113    LOG_JIT(DEBUG) << "JitFort:: Allocate " << (void *)ret << " - " << (void *)(ret+size-1) <<
114        " size " << size << " instructionsSize " << desc->instructionsSize;
115    return ret;
116}
117
118// Called by GC thread during Mark to mark Fort buf alive.
119// Encoding details in GCBitset for marking live Fort buffers:
120//   Begin of a live Fort buf:
121//     - 10: a live Jit Fort buf that has been installed
122//     - 11: a live Jit Fort buf that has not been installed yet
123//   End of a live Fort buf:
124//     - 01
125// This encoding requires 4 GCBitset bits to mark a live JitFort
126// buffer, which makes the minimum Fort buffer allocation size 32 bytes.
127void JitFort::MarkJitFortMemAlive(MachineCode *obj)
128{
129    size_t size = FortAllocSize(obj->GetInstructionsSize());
130    uintptr_t addr = obj->GetText();
131    uintptr_t endAddr = addr + size - 1;
132    uint32_t regionIdx = AddrToFortRegionIdx(addr);
133    regions_[regionIdx]->AtomicMark(reinterpret_cast<void *>(addr));
134    regions_[regionIdx]->AtomicMark(reinterpret_cast<void *>(endAddr));
135    LOG_JIT(DEBUG) << "MarkFortMemAlive: addr " << (void *)addr << " size " << size
136        << " regionIdx " << regionIdx << " instructionsSize " << obj->GetInstructionsSize();
137}
138
139// Called by Jit Compile thread during JitFort Allocate to mark Fort buf
140// awaiting install. Need mutex (JitGCLockHolder) for region gcBitset access.
141// See JitFort::MarkJitFortMemAlive comments for mark bit encoding in GC bitset.
142void JitFort::MarkJitFortMemAwaitInstall(uintptr_t addr, size_t size)
143{
144    uintptr_t endAddr = addr + size - 1;
145    uint32_t regionIdx = AddrToFortRegionIdx(addr);
146    regions_[regionIdx]->AtomicMark(reinterpret_cast<void *>(addr));
147    regions_[regionIdx]->AtomicMark(reinterpret_cast<void *>(addr + sizeof(uint64_t))); // mark next bit
148    regions_[regionIdx]->AtomicMark(reinterpret_cast<void *>(endAddr));
149    LOG_JIT(DEBUG) << "MarkFortMemAwaitInstall: addr " << (void *)addr << " size " << size
150        << " regionIdx " << regionIdx;
151}
152
153// Called by JS/Main thread during SafePoint to clear Fort buf AwaitInstall bit
154// See JitFort::MarkJitFortMemAlive comments for mark bit encoding in GC bitset.
155void JitFort::MarkJitFortMemInstalled(MachineCode *obj)
156{
157    size_t size = FortAllocSize(obj->GetInstructionsSize());
158    uintptr_t addr = obj->GetText();
159    uint32_t regionIdx = AddrToFortRegionIdx(addr);
160    regions_[regionIdx]->GetGCBitset()->ClearMark(addr + sizeof(uint64_t)); // clear next bit
161    LOG_JIT(DEBUG) << "MarkFortMemInstalled: addr " << (void *)addr << " size " << size
162        << " regionIdx " << regionIdx << " instructionsSize " << obj->GetInstructionsSize();
163}
164
165uint32_t JitFort::AddrToFortRegionIdx(uint64_t addr)
166{
167    ASSERT(InRange(addr));
168    uint32_t regionIdx = ((addr - JitFortBegin()) & ~(DEFAULT_REGION_MASK)) >> REGION_SIZE_LOG2;
169    ASSERT(regionIdx < MAX_JIT_FORT_REGIONS);
170    return regionIdx;
171}
172
173/*
174 * Called from GC worker thread duing Old/Full GC Sweep (AsyncSweep). Mutex is needed
175 * to ensure exclusive access to JitFort memory by GC thread when it frees JitFort mem
176 * blocks, and by Jit compiled thread when it allocates Fort mem.
177 */
178void JitFort::UpdateFreeSpace()
179{
180    if (!Jit::GetInstance()->IsEnableJitFort()) {
181        return;
182    }
183
184    LockHolder lock(mutex_);
185
186    if (!regionList_.GetLength()) { // LCOV_EXCL_BR_LINE
187        return;
188    }
189    LOG_JIT(DEBUG) << "UpdateFreeSpace enter: " << "Fort space allocated: "
190        << allocator_->GetAllocatedSize()
191        << " available: " << allocator_->GetAvailableSize();
192    allocator_->RebuildFreeList();
193    JitFortRegion *region = regionList_.GetFirst();
194    while (region) {
195        FreeRegion(region);
196        region = region->GetNext();
197    }
198    LOG_JIT(DEBUG) << "UpdateFreeSpace exit: allocator_->GetAvailableSize  "
199        << allocator_->GetAvailableSize();
200}
201
202void JitFort::FreeRegion(JitFortRegion *region)
203{
204    LOG_JIT(DEBUG) << "JitFort FreeRegion " << (void*)(region->GetBegin());
205
206    uintptr_t freeStart = region->GetBegin();
207    region->GetGCBitset()->IterateMarkedBitsConst(
208        region->GetBegin(), region->GetGCBitsetSize(),
209        [this, &region, &freeStart](void *mem, size_t size) {
210            ASSERT(region->InRange(ToUintPtr(mem)));
211            (void) region;
212            uintptr_t freeEnd = ToUintPtr(mem);
213            if (freeStart != freeEnd) {
214                allocator_->Free(freeStart, freeEnd - freeStart, true);
215            }
216            freeStart = freeEnd + size;
217        });
218    uintptr_t freeEnd = region->GetEnd();
219    if (freeStart != freeEnd) {
220        allocator_->Free(freeStart, freeEnd - freeStart, true);
221    }
222}
223
224bool JitFort::InRange(uintptr_t address) const
225{
226    return address >= jitFortBegin_ && (jitFortBegin_ + jitFortSize_) > 1 &&
227        address <= (jitFortBegin_ + jitFortSize_ - 1);
228}
229
230void JitFort::PrepareSweeping()
231{
232    isSweeping_.store(false, std::memory_order_release);
233}
234
235// concurrent sweep - only one of the AsyncSweep task will do JitFort sweep
236void JitFort::AsyncSweep()
237{
238    bool expect = false;
239    if (isSweeping_.compare_exchange_strong(expect, true, std::memory_order_seq_cst)) {
240        LOG_JIT(DEBUG) << "JitFort::AsyncSweep";
241        UpdateFreeSpace();
242    }
243}
244
245// non-concurrent sweep
246void JitFort::Sweep()
247{
248    LOG_JIT(DEBUG) << "JitFort::Sweep";
249    UpdateFreeSpace();
250}
251
252// Used by JitFort::UpdateFreeSpace call path to find corresponding
253// JitFortRegion for a free block in JitFort space, in order to put the blk into
254// the corresponding free set of the JitFortRegion the free block belongs.
255JitFortRegion *JitFort::ObjectAddressToRange(uintptr_t addr)
256{
257    return regions_[AddrToFortRegionIdx(addr)];
258}
259
260void JitFortGCBitset::MarkStartAddr(bool awaitInstall, uintptr_t startAddr, uint32_t index, uint32_t &word)
261{
262    if (!awaitInstall) {
263        ClearMark(startAddr);
264        word &= ~(1u << index);
265    } else {
266        word &= ~(1u << index);
267        word &= ~(1u << (index+1));
268    }
269}
270
271void JitFortGCBitset::MarkEndAddr(bool awaitInstall, uintptr_t endAddr, uint32_t index, uint32_t &word)
272{
273    if (!awaitInstall) {
274        ClearMark(endAddr - 1);
275    }
276    word &= ~(1u << index);
277}
278
279// See JitFort::MarkJitFortMemAlive comments for mark bit encoding in JitFort GC bitset.
280template <typename Visitor>
281void JitFortGCBitset::IterateMarkedBitsConst(uintptr_t regionAddr, size_t bitsetSize, Visitor visitor)
282{
283    bool awaitInstall = false;
284    uintptr_t startAddr = 0;
285    uintptr_t endAddr = 0;
286
287    auto words = Words();
288    uint32_t wordCount = WordCount(bitsetSize);
289    uint32_t index = BIT_PER_WORD;
290    for (uint32_t i = 0; i < wordCount; i++) {
291        uint32_t word = words[i];
292        while (word != 0) {
293            index = static_cast<uint32_t>(__builtin_ctz(word));
294            ASSERT(index < BIT_PER_WORD);
295            if (!startAddr) {
296                startAddr = regionAddr + (index << TAGGED_TYPE_SIZE_LOG);
297                awaitInstall = Test(regionAddr + ((index+1) << TAGGED_TYPE_SIZE_LOG));
298                MarkStartAddr(awaitInstall, startAddr, index, word);
299            } else {
300                endAddr = regionAddr + ((index+1) << TAGGED_TYPE_SIZE_LOG);
301                LOG_JIT(DEBUG) << "Live Jit Mem " << (void *)startAddr << " size " << endAddr-startAddr;
302                visitor(reinterpret_cast<void *>(startAddr), endAddr - startAddr);
303                MarkEndAddr(awaitInstall, endAddr, index, word);
304                awaitInstall = false;
305                startAddr = 0;
306            }
307        }
308        regionAddr += TAGGED_TYPE_SIZE * BIT_PER_WORD;
309    }
310}
311
312bool JitFort::isResourceAvailable_ = true;
313bool JitFort::IsResourceAvailable()
314{
315    return isResourceAvailable_;
316}
317void JitFort::InitJitFortResource()
318{
319#if defined(JIT_ENABLE_CODE_SIGN) && !defined(JIT_FORT_DISABLE)
320    ECMA_BYTRACE_NAME(HITRACE_TAG_ARK, "JIT::InitJitFortResource");
321    if (!Jit::GetInstance()->IsAppJit()) {
322        int fd = open("/dev/xpm", O_RDWR);
323        if (fd < 0) {
324            isResourceAvailable_ = false;
325            LOG_JIT(ERROR) << "Failed to init jitfort resource, open xpm failed: " << strerror(errno);
326            return;
327        }
328        int rc = ioctl(fd, XPM_SET_JITFORT_ENABLE, 0);
329        if (rc < 0) {
330            isResourceAvailable_ = false;
331            LOG_JIT(ERROR) << "Failed to init jitfort resource, enable xpm failed: " << strerror(errno);
332            close(fd);
333            return;
334        }
335        close(fd);
336    }
337    constexpr int prSetJitFort = 0x6a6974;
338    constexpr int jitFortInit = 5;
339    int res = prctl(prSetJitFort, jitFortInit, 0);
340    if (res < 0) {
341        isResourceAvailable_ = false;
342        LOG_JIT(ERROR) << "Failed to init jitfort resource: " << strerror(errno);
343        return;
344    }
345    res = prctl(prSetJitFort, jitFortInit, 0);
346    if (res >= 0 || errno != EEXIST) {
347        isResourceAvailable_ = false;
348        LOG_JIT(ERROR) << "jitfort not support";
349    }
350#endif
351}
352
353MemDescPool::MemDescPool(uintptr_t fortBegin, size_t fortSize)
354    : fortBegin_(fortBegin), fortSize_(fortSize)
355{
356    Expand();
357}
358
359MemDescPool::~MemDescPool()
360{
361    for (const auto& block : memDescBlocks_) {
362        if (block) {
363            free(block);
364        }
365    }
366}
367
368MemDesc *MemDescPool::GetDesc()
369{
370    if (IsEmpty(freeList_)) {
371        Expand();
372    }
373    if (!IsEmpty(freeList_)) { // LCOV_EXCL_BR_LINE
374        MemDesc *res = freeList_;
375        freeList_ = freeList_->GetNext();
376        allocated_++;
377        if (allocated_-returned_ > highwater_) {
378            highwater_ = allocated_ - returned_;
379        }
380        return res;
381    }
382    return nullptr;
383}
384
385void MemDescPool::Expand()
386{
387    void *block = malloc(sizeof(MemDesc) * MEMDESCS_PER_BLOCK);
388    if (block) {
389        memDescBlocks_.push_back(block);
390        for (size_t i = 0; i < MEMDESCS_PER_BLOCK; ++i) {
391            Add(new (ToVoidPtr(reinterpret_cast<uintptr_t>(block) + i*sizeof(MemDesc))) MemDesc());
392        }
393    }
394}
395
396void MemDescPool::Add(MemDesc *desc)
397{
398    desc->SetNext(freeList_);
399    freeList_ = desc;
400}
401
402}  // namespace panda::ecmascript
403