1/*
2 * Copyright (c) 2022-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#ifndef ECMASCRIPT_MEM_REGION_INL_H
17#define ECMASCRIPT_MEM_REGION_INL_H
18
19#include "ecmascript/mem/region.h"
20
21#include "ecmascript/mem/mem.h"
22#include "ecmascript/mem/native_area_allocator.h"
23
24namespace panda::ecmascript {
25inline RememberedSet *Region::CreateRememberedSet()
26{
27    auto bitSize = GCBitset::SizeOfGCBitset(GetCapacity());
28    auto setAddr = nativeAreaAllocator_->Allocate(bitSize + RememberedSet::GCBITSET_DATA_OFFSET);
29    auto ret = new (setAddr) RememberedSet(bitSize);
30    ret->ClearAll();
31    std::atomic_thread_fence(std::memory_order_seq_cst);
32    return ret;
33}
34
35inline RememberedSet *Region::GetOrCreateCrossRegionRememberedSet()
36{
37    if (UNLIKELY(crossRegionSet_ == nullptr)) {
38        LockHolder lock(*lock_);
39        if (crossRegionSet_ == nullptr) {
40            crossRegionSet_ = CreateRememberedSet();
41        }
42    }
43    return crossRegionSet_;
44}
45
46ARK_NOINLINE RememberedSet* Region::CreateNewToEdenRememberedSet()
47{
48    LockHolder lock(*lock_);
49    if (packedData_.newToEdenSet_ == nullptr) {
50        packedData_.newToEdenSet_ = CreateRememberedSet();
51    }
52    return packedData_.newToEdenSet_;
53}
54
55inline RememberedSet *Region::GetOrCreateNewToEdenRememberedSet()
56{
57    if (UNLIKELY(packedData_.newToEdenSet_ == nullptr)) {
58        return CreateNewToEdenRememberedSet();
59    }
60    return packedData_.newToEdenSet_;
61}
62
63ARK_NOINLINE RememberedSet* Region::CreateOldToNewRememberedSet()
64{
65    LockHolder lock(*lock_);
66    if (packedData_.oldToNewSet_ == nullptr) {
67        if (sweepingOldToNewRSet_ != nullptr && IsGCFlagSet(RegionGCFlags::HAS_BEEN_SWEPT)) {
68            packedData_.oldToNewSet_ = sweepingOldToNewRSet_;
69            sweepingOldToNewRSet_ = nullptr;
70        } else {
71            packedData_.oldToNewSet_ = CreateRememberedSet();
72        }
73    }
74    return packedData_.oldToNewSet_;
75}
76
77inline RememberedSet* Region::GetOrCreateOldToNewRememberedSet()
78{
79    if (UNLIKELY(packedData_.oldToNewSet_ == nullptr)) {
80        return CreateOldToNewRememberedSet();
81    }
82    return packedData_.oldToNewSet_;
83}
84
85ARK_NOINLINE RememberedSet* Region::CreateLocalToShareRememberedSet()
86{
87    LockHolder lock(*lock_);
88    if (packedData_.localToShareSet_ == nullptr) {
89        if (sweepingLocalToShareRSet_ != nullptr && IsGCFlagSet(RegionGCFlags::HAS_BEEN_SWEPT)) {
90            packedData_.localToShareSet_ = sweepingLocalToShareRSet_;
91            sweepingLocalToShareRSet_ = nullptr;
92        } else {
93            packedData_.localToShareSet_ = CreateRememberedSet();
94        }
95    }
96    return packedData_.localToShareSet_;
97}
98
99inline RememberedSet *Region::GetOrCreateLocalToShareRememberedSet()
100{
101    if (UNLIKELY(packedData_.localToShareSet_ == nullptr)) {
102        return CreateLocalToShareRememberedSet();
103    }
104    return packedData_.localToShareSet_;
105}
106
107inline void Region::MergeLocalToShareRSetForCS()
108{
109    if (sweepingLocalToShareRSet_ == nullptr) {
110        return;
111    }
112    if (packedData_.localToShareSet_ == nullptr) {
113        packedData_.localToShareSet_ = sweepingLocalToShareRSet_;
114        sweepingLocalToShareRSet_ = nullptr;
115    } else {
116        packedData_.localToShareSet_->Merge(sweepingLocalToShareRSet_);
117        DeleteSweepingLocalToShareRSet();
118        sweepingLocalToShareRSet_ = nullptr;
119    }
120}
121
122inline void Region::MergeOldToNewRSetForCS()
123{
124    if (sweepingOldToNewRSet_ == nullptr) {
125        return;
126    }
127    if (packedData_.oldToNewSet_ == nullptr) {
128        packedData_.oldToNewSet_ = sweepingOldToNewRSet_;
129        sweepingOldToNewRSet_   = nullptr;
130    } else {
131        packedData_.oldToNewSet_->Merge(sweepingOldToNewRSet_);
132        DeleteSweepingOldToNewRSet();
133        sweepingOldToNewRSet_ = nullptr;
134    }
135}
136
137inline void Region::MergeLocalToShareRSetForCM(RememberedSet *set)
138{
139    if (packedData_.localToShareSet_ == nullptr) {
140        packedData_.localToShareSet_ = set;
141    } else {
142        packedData_.localToShareSet_->Merge(set);
143        nativeAreaAllocator_->Free(set, set->Size());
144    }
145}
146
147inline GCBitset *Region::GetMarkGCBitset() const
148{
149    return packedData_.markGCBitset_;
150}
151
152inline bool Region::AtomicMark(void *address)
153{
154    auto addrPtr = reinterpret_cast<uintptr_t>(address);
155    ASSERT(InRange(addrPtr));
156    return packedData_.markGCBitset_->SetBit<AccessType::ATOMIC>(
157        (addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
158}
159
160inline bool Region::NonAtomicMark(void *address)
161{
162    ASSERT(IsFreshRegion());
163    auto addrPtr = reinterpret_cast<uintptr_t>(address);
164    ASSERT(InRange(addrPtr));
165    return packedData_.markGCBitset_->SetBit<AccessType::NON_ATOMIC>(
166        (addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
167}
168
169inline void Region::ClearMark(void *address)
170{
171    auto addrPtr = reinterpret_cast<uintptr_t>(address);
172    ASSERT(InRange(addrPtr));
173    packedData_.markGCBitset_->ClearBit((addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
174}
175
176inline bool Region::Test(void *addr) const
177{
178    auto addrPtr = reinterpret_cast<uintptr_t>(addr);
179    return Test(addrPtr);
180}
181
182inline bool Region::Test(uintptr_t addrPtr) const
183{
184    ASSERT(InRange(addrPtr));
185    return packedData_.markGCBitset_->TestBit((addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
186}
187
188// ONLY used for heap verification.
189inline bool Region::TestNewToEden(uintptr_t addr)
190{
191    ASSERT(InRange(addr));
192    // Only used for heap verification, so donot need to use lock
193    auto set = packedData_.newToEdenSet_;
194    if (set == nullptr) {
195        return false;
196    }
197    return set->TestBit(ToUintPtr(this), addr);
198}
199
200// ONLY used for heap verification.
201inline bool Region::TestOldToNew(uintptr_t addr)
202{
203    ASSERT(InRange(addr));
204    // Only used for heap verification, so donot need to use lock
205    auto set = packedData_.oldToNewSet_;
206    if (set == nullptr) {
207        return false;
208    }
209    return set->TestBit(ToUintPtr(this), addr);
210}
211
212// ONLY used for heap verification.
213inline bool Region::TestLocalToShare(uintptr_t addr)
214{
215    ASSERT(InRange(addr));
216    // Only used for heap verification, so donot need to use lock
217    if (packedData_.localToShareSet_ == nullptr) {
218        return false;
219    }
220    return packedData_.localToShareSet_->TestBit(ToUintPtr(this), addr);
221}
222
223template <typename Visitor>
224inline void Region::IterateAllMarkedBits(Visitor visitor) const
225{
226    packedData_.markGCBitset_->IterateMarkedBitsConst(
227        reinterpret_cast<uintptr_t>(this), packedData_.bitsetSize_, visitor);
228}
229
230inline void Region::ClearMarkGCBitset()
231{
232    if (packedData_.markGCBitset_ != nullptr) {
233        packedData_.markGCBitset_->Clear(packedData_.bitsetSize_);
234    }
235}
236
237inline void Region::InsertCrossRegionRSet(uintptr_t addr)
238{
239    auto set = GetOrCreateCrossRegionRememberedSet();
240    set->Insert(ToUintPtr(this), addr);
241}
242
243inline void Region::AtomicInsertCrossRegionRSet(uintptr_t addr)
244{
245    auto set = GetOrCreateCrossRegionRememberedSet();
246    set->AtomicInsert(ToUintPtr(this), addr);
247}
248
249inline bool Region::HasLocalToShareRememberedSet() const
250{
251    return packedData_.localToShareSet_ != nullptr;
252}
253
254inline RememberedSet *Region::ExtractLocalToShareRSet()
255{
256    RememberedSet *set = packedData_.localToShareSet_;
257    packedData_.localToShareSet_ = nullptr;
258    return set;
259}
260
261inline void Region::InsertLocalToShareRSet(uintptr_t addr)
262{
263    auto set = GetOrCreateLocalToShareRememberedSet();
264    set->Insert(ToUintPtr(this), addr);
265}
266
267template <Region::RegionSpaceKind kind>
268Region::Updater<kind> Region::GetBatchRSetUpdater(uintptr_t addr)
269{
270    return Region::Updater<kind>(addr, *this);
271}
272
273inline void Region::AtomicInsertLocalToShareRSet(uintptr_t addr)
274{
275    auto set = GetOrCreateLocalToShareRememberedSet();
276    set->AtomicInsert(ToUintPtr(this), addr);
277}
278
279inline void Region::ClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
280{
281    if (packedData_.localToShareSet_ != nullptr) {
282        packedData_.localToShareSet_->ClearRange(ToUintPtr(this), start, end);
283    }
284}
285
286inline void Region::AtomicClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
287{
288    if (packedData_.localToShareSet_ != nullptr) {
289        packedData_.localToShareSet_->AtomicClearRange(ToUintPtr(this), start, end);
290    }
291}
292
293inline void Region::DeleteLocalToShareRSet()
294{
295    if (packedData_.localToShareSet_ != nullptr) {
296        nativeAreaAllocator_->Free(packedData_.localToShareSet_, packedData_.localToShareSet_->Size());
297        packedData_.localToShareSet_ = nullptr;
298    }
299}
300
301inline void Region::AtomicClearSweepingLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
302{
303    if (sweepingLocalToShareRSet_ != nullptr) {
304        sweepingLocalToShareRSet_->AtomicClearRange(ToUintPtr(this), start, end);
305    }
306}
307
308inline void Region::DeleteSweepingLocalToShareRSet()
309{
310    if (sweepingLocalToShareRSet_!= nullptr) {
311        nativeAreaAllocator_->Free(sweepingLocalToShareRSet_, sweepingLocalToShareRSet_->Size());
312        sweepingLocalToShareRSet_ = nullptr;
313    }
314}
315
316template <typename Visitor>
317inline void Region::IterateAllLocalToShareBits(Visitor visitor)
318{
319    if (packedData_.localToShareSet_ != nullptr) {
320        packedData_.localToShareSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
321    }
322}
323
324template <typename Visitor>
325inline void Region::IterateAllCrossRegionBits(Visitor visitor) const
326{
327    if (crossRegionSet_ != nullptr) {
328        crossRegionSet_->IterateAllMarkedBitsConst(ToUintPtr(this), visitor);
329    }
330}
331
332inline void Region::ClearCrossRegionRSet()
333{
334    if (crossRegionSet_ != nullptr) {
335        crossRegionSet_->ClearAll();
336    }
337}
338
339inline void Region::ClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)
340{
341    if (crossRegionSet_ != nullptr) {
342        crossRegionSet_->ClearRange(ToUintPtr(this), start, end);
343    }
344}
345
346inline void Region::AtomicClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)
347{
348    if (crossRegionSet_ != nullptr) {
349        crossRegionSet_->AtomicClearRange(ToUintPtr(this), start, end);
350    }
351}
352
353inline void Region::DeleteCrossRegionRSet()
354{
355    if (crossRegionSet_ != nullptr) {
356        nativeAreaAllocator_->Free(crossRegionSet_, crossRegionSet_->Size());
357        crossRegionSet_ = nullptr;
358    }
359}
360
361inline void Region::InsertNewToEdenRSet(uintptr_t addr)
362{
363    auto set = GetOrCreateNewToEdenRememberedSet();
364    set->Insert(ToUintPtr(this), addr);
365}
366
367inline void Region::AtomicInsertNewToEdenRSet(uintptr_t addr)
368{
369    auto set = GetOrCreateNewToEdenRememberedSet();
370    set->AtomicInsert(ToUintPtr(this), addr);
371}
372
373inline void Region::ClearNewToEdenRSet(uintptr_t addr)
374{
375    auto set = GetOrCreateNewToEdenRememberedSet();
376    set->ClearBit(ToUintPtr(this), addr);
377}
378
379inline void Region::InsertOldToNewRSet(uintptr_t addr)
380{
381    auto set = GetOrCreateOldToNewRememberedSet();
382    set->Insert(ToUintPtr(this), addr);
383}
384
385inline void Region::ClearOldToNewRSet(uintptr_t addr)
386{
387    auto set = GetOrCreateOldToNewRememberedSet();
388    set->ClearBit(ToUintPtr(this), addr);
389}
390
391template <typename Visitor>
392inline void Region::IterateAllNewToEdenBits(Visitor visitor)
393{
394    if (packedData_.newToEdenSet_ != nullptr) {
395        packedData_.newToEdenSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
396    }
397}
398
399template <typename Visitor>
400inline void Region::IterateAllOldToNewBits(Visitor visitor)
401{
402    if (packedData_.oldToNewSet_ != nullptr) {
403        packedData_.oldToNewSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
404    }
405}
406
407template <typename Visitor>
408inline void Region::AtomicIterateAllSweepingRSetBits(Visitor visitor)
409{
410    if (sweepingOldToNewRSet_ != nullptr) {
411        sweepingOldToNewRSet_->AtomicIterateAllMarkedBits(ToUintPtr(this), visitor);
412    }
413}
414
415template <typename Visitor>
416inline void Region::IterateAllSweepingRSetBits(Visitor visitor)
417{
418    if (sweepingOldToNewRSet_ != nullptr) {
419        sweepingOldToNewRSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
420    }
421}
422
423inline RememberedSet *Region::GetNewToEdenRSet()
424{
425    return  packedData_.newToEdenSet_;
426}
427
428inline void Region::ClearNewToEdenRSet()
429{
430    if (packedData_.newToEdenSet_ != nullptr) {
431        packedData_.newToEdenSet_->ClearAll();
432    }
433}
434
435inline void Region::ClearNewToEdenRSetInRange(uintptr_t start, uintptr_t end)
436{
437    if (packedData_.newToEdenSet_ != nullptr) {
438        packedData_.newToEdenSet_->ClearRange(ToUintPtr(this), start, end);
439    }
440}
441
442inline void Region::DeleteNewToEdenRSet()
443{
444    if (packedData_.newToEdenSet_ != nullptr) {
445        nativeAreaAllocator_->Free(packedData_.newToEdenSet_, packedData_.newToEdenSet_->Size());
446        packedData_.newToEdenSet_ = nullptr;
447    }
448}
449
450inline void Region::ClearOldToNewRSet()
451{
452    if (packedData_.oldToNewSet_ != nullptr) {
453        packedData_.oldToNewSet_->ClearAll();
454    }
455}
456
457inline void Region::ClearOldToNewRSetInRange(uintptr_t start, uintptr_t end)
458{
459    if (packedData_.oldToNewSet_ != nullptr) {
460        packedData_.oldToNewSet_->ClearRange(ToUintPtr(this), start, end);
461    }
462}
463
464inline void Region::DeleteOldToNewRSet()
465{
466    if (packedData_.oldToNewSet_ != nullptr) {
467        nativeAreaAllocator_->Free(packedData_.oldToNewSet_, packedData_.oldToNewSet_->Size());
468        packedData_.oldToNewSet_ = nullptr;
469    }
470}
471
472inline void Region::AtomicClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)
473{
474    if (sweepingOldToNewRSet_ != nullptr) {
475        sweepingOldToNewRSet_->AtomicClearRange(ToUintPtr(this), start, end);
476    }
477}
478
479inline void Region::ClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)
480{
481    if (sweepingOldToNewRSet_ != nullptr) {
482        sweepingOldToNewRSet_->ClearRange(ToUintPtr(this), start, end);
483    }
484}
485
486inline void Region::DeleteSweepingOldToNewRSet()
487{
488    if (sweepingOldToNewRSet_ != nullptr) {
489        nativeAreaAllocator_->Free(sweepingOldToNewRSet_, sweepingOldToNewRSet_->Size());
490        sweepingOldToNewRSet_ = nullptr;
491    }
492}
493
494inline uint8_t Region::GetRegionSpaceFlag()
495{
496    return packedData_.flags_.spaceFlag_;
497}
498
499template <Region::RegionSpaceKind kind>
500ARK_INLINE void Region::Updater<kind>::Flush()
501{
502    uintptr_t updateAddress = 0;
503    std::array<std::bitset<GCBitset::BIT_PER_WORD>, BitSetNum> bitsets = bitsetUpdater_.GetAndResetAll(updateAddress);
504    for (size_t idx = 0; idx < BitSetNum; idx++) {
505        if (bitsets[idx].none()) {
506            continue;
507        }
508        Consume(idx, updateAddress, static_cast<uint32_t>(bitsets[idx].to_ulong()));
509    }
510}
511
512template <Region::RegionSpaceKind kind>
513ARK_INLINE void Region::Updater<kind>::Consume(size_t idx, uintptr_t updateAddress, uint32_t mask)
514{
515    if (idx == LocalToShareIdx) {
516        auto set = region_.GetOrCreateLocalToShareRememberedSet();
517        set->InsertRange(ToUintPtr(&region_), updateAddress, mask);
518    }
519    if (kind == InYoung && idx == NewToEdenIdx) {
520        auto set = region_.GetOrCreateNewToEdenRememberedSet();
521        set->InsertRange(ToUintPtr(&region_), updateAddress, mask);
522    }
523    if (kind == InGeneralOld && idx == OldToNewIdx) {
524        auto set = region_.GetOrCreateOldToNewRememberedSet();
525        set->InsertRange(ToUintPtr(&region_), updateAddress, mask);
526    }
527}
528
529} // namespace panda::ecmascript
530#endif  // ECMASCRIPT_MEM_REGION_INL_H
531