1 /*
2 * Copyright (c) 2022-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #ifndef ECMASCRIPT_MEM_REGION_INL_H
17 #define ECMASCRIPT_MEM_REGION_INL_H
18
19 #include "ecmascript/mem/region.h"
20
21 #include "ecmascript/mem/mem.h"
22 #include "ecmascript/mem/native_area_allocator.h"
23
24 namespace panda::ecmascript {
CreateRememberedSet()25 inline RememberedSet *Region::CreateRememberedSet()
26 {
27 auto bitSize = GCBitset::SizeOfGCBitset(GetCapacity());
28 auto setAddr = nativeAreaAllocator_->Allocate(bitSize + RememberedSet::GCBITSET_DATA_OFFSET);
29 auto ret = new (setAddr) RememberedSet(bitSize);
30 ret->ClearAll();
31 std::atomic_thread_fence(std::memory_order_seq_cst);
32 return ret;
33 }
34
GetOrCreateCrossRegionRememberedSet()35 inline RememberedSet *Region::GetOrCreateCrossRegionRememberedSet()
36 {
37 if (UNLIKELY(crossRegionSet_ == nullptr)) {
38 LockHolder lock(*lock_);
39 if (crossRegionSet_ == nullptr) {
40 crossRegionSet_ = CreateRememberedSet();
41 }
42 }
43 return crossRegionSet_;
44 }
45
CreateNewToEdenRememberedSet()46 ARK_NOINLINE RememberedSet* Region::CreateNewToEdenRememberedSet()
47 {
48 LockHolder lock(*lock_);
49 if (packedData_.newToEdenSet_ == nullptr) {
50 packedData_.newToEdenSet_ = CreateRememberedSet();
51 }
52 return packedData_.newToEdenSet_;
53 }
54
GetOrCreateNewToEdenRememberedSet()55 inline RememberedSet *Region::GetOrCreateNewToEdenRememberedSet()
56 {
57 if (UNLIKELY(packedData_.newToEdenSet_ == nullptr)) {
58 return CreateNewToEdenRememberedSet();
59 }
60 return packedData_.newToEdenSet_;
61 }
62
CreateOldToNewRememberedSet()63 ARK_NOINLINE RememberedSet* Region::CreateOldToNewRememberedSet()
64 {
65 LockHolder lock(*lock_);
66 if (packedData_.oldToNewSet_ == nullptr) {
67 if (sweepingOldToNewRSet_ != nullptr && IsGCFlagSet(RegionGCFlags::HAS_BEEN_SWEPT)) {
68 packedData_.oldToNewSet_ = sweepingOldToNewRSet_;
69 sweepingOldToNewRSet_ = nullptr;
70 } else {
71 packedData_.oldToNewSet_ = CreateRememberedSet();
72 }
73 }
74 return packedData_.oldToNewSet_;
75 }
76
GetOrCreateOldToNewRememberedSet()77 inline RememberedSet* Region::GetOrCreateOldToNewRememberedSet()
78 {
79 if (UNLIKELY(packedData_.oldToNewSet_ == nullptr)) {
80 return CreateOldToNewRememberedSet();
81 }
82 return packedData_.oldToNewSet_;
83 }
84
CreateLocalToShareRememberedSet()85 ARK_NOINLINE RememberedSet* Region::CreateLocalToShareRememberedSet()
86 {
87 LockHolder lock(*lock_);
88 if (packedData_.localToShareSet_ == nullptr) {
89 if (sweepingLocalToShareRSet_ != nullptr && IsGCFlagSet(RegionGCFlags::HAS_BEEN_SWEPT)) {
90 packedData_.localToShareSet_ = sweepingLocalToShareRSet_;
91 sweepingLocalToShareRSet_ = nullptr;
92 } else {
93 packedData_.localToShareSet_ = CreateRememberedSet();
94 }
95 }
96 return packedData_.localToShareSet_;
97 }
98
GetOrCreateLocalToShareRememberedSet()99 inline RememberedSet *Region::GetOrCreateLocalToShareRememberedSet()
100 {
101 if (UNLIKELY(packedData_.localToShareSet_ == nullptr)) {
102 return CreateLocalToShareRememberedSet();
103 }
104 return packedData_.localToShareSet_;
105 }
106
MergeLocalToShareRSetForCS()107 inline void Region::MergeLocalToShareRSetForCS()
108 {
109 if (sweepingLocalToShareRSet_ == nullptr) {
110 return;
111 }
112 if (packedData_.localToShareSet_ == nullptr) {
113 packedData_.localToShareSet_ = sweepingLocalToShareRSet_;
114 sweepingLocalToShareRSet_ = nullptr;
115 } else {
116 packedData_.localToShareSet_->Merge(sweepingLocalToShareRSet_);
117 DeleteSweepingLocalToShareRSet();
118 sweepingLocalToShareRSet_ = nullptr;
119 }
120 }
121
MergeOldToNewRSetForCS()122 inline void Region::MergeOldToNewRSetForCS()
123 {
124 if (sweepingOldToNewRSet_ == nullptr) {
125 return;
126 }
127 if (packedData_.oldToNewSet_ == nullptr) {
128 packedData_.oldToNewSet_ = sweepingOldToNewRSet_;
129 sweepingOldToNewRSet_ = nullptr;
130 } else {
131 packedData_.oldToNewSet_->Merge(sweepingOldToNewRSet_);
132 DeleteSweepingOldToNewRSet();
133 sweepingOldToNewRSet_ = nullptr;
134 }
135 }
136
MergeLocalToShareRSetForCM(RememberedSet *set)137 inline void Region::MergeLocalToShareRSetForCM(RememberedSet *set)
138 {
139 if (packedData_.localToShareSet_ == nullptr) {
140 packedData_.localToShareSet_ = set;
141 } else {
142 packedData_.localToShareSet_->Merge(set);
143 nativeAreaAllocator_->Free(set, set->Size());
144 }
145 }
146
GetMarkGCBitset() const147 inline GCBitset *Region::GetMarkGCBitset() const
148 {
149 return packedData_.markGCBitset_;
150 }
151
AtomicMark(void *address)152 inline bool Region::AtomicMark(void *address)
153 {
154 auto addrPtr = reinterpret_cast<uintptr_t>(address);
155 ASSERT(InRange(addrPtr));
156 return packedData_.markGCBitset_->SetBit<AccessType::ATOMIC>(
157 (addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
158 }
159
NonAtomicMark(void *address)160 inline bool Region::NonAtomicMark(void *address)
161 {
162 ASSERT(IsFreshRegion());
163 auto addrPtr = reinterpret_cast<uintptr_t>(address);
164 ASSERT(InRange(addrPtr));
165 return packedData_.markGCBitset_->SetBit<AccessType::NON_ATOMIC>(
166 (addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
167 }
168
ClearMark(void *address)169 inline void Region::ClearMark(void *address)
170 {
171 auto addrPtr = reinterpret_cast<uintptr_t>(address);
172 ASSERT(InRange(addrPtr));
173 packedData_.markGCBitset_->ClearBit((addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
174 }
175
Test(void *addr) const176 inline bool Region::Test(void *addr) const
177 {
178 auto addrPtr = reinterpret_cast<uintptr_t>(addr);
179 return Test(addrPtr);
180 }
181
Test(uintptr_t addrPtr) const182 inline bool Region::Test(uintptr_t addrPtr) const
183 {
184 ASSERT(InRange(addrPtr));
185 return packedData_.markGCBitset_->TestBit((addrPtr & DEFAULT_REGION_MASK) >> TAGGED_TYPE_SIZE_LOG);
186 }
187
188 // ONLY used for heap verification.
TestNewToEden(uintptr_t addr)189 inline bool Region::TestNewToEden(uintptr_t addr)
190 {
191 ASSERT(InRange(addr));
192 // Only used for heap verification, so donot need to use lock
193 auto set = packedData_.newToEdenSet_;
194 if (set == nullptr) {
195 return false;
196 }
197 return set->TestBit(ToUintPtr(this), addr);
198 }
199
200 // ONLY used for heap verification.
TestOldToNew(uintptr_t addr)201 inline bool Region::TestOldToNew(uintptr_t addr)
202 {
203 ASSERT(InRange(addr));
204 // Only used for heap verification, so donot need to use lock
205 auto set = packedData_.oldToNewSet_;
206 if (set == nullptr) {
207 return false;
208 }
209 return set->TestBit(ToUintPtr(this), addr);
210 }
211
212 // ONLY used for heap verification.
TestLocalToShare(uintptr_t addr)213 inline bool Region::TestLocalToShare(uintptr_t addr)
214 {
215 ASSERT(InRange(addr));
216 // Only used for heap verification, so donot need to use lock
217 if (packedData_.localToShareSet_ == nullptr) {
218 return false;
219 }
220 return packedData_.localToShareSet_->TestBit(ToUintPtr(this), addr);
221 }
222
223 template <typename Visitor>
IterateAllMarkedBits(Visitor visitor) const224 inline void Region::IterateAllMarkedBits(Visitor visitor) const
225 {
226 packedData_.markGCBitset_->IterateMarkedBitsConst(
227 reinterpret_cast<uintptr_t>(this), packedData_.bitsetSize_, visitor);
228 }
229
ClearMarkGCBitset()230 inline void Region::ClearMarkGCBitset()
231 {
232 if (packedData_.markGCBitset_ != nullptr) {
233 packedData_.markGCBitset_->Clear(packedData_.bitsetSize_);
234 }
235 }
236
InsertCrossRegionRSet(uintptr_t addr)237 inline void Region::InsertCrossRegionRSet(uintptr_t addr)
238 {
239 auto set = GetOrCreateCrossRegionRememberedSet();
240 set->Insert(ToUintPtr(this), addr);
241 }
242
AtomicInsertCrossRegionRSet(uintptr_t addr)243 inline void Region::AtomicInsertCrossRegionRSet(uintptr_t addr)
244 {
245 auto set = GetOrCreateCrossRegionRememberedSet();
246 set->AtomicInsert(ToUintPtr(this), addr);
247 }
248
HasLocalToShareRememberedSet() const249 inline bool Region::HasLocalToShareRememberedSet() const
250 {
251 return packedData_.localToShareSet_ != nullptr;
252 }
253
ExtractLocalToShareRSet()254 inline RememberedSet *Region::ExtractLocalToShareRSet()
255 {
256 RememberedSet *set = packedData_.localToShareSet_;
257 packedData_.localToShareSet_ = nullptr;
258 return set;
259 }
260
InsertLocalToShareRSet(uintptr_t addr)261 inline void Region::InsertLocalToShareRSet(uintptr_t addr)
262 {
263 auto set = GetOrCreateLocalToShareRememberedSet();
264 set->Insert(ToUintPtr(this), addr);
265 }
266
267 template <Region::RegionSpaceKind kind>
GetBatchRSetUpdater(uintptr_t addr)268 Region::Updater<kind> Region::GetBatchRSetUpdater(uintptr_t addr)
269 {
270 return Region::Updater<kind>(addr, *this);
271 }
272
AtomicInsertLocalToShareRSet(uintptr_t addr)273 inline void Region::AtomicInsertLocalToShareRSet(uintptr_t addr)
274 {
275 auto set = GetOrCreateLocalToShareRememberedSet();
276 set->AtomicInsert(ToUintPtr(this), addr);
277 }
278
ClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)279 inline void Region::ClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
280 {
281 if (packedData_.localToShareSet_ != nullptr) {
282 packedData_.localToShareSet_->ClearRange(ToUintPtr(this), start, end);
283 }
284 }
285
AtomicClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)286 inline void Region::AtomicClearLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
287 {
288 if (packedData_.localToShareSet_ != nullptr) {
289 packedData_.localToShareSet_->AtomicClearRange(ToUintPtr(this), start, end);
290 }
291 }
292
DeleteLocalToShareRSet()293 inline void Region::DeleteLocalToShareRSet()
294 {
295 if (packedData_.localToShareSet_ != nullptr) {
296 nativeAreaAllocator_->Free(packedData_.localToShareSet_, packedData_.localToShareSet_->Size());
297 packedData_.localToShareSet_ = nullptr;
298 }
299 }
300
AtomicClearSweepingLocalToShareRSetInRange(uintptr_t start, uintptr_t end)301 inline void Region::AtomicClearSweepingLocalToShareRSetInRange(uintptr_t start, uintptr_t end)
302 {
303 if (sweepingLocalToShareRSet_ != nullptr) {
304 sweepingLocalToShareRSet_->AtomicClearRange(ToUintPtr(this), start, end);
305 }
306 }
307
DeleteSweepingLocalToShareRSet()308 inline void Region::DeleteSweepingLocalToShareRSet()
309 {
310 if (sweepingLocalToShareRSet_!= nullptr) {
311 nativeAreaAllocator_->Free(sweepingLocalToShareRSet_, sweepingLocalToShareRSet_->Size());
312 sweepingLocalToShareRSet_ = nullptr;
313 }
314 }
315
316 template <typename Visitor>
IterateAllLocalToShareBits(Visitor visitor)317 inline void Region::IterateAllLocalToShareBits(Visitor visitor)
318 {
319 if (packedData_.localToShareSet_ != nullptr) {
320 packedData_.localToShareSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
321 }
322 }
323
324 template <typename Visitor>
IterateAllCrossRegionBits(Visitor visitor) const325 inline void Region::IterateAllCrossRegionBits(Visitor visitor) const
326 {
327 if (crossRegionSet_ != nullptr) {
328 crossRegionSet_->IterateAllMarkedBitsConst(ToUintPtr(this), visitor);
329 }
330 }
331
ClearCrossRegionRSet()332 inline void Region::ClearCrossRegionRSet()
333 {
334 if (crossRegionSet_ != nullptr) {
335 crossRegionSet_->ClearAll();
336 }
337 }
338
ClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)339 inline void Region::ClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)
340 {
341 if (crossRegionSet_ != nullptr) {
342 crossRegionSet_->ClearRange(ToUintPtr(this), start, end);
343 }
344 }
345
AtomicClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)346 inline void Region::AtomicClearCrossRegionRSetInRange(uintptr_t start, uintptr_t end)
347 {
348 if (crossRegionSet_ != nullptr) {
349 crossRegionSet_->AtomicClearRange(ToUintPtr(this), start, end);
350 }
351 }
352
DeleteCrossRegionRSet()353 inline void Region::DeleteCrossRegionRSet()
354 {
355 if (crossRegionSet_ != nullptr) {
356 nativeAreaAllocator_->Free(crossRegionSet_, crossRegionSet_->Size());
357 crossRegionSet_ = nullptr;
358 }
359 }
360
InsertNewToEdenRSet(uintptr_t addr)361 inline void Region::InsertNewToEdenRSet(uintptr_t addr)
362 {
363 auto set = GetOrCreateNewToEdenRememberedSet();
364 set->Insert(ToUintPtr(this), addr);
365 }
366
AtomicInsertNewToEdenRSet(uintptr_t addr)367 inline void Region::AtomicInsertNewToEdenRSet(uintptr_t addr)
368 {
369 auto set = GetOrCreateNewToEdenRememberedSet();
370 set->AtomicInsert(ToUintPtr(this), addr);
371 }
372
ClearNewToEdenRSet(uintptr_t addr)373 inline void Region::ClearNewToEdenRSet(uintptr_t addr)
374 {
375 auto set = GetOrCreateNewToEdenRememberedSet();
376 set->ClearBit(ToUintPtr(this), addr);
377 }
378
InsertOldToNewRSet(uintptr_t addr)379 inline void Region::InsertOldToNewRSet(uintptr_t addr)
380 {
381 auto set = GetOrCreateOldToNewRememberedSet();
382 set->Insert(ToUintPtr(this), addr);
383 }
384
ClearOldToNewRSet(uintptr_t addr)385 inline void Region::ClearOldToNewRSet(uintptr_t addr)
386 {
387 auto set = GetOrCreateOldToNewRememberedSet();
388 set->ClearBit(ToUintPtr(this), addr);
389 }
390
391 template <typename Visitor>
IterateAllNewToEdenBits(Visitor visitor)392 inline void Region::IterateAllNewToEdenBits(Visitor visitor)
393 {
394 if (packedData_.newToEdenSet_ != nullptr) {
395 packedData_.newToEdenSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
396 }
397 }
398
399 template <typename Visitor>
IterateAllOldToNewBits(Visitor visitor)400 inline void Region::IterateAllOldToNewBits(Visitor visitor)
401 {
402 if (packedData_.oldToNewSet_ != nullptr) {
403 packedData_.oldToNewSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
404 }
405 }
406
407 template <typename Visitor>
AtomicIterateAllSweepingRSetBits(Visitor visitor)408 inline void Region::AtomicIterateAllSweepingRSetBits(Visitor visitor)
409 {
410 if (sweepingOldToNewRSet_ != nullptr) {
411 sweepingOldToNewRSet_->AtomicIterateAllMarkedBits(ToUintPtr(this), visitor);
412 }
413 }
414
415 template <typename Visitor>
IterateAllSweepingRSetBits(Visitor visitor)416 inline void Region::IterateAllSweepingRSetBits(Visitor visitor)
417 {
418 if (sweepingOldToNewRSet_ != nullptr) {
419 sweepingOldToNewRSet_->IterateAllMarkedBits(ToUintPtr(this), visitor);
420 }
421 }
422
GetNewToEdenRSet()423 inline RememberedSet *Region::GetNewToEdenRSet()
424 {
425 return packedData_.newToEdenSet_;
426 }
427
ClearNewToEdenRSet()428 inline void Region::ClearNewToEdenRSet()
429 {
430 if (packedData_.newToEdenSet_ != nullptr) {
431 packedData_.newToEdenSet_->ClearAll();
432 }
433 }
434
ClearNewToEdenRSetInRange(uintptr_t start, uintptr_t end)435 inline void Region::ClearNewToEdenRSetInRange(uintptr_t start, uintptr_t end)
436 {
437 if (packedData_.newToEdenSet_ != nullptr) {
438 packedData_.newToEdenSet_->ClearRange(ToUintPtr(this), start, end);
439 }
440 }
441
DeleteNewToEdenRSet()442 inline void Region::DeleteNewToEdenRSet()
443 {
444 if (packedData_.newToEdenSet_ != nullptr) {
445 nativeAreaAllocator_->Free(packedData_.newToEdenSet_, packedData_.newToEdenSet_->Size());
446 packedData_.newToEdenSet_ = nullptr;
447 }
448 }
449
ClearOldToNewRSet()450 inline void Region::ClearOldToNewRSet()
451 {
452 if (packedData_.oldToNewSet_ != nullptr) {
453 packedData_.oldToNewSet_->ClearAll();
454 }
455 }
456
ClearOldToNewRSetInRange(uintptr_t start, uintptr_t end)457 inline void Region::ClearOldToNewRSetInRange(uintptr_t start, uintptr_t end)
458 {
459 if (packedData_.oldToNewSet_ != nullptr) {
460 packedData_.oldToNewSet_->ClearRange(ToUintPtr(this), start, end);
461 }
462 }
463
DeleteOldToNewRSet()464 inline void Region::DeleteOldToNewRSet()
465 {
466 if (packedData_.oldToNewSet_ != nullptr) {
467 nativeAreaAllocator_->Free(packedData_.oldToNewSet_, packedData_.oldToNewSet_->Size());
468 packedData_.oldToNewSet_ = nullptr;
469 }
470 }
471
AtomicClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)472 inline void Region::AtomicClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)
473 {
474 if (sweepingOldToNewRSet_ != nullptr) {
475 sweepingOldToNewRSet_->AtomicClearRange(ToUintPtr(this), start, end);
476 }
477 }
478
ClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)479 inline void Region::ClearSweepingOldToNewRSetInRange(uintptr_t start, uintptr_t end)
480 {
481 if (sweepingOldToNewRSet_ != nullptr) {
482 sweepingOldToNewRSet_->ClearRange(ToUintPtr(this), start, end);
483 }
484 }
485
DeleteSweepingOldToNewRSet()486 inline void Region::DeleteSweepingOldToNewRSet()
487 {
488 if (sweepingOldToNewRSet_ != nullptr) {
489 nativeAreaAllocator_->Free(sweepingOldToNewRSet_, sweepingOldToNewRSet_->Size());
490 sweepingOldToNewRSet_ = nullptr;
491 }
492 }
493
GetRegionSpaceFlag()494 inline uint8_t Region::GetRegionSpaceFlag()
495 {
496 return packedData_.flags_.spaceFlag_;
497 }
498
499 template <Region::RegionSpaceKind kind>
Flush()500 ARK_INLINE void Region::Updater<kind>::Flush()
501 {
502 uintptr_t updateAddress = 0;
503 std::array<std::bitset<GCBitset::BIT_PER_WORD>, BitSetNum> bitsets = bitsetUpdater_.GetAndResetAll(updateAddress);
504 for (size_t idx = 0; idx < BitSetNum; idx++) {
505 if (bitsets[idx].none()) {
506 continue;
507 }
508 Consume(idx, updateAddress, static_cast<uint32_t>(bitsets[idx].to_ulong()));
509 }
510 }
511
512 template <Region::RegionSpaceKind kind>
Consume(size_t idx, uintptr_t updateAddress, uint32_t mask)513 ARK_INLINE void Region::Updater<kind>::Consume(size_t idx, uintptr_t updateAddress, uint32_t mask)
514 {
515 if (idx == LocalToShareIdx) {
516 auto set = region_.GetOrCreateLocalToShareRememberedSet();
517 set->InsertRange(ToUintPtr(®ion_), updateAddress, mask);
518 }
519 if (kind == InYoung && idx == NewToEdenIdx) {
520 auto set = region_.GetOrCreateNewToEdenRememberedSet();
521 set->InsertRange(ToUintPtr(®ion_), updateAddress, mask);
522 }
523 if (kind == InGeneralOld && idx == OldToNewIdx) {
524 auto set = region_.GetOrCreateOldToNewRememberedSet();
525 set->InsertRange(ToUintPtr(®ion_), updateAddress, mask);
526 }
527 }
528
529 } // namespace panda::ecmascript
530 #endif // ECMASCRIPT_MEM_REGION_INL_H
531