1/*
2 * Copyright (c) 2021-2024 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#ifndef ECMASCRIPT_MEM_BARRIERS_INL_H
17#define ECMASCRIPT_MEM_BARRIERS_INL_H
18
19#include "ecmascript/base/config.h"
20#include "ecmascript/daemon/daemon_thread.h"
21#include "ecmascript/js_tagged_value.h"
22#include "ecmascript/js_thread.h"
23#include "ecmascript/mem/assert_scope.h"
24#include "ecmascript/mem/barriers.h"
25#include "ecmascript/mem/region-inl.h"
26#include "ecmascript/mem/heap.h"
27#include "ecmascript/ecma_vm.h"
28
29namespace panda::ecmascript {
30template<WriteBarrierType writeType = WriteBarrierType::NORMAL>
31static ARK_INLINE void WriteBarrier(const JSThread *thread, void *obj, size_t offset, JSTaggedType value)
32{
33    // NOTE: The logic in WriteBarrier should be synced with CopyObject.
34    // if any new feature/bugfix be added in WriteBarrier, it should also be added to CopyObject.
35    ASSERT(value != JSTaggedValue::VALUE_UNDEFINED);
36    Region *objectRegion = Region::ObjectAddressToRange(static_cast<TaggedObject *>(obj));
37    Region *valueRegion = Region::ObjectAddressToRange(reinterpret_cast<TaggedObject *>(value));
38#if ECMASCRIPT_ENABLE_BARRIER_CHECK
39    if (!thread->GetEcmaVM()->GetHeap()->IsAlive(JSTaggedValue(value).GetHeapObject())) {
40        LOG_FULL(FATAL) << "WriteBarrier checked value:" << value << " is invalid!";
41    }
42#endif
43    uintptr_t slotAddr = ToUintPtr(obj) + offset;
44    if (objectRegion->InGeneralOldSpace() && valueRegion->InGeneralNewSpace()) {
45        // Should align with '8' in 64 and 32 bit platform
46        ASSERT((slotAddr % static_cast<uint8_t>(MemAlignment::MEM_ALIGN_OBJECT)) == 0);
47        objectRegion->InsertOldToNewRSet(slotAddr);
48    } else if (!objectRegion->InSharedHeap() && valueRegion->InSharedSweepableSpace()) {
49#ifndef NDEBUG
50        if (UNLIKELY(JSTaggedValue(value).IsWeakForHeapObject())) {
51            CHECK_NO_LOCAL_TO_SHARE_WEAK_REF_HANDLE;
52        }
53#endif
54        objectRegion->InsertLocalToShareRSet(slotAddr);
55    } else if (valueRegion->InEdenSpace() && objectRegion->InYoungSpace()) {
56        objectRegion->InsertNewToEdenRSet(slotAddr);
57    }
58    ASSERT(!objectRegion->InSharedHeap() || valueRegion->InSharedHeap());
59    if (!valueRegion->InSharedHeap() && thread->IsConcurrentMarkingOrFinished()) {
60        Barriers::Update(thread, slotAddr, objectRegion, reinterpret_cast<TaggedObject *>(value),
61                         valueRegion, writeType);
62        // NOTE: ConcurrentMarking and SharedConcurrentMarking can be enabled at the same time, but a specific value
63        // can't be "not shared heap" and "in SharedSweepableSpace" at the same time. So using "if - else if" is safe.
64    } else if (valueRegion->InSharedSweepableSpace() && thread->IsSharedConcurrentMarkingOrFinished()) {
65        if constexpr (writeType != WriteBarrierType::DESERIALIZE) {
66            Barriers::UpdateShared(thread, reinterpret_cast<TaggedObject *>(value), valueRegion);
67        } else {
68            // In deserialize, will never add references from old object(not allocated by deserialing) to
69            // new object(allocated by deserializing), only two kinds of references(new->old, new->new) will
70            // be added, the old object is considered as serialize_root, and be marked and pushed in
71            // SharedGC::MarkRoots, so just mark all the new object is enough, do not need to push them to
72            // workmanager and recursively visit slots of that.
73            ASSERT(DaemonThread::GetInstance()->IsConcurrentMarkingOrFinished());
74            valueRegion->AtomicMark(JSTaggedValue(value).GetHeapObject());
75        }
76    }
77}
78
79template<bool needWriteBarrier>
80inline void Barriers::SetObject(const JSThread *thread, void *obj, size_t offset, JSTaggedType value)
81{
82    // NOLINTNEXTLINE(clang-analyzer-core.NullDereference)
83    *reinterpret_cast<JSTaggedType *>(reinterpret_cast<uintptr_t>(obj) + offset) = value;
84    if constexpr (needWriteBarrier) {
85        WriteBarrier(thread, obj, offset, value);
86    }
87}
88
89inline void Barriers::SynchronizedSetClass(const JSThread *thread, void *obj, JSTaggedType value)
90{
91    reinterpret_cast<volatile std::atomic<JSTaggedType> *>(obj)->store(value, std::memory_order_release);
92    WriteBarrier(thread, obj, 0, value);
93}
94
95inline void Barriers::SynchronizedSetObject(const JSThread *thread, void *obj, size_t offset, JSTaggedType value,
96                                            bool isPrimitive)
97{
98    reinterpret_cast<volatile std::atomic<JSTaggedType> *>(ToUintPtr(obj) + offset)->store(value,
99        std::memory_order_release);
100    if (!isPrimitive) {
101        WriteBarrier(thread, obj, offset, value);
102    }
103}
104
105static inline void CopyMaybeOverlap(JSTaggedValue* dst, JSTaggedValue* src, size_t count)
106{
107    std::copy_n(src, count, dst);
108}
109
110static inline void CopyNoOverlap(JSTaggedValue* __restrict__ dst, JSTaggedValue* __restrict__ src, size_t count)
111{
112    std::copy_n(src, count, dst);
113}
114
115template <Region::RegionSpaceKind kind>
116ARK_NOINLINE bool BatchBitSet(const JSThread* thread, Region* objectRegion, JSTaggedValue* dst, size_t count);
117
118template <bool needWriteBarrier, bool maybeOverlap>
119void Barriers::CopyObject(const JSThread* thread, JSTaggedValue* dst, JSTaggedValue* src, size_t count)
120{
121    // NOTE: The logic in CopyObject should be synced with WriteBarrier.
122    // if any new feature/bugfix be added in CopyObject, it should also be added to WriteBarrier.
123
124    // step 1. copy from src to dst directly.
125    CopyObjectPrimitive<maybeOverlap>(dst, src, count);
126    if constexpr (!needWriteBarrier) {
127        return;
128    }
129    // step 2. According to object region, update the corresponding bit set batch.
130    Region* objectRegion = Region::ObjectAddressToRange(ToUintPtr(dst));
131    if (!objectRegion->InSharedHeap()) {
132        bool allValueNotHeap = false;
133        if (objectRegion->InYoungSpace()) {
134            allValueNotHeap = BatchBitSet<Region::InYoung>(thread, objectRegion, dst, count);
135        } else if (objectRegion->InGeneralOldSpace()) {
136            allValueNotHeap = BatchBitSet<Region::InGeneralOld>(thread, objectRegion, dst, count);
137        } else {
138            allValueNotHeap = BatchBitSet<Region::Other>(thread, objectRegion, dst, count);
139        }
140        if (allValueNotHeap) {
141            return;
142        }
143    }
144    // step 3. According to marking status, update the barriers.
145    const bool marking = thread->IsConcurrentMarkingOrFinished();
146    const bool sharedMarking = thread->IsSharedConcurrentMarkingOrFinished();
147    if (!marking && !sharedMarking) {
148        return;
149    }
150    for (uint32_t i = 0; i < count; i++) {
151        JSTaggedValue taggedValue = *(dst + i);
152        if (!taggedValue.IsHeapObject()) {
153            continue;
154        }
155        Region* valueRegion = Region::ObjectAddressToRange(taggedValue.GetTaggedObject());
156        ASSERT(!objectRegion->InSharedHeap() || valueRegion->InSharedHeap());
157        if (marking && !valueRegion->InSharedHeap()) {
158            const uintptr_t slotAddr = ToUintPtr(dst) + JSTaggedValue::TaggedTypeSize() * i;
159            Barriers::Update(thread, slotAddr, objectRegion, taggedValue.GetTaggedObject(), valueRegion);
160            // NOTE: ConcurrentMarking and SharedConcurrentMarking can be enabled at the same time, but a specific
161            // value can't be "not shared heap" and "in SharedSweepableSpace" at the same time. So using "if - else if"
162            // is safe.
163        } else if (sharedMarking && valueRegion->InSharedSweepableSpace()) {
164            Barriers::UpdateShared(thread, taggedValue.GetTaggedObject(), valueRegion);
165        }
166    }
167}
168
169template <bool maybeOverlap>
170inline void Barriers::CopyObjectPrimitive(JSTaggedValue* dst, JSTaggedValue* src, size_t count)
171{
172    // Copy Primitive value don't need thread.
173    ASSERT((ToUintPtr(dst) % static_cast<uint8_t>(MemAlignment::MEM_ALIGN_OBJECT)) == 0);
174    if constexpr (maybeOverlap) {
175        CopyMaybeOverlap(dst, src, count);
176    } else {
177        CopyNoOverlap(dst, src, count);
178    }
179}
180} // namespace panda::ecmascript
181
182#endif  // ECMASCRIPT_MEM_BARRIERS_INL_H
183