1 /**
2  * Copyright (c) 2024 Huawei Device Co., Ltd.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef PANDA_RUNTIME_MEM_TLAB_H
16 #define PANDA_RUNTIME_MEM_TLAB_H
17 
18 #include "libpandabase/utils/logger.h"
19 #include "libpandabase/mem/mem.h"
20 #include "libpandabase/mem/pool_map.h"
21 #include "libpandabase/mem/mem_range.h"
22 
23 namespace ark::mem {
24 
25 // NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
26 #define LOG_TLAB_ALLOCATOR(level) LOG(level, ALLOC) << "TLAB: "
27 
28 #ifdef NDEBUG
29 static constexpr bool PANDA_TRACK_TLAB_ALLOCATIONS = false;
30 #else
31 static constexpr bool PANDA_TRACK_TLAB_ALLOCATIONS = true;
32 #endif
33 // Current TLAB structure looks like that:
34 //
35 // |--------------------------|
36 // |........TLAB class........|
37 // |--------------------------|
38 // |.........end addr.........|------------|
39 // |.......free pointer.......|--------|   |
40 // |........start addr........|----|   |   |
41 // |--------------------------|    |   |   |
42 //                                 |   |   |
43 //                                 |   |   |
44 // |--------------------------|    |   |   |
45 // |..Memory for allocations..|    |   |   |
46 // |--------------------------|    |   |   |
47 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|<---|   |   |
48 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|        |   |
49 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|        |   |
50 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|        |   |
51 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|        |   |
52 // |xxxxxxxxxxxxxxxxxxxxxxxxxx|<-------|   |
53 // |..........................|            |
54 // |..........................|            |
55 // |..........................|            |
56 // |..........................|            |
57 // |..........................|<-----------|
58 // |--------------------------|
59 //
60 // Each TLAB is connected with certain thread:
61 // (NOTE: In current implementation, we can reach max one TLAB from a thread metadata)
62 // NOTE(aemelenko): If we don't use links on next, prev TLABS,
63 // it is better to remove these fields.
64 // |------------------------|              |---------------|
65 // | Thread Metainformation | ---------->  | Current  TLAB |----
66 // |------------------------|              |---------------|    |
67 //                                                              |
68 //                                                              |
69 //                                         |---------------|    |
70 //                                         | Previous TLAB |<---|
71 //                                         |---------------|
72 
73 // How to use TLAB from the compiler if we want to allocate an object for class 'cls' with size 'allocation_size':
74 // NOTE: If we have PANDA_TRACK_TLAB_ALLOCATIONS option on, JIT should always call Runtime at AllocateObject calls.
75 // Pseudocode:
76 // IF  allocation_size > TLAB::GetMaxSize() || IsFinalizable(cls)
77 //     call HeapManager::AllocateObject(obj_class, allocation_size)
78 // ELSE
79 //     // We should use TLS for this purpose.
80 //     // Read current TLAB pointer from TLS:
81 //     load TLS.TLAB -> cur_TLAB
82 //     // Read uintptr_t value from TLAB structure:
83 //     load (AddressOf(cur_TLAB) + TLAB::TLABFreePointerOffset) -> free_pointer
84 //     // Read uintptr_t value from TLAB structure:
85 //     load (AddressOf(cur_TLAB) + TLAB::TLABEndAddrOffset) -> end_pointer
86 //     // Align the size of an object to DEFAULT_ALIGNMENT_IN_BYTES
87 //     // One can use GetAlignedObjectSize() method for that.
88 //     align (allocation_size, DEFAULT_ALIGNMENT_IN_BYTES) -> allocation_size
89 //     IF  free_pointer + allocation_size > end_pointer
90 //         // Goto slow path
91 //         call HeapManager::AllocateObject(obj_class, allocation_size)
92 //     // Calculate new_free_pointer:
93 //     new_free_pointer = AddressOf(free_pointer) + allocation_size
94 //     // Store new_free_pointer to (cur_TLAB + TLAB::TLABFreePointerOffset):
95 //     store (AddressOf(cur_TLAB) + TLAB::TLABFreePointerOffset) <- new_free_pointer
96 //     return free_pointer
97 //
98 // After that the Compiler should initialize class word inside new object and
99 // set correct GC bits in the mark word:
100 //     ObjectHeader obj_header
101 //     obj_header.SetClass(cls)
102 //     GetGC()->InitGCBitsForAllocationInTLAB(&obj_header)
103 //     free_pointer <- obj_header
104 //
105 // Runtime should provide these parameters:
106 // HeapManager::GetTLABMaxAllocSize() - max size that can be allocated via TLAB. (depends on the allocator used by GC)
107 // HeapManager::UseTLABForAllocations() - do we need to use TLABs for allocations. (it is a runtime option)
108 // GC::InitGCBitsForAllocationInTLAB() - method for initialize GC bits inside the object header
109 //                                       during allocations through TLAB
110 // TLAB::TLABFreePointerOffset() - an offset of a free pointer field inside TLAB.
111 // TLAB::TLABEndAddrOffset() - an offset of a end buffer pointer field inside TLAB.
112 //
113 
114 class TLAB {
115 public:
116     /**
117      * @brief Construct TLAB with the buffer at @param address with @param size
118      * @param address - a pointer into the memory where TLAB memory will be created
119      * @param size - a size of the allocated memory for the TLAB
120      */
121     explicit TLAB(void *address = nullptr, size_t size = 0);
122     ~TLAB();
123 
124     void Destroy();
125 
126     /**
127      * @brief Fill a TLAB with the buffer at @param address with @param size
128      * @param address - a pointer into the memory where TLAB memory will be created
129      * @param size - a size of the allocated memory for the TLAB
130      */
131     void Fill(void *address, size_t size);
132 
133     /// @brief Set TLAB to be empty
Reset()134     void Reset()
135     {
136         Fill(nullptr, 0U);
137     }
138 
IsUninitialized()139     bool IsUninitialized()
140     {
141         return (memoryStartAddr_ == nullptr) || (curFreePosition_ == nullptr) || (memoryEndAddr_ == nullptr);
142     }
143 
144     NO_MOVE_SEMANTIC(TLAB);
145     NO_COPY_SEMANTIC(TLAB);
146 
147     /**
148      * @brief Allocates memory with size @param size and aligned with DEFAULT_ALIGNMENT alignment
149      * @param size - size of the allocated memory
150      * @return pointer to the allocated memory on success, or nullptr on fail
151      */
152     void *Alloc(size_t size);
153 
154     /**
155      * @brief Iterates over all objects in this TLAB
156      * @param object_visitor
157      */
158     void IterateOverObjects(const std::function<void(ObjectHeader *objectHeader)> &objectVisitor);
159 
160     /**
161      * @brief Iterates over objects in the range inclusively.
162      * @param mem_visitor - function pointer or functor
163      * @param mem_range - memory range
164      */
165     void IterateOverObjectsInRange(const std::function<void(ObjectHeader *objectHeader)> &memVisitor,
166                                    const MemRange &memRange);
167 
168     /**
169      * Collects dead objects and move alive with provided visitor
170      * @param death_checker - functor for check if object alive
171      * @param object_move_visitor - object visitor
172      */
173     template <typename ObjectMoveVisitorT>
CollectAndMove(const GCObjectVisitor &deathChecker, const ObjectMoveVisitorT &objectMoveVisitor)174     void CollectAndMove(const GCObjectVisitor &deathChecker, const ObjectMoveVisitorT &objectMoveVisitor)
175     {
176         LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove started";
177         IterateOverObjects([&](ObjectHeader *objectHeader) {
178             // We are interested only in moving alive objects, after that we cleanup this buffer
179             if (deathChecker(objectHeader) == ObjectStatus::ALIVE_OBJECT) {
180                 LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove found alive object with addr " << objectHeader;
181                 objectMoveVisitor(objectHeader);
182             }
183         });
184         LOG_TLAB_ALLOCATOR(DEBUG) << "CollectAndMove finished";
185     }
186 
187     bool ContainObject(const ObjectHeader *obj);
188 
189     bool IsLive(const ObjectHeader *obj);
190 
GetNextTLAB()191     TLAB *GetNextTLAB()
192     {
193         return nextTlab_;
194     }
195 
GetPrevTLAB()196     TLAB *GetPrevTLAB()
197     {
198         return prevTlab_;
199     }
200 
SetNextTLAB(TLAB *tlabPointer)201     void SetNextTLAB(TLAB *tlabPointer)
202     {
203         nextTlab_ = tlabPointer;
204     }
205 
SetPrevTLAB(TLAB *tlabPointer)206     void SetPrevTLAB(TLAB *tlabPointer)
207     {
208         prevTlab_ = tlabPointer;
209     }
210 
GetStartAddr() const211     void *GetStartAddr() const
212     {
213         return memoryStartAddr_;
214     }
215 
GetEndAddr() const216     void *GetEndAddr() const
217     {
218         return memoryEndAddr_;
219     }
220 
GetCurPos() const221     void *GetCurPos() const
222     {
223         return curFreePosition_;
224     }
225 
GetOccupiedSize() const226     size_t GetOccupiedSize() const
227     {
228         ASSERT(ToUintPtr(curFreePosition_) >= ToUintPtr(memoryStartAddr_));
229         return ToUintPtr(curFreePosition_) - ToUintPtr(memoryStartAddr_);
230     }
231 
GetMemRangeForOccupiedMemory() const232     MemRange GetMemRangeForOccupiedMemory() const
233     {
234         return MemRange(ToUintPtr(memoryStartAddr_), ToUintPtr(curFreePosition_) - 1);
235     }
236 
TLABStartAddrOffset()237     static constexpr size_t TLABStartAddrOffset()
238     {
239         return MEMBER_OFFSET(TLAB, memoryStartAddr_);
240     }
241 
TLABFreePointerOffset()242     static constexpr size_t TLABFreePointerOffset()
243     {
244         return MEMBER_OFFSET(TLAB, curFreePosition_);
245     }
246 
TLABEndAddrOffset()247     static constexpr size_t TLABEndAddrOffset()
248     {
249         return MEMBER_OFFSET(TLAB, memoryEndAddr_);
250     }
251 
GetAllocatorType()252     static constexpr AllocatorType GetAllocatorType()
253     {
254         return AllocatorType::TLAB_ALLOCATOR;
255     }
256 
257     static constexpr float MIN_DESIRED_FILL_FRACTION = 0.5;
258 
GetSize()259     size_t GetSize()
260     {
261         ASSERT(ToUintPtr(memoryEndAddr_) >= ToUintPtr(memoryStartAddr_));
262         return ToUintPtr(memoryEndAddr_) - ToUintPtr(memoryStartAddr_);
263     }
264 
GetFillFraction()265     float GetFillFraction()
266     {
267         size_t size = GetSize();
268         if (size == 0) {
269             // ZERO tlab case
270             // consider it is always full
271             return 1.0;
272         }
273         float fillFraction = static_cast<float>(GetOccupiedSize()) / static_cast<float>(size);
274         ASSERT(fillFraction <= 1.0);
275         return fillFraction;
276     }
277 
278 private:
GetFreeSize()279     size_t GetFreeSize()
280     {
281         ASSERT(ToUintPtr(curFreePosition_) >= ToUintPtr(memoryStartAddr_));
282         ASSERT(ToUintPtr(curFreePosition_) <= ToUintPtr(memoryEndAddr_));
283         return ToUintPtr(memoryEndAddr_) - ToUintPtr(curFreePosition_);
284     }
285 
286     TLAB *nextTlab_;
287     TLAB *prevTlab_;
288     // NOTE(aemelenko): Maybe use OBJECT_POINTER_SIZE here for heap allocation.
289     void *memoryStartAddr_ {nullptr};
290     void *memoryEndAddr_ {nullptr};
291     void *curFreePosition_ {nullptr};
292 };
293 
294 #undef LOG_TLAB_ALLOCATOR
295 
296 }  // namespace ark::mem
297 
298 #endif  // PANDA_RUNTIME_MEM_TLAB_H
299