/** * Copyright (c) 2021-2024 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include "runtime/mem/runslots.h" #include #include "runtime/include/object_header.h" namespace ark::mem { // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) #define LOG_RUNSLOTS(level) LOG(level, ALLOC) << "RunSlots: " template void RunSlots::Initialize(size_t slotSize, uintptr_t poolPointer, bool initializeLock) { ASAN_UNPOISON_MEMORY_REGION(this, RUNSLOTS_SIZE); LOG_RUNSLOTS(DEBUG) << "Initializing RunSlots:"; ASSERT_PRINT((slotSize >= SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES)), "Size of slot in RunSlots is too small"); ASSERT_PRINT((slotSize <= SlotToSize(SlotsSizes::SLOT_MAX_SIZE_BYTES)), "Size of slot in RunSlots is too big"); ASSERT(poolPointer != 0); poolPointer_ = poolPointer; ASSERT_PRINT(!(ToUintPtr(this) & RUNSLOTS_ALIGNMENT_MASK), "RunSlots object must have alignment"); slotSize_ = slotSize; size_t firstSlotOffset = ComputeFirstSlotOffset(slotSize); firstUninitializedSlotOffset_ = firstSlotOffset; ASSERT(firstUninitializedSlotOffset_ != 0); nextFree_ = nullptr; usedSlots_ = 0; nextRunslot_ = nullptr; prevRunslot_ = nullptr; if (initializeLock) { new (&lock_) LockTypeT(); } memset_s(bitmap_.data(), BITMAP_ARRAY_SIZE, 0x0, BITMAP_ARRAY_SIZE); LOG_RUNSLOTS(DEBUG) << "- Memory started from = 0x" << std::hex << ToUintPtr(this); LOG_RUNSLOTS(DEBUG) << "- Pool size = " << RUNSLOTS_SIZE << " bytes"; LOG_RUNSLOTS(DEBUG) << "- Slots size = " << slotSize_ << " bytes"; LOG_RUNSLOTS(DEBUG) << "- First free slot = " << std::hex << static_cast(nextFree_); LOG_RUNSLOTS(DEBUG) << "- First uninitialized slot offset = " << std::hex << static_cast(ToVoidPtr(firstUninitializedSlotOffset_)); LOG_RUNSLOTS(DEBUG) << "- Pool pointer = " << std::hex << static_cast(ToVoidPtr(poolPointer_)); LOG_RUNSLOTS(DEBUG) << "Successful finished RunSlots init"; ASAN_POISON_MEMORY_REGION(this, RUNSLOTS_SIZE); } template FreeSlot *RunSlots::PopFreeSlot() { ASAN_UNPOISON_MEMORY_REGION(this, GetHeaderSize()); FreeSlot *freeSlot = nullptr; if (nextFree_ == nullptr) { void *uninitializedSlot = PopUninitializedSlot(); if (uninitializedSlot == nullptr) { LOG_RUNSLOTS(DEBUG) << "Failed to get free slot - there are no free slots in RunSlots"; ASAN_POISON_MEMORY_REGION(this, GetHeaderSize()); return nullptr; } freeSlot = static_cast(uninitializedSlot); } else { freeSlot = nextFree_; ASAN_UNPOISON_MEMORY_REGION(freeSlot, sizeof(FreeSlot)); nextFree_ = nextFree_->GetNext(); ASAN_POISON_MEMORY_REGION(freeSlot, sizeof(FreeSlot)); } MarkAsOccupied(freeSlot); usedSlots_++; LOG_RUNSLOTS(DEBUG) << "Successfully get free slot " << std::hex << static_cast(freeSlot) << ". Used slots in this RunSlots = " << std::dec << usedSlots_; ASAN_POISON_MEMORY_REGION(this, GetHeaderSize()); return freeSlot; } template void RunSlots::PushFreeSlot(FreeSlot *memSlot) { ASAN_UNPOISON_MEMORY_REGION(this, GetHeaderSize()); LOG_RUNSLOTS(DEBUG) << "Free slot in RunSlots at addr " << std::hex << static_cast(memSlot); // We need to poison/unpoison mem_slot here because we could allocate an object with size less than FreeSlot size ASAN_UNPOISON_MEMORY_REGION(memSlot, sizeof(FreeSlot)); memSlot->SetNext(nextFree_); ASAN_POISON_MEMORY_REGION(memSlot, sizeof(FreeSlot)); nextFree_ = memSlot; MarkAsFree(memSlot); usedSlots_--; LOG_RUNSLOTS(DEBUG) << "Used slots in RunSlots = " << usedSlots_; ASAN_POISON_MEMORY_REGION(this, GetHeaderSize()); } template size_t RunSlots::ComputeFirstSlotOffset(size_t slotSize) { size_t slotsForHeader = (GetHeaderSize() / slotSize); if ((GetHeaderSize() % slotSize) > 0) { slotsForHeader++; } return slotsForHeader * slotSize; } template void *RunSlots::PopUninitializedSlot() { if (firstUninitializedSlotOffset_ != 0) { ASSERT(RUNSLOTS_SIZE > firstUninitializedSlotOffset_); void *uninitializedSlot = ToVoidPtr(ToUintPtr(this) + firstUninitializedSlotOffset_); firstUninitializedSlotOffset_ += slotSize_; if (firstUninitializedSlotOffset_ >= RUNSLOTS_SIZE) { ASSERT(firstUninitializedSlotOffset_ == RUNSLOTS_SIZE); firstUninitializedSlotOffset_ = 0; } return uninitializedSlot; } return nullptr; } template void RunSlots::MarkAsOccupied(const FreeSlot *slotMem) { uintptr_t bitIndex = (ToUintPtr(slotMem) & (RUNSLOTS_SIZE - 1U)) >> SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES_POWER_OF_TWO); uintptr_t arrayIndex = bitIndex >> BITS_IN_BYTE_POWER_OF_TWO; uintptr_t bitInArrayElement = bitIndex & ((1U << BITS_IN_BYTE_POWER_OF_TWO) - 1U); ASSERT(!(bitmap_[arrayIndex] & (1U << bitInArrayElement))); bitmap_[arrayIndex] |= 1U << bitInArrayElement; } template void RunSlots::MarkAsFree(const FreeSlot *slotMem) { uintptr_t bitIndex = (ToUintPtr(slotMem) & (RUNSLOTS_SIZE - 1U)) >> SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES_POWER_OF_TWO); uintptr_t arrayIndex = bitIndex >> BITS_IN_BYTE_POWER_OF_TWO; uintptr_t bitInArrayElement = bitIndex & ((1U << BITS_IN_BYTE_POWER_OF_TWO) - 1U); ASSERT(bitmap_[arrayIndex] & (1U << bitInArrayElement)); bitmap_[arrayIndex] ^= 1U << bitInArrayElement; } template FreeSlot *RunSlots::BitMapToSlot(size_t arrayIndex, size_t bit) { return static_cast( ToVoidPtr(ToUintPtr(this) + (((arrayIndex << BITS_IN_BYTE_POWER_OF_TWO) + bit) << SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES_POWER_OF_TWO)))); } template size_t RunSlots::RunVerifier::operator()(RunSlots *run) { // 1. should verify whether run's bracket size is the same as recorded in RunSlotsAllocator, but RunSlotsAllocator // does not record this // 2. should verify thread local run's ownership, but thread local run not implemented yet // check alloc'ed size auto sizeCheckFunc = [this, &run](const ObjectHeader *obj) { auto sizePowerOfTwo = ConvertToPowerOfTwoUnsafe(obj->ObjectSize()); if ((1U << sizePowerOfTwo) != run->GetSlotsSize()) { ++(this->failCnt_); } }; run->IterateOverOccupiedSlots(sizeCheckFunc); return failCnt_; } template bool RunSlots::IsLive(const ObjectHeader *obj) const { ASAN_UNPOISON_MEMORY_REGION(this, GetHeaderSize()); uintptr_t memTailByRunslots = ToUintPtr(obj) & (RUNSLOTS_SIZE - 1U); if ((memTailByRunslots & (static_cast(slotSize_) - 1)) != 0) { ASAN_POISON_MEMORY_REGION(this, GetHeaderSize()); return false; } uintptr_t bitIndex = memTailByRunslots >> SlotToSize(SlotsSizes::SLOT_MIN_SIZE_BYTES_POWER_OF_TWO); uintptr_t arrayIndex = bitIndex >> BITS_IN_BYTE_POWER_OF_TWO; uintptr_t bitInArrayElement = bitIndex & ((1U << BITS_IN_BYTE_POWER_OF_TWO) - 1U); auto liveWord = bitmap_[arrayIndex] & (1U << bitInArrayElement); ASAN_POISON_MEMORY_REGION(this, GetHeaderSize()); return liveWord != 0; } template class RunSlots; template class RunSlots; } // namespace ark::mem