1/* Alloc.c -- Memory allocation functions 22023-04-02 : Igor Pavlov : Public domain */ 3 4#include "Precomp.h" 5 6#ifdef _WIN32 7#include "7zWindows.h" 8#endif 9#include <stdlib.h> 10 11#include "Alloc.h" 12 13#ifdef _WIN32 14#ifdef Z7_LARGE_PAGES 15#if defined(__clang__) || defined(__GNUC__) 16typedef void (*Z7_voidFunction)(void); 17#define MY_CAST_FUNC (Z7_voidFunction) 18#elif defined(_MSC_VER) && _MSC_VER > 1920 19#define MY_CAST_FUNC (void *) 20// #pragma warning(disable : 4191) // 'type cast': unsafe conversion from 'FARPROC' to 'void (__cdecl *)()' 21#else 22#define MY_CAST_FUNC 23#endif 24#endif // Z7_LARGE_PAGES 25#endif // _WIN32 26 27// #define SZ_ALLOC_DEBUG 28/* #define SZ_ALLOC_DEBUG */ 29 30/* use SZ_ALLOC_DEBUG to debug alloc/free operations */ 31#ifdef SZ_ALLOC_DEBUG 32 33#include <string.h> 34#include <stdio.h> 35static int g_allocCount = 0; 36#ifdef _WIN32 37static int g_allocCountMid = 0; 38static int g_allocCountBig = 0; 39#endif 40 41 42#define CONVERT_INT_TO_STR(charType, tempSize) \ 43 char temp[tempSize]; unsigned i = 0; \ 44 while (val >= 10) { temp[i++] = (char)('0' + (unsigned)(val % 10)); val /= 10; } \ 45 *s++ = (charType)('0' + (unsigned)val); \ 46 while (i != 0) { i--; *s++ = temp[i]; } \ 47 *s = 0; 48 49static void ConvertUInt64ToString(UInt64 val, char *s) 50{ 51 CONVERT_INT_TO_STR(char, 24) 52} 53 54#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) 55 56static void ConvertUInt64ToHex(UInt64 val, char *s) 57{ 58 UInt64 v = val; 59 unsigned i; 60 for (i = 1;; i++) 61 { 62 v >>= 4; 63 if (v == 0) 64 break; 65 } 66 s[i] = 0; 67 do 68 { 69 unsigned t = (unsigned)(val & 0xF); 70 val >>= 4; 71 s[--i] = GET_HEX_CHAR(t); 72 } 73 while (i); 74} 75 76#define DEBUG_OUT_STREAM stderr 77 78static void Print(const char *s) 79{ 80 fputs(s, DEBUG_OUT_STREAM); 81} 82 83static void PrintAligned(const char *s, size_t align) 84{ 85 size_t len = strlen(s); 86 for(;;) 87 { 88 fputc(' ', DEBUG_OUT_STREAM); 89 if (len >= align) 90 break; 91 ++len; 92 } 93 Print(s); 94} 95 96static void PrintLn(void) 97{ 98 Print("\n"); 99} 100 101static void PrintHex(UInt64 v, size_t align) 102{ 103 char s[32]; 104 ConvertUInt64ToHex(v, s); 105 PrintAligned(s, align); 106} 107 108static void PrintDec(int v, size_t align) 109{ 110 char s[32]; 111 ConvertUInt64ToString((unsigned)v, s); 112 PrintAligned(s, align); 113} 114 115static void PrintAddr(void *p) 116{ 117 PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12); 118} 119 120 121#define PRINT_REALLOC(name, cnt, size, ptr) { \ 122 Print(name " "); \ 123 if (!ptr) PrintDec(cnt++, 10); \ 124 PrintHex(size, 10); \ 125 PrintAddr(ptr); \ 126 PrintLn(); } 127 128#define PRINT_ALLOC(name, cnt, size, ptr) { \ 129 Print(name " "); \ 130 PrintDec(cnt++, 10); \ 131 PrintHex(size, 10); \ 132 PrintAddr(ptr); \ 133 PrintLn(); } 134 135#define PRINT_FREE(name, cnt, ptr) if (ptr) { \ 136 Print(name " "); \ 137 PrintDec(--cnt, 10); \ 138 PrintAddr(ptr); \ 139 PrintLn(); } 140 141#else 142 143#ifdef _WIN32 144#define PRINT_ALLOC(name, cnt, size, ptr) 145#endif 146#define PRINT_FREE(name, cnt, ptr) 147#define Print(s) 148#define PrintLn() 149#define PrintHex(v, align) 150#define PrintAddr(p) 151 152#endif 153 154 155/* 156by specification: 157 malloc(non_NULL, 0) : returns NULL or a unique pointer value that can later be successfully passed to free() 158 realloc(NULL, size) : the call is equivalent to malloc(size) 159 realloc(non_NULL, 0) : the call is equivalent to free(ptr) 160 161in main compilers: 162 malloc(0) : returns non_NULL 163 realloc(NULL, 0) : returns non_NULL 164 realloc(non_NULL, 0) : returns NULL 165*/ 166 167 168void *MyAlloc(size_t size) 169{ 170 if (size == 0) 171 return NULL; 172 // PRINT_ALLOC("Alloc ", g_allocCount, size, NULL) 173 #ifdef SZ_ALLOC_DEBUG 174 { 175 void *p = malloc(size); 176 if (p) 177 { 178 PRINT_ALLOC("Alloc ", g_allocCount, size, p) 179 } 180 return p; 181 } 182 #else 183 return malloc(size); 184 #endif 185} 186 187void MyFree(void *address) 188{ 189 PRINT_FREE("Free ", g_allocCount, address) 190 191 free(address); 192} 193 194void *MyRealloc(void *address, size_t size) 195{ 196 if (size == 0) 197 { 198 MyFree(address); 199 return NULL; 200 } 201 // PRINT_REALLOC("Realloc ", g_allocCount, size, address) 202 #ifdef SZ_ALLOC_DEBUG 203 { 204 void *p = realloc(address, size); 205 if (p) 206 { 207 PRINT_REALLOC("Realloc ", g_allocCount, size, address) 208 } 209 return p; 210 } 211 #else 212 return realloc(address, size); 213 #endif 214} 215 216 217#ifdef _WIN32 218 219void *MidAlloc(size_t size) 220{ 221 if (size == 0) 222 return NULL; 223 #ifdef SZ_ALLOC_DEBUG 224 { 225 void *p = VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); 226 if (p) 227 { 228 PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, p) 229 } 230 return p; 231 } 232 #else 233 return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); 234 #endif 235} 236 237void MidFree(void *address) 238{ 239 PRINT_FREE("Free-Mid", g_allocCountMid, address) 240 241 if (!address) 242 return; 243 VirtualFree(address, 0, MEM_RELEASE); 244} 245 246#ifdef Z7_LARGE_PAGES 247 248#ifdef MEM_LARGE_PAGES 249 #define MY__MEM_LARGE_PAGES MEM_LARGE_PAGES 250#else 251 #define MY__MEM_LARGE_PAGES 0x20000000 252#endif 253 254extern 255SIZE_T g_LargePageSize; 256SIZE_T g_LargePageSize = 0; 257typedef SIZE_T (WINAPI *Func_GetLargePageMinimum)(VOID); 258 259void SetLargePageSize(void) 260{ 261 #ifdef Z7_LARGE_PAGES 262 SIZE_T size; 263 const 264 Func_GetLargePageMinimum fn = 265 (Func_GetLargePageMinimum) MY_CAST_FUNC GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), 266 "GetLargePageMinimum"); 267 if (!fn) 268 return; 269 size = fn(); 270 if (size == 0 || (size & (size - 1)) != 0) 271 return; 272 g_LargePageSize = size; 273 #endif 274} 275 276#endif // Z7_LARGE_PAGES 277 278void *BigAlloc(size_t size) 279{ 280 if (size == 0) 281 return NULL; 282 283 PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL) 284 285 #ifdef Z7_LARGE_PAGES 286 { 287 SIZE_T ps = g_LargePageSize; 288 if (ps != 0 && ps <= (1 << 30) && size > (ps / 2)) 289 { 290 size_t size2; 291 ps--; 292 size2 = (size + ps) & ~ps; 293 if (size2 >= size) 294 { 295 void *p = VirtualAlloc(NULL, size2, MEM_COMMIT | MY__MEM_LARGE_PAGES, PAGE_READWRITE); 296 if (p) 297 { 298 PRINT_ALLOC("Alloc-BM ", g_allocCountMid, size2, p) 299 return p; 300 } 301 } 302 } 303 } 304 #endif 305 306 return MidAlloc(size); 307} 308 309void BigFree(void *address) 310{ 311 PRINT_FREE("Free-Big", g_allocCountBig, address) 312 MidFree(address); 313} 314 315#endif // _WIN32 316 317 318static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p) return MyAlloc(size); } 319static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p) MyFree(address); } 320const ISzAlloc g_Alloc = { SzAlloc, SzFree }; 321 322#ifdef _WIN32 323static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p) return MidAlloc(size); } 324static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p) MidFree(address); } 325static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p) return BigAlloc(size); } 326static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p) BigFree(address); } 327const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; 328const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; 329#endif 330 331/* 332 uintptr_t : <stdint.h> C99 (optional) 333 : unsupported in VS6 334*/ 335 336#ifdef _WIN32 337 typedef UINT_PTR UIntPtr; 338#else 339 /* 340 typedef uintptr_t UIntPtr; 341 */ 342 typedef ptrdiff_t UIntPtr; 343#endif 344 345 346#define ADJUST_ALLOC_SIZE 0 347/* 348#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1) 349*/ 350/* 351 Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if 352 MyAlloc() can return address that is NOT multiple of sizeof(void *). 353*/ 354 355 356/* 357#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) 358*/ 359#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) 360 361 362#if !defined(_WIN32) && defined(_POSIX_C_SOURCE) && (_POSIX_C_SOURCE >= 200112L) 363 #define USE_posix_memalign 364#endif 365 366#ifndef USE_posix_memalign 367#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) 368#endif 369 370/* 371 This posix_memalign() is for test purposes only. 372 We also need special Free() function instead of free(), 373 if this posix_memalign() is used. 374*/ 375 376/* 377static int posix_memalign(void **ptr, size_t align, size_t size) 378{ 379 size_t newSize = size + align; 380 void *p; 381 void *pAligned; 382 *ptr = NULL; 383 if (newSize < size) 384 return 12; // ENOMEM 385 p = MyAlloc(newSize); 386 if (!p) 387 return 12; // ENOMEM 388 pAligned = MY_ALIGN_PTR_UP_PLUS(p, align); 389 ((void **)pAligned)[-1] = p; 390 *ptr = pAligned; 391 return 0; 392} 393*/ 394 395/* 396 ALLOC_ALIGN_SIZE >= sizeof(void *) 397 ALLOC_ALIGN_SIZE >= cache_line_size 398*/ 399 400#define ALLOC_ALIGN_SIZE ((size_t)1 << 7) 401 402static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) 403{ 404 #ifndef USE_posix_memalign 405 406 void *p; 407 void *pAligned; 408 size_t newSize; 409 UNUSED_VAR(pp) 410 411 /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned 412 block to prevent cache line sharing with another allocated blocks */ 413 414 newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE; 415 if (newSize < size) 416 return NULL; 417 418 p = MyAlloc(newSize); 419 420 if (!p) 421 return NULL; 422 pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE); 423 424 Print(" size="); PrintHex(size, 8); 425 Print(" a_size="); PrintHex(newSize, 8); 426 Print(" ptr="); PrintAddr(p); 427 Print(" a_ptr="); PrintAddr(pAligned); 428 PrintLn(); 429 430 ((void **)pAligned)[-1] = p; 431 432 return pAligned; 433 434 #else 435 436 void *p; 437 UNUSED_VAR(pp) 438 if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) 439 return NULL; 440 441 Print(" posix_memalign="); PrintAddr(p); 442 PrintLn(); 443 444 return p; 445 446 #endif 447} 448 449 450static void SzAlignedFree(ISzAllocPtr pp, void *address) 451{ 452 UNUSED_VAR(pp) 453 #ifndef USE_posix_memalign 454 if (address) 455 MyFree(((void **)address)[-1]); 456 #else 457 free(address); 458 #endif 459} 460 461 462const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; 463 464 465 466#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) 467 468/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ 469#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] 470/* 471#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] 472*/ 473 474static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) 475{ 476 const CAlignOffsetAlloc *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CAlignOffsetAlloc, vt); 477 void *adr; 478 void *pAligned; 479 size_t newSize; 480 size_t extra; 481 size_t alignSize = (size_t)1 << p->numAlignBits; 482 483 if (alignSize < sizeof(void *)) 484 alignSize = sizeof(void *); 485 486 if (p->offset >= alignSize) 487 return NULL; 488 489 /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned 490 block to prevent cache line sharing with another allocated blocks */ 491 extra = p->offset & (sizeof(void *) - 1); 492 newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE; 493 if (newSize < size) 494 return NULL; 495 496 adr = ISzAlloc_Alloc(p->baseAlloc, newSize); 497 498 if (!adr) 499 return NULL; 500 501 pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + 502 alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; 503 504 PrintLn(); 505 Print("- Aligned: "); 506 Print(" size="); PrintHex(size, 8); 507 Print(" a_size="); PrintHex(newSize, 8); 508 Print(" ptr="); PrintAddr(adr); 509 Print(" a_ptr="); PrintAddr(pAligned); 510 PrintLn(); 511 512 REAL_BLOCK_PTR_VAR(pAligned) = adr; 513 514 return pAligned; 515} 516 517 518static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) 519{ 520 if (address) 521 { 522 const CAlignOffsetAlloc *p = Z7_CONTAINER_FROM_VTBL_CONST(pp, CAlignOffsetAlloc, vt); 523 PrintLn(); 524 Print("- Aligned Free: "); 525 PrintLn(); 526 ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); 527 } 528} 529 530 531void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p) 532{ 533 p->vt.Alloc = AlignOffsetAlloc_Alloc; 534 p->vt.Free = AlignOffsetAlloc_Free; 535} 536