1 #include <stdlib.h>
2 #include <stdint.h>
3 #include <limits.h>
4 #include <string.h>
5 #include <sys/mman.h>
6 #include <sys/prctl.h>
7 #include <errno.h>
8
9 #include "meta.h"
10
11 #ifdef USE_JEMALLOC
12 #ifdef USE_JEMALLOC_DFX_INTF
13 extern void je_malloc_disable();
14 extern void je_malloc_enable();
15 extern int je_iterate(uintptr_t base, size_t size,
16 void (*callback)(uintptr_t ptr, size_t size, void* arg), void* arg);
17 extern int je_mallopt(int param, int value);
18 #endif
19 #endif
20
21 #ifdef MALLOC_SECURE_ALL
22 #include <fcntl.h>
23 #define RANDOM_BUFFER_LEN 512
24 static uint8_t buffer[RANDOM_BUFFER_LEN] = { 0 };
25 static size_t ri = RANDOM_BUFFER_LEN;
26
get_random8null27 static uint8_t get_random8()
28 {
29 uint8_t num;
30 if ((ri >= RANDOM_BUFFER_LEN) || (buffer[0] == 0)) {
31 int fd = open("/dev/urandom", O_RDONLY);
32 if (fd < 0) {
33 num = (uint8_t)get_random_secret();
34 return num;
35 }
36
37 read(fd, buffer, RANDOM_BUFFER_LEN);
38 close(fd);
39 ri = 0;
40 }
41 num = buffer[ri];
42 ri++;
43 return num;
44 }
45
get_randomIdx(int avail_mask, int last_idx)46 static int get_randomIdx(int avail_mask, int last_idx)
47 {
48 uint32_t mask;
49 uint32_t r;
50 uint32_t cmask;
51 int idx;
52
53 mask = avail_mask;
54 r = get_random8() % last_idx;
55 cmask = ~((2u << (last_idx - r)) - 1);
56
57 if (mask & cmask) {
58 idx = 31 - a_clz_32(mask & cmask);
59 } else {
60 idx = a_ctz_32(mask);
61 }
62
63 return idx;
64 }
65 #endif
66
67 LOCK_OBJ_DEF;
68
69 const uint16_t size_classes[] = {
70 1, 2, 3, 4, 5, 6, 7, 8,
71 9, 10, 12, 15,
72 18, 20, 25, 31,
73 36, 42, 50, 63,
74 72, 84, 102, 127,
75 146, 170, 204, 255,
76 292, 340, 409, 511,
77 584, 682, 818, 1023,
78 1169, 1364, 1637, 2047,
79 2340, 2730, 3276, 4095,
80 4680, 5460, 6552, 8191,
81 };
82
83 static const uint8_t small_cnt_tab[][3] = {
84 { 30, 30, 30 },
85 { 31, 15, 15 },
86 { 20, 10, 10 },
87 { 31, 15, 7 },
88 { 25, 12, 6 },
89 { 21, 10, 5 },
90 { 18, 8, 4 },
91 { 31, 15, 7 },
92 { 28, 14, 6 },
93 };
94
95 static const uint8_t med_cnt_tab[4] = { 28, 24, 20, 32 };
96
97 struct malloc_context ctx = { 0 };
98
alloc_meta(void)99 struct meta *alloc_meta(void)
100 {
101 struct meta *m;
102 unsigned char *p;
103 if (!ctx.init_done) {
104 #ifndef PAGESIZE
105 ctx.pagesize = get_page_size();
106 #endif
107 ctx.secret = get_random_secret();
108 ctx.init_done = 1;
109 }
110 size_t pagesize = PGSZ;
111 if (pagesize < 4096) pagesize = 4096;
112 if ((m = dequeue_head(&ctx.free_meta_head))) return m;
113 if (!ctx.avail_meta_count) {
114 int need_unprotect = 1;
115 if (!ctx.avail_meta_area_count && ctx.brk!=-1) {
116 uintptr_t new = ctx.brk + pagesize;
117 int need_guard = 0;
118 if (!ctx.brk) {
119 need_guard = 1;
120 ctx.brk = brk(0);
121 // some ancient kernels returned _ebss
122 // instead of next page as initial brk.
123 ctx.brk += -ctx.brk & (pagesize-1);
124 new = ctx.brk + 2*pagesize;
125 }
126 if (brk(new) != new) {
127 ctx.brk = -1;
128 } else {
129 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ctx.brk, new - ctx.brk, "native_heap:meta");
130 if (need_guard) mmap((void *)ctx.brk, pagesize,
131 PROT_NONE, MAP_ANON|MAP_PRIVATE|MAP_FIXED, -1, 0);
132 ctx.brk = new;
133 ctx.avail_meta_areas = (void *)(new - pagesize);
134 ctx.avail_meta_area_count = pagesize>>12;
135 need_unprotect = 0;
136 }
137 }
138 if (!ctx.avail_meta_area_count) {
139 size_t n = 2UL << ctx.meta_alloc_shift;
140 p = mmap(0, n*pagesize, PROT_NONE,
141 MAP_PRIVATE|MAP_ANON, -1, 0);
142 if (p==MAP_FAILED) return 0;
143 ctx.avail_meta_areas = p + pagesize;
144 ctx.avail_meta_area_count = (n-1)*(pagesize>>12);
145 ctx.meta_alloc_shift++;
146 }
147 p = ctx.avail_meta_areas;
148 if ((uintptr_t)p & (pagesize-1)) need_unprotect = 0;
149 if (need_unprotect)
150 if (mprotect(p, pagesize, PROT_READ|PROT_WRITE)
151 && errno != ENOSYS)
152 return 0;
153 ctx.avail_meta_area_count--;
154 ctx.avail_meta_areas = p + 4096;
155 if (ctx.meta_area_tail) {
156 ctx.meta_area_tail->next = (void *)p;
157 } else {
158 ctx.meta_area_head = (void *)p;
159 }
160 ctx.meta_area_tail = (void *)p;
161 ctx.meta_area_tail->check = ctx.secret;
162 ctx.avail_meta_count = ctx.meta_area_tail->nslots
163 = (4096-sizeof(struct meta_area))/sizeof *m;
164 ctx.avail_meta = ctx.meta_area_tail->slots;
165 }
166 ctx.avail_meta_count--;
167 m = ctx.avail_meta++;
168 m->prev = m->next = 0;
169 return m;
170 }
171
try_avail(struct meta **pm)172 static uint32_t try_avail(struct meta **pm)
173 {
174 struct meta *m = *pm;
175 uint32_t first;
176 if (!m) return 0;
177 uint32_t mask = m->avail_mask;
178 if (!mask) {
179 if (!m) return 0;
180 if (!m->freed_mask) {
181 dequeue(pm, m);
182 m = *pm;
183 if (!m) return 0;
184 } else {
185 m = m->next;
186 *pm = m;
187 }
188
189 mask = m->freed_mask;
190
191 // skip fully-free group unless it's the only one
192 // or it's a permanently non-freeable group
193 if (mask == (2u<<m->last_idx)-1 && m->freeable) {
194 m = m->next;
195 *pm = m;
196 mask = m->freed_mask;
197 }
198
199 // activate more slots in a not-fully-active group
200 // if needed, but only as a last resort. prefer using
201 // any other group with free slots. this avoids
202 // touching & dirtying as-yet-unused pages.
203 if (!(mask & ((2u<<m->mem->active_idx)-1))) {
204 if (m->next != m) {
205 m = m->next;
206 *pm = m;
207 } else {
208 int cnt = m->mem->active_idx + 2;
209 int size = size_classes[m->sizeclass]*UNIT;
210 int span = UNIT + size*cnt;
211 // activate up to next 4k boundary
212 while ((span^(span+size-1)) < 4096) {
213 cnt++;
214 span += size;
215 }
216 if (cnt > m->last_idx+1)
217 cnt = m->last_idx+1;
218 m->mem->active_idx = cnt-1;
219 }
220 }
221 mask = activate_group(m);
222 assert(mask);
223 decay_bounces(m->sizeclass);
224 }
225
226 #ifdef MALLOC_SECURE_ALL
227 int idx = get_randomIdx(mask, m->last_idx);
228 first = 1 << idx;
229 #else
230 first = mask&-mask;
231 #endif
232 m->avail_mask = mask-first;
233 return first;
234 }
235
236 static int alloc_slot(int, size_t);
237
alloc_group(int sc, size_t req)238 static struct meta *alloc_group(int sc, size_t req)
239 {
240 size_t size = UNIT*size_classes[sc];
241 int i = 0, cnt;
242 unsigned char *p;
243 struct meta *m = alloc_meta();
244 if (!m) return 0;
245 size_t usage = ctx.usage_by_class[sc];
246 size_t pagesize = PGSZ;
247 int active_idx;
248 if (sc < 9) {
249 while (i<2 && 4*small_cnt_tab[sc][i] > usage)
250 i++;
251 cnt = small_cnt_tab[sc][i];
252 } else {
253 // lookup max number of slots fitting in power-of-two size
254 // from a table, along with number of factors of two we
255 // can divide out without a remainder or reaching 1.
256 cnt = med_cnt_tab[sc&3];
257
258 // reduce cnt to avoid excessive eagar allocation.
259 while (!(cnt&1) && 4*cnt > usage)
260 cnt >>= 1;
261
262 // data structures don't support groups whose slot offsets
263 // in units don't fit in 16 bits.
264 while (size*cnt >= 65536*UNIT)
265 cnt >>= 1;
266 }
267
268 // If we selected a count of 1 above but it's not sufficient to use
269 // mmap, increase to 2. Then it might be; if not it will nest.
270 if (cnt==1 && size*cnt+UNIT <= pagesize/2) cnt = 2;
271
272 // All choices of size*cnt are "just below" a power of two, so anything
273 // larger than half the page size should be allocated as whole pages.
274 if (size*cnt+UNIT > pagesize/2) {
275 // check/update bounce counter to start/increase retention
276 // of freed maps, and inhibit use of low-count, odd-size
277 // small mappings and single-slot groups if activated.
278 int nosmall = is_bouncing(sc);
279 account_bounce(sc);
280 step_seq();
281
282 // since the following count reduction opportunities have
283 // an absolute memory usage cost, don't overdo them. count
284 // coarse usage as part of usage.
285 if (!(sc&1) && sc<32) usage += ctx.usage_by_class[sc+1];
286
287 // try to drop to a lower count if the one found above
288 // increases usage by more than 25%. these reduced counts
289 // roughly fill an integral number of pages, just not a
290 // power of two, limiting amount of unusable space.
291 if (4*cnt > usage && !nosmall) {
292 if (0);
293 else if ((sc&3)==1 && size*cnt>8*pagesize) cnt = 2;
294 else if ((sc&3)==2 && size*cnt>4*pagesize) cnt = 3;
295 else if ((sc&3)==0 && size*cnt>8*pagesize) cnt = 3;
296 else if ((sc&3)==0 && size*cnt>2*pagesize) cnt = 5;
297 }
298 size_t needed = size*cnt + UNIT;
299 needed += -needed & (pagesize-1);
300
301 // produce an individually-mmapped allocation if usage is low,
302 // bounce counter hasn't triggered, and either it saves memory
303 // or it avoids eagar slot allocation without wasting too much.
304 if (!nosmall && cnt<=7) {
305 req += IB + UNIT;
306 req += -req & (pagesize-1);
307 if (req<size+UNIT || (req>=4*pagesize && 2*cnt>usage)) {
308 cnt = 1;
309 needed = req;
310 }
311 }
312
313 p = mmap(0, needed, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANON, -1, 0);
314 if (p==MAP_FAILED) {
315 free_meta(m);
316 return 0;
317 }
318 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, needed, "native_heap:brk");
319 m->maplen = needed>>12;
320 ctx.mmap_counter++;
321 active_idx = (4096-UNIT)/size-1;
322 if (active_idx > cnt-1) active_idx = cnt-1;
323 if (active_idx < 0) active_idx = 0;
324 } else {
325 int j = size_to_class(UNIT+cnt*size-IB);
326 int idx = alloc_slot(j, UNIT+cnt*size-IB);
327 if (idx < 0) {
328 free_meta(m);
329 return 0;
330 }
331 struct meta *g = ctx.active[j];
332 p = enframe(g, idx, UNIT*size_classes[j]-IB, ctx.mmap_counter);
333 m->maplen = 0;
334 p[-3] = (p[-3]&31) | (6<<5);
335 for (int i=0; i<=cnt; i++)
336 p[UNIT+i*size-4] = 0;
337 active_idx = cnt-1;
338 }
339 ctx.usage_by_class[sc] += cnt;
340 m->avail_mask = (2u<<active_idx)-1;
341 m->freed_mask = (2u<<(cnt-1))-1 - m->avail_mask;
342 m->mem = (void *)p;
343 m->mem->meta = encode_ptr(m, ctx.secret);
344 m->mem->active_idx = active_idx;
345 m->last_idx = cnt-1;
346 m->freeable = 1;
347 m->sizeclass = sc;
348 return m;
349 }
350
alloc_slot(int sc, size_t req)351 static int alloc_slot(int sc, size_t req)
352 {
353 uint32_t first = try_avail(&ctx.active[sc]);
354 if (first) return a_ctz_32(first);
355
356 struct meta *g = alloc_group(sc, req);
357 if (!g) return -1;
358
359 g->avail_mask--;
360 queue(&ctx.active[sc], g);
361 return 0;
362 }
363
malloc(size_t n)364 void *malloc(size_t n)
365 {
366 if (size_overflows(n)) return 0;
367 struct meta *g;
368 uint32_t mask, first;
369 int sc;
370 int idx;
371 int ctr;
372
373 if (n >= MMAP_THRESHOLD) {
374 size_t needed = n + IB + UNIT;
375 void *p = mmap(0, needed, PROT_READ|PROT_WRITE,
376 MAP_PRIVATE|MAP_ANON, -1, 0);
377 if (p==MAP_FAILED) return 0;
378 wrlock();
379 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, p, needed, "native_heap:mmap");
380 step_seq();
381 g = alloc_meta();
382 if (!g) {
383 unlock();
384 munmap(p, needed);
385 return 0;
386 }
387 g->mem = p;
388 g->mem->meta = encode_ptr(g, ctx.secret);
389 g->last_idx = 0;
390 g->freeable = 1;
391 g->sizeclass = 63;
392 g->maplen = (needed+4095)/4096;
393 g->avail_mask = g->freed_mask = 0;
394 // use a global counter to cycle offset in
395 // individually-mmapped allocations.
396 ctx.mmap_counter++;
397 idx = 0;
398 goto success;
399 }
400
401 sc = size_to_class(n);
402
403 rdlock();
404 g = ctx.active[sc];
405
406 // use coarse size classes initially when there are not yet
407 // any groups of desired size. this allows counts of 2 or 3
408 // to be allocated at first rather than having to start with
409 // 7 or 5, the min counts for even size classes.
410 if (!g && sc>=4 && sc<32 && sc!=6 && !(sc&1) && !ctx.usage_by_class[sc]) {
411 size_t usage = ctx.usage_by_class[sc|1];
412 // if a new group may be allocated, count it toward
413 // usage in deciding if we can use coarse class.
414 if (!ctx.active[sc|1] || (!ctx.active[sc|1]->avail_mask
415 && !ctx.active[sc|1]->freed_mask))
416 usage += 3;
417 if (usage <= 12)
418 sc |= 1;
419 g = ctx.active[sc];
420 }
421
422 for (;;) {
423 mask = g ? g->avail_mask : 0;
424 #ifdef MALLOC_SECURE_ALL
425 if (!mask) break;
426 idx = get_randomIdx(mask, g->last_idx);
427 first = 1u << idx;
428
429 if (RDLOCK_IS_EXCLUSIVE || !MT)
430 g->avail_mask = mask-first;
431 else if (a_cas(&g->avail_mask, mask, mask-first)!=mask)
432 continue;
433 #else
434 first = mask&-mask;
435 if (!first) break;
436 if (RDLOCK_IS_EXCLUSIVE || !MT)
437 g->avail_mask = mask-first;
438 else if (a_cas(&g->avail_mask, mask, mask-first)!=mask)
439 continue;
440 idx = a_ctz_32(first);
441 #endif
442 goto success;
443 }
444 upgradelock();
445
446 idx = alloc_slot(sc, n);
447 if (idx < 0) {
448 unlock();
449 return 0;
450 }
451 g = ctx.active[sc];
452
453 success:
454 ctr = ctx.mmap_counter;
455 unlock();
456 return enframe(g, idx, n, ctr);
457 }
458
is_allzero(void *p)459 int is_allzero(void *p)
460 {
461 struct meta *g = get_meta(p);
462 return g->sizeclass >= 48 ||
463 get_stride(g) < UNIT*size_classes[g->sizeclass];
464 }
465
mallopt(int param, int value)466 int mallopt(int param, int value)
467 {
468 #ifdef USE_JEMALLOC_DFX_INTF
469 return je_mallopt(param, value);
470 #endif
471 return 0;
472 }
473
malloc_disable(void)474 void malloc_disable(void)
475 {
476 #ifdef USE_JEMALLOC_DFX_INTF
477 je_malloc_disable();
478 #endif
479 }
480
malloc_enable(void)481 void malloc_enable(void)
482 {
483 #ifdef USE_JEMALLOC_DFX_INTF
484 je_malloc_enable();
485 #endif
486 }
487
malloc_iterate(void* base, size_t size, void (*callback)(void* base, size_t size, void* arg), void* arg)488 int malloc_iterate(void* base, size_t size, void (*callback)(void* base, size_t size, void* arg), void* arg)
489 {
490 #ifdef USE_JEMALLOC_DFX_INTF
491 return je_iterate(base, size, callback, arg);
492 #endif
493 return 0;
494 }
495
malloc_backtrace(void* pointer, uintptr_t* frames, size_t frame_count)496 ssize_t malloc_backtrace(void* pointer, uintptr_t* frames, size_t frame_count)
497 {
498 return 0;
499 }
500