1#define _GNU_SOURCE 2#define ANON_STACK_NAME_SIZE 50 3#include "musl_log.h" 4#include "pthread_impl.h" 5#include "stdio_impl.h" 6#include "libc.h" 7#include "lock.h" 8#include <sys/mman.h> 9#include <sys/prctl.h> 10#include <string.h> 11#include <stddef.h> 12#include <stdarg.h> 13 14pid_t getpid(void); 15 16void log_print(const char* info, ...) 17{ 18 va_list ap; 19 va_start(ap, info); 20 vfprintf(stdout, info, ap); 21 va_end(ap); 22} 23 24void stack_naming(struct pthread *new) { 25 size_t size_len; 26 unsigned char *start_addr; 27 char name[ANON_STACK_NAME_SIZE]; 28 if (new->guard_size) { 29 snprintf(name, ANON_STACK_NAME_SIZE, "guard:%d", new->tid); 30 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, new->map_base, new->guard_size, name); 31 start_addr = new->map_base + new->guard_size; 32 size_len = new->map_size - new->guard_size; 33 memset(name, 0, ANON_STACK_NAME_SIZE); 34 } else { 35 start_addr = new->map_base; 36 size_len = new->map_size; 37 } 38 snprintf(name, ANON_STACK_NAME_SIZE, "stack:%d", new->tid); 39 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, start_addr, size_len, name); 40}; 41 42#ifdef RESERVE_SIGNAL_STACK 43#if defined (__LP64__) 44#define RESERVE_SIGNAL_STACK_SIZE (32 * 1024) 45#else 46#define RESERVE_SIGNAL_STACK_SIZE (20 * 1024) 47#endif 48void __pthread_reserve_signal_stack() 49{ 50 void* stack = mmap(NULL, RESERVE_SIGNAL_STACK_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 51 if (stack != MAP_FAILED) { 52 if (mprotect(stack, __default_guardsize, PROT_NONE) == -1) { 53 munmap(stack, RESERVE_SIGNAL_STACK_SIZE); 54 return; 55 } 56 } 57 58 stack_t signal_stack; 59 signal_stack.ss_sp = (uint8_t*)stack + __default_guardsize; 60 signal_stack.ss_size = RESERVE_SIGNAL_STACK_SIZE - __default_guardsize; 61 signal_stack.ss_flags = 0; 62 sigaltstack(&signal_stack, NULL); 63 64 pthread_t self = __pthread_self(); 65 self->signal_stack = stack; 66 char name[ANON_STACK_NAME_SIZE]; 67 snprintf(name, ANON_STACK_NAME_SIZE, "signal_stack:%d", __pthread_self()->tid); 68 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, signal_stack.ss_sp, signal_stack.ss_size, name); 69 return; 70} 71 72void __pthread_release_signal_stack() 73{ 74 pthread_t self = __pthread_self(); 75 if (self->signal_stack == NULL) { 76 return; 77 } 78 79 stack_t signal_stack, old_stack; 80 memset(&signal_stack, 0, sizeof(signal_stack)); 81 signal_stack.ss_flags = SS_DISABLE; 82 sigaltstack(&signal_stack, &old_stack); 83 munmap(self->signal_stack, __default_guardsize); 84 if (old_stack.ss_flags != SS_DISABLE) { 85 munmap(old_stack.ss_sp, old_stack.ss_size); 86 } 87 self->signal_stack = NULL; 88} 89 90weak_alias(__pthread_reserve_signal_stack, pthread_reserve_signal_stack); 91weak_alias(__pthread_release_signal_stack, pthread_release_signal_stack); 92#endif 93 94static void dummy_0() 95{ 96} 97weak_alias(dummy_0, __acquire_ptc); 98weak_alias(dummy_0, __release_ptc); 99weak_alias(dummy_0, __pthread_tsd_run_dtors); 100weak_alias(dummy_0, __do_orphaned_stdio_locks); 101weak_alias(dummy_0, __dl_thread_cleanup); 102weak_alias(dummy_0, __membarrier_init); 103 104#define TID_ERROR_0 (0) 105#define TID_ERROR_INIT (-1) 106#define COUNT_ERROR_INIT (-10000) 107 108static int tl_lock_count; 109static int tl_lock_waiters; 110static int tl_lock_tid_fail = TID_ERROR_INIT; 111static int tl_lock_count_tid = TID_ERROR_INIT; 112static int tl_lock_count_fail = COUNT_ERROR_INIT; 113static int thread_list_lock_pre_unlock = TID_ERROR_INIT; 114 115int get_tl_lock_count(void) 116{ 117 return tl_lock_count; 118} 119 120int get_tl_lock_waiters(void) 121{ 122 return tl_lock_waiters; 123} 124 125int get_tl_lock_tid_fail(void) 126{ 127 return tl_lock_tid_fail; 128} 129 130int get_tl_lock_count_tid(void) 131{ 132 return tl_lock_count_tid; 133} 134 135int get_tl_lock_count_fail(void) 136{ 137 return tl_lock_count_fail; 138} 139 140int get_thread_list_lock_pre_unlock(void) 141{ 142 return thread_list_lock_pre_unlock; 143} 144 145void __tl_lock(void) 146{ 147 int tid = __pthread_self()->tid; 148 if (tid == TID_ERROR_0 || tid == TID_ERROR_INIT) { 149 tl_lock_tid_fail = TID_ERROR_0; 150 tid = __syscall(SYS_gettid); 151 } 152 int val = __thread_list_lock; 153 if (val == tid) { 154 tl_lock_count++; 155 tl_lock_count_tid = val; 156 return; 157 } 158 while ((val = a_cas(&__thread_list_lock, 0, tid))) 159 __wait(&__thread_list_lock, &tl_lock_waiters, val, 0); 160} 161 162void __tl_unlock(void) 163{ 164 if (tl_lock_count) { 165 tl_lock_count--; 166 return; 167 } 168 thread_list_lock_pre_unlock = __thread_list_lock; 169 a_store(&__thread_list_lock, 0); 170 if (tl_lock_waiters) __wake(&__thread_list_lock, 1, 0); 171} 172 173void __tl_sync(pthread_t td) 174{ 175 a_barrier(); 176 int val = __thread_list_lock; 177 if (!val) return; 178 __wait(&__thread_list_lock, &tl_lock_waiters, val, 0); 179 if (tl_lock_waiters) __wake(&__thread_list_lock, 1, 0); 180} 181 182#ifdef CXA_THREAD_USE_TLS 183extern void __cxa_thread_finalize(); 184#endif 185 186#ifdef ENABLE_HWASAN 187weak void __hwasan_thread_enter(); 188weak void __hwasan_thread_exit(); 189 190__attribute__((no_sanitize("hwaddress"))) 191#endif 192_Noreturn void __pthread_exit(void *result) 193{ 194#ifdef CXA_THREAD_USE_TLS 195 // Call thread_local dtors. 196 __cxa_thread_finalize(); 197#endif 198 pthread_t self = __pthread_self(); 199 sigset_t set; 200 201#ifdef FEATURE_PTHREAD_CANCEL 202 self->canceldisable = 1; 203 self->cancelasync = 0; 204#endif 205 self->result = result; 206 207 while (self->cancelbuf) { 208 void (*f)(void *) = self->cancelbuf->__f; 209 void *x = self->cancelbuf->__x; 210 self->cancelbuf = self->cancelbuf->__next; 211 f(x); 212 } 213 214 __pthread_tsd_run_dtors(); 215 216 __block_app_sigs(&set); 217 218 /* This atomic potentially competes with a concurrent pthread_detach 219 * call; the loser is responsible for freeing thread resources. */ 220 int state = a_cas(&self->detach_state, DT_JOINABLE, DT_EXITING); 221 222 if (state==DT_DETACHED && self->map_base) { 223 /* Since __unmapself bypasses the normal munmap code path, 224 * explicitly wait for vmlock holders first. This must be 225 * done before any locks are taken, to avoid lock ordering 226 * issues that could lead to deadlock. */ 227 __vm_wait(); 228 } 229 230 /* Access to target the exiting thread with syscalls that use 231 * its kernel tid is controlled by killlock. For detached threads, 232 * any use past this point would have undefined behavior, but for 233 * joinable threads it's a valid usage that must be handled. 234 * Signals must be blocked since pthread_kill must be AS-safe. */ 235 LOCK(self->killlock); 236 237 /* The thread list lock must be AS-safe, and thus depends on 238 * application signals being blocked above. */ 239 __tl_lock(); 240 241#ifdef RESERVE_SIGNAL_STACK 242 __pthread_release_signal_stack(); 243#endif 244 /* If this is the only thread in the list, don't proceed with 245 * termination of the thread, but restore the previous lock and 246 * signal state to prepare for exit to call atexit handlers. */ 247 if (self->next == self) { 248 __tl_unlock(); 249 UNLOCK(self->killlock); 250 self->detach_state = state; 251 __restore_sigs(&set); 252#ifdef ENABLE_HWASAN 253 __hwasan_thread_exit(); 254#endif 255 exit(0); 256 } 257 258 /* At this point we are committed to thread termination. */ 259 260 /* After the kernel thread exits, its tid may be reused. Clear it 261 * to prevent inadvertent use and inform functions that would use 262 * it that it's no longer available. At this point the killlock 263 * may be released, since functions that use it will consistently 264 * see the thread as having exited. Release it now so that no 265 * remaining locks (except thread list) are held if we end up 266 * resetting need_locks below. */ 267 self->tid = 0; 268 UNLOCK(self->killlock); 269 270 /* Process robust list in userspace to handle non-pshared mutexes 271 * and the detached thread case where the robust list head will 272 * be invalid when the kernel would process it. */ 273 __vm_lock(); 274 volatile void *volatile *rp; 275 while ((rp=self->robust_list.head) && rp != &self->robust_list.head) { 276 pthread_mutex_t *m = (void *)((char *)rp 277 - offsetof(pthread_mutex_t, _m_next)); 278 int waiters = m->_m_waiters; 279 int priv = (m->_m_type & 128) ^ 128; 280 self->robust_list.pending = rp; 281 self->robust_list.head = *rp; 282 int cont = a_swap(&m->_m_lock, 0x40000000); 283 self->robust_list.pending = 0; 284 if (cont < 0 || waiters) 285 __wake(&m->_m_lock, 1, priv); 286 } 287 __vm_unlock(); 288 289 __do_orphaned_stdio_locks(); 290 __dl_thread_cleanup(); 291 292 /* Last, unlink thread from the list. This change will not be visible 293 * until the lock is released, which only happens after SYS_exit 294 * has been called, via the exit futex address pointing at the lock. 295 * This needs to happen after any possible calls to LOCK() that might 296 * skip locking if process appears single-threaded. */ 297 if (!--libc.threads_minus_1) libc.need_locks = -1; 298 self->next->prev = self->prev; 299 self->prev->next = self->next; 300 self->prev = self->next = self; 301 302 if (state==DT_DETACHED && self->map_base) { 303 /* Detached threads must block even implementation-internal 304 * signals, since they will not have a stack in their last 305 * moments of existence. */ 306 __block_all_sigs(&set); 307 308 /* Robust list will no longer be valid, and was already 309 * processed above, so unregister it with the kernel. */ 310 if (self->robust_list.off) 311 __syscall(SYS_set_robust_list, 0, 3*sizeof(long)); 312 313 /* The following call unmaps the thread's stack mapping 314 * and then exits without touching the stack. */ 315 __unmapself(self->map_base, self->map_size); 316 } 317 318 /* Wake any joiner. */ 319 a_store(&self->detach_state, DT_EXITED); 320 __wake(&self->detach_state, 1, 1); 321 322#ifdef ENABLE_HWASAN 323 __hwasan_thread_exit(); 324#endif 325 326 // If a thread call __tl_lock and call __pthread_exit without 327 // call __tl_unlock, the value of tl_lock_count will appear 328 // non-zero value, here set it to zero. 329 if(tl_lock_count != 0) { 330 tl_lock_count_fail = tl_lock_count; 331 tl_lock_count = 0; 332 } 333 334 for (;;) __syscall(SYS_exit, 0); 335} 336 337void __do_cleanup_push(struct __ptcb *cb) 338{ 339 struct pthread *self = __pthread_self(); 340 cb->__next = self->cancelbuf; 341 self->cancelbuf = cb; 342} 343 344void __do_cleanup_pop(struct __ptcb *cb) 345{ 346 __pthread_self()->cancelbuf = cb->__next; 347} 348 349struct start_args { 350 void *(*start_func)(void *); 351 void *start_arg; 352 volatile int control; 353 unsigned long sig_mask[_NSIG/8/sizeof(long)]; 354}; 355 356#ifdef ENABLE_HWASAN 357__attribute__((no_sanitize("hwaddress"))) 358#endif 359static int start(void *p) 360{ 361#ifdef ENABLE_HWASAN 362 __hwasan_thread_enter(); 363#endif 364 struct start_args *args = p; 365 int state = args->control; 366 if (state) { 367 if (a_cas(&args->control, 1, 2) == 1) 368 __wait(&args->control, 0, 2, 1); 369 if (args->control) { 370 __syscall(SYS_set_tid_address, &args->control); 371 for (;;) __syscall(SYS_exit, 0); 372 } 373 } 374 __syscall(SYS_rt_sigprocmask, SIG_SETMASK, &args->sig_mask, 0, _NSIG/8); 375#ifdef RESERVE_SIGNAL_STACK 376 __pthread_reserve_signal_stack(); 377#endif 378 __pthread_exit(args->start_func(args->start_arg)); 379 return 0; 380} 381 382#ifdef ENABLE_HWASAN 383__attribute__((no_sanitize("hwaddress"))) 384#endif 385static int start_c11(void *p) 386{ 387#ifdef RESERVE_SIGNAL_STACK 388 __pthread_reserve_signal_stack(); 389#endif 390#ifdef ENABLE_HWASAN 391 __hwasan_thread_enter(); 392#endif 393 struct start_args *args = p; 394 int (*start)(void*) = (int(*)(void*)) args->start_func; 395 __pthread_exit((void *)(uintptr_t)start(args->start_arg)); 396 return 0; 397} 398 399#define ROUND(x) (((x)+PAGE_SIZE-1)&-PAGE_SIZE) 400 401/* pthread_key_create.c overrides this */ 402static volatile size_t dummy = 0; 403weak_alias(dummy, __pthread_tsd_size); 404static void *dummy_tsd[1] = { 0 }; 405weak_alias(dummy_tsd, __pthread_tsd_main); 406 407static FILE *volatile dummy_file = 0; 408weak_alias(dummy_file, __stdin_used); 409weak_alias(dummy_file, __stdout_used); 410weak_alias(dummy_file, __stderr_used); 411 412static void init_file_lock(FILE *f) 413{ 414 if (f && f->lock<0) f->lock = 0; 415} 416 417#ifdef ENABLE_HWASAN 418__attribute__((no_sanitize("hwaddress"))) 419#endif 420int __pthread_create(pthread_t *restrict res, const pthread_attr_t *restrict attrp, void *(*entry)(void *), void *restrict arg) 421{ 422 int ret, c11 = (attrp == __ATTRP_C11_THREAD); 423 size_t size, guard, size_len; 424 struct pthread *self, *new; 425 unsigned char *map = 0, *stack = 0, *tsd = 0, *stack_limit, *start_addr; 426 unsigned flags = CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGHAND 427 | CLONE_THREAD | CLONE_SYSVSEM | CLONE_SETTLS 428 | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_DETACHED; 429 pthread_attr_t attr = { 0 }; 430 sigset_t set; 431 432 if (!libc.can_do_threads) { 433 MUSL_LOGE("pthread_create: can't do threads, err: %{public}s", strerror(errno)); 434 return ENOSYS; 435 } 436 self = __pthread_self(); 437 if (!libc.threaded) { 438 for (FILE *f = *__ofl_lock(); f; f = f->next) 439 init_file_lock(f); 440 __ofl_unlock(); 441 init_file_lock(__stdin_used); 442 init_file_lock(__stdout_used); 443 init_file_lock(__stderr_used); 444 __syscall(SYS_rt_sigprocmask, SIG_UNBLOCK, SIGPT_SET, 0, _NSIG/8); 445 self->tsd = (void **)__pthread_tsd_main; 446 __membarrier_init(); 447 libc.threaded = 1; 448 } 449 if (attrp && !c11) attr = *attrp; 450 451 __acquire_ptc(); 452 if (!attrp || c11) { 453 attr._a_stacksize = __default_stacksize; 454 attr._a_guardsize = __default_guardsize; 455 } 456 457 if (attr._a_stackaddr) { 458 size_t need = libc.tls_size + __pthread_tsd_size; 459 size = attr._a_stacksize; 460 stack = (void *)(attr._a_stackaddr & -16); 461 stack_limit = (void *)(attr._a_stackaddr - size); 462 /* Use application-provided stack for TLS only when 463 * it does not take more than ~12% or 2k of the 464 * application's stack space. */ 465 if (need < size / 8 && need < 2048) { 466 tsd = stack - __pthread_tsd_size; 467 stack = tsd - libc.tls_size; 468 memset(stack, 0, need); 469 } else { 470 size = ROUND(need); 471 } 472 guard = 0; 473 } else { 474 guard = ROUND(attr._a_guardsize); 475 size = guard + ROUND(attr._a_stacksize 476 + libc.tls_size + __pthread_tsd_size); 477 } 478 479 if (!tsd) { 480 if (guard) { 481 map = __mmap(0, size, PROT_NONE, MAP_PRIVATE | MAP_ANON, -1, 0); 482 if (map == MAP_FAILED) { 483 MUSL_LOGE("pthread_create: mmap PROT_NONE failed, err:%{public}s", strerror(errno)); 484 goto fail; 485 } 486 if (__mprotect(map+guard, size-guard, PROT_READ|PROT_WRITE) 487 && errno != ENOSYS) { 488 MUSL_LOGE("pthread_create: mprotect failed, err:%{public}s", strerror(errno)); 489 __munmap(map, size); 490 goto fail; 491 } 492 } else { 493 map = __mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0); 494 if (map == MAP_FAILED) { 495 MUSL_LOGE("pthread_create: mmap PROT_READ|PROT_WRITE failed, err:%{public}s", strerror(errno)); 496 goto fail; 497 } 498 } 499 tsd = map + size - __pthread_tsd_size; 500 if (!stack) { 501 stack = tsd - libc.tls_size; 502 stack_limit = map + guard; 503 } 504 } 505 506 new = __copy_tls(tsd - libc.tls_size); 507 new->map_base = map; 508 new->map_size = size; 509 new->stack = stack; 510 new->stack_size = stack - stack_limit; 511 new->guard_size = guard; 512 new->self = new; 513 new->pid = getpid(); 514 new->proc_tid = -1; 515 new->tsd = (void *)tsd; 516 new->locale = &libc.global_locale; 517 if (attr._a_detach) { 518 new->detach_state = DT_DETACHED; 519 } else { 520 new->detach_state = DT_JOINABLE; 521 } 522 new->robust_list.head = &new->robust_list.head; 523 new->canary = self->canary; 524 new->sysinfo = self->sysinfo; 525 526 /* Setup argument structure for the new thread on its stack. 527 * It's safe to access from the caller only until the thread 528 * list is unlocked. */ 529 stack -= (uintptr_t)stack % sizeof(uintptr_t); 530 stack -= sizeof(struct start_args); 531 struct start_args *args = (void *)stack; 532 args->start_func = entry; 533 args->start_arg = arg; 534 args->control = attr._a_sched ? 1 : 0; 535 536 /* Application signals (but not the synccall signal) must be 537 * blocked before the thread list lock can be taken, to ensure 538 * that the lock is AS-safe. */ 539 __block_app_sigs(&set); 540 541 /* Ensure SIGCANCEL is unblocked in new thread. This requires 542 * working with a copy of the set so we can restore the 543 * original mask in the calling thread. */ 544 memcpy(&args->sig_mask, &set, sizeof args->sig_mask); 545 args->sig_mask[(SIGCANCEL-1)/8/sizeof(long)] &= 546 ~(1UL<<((SIGCANCEL-1)%(8*sizeof(long)))); 547 548 __tl_lock(); 549 if (!libc.threads_minus_1++) libc.need_locks = 1; 550 ret = __clone((c11 ? start_c11 : start), stack, flags, args, &new->tid, TP_ADJ(new), &__thread_list_lock); 551 552 /* All clone failures translate to EAGAIN. If explicit scheduling 553 * was requested, attempt it before unlocking the thread list so 554 * that the failed thread is never exposed and so that we can 555 * clean up all transient resource usage before returning. */ 556 if (ret < 0) { 557 ret = -EAGAIN; 558 } else if (attr._a_sched) { 559 ret = __syscall(SYS_sched_setscheduler, 560 new->tid, attr._a_policy, &attr._a_prio); 561 if (a_swap(&args->control, ret ? 3 : 0) == 2) 562 __wake(&args->control, 1, 1); 563 if (ret) 564 __wait(&args->control, 0, 3, 0); 565 } 566 567 if (ret >= 0) { 568 stack_naming(new); 569 570 new->next = self->next; 571 new->prev = self; 572 new->next->prev = new; 573 new->prev->next = new; 574 } else { 575 if (!--libc.threads_minus_1) libc.need_locks = 0; 576 } 577 __tl_unlock(); 578 __restore_sigs(&set); 579 __release_ptc(); 580 581 if (ret < 0) { 582 if (map) __munmap(map, size); 583 MUSL_LOGE("pthread_create: ret:%{public}d, err:%{public}s", ret, strerror(errno)); 584 return -ret; 585 } 586 587 *res = new; 588 return 0; 589fail: 590 __release_ptc(); 591 return EAGAIN; 592} 593 594weak_alias(__pthread_exit, pthread_exit); 595weak_alias(__pthread_create, pthread_create); 596 597struct pthread* __pthread_list_find(pthread_t thread_id, const char* info) 598{ 599 struct pthread *thread = (struct pthread *)thread_id; 600 if (NULL == thread) { 601 log_print("invalid pthread_t (0) passed to %s\n", info); 602 return NULL; 603 } 604 605 struct pthread *self = __pthread_self(); 606 if (thread == self) { 607 return thread; 608 } 609 struct pthread *t = self; 610 t = t->next ; 611 while (t != self) { 612 if (t == thread) return thread; 613 t = t->next ; 614 } 615 log_print("invalid pthread_t %p passed to %s\n", thread, info); 616 return NULL; 617} 618 619pid_t __pthread_gettid_np(pthread_t t) 620{ 621 __tl_lock(); 622 struct pthread* thread = __pthread_list_find(t, "pthread_gettid_np"); 623 __tl_unlock(); 624 return thread ? thread->tid : -1; 625} 626weak_alias(__pthread_gettid_np, pthread_gettid_np); 627