1/* Copyright (C) 2002-2018 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <signal.h>
22#include <stdint.h>
23#include <string.h>
24#include <unistd.h>
25#include <sys/mman.h>
26#include <sys/param.h>
27#include <dl-sysdep.h>
28#include <dl-tls.h>
29#include <tls.h>
30#include <list.h>
31#include <lowlevellock.h>
32#include <futex-internal.h>
33#include <kernel-features.h>
34#include <stack-aliasing.h>
35
36
37#ifndef NEED_SEPARATE_REGISTER_STACK
38
39/* Most architectures have exactly one stack pointer. Some have more. */
40# define STACK_VARIABLES void *stackaddr = NULL
41
42/* How to pass the values to the 'create_thread' function. */
43# define STACK_VARIABLES_ARGS stackaddr
44
45/* How to declare function which gets there parameters. */
46# define STACK_VARIABLES_PARMS void *stackaddr
47
48/* How to declare allocate_stack. */
49# define ALLOCATE_STACK_PARMS void **stack
50
51/* This is how the function is called. We do it this way to allow
52 other variants of the function to have more parameters. */
53# define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
54
55#else
56
57/* We need two stacks. The kernel will place them but we have to tell
58 the kernel about the size of the reserved address space. */
59# define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
60
61/* How to pass the values to the 'create_thread' function. */
62# define STACK_VARIABLES_ARGS stackaddr, stacksize
63
64/* How to declare function which gets there parameters. */
65# define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
66
67/* How to declare allocate_stack. */
68# define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
69
70/* This is how the function is called. We do it this way to allow
71 other variants of the function to have more parameters. */
72# define ALLOCATE_STACK(attr, pd) \
73 allocate_stack (attr, pd, &stackaddr, &stacksize)
74
75#endif
76
77
78/* Default alignment of stack. */
79#ifndef STACK_ALIGN
80# define STACK_ALIGN __alignof__ (long double)
81#endif
82
83/* Default value for minimal stack size after allocating thread
84 descriptor and guard. */
85#ifndef MINIMAL_REST_STACK
86# define MINIMAL_REST_STACK 4096
87#endif
88
89
90/* Newer kernels have the MAP_STACK flag to indicate a mapping is used for
91 a stack. Use it when possible. */
92#ifndef MAP_STACK
93# define MAP_STACK 0
94#endif
95
96/* This yields the pointer that TLS support code calls the thread pointer. */
97#if TLS_TCB_AT_TP
98# define TLS_TPADJ(pd) (pd)
99#elif TLS_DTV_AT_TP
100# define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
101#endif
102
103/* Cache handling for not-yet free stacks. */
104
105/* Maximum size in kB of cache. */
106static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */
107static size_t stack_cache_actsize;
108
109/* Mutex protecting this variable. */
110static int stack_cache_lock = LLL_LOCK_INITIALIZER;
111
112/* List of queued stack frames. */
113static LIST_HEAD (stack_cache);
114
115/* List of the stacks in use. */
116static LIST_HEAD (stack_used);
117
118/* We need to record what list operations we are going to do so that,
119 in case of an asynchronous interruption due to a fork() call, we
120 can correct for the work. */
121static uintptr_t in_flight_stack;
122
123/* List of the threads with user provided stacks in use. No need to
124 initialize this, since it's done in __pthread_initialize_minimal. */
125list_t __stack_user __attribute__ ((nocommon));
126hidden_data_def (__stack_user)
127
128
129/* Check whether the stack is still used or not. */
130#define FREE_P(descr) ((descr)->tid <= 0)
131
132
133static void
134stack_list_del (list_t *elem)
135{
136 in_flight_stack = (uintptr_t) elem;
137
138 atomic_write_barrier ();
139
140 list_del (elem);
141
142 atomic_write_barrier ();
143
144 in_flight_stack = 0;
145}
146
147
148static void
149stack_list_add (list_t *elem, list_t *list)
150{
151 in_flight_stack = (uintptr_t) elem | 1;
152
153 atomic_write_barrier ();
154
155 list_add (elem, list);
156
157 atomic_write_barrier ();
158
159 in_flight_stack = 0;
160}
161
162
163/* We create a double linked list of all cache entries. Double linked
164 because this allows removing entries from the end. */
165
166
167/* Get a stack frame from the cache. We have to match by size since
168 some blocks might be too small or far too large. */
169static struct pthread *
170get_cached_stack (size_t *sizep, void **memp)
171{
172 size_t size = *sizep;
173 struct pthread *result = NULL;
174 list_t *entry;
175
176 lll_lock (stack_cache_lock, LLL_PRIVATE);
177
178 /* Search the cache for a matching entry. We search for the
179 smallest stack which has at least the required size. Note that
180 in normal situations the size of all allocated stacks is the
181 same. As the very least there are only a few different sizes.
182 Therefore this loop will exit early most of the time with an
183 exact match. */
184 list_for_each (entry, &stack_cache)
185 {
186 struct pthread *curr;
187
188 curr = list_entry (entry, struct pthread, list);
189 if (FREE_P (curr) && curr->stackblock_size >= size)
190 {
191 if (curr->stackblock_size == size)
192 {
193 result = curr;
194 break;
195 }
196
197 if (result == NULL
198 || result->stackblock_size > curr->stackblock_size)
199 result = curr;
200 }
201 }
202
203 if (__builtin_expect (result == NULL, 0)
204 /* Make sure the size difference is not too excessive. In that
205 case we do not use the block. */
206 || __builtin_expect (result->stackblock_size > 4 * size, 0))
207 {
208 /* Release the lock. */
209 lll_unlock (stack_cache_lock, LLL_PRIVATE);
210
211 return NULL;
212 }
213
214 /* Don't allow setxid until cloned. */
215 result->setxid_futex = -1;
216
217 /* Dequeue the entry. */
218 stack_list_del (&result->list);
219
220 /* And add to the list of stacks in use. */
221 stack_list_add (&result->list, &stack_used);
222
223 /* And decrease the cache size. */
224 stack_cache_actsize -= result->stackblock_size;
225
226 /* Release the lock early. */
227 lll_unlock (stack_cache_lock, LLL_PRIVATE);
228
229 /* Report size and location of the stack to the caller. */
230 *sizep = result->stackblock_size;
231 *memp = result->stackblock;
232
233 /* Cancellation handling is back to the default. */
234 result->cancelhandling = 0;
235 result->cleanup = NULL;
236
237 /* No pending event. */
238 result->nextevent = NULL;
239
240 /* Clear the DTV. */
241 dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
242 for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
243 free (dtv[1 + cnt].pointer.to_free);
244 memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
245
246 /* Re-initialize the TLS. */
247 _dl_allocate_tls_init (TLS_TPADJ (result));
248
249 return result;
250}
251
252
253/* Free stacks until cache size is lower than LIMIT. */
254static void
255free_stacks (size_t limit)
256{
257 /* We reduce the size of the cache. Remove the last entries until
258 the size is below the limit. */
259 list_t *entry;
260 list_t *prev;
261
262 /* Search from the end of the list. */
263 list_for_each_prev_safe (entry, prev, &stack_cache)
264 {
265 struct pthread *curr;
266
267 curr = list_entry (entry, struct pthread, list);
268 if (FREE_P (curr))
269 {
270 /* Unlink the block. */
271 stack_list_del (entry);
272
273 /* Account for the freed memory. */
274 stack_cache_actsize -= curr->stackblock_size;
275
276 /* Free the memory associated with the ELF TLS. */
277 _dl_deallocate_tls (TLS_TPADJ (curr), false);
278
279 /* Remove this block. This should never fail. If it does
280 something is really wrong. */
281 if (__munmap (curr->stackblock, curr->stackblock_size) != 0)
282 abort ();
283
284 /* Maybe we have freed enough. */
285 if (stack_cache_actsize <= limit)
286 break;
287 }
288 }
289}
290
291/* Free all the stacks on cleanup. */
292void
293__nptl_stacks_freeres (void)
294{
295 free_stacks (0);
296}
297
298/* Add a stack frame which is not used anymore to the stack. Must be
299 called with the cache lock held. */
300static inline void
301__attribute ((always_inline))
302queue_stack (struct pthread *stack)
303{
304 /* We unconditionally add the stack to the list. The memory may
305 still be in use but it will not be reused until the kernel marks
306 the stack as not used anymore. */
307 stack_list_add (&stack->list, &stack_cache);
308
309 stack_cache_actsize += stack->stackblock_size;
310 if (__glibc_unlikely (stack_cache_actsize > stack_cache_maxsize))
311 free_stacks (stack_cache_maxsize);
312}
313
314
315static int
316change_stack_perm (struct pthread *pd
317#ifdef NEED_SEPARATE_REGISTER_STACK
318 , size_t pagemask
319#endif
320 )
321{
322#ifdef NEED_SEPARATE_REGISTER_STACK
323 void *stack = (pd->stackblock
324 + (((((pd->stackblock_size - pd->guardsize) / 2)
325 & pagemask) + pd->guardsize) & pagemask));
326 size_t len = pd->stackblock + pd->stackblock_size - stack;
327#elif _STACK_GROWS_DOWN
328 void *stack = pd->stackblock + pd->guardsize;
329 size_t len = pd->stackblock_size - pd->guardsize;
330#elif _STACK_GROWS_UP
331 void *stack = pd->stackblock;
332 size_t len = (uintptr_t) pd - pd->guardsize - (uintptr_t) pd->stackblock;
333#else
334# error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
335#endif
336 if (__mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
337 return errno;
338
339 return 0;
340}
341
342/* Return the guard page position on allocated stack. */
343static inline char *
344__attribute ((always_inline))
345guard_position (void *mem, size_t size, size_t guardsize, struct pthread *pd,
346 size_t pagesize_m1)
347{
348#ifdef NEED_SEPARATE_REGISTER_STACK
349 return mem + (((size - guardsize) / 2) & ~pagesize_m1);
350#elif _STACK_GROWS_DOWN
351 return mem;
352#elif _STACK_GROWS_UP
353 return (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
354#endif
355}
356
357/* Based on stack allocated with PROT_NONE, setup the required portions with
358 'prot' flags based on the guard page position. */
359static inline int
360setup_stack_prot (char *mem, size_t size, char *guard, size_t guardsize,
361 const int prot)
362{
363 char *guardend = guard + guardsize;
364#if _STACK_GROWS_DOWN && !defined(NEED_SEPARATE_REGISTER_STACK)
365 /* As defined at guard_position, for architectures with downward stack
366 the guard page is always at start of the allocated area. */
367 if (__mprotect (guardend, size - guardsize, prot) != 0)
368 return errno;
369#else
370 size_t mprots1 = (uintptr_t) guard - (uintptr_t) mem;
371 if (__mprotect (mem, mprots1, prot) != 0)
372 return errno;
373 size_t mprots2 = ((uintptr_t) mem + size) - (uintptr_t) guardend;
374 if (__mprotect (guardend, mprots2, prot) != 0)
375 return errno;
376#endif
377 return 0;
378}
379
380/* Mark the memory of the stack as usable to the kernel. It frees everything
381 except for the space used for the TCB itself. */
382static inline void
383__always_inline
384advise_stack_range (void *mem, size_t size, uintptr_t pd, size_t guardsize)
385{
386 uintptr_t sp = (uintptr_t) CURRENT_STACK_FRAME;
387 size_t pagesize_m1 = __getpagesize () - 1;
388#if _STACK_GROWS_DOWN && !defined(NEED_SEPARATE_REGISTER_STACK)
389 size_t freesize = (sp - (uintptr_t) mem) & ~pagesize_m1;
390 assert (freesize < size);
391 if (freesize > PTHREAD_STACK_MIN)
392 __madvise (mem, freesize - PTHREAD_STACK_MIN, MADV_DONTNEED);
393#else
394 /* Page aligned start of memory to free (higher than or equal
395 to current sp plus the minimum stack size). */
396 uintptr_t freeblock = (sp + PTHREAD_STACK_MIN + pagesize_m1) & ~pagesize_m1;
397 uintptr_t free_end = (pd - guardsize) & ~pagesize_m1;
398 if (free_end > freeblock)
399 {
400 size_t freesize = free_end - freeblock;
401 assert (freesize < size);
402 __madvise ((void*) freeblock, freesize, MADV_DONTNEED);
403 }
404#endif
405}
406
407/* Returns a usable stack for a new thread either by allocating a
408 new stack or reusing a cached stack of sufficient size.
409 ATTR must be non-NULL and point to a valid pthread_attr.
410 PDP must be non-NULL. */
411static int
412allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
413 ALLOCATE_STACK_PARMS)
414{
415 struct pthread *pd;
416 size_t size;
417 size_t pagesize_m1 = __getpagesize () - 1;
418
419 assert (powerof2 (pagesize_m1 + 1));
420 assert (TCB_ALIGNMENT >= STACK_ALIGN);
421
422 /* Get the stack size from the attribute if it is set. Otherwise we
423 use the default we determined at start time. */
424 if (attr->stacksize != 0)
425 size = attr->stacksize;
426 else
427 {
428 lll_lock (__default_pthread_attr_lock, LLL_PRIVATE);
429 size = __default_pthread_attr.stacksize;
430 lll_unlock (__default_pthread_attr_lock, LLL_PRIVATE);
431 }
432
433 /* Get memory for the stack. */
434 if (__glibc_unlikely (attr->flags & ATTR_FLAG_STACKADDR))
435 {
436 uintptr_t adj;
437 char *stackaddr = (char *) attr->stackaddr;
438
439 /* Assume the same layout as the _STACK_GROWS_DOWN case, with struct
440 pthread at the top of the stack block. Later we adjust the guard
441 location and stack address to match the _STACK_GROWS_UP case. */
442 if (_STACK_GROWS_UP)
443 stackaddr += attr->stacksize;
444
445 /* If the user also specified the size of the stack make sure it
446 is large enough. */
447 if (attr->stacksize != 0
448 && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
449 return EINVAL;
450
451 /* Adjust stack size for alignment of the TLS block. */
452#if TLS_TCB_AT_TP
453 adj = ((uintptr_t) stackaddr - TLS_TCB_SIZE)
454 & __static_tls_align_m1;
455 assert (size > adj + TLS_TCB_SIZE);
456#elif TLS_DTV_AT_TP
457 adj = ((uintptr_t) stackaddr - __static_tls_size)
458 & __static_tls_align_m1;
459 assert (size > adj);
460#endif
461
462 /* The user provided some memory. Let's hope it matches the
463 size... We do not allocate guard pages if the user provided
464 the stack. It is the user's responsibility to do this if it
465 is wanted. */
466#if TLS_TCB_AT_TP
467 pd = (struct pthread *) ((uintptr_t) stackaddr
468 - TLS_TCB_SIZE - adj);
469#elif TLS_DTV_AT_TP
470 pd = (struct pthread *) (((uintptr_t) stackaddr
471 - __static_tls_size - adj)
472 - TLS_PRE_TCB_SIZE);
473#endif
474
475 /* The user provided stack memory needs to be cleared. */
476 memset (pd, '\0', sizeof (struct pthread));
477
478 /* The first TSD block is included in the TCB. */
479 pd->specific[0] = pd->specific_1stblock;
480
481 /* Remember the stack-related values. */
482 pd->stackblock = (char *) stackaddr - size;
483 pd->stackblock_size = size;
484
485 /* This is a user-provided stack. It will not be queued in the
486 stack cache nor will the memory (except the TLS memory) be freed. */
487 pd->user_stack = true;
488
489 /* This is at least the second thread. */
490 pd->header.multiple_threads = 1;
491#ifndef TLS_MULTIPLE_THREADS_IN_TCB
492 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
493#endif
494
495#ifdef NEED_DL_SYSINFO
496 SETUP_THREAD_SYSINFO (pd);
497#endif
498
499 /* Don't allow setxid until cloned. */
500 pd->setxid_futex = -1;
501
502 /* Allocate the DTV for this thread. */
503 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
504 {
505 /* Something went wrong. */
506 assert (errno == ENOMEM);
507 return errno;
508 }
509
510
511 /* Prepare to modify global data. */
512 lll_lock (stack_cache_lock, LLL_PRIVATE);
513
514 /* And add to the list of stacks in use. */
515 list_add (&pd->list, &__stack_user);
516
517 lll_unlock (stack_cache_lock, LLL_PRIVATE);
518 }
519 else
520 {
521 /* Allocate some anonymous memory. If possible use the cache. */
522 size_t guardsize;
523 size_t reqsize;
524 void *mem;
525 const int prot = (PROT_READ | PROT_WRITE
526 | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
527
528 /* Adjust the stack size for alignment. */
529 size &= ~__static_tls_align_m1;
530 assert (size != 0);
531
532 /* Make sure the size of the stack is enough for the guard and
533 eventually the thread descriptor. */
534 guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
535 if (guardsize < attr->guardsize || size + guardsize < guardsize)
536 /* Arithmetic overflow. */
537 return EINVAL;
538 size += guardsize;
539 if (__builtin_expect (size < ((guardsize + __static_tls_size
540 + MINIMAL_REST_STACK + pagesize_m1)
541 & ~pagesize_m1),
542 0))
543 /* The stack is too small (or the guard too large). */
544 return EINVAL;
545
546 /* Try to get a stack from the cache. */
547 reqsize = size;
548 pd = get_cached_stack (&size, &mem);
549 if (pd == NULL)
550 {
551 /* To avoid aliasing effects on a larger scale than pages we
552 adjust the allocated stack size if necessary. This way
553 allocations directly following each other will not have
554 aliasing problems. */
555#if MULTI_PAGE_ALIASING != 0
556 if ((size % MULTI_PAGE_ALIASING) == 0)
557 size += pagesize_m1 + 1;
558#endif
559
560 /* If a guard page is required, avoid committing memory by first
561 allocate with PROT_NONE and then reserve with required permission
562 excluding the guard page. */
563 mem = __mmap (NULL, size, (guardsize == 0) ? prot : PROT_NONE,
564 MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
565
566 if (__glibc_unlikely (mem == MAP_FAILED))
567 return errno;
568
569 /* SIZE is guaranteed to be greater than zero.
570 So we can never get a null pointer back from mmap. */
571 assert (mem != NULL);
572
573 /* Place the thread descriptor at the end of the stack. */
574#if TLS_TCB_AT_TP
575 pd = (struct pthread *) ((((uintptr_t) mem + size)
576 - TLS_TCB_SIZE)
577 & ~__static_tls_align_m1);
578#elif TLS_DTV_AT_TP
579 pd = (struct pthread *) ((((uintptr_t) mem + size
580 - __static_tls_size)
581 & ~__static_tls_align_m1)
582 - TLS_PRE_TCB_SIZE);
583#endif
584
585 /* Now mprotect the required region excluding the guard area. */
586 if (__glibc_likely (guardsize > 0))
587 {
588 char *guard = guard_position (mem, size, guardsize, pd,
589 pagesize_m1);
590 if (setup_stack_prot (mem, size, guard, guardsize, prot) != 0)
591 {
592 __munmap (mem, size);
593 return errno;
594 }
595 }
596
597 /* Remember the stack-related values. */
598 pd->stackblock = mem;
599 pd->stackblock_size = size;
600 /* Update guardsize for newly allocated guardsize to avoid
601 an mprotect in guard resize below. */
602 pd->guardsize = guardsize;
603
604 /* We allocated the first block thread-specific data array.
605 This address will not change for the lifetime of this
606 descriptor. */
607 pd->specific[0] = pd->specific_1stblock;
608
609 /* This is at least the second thread. */
610 pd->header.multiple_threads = 1;
611#ifndef TLS_MULTIPLE_THREADS_IN_TCB
612 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
613#endif
614
615#ifdef NEED_DL_SYSINFO
616 SETUP_THREAD_SYSINFO (pd);
617#endif
618
619 /* Don't allow setxid until cloned. */
620 pd->setxid_futex = -1;
621
622 /* Allocate the DTV for this thread. */
623 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
624 {
625 /* Something went wrong. */
626 assert (errno == ENOMEM);
627
628 /* Free the stack memory we just allocated. */
629 (void) __munmap (mem, size);
630
631 return errno;
632 }
633
634
635 /* Prepare to modify global data. */
636 lll_lock (stack_cache_lock, LLL_PRIVATE);
637
638 /* And add to the list of stacks in use. */
639 stack_list_add (&pd->list, &stack_used);
640
641 lll_unlock (stack_cache_lock, LLL_PRIVATE);
642
643
644 /* There might have been a race. Another thread might have
645 caused the stacks to get exec permission while this new
646 stack was prepared. Detect if this was possible and
647 change the permission if necessary. */
648 if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
649 && (prot & PROT_EXEC) == 0, 0))
650 {
651 int err = change_stack_perm (pd
652#ifdef NEED_SEPARATE_REGISTER_STACK
653 , ~pagesize_m1
654#endif
655 );
656 if (err != 0)
657 {
658 /* Free the stack memory we just allocated. */
659 (void) __munmap (mem, size);
660
661 return err;
662 }
663 }
664
665
666 /* Note that all of the stack and the thread descriptor is
667 zeroed. This means we do not have to initialize fields
668 with initial value zero. This is specifically true for
669 the 'tid' field which is always set back to zero once the
670 stack is not used anymore and for the 'guardsize' field
671 which will be read next. */
672 }
673
674 /* Create or resize the guard area if necessary. */
675 if (__glibc_unlikely (guardsize > pd->guardsize))
676 {
677 char *guard = guard_position (mem, size, guardsize, pd,
678 pagesize_m1);
679 if (__mprotect (guard, guardsize, PROT_NONE) != 0)
680 {
681 mprot_error:
682 lll_lock (stack_cache_lock, LLL_PRIVATE);
683
684 /* Remove the thread from the list. */
685 stack_list_del (&pd->list);
686
687 lll_unlock (stack_cache_lock, LLL_PRIVATE);
688
689 /* Get rid of the TLS block we allocated. */
690 _dl_deallocate_tls (TLS_TPADJ (pd), false);
691
692 /* Free the stack memory regardless of whether the size
693 of the cache is over the limit or not. If this piece
694 of memory caused problems we better do not use it
695 anymore. Uh, and we ignore possible errors. There
696 is nothing we could do. */
697 (void) __munmap (mem, size);
698
699 return errno;
700 }
701
702 pd->guardsize = guardsize;
703 }
704 else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
705 0))
706 {
707 /* The old guard area is too large. */
708
709#ifdef NEED_SEPARATE_REGISTER_STACK
710 char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
711 char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
712
713 if (oldguard < guard
714 && __mprotect (oldguard, guard - oldguard, prot) != 0)
715 goto mprot_error;
716
717 if (__mprotect (guard + guardsize,
718 oldguard + pd->guardsize - guard - guardsize,
719 prot) != 0)
720 goto mprot_error;
721#elif _STACK_GROWS_DOWN
722 if (__mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
723 prot) != 0)
724 goto mprot_error;
725#elif _STACK_GROWS_UP
726 char *new_guard = (char *)(((uintptr_t) pd - guardsize)
727 & ~pagesize_m1);
728 char *old_guard = (char *)(((uintptr_t) pd - pd->guardsize)
729 & ~pagesize_m1);
730 /* The guard size difference might be > 0, but once rounded
731 to the nearest page the size difference might be zero. */
732 if (new_guard > old_guard
733 && __mprotect (old_guard, new_guard - old_guard, prot) != 0)
734 goto mprot_error;
735#endif
736
737 pd->guardsize = guardsize;
738 }
739 /* The pthread_getattr_np() calls need to get passed the size
740 requested in the attribute, regardless of how large the
741 actually used guardsize is. */
742 pd->reported_guardsize = guardsize;
743 }
744
745 /* Initialize the lock. We have to do this unconditionally since the
746 stillborn thread could be canceled while the lock is taken. */
747 pd->lock = LLL_LOCK_INITIALIZER;
748
749 /* The robust mutex lists also need to be initialized
750 unconditionally because the cleanup for the previous stack owner
751 might have happened in the kernel. */
752 pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
753 - offsetof (pthread_mutex_t,
754 __data.__list.__next));
755 pd->robust_head.list_op_pending = NULL;
756#if __PTHREAD_MUTEX_HAVE_PREV
757 pd->robust_prev = &pd->robust_head;
758#endif
759 pd->robust_head.list = &pd->robust_head;
760
761 /* We place the thread descriptor at the end of the stack. */
762 *pdp = pd;
763
764#if _STACK_GROWS_DOWN
765 void *stacktop;
766
767# if TLS_TCB_AT_TP
768 /* The stack begins before the TCB and the static TLS block. */
769 stacktop = ((char *) (pd + 1) - __static_tls_size);
770# elif TLS_DTV_AT_TP
771 stacktop = (char *) (pd - 1);
772# endif
773
774# ifdef NEED_SEPARATE_REGISTER_STACK
775 *stack = pd->stackblock;
776 *stacksize = stacktop - *stack;
777# else
778 *stack = stacktop;
779# endif
780#else
781 *stack = pd->stackblock;
782#endif
783
784 return 0;
785}
786
787
788void
789__deallocate_stack (struct pthread *pd)
790{
791 lll_lock (stack_cache_lock, LLL_PRIVATE);
792
793 /* Remove the thread from the list of threads with user defined
794 stacks. */
795 stack_list_del (&pd->list);
796
797 /* Not much to do. Just free the mmap()ed memory. Note that we do
798 not reset the 'used' flag in the 'tid' field. This is done by
799 the kernel. If no thread has been created yet this field is
800 still zero. */
801 if (__glibc_likely (! pd->user_stack))
802 (void) queue_stack (pd);
803 else
804 /* Free the memory associated with the ELF TLS. */
805 _dl_deallocate_tls (TLS_TPADJ (pd), false);
806
807 lll_unlock (stack_cache_lock, LLL_PRIVATE);
808}
809
810
811int
812__make_stacks_executable (void **stack_endp)
813{
814 /* First the main thread's stack. */
815 int err = _dl_make_stack_executable (stack_endp);
816 if (err != 0)
817 return err;
818
819#ifdef NEED_SEPARATE_REGISTER_STACK
820 const size_t pagemask = ~(__getpagesize () - 1);
821#endif
822
823 lll_lock (stack_cache_lock, LLL_PRIVATE);
824
825 list_t *runp;
826 list_for_each (runp, &stack_used)
827 {
828 err = change_stack_perm (list_entry (runp, struct pthread, list)
829#ifdef NEED_SEPARATE_REGISTER_STACK
830 , pagemask
831#endif
832 );
833 if (err != 0)
834 break;
835 }
836
837 /* Also change the permission for the currently unused stacks. This
838 might be wasted time but better spend it here than adding a check
839 in the fast path. */
840 if (err == 0)
841 list_for_each (runp, &stack_cache)
842 {
843 err = change_stack_perm (list_entry (runp, struct pthread, list)
844#ifdef NEED_SEPARATE_REGISTER_STACK
845 , pagemask
846#endif
847 );
848 if (err != 0)
849 break;
850 }
851
852 lll_unlock (stack_cache_lock, LLL_PRIVATE);
853
854 return err;
855}
856
857
858/* In case of a fork() call the memory allocation in the child will be
859 the same but only one thread is running. All stacks except that of
860 the one running thread are not used anymore. We have to recycle
861 them. */
862void
863__reclaim_stacks (void)
864{
865 struct pthread *self = (struct pthread *) THREAD_SELF;
866
867 /* No locking necessary. The caller is the only stack in use. But
868 we have to be aware that we might have interrupted a list
869 operation. */
870
871 if (in_flight_stack != 0)
872 {
873 bool add_p = in_flight_stack & 1;
874 list_t *elem = (list_t *) (in_flight_stack & ~(uintptr_t) 1);
875
876 if (add_p)
877 {
878 /* We always add at the beginning of the list. So in this case we
879 only need to check the beginning of these lists to see if the
880 pointers at the head of the list are inconsistent. */
881 list_t *l = NULL;
882
883 if (stack_used.next->prev != &stack_used)
884 l = &stack_used;
885 else if (stack_cache.next->prev != &stack_cache)
886 l = &stack_cache;
887
888 if (l != NULL)
889 {
890 assert (l->next->prev == elem);
891 elem->next = l->next;
892 elem->prev = l;
893 l->next = elem;
894 }
895 }
896 else
897 {
898 /* We can simply always replay the delete operation. */
899 elem->next->prev = elem->prev;
900 elem->prev->next = elem->next;
901 }
902 }
903
904 /* Mark all stacks except the still running one as free. */
905 list_t *runp;
906 list_for_each (runp, &stack_used)
907 {
908 struct pthread *curp = list_entry (runp, struct pthread, list);
909 if (curp != self)
910 {
911 /* This marks the stack as free. */
912 curp->tid = 0;
913
914 /* Account for the size of the stack. */
915 stack_cache_actsize += curp->stackblock_size;
916
917 if (curp->specific_used)
918 {
919 /* Clear the thread-specific data. */
920 memset (curp->specific_1stblock, '\0',
921 sizeof (curp->specific_1stblock));
922
923 curp->specific_used = false;
924
925 for (size_t cnt = 1; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt)
926 if (curp->specific[cnt] != NULL)
927 {
928 memset (curp->specific[cnt], '\0',
929 sizeof (curp->specific_1stblock));
930
931 /* We have allocated the block which we do not
932 free here so re-set the bit. */
933 curp->specific_used = true;
934 }
935 }
936 }
937 }
938
939 /* Add the stack of all running threads to the cache. */
940 list_splice (&stack_used, &stack_cache);
941
942 /* Remove the entry for the current thread to from the cache list
943 and add it to the list of running threads. Which of the two
944 lists is decided by the user_stack flag. */
945 stack_list_del (&self->list);
946
947 /* Re-initialize the lists for all the threads. */
948 INIT_LIST_HEAD (&stack_used);
949 INIT_LIST_HEAD (&__stack_user);
950
951 if (__glibc_unlikely (THREAD_GETMEM (self, user_stack)))
952 list_add (&self->list, &__stack_user);
953 else
954 list_add (&self->list, &stack_used);
955
956 /* There is one thread running. */
957 __nptl_nthreads = 1;
958
959 in_flight_stack = 0;
960
961 /* Initialize locks. */
962 stack_cache_lock = LLL_LOCK_INITIALIZER;
963 __default_pthread_attr_lock = LLL_LOCK_INITIALIZER;
964}
965
966
967#if HP_TIMING_AVAIL
968# undef __find_thread_by_id
969/* Find a thread given the thread ID. */
970attribute_hidden
971struct pthread *
972__find_thread_by_id (pid_t tid)
973{
974 struct pthread *result = NULL;
975
976 lll_lock (stack_cache_lock, LLL_PRIVATE);
977
978 /* Iterate over the list with system-allocated threads first. */
979 list_t *runp;
980 list_for_each (runp, &stack_used)
981 {
982 struct pthread *curp;
983
984 curp = list_entry (runp, struct pthread, list);
985
986 if (curp->tid == tid)
987 {
988 result = curp;
989 goto out;
990 }
991 }
992
993 /* Now the list with threads using user-allocated stacks. */
994 list_for_each (runp, &__stack_user)
995 {
996 struct pthread *curp;
997
998 curp = list_entry (runp, struct pthread, list);
999
1000 if (curp->tid == tid)
1001 {
1002 result = curp;
1003 goto out;
1004 }
1005 }
1006
1007 out:
1008 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1009
1010 return result;
1011}
1012#endif
1013
1014
1015#ifdef SIGSETXID
1016static void
1017setxid_mark_thread (struct xid_command *cmdp, struct pthread *t)
1018{
1019 int ch;
1020
1021 /* Wait until this thread is cloned. */
1022 if (t->setxid_futex == -1
1023 && ! atomic_compare_and_exchange_bool_acq (&t->setxid_futex, -2, -1))
1024 do
1025 futex_wait_simple (&t->setxid_futex, -2, FUTEX_PRIVATE);
1026 while (t->setxid_futex == -2);
1027
1028 /* Don't let the thread exit before the setxid handler runs. */
1029 t->setxid_futex = 0;
1030
1031 do
1032 {
1033 ch = t->cancelhandling;
1034
1035 /* If the thread is exiting right now, ignore it. */
1036 if ((ch & EXITING_BITMASK) != 0)
1037 {
1038 /* Release the futex if there is no other setxid in
1039 progress. */
1040 if ((ch & SETXID_BITMASK) == 0)
1041 {
1042 t->setxid_futex = 1;
1043 futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE);
1044 }
1045 return;
1046 }
1047 }
1048 while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
1049 ch | SETXID_BITMASK, ch));
1050}
1051
1052
1053static void
1054setxid_unmark_thread (struct xid_command *cmdp, struct pthread *t)
1055{
1056 int ch;
1057
1058 do
1059 {
1060 ch = t->cancelhandling;
1061 if ((ch & SETXID_BITMASK) == 0)
1062 return;
1063 }
1064 while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
1065 ch & ~SETXID_BITMASK, ch));
1066
1067 /* Release the futex just in case. */
1068 t->setxid_futex = 1;
1069 futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE);
1070}
1071
1072
1073static int
1074setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
1075{
1076 if ((t->cancelhandling & SETXID_BITMASK) == 0)
1077 return 0;
1078
1079 int val;
1080 pid_t pid = __getpid ();
1081 INTERNAL_SYSCALL_DECL (err);
1082 val = INTERNAL_SYSCALL_CALL (tgkill, err, pid, t->tid, SIGSETXID);
1083
1084 /* If this failed, it must have had not started yet or else exited. */
1085 if (!INTERNAL_SYSCALL_ERROR_P (val, err))
1086 {
1087 atomic_increment (&cmdp->cntr);
1088 return 1;
1089 }
1090 else
1091 return 0;
1092}
1093
1094/* Check for consistency across set*id system call results. The abort
1095 should not happen as long as all privileges changes happen through
1096 the glibc wrappers. ERROR must be 0 (no error) or an errno
1097 code. */
1098void
1099attribute_hidden
1100__nptl_setxid_error (struct xid_command *cmdp, int error)
1101{
1102 do
1103 {
1104 int olderror = cmdp->error;
1105 if (olderror == error)
1106 break;
1107 if (olderror != -1)
1108 {
1109 /* Mismatch between current and previous results. Save the
1110 error value to memory so that is not clobbered by the
1111 abort function and preserved in coredumps. */
1112 volatile int xid_err __attribute__((unused)) = error;
1113 abort ();
1114 }
1115 }
1116 while (atomic_compare_and_exchange_bool_acq (&cmdp->error, error, -1));
1117}
1118
1119int
1120attribute_hidden
1121__nptl_setxid (struct xid_command *cmdp)
1122{
1123 int signalled;
1124 int result;
1125 lll_lock (stack_cache_lock, LLL_PRIVATE);
1126
1127 __xidcmd = cmdp;
1128 cmdp->cntr = 0;
1129 cmdp->error = -1;
1130
1131 struct pthread *self = THREAD_SELF;
1132
1133 /* Iterate over the list with system-allocated threads first. */
1134 list_t *runp;
1135 list_for_each (runp, &stack_used)
1136 {
1137 struct pthread *t = list_entry (runp, struct pthread, list);
1138 if (t == self)
1139 continue;
1140
1141 setxid_mark_thread (cmdp, t);
1142 }
1143
1144 /* Now the list with threads using user-allocated stacks. */
1145 list_for_each (runp, &__stack_user)
1146 {
1147 struct pthread *t = list_entry (runp, struct pthread, list);
1148 if (t == self)
1149 continue;
1150
1151 setxid_mark_thread (cmdp, t);
1152 }
1153
1154 /* Iterate until we don't succeed in signalling anyone. That means
1155 we have gotten all running threads, and their children will be
1156 automatically correct once started. */
1157 do
1158 {
1159 signalled = 0;
1160
1161 list_for_each (runp, &stack_used)
1162 {
1163 struct pthread *t = list_entry (runp, struct pthread, list);
1164 if (t == self)
1165 continue;
1166
1167 signalled += setxid_signal_thread (cmdp, t);
1168 }
1169
1170 list_for_each (runp, &__stack_user)
1171 {
1172 struct pthread *t = list_entry (runp, struct pthread, list);
1173 if (t == self)
1174 continue;
1175
1176 signalled += setxid_signal_thread (cmdp, t);
1177 }
1178
1179 int cur = cmdp->cntr;
1180 while (cur != 0)
1181 {
1182 futex_wait_simple ((unsigned int *) &cmdp->cntr, cur,
1183 FUTEX_PRIVATE);
1184 cur = cmdp->cntr;
1185 }
1186 }
1187 while (signalled != 0);
1188
1189 /* Clean up flags, so that no thread blocks during exit waiting
1190 for a signal which will never come. */
1191 list_for_each (runp, &stack_used)
1192 {
1193 struct pthread *t = list_entry (runp, struct pthread, list);
1194 if (t == self)
1195 continue;
1196
1197 setxid_unmark_thread (cmdp, t);
1198 }
1199
1200 list_for_each (runp, &__stack_user)
1201 {
1202 struct pthread *t = list_entry (runp, struct pthread, list);
1203 if (t == self)
1204 continue;
1205
1206 setxid_unmark_thread (cmdp, t);
1207 }
1208
1209 /* This must be last, otherwise the current thread might not have
1210 permissions to send SIGSETXID syscall to the other threads. */
1211 INTERNAL_SYSCALL_DECL (err);
1212 result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
1213 cmdp->id[0], cmdp->id[1], cmdp->id[2]);
1214 int error = 0;
1215 if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, err)))
1216 {
1217 error = INTERNAL_SYSCALL_ERRNO (result, err);
1218 __set_errno (error);
1219 result = -1;
1220 }
1221 __nptl_setxid_error (cmdp, error);
1222
1223 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1224 return result;
1225}
1226#endif /* SIGSETXID. */
1227
1228
1229static inline void __attribute__((always_inline))
1230init_one_static_tls (struct pthread *curp, struct link_map *map)
1231{
1232# if TLS_TCB_AT_TP
1233 void *dest = (char *) curp - map->l_tls_offset;
1234# elif TLS_DTV_AT_TP
1235 void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
1236# else
1237# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
1238# endif
1239
1240 /* Initialize the memory. */
1241 memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
1242 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
1243}
1244
1245void
1246attribute_hidden
1247__pthread_init_static_tls (struct link_map *map)
1248{
1249 lll_lock (stack_cache_lock, LLL_PRIVATE);
1250
1251 /* Iterate over the list with system-allocated threads first. */
1252 list_t *runp;
1253 list_for_each (runp, &stack_used)
1254 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1255
1256 /* Now the list with threads using user-allocated stacks. */
1257 list_for_each (runp, &__stack_user)
1258 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1259
1260 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1261}
1262
1263
1264void
1265attribute_hidden
1266__wait_lookup_done (void)
1267{
1268 lll_lock (stack_cache_lock, LLL_PRIVATE);
1269
1270 struct pthread *self = THREAD_SELF;
1271
1272 /* Iterate over the list with system-allocated threads first. */
1273 list_t *runp;
1274 list_for_each (runp, &stack_used)
1275 {
1276 struct pthread *t = list_entry (runp, struct pthread, list);
1277 if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1278 continue;
1279
1280 int *const gscope_flagp = &t->header.gscope_flag;
1281
1282 /* We have to wait until this thread is done with the global
1283 scope. First tell the thread that we are waiting and
1284 possibly have to be woken. */
1285 if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1286 THREAD_GSCOPE_FLAG_WAIT,
1287 THREAD_GSCOPE_FLAG_USED))
1288 continue;
1289
1290 do
1291 futex_wait_simple ((unsigned int *) gscope_flagp,
1292 THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
1293 while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1294 }
1295
1296 /* Now the list with threads using user-allocated stacks. */
1297 list_for_each (runp, &__stack_user)
1298 {
1299 struct pthread *t = list_entry (runp, struct pthread, list);
1300 if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1301 continue;
1302
1303 int *const gscope_flagp = &t->header.gscope_flag;
1304
1305 /* We have to wait until this thread is done with the global
1306 scope. First tell the thread that we are waiting and
1307 possibly have to be woken. */
1308 if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1309 THREAD_GSCOPE_FLAG_WAIT,
1310 THREAD_GSCOPE_FLAG_USED))
1311 continue;
1312
1313 do
1314 futex_wait_simple ((unsigned int *) gscope_flagp,
1315 THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE);
1316 while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1317 }
1318
1319 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1320}
1321