1 | /* Copyright (C) 2002-2018 Free Software Foundation, Inc. |
2 | This file is part of the GNU C Library. |
3 | Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ |
18 | |
19 | #include <assert.h> |
20 | #include <errno.h> |
21 | #include <signal.h> |
22 | #include <stdint.h> |
23 | #include <string.h> |
24 | #include <unistd.h> |
25 | #include <sys/mman.h> |
26 | #include <sys/param.h> |
27 | #include <dl-sysdep.h> |
28 | #include <dl-tls.h> |
29 | #include <tls.h> |
30 | #include <list.h> |
31 | #include <lowlevellock.h> |
32 | #include <futex-internal.h> |
33 | #include <kernel-features.h> |
34 | #include <stack-aliasing.h> |
35 | |
36 | |
37 | #ifndef NEED_SEPARATE_REGISTER_STACK |
38 | |
39 | /* Most architectures have exactly one stack pointer. Some have more. */ |
40 | # define STACK_VARIABLES void *stackaddr = NULL |
41 | |
42 | /* How to pass the values to the 'create_thread' function. */ |
43 | # define STACK_VARIABLES_ARGS stackaddr |
44 | |
45 | /* How to declare function which gets there parameters. */ |
46 | # define STACK_VARIABLES_PARMS void *stackaddr |
47 | |
48 | /* How to declare allocate_stack. */ |
49 | # define ALLOCATE_STACK_PARMS void **stack |
50 | |
51 | /* This is how the function is called. We do it this way to allow |
52 | other variants of the function to have more parameters. */ |
53 | # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr) |
54 | |
55 | #else |
56 | |
57 | /* We need two stacks. The kernel will place them but we have to tell |
58 | the kernel about the size of the reserved address space. */ |
59 | # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0 |
60 | |
61 | /* How to pass the values to the 'create_thread' function. */ |
62 | # define STACK_VARIABLES_ARGS stackaddr, stacksize |
63 | |
64 | /* How to declare function which gets there parameters. */ |
65 | # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize |
66 | |
67 | /* How to declare allocate_stack. */ |
68 | # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize |
69 | |
70 | /* This is how the function is called. We do it this way to allow |
71 | other variants of the function to have more parameters. */ |
72 | # define ALLOCATE_STACK(attr, pd) \ |
73 | allocate_stack (attr, pd, &stackaddr, &stacksize) |
74 | |
75 | #endif |
76 | |
77 | |
78 | /* Default alignment of stack. */ |
79 | #ifndef STACK_ALIGN |
80 | # define STACK_ALIGN __alignof__ (long double) |
81 | #endif |
82 | |
83 | /* Default value for minimal stack size after allocating thread |
84 | descriptor and guard. */ |
85 | #ifndef MINIMAL_REST_STACK |
86 | # define MINIMAL_REST_STACK 4096 |
87 | #endif |
88 | |
89 | |
90 | /* Newer kernels have the MAP_STACK flag to indicate a mapping is used for |
91 | a stack. Use it when possible. */ |
92 | #ifndef MAP_STACK |
93 | # define MAP_STACK 0 |
94 | #endif |
95 | |
96 | /* This yields the pointer that TLS support code calls the thread pointer. */ |
97 | #if TLS_TCB_AT_TP |
98 | # define TLS_TPADJ(pd) (pd) |
99 | #elif TLS_DTV_AT_TP |
100 | # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE)) |
101 | #endif |
102 | |
103 | /* Cache handling for not-yet free stacks. */ |
104 | |
105 | /* Maximum size in kB of cache. */ |
106 | static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */ |
107 | static size_t stack_cache_actsize; |
108 | |
109 | /* Mutex protecting this variable. */ |
110 | static int stack_cache_lock = LLL_LOCK_INITIALIZER; |
111 | |
112 | /* List of queued stack frames. */ |
113 | static LIST_HEAD (stack_cache); |
114 | |
115 | /* List of the stacks in use. */ |
116 | static LIST_HEAD (stack_used); |
117 | |
118 | /* We need to record what list operations we are going to do so that, |
119 | in case of an asynchronous interruption due to a fork() call, we |
120 | can correct for the work. */ |
121 | static uintptr_t in_flight_stack; |
122 | |
123 | /* List of the threads with user provided stacks in use. No need to |
124 | initialize this, since it's done in __pthread_initialize_minimal. */ |
125 | list_t __stack_user __attribute__ ((nocommon)); |
126 | hidden_data_def (__stack_user) |
127 | |
128 | |
129 | /* Check whether the stack is still used or not. */ |
130 | #define FREE_P(descr) ((descr)->tid <= 0) |
131 | |
132 | |
133 | static void |
134 | stack_list_del (list_t *elem) |
135 | { |
136 | in_flight_stack = (uintptr_t) elem; |
137 | |
138 | atomic_write_barrier (); |
139 | |
140 | list_del (elem); |
141 | |
142 | atomic_write_barrier (); |
143 | |
144 | in_flight_stack = 0; |
145 | } |
146 | |
147 | |
148 | static void |
149 | stack_list_add (list_t *elem, list_t *list) |
150 | { |
151 | in_flight_stack = (uintptr_t) elem | 1; |
152 | |
153 | atomic_write_barrier (); |
154 | |
155 | list_add (elem, list); |
156 | |
157 | atomic_write_barrier (); |
158 | |
159 | in_flight_stack = 0; |
160 | } |
161 | |
162 | |
163 | /* We create a double linked list of all cache entries. Double linked |
164 | because this allows removing entries from the end. */ |
165 | |
166 | |
167 | /* Get a stack frame from the cache. We have to match by size since |
168 | some blocks might be too small or far too large. */ |
169 | static struct pthread * |
170 | get_cached_stack (size_t *sizep, void **memp) |
171 | { |
172 | size_t size = *sizep; |
173 | struct pthread *result = NULL; |
174 | list_t *entry; |
175 | |
176 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
177 | |
178 | /* Search the cache for a matching entry. We search for the |
179 | smallest stack which has at least the required size. Note that |
180 | in normal situations the size of all allocated stacks is the |
181 | same. As the very least there are only a few different sizes. |
182 | Therefore this loop will exit early most of the time with an |
183 | exact match. */ |
184 | list_for_each (entry, &stack_cache) |
185 | { |
186 | struct pthread *curr; |
187 | |
188 | curr = list_entry (entry, struct pthread, list); |
189 | if (FREE_P (curr) && curr->stackblock_size >= size) |
190 | { |
191 | if (curr->stackblock_size == size) |
192 | { |
193 | result = curr; |
194 | break; |
195 | } |
196 | |
197 | if (result == NULL |
198 | || result->stackblock_size > curr->stackblock_size) |
199 | result = curr; |
200 | } |
201 | } |
202 | |
203 | if (__builtin_expect (result == NULL, 0) |
204 | /* Make sure the size difference is not too excessive. In that |
205 | case we do not use the block. */ |
206 | || __builtin_expect (result->stackblock_size > 4 * size, 0)) |
207 | { |
208 | /* Release the lock. */ |
209 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
210 | |
211 | return NULL; |
212 | } |
213 | |
214 | /* Don't allow setxid until cloned. */ |
215 | result->setxid_futex = -1; |
216 | |
217 | /* Dequeue the entry. */ |
218 | stack_list_del (&result->list); |
219 | |
220 | /* And add to the list of stacks in use. */ |
221 | stack_list_add (&result->list, &stack_used); |
222 | |
223 | /* And decrease the cache size. */ |
224 | stack_cache_actsize -= result->stackblock_size; |
225 | |
226 | /* Release the lock early. */ |
227 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
228 | |
229 | /* Report size and location of the stack to the caller. */ |
230 | *sizep = result->stackblock_size; |
231 | *memp = result->stackblock; |
232 | |
233 | /* Cancellation handling is back to the default. */ |
234 | result->cancelhandling = 0; |
235 | result->cleanup = NULL; |
236 | |
237 | /* No pending event. */ |
238 | result->nextevent = NULL; |
239 | |
240 | /* Clear the DTV. */ |
241 | dtv_t *dtv = GET_DTV (TLS_TPADJ (result)); |
242 | for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt) |
243 | free (dtv[1 + cnt].pointer.to_free); |
244 | memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t)); |
245 | |
246 | /* Re-initialize the TLS. */ |
247 | _dl_allocate_tls_init (TLS_TPADJ (result)); |
248 | |
249 | return result; |
250 | } |
251 | |
252 | |
253 | /* Free stacks until cache size is lower than LIMIT. */ |
254 | void |
255 | __free_stacks (size_t limit) |
256 | { |
257 | /* We reduce the size of the cache. Remove the last entries until |
258 | the size is below the limit. */ |
259 | list_t *entry; |
260 | list_t *prev; |
261 | |
262 | /* Search from the end of the list. */ |
263 | list_for_each_prev_safe (entry, prev, &stack_cache) |
264 | { |
265 | struct pthread *curr; |
266 | |
267 | curr = list_entry (entry, struct pthread, list); |
268 | if (FREE_P (curr)) |
269 | { |
270 | /* Unlink the block. */ |
271 | stack_list_del (entry); |
272 | |
273 | /* Account for the freed memory. */ |
274 | stack_cache_actsize -= curr->stackblock_size; |
275 | |
276 | /* Free the memory associated with the ELF TLS. */ |
277 | _dl_deallocate_tls (TLS_TPADJ (curr), false); |
278 | |
279 | /* Remove this block. This should never fail. If it does |
280 | something is really wrong. */ |
281 | if (__munmap (curr->stackblock, curr->stackblock_size) != 0) |
282 | abort (); |
283 | |
284 | /* Maybe we have freed enough. */ |
285 | if (stack_cache_actsize <= limit) |
286 | break; |
287 | } |
288 | } |
289 | } |
290 | |
291 | |
292 | /* Add a stack frame which is not used anymore to the stack. Must be |
293 | called with the cache lock held. */ |
294 | static inline void |
295 | __attribute ((always_inline)) |
296 | queue_stack (struct pthread *stack) |
297 | { |
298 | /* We unconditionally add the stack to the list. The memory may |
299 | still be in use but it will not be reused until the kernel marks |
300 | the stack as not used anymore. */ |
301 | stack_list_add (&stack->list, &stack_cache); |
302 | |
303 | stack_cache_actsize += stack->stackblock_size; |
304 | if (__glibc_unlikely (stack_cache_actsize > stack_cache_maxsize)) |
305 | __free_stacks (stack_cache_maxsize); |
306 | } |
307 | |
308 | |
309 | static int |
310 | change_stack_perm (struct pthread *pd |
311 | #ifdef NEED_SEPARATE_REGISTER_STACK |
312 | , size_t pagemask |
313 | #endif |
314 | ) |
315 | { |
316 | #ifdef NEED_SEPARATE_REGISTER_STACK |
317 | void *stack = (pd->stackblock |
318 | + (((((pd->stackblock_size - pd->guardsize) / 2) |
319 | & pagemask) + pd->guardsize) & pagemask)); |
320 | size_t len = pd->stackblock + pd->stackblock_size - stack; |
321 | #elif _STACK_GROWS_DOWN |
322 | void *stack = pd->stackblock + pd->guardsize; |
323 | size_t len = pd->stackblock_size - pd->guardsize; |
324 | #elif _STACK_GROWS_UP |
325 | void *stack = pd->stackblock; |
326 | size_t len = (uintptr_t) pd - pd->guardsize - (uintptr_t) pd->stackblock; |
327 | #else |
328 | # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP" |
329 | #endif |
330 | if (__mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0) |
331 | return errno; |
332 | |
333 | return 0; |
334 | } |
335 | |
336 | /* Return the guard page position on allocated stack. */ |
337 | static inline char * |
338 | __attribute ((always_inline)) |
339 | guard_position (void *mem, size_t size, size_t guardsize, struct pthread *pd, |
340 | size_t pagesize_m1) |
341 | { |
342 | #ifdef NEED_SEPARATE_REGISTER_STACK |
343 | return mem + (((size - guardsize) / 2) & ~pagesize_m1); |
344 | #elif _STACK_GROWS_DOWN |
345 | return mem; |
346 | #elif _STACK_GROWS_UP |
347 | return (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1); |
348 | #endif |
349 | } |
350 | |
351 | /* Based on stack allocated with PROT_NONE, setup the required portions with |
352 | 'prot' flags based on the guard page position. */ |
353 | static inline int |
354 | setup_stack_prot (char *mem, size_t size, char *guard, size_t guardsize, |
355 | const int prot) |
356 | { |
357 | char *guardend = guard + guardsize; |
358 | #if _STACK_GROWS_DOWN && !defined(NEED_SEPARATE_REGISTER_STACK) |
359 | /* As defined at guard_position, for architectures with downward stack |
360 | the guard page is always at start of the allocated area. */ |
361 | if (__mprotect (guardend, size - guardsize, prot) != 0) |
362 | return errno; |
363 | #else |
364 | size_t mprots1 = (uintptr_t) guard - (uintptr_t) mem; |
365 | if (__mprotect (mem, mprots1, prot) != 0) |
366 | return errno; |
367 | size_t mprots2 = ((uintptr_t) mem + size) - (uintptr_t) guardend; |
368 | if (__mprotect (guardend, mprots2, prot) != 0) |
369 | return errno; |
370 | #endif |
371 | return 0; |
372 | } |
373 | |
374 | /* Mark the memory of the stack as usable to the kernel. It frees everything |
375 | except for the space used for the TCB itself. */ |
376 | static inline void |
377 | __always_inline |
378 | advise_stack_range (void *mem, size_t size, uintptr_t pd, size_t guardsize) |
379 | { |
380 | uintptr_t sp = (uintptr_t) CURRENT_STACK_FRAME; |
381 | size_t pagesize_m1 = __getpagesize () - 1; |
382 | #if _STACK_GROWS_DOWN && !defined(NEED_SEPARATE_REGISTER_STACK) |
383 | size_t freesize = (sp - (uintptr_t) mem) & ~pagesize_m1; |
384 | assert (freesize < size); |
385 | if (freesize > PTHREAD_STACK_MIN) |
386 | __madvise (mem, freesize - PTHREAD_STACK_MIN, MADV_DONTNEED); |
387 | #else |
388 | /* Page aligned start of memory to free (higher than or equal |
389 | to current sp plus the minimum stack size). */ |
390 | uintptr_t freeblock = (sp + PTHREAD_STACK_MIN + pagesize_m1) & ~pagesize_m1; |
391 | uintptr_t free_end = (pd - guardsize) & ~pagesize_m1; |
392 | if (free_end > freeblock) |
393 | { |
394 | size_t freesize = free_end - freeblock; |
395 | assert (freesize < size); |
396 | __madvise ((void*) freeblock, freesize, MADV_DONTNEED); |
397 | } |
398 | #endif |
399 | } |
400 | |
401 | /* Returns a usable stack for a new thread either by allocating a |
402 | new stack or reusing a cached stack of sufficient size. |
403 | ATTR must be non-NULL and point to a valid pthread_attr. |
404 | PDP must be non-NULL. */ |
405 | static int |
406 | allocate_stack (const struct pthread_attr *attr, struct pthread **pdp, |
407 | ALLOCATE_STACK_PARMS) |
408 | { |
409 | struct pthread *pd; |
410 | size_t size; |
411 | size_t pagesize_m1 = __getpagesize () - 1; |
412 | |
413 | assert (powerof2 (pagesize_m1 + 1)); |
414 | assert (TCB_ALIGNMENT >= STACK_ALIGN); |
415 | |
416 | /* Get the stack size from the attribute if it is set. Otherwise we |
417 | use the default we determined at start time. */ |
418 | if (attr->stacksize != 0) |
419 | size = attr->stacksize; |
420 | else |
421 | { |
422 | lll_lock (__default_pthread_attr_lock, LLL_PRIVATE); |
423 | size = __default_pthread_attr.stacksize; |
424 | lll_unlock (__default_pthread_attr_lock, LLL_PRIVATE); |
425 | } |
426 | |
427 | /* Get memory for the stack. */ |
428 | if (__glibc_unlikely (attr->flags & ATTR_FLAG_STACKADDR)) |
429 | { |
430 | uintptr_t adj; |
431 | char *stackaddr = (char *) attr->stackaddr; |
432 | |
433 | /* Assume the same layout as the _STACK_GROWS_DOWN case, with struct |
434 | pthread at the top of the stack block. Later we adjust the guard |
435 | location and stack address to match the _STACK_GROWS_UP case. */ |
436 | if (_STACK_GROWS_UP) |
437 | stackaddr += attr->stacksize; |
438 | |
439 | /* If the user also specified the size of the stack make sure it |
440 | is large enough. */ |
441 | if (attr->stacksize != 0 |
442 | && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK)) |
443 | return EINVAL; |
444 | |
445 | /* Adjust stack size for alignment of the TLS block. */ |
446 | #if TLS_TCB_AT_TP |
447 | adj = ((uintptr_t) stackaddr - TLS_TCB_SIZE) |
448 | & __static_tls_align_m1; |
449 | assert (size > adj + TLS_TCB_SIZE); |
450 | #elif TLS_DTV_AT_TP |
451 | adj = ((uintptr_t) stackaddr - __static_tls_size) |
452 | & __static_tls_align_m1; |
453 | assert (size > adj); |
454 | #endif |
455 | |
456 | /* The user provided some memory. Let's hope it matches the |
457 | size... We do not allocate guard pages if the user provided |
458 | the stack. It is the user's responsibility to do this if it |
459 | is wanted. */ |
460 | #if TLS_TCB_AT_TP |
461 | pd = (struct pthread *) ((uintptr_t) stackaddr |
462 | - TLS_TCB_SIZE - adj); |
463 | #elif TLS_DTV_AT_TP |
464 | pd = (struct pthread *) (((uintptr_t) stackaddr |
465 | - __static_tls_size - adj) |
466 | - TLS_PRE_TCB_SIZE); |
467 | #endif |
468 | |
469 | /* The user provided stack memory needs to be cleared. */ |
470 | memset (pd, '\0', sizeof (struct pthread)); |
471 | |
472 | /* The first TSD block is included in the TCB. */ |
473 | pd->specific[0] = pd->specific_1stblock; |
474 | |
475 | /* Remember the stack-related values. */ |
476 | pd->stackblock = (char *) stackaddr - size; |
477 | pd->stackblock_size = size; |
478 | |
479 | /* This is a user-provided stack. It will not be queued in the |
480 | stack cache nor will the memory (except the TLS memory) be freed. */ |
481 | pd->user_stack = true; |
482 | |
483 | /* This is at least the second thread. */ |
484 | pd->header.multiple_threads = 1; |
485 | #ifndef TLS_MULTIPLE_THREADS_IN_TCB |
486 | __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1; |
487 | #endif |
488 | |
489 | #ifndef __ASSUME_PRIVATE_FUTEX |
490 | /* The thread must know when private futexes are supported. */ |
491 | pd->header.private_futex = THREAD_GETMEM (THREAD_SELF, |
492 | header.private_futex); |
493 | #endif |
494 | |
495 | #ifdef NEED_DL_SYSINFO |
496 | SETUP_THREAD_SYSINFO (pd); |
497 | #endif |
498 | |
499 | /* Don't allow setxid until cloned. */ |
500 | pd->setxid_futex = -1; |
501 | |
502 | /* Allocate the DTV for this thread. */ |
503 | if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL) |
504 | { |
505 | /* Something went wrong. */ |
506 | assert (errno == ENOMEM); |
507 | return errno; |
508 | } |
509 | |
510 | |
511 | /* Prepare to modify global data. */ |
512 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
513 | |
514 | /* And add to the list of stacks in use. */ |
515 | list_add (&pd->list, &__stack_user); |
516 | |
517 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
518 | } |
519 | else |
520 | { |
521 | /* Allocate some anonymous memory. If possible use the cache. */ |
522 | size_t guardsize; |
523 | size_t reqsize; |
524 | void *mem; |
525 | const int prot = (PROT_READ | PROT_WRITE |
526 | | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0)); |
527 | |
528 | /* Adjust the stack size for alignment. */ |
529 | size &= ~__static_tls_align_m1; |
530 | assert (size != 0); |
531 | |
532 | /* Make sure the size of the stack is enough for the guard and |
533 | eventually the thread descriptor. */ |
534 | guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1; |
535 | if (guardsize < attr->guardsize || size + guardsize < guardsize) |
536 | /* Arithmetic overflow. */ |
537 | return EINVAL; |
538 | size += guardsize; |
539 | if (__builtin_expect (size < ((guardsize + __static_tls_size |
540 | + MINIMAL_REST_STACK + pagesize_m1) |
541 | & ~pagesize_m1), |
542 | 0)) |
543 | /* The stack is too small (or the guard too large). */ |
544 | return EINVAL; |
545 | |
546 | /* Try to get a stack from the cache. */ |
547 | reqsize = size; |
548 | pd = get_cached_stack (&size, &mem); |
549 | if (pd == NULL) |
550 | { |
551 | /* To avoid aliasing effects on a larger scale than pages we |
552 | adjust the allocated stack size if necessary. This way |
553 | allocations directly following each other will not have |
554 | aliasing problems. */ |
555 | #if MULTI_PAGE_ALIASING != 0 |
556 | if ((size % MULTI_PAGE_ALIASING) == 0) |
557 | size += pagesize_m1 + 1; |
558 | #endif |
559 | |
560 | /* If a guard page is required, avoid committing memory by first |
561 | allocate with PROT_NONE and then reserve with required permission |
562 | excluding the guard page. */ |
563 | mem = __mmap (NULL, size, (guardsize == 0) ? prot : PROT_NONE, |
564 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); |
565 | |
566 | if (__glibc_unlikely (mem == MAP_FAILED)) |
567 | return errno; |
568 | |
569 | /* SIZE is guaranteed to be greater than zero. |
570 | So we can never get a null pointer back from mmap. */ |
571 | assert (mem != NULL); |
572 | |
573 | /* Place the thread descriptor at the end of the stack. */ |
574 | #if TLS_TCB_AT_TP |
575 | pd = (struct pthread *) ((char *) mem + size) - 1; |
576 | #elif TLS_DTV_AT_TP |
577 | pd = (struct pthread *) ((((uintptr_t) mem + size |
578 | - __static_tls_size) |
579 | & ~__static_tls_align_m1) |
580 | - TLS_PRE_TCB_SIZE); |
581 | #endif |
582 | |
583 | /* Now mprotect the required region excluding the guard area. */ |
584 | if (__glibc_likely (guardsize > 0)) |
585 | { |
586 | char *guard = guard_position (mem, size, guardsize, pd, |
587 | pagesize_m1); |
588 | if (setup_stack_prot (mem, size, guard, guardsize, prot) != 0) |
589 | { |
590 | __munmap (mem, size); |
591 | return errno; |
592 | } |
593 | } |
594 | |
595 | /* Remember the stack-related values. */ |
596 | pd->stackblock = mem; |
597 | pd->stackblock_size = size; |
598 | /* Update guardsize for newly allocated guardsize to avoid |
599 | an mprotect in guard resize below. */ |
600 | pd->guardsize = guardsize; |
601 | |
602 | /* We allocated the first block thread-specific data array. |
603 | This address will not change for the lifetime of this |
604 | descriptor. */ |
605 | pd->specific[0] = pd->specific_1stblock; |
606 | |
607 | /* This is at least the second thread. */ |
608 | pd->header.multiple_threads = 1; |
609 | #ifndef TLS_MULTIPLE_THREADS_IN_TCB |
610 | __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1; |
611 | #endif |
612 | |
613 | #ifndef __ASSUME_PRIVATE_FUTEX |
614 | /* The thread must know when private futexes are supported. */ |
615 | pd->header.private_futex = THREAD_GETMEM (THREAD_SELF, |
616 | header.private_futex); |
617 | #endif |
618 | |
619 | #ifdef NEED_DL_SYSINFO |
620 | SETUP_THREAD_SYSINFO (pd); |
621 | #endif |
622 | |
623 | /* Don't allow setxid until cloned. */ |
624 | pd->setxid_futex = -1; |
625 | |
626 | /* Allocate the DTV for this thread. */ |
627 | if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL) |
628 | { |
629 | /* Something went wrong. */ |
630 | assert (errno == ENOMEM); |
631 | |
632 | /* Free the stack memory we just allocated. */ |
633 | (void) __munmap (mem, size); |
634 | |
635 | return errno; |
636 | } |
637 | |
638 | |
639 | /* Prepare to modify global data. */ |
640 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
641 | |
642 | /* And add to the list of stacks in use. */ |
643 | stack_list_add (&pd->list, &stack_used); |
644 | |
645 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
646 | |
647 | |
648 | /* There might have been a race. Another thread might have |
649 | caused the stacks to get exec permission while this new |
650 | stack was prepared. Detect if this was possible and |
651 | change the permission if necessary. */ |
652 | if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0 |
653 | && (prot & PROT_EXEC) == 0, 0)) |
654 | { |
655 | int err = change_stack_perm (pd |
656 | #ifdef NEED_SEPARATE_REGISTER_STACK |
657 | , ~pagesize_m1 |
658 | #endif |
659 | ); |
660 | if (err != 0) |
661 | { |
662 | /* Free the stack memory we just allocated. */ |
663 | (void) __munmap (mem, size); |
664 | |
665 | return err; |
666 | } |
667 | } |
668 | |
669 | |
670 | /* Note that all of the stack and the thread descriptor is |
671 | zeroed. This means we do not have to initialize fields |
672 | with initial value zero. This is specifically true for |
673 | the 'tid' field which is always set back to zero once the |
674 | stack is not used anymore and for the 'guardsize' field |
675 | which will be read next. */ |
676 | } |
677 | |
678 | /* Create or resize the guard area if necessary. */ |
679 | if (__glibc_unlikely (guardsize > pd->guardsize)) |
680 | { |
681 | char *guard = guard_position (mem, size, guardsize, pd, |
682 | pagesize_m1); |
683 | if (__mprotect (guard, guardsize, PROT_NONE) != 0) |
684 | { |
685 | mprot_error: |
686 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
687 | |
688 | /* Remove the thread from the list. */ |
689 | stack_list_del (&pd->list); |
690 | |
691 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
692 | |
693 | /* Get rid of the TLS block we allocated. */ |
694 | _dl_deallocate_tls (TLS_TPADJ (pd), false); |
695 | |
696 | /* Free the stack memory regardless of whether the size |
697 | of the cache is over the limit or not. If this piece |
698 | of memory caused problems we better do not use it |
699 | anymore. Uh, and we ignore possible errors. There |
700 | is nothing we could do. */ |
701 | (void) __munmap (mem, size); |
702 | |
703 | return errno; |
704 | } |
705 | |
706 | pd->guardsize = guardsize; |
707 | } |
708 | else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize, |
709 | 0)) |
710 | { |
711 | /* The old guard area is too large. */ |
712 | |
713 | #ifdef NEED_SEPARATE_REGISTER_STACK |
714 | char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1); |
715 | char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1); |
716 | |
717 | if (oldguard < guard |
718 | && __mprotect (oldguard, guard - oldguard, prot) != 0) |
719 | goto mprot_error; |
720 | |
721 | if (__mprotect (guard + guardsize, |
722 | oldguard + pd->guardsize - guard - guardsize, |
723 | prot) != 0) |
724 | goto mprot_error; |
725 | #elif _STACK_GROWS_DOWN |
726 | if (__mprotect ((char *) mem + guardsize, pd->guardsize - guardsize, |
727 | prot) != 0) |
728 | goto mprot_error; |
729 | #elif _STACK_GROWS_UP |
730 | char *new_guard = (char *)(((uintptr_t) pd - guardsize) |
731 | & ~pagesize_m1); |
732 | char *old_guard = (char *)(((uintptr_t) pd - pd->guardsize) |
733 | & ~pagesize_m1); |
734 | /* The guard size difference might be > 0, but once rounded |
735 | to the nearest page the size difference might be zero. */ |
736 | if (new_guard > old_guard |
737 | && mprotect (old_guard, new_guard - old_guard, prot) != 0) |
738 | goto mprot_error; |
739 | #endif |
740 | |
741 | pd->guardsize = guardsize; |
742 | } |
743 | /* The pthread_getattr_np() calls need to get passed the size |
744 | requested in the attribute, regardless of how large the |
745 | actually used guardsize is. */ |
746 | pd->reported_guardsize = guardsize; |
747 | } |
748 | |
749 | /* Initialize the lock. We have to do this unconditionally since the |
750 | stillborn thread could be canceled while the lock is taken. */ |
751 | pd->lock = LLL_LOCK_INITIALIZER; |
752 | |
753 | /* The robust mutex lists also need to be initialized |
754 | unconditionally because the cleanup for the previous stack owner |
755 | might have happened in the kernel. */ |
756 | pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock) |
757 | - offsetof (pthread_mutex_t, |
758 | __data.__list.__next)); |
759 | pd->robust_head.list_op_pending = NULL; |
760 | #if __PTHREAD_MUTEX_HAVE_PREV |
761 | pd->robust_prev = &pd->robust_head; |
762 | #endif |
763 | pd->robust_head.list = &pd->robust_head; |
764 | |
765 | /* We place the thread descriptor at the end of the stack. */ |
766 | *pdp = pd; |
767 | |
768 | #if _STACK_GROWS_DOWN |
769 | void *stacktop; |
770 | |
771 | # if TLS_TCB_AT_TP |
772 | /* The stack begins before the TCB and the static TLS block. */ |
773 | stacktop = ((char *) (pd + 1) - __static_tls_size); |
774 | # elif TLS_DTV_AT_TP |
775 | stacktop = (char *) (pd - 1); |
776 | # endif |
777 | |
778 | # ifdef NEED_SEPARATE_REGISTER_STACK |
779 | *stack = pd->stackblock; |
780 | *stacksize = stacktop - *stack; |
781 | # else |
782 | *stack = stacktop; |
783 | # endif |
784 | #else |
785 | *stack = pd->stackblock; |
786 | #endif |
787 | |
788 | return 0; |
789 | } |
790 | |
791 | |
792 | void |
793 | __deallocate_stack (struct pthread *pd) |
794 | { |
795 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
796 | |
797 | /* Remove the thread from the list of threads with user defined |
798 | stacks. */ |
799 | stack_list_del (&pd->list); |
800 | |
801 | /* Not much to do. Just free the mmap()ed memory. Note that we do |
802 | not reset the 'used' flag in the 'tid' field. This is done by |
803 | the kernel. If no thread has been created yet this field is |
804 | still zero. */ |
805 | if (__glibc_likely (! pd->user_stack)) |
806 | (void) queue_stack (pd); |
807 | else |
808 | /* Free the memory associated with the ELF TLS. */ |
809 | _dl_deallocate_tls (TLS_TPADJ (pd), false); |
810 | |
811 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
812 | } |
813 | |
814 | |
815 | int |
816 | __make_stacks_executable (void **stack_endp) |
817 | { |
818 | /* First the main thread's stack. */ |
819 | int err = _dl_make_stack_executable (stack_endp); |
820 | if (err != 0) |
821 | return err; |
822 | |
823 | #ifdef NEED_SEPARATE_REGISTER_STACK |
824 | const size_t pagemask = ~(__getpagesize () - 1); |
825 | #endif |
826 | |
827 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
828 | |
829 | list_t *runp; |
830 | list_for_each (runp, &stack_used) |
831 | { |
832 | err = change_stack_perm (list_entry (runp, struct pthread, list) |
833 | #ifdef NEED_SEPARATE_REGISTER_STACK |
834 | , pagemask |
835 | #endif |
836 | ); |
837 | if (err != 0) |
838 | break; |
839 | } |
840 | |
841 | /* Also change the permission for the currently unused stacks. This |
842 | might be wasted time but better spend it here than adding a check |
843 | in the fast path. */ |
844 | if (err == 0) |
845 | list_for_each (runp, &stack_cache) |
846 | { |
847 | err = change_stack_perm (list_entry (runp, struct pthread, list) |
848 | #ifdef NEED_SEPARATE_REGISTER_STACK |
849 | , pagemask |
850 | #endif |
851 | ); |
852 | if (err != 0) |
853 | break; |
854 | } |
855 | |
856 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
857 | |
858 | return err; |
859 | } |
860 | |
861 | |
862 | /* In case of a fork() call the memory allocation in the child will be |
863 | the same but only one thread is running. All stacks except that of |
864 | the one running thread are not used anymore. We have to recycle |
865 | them. */ |
866 | void |
867 | __reclaim_stacks (void) |
868 | { |
869 | struct pthread *self = (struct pthread *) THREAD_SELF; |
870 | |
871 | /* No locking necessary. The caller is the only stack in use. But |
872 | we have to be aware that we might have interrupted a list |
873 | operation. */ |
874 | |
875 | if (in_flight_stack != 0) |
876 | { |
877 | bool add_p = in_flight_stack & 1; |
878 | list_t *elem = (list_t *) (in_flight_stack & ~(uintptr_t) 1); |
879 | |
880 | if (add_p) |
881 | { |
882 | /* We always add at the beginning of the list. So in this case we |
883 | only need to check the beginning of these lists to see if the |
884 | pointers at the head of the list are inconsistent. */ |
885 | list_t *l = NULL; |
886 | |
887 | if (stack_used.next->prev != &stack_used) |
888 | l = &stack_used; |
889 | else if (stack_cache.next->prev != &stack_cache) |
890 | l = &stack_cache; |
891 | |
892 | if (l != NULL) |
893 | { |
894 | assert (l->next->prev == elem); |
895 | elem->next = l->next; |
896 | elem->prev = l; |
897 | l->next = elem; |
898 | } |
899 | } |
900 | else |
901 | { |
902 | /* We can simply always replay the delete operation. */ |
903 | elem->next->prev = elem->prev; |
904 | elem->prev->next = elem->next; |
905 | } |
906 | } |
907 | |
908 | /* Mark all stacks except the still running one as free. */ |
909 | list_t *runp; |
910 | list_for_each (runp, &stack_used) |
911 | { |
912 | struct pthread *curp = list_entry (runp, struct pthread, list); |
913 | if (curp != self) |
914 | { |
915 | /* This marks the stack as free. */ |
916 | curp->tid = 0; |
917 | |
918 | /* Account for the size of the stack. */ |
919 | stack_cache_actsize += curp->stackblock_size; |
920 | |
921 | if (curp->specific_used) |
922 | { |
923 | /* Clear the thread-specific data. */ |
924 | memset (curp->specific_1stblock, '\0', |
925 | sizeof (curp->specific_1stblock)); |
926 | |
927 | curp->specific_used = false; |
928 | |
929 | for (size_t cnt = 1; cnt < PTHREAD_KEY_1STLEVEL_SIZE; ++cnt) |
930 | if (curp->specific[cnt] != NULL) |
931 | { |
932 | memset (curp->specific[cnt], '\0', |
933 | sizeof (curp->specific_1stblock)); |
934 | |
935 | /* We have allocated the block which we do not |
936 | free here so re-set the bit. */ |
937 | curp->specific_used = true; |
938 | } |
939 | } |
940 | } |
941 | } |
942 | |
943 | /* Add the stack of all running threads to the cache. */ |
944 | list_splice (&stack_used, &stack_cache); |
945 | |
946 | /* Remove the entry for the current thread to from the cache list |
947 | and add it to the list of running threads. Which of the two |
948 | lists is decided by the user_stack flag. */ |
949 | stack_list_del (&self->list); |
950 | |
951 | /* Re-initialize the lists for all the threads. */ |
952 | INIT_LIST_HEAD (&stack_used); |
953 | INIT_LIST_HEAD (&__stack_user); |
954 | |
955 | if (__glibc_unlikely (THREAD_GETMEM (self, user_stack))) |
956 | list_add (&self->list, &__stack_user); |
957 | else |
958 | list_add (&self->list, &stack_used); |
959 | |
960 | /* There is one thread running. */ |
961 | __nptl_nthreads = 1; |
962 | |
963 | in_flight_stack = 0; |
964 | |
965 | /* Initialize locks. */ |
966 | stack_cache_lock = LLL_LOCK_INITIALIZER; |
967 | __default_pthread_attr_lock = LLL_LOCK_INITIALIZER; |
968 | } |
969 | |
970 | |
971 | #if HP_TIMING_AVAIL |
972 | # undef __find_thread_by_id |
973 | /* Find a thread given the thread ID. */ |
974 | attribute_hidden |
975 | struct pthread * |
976 | __find_thread_by_id (pid_t tid) |
977 | { |
978 | struct pthread *result = NULL; |
979 | |
980 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
981 | |
982 | /* Iterate over the list with system-allocated threads first. */ |
983 | list_t *runp; |
984 | list_for_each (runp, &stack_used) |
985 | { |
986 | struct pthread *curp; |
987 | |
988 | curp = list_entry (runp, struct pthread, list); |
989 | |
990 | if (curp->tid == tid) |
991 | { |
992 | result = curp; |
993 | goto out; |
994 | } |
995 | } |
996 | |
997 | /* Now the list with threads using user-allocated stacks. */ |
998 | list_for_each (runp, &__stack_user) |
999 | { |
1000 | struct pthread *curp; |
1001 | |
1002 | curp = list_entry (runp, struct pthread, list); |
1003 | |
1004 | if (curp->tid == tid) |
1005 | { |
1006 | result = curp; |
1007 | goto out; |
1008 | } |
1009 | } |
1010 | |
1011 | out: |
1012 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
1013 | |
1014 | return result; |
1015 | } |
1016 | #endif |
1017 | |
1018 | |
1019 | #ifdef SIGSETXID |
1020 | static void |
1021 | setxid_mark_thread (struct xid_command *cmdp, struct pthread *t) |
1022 | { |
1023 | int ch; |
1024 | |
1025 | /* Wait until this thread is cloned. */ |
1026 | if (t->setxid_futex == -1 |
1027 | && ! atomic_compare_and_exchange_bool_acq (&t->setxid_futex, -2, -1)) |
1028 | do |
1029 | futex_wait_simple (&t->setxid_futex, -2, FUTEX_PRIVATE); |
1030 | while (t->setxid_futex == -2); |
1031 | |
1032 | /* Don't let the thread exit before the setxid handler runs. */ |
1033 | t->setxid_futex = 0; |
1034 | |
1035 | do |
1036 | { |
1037 | ch = t->cancelhandling; |
1038 | |
1039 | /* If the thread is exiting right now, ignore it. */ |
1040 | if ((ch & EXITING_BITMASK) != 0) |
1041 | { |
1042 | /* Release the futex if there is no other setxid in |
1043 | progress. */ |
1044 | if ((ch & SETXID_BITMASK) == 0) |
1045 | { |
1046 | t->setxid_futex = 1; |
1047 | futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE); |
1048 | } |
1049 | return; |
1050 | } |
1051 | } |
1052 | while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling, |
1053 | ch | SETXID_BITMASK, ch)); |
1054 | } |
1055 | |
1056 | |
1057 | static void |
1058 | setxid_unmark_thread (struct xid_command *cmdp, struct pthread *t) |
1059 | { |
1060 | int ch; |
1061 | |
1062 | do |
1063 | { |
1064 | ch = t->cancelhandling; |
1065 | if ((ch & SETXID_BITMASK) == 0) |
1066 | return; |
1067 | } |
1068 | while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling, |
1069 | ch & ~SETXID_BITMASK, ch)); |
1070 | |
1071 | /* Release the futex just in case. */ |
1072 | t->setxid_futex = 1; |
1073 | futex_wake (&t->setxid_futex, 1, FUTEX_PRIVATE); |
1074 | } |
1075 | |
1076 | |
1077 | static int |
1078 | setxid_signal_thread (struct xid_command *cmdp, struct pthread *t) |
1079 | { |
1080 | if ((t->cancelhandling & SETXID_BITMASK) == 0) |
1081 | return 0; |
1082 | |
1083 | int val; |
1084 | pid_t pid = __getpid (); |
1085 | INTERNAL_SYSCALL_DECL (err); |
1086 | val = INTERNAL_SYSCALL_CALL (tgkill, err, pid, t->tid, SIGSETXID); |
1087 | |
1088 | /* If this failed, it must have had not started yet or else exited. */ |
1089 | if (!INTERNAL_SYSCALL_ERROR_P (val, err)) |
1090 | { |
1091 | atomic_increment (&cmdp->cntr); |
1092 | return 1; |
1093 | } |
1094 | else |
1095 | return 0; |
1096 | } |
1097 | |
1098 | /* Check for consistency across set*id system call results. The abort |
1099 | should not happen as long as all privileges changes happen through |
1100 | the glibc wrappers. ERROR must be 0 (no error) or an errno |
1101 | code. */ |
1102 | void |
1103 | attribute_hidden |
1104 | __nptl_setxid_error (struct xid_command *cmdp, int error) |
1105 | { |
1106 | do |
1107 | { |
1108 | int olderror = cmdp->error; |
1109 | if (olderror == error) |
1110 | break; |
1111 | if (olderror != -1) |
1112 | { |
1113 | /* Mismatch between current and previous results. Save the |
1114 | error value to memory so that is not clobbered by the |
1115 | abort function and preserved in coredumps. */ |
1116 | volatile int xid_err __attribute__((unused)) = error; |
1117 | abort (); |
1118 | } |
1119 | } |
1120 | while (atomic_compare_and_exchange_bool_acq (&cmdp->error, error, -1)); |
1121 | } |
1122 | |
1123 | int |
1124 | attribute_hidden |
1125 | __nptl_setxid (struct xid_command *cmdp) |
1126 | { |
1127 | int signalled; |
1128 | int result; |
1129 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
1130 | |
1131 | __xidcmd = cmdp; |
1132 | cmdp->cntr = 0; |
1133 | cmdp->error = -1; |
1134 | |
1135 | struct pthread *self = THREAD_SELF; |
1136 | |
1137 | /* Iterate over the list with system-allocated threads first. */ |
1138 | list_t *runp; |
1139 | list_for_each (runp, &stack_used) |
1140 | { |
1141 | struct pthread *t = list_entry (runp, struct pthread, list); |
1142 | if (t == self) |
1143 | continue; |
1144 | |
1145 | setxid_mark_thread (cmdp, t); |
1146 | } |
1147 | |
1148 | /* Now the list with threads using user-allocated stacks. */ |
1149 | list_for_each (runp, &__stack_user) |
1150 | { |
1151 | struct pthread *t = list_entry (runp, struct pthread, list); |
1152 | if (t == self) |
1153 | continue; |
1154 | |
1155 | setxid_mark_thread (cmdp, t); |
1156 | } |
1157 | |
1158 | /* Iterate until we don't succeed in signalling anyone. That means |
1159 | we have gotten all running threads, and their children will be |
1160 | automatically correct once started. */ |
1161 | do |
1162 | { |
1163 | signalled = 0; |
1164 | |
1165 | list_for_each (runp, &stack_used) |
1166 | { |
1167 | struct pthread *t = list_entry (runp, struct pthread, list); |
1168 | if (t == self) |
1169 | continue; |
1170 | |
1171 | signalled += setxid_signal_thread (cmdp, t); |
1172 | } |
1173 | |
1174 | list_for_each (runp, &__stack_user) |
1175 | { |
1176 | struct pthread *t = list_entry (runp, struct pthread, list); |
1177 | if (t == self) |
1178 | continue; |
1179 | |
1180 | signalled += setxid_signal_thread (cmdp, t); |
1181 | } |
1182 | |
1183 | int cur = cmdp->cntr; |
1184 | while (cur != 0) |
1185 | { |
1186 | futex_wait_simple ((unsigned int *) &cmdp->cntr, cur, |
1187 | FUTEX_PRIVATE); |
1188 | cur = cmdp->cntr; |
1189 | } |
1190 | } |
1191 | while (signalled != 0); |
1192 | |
1193 | /* Clean up flags, so that no thread blocks during exit waiting |
1194 | for a signal which will never come. */ |
1195 | list_for_each (runp, &stack_used) |
1196 | { |
1197 | struct pthread *t = list_entry (runp, struct pthread, list); |
1198 | if (t == self) |
1199 | continue; |
1200 | |
1201 | setxid_unmark_thread (cmdp, t); |
1202 | } |
1203 | |
1204 | list_for_each (runp, &__stack_user) |
1205 | { |
1206 | struct pthread *t = list_entry (runp, struct pthread, list); |
1207 | if (t == self) |
1208 | continue; |
1209 | |
1210 | setxid_unmark_thread (cmdp, t); |
1211 | } |
1212 | |
1213 | /* This must be last, otherwise the current thread might not have |
1214 | permissions to send SIGSETXID syscall to the other threads. */ |
1215 | INTERNAL_SYSCALL_DECL (err); |
1216 | result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3, |
1217 | cmdp->id[0], cmdp->id[1], cmdp->id[2]); |
1218 | int error = 0; |
1219 | if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, err))) |
1220 | { |
1221 | error = INTERNAL_SYSCALL_ERRNO (result, err); |
1222 | __set_errno (error); |
1223 | result = -1; |
1224 | } |
1225 | __nptl_setxid_error (cmdp, error); |
1226 | |
1227 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
1228 | return result; |
1229 | } |
1230 | #endif /* SIGSETXID. */ |
1231 | |
1232 | |
1233 | static inline void __attribute__((always_inline)) |
1234 | init_one_static_tls (struct pthread *curp, struct link_map *map) |
1235 | { |
1236 | # if TLS_TCB_AT_TP |
1237 | void *dest = (char *) curp - map->l_tls_offset; |
1238 | # elif TLS_DTV_AT_TP |
1239 | void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE; |
1240 | # else |
1241 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" |
1242 | # endif |
1243 | |
1244 | /* Initialize the memory. */ |
1245 | memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size), |
1246 | '\0', map->l_tls_blocksize - map->l_tls_initimage_size); |
1247 | } |
1248 | |
1249 | void |
1250 | attribute_hidden |
1251 | __pthread_init_static_tls (struct link_map *map) |
1252 | { |
1253 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
1254 | |
1255 | /* Iterate over the list with system-allocated threads first. */ |
1256 | list_t *runp; |
1257 | list_for_each (runp, &stack_used) |
1258 | init_one_static_tls (list_entry (runp, struct pthread, list), map); |
1259 | |
1260 | /* Now the list with threads using user-allocated stacks. */ |
1261 | list_for_each (runp, &__stack_user) |
1262 | init_one_static_tls (list_entry (runp, struct pthread, list), map); |
1263 | |
1264 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
1265 | } |
1266 | |
1267 | |
1268 | void |
1269 | attribute_hidden |
1270 | __wait_lookup_done (void) |
1271 | { |
1272 | lll_lock (stack_cache_lock, LLL_PRIVATE); |
1273 | |
1274 | struct pthread *self = THREAD_SELF; |
1275 | |
1276 | /* Iterate over the list with system-allocated threads first. */ |
1277 | list_t *runp; |
1278 | list_for_each (runp, &stack_used) |
1279 | { |
1280 | struct pthread *t = list_entry (runp, struct pthread, list); |
1281 | if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED) |
1282 | continue; |
1283 | |
1284 | int *const gscope_flagp = &t->header.gscope_flag; |
1285 | |
1286 | /* We have to wait until this thread is done with the global |
1287 | scope. First tell the thread that we are waiting and |
1288 | possibly have to be woken. */ |
1289 | if (atomic_compare_and_exchange_bool_acq (gscope_flagp, |
1290 | THREAD_GSCOPE_FLAG_WAIT, |
1291 | THREAD_GSCOPE_FLAG_USED)) |
1292 | continue; |
1293 | |
1294 | do |
1295 | futex_wait_simple ((unsigned int *) gscope_flagp, |
1296 | THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE); |
1297 | while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); |
1298 | } |
1299 | |
1300 | /* Now the list with threads using user-allocated stacks. */ |
1301 | list_for_each (runp, &__stack_user) |
1302 | { |
1303 | struct pthread *t = list_entry (runp, struct pthread, list); |
1304 | if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED) |
1305 | continue; |
1306 | |
1307 | int *const gscope_flagp = &t->header.gscope_flag; |
1308 | |
1309 | /* We have to wait until this thread is done with the global |
1310 | scope. First tell the thread that we are waiting and |
1311 | possibly have to be woken. */ |
1312 | if (atomic_compare_and_exchange_bool_acq (gscope_flagp, |
1313 | THREAD_GSCOPE_FLAG_WAIT, |
1314 | THREAD_GSCOPE_FLAG_USED)) |
1315 | continue; |
1316 | |
1317 | do |
1318 | futex_wait_simple ((unsigned int *) gscope_flagp, |
1319 | THREAD_GSCOPE_FLAG_WAIT, FUTEX_PRIVATE); |
1320 | while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT); |
1321 | } |
1322 | |
1323 | lll_unlock (stack_cache_lock, LLL_PRIVATE); |
1324 | } |
1325 | |