1/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <limits.h>
22#include <signal.h>
23#include <stdlib.h>
24#include <unistd.h>
25#include <sys/param.h>
26#include <sys/resource.h>
27#include <pthreadP.h>
28#include <atomic.h>
29#include <ldsodefs.h>
30#include <tls.h>
31#include <list.h>
32#include <fork.h>
33#include <version.h>
34#include <shlib-compat.h>
35#include <smp.h>
36#include <lowlevellock.h>
37#include <futex-internal.h>
38#include <kernel-features.h>
39#include <libc-internal.h>
40#include <pthread-pids.h>
41
42#ifndef TLS_MULTIPLE_THREADS_IN_TCB
43/* Pointer to the corresponding variable in libc. */
44int *__libc_multiple_threads_ptr attribute_hidden;
45#endif
46
47/* Size and alignment of static TLS block. */
48size_t __static_tls_size;
49size_t __static_tls_align_m1;
50
51#ifndef __ASSUME_SET_ROBUST_LIST
52/* Negative if we do not have the system call and we can use it. */
53int __set_robust_list_avail;
54# define set_robust_list_not_avail() \
55 __set_robust_list_avail = -1
56#else
57# define set_robust_list_not_avail() do { } while (0)
58#endif
59
60#ifndef __ASSUME_FUTEX_CLOCK_REALTIME
61/* Nonzero if we do not have FUTEX_CLOCK_REALTIME. */
62int __have_futex_clock_realtime;
63# define __set_futex_clock_realtime() \
64 __have_futex_clock_realtime = 1
65#else
66#define __set_futex_clock_realtime() do { } while (0)
67#endif
68
69/* Version of the library, used in libthread_db to detect mismatches. */
70static const char nptl_version[] __attribute_used__ = VERSION;
71
72
73#ifndef SHARED
74extern void __libc_setup_tls (size_t tcbsize, size_t tcbalign);
75#endif
76
77#ifdef SHARED
78static
79#else
80extern
81#endif
82void __nptl_set_robust (struct pthread *);
83
84#ifdef SHARED
85static void nptl_freeres (void);
86
87
88static const struct pthread_functions pthread_functions =
89 {
90 .ptr_pthread_attr_destroy = __pthread_attr_destroy,
91# if SHLIB_COMPAT(libpthread, GLIBC_2_0, GLIBC_2_1)
92 .ptr___pthread_attr_init_2_0 = __pthread_attr_init_2_0,
93# endif
94 .ptr___pthread_attr_init_2_1 = __pthread_attr_init_2_1,
95 .ptr_pthread_attr_getdetachstate = __pthread_attr_getdetachstate,
96 .ptr_pthread_attr_setdetachstate = __pthread_attr_setdetachstate,
97 .ptr_pthread_attr_getinheritsched = __pthread_attr_getinheritsched,
98 .ptr_pthread_attr_setinheritsched = __pthread_attr_setinheritsched,
99 .ptr_pthread_attr_getschedparam = __pthread_attr_getschedparam,
100 .ptr_pthread_attr_setschedparam = __pthread_attr_setschedparam,
101 .ptr_pthread_attr_getschedpolicy = __pthread_attr_getschedpolicy,
102 .ptr_pthread_attr_setschedpolicy = __pthread_attr_setschedpolicy,
103 .ptr_pthread_attr_getscope = __pthread_attr_getscope,
104 .ptr_pthread_attr_setscope = __pthread_attr_setscope,
105 .ptr_pthread_condattr_destroy = __pthread_condattr_destroy,
106 .ptr_pthread_condattr_init = __pthread_condattr_init,
107 .ptr___pthread_cond_broadcast = __pthread_cond_broadcast,
108 .ptr___pthread_cond_destroy = __pthread_cond_destroy,
109 .ptr___pthread_cond_init = __pthread_cond_init,
110 .ptr___pthread_cond_signal = __pthread_cond_signal,
111 .ptr___pthread_cond_wait = __pthread_cond_wait,
112 .ptr___pthread_cond_timedwait = __pthread_cond_timedwait,
113# if SHLIB_COMPAT(libpthread, GLIBC_2_0, GLIBC_2_3_2)
114 .ptr___pthread_cond_broadcast_2_0 = __pthread_cond_broadcast_2_0,
115 .ptr___pthread_cond_destroy_2_0 = __pthread_cond_destroy_2_0,
116 .ptr___pthread_cond_init_2_0 = __pthread_cond_init_2_0,
117 .ptr___pthread_cond_signal_2_0 = __pthread_cond_signal_2_0,
118 .ptr___pthread_cond_wait_2_0 = __pthread_cond_wait_2_0,
119 .ptr___pthread_cond_timedwait_2_0 = __pthread_cond_timedwait_2_0,
120# endif
121 .ptr_pthread_equal = __pthread_equal,
122 .ptr___pthread_exit = __pthread_exit,
123 .ptr_pthread_getschedparam = __pthread_getschedparam,
124 .ptr_pthread_setschedparam = __pthread_setschedparam,
125 .ptr_pthread_mutex_destroy = __pthread_mutex_destroy,
126 .ptr_pthread_mutex_init = __pthread_mutex_init,
127 .ptr_pthread_mutex_lock = __pthread_mutex_lock,
128 .ptr_pthread_mutex_unlock = __pthread_mutex_unlock,
129 .ptr_pthread_self = __pthread_self,
130 .ptr___pthread_setcancelstate = __pthread_setcancelstate,
131 .ptr_pthread_setcanceltype = __pthread_setcanceltype,
132 .ptr___pthread_cleanup_upto = __pthread_cleanup_upto,
133 .ptr___pthread_once = __pthread_once,
134 .ptr___pthread_rwlock_rdlock = __pthread_rwlock_rdlock,
135 .ptr___pthread_rwlock_wrlock = __pthread_rwlock_wrlock,
136 .ptr___pthread_rwlock_unlock = __pthread_rwlock_unlock,
137 .ptr___pthread_key_create = __pthread_key_create,
138 .ptr___pthread_getspecific = __pthread_getspecific,
139 .ptr___pthread_setspecific = __pthread_setspecific,
140 .ptr__pthread_cleanup_push_defer = __pthread_cleanup_push_defer,
141 .ptr__pthread_cleanup_pop_restore = __pthread_cleanup_pop_restore,
142 .ptr_nthreads = &__nptl_nthreads,
143 .ptr___pthread_unwind = &__pthread_unwind,
144 .ptr__nptl_deallocate_tsd = __nptl_deallocate_tsd,
145# ifdef SIGSETXID
146 .ptr__nptl_setxid = __nptl_setxid,
147# endif
148 /* For now only the stack cache needs to be freed. */
149 .ptr_freeres = nptl_freeres,
150 .ptr_set_robust = __nptl_set_robust
151 };
152# define ptr_pthread_functions &pthread_functions
153#else
154# define ptr_pthread_functions NULL
155#endif
156
157
158#ifdef SHARED
159/* This function is called indirectly from the freeres code in libc. */
160static void
161__libc_freeres_fn_section
162nptl_freeres (void)
163{
164 __unwind_freeres ();
165 __free_stacks (0);
166}
167
168
169static
170#endif
171void
172__nptl_set_robust (struct pthread *self)
173{
174#ifdef __NR_set_robust_list
175 INTERNAL_SYSCALL_DECL (err);
176 INTERNAL_SYSCALL (set_robust_list, err, 2, &self->robust_head,
177 sizeof (struct robust_list_head));
178#endif
179}
180
181
182#ifdef SIGCANCEL
183/* For asynchronous cancellation we use a signal. This is the handler. */
184static void
185sigcancel_handler (int sig, siginfo_t *si, void *ctx)
186{
187 /* Safety check. It would be possible to call this function for
188 other signals and send a signal from another process. This is not
189 correct and might even be a security problem. Try to catch as
190 many incorrect invocations as possible. */
191 if (sig != SIGCANCEL
192 || si->si_pid != __getpid()
193 || si->si_code != SI_TKILL)
194 return;
195
196 struct pthread *self = THREAD_SELF;
197
198 int oldval = THREAD_GETMEM (self, cancelhandling);
199 while (1)
200 {
201 /* We are canceled now. When canceled by another thread this flag
202 is already set but if the signal is directly send (internally or
203 from another process) is has to be done here. */
204 int newval = oldval | CANCELING_BITMASK | CANCELED_BITMASK;
205
206 if (oldval == newval || (oldval & EXITING_BITMASK) != 0)
207 /* Already canceled or exiting. */
208 break;
209
210 int curval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling, newval,
211 oldval);
212 if (curval == oldval)
213 {
214 /* Set the return value. */
215 THREAD_SETMEM (self, result, PTHREAD_CANCELED);
216
217 /* Make sure asynchronous cancellation is still enabled. */
218 if ((newval & CANCELTYPE_BITMASK) != 0)
219 /* Run the registered destructors and terminate the thread. */
220 __do_cancel ();
221
222 break;
223 }
224
225 oldval = curval;
226 }
227}
228#endif
229
230
231#ifdef SIGSETXID
232struct xid_command *__xidcmd attribute_hidden;
233
234/* We use the SIGSETXID signal in the setuid, setgid, etc. implementations to
235 tell each thread to call the respective setxid syscall on itself. This is
236 the handler. */
237static void
238sighandler_setxid (int sig, siginfo_t *si, void *ctx)
239{
240 int result;
241
242 /* Safety check. It would be possible to call this function for
243 other signals and send a signal from another process. This is not
244 correct and might even be a security problem. Try to catch as
245 many incorrect invocations as possible. */
246 if (sig != SIGSETXID
247 || si->si_pid != __getpid ()
248 || si->si_code != SI_TKILL)
249 return;
250
251 INTERNAL_SYSCALL_DECL (err);
252 result = INTERNAL_SYSCALL_NCS (__xidcmd->syscall_no, err, 3, __xidcmd->id[0],
253 __xidcmd->id[1], __xidcmd->id[2]);
254 int error = 0;
255 if (__glibc_unlikely (INTERNAL_SYSCALL_ERROR_P (result, err)))
256 error = INTERNAL_SYSCALL_ERRNO (result, err);
257 __nptl_setxid_error (__xidcmd, error);
258
259 /* Reset the SETXID flag. */
260 struct pthread *self = THREAD_SELF;
261 int flags, newval;
262 do
263 {
264 flags = THREAD_GETMEM (self, cancelhandling);
265 newval = THREAD_ATOMIC_CMPXCHG_VAL (self, cancelhandling,
266 flags & ~SETXID_BITMASK, flags);
267 }
268 while (flags != newval);
269
270 /* And release the futex. */
271 self->setxid_futex = 1;
272 futex_wake (&self->setxid_futex, 1, FUTEX_PRIVATE);
273
274 if (atomic_decrement_val (&__xidcmd->cntr) == 0)
275 futex_wake ((unsigned int *) &__xidcmd->cntr, 1, FUTEX_PRIVATE);
276}
277#endif
278
279
280/* When using __thread for this, we do it in libc so as not
281 to give libpthread its own TLS segment just for this. */
282extern void **__libc_dl_error_tsd (void) __attribute__ ((const));
283
284
285/* This can be set by the debugger before initialization is complete. */
286static bool __nptl_initial_report_events __attribute_used__;
287
288void
289__pthread_initialize_minimal_internal (void)
290{
291#ifndef SHARED
292 /* Unlike in the dynamically linked case the dynamic linker has not
293 taken care of initializing the TLS data structures. */
294 __libc_setup_tls (TLS_TCB_SIZE, TLS_TCB_ALIGN);
295
296 /* We must prevent gcc from being clever and move any of the
297 following code ahead of the __libc_setup_tls call. This function
298 will initialize the thread register which is subsequently
299 used. */
300 __asm __volatile ("");
301#endif
302
303 /* Minimal initialization of the thread descriptor. */
304 struct pthread *pd = THREAD_SELF;
305 __pthread_initialize_pids (pd);
306 THREAD_SETMEM (pd, specific[0], &pd->specific_1stblock[0]);
307 THREAD_SETMEM (pd, user_stack, true);
308 if (LLL_LOCK_INITIALIZER != 0)
309 THREAD_SETMEM (pd, lock, LLL_LOCK_INITIALIZER);
310#if HP_TIMING_AVAIL
311 THREAD_SETMEM (pd, cpuclock_offset, GL(dl_cpuclock_offset));
312#endif
313
314 /* Initialize the robust mutex data. */
315 {
316#ifdef __PTHREAD_MUTEX_HAVE_PREV
317 pd->robust_prev = &pd->robust_head;
318#endif
319 pd->robust_head.list = &pd->robust_head;
320#ifdef __NR_set_robust_list
321 pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
322 - offsetof (pthread_mutex_t,
323 __data.__list.__next));
324 INTERNAL_SYSCALL_DECL (err);
325 int res = INTERNAL_SYSCALL (set_robust_list, err, 2, &pd->robust_head,
326 sizeof (struct robust_list_head));
327 if (INTERNAL_SYSCALL_ERROR_P (res, err))
328#endif
329 set_robust_list_not_avail ();
330 }
331
332#ifdef __NR_futex
333# ifndef __ASSUME_PRIVATE_FUTEX
334 /* Private futexes are always used (at least internally) so that
335 doing the test once this early is beneficial. */
336 {
337 int word = 0;
338 INTERNAL_SYSCALL_DECL (err);
339 word = INTERNAL_SYSCALL (futex, err, 3, &word,
340 FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1);
341 if (!INTERNAL_SYSCALL_ERROR_P (word, err))
342 THREAD_SETMEM (pd, header.private_futex, FUTEX_PRIVATE_FLAG);
343 }
344
345 /* Private futexes have been introduced earlier than the
346 FUTEX_CLOCK_REALTIME flag. We don't have to run the test if we
347 know the former are not supported. This also means we know the
348 kernel will return ENOSYS for unknown operations. */
349 if (THREAD_GETMEM (pd, header.private_futex) != 0)
350# endif
351# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
352 {
353 int word = 0;
354 /* NB: the syscall actually takes six parameters. The last is the
355 bit mask. But since we will not actually wait at all the value
356 is irrelevant. Given that passing six parameters is difficult
357 on some architectures we just pass whatever random value the
358 calling convention calls for to the kernel. It causes no harm. */
359 INTERNAL_SYSCALL_DECL (err);
360 word = INTERNAL_SYSCALL (futex, err, 5, &word,
361 FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME
362 | FUTEX_PRIVATE_FLAG, 1, NULL, 0);
363 assert (INTERNAL_SYSCALL_ERROR_P (word, err));
364 if (INTERNAL_SYSCALL_ERRNO (word, err) != ENOSYS)
365 __set_futex_clock_realtime ();
366 }
367# endif
368#endif
369
370 /* Set initial thread's stack block from 0 up to __libc_stack_end.
371 It will be bigger than it actually is, but for unwind.c/pt-longjmp.c
372 purposes this is good enough. */
373 THREAD_SETMEM (pd, stackblock_size, (size_t) __libc_stack_end);
374
375 /* Initialize the list of all running threads with the main thread. */
376 INIT_LIST_HEAD (&__stack_user);
377 list_add (&pd->list, &__stack_user);
378
379 /* Before initializing __stack_user, the debugger could not find us and
380 had to set __nptl_initial_report_events. Propagate its setting. */
381 THREAD_SETMEM (pd, report_events, __nptl_initial_report_events);
382
383#if defined SIGCANCEL || defined SIGSETXID
384 struct sigaction sa;
385 __sigemptyset (&sa.sa_mask);
386
387# ifdef SIGCANCEL
388 /* Install the cancellation signal handler. If for some reason we
389 cannot install the handler we do not abort. Maybe we should, but
390 it is only asynchronous cancellation which is affected. */
391 sa.sa_sigaction = sigcancel_handler;
392 sa.sa_flags = SA_SIGINFO;
393 (void) __libc_sigaction (SIGCANCEL, &sa, NULL);
394# endif
395
396# ifdef SIGSETXID
397 /* Install the handle to change the threads' uid/gid. */
398 sa.sa_sigaction = sighandler_setxid;
399 sa.sa_flags = SA_SIGINFO | SA_RESTART;
400 (void) __libc_sigaction (SIGSETXID, &sa, NULL);
401# endif
402
403 /* The parent process might have left the signals blocked. Just in
404 case, unblock it. We reuse the signal mask in the sigaction
405 structure. It is already cleared. */
406# ifdef SIGCANCEL
407 __sigaddset (&sa.sa_mask, SIGCANCEL);
408# endif
409# ifdef SIGSETXID
410 __sigaddset (&sa.sa_mask, SIGSETXID);
411# endif
412 {
413 INTERNAL_SYSCALL_DECL (err);
414 (void) INTERNAL_SYSCALL (rt_sigprocmask, err, 4, SIG_UNBLOCK, &sa.sa_mask,
415 NULL, _NSIG / 8);
416 }
417#endif
418
419 /* Get the size of the static and alignment requirements for the TLS
420 block. */
421 size_t static_tls_align;
422 _dl_get_tls_static_info (&__static_tls_size, &static_tls_align);
423
424 /* Make sure the size takes all the alignments into account. */
425 if (STACK_ALIGN > static_tls_align)
426 static_tls_align = STACK_ALIGN;
427 __static_tls_align_m1 = static_tls_align - 1;
428
429 __static_tls_size = roundup (__static_tls_size, static_tls_align);
430
431 /* Determine the default allowed stack size. This is the size used
432 in case the user does not specify one. */
433 struct rlimit limit;
434 if (__getrlimit (RLIMIT_STACK, &limit) != 0
435 || limit.rlim_cur == RLIM_INFINITY)
436 /* The system limit is not usable. Use an architecture-specific
437 default. */
438 limit.rlim_cur = ARCH_STACK_DEFAULT_SIZE;
439 else if (limit.rlim_cur < PTHREAD_STACK_MIN)
440 /* The system limit is unusably small.
441 Use the minimal size acceptable. */
442 limit.rlim_cur = PTHREAD_STACK_MIN;
443
444 /* Make sure it meets the minimum size that allocate_stack
445 (allocatestack.c) will demand, which depends on the page size. */
446 const uintptr_t pagesz = GLRO(dl_pagesize);
447 const size_t minstack = pagesz + __static_tls_size + MINIMAL_REST_STACK;
448 if (limit.rlim_cur < minstack)
449 limit.rlim_cur = minstack;
450
451 /* Round the resource limit up to page size. */
452 limit.rlim_cur = ALIGN_UP (limit.rlim_cur, pagesz);
453 lll_lock (__default_pthread_attr_lock, LLL_PRIVATE);
454 __default_pthread_attr.stacksize = limit.rlim_cur;
455 __default_pthread_attr.guardsize = GLRO (dl_pagesize);
456 lll_unlock (__default_pthread_attr_lock, LLL_PRIVATE);
457
458#ifdef SHARED
459 /* Transfer the old value from the dynamic linker's internal location. */
460 *__libc_dl_error_tsd () = *(*GL(dl_error_catch_tsd)) ();
461 GL(dl_error_catch_tsd) = &__libc_dl_error_tsd;
462
463 /* Make __rtld_lock_{,un}lock_recursive use pthread_mutex_{,un}lock,
464 keep the lock count from the ld.so implementation. */
465 GL(dl_rtld_lock_recursive) = (void *) __pthread_mutex_lock;
466 GL(dl_rtld_unlock_recursive) = (void *) __pthread_mutex_unlock;
467 unsigned int rtld_lock_count = GL(dl_load_lock).mutex.__data.__count;
468 GL(dl_load_lock).mutex.__data.__count = 0;
469 while (rtld_lock_count-- > 0)
470 __pthread_mutex_lock (&GL(dl_load_lock).mutex);
471
472 GL(dl_make_stack_executable_hook) = &__make_stacks_executable;
473#endif
474
475 GL(dl_init_static_tls) = &__pthread_init_static_tls;
476
477 GL(dl_wait_lookup_done) = &__wait_lookup_done;
478
479 /* Register the fork generation counter with the libc. */
480#ifndef TLS_MULTIPLE_THREADS_IN_TCB
481 __libc_multiple_threads_ptr =
482#endif
483 __libc_pthread_init (&__fork_generation, __reclaim_stacks,
484 ptr_pthread_functions);
485
486 /* Determine whether the machine is SMP or not. */
487 __is_smp = is_smp_system ();
488}
489strong_alias (__pthread_initialize_minimal_internal,
490 __pthread_initialize_minimal)
491
492
493size_t
494__pthread_get_minstack (const pthread_attr_t *attr)
495{
496 return GLRO(dl_pagesize) + __static_tls_size + PTHREAD_STACK_MIN;
497}
498