1/* Copyright (C) 2002-2017 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <stdlib.h>
22#include <unistd.h>
23#include <sys/param.h>
24#include <not-cancel.h>
25#include "pthreadP.h"
26#include <atomic.h>
27#include <lowlevellock.h>
28#include <stap-probe.h>
29
30#ifndef lll_lock_elision
31#define lll_lock_elision(lock, try_lock, private) ({ \
32 lll_lock (lock, private); 0; })
33#endif
34
35#ifndef lll_trylock_elision
36#define lll_trylock_elision(a,t) lll_trylock(a)
37#endif
38
39/* Some of the following definitions differ when pthread_mutex_cond_lock.c
40 includes this file. */
41#ifndef LLL_MUTEX_LOCK
42# define LLL_MUTEX_LOCK(mutex) \
43 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
44# define LLL_MUTEX_TRYLOCK(mutex) \
45 lll_trylock ((mutex)->__data.__lock)
46# define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0
47# define LLL_MUTEX_LOCK_ELISION(mutex) \
48 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
49 PTHREAD_MUTEX_PSHARED (mutex))
50# define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
51 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
52 PTHREAD_MUTEX_PSHARED (mutex))
53#endif
54
55#ifndef FORCE_ELISION
56#define FORCE_ELISION(m, s)
57#endif
58
59static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
60 __attribute_noinline__;
61
62int
63__pthread_mutex_lock (pthread_mutex_t *mutex)
64{
65 assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
66
67 unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
68
69 LIBC_PROBE (mutex_entry, 1, mutex);
70
71 if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
72 | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
73 return __pthread_mutex_lock_full (mutex);
74
75 if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
76 {
77 FORCE_ELISION (mutex, goto elision);
78 simple:
79 /* Normal mutex. */
80 LLL_MUTEX_LOCK (mutex);
81 assert (mutex->__data.__owner == 0);
82 }
83#ifdef HAVE_ELISION
84 else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
85 {
86 elision: __attribute__((unused))
87 /* This case can never happen on a system without elision,
88 as the mutex type initialization functions will not
89 allow to set the elision flags. */
90 /* Don't record owner or users for elision case. This is a
91 tail call. */
92 return LLL_MUTEX_LOCK_ELISION (mutex);
93 }
94#endif
95 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
96 == PTHREAD_MUTEX_RECURSIVE_NP, 1))
97 {
98 /* Recursive mutex. */
99 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
100
101 /* Check whether we already hold the mutex. */
102 if (mutex->__data.__owner == id)
103 {
104 /* Just bump the counter. */
105 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
106 /* Overflow of the counter. */
107 return EAGAIN;
108
109 ++mutex->__data.__count;
110
111 return 0;
112 }
113
114 /* We have to get the mutex. */
115 LLL_MUTEX_LOCK (mutex);
116
117 assert (mutex->__data.__owner == 0);
118 mutex->__data.__count = 1;
119 }
120 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
121 == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
122 {
123 if (! __is_smp)
124 goto simple;
125
126 if (LLL_MUTEX_TRYLOCK (mutex) != 0)
127 {
128 int cnt = 0;
129 int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
130 mutex->__data.__spins * 2 + 10);
131 do
132 {
133 if (cnt++ >= max_cnt)
134 {
135 LLL_MUTEX_LOCK (mutex);
136 break;
137 }
138 atomic_spin_nop ();
139 }
140 while (LLL_MUTEX_TRYLOCK (mutex) != 0);
141
142 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
143 }
144 assert (mutex->__data.__owner == 0);
145 }
146 else
147 {
148 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
149 assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
150 /* Check whether we already hold the mutex. */
151 if (__glibc_unlikely (mutex->__data.__owner == id))
152 return EDEADLK;
153 goto simple;
154 }
155
156 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
157
158 /* Record the ownership. */
159 mutex->__data.__owner = id;
160#ifndef NO_INCR
161 ++mutex->__data.__nusers;
162#endif
163
164 LIBC_PROBE (mutex_acquired, 1, mutex);
165
166 return 0;
167}
168
169static int
170__pthread_mutex_lock_full (pthread_mutex_t *mutex)
171{
172 int oldval;
173 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
174
175 switch (PTHREAD_MUTEX_TYPE (mutex))
176 {
177 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
178 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
179 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
180 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
181 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
182 &mutex->__data.__list.__next);
183 /* We need to set op_pending before starting the operation. Also
184 see comments at ENQUEUE_MUTEX. */
185 __asm ("" ::: "memory");
186
187 oldval = mutex->__data.__lock;
188 /* This is set to FUTEX_WAITERS iff we might have shared the
189 FUTEX_WAITERS flag with other threads, and therefore need to keep it
190 set to avoid lost wake-ups. We have the same requirement in the
191 simple mutex algorithm.
192 We start with value zero for a normal mutex, and FUTEX_WAITERS if we
193 are building the special case mutexes for use from within condition
194 variables. */
195 unsigned int assume_other_futex_waiters = LLL_ROBUST_MUTEX_LOCK_MODIFIER;
196 while (1)
197 {
198 /* Try to acquire the lock through a CAS from 0 (not acquired) to
199 our TID | assume_other_futex_waiters. */
200 if (__glibc_likely (oldval == 0))
201 {
202 oldval
203 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
204 id | assume_other_futex_waiters, 0);
205 if (__glibc_likely (oldval == 0))
206 break;
207 }
208
209 if ((oldval & FUTEX_OWNER_DIED) != 0)
210 {
211 /* The previous owner died. Try locking the mutex. */
212 int newval = id;
213#ifdef NO_INCR
214 /* We are not taking assume_other_futex_waiters into accoount
215 here simply because we'll set FUTEX_WAITERS anyway. */
216 newval |= FUTEX_WAITERS;
217#else
218 newval |= (oldval & FUTEX_WAITERS) | assume_other_futex_waiters;
219#endif
220
221 newval
222 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
223 newval, oldval);
224
225 if (newval != oldval)
226 {
227 oldval = newval;
228 continue;
229 }
230
231 /* We got the mutex. */
232 mutex->__data.__count = 1;
233 /* But it is inconsistent unless marked otherwise. */
234 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
235
236 /* We must not enqueue the mutex before we have acquired it.
237 Also see comments at ENQUEUE_MUTEX. */
238 __asm ("" ::: "memory");
239 ENQUEUE_MUTEX (mutex);
240 /* We need to clear op_pending after we enqueue the mutex. */
241 __asm ("" ::: "memory");
242 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
243
244 /* Note that we deliberately exit here. If we fall
245 through to the end of the function __nusers would be
246 incremented which is not correct because the old
247 owner has to be discounted. If we are not supposed
248 to increment __nusers we actually have to decrement
249 it here. */
250#ifdef NO_INCR
251 --mutex->__data.__nusers;
252#endif
253
254 return EOWNERDEAD;
255 }
256
257 /* Check whether we already hold the mutex. */
258 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
259 {
260 int kind = PTHREAD_MUTEX_TYPE (mutex);
261 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
262 {
263 /* We do not need to ensure ordering wrt another memory
264 access. Also see comments at ENQUEUE_MUTEX. */
265 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
266 NULL);
267 return EDEADLK;
268 }
269
270 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
271 {
272 /* We do not need to ensure ordering wrt another memory
273 access. */
274 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
275 NULL);
276
277 /* Just bump the counter. */
278 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
279 /* Overflow of the counter. */
280 return EAGAIN;
281
282 ++mutex->__data.__count;
283
284 return 0;
285 }
286 }
287
288 /* We cannot acquire the mutex nor has its owner died. Thus, try
289 to block using futexes. Set FUTEX_WAITERS if necessary so that
290 other threads are aware that there are potentially threads
291 blocked on the futex. Restart if oldval changed in the
292 meantime. */
293 if ((oldval & FUTEX_WAITERS) == 0)
294 {
295 if (atomic_compare_and_exchange_bool_acq (&mutex->__data.__lock,
296 oldval | FUTEX_WAITERS,
297 oldval)
298 != 0)
299 {
300 oldval = mutex->__data.__lock;
301 continue;
302 }
303 oldval |= FUTEX_WAITERS;
304 }
305
306 /* It is now possible that we share the FUTEX_WAITERS flag with
307 another thread; therefore, update assume_other_futex_waiters so
308 that we do not forget about this when handling other cases
309 above and thus do not cause lost wake-ups. */
310 assume_other_futex_waiters |= FUTEX_WAITERS;
311
312 /* Block using the futex and reload current lock value. */
313 lll_futex_wait (&mutex->__data.__lock, oldval,
314 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
315 oldval = mutex->__data.__lock;
316 }
317
318 /* We have acquired the mutex; check if it is still consistent. */
319 if (__builtin_expect (mutex->__data.__owner
320 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
321 {
322 /* This mutex is now not recoverable. */
323 mutex->__data.__count = 0;
324 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
325 lll_unlock (mutex->__data.__lock, private);
326 /* FIXME This violates the mutex destruction requirements. See
327 __pthread_mutex_unlock_full. */
328 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
329 return ENOTRECOVERABLE;
330 }
331
332 mutex->__data.__count = 1;
333 /* We must not enqueue the mutex before we have acquired it.
334 Also see comments at ENQUEUE_MUTEX. */
335 __asm ("" ::: "memory");
336 ENQUEUE_MUTEX (mutex);
337 /* We need to clear op_pending after we enqueue the mutex. */
338 __asm ("" ::: "memory");
339 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
340 break;
341
342 /* The PI support requires the Linux futex system call. If that's not
343 available, pthread_mutex_init should never have allowed the type to
344 be set. So it will get the default case for an invalid type. */
345#ifdef __NR_futex
346 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
347 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
348 case PTHREAD_MUTEX_PI_NORMAL_NP:
349 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
350 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
351 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
352 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
353 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
354 {
355 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
356 int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
357
358 if (robust)
359 {
360 /* Note: robust PI futexes are signaled by setting bit 0. */
361 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
362 (void *) (((uintptr_t) &mutex->__data.__list.__next)
363 | 1));
364 /* We need to set op_pending before starting the operation. Also
365 see comments at ENQUEUE_MUTEX. */
366 __asm ("" ::: "memory");
367 }
368
369 oldval = mutex->__data.__lock;
370
371 /* Check whether we already hold the mutex. */
372 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
373 {
374 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
375 {
376 /* We do not need to ensure ordering wrt another memory
377 access. */
378 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
379 return EDEADLK;
380 }
381
382 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
383 {
384 /* We do not need to ensure ordering wrt another memory
385 access. */
386 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
387
388 /* Just bump the counter. */
389 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
390 /* Overflow of the counter. */
391 return EAGAIN;
392
393 ++mutex->__data.__count;
394
395 return 0;
396 }
397 }
398
399 int newval = id;
400# ifdef NO_INCR
401 newval |= FUTEX_WAITERS;
402# endif
403 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
404 newval, 0);
405
406 if (oldval != 0)
407 {
408 /* The mutex is locked. The kernel will now take care of
409 everything. */
410 int private = (robust
411 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
412 : PTHREAD_MUTEX_PSHARED (mutex));
413 INTERNAL_SYSCALL_DECL (__err);
414 int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
415 __lll_private_flag (FUTEX_LOCK_PI,
416 private), 1, 0);
417
418 if (INTERNAL_SYSCALL_ERROR_P (e, __err)
419 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
420 || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
421 {
422 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
423 || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
424 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
425 /* ESRCH can happen only for non-robust PI mutexes where
426 the owner of the lock died. */
427 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
428
429 /* Delay the thread indefinitely. */
430 while (1)
431 pause_not_cancel ();
432 }
433
434 oldval = mutex->__data.__lock;
435
436 assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
437 }
438
439 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
440 {
441 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
442
443 /* We got the mutex. */
444 mutex->__data.__count = 1;
445 /* But it is inconsistent unless marked otherwise. */
446 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
447
448 /* We must not enqueue the mutex before we have acquired it.
449 Also see comments at ENQUEUE_MUTEX. */
450 __asm ("" ::: "memory");
451 ENQUEUE_MUTEX_PI (mutex);
452 /* We need to clear op_pending after we enqueue the mutex. */
453 __asm ("" ::: "memory");
454 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
455
456 /* Note that we deliberately exit here. If we fall
457 through to the end of the function __nusers would be
458 incremented which is not correct because the old owner
459 has to be discounted. If we are not supposed to
460 increment __nusers we actually have to decrement it here. */
461# ifdef NO_INCR
462 --mutex->__data.__nusers;
463# endif
464
465 return EOWNERDEAD;
466 }
467
468 if (robust
469 && __builtin_expect (mutex->__data.__owner
470 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
471 {
472 /* This mutex is now not recoverable. */
473 mutex->__data.__count = 0;
474
475 INTERNAL_SYSCALL_DECL (__err);
476 INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
477 __lll_private_flag (FUTEX_UNLOCK_PI,
478 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
479 0, 0);
480
481 /* To the kernel, this will be visible after the kernel has
482 acquired the mutex in the syscall. */
483 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
484 return ENOTRECOVERABLE;
485 }
486
487 mutex->__data.__count = 1;
488 if (robust)
489 {
490 /* We must not enqueue the mutex before we have acquired it.
491 Also see comments at ENQUEUE_MUTEX. */
492 __asm ("" ::: "memory");
493 ENQUEUE_MUTEX_PI (mutex);
494 /* We need to clear op_pending after we enqueue the mutex. */
495 __asm ("" ::: "memory");
496 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
497 }
498 }
499 break;
500#endif /* __NR_futex. */
501
502 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
503 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
504 case PTHREAD_MUTEX_PP_NORMAL_NP:
505 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
506 {
507 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
508
509 oldval = mutex->__data.__lock;
510
511 /* Check whether we already hold the mutex. */
512 if (mutex->__data.__owner == id)
513 {
514 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
515 return EDEADLK;
516
517 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
518 {
519 /* Just bump the counter. */
520 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
521 /* Overflow of the counter. */
522 return EAGAIN;
523
524 ++mutex->__data.__count;
525
526 return 0;
527 }
528 }
529
530 int oldprio = -1, ceilval;
531 do
532 {
533 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
534 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
535
536 if (__pthread_current_priority () > ceiling)
537 {
538 if (oldprio != -1)
539 __pthread_tpp_change_priority (oldprio, -1);
540 return EINVAL;
541 }
542
543 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
544 if (retval)
545 return retval;
546
547 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
548 oldprio = ceiling;
549
550 oldval
551 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
552#ifdef NO_INCR
553 ceilval | 2,
554#else
555 ceilval | 1,
556#endif
557 ceilval);
558
559 if (oldval == ceilval)
560 break;
561
562 do
563 {
564 oldval
565 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
566 ceilval | 2,
567 ceilval | 1);
568
569 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
570 break;
571
572 if (oldval != ceilval)
573 lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
574 PTHREAD_MUTEX_PSHARED (mutex));
575 }
576 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
577 ceilval | 2, ceilval)
578 != ceilval);
579 }
580 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
581
582 assert (mutex->__data.__owner == 0);
583 mutex->__data.__count = 1;
584 }
585 break;
586
587 default:
588 /* Correct code cannot set any other type. */
589 return EINVAL;
590 }
591
592 /* Record the ownership. */
593 mutex->__data.__owner = id;
594#ifndef NO_INCR
595 ++mutex->__data.__nusers;
596#endif
597
598 LIBC_PROBE (mutex_acquired, 1, mutex);
599
600 return 0;
601}
602#ifndef __pthread_mutex_lock
603strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
604hidden_def (__pthread_mutex_lock)
605#endif
606
607
608#ifdef NO_INCR
609void
610internal_function
611__pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
612{
613 assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
614 assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
615 assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
616
617 /* Record the ownership. */
618 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
619 mutex->__data.__owner = id;
620
621 if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
622 ++mutex->__data.__count;
623}
624#endif
625