1/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <stdlib.h>
22#include <unistd.h>
23#include <sys/param.h>
24#include <not-cancel.h>
25#include "pthreadP.h"
26#include <atomic.h>
27#include <lowlevellock.h>
28#include <stap-probe.h>
29
30#ifndef lll_lock_elision
31#define lll_lock_elision(lock, try_lock, private) ({ \
32 lll_lock (lock, private); 0; })
33#endif
34
35#ifndef lll_trylock_elision
36#define lll_trylock_elision(a,t) lll_trylock(a)
37#endif
38
39#ifndef LLL_MUTEX_LOCK
40# define LLL_MUTEX_LOCK(mutex) \
41 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
42# define LLL_MUTEX_TRYLOCK(mutex) \
43 lll_trylock ((mutex)->__data.__lock)
44# define LLL_ROBUST_MUTEX_LOCK(mutex, id) \
45 lll_robust_lock ((mutex)->__data.__lock, id, \
46 PTHREAD_ROBUST_MUTEX_PSHARED (mutex))
47# define LLL_MUTEX_LOCK_ELISION(mutex) \
48 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
49 PTHREAD_MUTEX_PSHARED (mutex))
50# define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
51 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
52 PTHREAD_MUTEX_PSHARED (mutex))
53#endif
54
55#ifndef FORCE_ELISION
56#define FORCE_ELISION(m, s)
57#endif
58
59static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
60 __attribute_noinline__;
61
62int
63__pthread_mutex_lock (pthread_mutex_t *mutex)
64{
65 assert (sizeof (mutex->__size) >= sizeof (mutex->__data));
66
67 unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
68
69 LIBC_PROBE (mutex_entry, 1, mutex);
70
71 if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
72 | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
73 return __pthread_mutex_lock_full (mutex);
74
75 if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
76 {
77 FORCE_ELISION (mutex, goto elision);
78 simple:
79 /* Normal mutex. */
80 LLL_MUTEX_LOCK (mutex);
81 assert (mutex->__data.__owner == 0);
82 }
83#ifdef HAVE_ELISION
84 else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
85 {
86 elision: __attribute__((unused))
87 /* This case can never happen on a system without elision,
88 as the mutex type initialization functions will not
89 allow to set the elision flags. */
90 /* Don't record owner or users for elision case. This is a
91 tail call. */
92 return LLL_MUTEX_LOCK_ELISION (mutex);
93 }
94#endif
95 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
96 == PTHREAD_MUTEX_RECURSIVE_NP, 1))
97 {
98 /* Recursive mutex. */
99 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
100
101 /* Check whether we already hold the mutex. */
102 if (mutex->__data.__owner == id)
103 {
104 /* Just bump the counter. */
105 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
106 /* Overflow of the counter. */
107 return EAGAIN;
108
109 ++mutex->__data.__count;
110
111 return 0;
112 }
113
114 /* We have to get the mutex. */
115 LLL_MUTEX_LOCK (mutex);
116
117 assert (mutex->__data.__owner == 0);
118 mutex->__data.__count = 1;
119 }
120 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
121 == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
122 {
123 if (! __is_smp)
124 goto simple;
125
126 if (LLL_MUTEX_TRYLOCK (mutex) != 0)
127 {
128 int cnt = 0;
129 int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
130 mutex->__data.__spins * 2 + 10);
131 do
132 {
133 if (cnt++ >= max_cnt)
134 {
135 LLL_MUTEX_LOCK (mutex);
136 break;
137 }
138 atomic_spin_nop ();
139 }
140 while (LLL_MUTEX_TRYLOCK (mutex) != 0);
141
142 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
143 }
144 assert (mutex->__data.__owner == 0);
145 }
146 else
147 {
148 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
149 assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
150 /* Check whether we already hold the mutex. */
151 if (__glibc_unlikely (mutex->__data.__owner == id))
152 return EDEADLK;
153 goto simple;
154 }
155
156 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
157
158 /* Record the ownership. */
159 mutex->__data.__owner = id;
160#ifndef NO_INCR
161 ++mutex->__data.__nusers;
162#endif
163
164 LIBC_PROBE (mutex_acquired, 1, mutex);
165
166 return 0;
167}
168
169static int
170__pthread_mutex_lock_full (pthread_mutex_t *mutex)
171{
172 int oldval;
173 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
174
175 switch (PTHREAD_MUTEX_TYPE (mutex))
176 {
177 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
178 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
179 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
180 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
181 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
182 &mutex->__data.__list.__next);
183
184 oldval = mutex->__data.__lock;
185 do
186 {
187 again:
188 if ((oldval & FUTEX_OWNER_DIED) != 0)
189 {
190 /* The previous owner died. Try locking the mutex. */
191 int newval = id;
192#ifdef NO_INCR
193 newval |= FUTEX_WAITERS;
194#else
195 newval |= (oldval & FUTEX_WAITERS);
196#endif
197
198 newval
199 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
200 newval, oldval);
201
202 if (newval != oldval)
203 {
204 oldval = newval;
205 goto again;
206 }
207
208 /* We got the mutex. */
209 mutex->__data.__count = 1;
210 /* But it is inconsistent unless marked otherwise. */
211 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
212
213 ENQUEUE_MUTEX (mutex);
214 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
215
216 /* Note that we deliberately exit here. If we fall
217 through to the end of the function __nusers would be
218 incremented which is not correct because the old
219 owner has to be discounted. If we are not supposed
220 to increment __nusers we actually have to decrement
221 it here. */
222#ifdef NO_INCR
223 --mutex->__data.__nusers;
224#endif
225
226 return EOWNERDEAD;
227 }
228
229 /* Check whether we already hold the mutex. */
230 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
231 {
232 int kind = PTHREAD_MUTEX_TYPE (mutex);
233 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
234 {
235 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
236 NULL);
237 return EDEADLK;
238 }
239
240 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
241 {
242 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
243 NULL);
244
245 /* Just bump the counter. */
246 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
247 /* Overflow of the counter. */
248 return EAGAIN;
249
250 ++mutex->__data.__count;
251
252 return 0;
253 }
254 }
255
256 oldval = LLL_ROBUST_MUTEX_LOCK (mutex, id);
257
258 if (__builtin_expect (mutex->__data.__owner
259 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
260 {
261 /* This mutex is now not recoverable. */
262 mutex->__data.__count = 0;
263 lll_unlock (mutex->__data.__lock,
264 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
265 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
266 return ENOTRECOVERABLE;
267 }
268 }
269 while ((oldval & FUTEX_OWNER_DIED) != 0);
270
271 mutex->__data.__count = 1;
272 ENQUEUE_MUTEX (mutex);
273 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
274 break;
275
276 /* The PI support requires the Linux futex system call. If that's not
277 available, pthread_mutex_init should never have allowed the type to
278 be set. So it will get the default case for an invalid type. */
279#ifdef __NR_futex
280 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
281 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
282 case PTHREAD_MUTEX_PI_NORMAL_NP:
283 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
284 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
285 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
286 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
287 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
288 {
289 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
290 int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
291
292 if (robust)
293 /* Note: robust PI futexes are signaled by setting bit 0. */
294 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
295 (void *) (((uintptr_t) &mutex->__data.__list.__next)
296 | 1));
297
298 oldval = mutex->__data.__lock;
299
300 /* Check whether we already hold the mutex. */
301 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
302 {
303 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
304 {
305 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
306 return EDEADLK;
307 }
308
309 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
310 {
311 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
312
313 /* Just bump the counter. */
314 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
315 /* Overflow of the counter. */
316 return EAGAIN;
317
318 ++mutex->__data.__count;
319
320 return 0;
321 }
322 }
323
324 int newval = id;
325# ifdef NO_INCR
326 newval |= FUTEX_WAITERS;
327# endif
328 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
329 newval, 0);
330
331 if (oldval != 0)
332 {
333 /* The mutex is locked. The kernel will now take care of
334 everything. */
335 int private = (robust
336 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
337 : PTHREAD_MUTEX_PSHARED (mutex));
338 INTERNAL_SYSCALL_DECL (__err);
339 int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
340 __lll_private_flag (FUTEX_LOCK_PI,
341 private), 1, 0);
342
343 if (INTERNAL_SYSCALL_ERROR_P (e, __err)
344 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
345 || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
346 {
347 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
348 || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
349 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
350 /* ESRCH can happen only for non-robust PI mutexes where
351 the owner of the lock died. */
352 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
353
354 /* Delay the thread indefinitely. */
355 while (1)
356 pause_not_cancel ();
357 }
358
359 oldval = mutex->__data.__lock;
360
361 assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
362 }
363
364 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
365 {
366 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
367
368 /* We got the mutex. */
369 mutex->__data.__count = 1;
370 /* But it is inconsistent unless marked otherwise. */
371 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
372
373 ENQUEUE_MUTEX_PI (mutex);
374 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
375
376 /* Note that we deliberately exit here. If we fall
377 through to the end of the function __nusers would be
378 incremented which is not correct because the old owner
379 has to be discounted. If we are not supposed to
380 increment __nusers we actually have to decrement it here. */
381# ifdef NO_INCR
382 --mutex->__data.__nusers;
383# endif
384
385 return EOWNERDEAD;
386 }
387
388 if (robust
389 && __builtin_expect (mutex->__data.__owner
390 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
391 {
392 /* This mutex is now not recoverable. */
393 mutex->__data.__count = 0;
394
395 INTERNAL_SYSCALL_DECL (__err);
396 INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
397 __lll_private_flag (FUTEX_UNLOCK_PI,
398 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
399 0, 0);
400
401 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
402 return ENOTRECOVERABLE;
403 }
404
405 mutex->__data.__count = 1;
406 if (robust)
407 {
408 ENQUEUE_MUTEX_PI (mutex);
409 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
410 }
411 }
412 break;
413#endif /* __NR_futex. */
414
415 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
416 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
417 case PTHREAD_MUTEX_PP_NORMAL_NP:
418 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
419 {
420 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
421
422 oldval = mutex->__data.__lock;
423
424 /* Check whether we already hold the mutex. */
425 if (mutex->__data.__owner == id)
426 {
427 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
428 return EDEADLK;
429
430 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
431 {
432 /* Just bump the counter. */
433 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
434 /* Overflow of the counter. */
435 return EAGAIN;
436
437 ++mutex->__data.__count;
438
439 return 0;
440 }
441 }
442
443 int oldprio = -1, ceilval;
444 do
445 {
446 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
447 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
448
449 if (__pthread_current_priority () > ceiling)
450 {
451 if (oldprio != -1)
452 __pthread_tpp_change_priority (oldprio, -1);
453 return EINVAL;
454 }
455
456 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
457 if (retval)
458 return retval;
459
460 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
461 oldprio = ceiling;
462
463 oldval
464 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
465#ifdef NO_INCR
466 ceilval | 2,
467#else
468 ceilval | 1,
469#endif
470 ceilval);
471
472 if (oldval == ceilval)
473 break;
474
475 do
476 {
477 oldval
478 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
479 ceilval | 2,
480 ceilval | 1);
481
482 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
483 break;
484
485 if (oldval != ceilval)
486 lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
487 PTHREAD_MUTEX_PSHARED (mutex));
488 }
489 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
490 ceilval | 2, ceilval)
491 != ceilval);
492 }
493 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
494
495 assert (mutex->__data.__owner == 0);
496 mutex->__data.__count = 1;
497 }
498 break;
499
500 default:
501 /* Correct code cannot set any other type. */
502 return EINVAL;
503 }
504
505 /* Record the ownership. */
506 mutex->__data.__owner = id;
507#ifndef NO_INCR
508 ++mutex->__data.__nusers;
509#endif
510
511 LIBC_PROBE (mutex_acquired, 1, mutex);
512
513 return 0;
514}
515#ifndef __pthread_mutex_lock
516strong_alias (__pthread_mutex_lock, pthread_mutex_lock)
517hidden_def (__pthread_mutex_lock)
518#endif
519
520
521#ifdef NO_INCR
522void
523internal_function
524__pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
525{
526 assert ((mutex->__data.__kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
527 assert ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
528 assert ((mutex->__data.__kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
529
530 /* Record the ownership. */
531 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
532 mutex->__data.__owner = id;
533
534 if (mutex->__data.__kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
535 ++mutex->__data.__count;
536}
537#endif
538