1/* Copyright (C) 2002-2020 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <stdlib.h>
22#include "pthreadP.h"
23#include <lowlevellock.h>
24#include <futex-internal.h>
25
26#ifndef lll_trylock_elision
27#define lll_trylock_elision(a,t) lll_trylock(a)
28#endif
29
30#ifndef FORCE_ELISION
31#define FORCE_ELISION(m, s)
32#endif
33
34int
35__pthread_mutex_trylock (pthread_mutex_t *mutex)
36{
37 int oldval;
38 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
39
40 /* See concurrency notes regarding mutex type which is loaded from __kind
41 in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */
42 switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
43 PTHREAD_MUTEX_TIMED_NP))
44 {
45 /* Recursive mutex. */
46 case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
47 case PTHREAD_MUTEX_RECURSIVE_NP:
48 /* Check whether we already hold the mutex. */
49 if (mutex->__data.__owner == id)
50 {
51 /* Just bump the counter. */
52 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
53 /* Overflow of the counter. */
54 return EAGAIN;
55
56 ++mutex->__data.__count;
57 return 0;
58 }
59
60 if (lll_trylock (mutex->__data.__lock) == 0)
61 {
62 /* Record the ownership. */
63 mutex->__data.__owner = id;
64 mutex->__data.__count = 1;
65 ++mutex->__data.__nusers;
66 return 0;
67 }
68 break;
69
70 case PTHREAD_MUTEX_TIMED_ELISION_NP:
71 elision: __attribute__((unused))
72 if (lll_trylock_elision (mutex->__data.__lock,
73 mutex->__data.__elision) != 0)
74 break;
75 /* Don't record the ownership. */
76 return 0;
77
78 case PTHREAD_MUTEX_TIMED_NP:
79 FORCE_ELISION (mutex, goto elision);
80 /*FALL THROUGH*/
81 case PTHREAD_MUTEX_ADAPTIVE_NP:
82 case PTHREAD_MUTEX_ERRORCHECK_NP:
83 if (lll_trylock (mutex->__data.__lock) != 0)
84 break;
85
86 /* Record the ownership. */
87 mutex->__data.__owner = id;
88 ++mutex->__data.__nusers;
89
90 return 0;
91
92 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
93 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
94 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
95 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
96 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
97 &mutex->__data.__list.__next);
98 /* We need to set op_pending before starting the operation. Also
99 see comments at ENQUEUE_MUTEX. */
100 __asm ("" ::: "memory");
101
102 oldval = mutex->__data.__lock;
103 do
104 {
105 again:
106 if ((oldval & FUTEX_OWNER_DIED) != 0)
107 {
108 /* The previous owner died. Try locking the mutex. */
109 int newval = id | (oldval & FUTEX_WAITERS);
110
111 newval
112 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
113 newval, oldval);
114
115 if (newval != oldval)
116 {
117 oldval = newval;
118 goto again;
119 }
120
121 /* We got the mutex. */
122 mutex->__data.__count = 1;
123 /* But it is inconsistent unless marked otherwise. */
124 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
125
126 /* We must not enqueue the mutex before we have acquired it.
127 Also see comments at ENQUEUE_MUTEX. */
128 __asm ("" ::: "memory");
129 ENQUEUE_MUTEX (mutex);
130 /* We need to clear op_pending after we enqueue the mutex. */
131 __asm ("" ::: "memory");
132 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
133
134 /* Note that we deliberately exit here. If we fall
135 through to the end of the function __nusers would be
136 incremented which is not correct because the old
137 owner has to be discounted. */
138 return EOWNERDEAD;
139 }
140
141 /* Check whether we already hold the mutex. */
142 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
143 {
144 int kind = PTHREAD_MUTEX_TYPE (mutex);
145 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
146 {
147 /* We do not need to ensure ordering wrt another memory
148 access. Also see comments at ENQUEUE_MUTEX. */
149 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
150 NULL);
151 return EDEADLK;
152 }
153
154 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
155 {
156 /* We do not need to ensure ordering wrt another memory
157 access. */
158 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
159 NULL);
160
161 /* Just bump the counter. */
162 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
163 /* Overflow of the counter. */
164 return EAGAIN;
165
166 ++mutex->__data.__count;
167
168 return 0;
169 }
170 }
171
172 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
173 id, 0);
174 if (oldval != 0 && (oldval & FUTEX_OWNER_DIED) == 0)
175 {
176 /* We haven't acquired the lock as it is already acquired by
177 another owner. We do not need to ensure ordering wrt another
178 memory access. */
179 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
180
181 return EBUSY;
182 }
183
184 if (__builtin_expect (mutex->__data.__owner
185 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
186 {
187 /* This mutex is now not recoverable. */
188 mutex->__data.__count = 0;
189 if (oldval == id)
190 lll_unlock (mutex->__data.__lock,
191 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
192 /* FIXME This violates the mutex destruction requirements. See
193 __pthread_mutex_unlock_full. */
194 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
195 return ENOTRECOVERABLE;
196 }
197 }
198 while ((oldval & FUTEX_OWNER_DIED) != 0);
199
200 /* We must not enqueue the mutex before we have acquired it.
201 Also see comments at ENQUEUE_MUTEX. */
202 __asm ("" ::: "memory");
203 ENQUEUE_MUTEX (mutex);
204 /* We need to clear op_pending after we enqueue the mutex. */
205 __asm ("" ::: "memory");
206 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
207
208 mutex->__data.__owner = id;
209 ++mutex->__data.__nusers;
210 mutex->__data.__count = 1;
211
212 return 0;
213
214 /* The PI support requires the Linux futex system call. If that's not
215 available, pthread_mutex_init should never have allowed the type to
216 be set. So it will get the default case for an invalid type. */
217#ifdef __NR_futex
218 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
219 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
220 case PTHREAD_MUTEX_PI_NORMAL_NP:
221 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
222 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
223 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
224 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
225 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
226 {
227 int kind, robust;
228 {
229 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
230 in sysdeps/nptl/bits/thread-shared-types.h. */
231 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
232 kind = mutex_kind & PTHREAD_MUTEX_KIND_MASK_NP;
233 robust = mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
234 }
235
236 if (robust)
237 {
238 /* Note: robust PI futexes are signaled by setting bit 0. */
239 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
240 (void *) (((uintptr_t) &mutex->__data.__list.__next)
241 | 1));
242 /* We need to set op_pending before starting the operation. Also
243 see comments at ENQUEUE_MUTEX. */
244 __asm ("" ::: "memory");
245 }
246
247 oldval = mutex->__data.__lock;
248
249 /* Check whether we already hold the mutex. */
250 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
251 {
252 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
253 {
254 /* We do not need to ensure ordering wrt another memory
255 access. */
256 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
257 return EDEADLK;
258 }
259
260 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
261 {
262 /* We do not need to ensure ordering wrt another memory
263 access. */
264 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
265
266 /* Just bump the counter. */
267 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
268 /* Overflow of the counter. */
269 return EAGAIN;
270
271 ++mutex->__data.__count;
272
273 return 0;
274 }
275 }
276
277 oldval
278 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
279 id, 0);
280
281 if (oldval != 0)
282 {
283 if ((oldval & FUTEX_OWNER_DIED) == 0)
284 {
285 /* We haven't acquired the lock as it is already acquired by
286 another owner. We do not need to ensure ordering wrt another
287 memory access. */
288 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
289
290 return EBUSY;
291 }
292
293 assert (robust);
294
295 /* The mutex owner died. The kernel will now take care of
296 everything. */
297 int private = (robust
298 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
299 : PTHREAD_MUTEX_PSHARED (mutex));
300 INTERNAL_SYSCALL_DECL (__err);
301 int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
302 __lll_private_flag (FUTEX_TRYLOCK_PI,
303 private), 0, 0);
304
305 if (INTERNAL_SYSCALL_ERROR_P (e, __err)
306 && INTERNAL_SYSCALL_ERRNO (e, __err) == EWOULDBLOCK)
307 {
308 /* The kernel has not yet finished the mutex owner death.
309 We do not need to ensure ordering wrt another memory
310 access. */
311 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
312
313 return EBUSY;
314 }
315
316 oldval = mutex->__data.__lock;
317 }
318
319 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
320 {
321 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
322
323 /* We got the mutex. */
324 mutex->__data.__count = 1;
325 /* But it is inconsistent unless marked otherwise. */
326 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
327
328 /* We must not enqueue the mutex before we have acquired it.
329 Also see comments at ENQUEUE_MUTEX. */
330 __asm ("" ::: "memory");
331 ENQUEUE_MUTEX (mutex);
332 /* We need to clear op_pending after we enqueue the mutex. */
333 __asm ("" ::: "memory");
334 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
335
336 /* Note that we deliberately exit here. If we fall
337 through to the end of the function __nusers would be
338 incremented which is not correct because the old owner
339 has to be discounted. */
340 return EOWNERDEAD;
341 }
342
343 if (robust
344 && __builtin_expect (mutex->__data.__owner
345 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
346 {
347 /* This mutex is now not recoverable. */
348 mutex->__data.__count = 0;
349
350 futex_unlock_pi ((unsigned int *) &mutex->__data.__lock,
351 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
352
353 /* To the kernel, this will be visible after the kernel has
354 acquired the mutex in the syscall. */
355 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
356 return ENOTRECOVERABLE;
357 }
358
359 if (robust)
360 {
361 /* We must not enqueue the mutex before we have acquired it.
362 Also see comments at ENQUEUE_MUTEX. */
363 __asm ("" ::: "memory");
364 ENQUEUE_MUTEX_PI (mutex);
365 /* We need to clear op_pending after we enqueue the mutex. */
366 __asm ("" ::: "memory");
367 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
368 }
369
370 mutex->__data.__owner = id;
371 ++mutex->__data.__nusers;
372 mutex->__data.__count = 1;
373
374 return 0;
375 }
376#endif /* __NR_futex. */
377
378 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
379 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
380 case PTHREAD_MUTEX_PP_NORMAL_NP:
381 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
382 {
383 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
384 in sysdeps/nptl/bits/thread-shared-types.h. */
385 int kind = atomic_load_relaxed (&(mutex->__data.__kind))
386 & PTHREAD_MUTEX_KIND_MASK_NP;
387
388 oldval = mutex->__data.__lock;
389
390 /* Check whether we already hold the mutex. */
391 if (mutex->__data.__owner == id)
392 {
393 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
394 return EDEADLK;
395
396 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
397 {
398 /* Just bump the counter. */
399 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
400 /* Overflow of the counter. */
401 return EAGAIN;
402
403 ++mutex->__data.__count;
404
405 return 0;
406 }
407 }
408
409 int oldprio = -1, ceilval;
410 do
411 {
412 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
413 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
414
415 if (__pthread_current_priority () > ceiling)
416 {
417 if (oldprio != -1)
418 __pthread_tpp_change_priority (oldprio, -1);
419 return EINVAL;
420 }
421
422 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
423 if (retval)
424 return retval;
425
426 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
427 oldprio = ceiling;
428
429 oldval
430 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
431 ceilval | 1, ceilval);
432
433 if (oldval == ceilval)
434 break;
435 }
436 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
437
438 if (oldval != ceilval)
439 {
440 __pthread_tpp_change_priority (oldprio, -1);
441 break;
442 }
443
444 assert (mutex->__data.__owner == 0);
445 /* Record the ownership. */
446 mutex->__data.__owner = id;
447 ++mutex->__data.__nusers;
448 mutex->__data.__count = 1;
449
450 return 0;
451 }
452 break;
453
454 default:
455 /* Correct code cannot set any other type. */
456 return EINVAL;
457 }
458
459 return EBUSY;
460}
461
462#ifndef __pthread_mutex_trylock
463#ifndef pthread_mutex_trylock
464weak_alias (__pthread_mutex_trylock, pthread_mutex_trylock)
465hidden_def (__pthread_mutex_trylock)
466#endif
467#endif
468