1 | /* Copyright (C) 2002-2016 Free Software Foundation, Inc. |
2 | This file is part of the GNU C Library. |
3 | Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ |
18 | |
19 | #include <assert.h> |
20 | #include <errno.h> |
21 | #include <time.h> |
22 | #include <sys/param.h> |
23 | #include <sys/time.h> |
24 | #include "pthreadP.h" |
25 | #include <atomic.h> |
26 | #include <lowlevellock.h> |
27 | #include <not-cancel.h> |
28 | |
29 | #include <stap-probe.h> |
30 | |
31 | #ifndef lll_timedlock_elision |
32 | #define lll_timedlock_elision(a,dummy,b,c) lll_timedlock(a, b, c) |
33 | #endif |
34 | |
35 | #ifndef lll_trylock_elision |
36 | #define lll_trylock_elision(a,t) lll_trylock(a) |
37 | #endif |
38 | |
39 | #ifndef FORCE_ELISION |
40 | #define FORCE_ELISION(m, s) |
41 | #endif |
42 | |
43 | int |
44 | pthread_mutex_timedlock (pthread_mutex_t *mutex, |
45 | const struct timespec *abstime) |
46 | { |
47 | int oldval; |
48 | pid_t id = THREAD_GETMEM (THREAD_SELF, tid); |
49 | int result = 0; |
50 | |
51 | LIBC_PROBE (mutex_timedlock_entry, 2, mutex, abstime); |
52 | |
53 | /* We must not check ABSTIME here. If the thread does not block |
54 | abstime must not be checked for a valid value. */ |
55 | |
56 | switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex), |
57 | PTHREAD_MUTEX_TIMED_NP)) |
58 | { |
59 | /* Recursive mutex. */ |
60 | case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP: |
61 | case PTHREAD_MUTEX_RECURSIVE_NP: |
62 | /* Check whether we already hold the mutex. */ |
63 | if (mutex->__data.__owner == id) |
64 | { |
65 | /* Just bump the counter. */ |
66 | if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) |
67 | /* Overflow of the counter. */ |
68 | return EAGAIN; |
69 | |
70 | ++mutex->__data.__count; |
71 | |
72 | goto out; |
73 | } |
74 | |
75 | /* We have to get the mutex. */ |
76 | result = lll_timedlock (mutex->__data.__lock, abstime, |
77 | PTHREAD_MUTEX_PSHARED (mutex)); |
78 | |
79 | if (result != 0) |
80 | goto out; |
81 | |
82 | /* Only locked once so far. */ |
83 | mutex->__data.__count = 1; |
84 | break; |
85 | |
86 | /* Error checking mutex. */ |
87 | case PTHREAD_MUTEX_ERRORCHECK_NP: |
88 | /* Check whether we already hold the mutex. */ |
89 | if (__glibc_unlikely (mutex->__data.__owner == id)) |
90 | return EDEADLK; |
91 | |
92 | /* Don't do lock elision on an error checking mutex. */ |
93 | goto simple; |
94 | |
95 | case PTHREAD_MUTEX_TIMED_NP: |
96 | FORCE_ELISION (mutex, goto elision); |
97 | simple: |
98 | /* Normal mutex. */ |
99 | result = lll_timedlock (mutex->__data.__lock, abstime, |
100 | PTHREAD_MUTEX_PSHARED (mutex)); |
101 | break; |
102 | |
103 | case PTHREAD_MUTEX_TIMED_ELISION_NP: |
104 | elision: __attribute__((unused)) |
105 | /* Don't record ownership */ |
106 | return lll_timedlock_elision (mutex->__data.__lock, |
107 | mutex->__data.__spins, |
108 | abstime, |
109 | PTHREAD_MUTEX_PSHARED (mutex)); |
110 | |
111 | |
112 | case PTHREAD_MUTEX_ADAPTIVE_NP: |
113 | if (! __is_smp) |
114 | goto simple; |
115 | |
116 | if (lll_trylock (mutex->__data.__lock) != 0) |
117 | { |
118 | int cnt = 0; |
119 | int max_cnt = MIN (MAX_ADAPTIVE_COUNT, |
120 | mutex->__data.__spins * 2 + 10); |
121 | do |
122 | { |
123 | if (cnt++ >= max_cnt) |
124 | { |
125 | result = lll_timedlock (mutex->__data.__lock, abstime, |
126 | PTHREAD_MUTEX_PSHARED (mutex)); |
127 | break; |
128 | } |
129 | atomic_spin_nop (); |
130 | } |
131 | while (lll_trylock (mutex->__data.__lock) != 0); |
132 | |
133 | mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8; |
134 | } |
135 | break; |
136 | |
137 | case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP: |
138 | case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP: |
139 | case PTHREAD_MUTEX_ROBUST_NORMAL_NP: |
140 | case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP: |
141 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, |
142 | &mutex->__data.__list.__next); |
143 | |
144 | oldval = mutex->__data.__lock; |
145 | do |
146 | { |
147 | again: |
148 | if ((oldval & FUTEX_OWNER_DIED) != 0) |
149 | { |
150 | /* The previous owner died. Try locking the mutex. */ |
151 | int newval = id | (oldval & FUTEX_WAITERS); |
152 | |
153 | newval |
154 | = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
155 | newval, oldval); |
156 | if (newval != oldval) |
157 | { |
158 | oldval = newval; |
159 | goto again; |
160 | } |
161 | |
162 | /* We got the mutex. */ |
163 | mutex->__data.__count = 1; |
164 | /* But it is inconsistent unless marked otherwise. */ |
165 | mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; |
166 | |
167 | ENQUEUE_MUTEX (mutex); |
168 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
169 | |
170 | /* Note that we deliberately exit here. If we fall |
171 | through to the end of the function __nusers would be |
172 | incremented which is not correct because the old |
173 | owner has to be discounted. */ |
174 | return EOWNERDEAD; |
175 | } |
176 | |
177 | /* Check whether we already hold the mutex. */ |
178 | if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id)) |
179 | { |
180 | int kind = PTHREAD_MUTEX_TYPE (mutex); |
181 | if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP) |
182 | { |
183 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, |
184 | NULL); |
185 | return EDEADLK; |
186 | } |
187 | |
188 | if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP) |
189 | { |
190 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, |
191 | NULL); |
192 | |
193 | /* Just bump the counter. */ |
194 | if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) |
195 | /* Overflow of the counter. */ |
196 | return EAGAIN; |
197 | |
198 | ++mutex->__data.__count; |
199 | |
200 | LIBC_PROBE (mutex_timedlock_acquired, 1, mutex); |
201 | |
202 | return 0; |
203 | } |
204 | } |
205 | |
206 | result = lll_robust_timedlock (mutex->__data.__lock, abstime, id, |
207 | PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); |
208 | |
209 | if (__builtin_expect (mutex->__data.__owner |
210 | == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) |
211 | { |
212 | /* This mutex is now not recoverable. */ |
213 | mutex->__data.__count = 0; |
214 | lll_unlock (mutex->__data.__lock, |
215 | PTHREAD_ROBUST_MUTEX_PSHARED (mutex)); |
216 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
217 | return ENOTRECOVERABLE; |
218 | } |
219 | |
220 | if (result == ETIMEDOUT || result == EINVAL) |
221 | goto out; |
222 | |
223 | oldval = result; |
224 | } |
225 | while ((oldval & FUTEX_OWNER_DIED) != 0); |
226 | |
227 | mutex->__data.__count = 1; |
228 | ENQUEUE_MUTEX (mutex); |
229 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
230 | break; |
231 | |
232 | /* The PI support requires the Linux futex system call. If that's not |
233 | available, pthread_mutex_init should never have allowed the type to |
234 | be set. So it will get the default case for an invalid type. */ |
235 | #ifdef __NR_futex |
236 | case PTHREAD_MUTEX_PI_RECURSIVE_NP: |
237 | case PTHREAD_MUTEX_PI_ERRORCHECK_NP: |
238 | case PTHREAD_MUTEX_PI_NORMAL_NP: |
239 | case PTHREAD_MUTEX_PI_ADAPTIVE_NP: |
240 | case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP: |
241 | case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP: |
242 | case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP: |
243 | case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP: |
244 | { |
245 | int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP; |
246 | int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP; |
247 | |
248 | if (robust) |
249 | /* Note: robust PI futexes are signaled by setting bit 0. */ |
250 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, |
251 | (void *) (((uintptr_t) &mutex->__data.__list.__next) |
252 | | 1)); |
253 | |
254 | oldval = mutex->__data.__lock; |
255 | |
256 | /* Check whether we already hold the mutex. */ |
257 | if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id)) |
258 | { |
259 | if (kind == PTHREAD_MUTEX_ERRORCHECK_NP) |
260 | { |
261 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
262 | return EDEADLK; |
263 | } |
264 | |
265 | if (kind == PTHREAD_MUTEX_RECURSIVE_NP) |
266 | { |
267 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
268 | |
269 | /* Just bump the counter. */ |
270 | if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) |
271 | /* Overflow of the counter. */ |
272 | return EAGAIN; |
273 | |
274 | ++mutex->__data.__count; |
275 | |
276 | LIBC_PROBE (mutex_timedlock_acquired, 1, mutex); |
277 | |
278 | return 0; |
279 | } |
280 | } |
281 | |
282 | oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
283 | id, 0); |
284 | |
285 | if (oldval != 0) |
286 | { |
287 | /* The mutex is locked. The kernel will now take care of |
288 | everything. The timeout value must be a relative value. |
289 | Convert it. */ |
290 | int private = (robust |
291 | ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex) |
292 | : PTHREAD_MUTEX_PSHARED (mutex)); |
293 | INTERNAL_SYSCALL_DECL (__err); |
294 | |
295 | int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock, |
296 | __lll_private_flag (FUTEX_LOCK_PI, |
297 | private), 1, |
298 | abstime); |
299 | if (INTERNAL_SYSCALL_ERROR_P (e, __err)) |
300 | { |
301 | if (INTERNAL_SYSCALL_ERRNO (e, __err) == ETIMEDOUT) |
302 | return ETIMEDOUT; |
303 | |
304 | if (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH |
305 | || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK) |
306 | { |
307 | assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK |
308 | || (kind != PTHREAD_MUTEX_ERRORCHECK_NP |
309 | && kind != PTHREAD_MUTEX_RECURSIVE_NP)); |
310 | /* ESRCH can happen only for non-robust PI mutexes where |
311 | the owner of the lock died. */ |
312 | assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH |
313 | || !robust); |
314 | |
315 | /* Delay the thread until the timeout is reached. |
316 | Then return ETIMEDOUT. */ |
317 | struct timespec reltime; |
318 | struct timespec now; |
319 | |
320 | INTERNAL_SYSCALL (clock_gettime, __err, 2, CLOCK_REALTIME, |
321 | &now); |
322 | reltime.tv_sec = abstime->tv_sec - now.tv_sec; |
323 | reltime.tv_nsec = abstime->tv_nsec - now.tv_nsec; |
324 | if (reltime.tv_nsec < 0) |
325 | { |
326 | reltime.tv_nsec += 1000000000; |
327 | --reltime.tv_sec; |
328 | } |
329 | if (reltime.tv_sec >= 0) |
330 | while (nanosleep_not_cancel (&reltime, &reltime) != 0) |
331 | continue; |
332 | |
333 | return ETIMEDOUT; |
334 | } |
335 | |
336 | return INTERNAL_SYSCALL_ERRNO (e, __err); |
337 | } |
338 | |
339 | oldval = mutex->__data.__lock; |
340 | |
341 | assert (robust || (oldval & FUTEX_OWNER_DIED) == 0); |
342 | } |
343 | |
344 | if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED)) |
345 | { |
346 | atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED); |
347 | |
348 | /* We got the mutex. */ |
349 | mutex->__data.__count = 1; |
350 | /* But it is inconsistent unless marked otherwise. */ |
351 | mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT; |
352 | |
353 | ENQUEUE_MUTEX_PI (mutex); |
354 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
355 | |
356 | /* Note that we deliberately exit here. If we fall |
357 | through to the end of the function __nusers would be |
358 | incremented which is not correct because the old owner |
359 | has to be discounted. */ |
360 | return EOWNERDEAD; |
361 | } |
362 | |
363 | if (robust |
364 | && __builtin_expect (mutex->__data.__owner |
365 | == PTHREAD_MUTEX_NOTRECOVERABLE, 0)) |
366 | { |
367 | /* This mutex is now not recoverable. */ |
368 | mutex->__data.__count = 0; |
369 | |
370 | INTERNAL_SYSCALL_DECL (__err); |
371 | INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock, |
372 | __lll_private_flag (FUTEX_UNLOCK_PI, |
373 | PTHREAD_ROBUST_MUTEX_PSHARED (mutex)), |
374 | 0, 0); |
375 | |
376 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
377 | return ENOTRECOVERABLE; |
378 | } |
379 | |
380 | mutex->__data.__count = 1; |
381 | if (robust) |
382 | { |
383 | ENQUEUE_MUTEX_PI (mutex); |
384 | THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL); |
385 | } |
386 | } |
387 | break; |
388 | #endif /* __NR_futex. */ |
389 | |
390 | case PTHREAD_MUTEX_PP_RECURSIVE_NP: |
391 | case PTHREAD_MUTEX_PP_ERRORCHECK_NP: |
392 | case PTHREAD_MUTEX_PP_NORMAL_NP: |
393 | case PTHREAD_MUTEX_PP_ADAPTIVE_NP: |
394 | { |
395 | int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP; |
396 | |
397 | oldval = mutex->__data.__lock; |
398 | |
399 | /* Check whether we already hold the mutex. */ |
400 | if (mutex->__data.__owner == id) |
401 | { |
402 | if (kind == PTHREAD_MUTEX_ERRORCHECK_NP) |
403 | return EDEADLK; |
404 | |
405 | if (kind == PTHREAD_MUTEX_RECURSIVE_NP) |
406 | { |
407 | /* Just bump the counter. */ |
408 | if (__glibc_unlikely (mutex->__data.__count + 1 == 0)) |
409 | /* Overflow of the counter. */ |
410 | return EAGAIN; |
411 | |
412 | ++mutex->__data.__count; |
413 | |
414 | LIBC_PROBE (mutex_timedlock_acquired, 1, mutex); |
415 | |
416 | return 0; |
417 | } |
418 | } |
419 | |
420 | int oldprio = -1, ceilval; |
421 | do |
422 | { |
423 | int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) |
424 | >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT; |
425 | |
426 | if (__pthread_current_priority () > ceiling) |
427 | { |
428 | result = EINVAL; |
429 | failpp: |
430 | if (oldprio != -1) |
431 | __pthread_tpp_change_priority (oldprio, -1); |
432 | return result; |
433 | } |
434 | |
435 | result = __pthread_tpp_change_priority (oldprio, ceiling); |
436 | if (result) |
437 | return result; |
438 | |
439 | ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT; |
440 | oldprio = ceiling; |
441 | |
442 | oldval |
443 | = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
444 | ceilval | 1, ceilval); |
445 | |
446 | if (oldval == ceilval) |
447 | break; |
448 | |
449 | do |
450 | { |
451 | oldval |
452 | = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
453 | ceilval | 2, |
454 | ceilval | 1); |
455 | |
456 | if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval) |
457 | break; |
458 | |
459 | if (oldval != ceilval) |
460 | { |
461 | /* Reject invalid timeouts. */ |
462 | if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000) |
463 | { |
464 | result = EINVAL; |
465 | goto failpp; |
466 | } |
467 | |
468 | struct timeval tv; |
469 | struct timespec rt; |
470 | |
471 | /* Get the current time. */ |
472 | (void) __gettimeofday (&tv, NULL); |
473 | |
474 | /* Compute relative timeout. */ |
475 | rt.tv_sec = abstime->tv_sec - tv.tv_sec; |
476 | rt.tv_nsec = abstime->tv_nsec - tv.tv_usec * 1000; |
477 | if (rt.tv_nsec < 0) |
478 | { |
479 | rt.tv_nsec += 1000000000; |
480 | --rt.tv_sec; |
481 | } |
482 | |
483 | /* Already timed out? */ |
484 | if (rt.tv_sec < 0) |
485 | { |
486 | result = ETIMEDOUT; |
487 | goto failpp; |
488 | } |
489 | |
490 | lll_futex_timed_wait (&mutex->__data.__lock, |
491 | ceilval | 2, &rt, |
492 | PTHREAD_MUTEX_PSHARED (mutex)); |
493 | } |
494 | } |
495 | while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock, |
496 | ceilval | 2, ceilval) |
497 | != ceilval); |
498 | } |
499 | while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval); |
500 | |
501 | assert (mutex->__data.__owner == 0); |
502 | mutex->__data.__count = 1; |
503 | } |
504 | break; |
505 | |
506 | default: |
507 | /* Correct code cannot set any other type. */ |
508 | return EINVAL; |
509 | } |
510 | |
511 | if (result == 0) |
512 | { |
513 | /* Record the ownership. */ |
514 | mutex->__data.__owner = id; |
515 | ++mutex->__data.__nusers; |
516 | |
517 | LIBC_PROBE (mutex_timedlock_acquired, 1, mutex); |
518 | } |
519 | |
520 | out: |
521 | return result; |
522 | } |
523 | |