1/* Copyright (C) 2002-2018 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <stdlib.h>
22#include <unistd.h>
23#include <sys/param.h>
24#include <not-cancel.h>
25#include "pthreadP.h"
26#include <atomic.h>
27#include <lowlevellock.h>
28#include <stap-probe.h>
29
30#ifndef lll_lock_elision
31#define lll_lock_elision(lock, try_lock, private) ({ \
32 lll_lock (lock, private); 0; })
33#endif
34
35#ifndef lll_trylock_elision
36#define lll_trylock_elision(a,t) lll_trylock(a)
37#endif
38
39/* Some of the following definitions differ when pthread_mutex_cond_lock.c
40 includes this file. */
41#ifndef LLL_MUTEX_LOCK
42# define LLL_MUTEX_LOCK(mutex) \
43 lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
44# define LLL_MUTEX_TRYLOCK(mutex) \
45 lll_trylock ((mutex)->__data.__lock)
46# define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0
47# define LLL_MUTEX_LOCK_ELISION(mutex) \
48 lll_lock_elision ((mutex)->__data.__lock, (mutex)->__data.__elision, \
49 PTHREAD_MUTEX_PSHARED (mutex))
50# define LLL_MUTEX_TRYLOCK_ELISION(mutex) \
51 lll_trylock_elision((mutex)->__data.__lock, (mutex)->__data.__elision, \
52 PTHREAD_MUTEX_PSHARED (mutex))
53#endif
54
55#ifndef FORCE_ELISION
56#define FORCE_ELISION(m, s)
57#endif
58
59static int __pthread_mutex_lock_full (pthread_mutex_t *mutex)
60 __attribute_noinline__;
61
62int
63__pthread_mutex_lock (pthread_mutex_t *mutex)
64{
65 /* See concurrency notes regarding mutex type which is loaded from __kind
66 in struct __pthread_mutex_s in sysdeps/nptl/bits/thread-shared-types.h. */
67 unsigned int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
68
69 LIBC_PROBE (mutex_entry, 1, mutex);
70
71 if (__builtin_expect (type & ~(PTHREAD_MUTEX_KIND_MASK_NP
72 | PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
73 return __pthread_mutex_lock_full (mutex);
74
75 if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_NP))
76 {
77 FORCE_ELISION (mutex, goto elision);
78 simple:
79 /* Normal mutex. */
80 LLL_MUTEX_LOCK (mutex);
81 assert (mutex->__data.__owner == 0);
82 }
83#ifdef HAVE_ELISION
84 else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
85 {
86 elision: __attribute__((unused))
87 /* This case can never happen on a system without elision,
88 as the mutex type initialization functions will not
89 allow to set the elision flags. */
90 /* Don't record owner or users for elision case. This is a
91 tail call. */
92 return LLL_MUTEX_LOCK_ELISION (mutex);
93 }
94#endif
95 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
96 == PTHREAD_MUTEX_RECURSIVE_NP, 1))
97 {
98 /* Recursive mutex. */
99 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
100
101 /* Check whether we already hold the mutex. */
102 if (mutex->__data.__owner == id)
103 {
104 /* Just bump the counter. */
105 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
106 /* Overflow of the counter. */
107 return EAGAIN;
108
109 ++mutex->__data.__count;
110
111 return 0;
112 }
113
114 /* We have to get the mutex. */
115 LLL_MUTEX_LOCK (mutex);
116
117 assert (mutex->__data.__owner == 0);
118 mutex->__data.__count = 1;
119 }
120 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
121 == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
122 {
123 if (! __is_smp)
124 goto simple;
125
126 if (LLL_MUTEX_TRYLOCK (mutex) != 0)
127 {
128 int cnt = 0;
129 int max_cnt = MIN (MAX_ADAPTIVE_COUNT,
130 mutex->__data.__spins * 2 + 10);
131 do
132 {
133 if (cnt++ >= max_cnt)
134 {
135 LLL_MUTEX_LOCK (mutex);
136 break;
137 }
138 atomic_spin_nop ();
139 }
140 while (LLL_MUTEX_TRYLOCK (mutex) != 0);
141
142 mutex->__data.__spins += (cnt - mutex->__data.__spins) / 8;
143 }
144 assert (mutex->__data.__owner == 0);
145 }
146 else
147 {
148 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
149 assert (PTHREAD_MUTEX_TYPE (mutex) == PTHREAD_MUTEX_ERRORCHECK_NP);
150 /* Check whether we already hold the mutex. */
151 if (__glibc_unlikely (mutex->__data.__owner == id))
152 return EDEADLK;
153 goto simple;
154 }
155
156 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
157
158 /* Record the ownership. */
159 mutex->__data.__owner = id;
160#ifndef NO_INCR
161 ++mutex->__data.__nusers;
162#endif
163
164 LIBC_PROBE (mutex_acquired, 1, mutex);
165
166 return 0;
167}
168
169static int
170__pthread_mutex_lock_full (pthread_mutex_t *mutex)
171{
172 int oldval;
173 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
174
175 switch (PTHREAD_MUTEX_TYPE (mutex))
176 {
177 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
178 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
179 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
180 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
181 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
182 &mutex->__data.__list.__next);
183 /* We need to set op_pending before starting the operation. Also
184 see comments at ENQUEUE_MUTEX. */
185 __asm ("" ::: "memory");
186
187 oldval = mutex->__data.__lock;
188 /* This is set to FUTEX_WAITERS iff we might have shared the
189 FUTEX_WAITERS flag with other threads, and therefore need to keep it
190 set to avoid lost wake-ups. We have the same requirement in the
191 simple mutex algorithm.
192 We start with value zero for a normal mutex, and FUTEX_WAITERS if we
193 are building the special case mutexes for use from within condition
194 variables. */
195 unsigned int assume_other_futex_waiters = LLL_ROBUST_MUTEX_LOCK_MODIFIER;
196 while (1)
197 {
198 /* Try to acquire the lock through a CAS from 0 (not acquired) to
199 our TID | assume_other_futex_waiters. */
200 if (__glibc_likely (oldval == 0))
201 {
202 oldval
203 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
204 id | assume_other_futex_waiters, 0);
205 if (__glibc_likely (oldval == 0))
206 break;
207 }
208
209 if ((oldval & FUTEX_OWNER_DIED) != 0)
210 {
211 /* The previous owner died. Try locking the mutex. */
212 int newval = id;
213#ifdef NO_INCR
214 /* We are not taking assume_other_futex_waiters into accoount
215 here simply because we'll set FUTEX_WAITERS anyway. */
216 newval |= FUTEX_WAITERS;
217#else
218 newval |= (oldval & FUTEX_WAITERS) | assume_other_futex_waiters;
219#endif
220
221 newval
222 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
223 newval, oldval);
224
225 if (newval != oldval)
226 {
227 oldval = newval;
228 continue;
229 }
230
231 /* We got the mutex. */
232 mutex->__data.__count = 1;
233 /* But it is inconsistent unless marked otherwise. */
234 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
235
236 /* We must not enqueue the mutex before we have acquired it.
237 Also see comments at ENQUEUE_MUTEX. */
238 __asm ("" ::: "memory");
239 ENQUEUE_MUTEX (mutex);
240 /* We need to clear op_pending after we enqueue the mutex. */
241 __asm ("" ::: "memory");
242 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
243
244 /* Note that we deliberately exit here. If we fall
245 through to the end of the function __nusers would be
246 incremented which is not correct because the old
247 owner has to be discounted. If we are not supposed
248 to increment __nusers we actually have to decrement
249 it here. */
250#ifdef NO_INCR
251 --mutex->__data.__nusers;
252#endif
253
254 return EOWNERDEAD;
255 }
256
257 /* Check whether we already hold the mutex. */
258 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
259 {
260 int kind = PTHREAD_MUTEX_TYPE (mutex);
261 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
262 {
263 /* We do not need to ensure ordering wrt another memory
264 access. Also see comments at ENQUEUE_MUTEX. */
265 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
266 NULL);
267 return EDEADLK;
268 }
269
270 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
271 {
272 /* We do not need to ensure ordering wrt another memory
273 access. */
274 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
275 NULL);
276
277 /* Just bump the counter. */
278 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
279 /* Overflow of the counter. */
280 return EAGAIN;
281
282 ++mutex->__data.__count;
283
284 return 0;
285 }
286 }
287
288 /* We cannot acquire the mutex nor has its owner died. Thus, try
289 to block using futexes. Set FUTEX_WAITERS if necessary so that
290 other threads are aware that there are potentially threads
291 blocked on the futex. Restart if oldval changed in the
292 meantime. */
293 if ((oldval & FUTEX_WAITERS) == 0)
294 {
295 if (atomic_compare_and_exchange_bool_acq (&mutex->__data.__lock,
296 oldval | FUTEX_WAITERS,
297 oldval)
298 != 0)
299 {
300 oldval = mutex->__data.__lock;
301 continue;
302 }
303 oldval |= FUTEX_WAITERS;
304 }
305
306 /* It is now possible that we share the FUTEX_WAITERS flag with
307 another thread; therefore, update assume_other_futex_waiters so
308 that we do not forget about this when handling other cases
309 above and thus do not cause lost wake-ups. */
310 assume_other_futex_waiters |= FUTEX_WAITERS;
311
312 /* Block using the futex and reload current lock value. */
313 lll_futex_wait (&mutex->__data.__lock, oldval,
314 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
315 oldval = mutex->__data.__lock;
316 }
317
318 /* We have acquired the mutex; check if it is still consistent. */
319 if (__builtin_expect (mutex->__data.__owner
320 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
321 {
322 /* This mutex is now not recoverable. */
323 mutex->__data.__count = 0;
324 int private = PTHREAD_ROBUST_MUTEX_PSHARED (mutex);
325 lll_unlock (mutex->__data.__lock, private);
326 /* FIXME This violates the mutex destruction requirements. See
327 __pthread_mutex_unlock_full. */
328 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
329 return ENOTRECOVERABLE;
330 }
331
332 mutex->__data.__count = 1;
333 /* We must not enqueue the mutex before we have acquired it.
334 Also see comments at ENQUEUE_MUTEX. */
335 __asm ("" ::: "memory");
336 ENQUEUE_MUTEX (mutex);
337 /* We need to clear op_pending after we enqueue the mutex. */
338 __asm ("" ::: "memory");
339 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
340 break;
341
342 /* The PI support requires the Linux futex system call. If that's not
343 available, pthread_mutex_init should never have allowed the type to
344 be set. So it will get the default case for an invalid type. */
345#ifdef __NR_futex
346 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
347 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
348 case PTHREAD_MUTEX_PI_NORMAL_NP:
349 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
350 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
351 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
352 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
353 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
354 {
355 int kind, robust;
356 {
357 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
358 in sysdeps/nptl/bits/thread-shared-types.h. */
359 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
360 kind = mutex_kind & PTHREAD_MUTEX_KIND_MASK_NP;
361 robust = mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
362 }
363
364 if (robust)
365 {
366 /* Note: robust PI futexes are signaled by setting bit 0. */
367 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
368 (void *) (((uintptr_t) &mutex->__data.__list.__next)
369 | 1));
370 /* We need to set op_pending before starting the operation. Also
371 see comments at ENQUEUE_MUTEX. */
372 __asm ("" ::: "memory");
373 }
374
375 oldval = mutex->__data.__lock;
376
377 /* Check whether we already hold the mutex. */
378 if (__glibc_unlikely ((oldval & FUTEX_TID_MASK) == id))
379 {
380 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
381 {
382 /* We do not need to ensure ordering wrt another memory
383 access. */
384 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
385 return EDEADLK;
386 }
387
388 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
389 {
390 /* We do not need to ensure ordering wrt another memory
391 access. */
392 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
393
394 /* Just bump the counter. */
395 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
396 /* Overflow of the counter. */
397 return EAGAIN;
398
399 ++mutex->__data.__count;
400
401 return 0;
402 }
403 }
404
405 int newval = id;
406# ifdef NO_INCR
407 newval |= FUTEX_WAITERS;
408# endif
409 oldval = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
410 newval, 0);
411
412 if (oldval != 0)
413 {
414 /* The mutex is locked. The kernel will now take care of
415 everything. */
416 int private = (robust
417 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
418 : PTHREAD_MUTEX_PSHARED (mutex));
419 INTERNAL_SYSCALL_DECL (__err);
420 int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
421 __lll_private_flag (FUTEX_LOCK_PI,
422 private), 1, 0);
423
424 if (INTERNAL_SYSCALL_ERROR_P (e, __err)
425 && (INTERNAL_SYSCALL_ERRNO (e, __err) == ESRCH
426 || INTERNAL_SYSCALL_ERRNO (e, __err) == EDEADLK))
427 {
428 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != EDEADLK
429 || (kind != PTHREAD_MUTEX_ERRORCHECK_NP
430 && kind != PTHREAD_MUTEX_RECURSIVE_NP));
431 /* ESRCH can happen only for non-robust PI mutexes where
432 the owner of the lock died. */
433 assert (INTERNAL_SYSCALL_ERRNO (e, __err) != ESRCH || !robust);
434
435 /* Delay the thread indefinitely. */
436 while (1)
437 __pause_nocancel ();
438 }
439
440 oldval = mutex->__data.__lock;
441
442 assert (robust || (oldval & FUTEX_OWNER_DIED) == 0);
443 }
444
445 if (__glibc_unlikely (oldval & FUTEX_OWNER_DIED))
446 {
447 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
448
449 /* We got the mutex. */
450 mutex->__data.__count = 1;
451 /* But it is inconsistent unless marked otherwise. */
452 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
453
454 /* We must not enqueue the mutex before we have acquired it.
455 Also see comments at ENQUEUE_MUTEX. */
456 __asm ("" ::: "memory");
457 ENQUEUE_MUTEX_PI (mutex);
458 /* We need to clear op_pending after we enqueue the mutex. */
459 __asm ("" ::: "memory");
460 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
461
462 /* Note that we deliberately exit here. If we fall
463 through to the end of the function __nusers would be
464 incremented which is not correct because the old owner
465 has to be discounted. If we are not supposed to
466 increment __nusers we actually have to decrement it here. */
467# ifdef NO_INCR
468 --mutex->__data.__nusers;
469# endif
470
471 return EOWNERDEAD;
472 }
473
474 if (robust
475 && __builtin_expect (mutex->__data.__owner
476 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
477 {
478 /* This mutex is now not recoverable. */
479 mutex->__data.__count = 0;
480
481 INTERNAL_SYSCALL_DECL (__err);
482 INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
483 __lll_private_flag (FUTEX_UNLOCK_PI,
484 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
485 0, 0);
486
487 /* To the kernel, this will be visible after the kernel has
488 acquired the mutex in the syscall. */
489 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
490 return ENOTRECOVERABLE;
491 }
492
493 mutex->__data.__count = 1;
494 if (robust)
495 {
496 /* We must not enqueue the mutex before we have acquired it.
497 Also see comments at ENQUEUE_MUTEX. */
498 __asm ("" ::: "memory");
499 ENQUEUE_MUTEX_PI (mutex);
500 /* We need to clear op_pending after we enqueue the mutex. */
501 __asm ("" ::: "memory");
502 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
503 }
504 }
505 break;
506#endif /* __NR_futex. */
507
508 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
509 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
510 case PTHREAD_MUTEX_PP_NORMAL_NP:
511 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
512 {
513 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
514 in sysdeps/nptl/bits/thread-shared-types.h. */
515 int kind = atomic_load_relaxed (&(mutex->__data.__kind))
516 & PTHREAD_MUTEX_KIND_MASK_NP;
517
518 oldval = mutex->__data.__lock;
519
520 /* Check whether we already hold the mutex. */
521 if (mutex->__data.__owner == id)
522 {
523 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
524 return EDEADLK;
525
526 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
527 {
528 /* Just bump the counter. */
529 if (__glibc_unlikely (mutex->__data.__count + 1 == 0))
530 /* Overflow of the counter. */
531 return EAGAIN;
532
533 ++mutex->__data.__count;
534
535 return 0;
536 }
537 }
538
539 int oldprio = -1, ceilval;
540 do
541 {
542 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
543 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
544
545 if (__pthread_current_priority () > ceiling)
546 {
547 if (oldprio != -1)
548 __pthread_tpp_change_priority (oldprio, -1);
549 return EINVAL;
550 }
551
552 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
553 if (retval)
554 return retval;
555
556 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
557 oldprio = ceiling;
558
559 oldval
560 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
561#ifdef NO_INCR
562 ceilval | 2,
563#else
564 ceilval | 1,
565#endif
566 ceilval);
567
568 if (oldval == ceilval)
569 break;
570
571 do
572 {
573 oldval
574 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
575 ceilval | 2,
576 ceilval | 1);
577
578 if ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval)
579 break;
580
581 if (oldval != ceilval)
582 lll_futex_wait (&mutex->__data.__lock, ceilval | 2,
583 PTHREAD_MUTEX_PSHARED (mutex));
584 }
585 while (atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
586 ceilval | 2, ceilval)
587 != ceilval);
588 }
589 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
590
591 assert (mutex->__data.__owner == 0);
592 mutex->__data.__count = 1;
593 }
594 break;
595
596 default:
597 /* Correct code cannot set any other type. */
598 return EINVAL;
599 }
600
601 /* Record the ownership. */
602 mutex->__data.__owner = id;
603#ifndef NO_INCR
604 ++mutex->__data.__nusers;
605#endif
606
607 LIBC_PROBE (mutex_acquired, 1, mutex);
608
609 return 0;
610}
611#ifndef __pthread_mutex_lock
612weak_alias (__pthread_mutex_lock, pthread_mutex_lock)
613hidden_def (__pthread_mutex_lock)
614#endif
615
616
617#ifdef NO_INCR
618void
619__pthread_mutex_cond_lock_adjust (pthread_mutex_t *mutex)
620{
621 /* See concurrency notes regarding __kind in struct __pthread_mutex_s
622 in sysdeps/nptl/bits/thread-shared-types.h. */
623 int mutex_kind = atomic_load_relaxed (&(mutex->__data.__kind));
624 assert ((mutex_kind & PTHREAD_MUTEX_PRIO_INHERIT_NP) != 0);
625 assert ((mutex_kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) == 0);
626 assert ((mutex_kind & PTHREAD_MUTEX_PSHARED_BIT) == 0);
627
628 /* Record the ownership. */
629 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
630 mutex->__data.__owner = id;
631
632 if (mutex_kind == PTHREAD_MUTEX_PI_RECURSIVE_NP)
633 ++mutex->__data.__count;
634}
635#endif
636