1/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <stdlib.h>
22#include "pthreadP.h"
23#include <lowlevellock.h>
24#include <stap-probe.h>
25
26#ifndef lll_unlock_elision
27#define lll_unlock_elision(a,b,c) ({ lll_unlock (a,c); 0; })
28#endif
29
30static int
31internal_function
32__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
33 __attribute_noinline__;
34
35int
36internal_function attribute_hidden
37__pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
38{
39 int type = PTHREAD_MUTEX_TYPE_ELISION (mutex);
40 if (__builtin_expect (type &
41 ~(PTHREAD_MUTEX_KIND_MASK_NP|PTHREAD_MUTEX_ELISION_FLAGS_NP), 0))
42 return __pthread_mutex_unlock_full (mutex, decr);
43
44 if (__builtin_expect (type, PTHREAD_MUTEX_TIMED_NP)
45 == PTHREAD_MUTEX_TIMED_NP)
46 {
47 /* Always reset the owner field. */
48 normal:
49 mutex->__data.__owner = 0;
50 if (decr)
51 /* One less user. */
52 --mutex->__data.__nusers;
53
54 /* Unlock. */
55 lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));
56
57 LIBC_PROBE (mutex_release, 1, mutex);
58
59 return 0;
60 }
61 else if (__glibc_likely (type == PTHREAD_MUTEX_TIMED_ELISION_NP))
62 {
63 /* Don't reset the owner/users fields for elision. */
64 return lll_unlock_elision (mutex->__data.__lock, mutex->__data.__elision,
65 PTHREAD_MUTEX_PSHARED (mutex));
66 }
67 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
68 == PTHREAD_MUTEX_RECURSIVE_NP, 1))
69 {
70 /* Recursive mutex. */
71 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
72 return EPERM;
73
74 if (--mutex->__data.__count != 0)
75 /* We still hold the mutex. */
76 return 0;
77 goto normal;
78 }
79 else if (__builtin_expect (PTHREAD_MUTEX_TYPE (mutex)
80 == PTHREAD_MUTEX_ADAPTIVE_NP, 1))
81 goto normal;
82 else
83 {
84 /* Error checking mutex. */
85 assert (type == PTHREAD_MUTEX_ERRORCHECK_NP);
86 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
87 || ! lll_islocked (mutex->__data.__lock))
88 return EPERM;
89 goto normal;
90 }
91}
92
93
94static int
95internal_function
96__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
97{
98 int newowner = 0;
99
100 switch (PTHREAD_MUTEX_TYPE (mutex))
101 {
102 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
103 /* Recursive mutex. */
104 if ((mutex->__data.__lock & FUTEX_TID_MASK)
105 == THREAD_GETMEM (THREAD_SELF, tid)
106 && __builtin_expect (mutex->__data.__owner
107 == PTHREAD_MUTEX_INCONSISTENT, 0))
108 {
109 if (--mutex->__data.__count != 0)
110 /* We still hold the mutex. */
111 return ENOTRECOVERABLE;
112
113 goto notrecoverable;
114 }
115
116 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
117 return EPERM;
118
119 if (--mutex->__data.__count != 0)
120 /* We still hold the mutex. */
121 return 0;
122
123 goto robust;
124
125 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
126 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
127 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
128 if ((mutex->__data.__lock & FUTEX_TID_MASK)
129 != THREAD_GETMEM (THREAD_SELF, tid)
130 || ! lll_islocked (mutex->__data.__lock))
131 return EPERM;
132
133 /* If the previous owner died and the caller did not succeed in
134 making the state consistent, mark the mutex as unrecoverable
135 and make all waiters. */
136 if (__builtin_expect (mutex->__data.__owner
137 == PTHREAD_MUTEX_INCONSISTENT, 0))
138 notrecoverable:
139 newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
140
141 robust:
142 /* Remove mutex from the list. */
143 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
144 &mutex->__data.__list.__next);
145 DEQUEUE_MUTEX (mutex);
146
147 mutex->__data.__owner = newowner;
148 if (decr)
149 /* One less user. */
150 --mutex->__data.__nusers;
151
152 /* Unlock. */
153 lll_robust_unlock (mutex->__data.__lock,
154 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
155
156 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
157 break;
158
159 /* The PI support requires the Linux futex system call. If that's not
160 available, pthread_mutex_init should never have allowed the type to
161 be set. So it will get the default case for an invalid type. */
162#ifdef __NR_futex
163 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
164 /* Recursive mutex. */
165 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
166 return EPERM;
167
168 if (--mutex->__data.__count != 0)
169 /* We still hold the mutex. */
170 return 0;
171 goto continue_pi_non_robust;
172
173 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
174 /* Recursive mutex. */
175 if ((mutex->__data.__lock & FUTEX_TID_MASK)
176 == THREAD_GETMEM (THREAD_SELF, tid)
177 && __builtin_expect (mutex->__data.__owner
178 == PTHREAD_MUTEX_INCONSISTENT, 0))
179 {
180 if (--mutex->__data.__count != 0)
181 /* We still hold the mutex. */
182 return ENOTRECOVERABLE;
183
184 goto pi_notrecoverable;
185 }
186
187 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
188 return EPERM;
189
190 if (--mutex->__data.__count != 0)
191 /* We still hold the mutex. */
192 return 0;
193
194 goto continue_pi_robust;
195
196 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
197 case PTHREAD_MUTEX_PI_NORMAL_NP:
198 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
199 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
200 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
201 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
202 if ((mutex->__data.__lock & FUTEX_TID_MASK)
203 != THREAD_GETMEM (THREAD_SELF, tid)
204 || ! lll_islocked (mutex->__data.__lock))
205 return EPERM;
206
207 /* If the previous owner died and the caller did not succeed in
208 making the state consistent, mark the mutex as unrecoverable
209 and make all waiters. */
210 if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0
211 && __builtin_expect (mutex->__data.__owner
212 == PTHREAD_MUTEX_INCONSISTENT, 0))
213 pi_notrecoverable:
214 newowner = PTHREAD_MUTEX_NOTRECOVERABLE;
215
216 if ((mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP) != 0)
217 {
218 continue_pi_robust:
219 /* Remove mutex from the list.
220 Note: robust PI futexes are signaled by setting bit 0. */
221 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
222 (void *) (((uintptr_t) &mutex->__data.__list.__next)
223 | 1));
224 DEQUEUE_MUTEX (mutex);
225 }
226
227 continue_pi_non_robust:
228 mutex->__data.__owner = newowner;
229 if (decr)
230 /* One less user. */
231 --mutex->__data.__nusers;
232
233 /* Unlock. Load all necessary mutex data before releasing the mutex
234 to not violate the mutex destruction requirements (see
235 lll_unlock). */
236 int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
237 int private = (robust
238 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
239 : PTHREAD_MUTEX_PSHARED (mutex));
240 /* Unlock the mutex using a CAS unless there are futex waiters or our
241 TID is not the value of __lock anymore, in which case we let the
242 kernel take care of the situation. Use release MO in the CAS to
243 synchronize with acquire MO in lock acquisitions. */
244 int l = atomic_load_relaxed (&mutex->__data.__lock);
245 do
246 {
247 if (((l & FUTEX_WAITERS) != 0)
248 || (l != THREAD_GETMEM (THREAD_SELF, tid)))
249 {
250 INTERNAL_SYSCALL_DECL (__err);
251 INTERNAL_SYSCALL (futex, __err, 2, &mutex->__data.__lock,
252 __lll_private_flag (FUTEX_UNLOCK_PI, private));
253 break;
254 }
255 }
256 while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
257 &l, 0));
258
259 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
260 break;
261#endif /* __NR_futex. */
262
263 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
264 /* Recursive mutex. */
265 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid))
266 return EPERM;
267
268 if (--mutex->__data.__count != 0)
269 /* We still hold the mutex. */
270 return 0;
271 goto pp;
272
273 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
274 /* Error checking mutex. */
275 if (mutex->__data.__owner != THREAD_GETMEM (THREAD_SELF, tid)
276 || (mutex->__data.__lock & ~ PTHREAD_MUTEX_PRIO_CEILING_MASK) == 0)
277 return EPERM;
278 /* FALLTHROUGH */
279
280 case PTHREAD_MUTEX_PP_NORMAL_NP:
281 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
282 /* Always reset the owner field. */
283 pp:
284 mutex->__data.__owner = 0;
285
286 if (decr)
287 /* One less user. */
288 --mutex->__data.__nusers;
289
290 /* Unlock. Use release MO in the CAS to synchronize with acquire MO in
291 lock acquisitions. */
292 int newval;
293 int oldval = atomic_load_relaxed (&mutex->__data.__lock);
294 do
295 {
296 newval = oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK;
297 }
298 while (!atomic_compare_exchange_weak_release (&mutex->__data.__lock,
299 &oldval, newval));
300
301 if ((oldval & ~PTHREAD_MUTEX_PRIO_CEILING_MASK) > 1)
302 lll_futex_wake (&mutex->__data.__lock, 1,
303 PTHREAD_MUTEX_PSHARED (mutex));
304
305 int oldprio = newval >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
306
307 LIBC_PROBE (mutex_release, 1, mutex);
308
309 return __pthread_tpp_change_priority (oldprio, -1);
310
311 default:
312 /* Correct code cannot set any other type. */
313 return EINVAL;
314 }
315
316 LIBC_PROBE (mutex_release, 1, mutex);
317 return 0;
318}
319
320
321int
322__pthread_mutex_unlock (pthread_mutex_t *mutex)
323{
324 return __pthread_mutex_unlock_usercnt (mutex, 1);
325}
326strong_alias (__pthread_mutex_unlock, pthread_mutex_unlock)
327hidden_def (__pthread_mutex_unlock)
328