1 | /* Copyright (C) 2002-2017 Free Software Foundation, Inc. |
2 | This file is part of the GNU C Library. |
3 | Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ |
18 | |
19 | #ifndef _LOWLEVELLOCK_H |
20 | #define _LOWLEVELLOCK_H 1 |
21 | |
22 | #include <stap-probe.h> |
23 | |
24 | #ifndef __ASSEMBLER__ |
25 | # include <time.h> |
26 | # include <sys/param.h> |
27 | # include <bits/pthreadtypes.h> |
28 | # include <kernel-features.h> |
29 | # include <tcb-offsets.h> |
30 | |
31 | # ifndef LOCK_INSTR |
32 | # ifdef UP |
33 | # define LOCK_INSTR /* nothing */ |
34 | # else |
35 | # define LOCK_INSTR "lock;" |
36 | # endif |
37 | # endif |
38 | #else |
39 | # ifndef LOCK |
40 | # ifdef UP |
41 | # define LOCK |
42 | # else |
43 | # define LOCK lock |
44 | # endif |
45 | # endif |
46 | #endif |
47 | |
48 | #include <lowlevellock-futex.h> |
49 | |
50 | /* XXX Remove when no assembler code uses futexes anymore. */ |
51 | #define SYS_futex __NR_futex |
52 | |
53 | #ifndef __ASSEMBLER__ |
54 | |
55 | /* Initializer for lock. */ |
56 | #define LLL_LOCK_INITIALIZER (0) |
57 | #define LLL_LOCK_INITIALIZER_LOCKED (1) |
58 | #define LLL_LOCK_INITIALIZER_WAITERS (2) |
59 | |
60 | |
61 | /* NB: in the lll_trylock macro we simply return the value in %eax |
62 | after the cmpxchg instruction. In case the operation succeded this |
63 | value is zero. In case the operation failed, the cmpxchg instruction |
64 | has loaded the current value of the memory work which is guaranteed |
65 | to be nonzero. */ |
66 | #if !IS_IN (libc) || defined UP |
67 | # define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1" |
68 | #else |
69 | # define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ |
70 | "je 0f\n\t" \ |
71 | "lock; cmpxchgl %2, %1\n\t" \ |
72 | "jmp 1f\n\t" \ |
73 | "0:\tcmpxchgl %2, %1\n\t" \ |
74 | "1:" |
75 | #endif |
76 | |
77 | #define lll_trylock(futex) \ |
78 | ({ int ret; \ |
79 | __asm __volatile (__lll_trylock_asm \ |
80 | : "=a" (ret), "=m" (futex) \ |
81 | : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \ |
82 | "0" (LLL_LOCK_INITIALIZER) \ |
83 | : "memory"); \ |
84 | ret; }) |
85 | |
86 | #define lll_cond_trylock(futex) \ |
87 | ({ int ret; \ |
88 | __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ |
89 | : "=a" (ret), "=m" (futex) \ |
90 | : "r" (LLL_LOCK_INITIALIZER_WAITERS), \ |
91 | "m" (futex), "0" (LLL_LOCK_INITIALIZER) \ |
92 | : "memory"); \ |
93 | ret; }) |
94 | |
95 | #if !IS_IN (libc) || defined UP |
96 | # define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t" \ |
97 | "jz 24f\n\t" |
98 | #else |
99 | # define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ |
100 | "je 0f\n\t" \ |
101 | "lock; cmpxchgl %4, %2\n\t" \ |
102 | "jnz 1f\n\t" \ |
103 | "jmp 24f\n" \ |
104 | "0:\tcmpxchgl %4, %2\n\t" \ |
105 | "jz 24f\n\t" |
106 | #endif |
107 | |
108 | #define lll_lock(futex, private) \ |
109 | (void) \ |
110 | ({ int ignore1, ignore2, ignore3; \ |
111 | if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ |
112 | __asm __volatile (__lll_lock_asm_start \ |
113 | "1:\tlea %2, %%" RDI_LP "\n" \ |
114 | "2:\tsub $128, %%" RSP_LP "\n" \ |
115 | ".cfi_adjust_cfa_offset 128\n" \ |
116 | "3:\tcallq __lll_lock_wait_private\n" \ |
117 | "4:\tadd $128, %%" RSP_LP "\n" \ |
118 | ".cfi_adjust_cfa_offset -128\n" \ |
119 | "24:" \ |
120 | : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \ |
121 | "=a" (ignore3) \ |
122 | : "0" (1), "m" (futex), "3" (0) \ |
123 | : "cx", "r11", "cc", "memory"); \ |
124 | else \ |
125 | __asm __volatile (__lll_lock_asm_start \ |
126 | "1:\tlea %2, %%" RDI_LP "\n" \ |
127 | "2:\tsub $128, %%" RSP_LP "\n" \ |
128 | ".cfi_adjust_cfa_offset 128\n" \ |
129 | "3:\tcallq __lll_lock_wait\n" \ |
130 | "4:\tadd $128, %%" RSP_LP "\n" \ |
131 | ".cfi_adjust_cfa_offset -128\n" \ |
132 | "24:" \ |
133 | : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ |
134 | "=a" (ignore3) \ |
135 | : "1" (1), "m" (futex), "3" (0), "0" (private) \ |
136 | : "cx", "r11", "cc", "memory"); \ |
137 | }) \ |
138 | |
139 | #define lll_cond_lock(futex, private) \ |
140 | (void) \ |
141 | ({ int ignore1, ignore2, ignore3; \ |
142 | __asm __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t" \ |
143 | "jz 24f\n" \ |
144 | "1:\tlea %2, %%" RDI_LP "\n" \ |
145 | "2:\tsub $128, %%" RSP_LP "\n" \ |
146 | ".cfi_adjust_cfa_offset 128\n" \ |
147 | "3:\tcallq __lll_lock_wait\n" \ |
148 | "4:\tadd $128, %%" RSP_LP "\n" \ |
149 | ".cfi_adjust_cfa_offset -128\n" \ |
150 | "24:" \ |
151 | : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ |
152 | "=a" (ignore3) \ |
153 | : "1" (2), "m" (futex), "3" (0), "0" (private) \ |
154 | : "cx", "r11", "cc", "memory"); \ |
155 | }) |
156 | |
157 | #define lll_timedlock(futex, timeout, private) \ |
158 | ({ int result, ignore1, ignore2, ignore3; \ |
159 | __asm __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t" \ |
160 | "jz 24f\n" \ |
161 | "1:\tlea %4, %%" RDI_LP "\n" \ |
162 | "0:\tmov %8, %%" RDX_LP "\n" \ |
163 | "2:\tsub $128, %%" RSP_LP "\n" \ |
164 | ".cfi_adjust_cfa_offset 128\n" \ |
165 | "3:\tcallq __lll_timedlock_wait\n" \ |
166 | "4:\tadd $128, %%" RSP_LP "\n" \ |
167 | ".cfi_adjust_cfa_offset -128\n" \ |
168 | "24:" \ |
169 | : "=a" (result), "=D" (ignore1), "=S" (ignore2), \ |
170 | "=&d" (ignore3), "=m" (futex) \ |
171 | : "0" (0), "1" (1), "m" (futex), "m" (timeout), \ |
172 | "2" (private) \ |
173 | : "memory", "cx", "cc", "r10", "r11"); \ |
174 | result; }) |
175 | |
176 | extern int __lll_timedlock_elision (int *futex, short *adapt_count, |
177 | const struct timespec *timeout, |
178 | int private) attribute_hidden; |
179 | |
180 | #define lll_timedlock_elision(futex, adapt_count, timeout, private) \ |
181 | __lll_timedlock_elision(&(futex), &(adapt_count), timeout, private) |
182 | |
183 | #if !IS_IN (libc) || defined UP |
184 | # define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t" \ |
185 | "je 24f\n\t" |
186 | #else |
187 | # define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ |
188 | "je 0f\n\t" \ |
189 | "lock; decl %0\n\t" \ |
190 | "jne 1f\n\t" \ |
191 | "jmp 24f\n\t" \ |
192 | "0:\tdecl %0\n\t" \ |
193 | "je 24f\n\t" |
194 | #endif |
195 | |
196 | #define lll_unlock(futex, private) \ |
197 | (void) \ |
198 | ({ int ignore; \ |
199 | if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ |
200 | __asm __volatile (__lll_unlock_asm_start \ |
201 | "1:\tlea %0, %%" RDI_LP "\n" \ |
202 | "2:\tsub $128, %%" RSP_LP "\n" \ |
203 | ".cfi_adjust_cfa_offset 128\n" \ |
204 | "3:\tcallq __lll_unlock_wake_private\n" \ |
205 | "4:\tadd $128, %%" RSP_LP "\n" \ |
206 | ".cfi_adjust_cfa_offset -128\n" \ |
207 | "24:" \ |
208 | : "=m" (futex), "=&D" (ignore) \ |
209 | : "m" (futex) \ |
210 | : "ax", "cx", "r11", "cc", "memory"); \ |
211 | else \ |
212 | __asm __volatile (__lll_unlock_asm_start \ |
213 | "1:\tlea %0, %%" RDI_LP "\n" \ |
214 | "2:\tsub $128, %%" RSP_LP "\n" \ |
215 | ".cfi_adjust_cfa_offset 128\n" \ |
216 | "3:\tcallq __lll_unlock_wake\n" \ |
217 | "4:\tadd $128, %%" RSP_LP "\n" \ |
218 | ".cfi_adjust_cfa_offset -128\n" \ |
219 | "24:" \ |
220 | : "=m" (futex), "=&D" (ignore) \ |
221 | : "m" (futex), "S" (private) \ |
222 | : "ax", "cx", "r11", "cc", "memory"); \ |
223 | }) |
224 | |
225 | #define lll_islocked(futex) \ |
226 | (futex != LLL_LOCK_INITIALIZER) |
227 | |
228 | |
229 | /* The kernel notifies a process which uses CLONE_CHILD_CLEARTID via futex |
230 | wake-up when the clone terminates. The memory location contains the |
231 | thread ID while the clone is running and is reset to zero by the kernel |
232 | afterwards. The kernel up to version 3.16.3 does not use the private futex |
233 | operations for futex wake-up when the clone terminates. */ |
234 | #define lll_wait_tid(tid) \ |
235 | do { \ |
236 | __typeof (tid) __tid; \ |
237 | while ((__tid = (tid)) != 0) \ |
238 | lll_futex_wait (&(tid), __tid, LLL_SHARED);\ |
239 | } while (0) |
240 | |
241 | extern int __lll_timedwait_tid (int *, const struct timespec *) |
242 | attribute_hidden; |
243 | |
244 | /* As lll_wait_tid, but with a timeout. If the timeout occurs then return |
245 | ETIMEDOUT. If ABSTIME is invalid, return EINVAL. |
246 | XXX Note that this differs from the generic version in that we do the |
247 | error checking here and not in __lll_timedwait_tid. */ |
248 | #define lll_timedwait_tid(tid, abstime) \ |
249 | ({ \ |
250 | int __result = 0; \ |
251 | if ((tid) != 0) \ |
252 | { \ |
253 | if ((abstime)->tv_nsec < 0 || (abstime)->tv_nsec >= 1000000000) \ |
254 | __result = EINVAL; \ |
255 | else \ |
256 | __result = __lll_timedwait_tid (&(tid), (abstime)); \ |
257 | } \ |
258 | __result; }) |
259 | |
260 | extern int __lll_lock_elision (int *futex, short *adapt_count, int private) |
261 | attribute_hidden; |
262 | |
263 | extern int __lll_unlock_elision (int *lock, int private) |
264 | attribute_hidden; |
265 | |
266 | extern int __lll_trylock_elision (int *lock, short *adapt_count) |
267 | attribute_hidden; |
268 | |
269 | #define lll_lock_elision(futex, adapt_count, private) \ |
270 | __lll_lock_elision (&(futex), &(adapt_count), private) |
271 | #define lll_unlock_elision(futex, adapt_count, private) \ |
272 | __lll_unlock_elision (&(futex), private) |
273 | #define lll_trylock_elision(futex, adapt_count) \ |
274 | __lll_trylock_elision (&(futex), &(adapt_count)) |
275 | |
276 | #endif /* !__ASSEMBLER__ */ |
277 | |
278 | #endif /* lowlevellock.h */ |
279 | |