1/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <sysdep.h>
20#include <pthread-errnos.h>
21#include <kernel-features.h>
22#include <lowlevellock.h>
23
24#include <stap-probe.h>
25
26 .text
27
28#ifdef __ASSUME_PRIVATE_FUTEX
29# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
30 movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
31# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
32 movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
33# define LOAD_FUTEX_WAIT(reg) \
34 xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
35# define LOAD_FUTEX_WAIT_ABS(reg) \
36 xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
37# define LOAD_FUTEX_WAKE(reg) \
38 xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
39#else
40# if FUTEX_WAIT == 0
41# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
42 movl %fs:PRIVATE_FUTEX, reg
43# else
44# define LOAD_PRIVATE_FUTEX_WAIT(reg) \
45 movl %fs:PRIVATE_FUTEX, reg ; \
46 orl $FUTEX_WAIT, reg
47# endif
48# define LOAD_PRIVATE_FUTEX_WAKE(reg) \
49 movl %fs:PRIVATE_FUTEX, reg ; \
50 orl $FUTEX_WAKE, reg
51# if FUTEX_WAIT == 0
52# define LOAD_FUTEX_WAIT(reg) \
53 xorl $FUTEX_PRIVATE_FLAG, reg ; \
54 andl %fs:PRIVATE_FUTEX, reg
55# else
56# define LOAD_FUTEX_WAIT(reg) \
57 xorl $FUTEX_PRIVATE_FLAG, reg ; \
58 andl %fs:PRIVATE_FUTEX, reg ; \
59 orl $FUTEX_WAIT, reg
60# endif
61# define LOAD_FUTEX_WAIT_ABS(reg) \
62 xorl $FUTEX_PRIVATE_FLAG, reg ; \
63 andl %fs:PRIVATE_FUTEX, reg ; \
64 orl $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg
65# define LOAD_FUTEX_WAKE(reg) \
66 xorl $FUTEX_PRIVATE_FLAG, reg ; \
67 andl %fs:PRIVATE_FUTEX, reg ; \
68 orl $FUTEX_WAKE, reg
69#endif
70
71
72 .globl __lll_lock_wait_private
73 .type __lll_lock_wait_private,@function
74 .hidden __lll_lock_wait_private
75 .align 16
76__lll_lock_wait_private:
77 cfi_startproc
78 pushq %r10
79 cfi_adjust_cfa_offset(8)
80 pushq %rdx
81 cfi_adjust_cfa_offset(8)
82 cfi_offset(%r10, -16)
83 cfi_offset(%rdx, -24)
84 xorq %r10, %r10 /* No timeout. */
85 movl $2, %edx
86 LOAD_PRIVATE_FUTEX_WAIT (%esi)
87
88 cmpl %edx, %eax /* NB: %edx == 2 */
89 jne 2f
90
911: LIBC_PROBE (lll_lock_wait_private, 1, %rdi)
92 movl $SYS_futex, %eax
93 syscall
94
952: movl %edx, %eax
96 xchgl %eax, (%rdi) /* NB: lock is implied */
97
98 testl %eax, %eax
99 jnz 1b
100
101 popq %rdx
102 cfi_adjust_cfa_offset(-8)
103 cfi_restore(%rdx)
104 popq %r10
105 cfi_adjust_cfa_offset(-8)
106 cfi_restore(%r10)
107 retq
108 cfi_endproc
109 .size __lll_lock_wait_private,.-__lll_lock_wait_private
110
111#if !IS_IN (libc)
112 .globl __lll_lock_wait
113 .type __lll_lock_wait,@function
114 .hidden __lll_lock_wait
115 .align 16
116__lll_lock_wait:
117 cfi_startproc
118 pushq %r10
119 cfi_adjust_cfa_offset(8)
120 pushq %rdx
121 cfi_adjust_cfa_offset(8)
122 cfi_offset(%r10, -16)
123 cfi_offset(%rdx, -24)
124 xorq %r10, %r10 /* No timeout. */
125 movl $2, %edx
126 LOAD_FUTEX_WAIT (%esi)
127
128 cmpl %edx, %eax /* NB: %edx == 2 */
129 jne 2f
130
1311: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi)
132 movl $SYS_futex, %eax
133 syscall
134
1352: movl %edx, %eax
136 xchgl %eax, (%rdi) /* NB: lock is implied */
137
138 testl %eax, %eax
139 jnz 1b
140
141 popq %rdx
142 cfi_adjust_cfa_offset(-8)
143 cfi_restore(%rdx)
144 popq %r10
145 cfi_adjust_cfa_offset(-8)
146 cfi_restore(%r10)
147 retq
148 cfi_endproc
149 .size __lll_lock_wait,.-__lll_lock_wait
150
151 /* %rdi: futex
152 %rsi: flags
153 %rdx: timeout
154 %eax: futex value
155 */
156 .globl __lll_timedlock_wait
157 .type __lll_timedlock_wait,@function
158 .hidden __lll_timedlock_wait
159 .align 16
160__lll_timedlock_wait:
161 cfi_startproc
162# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
163# ifdef PIC
164 cmpl $0, __have_futex_clock_realtime(%rip)
165# else
166 cmpl $0, __have_futex_clock_realtime
167# endif
168 je .Lreltmo
169# endif
170
171 cmpq $0, (%rdx)
172 js 5f
173
174 pushq %r9
175 cfi_adjust_cfa_offset(8)
176 cfi_rel_offset(%r9, 0)
177
178 movq %rdx, %r10
179 movl $0xffffffff, %r9d
180 LOAD_FUTEX_WAIT_ABS (%esi)
181
182 movl $2, %edx
183 cmpl %edx, %eax
184 jne 2f
185
1861: movl $SYS_futex, %eax
187 movl $2, %edx
188 syscall
189
1902: xchgl %edx, (%rdi) /* NB: lock is implied */
191
192 testl %edx, %edx
193 jz 3f
194
195 cmpl $-ETIMEDOUT, %eax
196 je 4f
197 cmpl $-EINVAL, %eax
198 jne 1b
1994: movl %eax, %edx
200 negl %edx
201
2023: movl %edx, %eax
203 popq %r9
204 cfi_adjust_cfa_offset(-8)
205 cfi_restore(%r9)
206 retq
207
2085: movl $ETIMEDOUT, %eax
209 retq
210
211# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
212.Lreltmo:
213 /* Check for a valid timeout value. */
214 cmpq $1000000000, 8(%rdx)
215 jae 3f
216
217 pushq %r8
218 cfi_adjust_cfa_offset(8)
219 pushq %r9
220 cfi_adjust_cfa_offset(8)
221 pushq %r12
222 cfi_adjust_cfa_offset(8)
223 pushq %r13
224 cfi_adjust_cfa_offset(8)
225 pushq %r14
226 cfi_adjust_cfa_offset(8)
227 cfi_offset(%r8, -16)
228 cfi_offset(%r9, -24)
229 cfi_offset(%r12, -32)
230 cfi_offset(%r13, -40)
231 cfi_offset(%r14, -48)
232 pushq %rsi
233 cfi_adjust_cfa_offset(8)
234
235 /* Stack frame for the timespec and timeval structs. */
236 subq $24, %rsp
237 cfi_adjust_cfa_offset(24)
238
239 movq %rdi, %r12
240 movq %rdx, %r13
241
242 movl $2, %edx
243 xchgl %edx, (%r12)
244
245 testl %edx, %edx
246 je 6f
247
2481:
249 /* Get current time. */
250 movq %rsp, %rdi
251 xorl %esi, %esi
252 /* This call works because we directly jump to a system call entry
253 which preserves all the registers. */
254 call JUMPTARGET(__gettimeofday)
255
256 /* Compute relative timeout. */
257 movq 8(%rsp), %rax
258 movl $1000, %edi
259 mul %rdi /* Milli seconds to nano seconds. */
260 movq (%r13), %rdi
261 movq 8(%r13), %rsi
262 subq (%rsp), %rdi
263 subq %rax, %rsi
264 jns 4f
265 addq $1000000000, %rsi
266 decq %rdi
2674: testq %rdi, %rdi
268 js 2f /* Time is already up. */
269
270 /* Store relative timeout. */
271 movq %rdi, (%rsp)
272 movq %rsi, 8(%rsp)
273
274 /* Futex call. */
275 movl $2, %edx
276 movl $1, %eax
277 movq %rsp, %r10
278 movl 24(%rsp), %esi
279 LOAD_FUTEX_WAIT (%esi)
280 movq %r12, %rdi
281 movl $SYS_futex, %eax
282 syscall
283
284 /* NB: %edx == 2 */
285 xchgl %edx, (%r12)
286
287 testl %edx, %edx
288 je 6f
289
290 cmpl $-ETIMEDOUT, %eax
291 jne 1b
2922: movl $ETIMEDOUT, %edx
293
2946: addq $32, %rsp
295 cfi_adjust_cfa_offset(-32)
296 popq %r14
297 cfi_adjust_cfa_offset(-8)
298 cfi_restore(%r14)
299 popq %r13
300 cfi_adjust_cfa_offset(-8)
301 cfi_restore(%r13)
302 popq %r12
303 cfi_adjust_cfa_offset(-8)
304 cfi_restore(%r12)
305 popq %r9
306 cfi_adjust_cfa_offset(-8)
307 cfi_restore(%r9)
308 popq %r8
309 cfi_adjust_cfa_offset(-8)
310 cfi_restore(%r8)
311 movl %edx, %eax
312 retq
313
3143: movl $EINVAL, %eax
315 retq
316# endif
317 cfi_endproc
318 .size __lll_timedlock_wait,.-__lll_timedlock_wait
319#endif
320
321
322 .globl __lll_unlock_wake_private
323 .type __lll_unlock_wake_private,@function
324 .hidden __lll_unlock_wake_private
325 .align 16
326__lll_unlock_wake_private:
327 cfi_startproc
328 pushq %rsi
329 cfi_adjust_cfa_offset(8)
330 pushq %rdx
331 cfi_adjust_cfa_offset(8)
332 cfi_offset(%rsi, -16)
333 cfi_offset(%rdx, -24)
334
335 movl $0, (%rdi)
336 LOAD_PRIVATE_FUTEX_WAKE (%esi)
337 movl $1, %edx /* Wake one thread. */
338 movl $SYS_futex, %eax
339 syscall
340
341 popq %rdx
342 cfi_adjust_cfa_offset(-8)
343 cfi_restore(%rdx)
344 popq %rsi
345 cfi_adjust_cfa_offset(-8)
346 cfi_restore(%rsi)
347 retq
348 cfi_endproc
349 .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
350
351#if !IS_IN (libc)
352 .globl __lll_unlock_wake
353 .type __lll_unlock_wake,@function
354 .hidden __lll_unlock_wake
355 .align 16
356__lll_unlock_wake:
357 cfi_startproc
358 pushq %rsi
359 cfi_adjust_cfa_offset(8)
360 pushq %rdx
361 cfi_adjust_cfa_offset(8)
362 cfi_offset(%rsi, -16)
363 cfi_offset(%rdx, -24)
364
365 movl $0, (%rdi)
366 LOAD_FUTEX_WAKE (%esi)
367 movl $1, %edx /* Wake one thread. */
368 movl $SYS_futex, %eax
369 syscall
370
371 popq %rdx
372 cfi_adjust_cfa_offset(-8)
373 cfi_restore(%rdx)
374 popq %rsi
375 cfi_adjust_cfa_offset(-8)
376 cfi_restore(%rsi)
377 retq
378 cfi_endproc
379 .size __lll_unlock_wake,.-__lll_unlock_wake
380
381 .globl __lll_timedwait_tid
382 .type __lll_timedwait_tid,@function
383 .hidden __lll_timedwait_tid
384 .align 16
385__lll_timedwait_tid:
386 cfi_startproc
387 pushq %r12
388 cfi_adjust_cfa_offset(8)
389 pushq %r13
390 cfi_adjust_cfa_offset(8)
391 cfi_offset(%r12, -16)
392 cfi_offset(%r13, -24)
393
394 movq %rdi, %r12
395 movq %rsi, %r13
396
397 /* Align stack to 16 bytes when calling __gettimeofday. */
398 subq $24, %rsp
399 cfi_adjust_cfa_offset(24)
400
401 /* Get current time. */
4022: movq %rsp, %rdi
403 xorl %esi, %esi
404 /* This call works because we directly jump to a system call entry
405 which preserves all the registers. */
406 call JUMPTARGET(__gettimeofday)
407
408 /* Compute relative timeout. */
409 movq 8(%rsp), %rax
410 movl $1000, %edi
411 mul %rdi /* Milli seconds to nano seconds. */
412 movq (%r13), %rdi
413 movq 8(%r13), %rsi
414 subq (%rsp), %rdi
415 subq %rax, %rsi
416 jns 5f
417 addq $1000000000, %rsi
418 decq %rdi
4195: testq %rdi, %rdi
420 js 6f /* Time is already up. */
421
422 movq %rdi, (%rsp) /* Store relative timeout. */
423 movq %rsi, 8(%rsp)
424
425 movl (%r12), %edx
426 testl %edx, %edx
427 jz 4f
428
429 movq %rsp, %r10
430 /* XXX The kernel so far uses global futex for the wakeup at
431 all times. */
432#if FUTEX_WAIT == 0
433 xorl %esi, %esi
434#else
435 movl $FUTEX_WAIT, %esi
436#endif
437 movq %r12, %rdi
438 movl $SYS_futex, %eax
439 syscall
440
441 cmpl $0, (%rdi)
442 jne 1f
4434: xorl %eax, %eax
444
4458: addq $24, %rsp
446 cfi_adjust_cfa_offset(-24)
447 popq %r13
448 cfi_adjust_cfa_offset(-8)
449 cfi_restore(%r13)
450 popq %r12
451 cfi_adjust_cfa_offset(-8)
452 cfi_restore(%r12)
453 retq
454
455 cfi_adjust_cfa_offset(32)
4561: cmpq $-ETIMEDOUT, %rax
457 jne 2b
458
4596: movl $ETIMEDOUT, %eax
460 jmp 8b
461 cfi_endproc
462 .size __lll_timedwait_tid,.-__lll_timedwait_tid
463#endif
464