1/* Copyright (C) 2002-2016 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <sysdep.h>
20#include <pthread-errnos.h>
21#include <lowlevellock.h>
22#include <lowlevelrobustlock.h>
23#include <kernel-features.h>
24
25 .text
26
27#define FUTEX_WAITERS 0x80000000
28#define FUTEX_OWNER_DIED 0x40000000
29
30#ifdef __ASSUME_PRIVATE_FUTEX
31# define LOAD_FUTEX_WAIT(reg) \
32 xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
33# define LOAD_FUTEX_WAIT_ABS(reg) \
34 xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
35#else
36# if FUTEX_WAIT == 0
37# define LOAD_FUTEX_WAIT(reg) \
38 xorl $FUTEX_PRIVATE_FLAG, reg ; \
39 andl %fs:PRIVATE_FUTEX, reg
40# else
41# define LOAD_FUTEX_WAIT(reg) \
42 xorl $FUTEX_PRIVATE_FLAG, reg ; \
43 andl %fs:PRIVATE_FUTEX, reg ; \
44 orl $FUTEX_WAIT, reg
45# endif
46# define LOAD_FUTEX_WAIT_ABS(reg) \
47 xorl $FUTEX_PRIVATE_FLAG, reg ; \
48 andl %fs:PRIVATE_FUTEX, reg ; \
49 orl $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg
50#endif
51
52
53 .globl __lll_robust_lock_wait
54 .type __lll_robust_lock_wait,@function
55 .hidden __lll_robust_lock_wait
56 .align 16
57__lll_robust_lock_wait:
58 cfi_startproc
59 pushq %r10
60 cfi_adjust_cfa_offset(8)
61 pushq %rdx
62 cfi_adjust_cfa_offset(8)
63 cfi_offset(%r10, -16)
64 cfi_offset(%rdx, -24)
65
66 xorq %r10, %r10 /* No timeout. */
67 LOAD_FUTEX_WAIT (%esi)
68
694: movl %eax, %edx
70 orl $FUTEX_WAITERS, %edx
71
72 testl $FUTEX_OWNER_DIED, %eax
73 jnz 3f
74
75 cmpl %edx, %eax
76 je 1f
77
78 LOCK
79 cmpxchgl %edx, (%rdi)
80 jnz 2f
81
821: movl $SYS_futex, %eax
83 syscall
84
85 movl (%rdi), %eax
86
872: testl %eax, %eax
88 jne 4b
89
90 movl %fs:TID, %edx
91 orl $FUTEX_WAITERS, %edx
92 LOCK
93 cmpxchgl %edx, (%rdi)
94 jnz 4b
95 /* NB: %rax == 0 */
96
973: popq %rdx
98 cfi_adjust_cfa_offset(-8)
99 cfi_restore(%rdx)
100 popq %r10
101 cfi_adjust_cfa_offset(-8)
102 cfi_restore(%r10)
103 retq
104 cfi_endproc
105 .size __lll_robust_lock_wait,.-__lll_robust_lock_wait
106
107
108 .globl __lll_robust_timedlock_wait
109 .type __lll_robust_timedlock_wait,@function
110 .hidden __lll_robust_timedlock_wait
111 .align 16
112__lll_robust_timedlock_wait:
113 cfi_startproc
114# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
115# ifdef PIC
116 cmpl $0, __have_futex_clock_realtime(%rip)
117# else
118 cmpl $0, __have_futex_clock_realtime
119# endif
120 je .Lreltmo
121# endif
122
123 cmpq $0, (%rdx)
124 js 7f
125
126 pushq %r9
127 cfi_adjust_cfa_offset(8)
128 cfi_rel_offset(%r9, 0)
129 movq %rdx, %r10
130 movl $0xffffffff, %r9d
131 LOAD_FUTEX_WAIT_ABS (%esi)
132
1331: testl $FUTEX_OWNER_DIED, %eax
134 jnz 3f
135
136 movl %eax, %edx
137 orl $FUTEX_WAITERS, %edx
138
139 cmpl %eax, %edx
140 je 5f
141
142 LOCK
143 cmpxchgl %edx, (%rdi)
144 movq $0, %rcx /* Must use mov to avoid changing cc. */
145 jnz 6f
146
1475: movl $SYS_futex, %eax
148 syscall
149 movl %eax, %ecx
150
151 movl (%rdi), %eax
152
1536: testl %eax, %eax
154 jne 2f
155
156 movl %fs:TID, %edx
157 orl $FUTEX_WAITERS, %edx
158 LOCK
159 cmpxchgl %edx, (%rdi)
160 jnz 2f
161
1623: popq %r9
163 cfi_adjust_cfa_offset(-8)
164 cfi_restore(%r9)
165 retq
166
167 cfi_adjust_cfa_offset(8)
168 cfi_rel_offset(%r9, 0)
169 /* Check whether the time expired. */
1702: cmpl $-ETIMEDOUT, %ecx
171 je 4f
172 cmpl $-EINVAL, %ecx
173 jne 1b
174
1754: movl %ecx, %eax
176 negl %eax
177 jmp 3b
178 cfi_adjust_cfa_offset(-8)
179 cfi_restore(%r9)
180
1817: movl $ETIMEDOUT, %eax
182 retq
183
184
185# ifndef __ASSUME_FUTEX_CLOCK_REALTIME
186.Lreltmo:
187 /* Check for a valid timeout value. */
188 cmpq $1000000000, 8(%rdx)
189 jae 3f
190
191 pushq %r8
192 cfi_adjust_cfa_offset(8)
193 pushq %r9
194 cfi_adjust_cfa_offset(8)
195 pushq %r12
196 cfi_adjust_cfa_offset(8)
197 pushq %r13
198 cfi_adjust_cfa_offset(8)
199 cfi_offset(%r8, -16)
200 cfi_offset(%r9, -24)
201 cfi_offset(%r12, -32)
202 cfi_offset(%r13, -40)
203 pushq %rsi
204 cfi_adjust_cfa_offset(8)
205
206 /* Stack frame for the timespec and timeval structs. */
207 subq $32, %rsp
208 cfi_adjust_cfa_offset(32)
209
210 movq %rdi, %r12
211 movq %rdx, %r13
212
2131: movq %rax, 16(%rsp)
214
215 /* Get current time. */
216 movq %rsp, %rdi
217 xorl %esi, %esi
218 /* This call works because we directly jump to a system call entry
219 which preserves all the registers. */
220 call JUMPTARGET(__gettimeofday)
221
222 /* Compute relative timeout. */
223 movq 8(%rsp), %rax
224 movl $1000, %edi
225 mul %rdi /* Milli seconds to nano seconds. */
226 movq (%r13), %rdi
227 movq 8(%r13), %rsi
228 subq (%rsp), %rdi
229 subq %rax, %rsi
230 jns 4f
231 addq $1000000000, %rsi
232 decq %rdi
2334: testq %rdi, %rdi
234 js 8f /* Time is already up. */
235
236 /* Futex call. */
237 movq %rdi, (%rsp) /* Store relative timeout. */
238 movq %rsi, 8(%rsp)
239
240 movq 16(%rsp), %rdx
241 movl %edx, %eax
242 orl $FUTEX_WAITERS, %edx
243
244 testl $FUTEX_OWNER_DIED, %eax
245 jnz 6f
246
247 cmpl %eax, %edx
248 je 2f
249
250 LOCK
251 cmpxchgl %edx, (%r12)
252 movq $0, %rcx /* Must use mov to avoid changing cc. */
253 jnz 5f
254
2552: movq %rsp, %r10
256 movl 32(%rsp), %esi
257 LOAD_FUTEX_WAIT (%esi)
258 movq %r12, %rdi
259 movl $SYS_futex, %eax
260 syscall
261 movq %rax, %rcx
262
263 movl (%r12), %eax
264
2655: testl %eax, %eax
266 jne 7f
267
268 movl %fs:TID, %edx
269 orl $FUTEX_WAITERS, %edx
270 LOCK
271 cmpxchgl %edx, (%r12)
272 jnz 7f
273
2746: addq $40, %rsp
275 cfi_adjust_cfa_offset(-40)
276 popq %r13
277 cfi_adjust_cfa_offset(-8)
278 cfi_restore(%r13)
279 popq %r12
280 cfi_adjust_cfa_offset(-8)
281 cfi_restore(%r12)
282 popq %r9
283 cfi_adjust_cfa_offset(-8)
284 cfi_restore(%r9)
285 popq %r8
286 cfi_adjust_cfa_offset(-8)
287 cfi_restore(%r8)
288 retq
289
2903: movl $EINVAL, %eax
291 retq
292
293 cfi_adjust_cfa_offset(72)
294 cfi_offset(%r8, -16)
295 cfi_offset(%r9, -24)
296 cfi_offset(%r12, -32)
297 cfi_offset(%r13, -40)
298 /* Check whether the time expired. */
2997: cmpl $-ETIMEDOUT, %ecx
300 jne 1b
301
3028: movl $ETIMEDOUT, %eax
303 jmp 6b
304#endif
305 cfi_endproc
306 .size __lll_robust_timedlock_wait,.-__lll_robust_timedlock_wait
307