1 | /* Copyright (C) 2003-2016 Free Software Foundation, Inc. |
2 | This file is part of the GNU C Library. |
3 | Contributed by Martin Schwidefsky <schwidefsky@de.ibm.com>, 2003. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ |
18 | |
19 | #include <errno.h> |
20 | #include <sysdep.h> |
21 | #include <lowlevellock.h> |
22 | #include <futex-internal.h> |
23 | #include <pthread.h> |
24 | #include <pthreadP.h> |
25 | #include <stap-probe.h> |
26 | #include <elide.h> |
27 | #include <stdbool.h> |
28 | |
29 | |
30 | /* Acquire read lock for RWLOCK. Slow path. */ |
31 | static int __attribute__((noinline)) |
32 | __pthread_rwlock_rdlock_slow (pthread_rwlock_t *rwlock) |
33 | { |
34 | int result = 0; |
35 | bool wake = false; |
36 | int futex_shared = |
37 | rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; |
38 | |
39 | /* Lock is taken in caller. */ |
40 | |
41 | while (1) |
42 | { |
43 | /* Make sure we are not holding the rwlock as a writer. This is |
44 | a deadlock situation we recognize and report. */ |
45 | if (__builtin_expect (rwlock->__data.__writer |
46 | == THREAD_GETMEM (THREAD_SELF, tid), 0)) |
47 | { |
48 | result = EDEADLK; |
49 | break; |
50 | } |
51 | |
52 | /* Remember that we are a reader. */ |
53 | if (__glibc_unlikely (++rwlock->__data.__nr_readers_queued == 0)) |
54 | { |
55 | /* Overflow on number of queued readers. */ |
56 | --rwlock->__data.__nr_readers_queued; |
57 | result = EAGAIN; |
58 | break; |
59 | } |
60 | |
61 | int waitval = rwlock->__data.__readers_wakeup; |
62 | |
63 | /* Free the lock. */ |
64 | lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); |
65 | |
66 | /* Wait for the writer to finish. We do not check the return value |
67 | because we decide how to continue based on the state of the rwlock. */ |
68 | futex_wait_simple (&rwlock->__data.__readers_wakeup, waitval, |
69 | futex_shared); |
70 | |
71 | /* Get the lock. */ |
72 | lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); |
73 | |
74 | --rwlock->__data.__nr_readers_queued; |
75 | |
76 | /* Get the rwlock if there is no writer... */ |
77 | if (rwlock->__data.__writer == 0 |
78 | /* ...and if either no writer is waiting or we prefer readers. */ |
79 | && (!rwlock->__data.__nr_writers_queued |
80 | || PTHREAD_RWLOCK_PREFER_READER_P (rwlock))) |
81 | { |
82 | /* Increment the reader counter. Avoid overflow. */ |
83 | if (__glibc_unlikely (++rwlock->__data.__nr_readers == 0)) |
84 | { |
85 | /* Overflow on number of readers. */ |
86 | --rwlock->__data.__nr_readers; |
87 | result = EAGAIN; |
88 | } |
89 | else |
90 | { |
91 | LIBC_PROBE (rdlock_acquire_read, 1, rwlock); |
92 | /* See pthread_rwlock_rdlock. */ |
93 | if (rwlock->__data.__nr_readers == 1 |
94 | && rwlock->__data.__nr_readers_queued > 0 |
95 | && rwlock->__data.__nr_writers_queued > 0) |
96 | { |
97 | ++rwlock->__data.__readers_wakeup; |
98 | wake = true; |
99 | } |
100 | } |
101 | |
102 | break; |
103 | } |
104 | } |
105 | |
106 | /* We are done, free the lock. */ |
107 | lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); |
108 | |
109 | if (wake) |
110 | futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); |
111 | |
112 | return result; |
113 | } |
114 | |
115 | |
116 | /* Fast path of acquiring read lock on RWLOCK. */ |
117 | |
118 | int |
119 | __pthread_rwlock_rdlock (pthread_rwlock_t *rwlock) |
120 | { |
121 | int result = 0; |
122 | bool wake = false; |
123 | int futex_shared = |
124 | rwlock->__data.__shared == LLL_PRIVATE ? FUTEX_PRIVATE : FUTEX_SHARED; |
125 | |
126 | LIBC_PROBE (rdlock_entry, 1, rwlock); |
127 | |
128 | if (ELIDE_LOCK (rwlock->__data.__rwelision, |
129 | rwlock->__data.__lock == 0 |
130 | && rwlock->__data.__writer == 0 |
131 | && rwlock->__data.__nr_readers == 0)) |
132 | return 0; |
133 | |
134 | /* Make sure we are alone. */ |
135 | lll_lock (rwlock->__data.__lock, rwlock->__data.__shared); |
136 | |
137 | /* Get the rwlock if there is no writer... */ |
138 | if (rwlock->__data.__writer == 0 |
139 | /* ...and if either no writer is waiting or we prefer readers. */ |
140 | && (!rwlock->__data.__nr_writers_queued |
141 | || PTHREAD_RWLOCK_PREFER_READER_P (rwlock))) |
142 | { |
143 | /* Increment the reader counter. Avoid overflow. */ |
144 | if (__glibc_unlikely (++rwlock->__data.__nr_readers == 0)) |
145 | { |
146 | /* Overflow on number of readers. */ |
147 | --rwlock->__data.__nr_readers; |
148 | result = EAGAIN; |
149 | } |
150 | else |
151 | { |
152 | LIBC_PROBE (rdlock_acquire_read, 1, rwlock); |
153 | /* If we are the first reader, and there are blocked readers and |
154 | writers (which we don't prefer, see above), then it can be the |
155 | case that we stole the lock from a writer that was already woken |
156 | to acquire it. That means that we need to take over the writer's |
157 | responsibility to wake all readers (see pthread_rwlock_unlock). |
158 | Thus, wake all readers in this case. */ |
159 | if (rwlock->__data.__nr_readers == 1 |
160 | && rwlock->__data.__nr_readers_queued > 0 |
161 | && rwlock->__data.__nr_writers_queued > 0) |
162 | { |
163 | ++rwlock->__data.__readers_wakeup; |
164 | wake = true; |
165 | } |
166 | } |
167 | |
168 | /* We are done, free the lock. */ |
169 | lll_unlock (rwlock->__data.__lock, rwlock->__data.__shared); |
170 | |
171 | if (wake) |
172 | futex_wake (&rwlock->__data.__readers_wakeup, INT_MAX, futex_shared); |
173 | |
174 | return result; |
175 | } |
176 | |
177 | return __pthread_rwlock_rdlock_slow (rwlock); |
178 | } |
179 | |
180 | weak_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock) |
181 | hidden_def (__pthread_rwlock_rdlock) |
182 | |