1 | /* Internal macros for atomic operations for GNU C Library. |
2 | Copyright (C) 2002-2016 Free Software Foundation, Inc. |
3 | This file is part of the GNU C Library. |
4 | Contributed by Ulrich Drepper <drepper@redhat.com>, 2002. |
5 | |
6 | The GNU C Library is free software; you can redistribute it and/or |
7 | modify it under the terms of the GNU Lesser General Public |
8 | License as published by the Free Software Foundation; either |
9 | version 2.1 of the License, or (at your option) any later version. |
10 | |
11 | The GNU C Library is distributed in the hope that it will be useful, |
12 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
14 | Lesser General Public License for more details. |
15 | |
16 | You should have received a copy of the GNU Lesser General Public |
17 | License along with the GNU C Library; if not, see |
18 | <http://www.gnu.org/licenses/>. */ |
19 | |
20 | #ifndef _ATOMIC_H |
21 | #define _ATOMIC_H 1 |
22 | |
23 | /* This header defines three types of macros: |
24 | |
25 | - atomic arithmetic and logic operation on memory. They all |
26 | have the prefix "atomic_". |
27 | |
28 | - conditionally atomic operations of the same kinds. These |
29 | always behave identical but can be faster when atomicity |
30 | is not really needed since only one thread has access to |
31 | the memory location. In that case the code is slower in |
32 | the multi-thread case. The interfaces have the prefix |
33 | "catomic_". |
34 | |
35 | - support functions like barriers. They also have the prefix |
36 | "atomic_". |
37 | |
38 | Architectures must provide a few lowlevel macros (the compare |
39 | and exchange definitions). All others are optional. They |
40 | should only be provided if the architecture has specific |
41 | support for the operation. |
42 | |
43 | As <atomic.h> macros are usually heavily nested and often use local |
44 | variables to make sure side-effects are evaluated properly, use for |
45 | macro local variables a per-macro unique prefix. This file uses |
46 | __atgN_ prefix where N is different in each macro. */ |
47 | |
48 | #include <stdlib.h> |
49 | |
50 | #include <atomic-machine.h> |
51 | |
52 | /* Wrapper macros to call pre_NN_post (mem, ...) where NN is the |
53 | bit width of *MEM. The calling macro puts parens around MEM |
54 | and following args. */ |
55 | #define __atomic_val_bysize(pre, post, mem, ...) \ |
56 | ({ \ |
57 | __typeof (*mem) __atg1_result; \ |
58 | if (sizeof (*mem) == 1) \ |
59 | __atg1_result = pre##_8_##post (mem, __VA_ARGS__); \ |
60 | else if (sizeof (*mem) == 2) \ |
61 | __atg1_result = pre##_16_##post (mem, __VA_ARGS__); \ |
62 | else if (sizeof (*mem) == 4) \ |
63 | __atg1_result = pre##_32_##post (mem, __VA_ARGS__); \ |
64 | else if (sizeof (*mem) == 8) \ |
65 | __atg1_result = pre##_64_##post (mem, __VA_ARGS__); \ |
66 | else \ |
67 | abort (); \ |
68 | __atg1_result; \ |
69 | }) |
70 | #define __atomic_bool_bysize(pre, post, mem, ...) \ |
71 | ({ \ |
72 | int __atg2_result; \ |
73 | if (sizeof (*mem) == 1) \ |
74 | __atg2_result = pre##_8_##post (mem, __VA_ARGS__); \ |
75 | else if (sizeof (*mem) == 2) \ |
76 | __atg2_result = pre##_16_##post (mem, __VA_ARGS__); \ |
77 | else if (sizeof (*mem) == 4) \ |
78 | __atg2_result = pre##_32_##post (mem, __VA_ARGS__); \ |
79 | else if (sizeof (*mem) == 8) \ |
80 | __atg2_result = pre##_64_##post (mem, __VA_ARGS__); \ |
81 | else \ |
82 | abort (); \ |
83 | __atg2_result; \ |
84 | }) |
85 | |
86 | |
87 | /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. |
88 | Return the old *MEM value. */ |
89 | #if !defined atomic_compare_and_exchange_val_acq \ |
90 | && defined __arch_compare_and_exchange_val_32_acq |
91 | # define atomic_compare_and_exchange_val_acq(mem, newval, oldval) \ |
92 | __atomic_val_bysize (__arch_compare_and_exchange_val,acq, \ |
93 | mem, newval, oldval) |
94 | #endif |
95 | |
96 | |
97 | #ifndef catomic_compare_and_exchange_val_acq |
98 | # ifdef __arch_c_compare_and_exchange_val_32_acq |
99 | # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \ |
100 | __atomic_val_bysize (__arch_c_compare_and_exchange_val,acq, \ |
101 | mem, newval, oldval) |
102 | # else |
103 | # define catomic_compare_and_exchange_val_acq(mem, newval, oldval) \ |
104 | atomic_compare_and_exchange_val_acq (mem, newval, oldval) |
105 | # endif |
106 | #endif |
107 | |
108 | |
109 | #ifndef catomic_compare_and_exchange_val_rel |
110 | # ifndef atomic_compare_and_exchange_val_rel |
111 | # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \ |
112 | catomic_compare_and_exchange_val_acq (mem, newval, oldval) |
113 | # else |
114 | # define catomic_compare_and_exchange_val_rel(mem, newval, oldval) \ |
115 | atomic_compare_and_exchange_val_rel (mem, newval, oldval) |
116 | # endif |
117 | #endif |
118 | |
119 | |
120 | #ifndef atomic_compare_and_exchange_val_rel |
121 | # define atomic_compare_and_exchange_val_rel(mem, newval, oldval) \ |
122 | atomic_compare_and_exchange_val_acq (mem, newval, oldval) |
123 | #endif |
124 | |
125 | |
126 | /* Atomically store NEWVAL in *MEM if *MEM is equal to OLDVAL. |
127 | Return zero if *MEM was changed or non-zero if no exchange happened. */ |
128 | #ifndef atomic_compare_and_exchange_bool_acq |
129 | # ifdef __arch_compare_and_exchange_bool_32_acq |
130 | # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ |
131 | __atomic_bool_bysize (__arch_compare_and_exchange_bool,acq, \ |
132 | mem, newval, oldval) |
133 | # else |
134 | # define atomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ |
135 | ({ /* Cannot use __oldval here, because macros later in this file might \ |
136 | call this macro with __oldval argument. */ \ |
137 | __typeof (oldval) __atg3_old = (oldval); \ |
138 | atomic_compare_and_exchange_val_acq (mem, newval, __atg3_old) \ |
139 | != __atg3_old; \ |
140 | }) |
141 | # endif |
142 | #endif |
143 | |
144 | |
145 | #ifndef catomic_compare_and_exchange_bool_acq |
146 | # ifdef __arch_c_compare_and_exchange_bool_32_acq |
147 | # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ |
148 | __atomic_bool_bysize (__arch_c_compare_and_exchange_bool,acq, \ |
149 | mem, newval, oldval) |
150 | # else |
151 | # define catomic_compare_and_exchange_bool_acq(mem, newval, oldval) \ |
152 | ({ /* Cannot use __oldval here, because macros later in this file might \ |
153 | call this macro with __oldval argument. */ \ |
154 | __typeof (oldval) __atg4_old = (oldval); \ |
155 | catomic_compare_and_exchange_val_acq (mem, newval, __atg4_old) \ |
156 | != __atg4_old; \ |
157 | }) |
158 | # endif |
159 | #endif |
160 | |
161 | |
162 | #ifndef catomic_compare_and_exchange_bool_rel |
163 | # ifndef atomic_compare_and_exchange_bool_rel |
164 | # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \ |
165 | catomic_compare_and_exchange_bool_acq (mem, newval, oldval) |
166 | # else |
167 | # define catomic_compare_and_exchange_bool_rel(mem, newval, oldval) \ |
168 | atomic_compare_and_exchange_bool_rel (mem, newval, oldval) |
169 | # endif |
170 | #endif |
171 | |
172 | |
173 | #ifndef atomic_compare_and_exchange_bool_rel |
174 | # define atomic_compare_and_exchange_bool_rel(mem, newval, oldval) \ |
175 | atomic_compare_and_exchange_bool_acq (mem, newval, oldval) |
176 | #endif |
177 | |
178 | |
179 | /* Store NEWVALUE in *MEM and return the old value. */ |
180 | #ifndef atomic_exchange_acq |
181 | # define atomic_exchange_acq(mem, newvalue) \ |
182 | ({ __typeof (*(mem)) __atg5_oldval; \ |
183 | __typeof (mem) __atg5_memp = (mem); \ |
184 | __typeof (*(mem)) __atg5_value = (newvalue); \ |
185 | \ |
186 | do \ |
187 | __atg5_oldval = *__atg5_memp; \ |
188 | while (__builtin_expect \ |
189 | (atomic_compare_and_exchange_bool_acq (__atg5_memp, __atg5_value, \ |
190 | __atg5_oldval), 0)); \ |
191 | \ |
192 | __atg5_oldval; }) |
193 | #endif |
194 | |
195 | #ifndef atomic_exchange_rel |
196 | # define atomic_exchange_rel(mem, newvalue) atomic_exchange_acq (mem, newvalue) |
197 | #endif |
198 | |
199 | |
200 | /* Add VALUE to *MEM and return the old value of *MEM. */ |
201 | #ifndef atomic_exchange_and_add_acq |
202 | # ifdef atomic_exchange_and_add |
203 | # define atomic_exchange_and_add_acq(mem, value) \ |
204 | atomic_exchange_and_add (mem, value) |
205 | # else |
206 | # define atomic_exchange_and_add_acq(mem, value) \ |
207 | ({ __typeof (*(mem)) __atg6_oldval; \ |
208 | __typeof (mem) __atg6_memp = (mem); \ |
209 | __typeof (*(mem)) __atg6_value = (value); \ |
210 | \ |
211 | do \ |
212 | __atg6_oldval = *__atg6_memp; \ |
213 | while (__builtin_expect \ |
214 | (atomic_compare_and_exchange_bool_acq (__atg6_memp, \ |
215 | __atg6_oldval \ |
216 | + __atg6_value, \ |
217 | __atg6_oldval), 0)); \ |
218 | \ |
219 | __atg6_oldval; }) |
220 | # endif |
221 | #endif |
222 | |
223 | #ifndef atomic_exchange_and_add_rel |
224 | # define atomic_exchange_and_add_rel(mem, value) \ |
225 | atomic_exchange_and_add_acq(mem, value) |
226 | #endif |
227 | |
228 | #ifndef atomic_exchange_and_add |
229 | # define atomic_exchange_and_add(mem, value) \ |
230 | atomic_exchange_and_add_acq(mem, value) |
231 | #endif |
232 | |
233 | #ifndef catomic_exchange_and_add |
234 | # define catomic_exchange_and_add(mem, value) \ |
235 | ({ __typeof (*(mem)) __atg7_oldv; \ |
236 | __typeof (mem) __atg7_memp = (mem); \ |
237 | __typeof (*(mem)) __atg7_value = (value); \ |
238 | \ |
239 | do \ |
240 | __atg7_oldv = *__atg7_memp; \ |
241 | while (__builtin_expect \ |
242 | (catomic_compare_and_exchange_bool_acq (__atg7_memp, \ |
243 | __atg7_oldv \ |
244 | + __atg7_value, \ |
245 | __atg7_oldv), 0)); \ |
246 | \ |
247 | __atg7_oldv; }) |
248 | #endif |
249 | |
250 | |
251 | #ifndef atomic_max |
252 | # define atomic_max(mem, value) \ |
253 | do { \ |
254 | __typeof (*(mem)) __atg8_oldval; \ |
255 | __typeof (mem) __atg8_memp = (mem); \ |
256 | __typeof (*(mem)) __atg8_value = (value); \ |
257 | do { \ |
258 | __atg8_oldval = *__atg8_memp; \ |
259 | if (__atg8_oldval >= __atg8_value) \ |
260 | break; \ |
261 | } while (__builtin_expect \ |
262 | (atomic_compare_and_exchange_bool_acq (__atg8_memp, __atg8_value,\ |
263 | __atg8_oldval), 0)); \ |
264 | } while (0) |
265 | #endif |
266 | |
267 | |
268 | #ifndef catomic_max |
269 | # define catomic_max(mem, value) \ |
270 | do { \ |
271 | __typeof (*(mem)) __atg9_oldv; \ |
272 | __typeof (mem) __atg9_memp = (mem); \ |
273 | __typeof (*(mem)) __atg9_value = (value); \ |
274 | do { \ |
275 | __atg9_oldv = *__atg9_memp; \ |
276 | if (__atg9_oldv >= __atg9_value) \ |
277 | break; \ |
278 | } while (__builtin_expect \ |
279 | (catomic_compare_and_exchange_bool_acq (__atg9_memp, \ |
280 | __atg9_value, \ |
281 | __atg9_oldv), 0)); \ |
282 | } while (0) |
283 | #endif |
284 | |
285 | |
286 | #ifndef atomic_min |
287 | # define atomic_min(mem, value) \ |
288 | do { \ |
289 | __typeof (*(mem)) __atg10_oldval; \ |
290 | __typeof (mem) __atg10_memp = (mem); \ |
291 | __typeof (*(mem)) __atg10_value = (value); \ |
292 | do { \ |
293 | __atg10_oldval = *__atg10_memp; \ |
294 | if (__atg10_oldval <= __atg10_value) \ |
295 | break; \ |
296 | } while (__builtin_expect \ |
297 | (atomic_compare_and_exchange_bool_acq (__atg10_memp, \ |
298 | __atg10_value, \ |
299 | __atg10_oldval), 0)); \ |
300 | } while (0) |
301 | #endif |
302 | |
303 | |
304 | #ifndef atomic_add |
305 | # define atomic_add(mem, value) (void) atomic_exchange_and_add ((mem), (value)) |
306 | #endif |
307 | |
308 | |
309 | #ifndef catomic_add |
310 | # define catomic_add(mem, value) \ |
311 | (void) catomic_exchange_and_add ((mem), (value)) |
312 | #endif |
313 | |
314 | |
315 | #ifndef atomic_increment |
316 | # define atomic_increment(mem) atomic_add ((mem), 1) |
317 | #endif |
318 | |
319 | |
320 | #ifndef catomic_increment |
321 | # define catomic_increment(mem) catomic_add ((mem), 1) |
322 | #endif |
323 | |
324 | |
325 | #ifndef atomic_increment_val |
326 | # define atomic_increment_val(mem) (atomic_exchange_and_add ((mem), 1) + 1) |
327 | #endif |
328 | |
329 | |
330 | #ifndef catomic_increment_val |
331 | # define catomic_increment_val(mem) (catomic_exchange_and_add ((mem), 1) + 1) |
332 | #endif |
333 | |
334 | |
335 | /* Add one to *MEM and return true iff it's now zero. */ |
336 | #ifndef atomic_increment_and_test |
337 | # define atomic_increment_and_test(mem) \ |
338 | (atomic_exchange_and_add ((mem), 1) + 1 == 0) |
339 | #endif |
340 | |
341 | |
342 | #ifndef atomic_decrement |
343 | # define atomic_decrement(mem) atomic_add ((mem), -1) |
344 | #endif |
345 | |
346 | |
347 | #ifndef catomic_decrement |
348 | # define catomic_decrement(mem) catomic_add ((mem), -1) |
349 | #endif |
350 | |
351 | |
352 | #ifndef atomic_decrement_val |
353 | # define atomic_decrement_val(mem) (atomic_exchange_and_add ((mem), -1) - 1) |
354 | #endif |
355 | |
356 | |
357 | #ifndef catomic_decrement_val |
358 | # define catomic_decrement_val(mem) (catomic_exchange_and_add ((mem), -1) - 1) |
359 | #endif |
360 | |
361 | |
362 | /* Subtract 1 from *MEM and return true iff it's now zero. */ |
363 | #ifndef atomic_decrement_and_test |
364 | # define atomic_decrement_and_test(mem) \ |
365 | (atomic_exchange_and_add ((mem), -1) == 1) |
366 | #endif |
367 | |
368 | |
369 | /* Decrement *MEM if it is > 0, and return the old value. */ |
370 | #ifndef atomic_decrement_if_positive |
371 | # define atomic_decrement_if_positive(mem) \ |
372 | ({ __typeof (*(mem)) __atg11_oldval; \ |
373 | __typeof (mem) __atg11_memp = (mem); \ |
374 | \ |
375 | do \ |
376 | { \ |
377 | __atg11_oldval = *__atg11_memp; \ |
378 | if (__glibc_unlikely (__atg11_oldval <= 0)) \ |
379 | break; \ |
380 | } \ |
381 | while (__builtin_expect \ |
382 | (atomic_compare_and_exchange_bool_acq (__atg11_memp, \ |
383 | __atg11_oldval - 1, \ |
384 | __atg11_oldval), 0)); \ |
385 | __atg11_oldval; }) |
386 | #endif |
387 | |
388 | |
389 | #ifndef atomic_add_negative |
390 | # define atomic_add_negative(mem, value) \ |
391 | ({ __typeof (value) __atg12_value = (value); \ |
392 | atomic_exchange_and_add (mem, __atg12_value) < -__atg12_value; }) |
393 | #endif |
394 | |
395 | |
396 | #ifndef atomic_add_zero |
397 | # define atomic_add_zero(mem, value) \ |
398 | ({ __typeof (value) __atg13_value = (value); \ |
399 | atomic_exchange_and_add (mem, __atg13_value) == -__atg13_value; }) |
400 | #endif |
401 | |
402 | |
403 | #ifndef atomic_bit_set |
404 | # define atomic_bit_set(mem, bit) \ |
405 | (void) atomic_bit_test_set(mem, bit) |
406 | #endif |
407 | |
408 | |
409 | #ifndef atomic_bit_test_set |
410 | # define atomic_bit_test_set(mem, bit) \ |
411 | ({ __typeof (*(mem)) __atg14_old; \ |
412 | __typeof (mem) __atg14_memp = (mem); \ |
413 | __typeof (*(mem)) __atg14_mask = ((__typeof (*(mem))) 1 << (bit)); \ |
414 | \ |
415 | do \ |
416 | __atg14_old = (*__atg14_memp); \ |
417 | while (__builtin_expect \ |
418 | (atomic_compare_and_exchange_bool_acq (__atg14_memp, \ |
419 | __atg14_old | __atg14_mask,\ |
420 | __atg14_old), 0)); \ |
421 | \ |
422 | __atg14_old & __atg14_mask; }) |
423 | #endif |
424 | |
425 | /* Atomically *mem &= mask. */ |
426 | #ifndef atomic_and |
427 | # define atomic_and(mem, mask) \ |
428 | do { \ |
429 | __typeof (*(mem)) __atg15_old; \ |
430 | __typeof (mem) __atg15_memp = (mem); \ |
431 | __typeof (*(mem)) __atg15_mask = (mask); \ |
432 | \ |
433 | do \ |
434 | __atg15_old = (*__atg15_memp); \ |
435 | while (__builtin_expect \ |
436 | (atomic_compare_and_exchange_bool_acq (__atg15_memp, \ |
437 | __atg15_old & __atg15_mask, \ |
438 | __atg15_old), 0)); \ |
439 | } while (0) |
440 | #endif |
441 | |
442 | #ifndef catomic_and |
443 | # define catomic_and(mem, mask) \ |
444 | do { \ |
445 | __typeof (*(mem)) __atg20_old; \ |
446 | __typeof (mem) __atg20_memp = (mem); \ |
447 | __typeof (*(mem)) __atg20_mask = (mask); \ |
448 | \ |
449 | do \ |
450 | __atg20_old = (*__atg20_memp); \ |
451 | while (__builtin_expect \ |
452 | (catomic_compare_and_exchange_bool_acq (__atg20_memp, \ |
453 | __atg20_old & __atg20_mask,\ |
454 | __atg20_old), 0)); \ |
455 | } while (0) |
456 | #endif |
457 | |
458 | /* Atomically *mem &= mask and return the old value of *mem. */ |
459 | #ifndef atomic_and_val |
460 | # define atomic_and_val(mem, mask) \ |
461 | ({ __typeof (*(mem)) __atg16_old; \ |
462 | __typeof (mem) __atg16_memp = (mem); \ |
463 | __typeof (*(mem)) __atg16_mask = (mask); \ |
464 | \ |
465 | do \ |
466 | __atg16_old = (*__atg16_memp); \ |
467 | while (__builtin_expect \ |
468 | (atomic_compare_and_exchange_bool_acq (__atg16_memp, \ |
469 | __atg16_old & __atg16_mask,\ |
470 | __atg16_old), 0)); \ |
471 | \ |
472 | __atg16_old; }) |
473 | #endif |
474 | |
475 | /* Atomically *mem |= mask and return the old value of *mem. */ |
476 | #ifndef atomic_or |
477 | # define atomic_or(mem, mask) \ |
478 | do { \ |
479 | __typeof (*(mem)) __atg17_old; \ |
480 | __typeof (mem) __atg17_memp = (mem); \ |
481 | __typeof (*(mem)) __atg17_mask = (mask); \ |
482 | \ |
483 | do \ |
484 | __atg17_old = (*__atg17_memp); \ |
485 | while (__builtin_expect \ |
486 | (atomic_compare_and_exchange_bool_acq (__atg17_memp, \ |
487 | __atg17_old | __atg17_mask, \ |
488 | __atg17_old), 0)); \ |
489 | } while (0) |
490 | #endif |
491 | |
492 | #ifndef catomic_or |
493 | # define catomic_or(mem, mask) \ |
494 | do { \ |
495 | __typeof (*(mem)) __atg18_old; \ |
496 | __typeof (mem) __atg18_memp = (mem); \ |
497 | __typeof (*(mem)) __atg18_mask = (mask); \ |
498 | \ |
499 | do \ |
500 | __atg18_old = (*__atg18_memp); \ |
501 | while (__builtin_expect \ |
502 | (catomic_compare_and_exchange_bool_acq (__atg18_memp, \ |
503 | __atg18_old | __atg18_mask,\ |
504 | __atg18_old), 0)); \ |
505 | } while (0) |
506 | #endif |
507 | |
508 | /* Atomically *mem |= mask and return the old value of *mem. */ |
509 | #ifndef atomic_or_val |
510 | # define atomic_or_val(mem, mask) \ |
511 | ({ __typeof (*(mem)) __atg19_old; \ |
512 | __typeof (mem) __atg19_memp = (mem); \ |
513 | __typeof (*(mem)) __atg19_mask = (mask); \ |
514 | \ |
515 | do \ |
516 | __atg19_old = (*__atg19_memp); \ |
517 | while (__builtin_expect \ |
518 | (atomic_compare_and_exchange_bool_acq (__atg19_memp, \ |
519 | __atg19_old | __atg19_mask,\ |
520 | __atg19_old), 0)); \ |
521 | \ |
522 | __atg19_old; }) |
523 | #endif |
524 | |
525 | #ifndef atomic_full_barrier |
526 | # define atomic_full_barrier() __asm ("" ::: "memory") |
527 | #endif |
528 | |
529 | |
530 | #ifndef atomic_read_barrier |
531 | # define atomic_read_barrier() atomic_full_barrier () |
532 | #endif |
533 | |
534 | |
535 | #ifndef atomic_write_barrier |
536 | # define atomic_write_barrier() atomic_full_barrier () |
537 | #endif |
538 | |
539 | |
540 | #ifndef atomic_forced_read |
541 | # define atomic_forced_read(x) \ |
542 | ({ __typeof (x) __x; __asm ("" : "=r" (__x) : "0" (x)); __x; }) |
543 | #endif |
544 | |
545 | /* This is equal to 1 iff the architecture supports 64b atomic operations. */ |
546 | #ifndef __HAVE_64B_ATOMICS |
547 | #error Unable to determine if 64-bit atomics are present. |
548 | #endif |
549 | |
550 | /* The following functions are a subset of the atomic operations provided by |
551 | C11. Usually, a function named atomic_OP_MO(args) is equivalent to C11's |
552 | atomic_OP_explicit(args, memory_order_MO); exceptions noted below. */ |
553 | |
554 | /* Each arch can request to use compiler built-ins for C11 atomics. If it |
555 | does, all atomics will be based on these. */ |
556 | #if USE_ATOMIC_COMPILER_BUILTINS |
557 | |
558 | /* We require 32b atomic operations; some archs also support 64b atomic |
559 | operations. */ |
560 | void __atomic_link_error (void); |
561 | # if __HAVE_64B_ATOMICS == 1 |
562 | # define __atomic_check_size(mem) \ |
563 | if ((sizeof (*mem) != 4) && (sizeof (*mem) != 8)) \ |
564 | __atomic_link_error (); |
565 | # else |
566 | # define __atomic_check_size(mem) \ |
567 | if (sizeof (*mem) != 4) \ |
568 | __atomic_link_error (); |
569 | # endif |
570 | |
571 | # define atomic_thread_fence_acquire() \ |
572 | __atomic_thread_fence (__ATOMIC_ACQUIRE) |
573 | # define atomic_thread_fence_release() \ |
574 | __atomic_thread_fence (__ATOMIC_RELEASE) |
575 | # define atomic_thread_fence_seq_cst() \ |
576 | __atomic_thread_fence (__ATOMIC_SEQ_CST) |
577 | |
578 | # define atomic_load_relaxed(mem) \ |
579 | ({ __atomic_check_size((mem)); __atomic_load_n ((mem), __ATOMIC_RELAXED); }) |
580 | # define atomic_load_acquire(mem) \ |
581 | ({ __atomic_check_size((mem)); __atomic_load_n ((mem), __ATOMIC_ACQUIRE); }) |
582 | |
583 | # define atomic_store_relaxed(mem, val) \ |
584 | do { \ |
585 | __atomic_check_size((mem)); \ |
586 | __atomic_store_n ((mem), (val), __ATOMIC_RELAXED); \ |
587 | } while (0) |
588 | # define atomic_store_release(mem, val) \ |
589 | do { \ |
590 | __atomic_check_size((mem)); \ |
591 | __atomic_store_n ((mem), (val), __ATOMIC_RELEASE); \ |
592 | } while (0) |
593 | |
594 | /* On failure, this CAS has memory_order_relaxed semantics. */ |
595 | # define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \ |
596 | ({ __atomic_check_size((mem)); \ |
597 | __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \ |
598 | __ATOMIC_RELAXED, __ATOMIC_RELAXED); }) |
599 | # define atomic_compare_exchange_weak_acquire(mem, expected, desired) \ |
600 | ({ __atomic_check_size((mem)); \ |
601 | __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \ |
602 | __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); }) |
603 | # define atomic_compare_exchange_weak_release(mem, expected, desired) \ |
604 | ({ __atomic_check_size((mem)); \ |
605 | __atomic_compare_exchange_n ((mem), (expected), (desired), 1, \ |
606 | __ATOMIC_RELEASE, __ATOMIC_RELAXED); }) |
607 | |
608 | # define atomic_exchange_acquire(mem, desired) \ |
609 | ({ __atomic_check_size((mem)); \ |
610 | __atomic_exchange_n ((mem), (desired), __ATOMIC_ACQUIRE); }) |
611 | # define atomic_exchange_release(mem, desired) \ |
612 | ({ __atomic_check_size((mem)); \ |
613 | __atomic_exchange_n ((mem), (desired), __ATOMIC_RELEASE); }) |
614 | |
615 | # define atomic_fetch_add_relaxed(mem, operand) \ |
616 | ({ __atomic_check_size((mem)); \ |
617 | __atomic_fetch_add ((mem), (operand), __ATOMIC_RELAXED); }) |
618 | # define atomic_fetch_add_acquire(mem, operand) \ |
619 | ({ __atomic_check_size((mem)); \ |
620 | __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQUIRE); }) |
621 | # define atomic_fetch_add_release(mem, operand) \ |
622 | ({ __atomic_check_size((mem)); \ |
623 | __atomic_fetch_add ((mem), (operand), __ATOMIC_RELEASE); }) |
624 | # define atomic_fetch_add_acq_rel(mem, operand) \ |
625 | ({ __atomic_check_size((mem)); \ |
626 | __atomic_fetch_add ((mem), (operand), __ATOMIC_ACQ_REL); }) |
627 | |
628 | # define atomic_fetch_and_acquire(mem, operand) \ |
629 | ({ __atomic_check_size((mem)); \ |
630 | __atomic_fetch_and ((mem), (operand), __ATOMIC_ACQUIRE); }) |
631 | |
632 | # define atomic_fetch_or_relaxed(mem, operand) \ |
633 | ({ __atomic_check_size((mem)); \ |
634 | __atomic_fetch_or ((mem), (operand), __ATOMIC_RELAXED); }) |
635 | # define atomic_fetch_or_acquire(mem, operand) \ |
636 | ({ __atomic_check_size((mem)); \ |
637 | __atomic_fetch_or ((mem), (operand), __ATOMIC_ACQUIRE); }) |
638 | |
639 | #else /* !USE_ATOMIC_COMPILER_BUILTINS */ |
640 | |
641 | /* By default, we assume that read, write, and full barriers are equivalent |
642 | to acquire, release, and seq_cst barriers. Archs for which this does not |
643 | hold have to provide custom definitions of the fences. */ |
644 | # ifndef atomic_thread_fence_acquire |
645 | # define atomic_thread_fence_acquire() atomic_read_barrier () |
646 | # endif |
647 | # ifndef atomic_thread_fence_release |
648 | # define atomic_thread_fence_release() atomic_write_barrier () |
649 | # endif |
650 | # ifndef atomic_thread_fence_seq_cst |
651 | # define atomic_thread_fence_seq_cst() atomic_full_barrier () |
652 | # endif |
653 | |
654 | # ifndef atomic_load_relaxed |
655 | # define atomic_load_relaxed(mem) \ |
656 | ({ __typeof (*(mem)) __atg100_val; \ |
657 | __asm ("" : "=r" (__atg100_val) : "0" (*(mem))); \ |
658 | __atg100_val; }) |
659 | # endif |
660 | # ifndef atomic_load_acquire |
661 | # define atomic_load_acquire(mem) \ |
662 | ({ __typeof (*(mem)) __atg101_val = atomic_load_relaxed (mem); \ |
663 | atomic_thread_fence_acquire (); \ |
664 | __atg101_val; }) |
665 | # endif |
666 | |
667 | # ifndef atomic_store_relaxed |
668 | /* XXX Use inline asm here? */ |
669 | # define atomic_store_relaxed(mem, val) do { *(mem) = (val); } while (0) |
670 | # endif |
671 | # ifndef atomic_store_release |
672 | # define atomic_store_release(mem, val) \ |
673 | do { \ |
674 | atomic_thread_fence_release (); \ |
675 | atomic_store_relaxed ((mem), (val)); \ |
676 | } while (0) |
677 | # endif |
678 | |
679 | /* On failure, this CAS has memory_order_relaxed semantics. */ |
680 | /* XXX This potentially has one branch more than necessary, but archs |
681 | currently do not define a CAS that returns both the previous value and |
682 | the success flag. */ |
683 | # ifndef atomic_compare_exchange_weak_acquire |
684 | # define atomic_compare_exchange_weak_acquire(mem, expected, desired) \ |
685 | ({ typeof (*(expected)) __atg102_expected = *(expected); \ |
686 | *(expected) = \ |
687 | atomic_compare_and_exchange_val_acq ((mem), (desired), *(expected)); \ |
688 | *(expected) == __atg102_expected; }) |
689 | # endif |
690 | # ifndef atomic_compare_exchange_weak_relaxed |
691 | /* XXX Fall back to CAS with acquire MO because archs do not define a weaker |
692 | CAS. */ |
693 | # define atomic_compare_exchange_weak_relaxed(mem, expected, desired) \ |
694 | atomic_compare_exchange_weak_acquire ((mem), (expected), (desired)) |
695 | # endif |
696 | # ifndef atomic_compare_exchange_weak_release |
697 | # define atomic_compare_exchange_weak_release(mem, expected, desired) \ |
698 | ({ typeof (*(expected)) __atg103_expected = *(expected); \ |
699 | *(expected) = \ |
700 | atomic_compare_and_exchange_val_rel ((mem), (desired), *(expected)); \ |
701 | *(expected) == __atg103_expected; }) |
702 | # endif |
703 | |
704 | # ifndef atomic_exchange_acquire |
705 | # define atomic_exchange_acquire(mem, val) \ |
706 | atomic_exchange_acq ((mem), (val)) |
707 | # endif |
708 | # ifndef atomic_exchange_release |
709 | # define atomic_exchange_release(mem, val) \ |
710 | atomic_exchange_rel ((mem), (val)) |
711 | # endif |
712 | |
713 | # ifndef atomic_fetch_add_acquire |
714 | # define atomic_fetch_add_acquire(mem, operand) \ |
715 | atomic_exchange_and_add_acq ((mem), (operand)) |
716 | # endif |
717 | # ifndef atomic_fetch_add_relaxed |
718 | /* XXX Fall back to acquire MO because the MO semantics of |
719 | atomic_exchange_and_add are not documented; the generic version falls back |
720 | to atomic_exchange_and_add_acq if atomic_exchange_and_add is not defined, |
721 | and vice versa. */ |
722 | # define atomic_fetch_add_relaxed(mem, operand) \ |
723 | atomic_fetch_add_acquire ((mem), (operand)) |
724 | # endif |
725 | # ifndef atomic_fetch_add_release |
726 | # define atomic_fetch_add_release(mem, operand) \ |
727 | atomic_exchange_and_add_rel ((mem), (operand)) |
728 | # endif |
729 | # ifndef atomic_fetch_add_acq_rel |
730 | # define atomic_fetch_add_acq_rel(mem, operand) \ |
731 | ({ atomic_thread_fence_release (); \ |
732 | atomic_exchange_and_add_acq ((mem), (operand)); }) |
733 | # endif |
734 | |
735 | /* XXX The default for atomic_and_val has acquire semantics, but this is not |
736 | documented. */ |
737 | # ifndef atomic_fetch_and_acquire |
738 | # define atomic_fetch_and_acquire(mem, operand) \ |
739 | atomic_and_val ((mem), (operand)) |
740 | # endif |
741 | |
742 | /* XXX The default for atomic_or_val has acquire semantics, but this is not |
743 | documented. */ |
744 | # ifndef atomic_fetch_or_acquire |
745 | # define atomic_fetch_or_acquire(mem, operand) \ |
746 | atomic_or_val ((mem), (operand)) |
747 | # endif |
748 | /* XXX Fall back to acquire MO because archs do not define a weaker |
749 | atomic_or_val. */ |
750 | # ifndef atomic_fetch_or_relaxed |
751 | # define atomic_fetch_or_relaxed(mem, operand) \ |
752 | atomic_fetch_or_acquire ((mem), (operand)) |
753 | # endif |
754 | |
755 | #endif /* !USE_ATOMIC_COMPILER_BUILTINS */ |
756 | |
757 | /* This operation does not affect synchronization semantics but can be used |
758 | in the body of a spin loop to potentially improve its efficiency. */ |
759 | #ifndef atomic_spin_nop |
760 | # define atomic_spin_nop() do { /* nothing */ } while (0) |
761 | #endif |
762 | |
763 | #endif /* atomic.h */ |
764 | |