1 | /* Definition for thread-local data handling. nptl/x86_64 version. |
2 | Copyright (C) 2002-2016 Free Software Foundation, Inc. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ |
18 | |
19 | #ifndef _TLS_H |
20 | #define _TLS_H 1 |
21 | |
22 | #ifndef __ASSEMBLER__ |
23 | # include <asm/prctl.h> /* For ARCH_SET_FS. */ |
24 | # include <stdbool.h> |
25 | # include <stddef.h> |
26 | # include <stdint.h> |
27 | # include <stdlib.h> |
28 | # include <sysdep.h> |
29 | # include <libc-internal.h> |
30 | # include <kernel-features.h> |
31 | |
32 | /* Replacement type for __m128 since this file is included by ld.so, |
33 | which is compiled with -mno-sse. It must not change the alignment |
34 | of rtld_savespace_sse. */ |
35 | typedef struct |
36 | { |
37 | int i[4]; |
38 | } __128bits; |
39 | |
40 | |
41 | /* Type for the dtv. */ |
42 | typedef union dtv |
43 | { |
44 | size_t counter; |
45 | struct |
46 | { |
47 | void *val; |
48 | bool is_static; |
49 | } pointer; |
50 | } dtv_t; |
51 | |
52 | |
53 | typedef struct |
54 | { |
55 | void *tcb; /* Pointer to the TCB. Not necessarily the |
56 | thread descriptor used by libpthread. */ |
57 | dtv_t *dtv; |
58 | void *self; /* Pointer to the thread descriptor. */ |
59 | int multiple_threads; |
60 | int gscope_flag; |
61 | uintptr_t sysinfo; |
62 | uintptr_t stack_guard; |
63 | uintptr_t pointer_guard; |
64 | unsigned long int vgetcpu_cache[2]; |
65 | # ifndef __ASSUME_PRIVATE_FUTEX |
66 | int private_futex; |
67 | # else |
68 | int __glibc_reserved1; |
69 | # endif |
70 | int __glibc_unused1; |
71 | /* Reservation of some values for the TM ABI. */ |
72 | void *__private_tm[4]; |
73 | /* GCC split stack support. */ |
74 | void *__private_ss; |
75 | long int __glibc_reserved2; |
76 | /* Must be kept even if it is no longer used by glibc since programs, |
77 | like AddressSanitizer, depend on the size of tcbhead_t. */ |
78 | __128bits __glibc_unused2[8][4] __attribute__ ((aligned (32))); |
79 | |
80 | void *__padding[8]; |
81 | } tcbhead_t; |
82 | |
83 | #else /* __ASSEMBLER__ */ |
84 | # include <tcb-offsets.h> |
85 | #endif |
86 | |
87 | |
88 | /* Alignment requirement for the stack. */ |
89 | #define STACK_ALIGN 16 |
90 | |
91 | |
92 | #ifndef __ASSEMBLER__ |
93 | /* Get system call information. */ |
94 | # include <sysdep.h> |
95 | |
96 | #ifndef LOCK_PREFIX |
97 | # ifdef UP |
98 | # define LOCK_PREFIX /* nothing */ |
99 | # else |
100 | # define LOCK_PREFIX "lock;" |
101 | # endif |
102 | #endif |
103 | |
104 | /* This is the size of the initial TCB. Can't be just sizeof (tcbhead_t), |
105 | because NPTL getpid, __libc_alloca_cutoff etc. need (almost) the whole |
106 | struct pthread even when not linked with -lpthread. */ |
107 | # define TLS_INIT_TCB_SIZE sizeof (struct pthread) |
108 | |
109 | /* Alignment requirements for the initial TCB. */ |
110 | # define TLS_INIT_TCB_ALIGN __alignof__ (struct pthread) |
111 | |
112 | /* This is the size of the TCB. */ |
113 | # define TLS_TCB_SIZE sizeof (struct pthread) |
114 | |
115 | /* Alignment requirements for the TCB. */ |
116 | # define TLS_TCB_ALIGN __alignof__ (struct pthread) |
117 | |
118 | /* The TCB can have any size and the memory following the address the |
119 | thread pointer points to is unspecified. Allocate the TCB there. */ |
120 | # define TLS_TCB_AT_TP 1 |
121 | # define TLS_DTV_AT_TP 0 |
122 | |
123 | /* Get the thread descriptor definition. */ |
124 | # include <nptl/descr.h> |
125 | |
126 | |
127 | /* Install the dtv pointer. The pointer passed is to the element with |
128 | index -1 which contain the length. */ |
129 | # define INSTALL_DTV(descr, dtvp) \ |
130 | ((tcbhead_t *) (descr))->dtv = (dtvp) + 1 |
131 | |
132 | /* Install new dtv for current thread. */ |
133 | # define INSTALL_NEW_DTV(dtvp) \ |
134 | ({ struct pthread *__pd; \ |
135 | THREAD_SETMEM (__pd, header.dtv, (dtvp)); }) |
136 | |
137 | /* Return dtv of given thread descriptor. */ |
138 | # define GET_DTV(descr) \ |
139 | (((tcbhead_t *) (descr))->dtv) |
140 | |
141 | |
142 | /* Code to initially initialize the thread pointer. This might need |
143 | special attention since 'errno' is not yet available and if the |
144 | operation can cause a failure 'errno' must not be touched. |
145 | |
146 | We have to make the syscall for both uses of the macro since the |
147 | address might be (and probably is) different. */ |
148 | # define TLS_INIT_TP(thrdescr) \ |
149 | ({ void *_thrdescr = (thrdescr); \ |
150 | tcbhead_t *_head = _thrdescr; \ |
151 | int _result; \ |
152 | \ |
153 | _head->tcb = _thrdescr; \ |
154 | /* For now the thread descriptor is at the same address. */ \ |
155 | _head->self = _thrdescr; \ |
156 | \ |
157 | /* It is a simple syscall to set the %fs value for the thread. */ \ |
158 | asm volatile ("syscall" \ |
159 | : "=a" (_result) \ |
160 | : "0" ((unsigned long int) __NR_arch_prctl), \ |
161 | "D" ((unsigned long int) ARCH_SET_FS), \ |
162 | "S" (_thrdescr) \ |
163 | : "memory", "cc", "r11", "cx"); \ |
164 | \ |
165 | _result ? "cannot set %fs base address for thread-local storage" : 0; \ |
166 | }) |
167 | |
168 | # define TLS_DEFINE_INIT_TP(tp, pd) void *tp = (pd) |
169 | |
170 | |
171 | /* Return the address of the dtv for the current thread. */ |
172 | # define THREAD_DTV() \ |
173 | ({ struct pthread *__pd; \ |
174 | THREAD_GETMEM (__pd, header.dtv); }) |
175 | |
176 | |
177 | /* Return the thread descriptor for the current thread. |
178 | |
179 | The contained asm must *not* be marked volatile since otherwise |
180 | assignments like |
181 | pthread_descr self = thread_self(); |
182 | do not get optimized away. */ |
183 | # define THREAD_SELF \ |
184 | ({ struct pthread *__self; \ |
185 | asm ("mov %%fs:%c1,%0" : "=r" (__self) \ |
186 | : "i" (offsetof (struct pthread, header.self))); \ |
187 | __self;}) |
188 | |
189 | /* Magic for libthread_db to know how to do THREAD_SELF. */ |
190 | # define DB_THREAD_SELF_INCLUDE <sys/reg.h> /* For the FS constant. */ |
191 | # define DB_THREAD_SELF CONST_THREAD_AREA (64, FS) |
192 | |
193 | /* Read member of the thread descriptor directly. */ |
194 | # define THREAD_GETMEM(descr, member) \ |
195 | ({ __typeof (descr->member) __value; \ |
196 | if (sizeof (__value) == 1) \ |
197 | asm volatile ("movb %%fs:%P2,%b0" \ |
198 | : "=q" (__value) \ |
199 | : "0" (0), "i" (offsetof (struct pthread, member))); \ |
200 | else if (sizeof (__value) == 4) \ |
201 | asm volatile ("movl %%fs:%P1,%0" \ |
202 | : "=r" (__value) \ |
203 | : "i" (offsetof (struct pthread, member))); \ |
204 | else \ |
205 | { \ |
206 | if (sizeof (__value) != 8) \ |
207 | /* There should not be any value with a size other than 1, \ |
208 | 4 or 8. */ \ |
209 | abort (); \ |
210 | \ |
211 | asm volatile ("movq %%fs:%P1,%q0" \ |
212 | : "=r" (__value) \ |
213 | : "i" (offsetof (struct pthread, member))); \ |
214 | } \ |
215 | __value; }) |
216 | |
217 | |
218 | /* Same as THREAD_GETMEM, but the member offset can be non-constant. */ |
219 | # define THREAD_GETMEM_NC(descr, member, idx) \ |
220 | ({ __typeof (descr->member[0]) __value; \ |
221 | if (sizeof (__value) == 1) \ |
222 | asm volatile ("movb %%fs:%P2(%q3),%b0" \ |
223 | : "=q" (__value) \ |
224 | : "0" (0), "i" (offsetof (struct pthread, member[0])), \ |
225 | "r" (idx)); \ |
226 | else if (sizeof (__value) == 4) \ |
227 | asm volatile ("movl %%fs:%P1(,%q2,4),%0" \ |
228 | : "=r" (__value) \ |
229 | : "i" (offsetof (struct pthread, member[0])), "r" (idx));\ |
230 | else \ |
231 | { \ |
232 | if (sizeof (__value) != 8) \ |
233 | /* There should not be any value with a size other than 1, \ |
234 | 4 or 8. */ \ |
235 | abort (); \ |
236 | \ |
237 | asm volatile ("movq %%fs:%P1(,%q2,8),%q0" \ |
238 | : "=r" (__value) \ |
239 | : "i" (offsetof (struct pthread, member[0])), \ |
240 | "r" (idx)); \ |
241 | } \ |
242 | __value; }) |
243 | |
244 | |
245 | /* Loading addresses of objects on x86-64 needs to be treated special |
246 | when generating PIC code. */ |
247 | #ifdef __pic__ |
248 | # define IMM_MODE "nr" |
249 | #else |
250 | # define IMM_MODE "ir" |
251 | #endif |
252 | |
253 | |
254 | /* Set member of the thread descriptor directly. */ |
255 | # define THREAD_SETMEM(descr, member, value) \ |
256 | ({ if (sizeof (descr->member) == 1) \ |
257 | asm volatile ("movb %b0,%%fs:%P1" : \ |
258 | : "iq" (value), \ |
259 | "i" (offsetof (struct pthread, member))); \ |
260 | else if (sizeof (descr->member) == 4) \ |
261 | asm volatile ("movl %0,%%fs:%P1" : \ |
262 | : IMM_MODE (value), \ |
263 | "i" (offsetof (struct pthread, member))); \ |
264 | else \ |
265 | { \ |
266 | if (sizeof (descr->member) != 8) \ |
267 | /* There should not be any value with a size other than 1, \ |
268 | 4 or 8. */ \ |
269 | abort (); \ |
270 | \ |
271 | asm volatile ("movq %q0,%%fs:%P1" : \ |
272 | : IMM_MODE ((uint64_t) cast_to_integer (value)), \ |
273 | "i" (offsetof (struct pthread, member))); \ |
274 | }}) |
275 | |
276 | |
277 | /* Same as THREAD_SETMEM, but the member offset can be non-constant. */ |
278 | # define THREAD_SETMEM_NC(descr, member, idx, value) \ |
279 | ({ if (sizeof (descr->member[0]) == 1) \ |
280 | asm volatile ("movb %b0,%%fs:%P1(%q2)" : \ |
281 | : "iq" (value), \ |
282 | "i" (offsetof (struct pthread, member[0])), \ |
283 | "r" (idx)); \ |
284 | else if (sizeof (descr->member[0]) == 4) \ |
285 | asm volatile ("movl %0,%%fs:%P1(,%q2,4)" : \ |
286 | : IMM_MODE (value), \ |
287 | "i" (offsetof (struct pthread, member[0])), \ |
288 | "r" (idx)); \ |
289 | else \ |
290 | { \ |
291 | if (sizeof (descr->member[0]) != 8) \ |
292 | /* There should not be any value with a size other than 1, \ |
293 | 4 or 8. */ \ |
294 | abort (); \ |
295 | \ |
296 | asm volatile ("movq %q0,%%fs:%P1(,%q2,8)" : \ |
297 | : IMM_MODE ((uint64_t) cast_to_integer (value)), \ |
298 | "i" (offsetof (struct pthread, member[0])), \ |
299 | "r" (idx)); \ |
300 | }}) |
301 | |
302 | |
303 | /* Atomic compare and exchange on TLS, returning old value. */ |
304 | # define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \ |
305 | ({ __typeof (descr->member) __ret; \ |
306 | __typeof (oldval) __old = (oldval); \ |
307 | if (sizeof (descr->member) == 4) \ |
308 | asm volatile (LOCK_PREFIX "cmpxchgl %2, %%fs:%P3" \ |
309 | : "=a" (__ret) \ |
310 | : "0" (__old), "r" (newval), \ |
311 | "i" (offsetof (struct pthread, member))); \ |
312 | else \ |
313 | /* Not necessary for other sizes in the moment. */ \ |
314 | abort (); \ |
315 | __ret; }) |
316 | |
317 | |
318 | /* Atomic logical and. */ |
319 | # define THREAD_ATOMIC_AND(descr, member, val) \ |
320 | (void) ({ if (sizeof ((descr)->member) == 4) \ |
321 | asm volatile (LOCK_PREFIX "andl %1, %%fs:%P0" \ |
322 | :: "i" (offsetof (struct pthread, member)), \ |
323 | "ir" (val)); \ |
324 | else \ |
325 | /* Not necessary for other sizes in the moment. */ \ |
326 | abort (); }) |
327 | |
328 | |
329 | /* Atomic set bit. */ |
330 | # define THREAD_ATOMIC_BIT_SET(descr, member, bit) \ |
331 | (void) ({ if (sizeof ((descr)->member) == 4) \ |
332 | asm volatile (LOCK_PREFIX "orl %1, %%fs:%P0" \ |
333 | :: "i" (offsetof (struct pthread, member)), \ |
334 | "ir" (1 << (bit))); \ |
335 | else \ |
336 | /* Not necessary for other sizes in the moment. */ \ |
337 | abort (); }) |
338 | |
339 | |
340 | # define CALL_THREAD_FCT(descr) \ |
341 | ({ void *__res; \ |
342 | asm volatile ("movq %%fs:%P2, %%rdi\n\t" \ |
343 | "callq *%%fs:%P1" \ |
344 | : "=a" (__res) \ |
345 | : "i" (offsetof (struct pthread, start_routine)), \ |
346 | "i" (offsetof (struct pthread, arg)) \ |
347 | : "di", "si", "cx", "dx", "r8", "r9", "r10", "r11", \ |
348 | "memory", "cc"); \ |
349 | __res; }) |
350 | |
351 | |
352 | /* Set the stack guard field in TCB head. */ |
353 | # define THREAD_SET_STACK_GUARD(value) \ |
354 | THREAD_SETMEM (THREAD_SELF, header.stack_guard, value) |
355 | # define THREAD_COPY_STACK_GUARD(descr) \ |
356 | ((descr)->header.stack_guard \ |
357 | = THREAD_GETMEM (THREAD_SELF, header.stack_guard)) |
358 | |
359 | |
360 | /* Set the pointer guard field in the TCB head. */ |
361 | # define THREAD_SET_POINTER_GUARD(value) \ |
362 | THREAD_SETMEM (THREAD_SELF, header.pointer_guard, value) |
363 | # define THREAD_COPY_POINTER_GUARD(descr) \ |
364 | ((descr)->header.pointer_guard \ |
365 | = THREAD_GETMEM (THREAD_SELF, header.pointer_guard)) |
366 | |
367 | |
368 | /* Get and set the global scope generation counter in the TCB head. */ |
369 | # define THREAD_GSCOPE_FLAG_UNUSED 0 |
370 | # define THREAD_GSCOPE_FLAG_USED 1 |
371 | # define THREAD_GSCOPE_FLAG_WAIT 2 |
372 | # define THREAD_GSCOPE_RESET_FLAG() \ |
373 | do \ |
374 | { int __res; \ |
375 | asm volatile ("xchgl %0, %%fs:%P1" \ |
376 | : "=r" (__res) \ |
377 | : "i" (offsetof (struct pthread, header.gscope_flag)), \ |
378 | "0" (THREAD_GSCOPE_FLAG_UNUSED)); \ |
379 | if (__res == THREAD_GSCOPE_FLAG_WAIT) \ |
380 | lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1, LLL_PRIVATE); \ |
381 | } \ |
382 | while (0) |
383 | # define THREAD_GSCOPE_SET_FLAG() \ |
384 | THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED) |
385 | # define THREAD_GSCOPE_WAIT() \ |
386 | GL(dl_wait_lookup_done) () |
387 | |
388 | #endif /* __ASSEMBLER__ */ |
389 | |
390 | #endif /* tls.h */ |
391 | |