1 | /* Definition for thread-local data handling. nptl/x86_64 version. |
2 | Copyright (C) 2002-2018 Free Software Foundation, Inc. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ |
18 | |
19 | #ifndef _TLS_H |
20 | #define _TLS_H 1 |
21 | |
22 | #ifndef __ASSEMBLER__ |
23 | # include <asm/prctl.h> /* For ARCH_SET_FS. */ |
24 | # include <stdbool.h> |
25 | # include <stddef.h> |
26 | # include <stdint.h> |
27 | # include <stdlib.h> |
28 | # include <sysdep.h> |
29 | # include <libc-pointer-arith.h> /* For cast_to_integer. */ |
30 | # include <kernel-features.h> |
31 | # include <dl-dtv.h> |
32 | |
33 | /* Replacement type for __m128 since this file is included by ld.so, |
34 | which is compiled with -mno-sse. It must not change the alignment |
35 | of rtld_savespace_sse. */ |
36 | typedef struct |
37 | { |
38 | int i[4]; |
39 | } __128bits; |
40 | |
41 | |
42 | typedef struct |
43 | { |
44 | void *tcb; /* Pointer to the TCB. Not necessarily the |
45 | thread descriptor used by libpthread. */ |
46 | dtv_t *dtv; |
47 | void *self; /* Pointer to the thread descriptor. */ |
48 | int multiple_threads; |
49 | int gscope_flag; |
50 | uintptr_t sysinfo; |
51 | uintptr_t stack_guard; |
52 | uintptr_t pointer_guard; |
53 | unsigned long int vgetcpu_cache[2]; |
54 | /* Bit 0: X86_FEATURE_1_IBT. |
55 | Bit 1: X86_FEATURE_1_SHSTK. |
56 | */ |
57 | unsigned int feature_1; |
58 | int __glibc_unused1; |
59 | /* Reservation of some values for the TM ABI. */ |
60 | void *__private_tm[4]; |
61 | /* GCC split stack support. */ |
62 | void *__private_ss; |
63 | /* The lowest address of shadow stack, */ |
64 | unsigned long long int ssp_base; |
65 | /* Must be kept even if it is no longer used by glibc since programs, |
66 | like AddressSanitizer, depend on the size of tcbhead_t. */ |
67 | __128bits __glibc_unused2[8][4] __attribute__ ((aligned (32))); |
68 | |
69 | void *__padding[8]; |
70 | } tcbhead_t; |
71 | |
72 | # ifdef __ILP32__ |
73 | /* morestack.S in libgcc uses offset 0x40 to access __private_ss, */ |
74 | _Static_assert (offsetof (tcbhead_t, __private_ss) == 0x40, |
75 | "offset of __private_ss != 0x40" ); |
76 | /* NB: ssp_base used to be "long int __glibc_reserved2", which was |
77 | changed from 32 bits to 64 bits. Make sure that the offset of the |
78 | next field, __glibc_unused2, is unchanged. */ |
79 | _Static_assert (offsetof (tcbhead_t, __glibc_unused2) == 0x60, |
80 | "offset of __glibc_unused2 != 0x60" ); |
81 | # else |
82 | /* morestack.S in libgcc uses offset 0x70 to access __private_ss, */ |
83 | _Static_assert (offsetof (tcbhead_t, __private_ss) == 0x70, |
84 | "offset of __private_ss != 0x70" ); |
85 | _Static_assert (offsetof (tcbhead_t, __glibc_unused2) == 0x80, |
86 | "offset of __glibc_unused2 != 0x80" ); |
87 | # endif |
88 | |
89 | #else /* __ASSEMBLER__ */ |
90 | # include <tcb-offsets.h> |
91 | #endif |
92 | |
93 | |
94 | /* Alignment requirement for the stack. */ |
95 | #define STACK_ALIGN 16 |
96 | |
97 | |
98 | #ifndef __ASSEMBLER__ |
99 | /* Get system call information. */ |
100 | # include <sysdep.h> |
101 | |
102 | #ifndef LOCK_PREFIX |
103 | # ifdef UP |
104 | # define LOCK_PREFIX /* nothing */ |
105 | # else |
106 | # define LOCK_PREFIX "lock;" |
107 | # endif |
108 | #endif |
109 | |
110 | /* This is the size of the initial TCB. Can't be just sizeof (tcbhead_t), |
111 | because NPTL getpid, __libc_alloca_cutoff etc. need (almost) the whole |
112 | struct pthread even when not linked with -lpthread. */ |
113 | # define TLS_INIT_TCB_SIZE sizeof (struct pthread) |
114 | |
115 | /* Alignment requirements for the initial TCB. */ |
116 | # define TLS_INIT_TCB_ALIGN __alignof__ (struct pthread) |
117 | |
118 | /* This is the size of the TCB. */ |
119 | # define TLS_TCB_SIZE sizeof (struct pthread) |
120 | |
121 | /* Alignment requirements for the TCB. */ |
122 | # define TLS_TCB_ALIGN __alignof__ (struct pthread) |
123 | |
124 | /* The TCB can have any size and the memory following the address the |
125 | thread pointer points to is unspecified. Allocate the TCB there. */ |
126 | # define TLS_TCB_AT_TP 1 |
127 | # define TLS_DTV_AT_TP 0 |
128 | |
129 | /* Get the thread descriptor definition. */ |
130 | # include <nptl/descr.h> |
131 | |
132 | |
133 | /* Install the dtv pointer. The pointer passed is to the element with |
134 | index -1 which contain the length. */ |
135 | # define INSTALL_DTV(descr, dtvp) \ |
136 | ((tcbhead_t *) (descr))->dtv = (dtvp) + 1 |
137 | |
138 | /* Install new dtv for current thread. */ |
139 | # define INSTALL_NEW_DTV(dtvp) \ |
140 | ({ struct pthread *__pd; \ |
141 | THREAD_SETMEM (__pd, header.dtv, (dtvp)); }) |
142 | |
143 | /* Return dtv of given thread descriptor. */ |
144 | # define GET_DTV(descr) \ |
145 | (((tcbhead_t *) (descr))->dtv) |
146 | |
147 | |
148 | /* Code to initially initialize the thread pointer. This might need |
149 | special attention since 'errno' is not yet available and if the |
150 | operation can cause a failure 'errno' must not be touched. |
151 | |
152 | We have to make the syscall for both uses of the macro since the |
153 | address might be (and probably is) different. */ |
154 | # define TLS_INIT_TP(thrdescr) \ |
155 | ({ void *_thrdescr = (thrdescr); \ |
156 | tcbhead_t *_head = _thrdescr; \ |
157 | int _result; \ |
158 | \ |
159 | _head->tcb = _thrdescr; \ |
160 | /* For now the thread descriptor is at the same address. */ \ |
161 | _head->self = _thrdescr; \ |
162 | \ |
163 | /* It is a simple syscall to set the %fs value for the thread. */ \ |
164 | asm volatile ("syscall" \ |
165 | : "=a" (_result) \ |
166 | : "0" ((unsigned long int) __NR_arch_prctl), \ |
167 | "D" ((unsigned long int) ARCH_SET_FS), \ |
168 | "S" (_thrdescr) \ |
169 | : "memory", "cc", "r11", "cx"); \ |
170 | \ |
171 | _result ? "cannot set %fs base address for thread-local storage" : 0; \ |
172 | }) |
173 | |
174 | # define TLS_DEFINE_INIT_TP(tp, pd) void *tp = (pd) |
175 | |
176 | |
177 | /* Return the address of the dtv for the current thread. */ |
178 | # define THREAD_DTV() \ |
179 | ({ struct pthread *__pd; \ |
180 | THREAD_GETMEM (__pd, header.dtv); }) |
181 | |
182 | |
183 | /* Return the thread descriptor for the current thread. |
184 | |
185 | The contained asm must *not* be marked volatile since otherwise |
186 | assignments like |
187 | pthread_descr self = thread_self(); |
188 | do not get optimized away. */ |
189 | # define THREAD_SELF \ |
190 | ({ struct pthread *__self; \ |
191 | asm ("mov %%fs:%c1,%0" : "=r" (__self) \ |
192 | : "i" (offsetof (struct pthread, header.self))); \ |
193 | __self;}) |
194 | |
195 | /* Magic for libthread_db to know how to do THREAD_SELF. */ |
196 | # define DB_THREAD_SELF_INCLUDE <sys/reg.h> /* For the FS constant. */ |
197 | # define DB_THREAD_SELF CONST_THREAD_AREA (64, FS) |
198 | |
199 | /* Read member of the thread descriptor directly. */ |
200 | # define THREAD_GETMEM(descr, member) \ |
201 | ({ __typeof (descr->member) __value; \ |
202 | if (sizeof (__value) == 1) \ |
203 | asm volatile ("movb %%fs:%P2,%b0" \ |
204 | : "=q" (__value) \ |
205 | : "0" (0), "i" (offsetof (struct pthread, member))); \ |
206 | else if (sizeof (__value) == 4) \ |
207 | asm volatile ("movl %%fs:%P1,%0" \ |
208 | : "=r" (__value) \ |
209 | : "i" (offsetof (struct pthread, member))); \ |
210 | else \ |
211 | { \ |
212 | if (sizeof (__value) != 8) \ |
213 | /* There should not be any value with a size other than 1, \ |
214 | 4 or 8. */ \ |
215 | abort (); \ |
216 | \ |
217 | asm volatile ("movq %%fs:%P1,%q0" \ |
218 | : "=r" (__value) \ |
219 | : "i" (offsetof (struct pthread, member))); \ |
220 | } \ |
221 | __value; }) |
222 | |
223 | |
224 | /* Same as THREAD_GETMEM, but the member offset can be non-constant. */ |
225 | # define THREAD_GETMEM_NC(descr, member, idx) \ |
226 | ({ __typeof (descr->member[0]) __value; \ |
227 | if (sizeof (__value) == 1) \ |
228 | asm volatile ("movb %%fs:%P2(%q3),%b0" \ |
229 | : "=q" (__value) \ |
230 | : "0" (0), "i" (offsetof (struct pthread, member[0])), \ |
231 | "r" (idx)); \ |
232 | else if (sizeof (__value) == 4) \ |
233 | asm volatile ("movl %%fs:%P1(,%q2,4),%0" \ |
234 | : "=r" (__value) \ |
235 | : "i" (offsetof (struct pthread, member[0])), "r" (idx));\ |
236 | else \ |
237 | { \ |
238 | if (sizeof (__value) != 8) \ |
239 | /* There should not be any value with a size other than 1, \ |
240 | 4 or 8. */ \ |
241 | abort (); \ |
242 | \ |
243 | asm volatile ("movq %%fs:%P1(,%q2,8),%q0" \ |
244 | : "=r" (__value) \ |
245 | : "i" (offsetof (struct pthread, member[0])), \ |
246 | "r" (idx)); \ |
247 | } \ |
248 | __value; }) |
249 | |
250 | |
251 | /* Loading addresses of objects on x86-64 needs to be treated special |
252 | when generating PIC code. */ |
253 | #ifdef __pic__ |
254 | # define IMM_MODE "nr" |
255 | #else |
256 | # define IMM_MODE "ir" |
257 | #endif |
258 | |
259 | |
260 | /* Set member of the thread descriptor directly. */ |
261 | # define THREAD_SETMEM(descr, member, value) \ |
262 | ({ if (sizeof (descr->member) == 1) \ |
263 | asm volatile ("movb %b0,%%fs:%P1" : \ |
264 | : "iq" (value), \ |
265 | "i" (offsetof (struct pthread, member))); \ |
266 | else if (sizeof (descr->member) == 4) \ |
267 | asm volatile ("movl %0,%%fs:%P1" : \ |
268 | : IMM_MODE (value), \ |
269 | "i" (offsetof (struct pthread, member))); \ |
270 | else \ |
271 | { \ |
272 | if (sizeof (descr->member) != 8) \ |
273 | /* There should not be any value with a size other than 1, \ |
274 | 4 or 8. */ \ |
275 | abort (); \ |
276 | \ |
277 | asm volatile ("movq %q0,%%fs:%P1" : \ |
278 | : IMM_MODE ((uint64_t) cast_to_integer (value)), \ |
279 | "i" (offsetof (struct pthread, member))); \ |
280 | }}) |
281 | |
282 | |
283 | /* Same as THREAD_SETMEM, but the member offset can be non-constant. */ |
284 | # define THREAD_SETMEM_NC(descr, member, idx, value) \ |
285 | ({ if (sizeof (descr->member[0]) == 1) \ |
286 | asm volatile ("movb %b0,%%fs:%P1(%q2)" : \ |
287 | : "iq" (value), \ |
288 | "i" (offsetof (struct pthread, member[0])), \ |
289 | "r" (idx)); \ |
290 | else if (sizeof (descr->member[0]) == 4) \ |
291 | asm volatile ("movl %0,%%fs:%P1(,%q2,4)" : \ |
292 | : IMM_MODE (value), \ |
293 | "i" (offsetof (struct pthread, member[0])), \ |
294 | "r" (idx)); \ |
295 | else \ |
296 | { \ |
297 | if (sizeof (descr->member[0]) != 8) \ |
298 | /* There should not be any value with a size other than 1, \ |
299 | 4 or 8. */ \ |
300 | abort (); \ |
301 | \ |
302 | asm volatile ("movq %q0,%%fs:%P1(,%q2,8)" : \ |
303 | : IMM_MODE ((uint64_t) cast_to_integer (value)), \ |
304 | "i" (offsetof (struct pthread, member[0])), \ |
305 | "r" (idx)); \ |
306 | }}) |
307 | |
308 | |
309 | /* Atomic compare and exchange on TLS, returning old value. */ |
310 | # define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \ |
311 | ({ __typeof (descr->member) __ret; \ |
312 | __typeof (oldval) __old = (oldval); \ |
313 | if (sizeof (descr->member) == 4) \ |
314 | asm volatile (LOCK_PREFIX "cmpxchgl %2, %%fs:%P3" \ |
315 | : "=a" (__ret) \ |
316 | : "0" (__old), "r" (newval), \ |
317 | "i" (offsetof (struct pthread, member))); \ |
318 | else \ |
319 | /* Not necessary for other sizes in the moment. */ \ |
320 | abort (); \ |
321 | __ret; }) |
322 | |
323 | |
324 | /* Atomic logical and. */ |
325 | # define THREAD_ATOMIC_AND(descr, member, val) \ |
326 | (void) ({ if (sizeof ((descr)->member) == 4) \ |
327 | asm volatile (LOCK_PREFIX "andl %1, %%fs:%P0" \ |
328 | :: "i" (offsetof (struct pthread, member)), \ |
329 | "ir" (val)); \ |
330 | else \ |
331 | /* Not necessary for other sizes in the moment. */ \ |
332 | abort (); }) |
333 | |
334 | |
335 | /* Atomic set bit. */ |
336 | # define THREAD_ATOMIC_BIT_SET(descr, member, bit) \ |
337 | (void) ({ if (sizeof ((descr)->member) == 4) \ |
338 | asm volatile (LOCK_PREFIX "orl %1, %%fs:%P0" \ |
339 | :: "i" (offsetof (struct pthread, member)), \ |
340 | "ir" (1 << (bit))); \ |
341 | else \ |
342 | /* Not necessary for other sizes in the moment. */ \ |
343 | abort (); }) |
344 | |
345 | |
346 | /* Set the stack guard field in TCB head. */ |
347 | # define THREAD_SET_STACK_GUARD(value) \ |
348 | THREAD_SETMEM (THREAD_SELF, header.stack_guard, value) |
349 | # define THREAD_COPY_STACK_GUARD(descr) \ |
350 | ((descr)->header.stack_guard \ |
351 | = THREAD_GETMEM (THREAD_SELF, header.stack_guard)) |
352 | |
353 | |
354 | /* Set the pointer guard field in the TCB head. */ |
355 | # define THREAD_SET_POINTER_GUARD(value) \ |
356 | THREAD_SETMEM (THREAD_SELF, header.pointer_guard, value) |
357 | # define THREAD_COPY_POINTER_GUARD(descr) \ |
358 | ((descr)->header.pointer_guard \ |
359 | = THREAD_GETMEM (THREAD_SELF, header.pointer_guard)) |
360 | |
361 | |
362 | /* Get and set the global scope generation counter in the TCB head. */ |
363 | # define THREAD_GSCOPE_IN_TCB 1 |
364 | # define THREAD_GSCOPE_FLAG_UNUSED 0 |
365 | # define THREAD_GSCOPE_FLAG_USED 1 |
366 | # define THREAD_GSCOPE_FLAG_WAIT 2 |
367 | # define THREAD_GSCOPE_RESET_FLAG() \ |
368 | do \ |
369 | { int __res; \ |
370 | asm volatile ("xchgl %0, %%fs:%P1" \ |
371 | : "=r" (__res) \ |
372 | : "i" (offsetof (struct pthread, header.gscope_flag)), \ |
373 | "0" (THREAD_GSCOPE_FLAG_UNUSED)); \ |
374 | if (__res == THREAD_GSCOPE_FLAG_WAIT) \ |
375 | lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1, LLL_PRIVATE); \ |
376 | } \ |
377 | while (0) |
378 | # define THREAD_GSCOPE_SET_FLAG() \ |
379 | THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED) |
380 | # define THREAD_GSCOPE_WAIT() \ |
381 | GL(dl_wait_lookup_done) () |
382 | |
383 | #endif /* __ASSEMBLER__ */ |
384 | |
385 | #endif /* tls.h */ |
386 | |