1 | /* Thread-local storage handling in the ELF dynamic linker. Generic version. |
2 | Copyright (C) 2002-2020 Free Software Foundation, Inc. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <https://www.gnu.org/licenses/>. */ |
18 | |
19 | #include <assert.h> |
20 | #include <errno.h> |
21 | #include <libintl.h> |
22 | #include <signal.h> |
23 | #include <stdlib.h> |
24 | #include <unistd.h> |
25 | #include <sys/param.h> |
26 | #include <atomic.h> |
27 | |
28 | #include <tls.h> |
29 | #include <dl-tls.h> |
30 | #include <ldsodefs.h> |
31 | |
32 | /* Amount of excess space to allocate in the static TLS area |
33 | to allow dynamic loading of modules defining IE-model TLS data. */ |
34 | #define TLS_STATIC_SURPLUS 64 + DL_NNS * 100 |
35 | |
36 | |
37 | /* Out-of-memory handler. */ |
38 | static void |
39 | __attribute__ ((__noreturn__)) |
40 | oom (void) |
41 | { |
42 | _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n" ); |
43 | } |
44 | |
45 | |
46 | size_t |
47 | _dl_next_tls_modid (void) |
48 | { |
49 | size_t result; |
50 | |
51 | if (__builtin_expect (GL(dl_tls_dtv_gaps), false)) |
52 | { |
53 | size_t disp = 0; |
54 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); |
55 | |
56 | /* Note that this branch will never be executed during program |
57 | start since there are no gaps at that time. Therefore it |
58 | does not matter that the dl_tls_dtv_slotinfo is not allocated |
59 | yet when the function is called for the first times. |
60 | |
61 | NB: the offset +1 is due to the fact that DTV[0] is used |
62 | for something else. */ |
63 | result = GL(dl_tls_static_nelem) + 1; |
64 | if (result <= GL(dl_tls_max_dtv_idx)) |
65 | do |
66 | { |
67 | while (result - disp < runp->len) |
68 | { |
69 | if (runp->slotinfo[result - disp].map == NULL) |
70 | break; |
71 | |
72 | ++result; |
73 | assert (result <= GL(dl_tls_max_dtv_idx) + 1); |
74 | } |
75 | |
76 | if (result - disp < runp->len) |
77 | break; |
78 | |
79 | disp += runp->len; |
80 | } |
81 | while ((runp = runp->next) != NULL); |
82 | |
83 | if (result > GL(dl_tls_max_dtv_idx)) |
84 | { |
85 | /* The new index must indeed be exactly one higher than the |
86 | previous high. */ |
87 | assert (result == GL(dl_tls_max_dtv_idx) + 1); |
88 | /* There is no gap anymore. */ |
89 | GL(dl_tls_dtv_gaps) = false; |
90 | |
91 | goto nogaps; |
92 | } |
93 | } |
94 | else |
95 | { |
96 | /* No gaps, allocate a new entry. */ |
97 | nogaps: |
98 | |
99 | result = ++GL(dl_tls_max_dtv_idx); |
100 | } |
101 | |
102 | return result; |
103 | } |
104 | |
105 | |
106 | size_t |
107 | _dl_count_modids (void) |
108 | { |
109 | /* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where |
110 | we fail to load a module and unload it leaving a gap. If we don't |
111 | have gaps then the number of modids is the current maximum so |
112 | return that. */ |
113 | if (__glibc_likely (!GL(dl_tls_dtv_gaps))) |
114 | return GL(dl_tls_max_dtv_idx); |
115 | |
116 | /* We have gaps and are forced to count the non-NULL entries. */ |
117 | size_t n = 0; |
118 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); |
119 | while (runp != NULL) |
120 | { |
121 | for (size_t i = 0; i < runp->len; ++i) |
122 | if (runp->slotinfo[i].map != NULL) |
123 | ++n; |
124 | |
125 | runp = runp->next; |
126 | } |
127 | |
128 | return n; |
129 | } |
130 | |
131 | |
132 | #ifdef SHARED |
133 | void |
134 | _dl_determine_tlsoffset (void) |
135 | { |
136 | size_t max_align = TLS_TCB_ALIGN; |
137 | size_t freetop = 0; |
138 | size_t freebottom = 0; |
139 | |
140 | /* The first element of the dtv slot info list is allocated. */ |
141 | assert (GL(dl_tls_dtv_slotinfo_list) != NULL); |
142 | /* There is at this point only one element in the |
143 | dl_tls_dtv_slotinfo_list list. */ |
144 | assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL); |
145 | |
146 | struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo; |
147 | |
148 | /* Determining the offset of the various parts of the static TLS |
149 | block has several dependencies. In addition we have to work |
150 | around bugs in some toolchains. |
151 | |
152 | Each TLS block from the objects available at link time has a size |
153 | and an alignment requirement. The GNU ld computes the alignment |
154 | requirements for the data at the positions *in the file*, though. |
155 | I.e, it is not simply possible to allocate a block with the size |
156 | of the TLS program header entry. The data is layed out assuming |
157 | that the first byte of the TLS block fulfills |
158 | |
159 | p_vaddr mod p_align == &TLS_BLOCK mod p_align |
160 | |
161 | This means we have to add artificial padding at the beginning of |
162 | the TLS block. These bytes are never used for the TLS data in |
163 | this module but the first byte allocated must be aligned |
164 | according to mod p_align == 0 so that the first byte of the TLS |
165 | block is aligned according to p_vaddr mod p_align. This is ugly |
166 | and the linker can help by computing the offsets in the TLS block |
167 | assuming the first byte of the TLS block is aligned according to |
168 | p_align. |
169 | |
170 | The extra space which might be allocated before the first byte of |
171 | the TLS block need not go unused. The code below tries to use |
172 | that memory for the next TLS block. This can work if the total |
173 | memory requirement for the next TLS block is smaller than the |
174 | gap. */ |
175 | |
176 | #if TLS_TCB_AT_TP |
177 | /* We simply start with zero. */ |
178 | size_t offset = 0; |
179 | |
180 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) |
181 | { |
182 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); |
183 | |
184 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset |
185 | & (slotinfo[cnt].map->l_tls_align - 1)); |
186 | size_t off; |
187 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); |
188 | |
189 | if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize) |
190 | { |
191 | off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize |
192 | - firstbyte, slotinfo[cnt].map->l_tls_align) |
193 | + firstbyte; |
194 | if (off <= freebottom) |
195 | { |
196 | freetop = off; |
197 | |
198 | /* XXX For some architectures we perhaps should store the |
199 | negative offset. */ |
200 | slotinfo[cnt].map->l_tls_offset = off; |
201 | continue; |
202 | } |
203 | } |
204 | |
205 | off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte, |
206 | slotinfo[cnt].map->l_tls_align) + firstbyte; |
207 | if (off > offset + slotinfo[cnt].map->l_tls_blocksize |
208 | + (freebottom - freetop)) |
209 | { |
210 | freetop = offset; |
211 | freebottom = off - slotinfo[cnt].map->l_tls_blocksize; |
212 | } |
213 | offset = off; |
214 | |
215 | /* XXX For some architectures we perhaps should store the |
216 | negative offset. */ |
217 | slotinfo[cnt].map->l_tls_offset = off; |
218 | } |
219 | |
220 | GL(dl_tls_static_used) = offset; |
221 | GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align) |
222 | + TLS_TCB_SIZE); |
223 | #elif TLS_DTV_AT_TP |
224 | /* The TLS blocks start right after the TCB. */ |
225 | size_t offset = TLS_TCB_SIZE; |
226 | |
227 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) |
228 | { |
229 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); |
230 | |
231 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset |
232 | & (slotinfo[cnt].map->l_tls_align - 1)); |
233 | size_t off; |
234 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); |
235 | |
236 | if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom) |
237 | { |
238 | off = roundup (freebottom, slotinfo[cnt].map->l_tls_align); |
239 | if (off - freebottom < firstbyte) |
240 | off += slotinfo[cnt].map->l_tls_align; |
241 | if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop) |
242 | { |
243 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; |
244 | freebottom = (off + slotinfo[cnt].map->l_tls_blocksize |
245 | - firstbyte); |
246 | continue; |
247 | } |
248 | } |
249 | |
250 | off = roundup (offset, slotinfo[cnt].map->l_tls_align); |
251 | if (off - offset < firstbyte) |
252 | off += slotinfo[cnt].map->l_tls_align; |
253 | |
254 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; |
255 | if (off - firstbyte - offset > freetop - freebottom) |
256 | { |
257 | freebottom = offset; |
258 | freetop = off - firstbyte; |
259 | } |
260 | |
261 | offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte; |
262 | } |
263 | |
264 | GL(dl_tls_static_used) = offset; |
265 | GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS, |
266 | TLS_TCB_ALIGN); |
267 | #else |
268 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" |
269 | #endif |
270 | |
271 | /* The alignment requirement for the static TLS block. */ |
272 | GL(dl_tls_static_align) = max_align; |
273 | } |
274 | #endif /* SHARED */ |
275 | |
276 | static void * |
277 | allocate_dtv (void *result) |
278 | { |
279 | dtv_t *dtv; |
280 | size_t dtv_length; |
281 | |
282 | /* We allocate a few more elements in the dtv than are needed for the |
283 | initial set of modules. This should avoid in most cases expansions |
284 | of the dtv. */ |
285 | dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS; |
286 | dtv = calloc (dtv_length + 2, sizeof (dtv_t)); |
287 | if (dtv != NULL) |
288 | { |
289 | /* This is the initial length of the dtv. */ |
290 | dtv[0].counter = dtv_length; |
291 | |
292 | /* The rest of the dtv (including the generation counter) is |
293 | Initialize with zero to indicate nothing there. */ |
294 | |
295 | /* Add the dtv to the thread data structures. */ |
296 | INSTALL_DTV (result, dtv); |
297 | } |
298 | else |
299 | result = NULL; |
300 | |
301 | return result; |
302 | } |
303 | |
304 | |
305 | /* Get size and alignment requirements of the static TLS block. */ |
306 | void |
307 | _dl_get_tls_static_info (size_t *sizep, size_t *alignp) |
308 | { |
309 | *sizep = GL(dl_tls_static_size); |
310 | *alignp = GL(dl_tls_static_align); |
311 | } |
312 | |
313 | /* Derive the location of the pointer to the start of the original |
314 | allocation (before alignment) from the pointer to the TCB. */ |
315 | static inline void ** |
316 | tcb_to_pointer_to_free_location (void *tcb) |
317 | { |
318 | #if TLS_TCB_AT_TP |
319 | /* The TCB follows the TLS blocks, and the pointer to the front |
320 | follows the TCB. */ |
321 | void **original_pointer_location = tcb + TLS_TCB_SIZE; |
322 | #elif TLS_DTV_AT_TP |
323 | /* The TCB comes first, preceded by the pre-TCB, and the pointer is |
324 | before that. */ |
325 | void **original_pointer_location = tcb - TLS_PRE_TCB_SIZE - sizeof (void *); |
326 | #endif |
327 | return original_pointer_location; |
328 | } |
329 | |
330 | void * |
331 | _dl_allocate_tls_storage (void) |
332 | { |
333 | void *result; |
334 | size_t size = GL(dl_tls_static_size); |
335 | |
336 | #if TLS_DTV_AT_TP |
337 | /* Memory layout is: |
338 | [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ] |
339 | ^ This should be returned. */ |
340 | size += TLS_PRE_TCB_SIZE; |
341 | #endif |
342 | |
343 | /* Perform the allocation. Reserve space for the required alignment |
344 | and the pointer to the original allocation. */ |
345 | size_t alignment = GL(dl_tls_static_align); |
346 | void *allocated = malloc (size + alignment + sizeof (void *)); |
347 | if (__glibc_unlikely (allocated == NULL)) |
348 | return NULL; |
349 | |
350 | /* Perform alignment and allocate the DTV. */ |
351 | #if TLS_TCB_AT_TP |
352 | /* The TCB follows the TLS blocks, which determine the alignment. |
353 | (TCB alignment requirements have been taken into account when |
354 | calculating GL(dl_tls_static_align).) */ |
355 | void *aligned = (void *) roundup ((uintptr_t) allocated, alignment); |
356 | result = aligned + size - TLS_TCB_SIZE; |
357 | |
358 | /* Clear the TCB data structure. We can't ask the caller (i.e. |
359 | libpthread) to do it, because we will initialize the DTV et al. */ |
360 | memset (result, '\0', TLS_TCB_SIZE); |
361 | #elif TLS_DTV_AT_TP |
362 | /* Pre-TCB and TCB come before the TLS blocks. The layout computed |
363 | in _dl_determine_tlsoffset assumes that the TCB is aligned to the |
364 | TLS block alignment, and not just the TLS blocks after it. This |
365 | can leave an unused alignment gap between the TCB and the TLS |
366 | blocks. */ |
367 | result = (void *) roundup |
368 | (sizeof (void *) + TLS_PRE_TCB_SIZE + (uintptr_t) allocated, |
369 | alignment); |
370 | |
371 | /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before |
372 | it. We can't ask the caller (i.e. libpthread) to do it, because |
373 | we will initialize the DTV et al. */ |
374 | memset (result - TLS_PRE_TCB_SIZE, '\0', TLS_PRE_TCB_SIZE + TLS_TCB_SIZE); |
375 | #endif |
376 | |
377 | /* Record the value of the original pointer for later |
378 | deallocation. */ |
379 | *tcb_to_pointer_to_free_location (result) = allocated; |
380 | |
381 | result = allocate_dtv (result); |
382 | if (result == NULL) |
383 | free (allocated); |
384 | return result; |
385 | } |
386 | |
387 | |
388 | #ifndef SHARED |
389 | extern dtv_t _dl_static_dtv[]; |
390 | # define _dl_initial_dtv (&_dl_static_dtv[1]) |
391 | #endif |
392 | |
393 | static dtv_t * |
394 | _dl_resize_dtv (dtv_t *dtv) |
395 | { |
396 | /* Resize the dtv. */ |
397 | dtv_t *newp; |
398 | /* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by |
399 | other threads concurrently. */ |
400 | size_t newsize |
401 | = atomic_load_acquire (&GL(dl_tls_max_dtv_idx)) + DTV_SURPLUS; |
402 | size_t oldsize = dtv[-1].counter; |
403 | |
404 | if (dtv == GL(dl_initial_dtv)) |
405 | { |
406 | /* This is the initial dtv that was either statically allocated in |
407 | __libc_setup_tls or allocated during rtld startup using the |
408 | dl-minimal.c malloc instead of the real malloc. We can't free |
409 | it, we have to abandon the old storage. */ |
410 | |
411 | newp = malloc ((2 + newsize) * sizeof (dtv_t)); |
412 | if (newp == NULL) |
413 | oom (); |
414 | memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t)); |
415 | } |
416 | else |
417 | { |
418 | newp = realloc (&dtv[-1], |
419 | (2 + newsize) * sizeof (dtv_t)); |
420 | if (newp == NULL) |
421 | oom (); |
422 | } |
423 | |
424 | newp[0].counter = newsize; |
425 | |
426 | /* Clear the newly allocated part. */ |
427 | memset (newp + 2 + oldsize, '\0', |
428 | (newsize - oldsize) * sizeof (dtv_t)); |
429 | |
430 | /* Return the generation counter. */ |
431 | return &newp[1]; |
432 | } |
433 | |
434 | |
435 | void * |
436 | _dl_allocate_tls_init (void *result) |
437 | { |
438 | if (result == NULL) |
439 | /* The memory allocation failed. */ |
440 | return NULL; |
441 | |
442 | dtv_t *dtv = GET_DTV (result); |
443 | struct dtv_slotinfo_list *listp; |
444 | size_t total = 0; |
445 | size_t maxgen = 0; |
446 | |
447 | /* Check if the current dtv is big enough. */ |
448 | if (dtv[-1].counter < GL(dl_tls_max_dtv_idx)) |
449 | { |
450 | /* Resize the dtv. */ |
451 | dtv = _dl_resize_dtv (dtv); |
452 | |
453 | /* Install this new dtv in the thread data structures. */ |
454 | INSTALL_DTV (result, &dtv[-1]); |
455 | } |
456 | |
457 | /* We have to prepare the dtv for all currently loaded modules using |
458 | TLS. For those which are dynamically loaded we add the values |
459 | indicating deferred allocation. */ |
460 | listp = GL(dl_tls_dtv_slotinfo_list); |
461 | while (1) |
462 | { |
463 | size_t cnt; |
464 | |
465 | for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) |
466 | { |
467 | struct link_map *map; |
468 | void *dest; |
469 | |
470 | /* Check for the total number of used slots. */ |
471 | if (total + cnt > GL(dl_tls_max_dtv_idx)) |
472 | break; |
473 | |
474 | map = listp->slotinfo[cnt].map; |
475 | if (map == NULL) |
476 | /* Unused entry. */ |
477 | continue; |
478 | |
479 | /* Keep track of the maximum generation number. This might |
480 | not be the generation counter. */ |
481 | assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation)); |
482 | maxgen = MAX (maxgen, listp->slotinfo[cnt].gen); |
483 | |
484 | dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED; |
485 | dtv[map->l_tls_modid].pointer.to_free = NULL; |
486 | |
487 | if (map->l_tls_offset == NO_TLS_OFFSET |
488 | || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET) |
489 | continue; |
490 | |
491 | assert (map->l_tls_modid == total + cnt); |
492 | assert (map->l_tls_blocksize >= map->l_tls_initimage_size); |
493 | #if TLS_TCB_AT_TP |
494 | assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize); |
495 | dest = (char *) result - map->l_tls_offset; |
496 | #elif TLS_DTV_AT_TP |
497 | dest = (char *) result + map->l_tls_offset; |
498 | #else |
499 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" |
500 | #endif |
501 | |
502 | /* Set up the DTV entry. The simplified __tls_get_addr that |
503 | some platforms use in static programs requires it. */ |
504 | dtv[map->l_tls_modid].pointer.val = dest; |
505 | |
506 | /* Copy the initialization image and clear the BSS part. */ |
507 | memset (__mempcpy (dest, map->l_tls_initimage, |
508 | map->l_tls_initimage_size), '\0', |
509 | map->l_tls_blocksize - map->l_tls_initimage_size); |
510 | } |
511 | |
512 | total += cnt; |
513 | if (total >= GL(dl_tls_max_dtv_idx)) |
514 | break; |
515 | |
516 | listp = listp->next; |
517 | assert (listp != NULL); |
518 | } |
519 | |
520 | /* The DTV version is up-to-date now. */ |
521 | dtv[0].counter = maxgen; |
522 | |
523 | return result; |
524 | } |
525 | rtld_hidden_def (_dl_allocate_tls_init) |
526 | |
527 | void * |
528 | _dl_allocate_tls (void *mem) |
529 | { |
530 | return _dl_allocate_tls_init (mem == NULL |
531 | ? _dl_allocate_tls_storage () |
532 | : allocate_dtv (mem)); |
533 | } |
534 | rtld_hidden_def (_dl_allocate_tls) |
535 | |
536 | |
537 | void |
538 | _dl_deallocate_tls (void *tcb, bool dealloc_tcb) |
539 | { |
540 | dtv_t *dtv = GET_DTV (tcb); |
541 | |
542 | /* We need to free the memory allocated for non-static TLS. */ |
543 | for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt) |
544 | free (dtv[1 + cnt].pointer.to_free); |
545 | |
546 | /* The array starts with dtv[-1]. */ |
547 | if (dtv != GL(dl_initial_dtv)) |
548 | free (dtv - 1); |
549 | |
550 | if (dealloc_tcb) |
551 | free (*tcb_to_pointer_to_free_location (tcb)); |
552 | } |
553 | rtld_hidden_def (_dl_deallocate_tls) |
554 | |
555 | |
556 | #ifdef SHARED |
557 | /* The __tls_get_addr function has two basic forms which differ in the |
558 | arguments. The IA-64 form takes two parameters, the module ID and |
559 | offset. The form used, among others, on IA-32 takes a reference to |
560 | a special structure which contain the same information. The second |
561 | form seems to be more often used (in the moment) so we default to |
562 | it. Users of the IA-64 form have to provide adequate definitions |
563 | of the following macros. */ |
564 | # ifndef GET_ADDR_ARGS |
565 | # define GET_ADDR_ARGS tls_index *ti |
566 | # define GET_ADDR_PARAM ti |
567 | # endif |
568 | # ifndef GET_ADDR_MODULE |
569 | # define GET_ADDR_MODULE ti->ti_module |
570 | # endif |
571 | # ifndef GET_ADDR_OFFSET |
572 | # define GET_ADDR_OFFSET ti->ti_offset |
573 | # endif |
574 | |
575 | /* Allocate one DTV entry. */ |
576 | static struct dtv_pointer |
577 | allocate_dtv_entry (size_t alignment, size_t size) |
578 | { |
579 | if (powerof2 (alignment) && alignment <= _Alignof (max_align_t)) |
580 | { |
581 | /* The alignment is supported by malloc. */ |
582 | void *ptr = malloc (size); |
583 | return (struct dtv_pointer) { ptr, ptr }; |
584 | } |
585 | |
586 | /* Emulate memalign to by manually aligning a pointer returned by |
587 | malloc. First compute the size with an overflow check. */ |
588 | size_t alloc_size = size + alignment; |
589 | if (alloc_size < size) |
590 | return (struct dtv_pointer) {}; |
591 | |
592 | /* Perform the allocation. This is the pointer we need to free |
593 | later. */ |
594 | void *start = malloc (alloc_size); |
595 | if (start == NULL) |
596 | return (struct dtv_pointer) {}; |
597 | |
598 | /* Find the aligned position within the larger allocation. */ |
599 | void *aligned = (void *) roundup ((uintptr_t) start, alignment); |
600 | |
601 | return (struct dtv_pointer) { .val = aligned, .to_free = start }; |
602 | } |
603 | |
604 | static struct dtv_pointer |
605 | allocate_and_init (struct link_map *map) |
606 | { |
607 | struct dtv_pointer result = allocate_dtv_entry |
608 | (map->l_tls_align, map->l_tls_blocksize); |
609 | if (result.val == NULL) |
610 | oom (); |
611 | |
612 | /* Initialize the memory. */ |
613 | memset (__mempcpy (result.val, map->l_tls_initimage, |
614 | map->l_tls_initimage_size), |
615 | '\0', map->l_tls_blocksize - map->l_tls_initimage_size); |
616 | |
617 | return result; |
618 | } |
619 | |
620 | |
621 | struct link_map * |
622 | _dl_update_slotinfo (unsigned long int req_modid) |
623 | { |
624 | struct link_map *the_map = NULL; |
625 | dtv_t *dtv = THREAD_DTV (); |
626 | |
627 | /* The global dl_tls_dtv_slotinfo array contains for each module |
628 | index the generation counter current when the entry was created. |
629 | This array never shrinks so that all module indices which were |
630 | valid at some time can be used to access it. Before the first |
631 | use of a new module index in this function the array was extended |
632 | appropriately. Access also does not have to be guarded against |
633 | modifications of the array. It is assumed that pointer-size |
634 | values can be read atomically even in SMP environments. It is |
635 | possible that other threads at the same time dynamically load |
636 | code and therefore add to the slotinfo list. This is a problem |
637 | since we must not pick up any information about incomplete work. |
638 | The solution to this is to ignore all dtv slots which were |
639 | created after the one we are currently interested. We know that |
640 | dynamic loading for this module is completed and this is the last |
641 | load operation we know finished. */ |
642 | unsigned long int idx = req_modid; |
643 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
644 | |
645 | while (idx >= listp->len) |
646 | { |
647 | idx -= listp->len; |
648 | listp = listp->next; |
649 | } |
650 | |
651 | if (dtv[0].counter < listp->slotinfo[idx].gen) |
652 | { |
653 | /* The generation counter for the slot is higher than what the |
654 | current dtv implements. We have to update the whole dtv but |
655 | only those entries with a generation counter <= the one for |
656 | the entry we need. */ |
657 | size_t new_gen = listp->slotinfo[idx].gen; |
658 | size_t total = 0; |
659 | |
660 | /* We have to look through the entire dtv slotinfo list. */ |
661 | listp = GL(dl_tls_dtv_slotinfo_list); |
662 | do |
663 | { |
664 | for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) |
665 | { |
666 | size_t gen = listp->slotinfo[cnt].gen; |
667 | |
668 | if (gen > new_gen) |
669 | /* This is a slot for a generation younger than the |
670 | one we are handling now. It might be incompletely |
671 | set up so ignore it. */ |
672 | continue; |
673 | |
674 | /* If the entry is older than the current dtv layout we |
675 | know we don't have to handle it. */ |
676 | if (gen <= dtv[0].counter) |
677 | continue; |
678 | |
679 | /* If there is no map this means the entry is empty. */ |
680 | struct link_map *map = listp->slotinfo[cnt].map; |
681 | if (map == NULL) |
682 | { |
683 | if (dtv[-1].counter >= total + cnt) |
684 | { |
685 | /* If this modid was used at some point the memory |
686 | might still be allocated. */ |
687 | free (dtv[total + cnt].pointer.to_free); |
688 | dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED; |
689 | dtv[total + cnt].pointer.to_free = NULL; |
690 | } |
691 | |
692 | continue; |
693 | } |
694 | |
695 | /* Check whether the current dtv array is large enough. */ |
696 | size_t modid = map->l_tls_modid; |
697 | assert (total + cnt == modid); |
698 | if (dtv[-1].counter < modid) |
699 | { |
700 | /* Resize the dtv. */ |
701 | dtv = _dl_resize_dtv (dtv); |
702 | |
703 | assert (modid <= dtv[-1].counter); |
704 | |
705 | /* Install this new dtv in the thread data |
706 | structures. */ |
707 | INSTALL_NEW_DTV (dtv); |
708 | } |
709 | |
710 | /* If there is currently memory allocate for this |
711 | dtv entry free it. */ |
712 | /* XXX Ideally we will at some point create a memory |
713 | pool. */ |
714 | free (dtv[modid].pointer.to_free); |
715 | dtv[modid].pointer.val = TLS_DTV_UNALLOCATED; |
716 | dtv[modid].pointer.to_free = NULL; |
717 | |
718 | if (modid == req_modid) |
719 | the_map = map; |
720 | } |
721 | |
722 | total += listp->len; |
723 | } |
724 | while ((listp = listp->next) != NULL); |
725 | |
726 | /* This will be the new maximum generation counter. */ |
727 | dtv[0].counter = new_gen; |
728 | } |
729 | |
730 | return the_map; |
731 | } |
732 | |
733 | |
734 | static void * |
735 | __attribute_noinline__ |
736 | tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map) |
737 | { |
738 | /* The allocation was deferred. Do it now. */ |
739 | if (the_map == NULL) |
740 | { |
741 | /* Find the link map for this module. */ |
742 | size_t idx = GET_ADDR_MODULE; |
743 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
744 | |
745 | while (idx >= listp->len) |
746 | { |
747 | idx -= listp->len; |
748 | listp = listp->next; |
749 | } |
750 | |
751 | the_map = listp->slotinfo[idx].map; |
752 | } |
753 | |
754 | /* Make sure that, if a dlopen running in parallel forces the |
755 | variable into static storage, we'll wait until the address in the |
756 | static TLS block is set up, and use that. If we're undecided |
757 | yet, make sure we make the decision holding the lock as well. */ |
758 | if (__glibc_unlikely (the_map->l_tls_offset |
759 | != FORCED_DYNAMIC_TLS_OFFSET)) |
760 | { |
761 | __rtld_lock_lock_recursive (GL(dl_load_lock)); |
762 | if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET)) |
763 | { |
764 | the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET; |
765 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); |
766 | } |
767 | else if (__glibc_likely (the_map->l_tls_offset |
768 | != FORCED_DYNAMIC_TLS_OFFSET)) |
769 | { |
770 | #if TLS_TCB_AT_TP |
771 | void *p = (char *) THREAD_SELF - the_map->l_tls_offset; |
772 | #elif TLS_DTV_AT_TP |
773 | void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE; |
774 | #else |
775 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" |
776 | #endif |
777 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); |
778 | |
779 | dtv[GET_ADDR_MODULE].pointer.to_free = NULL; |
780 | dtv[GET_ADDR_MODULE].pointer.val = p; |
781 | |
782 | return (char *) p + GET_ADDR_OFFSET; |
783 | } |
784 | else |
785 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); |
786 | } |
787 | struct dtv_pointer result = allocate_and_init (the_map); |
788 | dtv[GET_ADDR_MODULE].pointer = result; |
789 | assert (result.to_free != NULL); |
790 | |
791 | return (char *) result.val + GET_ADDR_OFFSET; |
792 | } |
793 | |
794 | |
795 | static struct link_map * |
796 | __attribute_noinline__ |
797 | update_get_addr (GET_ADDR_ARGS) |
798 | { |
799 | struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE); |
800 | dtv_t *dtv = THREAD_DTV (); |
801 | |
802 | void *p = dtv[GET_ADDR_MODULE].pointer.val; |
803 | |
804 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
805 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map); |
806 | |
807 | return (void *) p + GET_ADDR_OFFSET; |
808 | } |
809 | |
810 | /* For all machines that have a non-macro version of __tls_get_addr, we |
811 | want to use rtld_hidden_proto/rtld_hidden_def in order to call the |
812 | internal alias for __tls_get_addr from ld.so. This avoids a PLT entry |
813 | in ld.so for __tls_get_addr. */ |
814 | |
815 | #ifndef __tls_get_addr |
816 | extern void * __tls_get_addr (GET_ADDR_ARGS); |
817 | rtld_hidden_proto (__tls_get_addr) |
818 | rtld_hidden_def (__tls_get_addr) |
819 | #endif |
820 | |
821 | /* The generic dynamic and local dynamic model cannot be used in |
822 | statically linked applications. */ |
823 | void * |
824 | __tls_get_addr (GET_ADDR_ARGS) |
825 | { |
826 | dtv_t *dtv = THREAD_DTV (); |
827 | |
828 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
829 | return update_get_addr (GET_ADDR_PARAM); |
830 | |
831 | void *p = dtv[GET_ADDR_MODULE].pointer.val; |
832 | |
833 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
834 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL); |
835 | |
836 | return (char *) p + GET_ADDR_OFFSET; |
837 | } |
838 | #endif |
839 | |
840 | |
841 | /* Look up the module's TLS block as for __tls_get_addr, |
842 | but never touch anything. Return null if it's not allocated yet. */ |
843 | void * |
844 | _dl_tls_get_addr_soft (struct link_map *l) |
845 | { |
846 | if (__glibc_unlikely (l->l_tls_modid == 0)) |
847 | /* This module has no TLS segment. */ |
848 | return NULL; |
849 | |
850 | dtv_t *dtv = THREAD_DTV (); |
851 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
852 | { |
853 | /* This thread's DTV is not completely current, |
854 | but it might already cover this module. */ |
855 | |
856 | if (l->l_tls_modid >= dtv[-1].counter) |
857 | /* Nope. */ |
858 | return NULL; |
859 | |
860 | size_t idx = l->l_tls_modid; |
861 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
862 | while (idx >= listp->len) |
863 | { |
864 | idx -= listp->len; |
865 | listp = listp->next; |
866 | } |
867 | |
868 | /* We've reached the slot for this module. |
869 | If its generation counter is higher than the DTV's, |
870 | this thread does not know about this module yet. */ |
871 | if (dtv[0].counter < listp->slotinfo[idx].gen) |
872 | return NULL; |
873 | } |
874 | |
875 | void *data = dtv[l->l_tls_modid].pointer.val; |
876 | if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED)) |
877 | /* The DTV is current, but this thread has not yet needed |
878 | to allocate this module's segment. */ |
879 | data = NULL; |
880 | |
881 | return data; |
882 | } |
883 | |
884 | |
885 | void |
886 | _dl_add_to_slotinfo (struct link_map *l, bool do_add) |
887 | { |
888 | /* Now that we know the object is loaded successfully add |
889 | modules containing TLS data to the dtv info table. We |
890 | might have to increase its size. */ |
891 | struct dtv_slotinfo_list *listp; |
892 | struct dtv_slotinfo_list *prevp; |
893 | size_t idx = l->l_tls_modid; |
894 | |
895 | /* Find the place in the dtv slotinfo list. */ |
896 | listp = GL(dl_tls_dtv_slotinfo_list); |
897 | prevp = NULL; /* Needed to shut up gcc. */ |
898 | do |
899 | { |
900 | /* Does it fit in the array of this list element? */ |
901 | if (idx < listp->len) |
902 | break; |
903 | idx -= listp->len; |
904 | prevp = listp; |
905 | listp = listp->next; |
906 | } |
907 | while (listp != NULL); |
908 | |
909 | if (listp == NULL) |
910 | { |
911 | /* When we come here it means we have to add a new element |
912 | to the slotinfo list. And the new module must be in |
913 | the first slot. */ |
914 | assert (idx == 0); |
915 | |
916 | listp = prevp->next = (struct dtv_slotinfo_list *) |
917 | malloc (sizeof (struct dtv_slotinfo_list) |
918 | + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); |
919 | if (listp == NULL) |
920 | { |
921 | /* We ran out of memory. We will simply fail this |
922 | call but don't undo anything we did so far. The |
923 | application will crash or be terminated anyway very |
924 | soon. */ |
925 | |
926 | /* We have to do this since some entries in the dtv |
927 | slotinfo array might already point to this |
928 | generation. */ |
929 | ++GL(dl_tls_generation); |
930 | |
931 | _dl_signal_error (ENOMEM, "dlopen" , NULL, N_("\ |
932 | cannot create TLS data structures" )); |
933 | } |
934 | |
935 | listp->len = TLS_SLOTINFO_SURPLUS; |
936 | listp->next = NULL; |
937 | memset (listp->slotinfo, '\0', |
938 | TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); |
939 | } |
940 | |
941 | /* Add the information into the slotinfo data structure. */ |
942 | if (do_add) |
943 | { |
944 | listp->slotinfo[idx].map = l; |
945 | listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1; |
946 | } |
947 | } |
948 | |