1/* Thread-local storage handling in the ELF dynamic linker. Generic version.
2 Copyright (C) 2002-2017 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <assert.h>
20#include <errno.h>
21#include <libintl.h>
22#include <signal.h>
23#include <stdlib.h>
24#include <unistd.h>
25#include <sys/param.h>
26#include <atomic.h>
27
28#include <tls.h>
29#include <dl-tls.h>
30#include <ldsodefs.h>
31
32/* Amount of excess space to allocate in the static TLS area
33 to allow dynamic loading of modules defining IE-model TLS data. */
34#define TLS_STATIC_SURPLUS 64 + DL_NNS * 100
35
36
37/* Out-of-memory handler. */
38static void
39__attribute__ ((__noreturn__))
40oom (void)
41{
42 _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n");
43}
44
45
46size_t
47internal_function
48_dl_next_tls_modid (void)
49{
50 size_t result;
51
52 if (__builtin_expect (GL(dl_tls_dtv_gaps), false))
53 {
54 size_t disp = 0;
55 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
56
57 /* Note that this branch will never be executed during program
58 start since there are no gaps at that time. Therefore it
59 does not matter that the dl_tls_dtv_slotinfo is not allocated
60 yet when the function is called for the first times.
61
62 NB: the offset +1 is due to the fact that DTV[0] is used
63 for something else. */
64 result = GL(dl_tls_static_nelem) + 1;
65 if (result <= GL(dl_tls_max_dtv_idx))
66 do
67 {
68 while (result - disp < runp->len)
69 {
70 if (runp->slotinfo[result - disp].map == NULL)
71 break;
72
73 ++result;
74 assert (result <= GL(dl_tls_max_dtv_idx) + 1);
75 }
76
77 if (result - disp < runp->len)
78 break;
79
80 disp += runp->len;
81 }
82 while ((runp = runp->next) != NULL);
83
84 if (result > GL(dl_tls_max_dtv_idx))
85 {
86 /* The new index must indeed be exactly one higher than the
87 previous high. */
88 assert (result == GL(dl_tls_max_dtv_idx) + 1);
89 /* There is no gap anymore. */
90 GL(dl_tls_dtv_gaps) = false;
91
92 goto nogaps;
93 }
94 }
95 else
96 {
97 /* No gaps, allocate a new entry. */
98 nogaps:
99
100 result = ++GL(dl_tls_max_dtv_idx);
101 }
102
103 return result;
104}
105
106
107size_t
108internal_function
109_dl_count_modids (void)
110{
111 /* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where
112 we fail to load a module and unload it leaving a gap. If we don't
113 have gaps then the number of modids is the current maximum so
114 return that. */
115 if (__glibc_likely (!GL(dl_tls_dtv_gaps)))
116 return GL(dl_tls_max_dtv_idx);
117
118 /* We have gaps and are forced to count the non-NULL entries. */
119 size_t n = 0;
120 struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list);
121 while (runp != NULL)
122 {
123 for (size_t i = 0; i < runp->len; ++i)
124 if (runp->slotinfo[i].map != NULL)
125 ++n;
126
127 runp = runp->next;
128 }
129
130 return n;
131}
132
133
134#ifdef SHARED
135void
136internal_function
137_dl_determine_tlsoffset (void)
138{
139 size_t max_align = TLS_TCB_ALIGN;
140 size_t freetop = 0;
141 size_t freebottom = 0;
142
143 /* The first element of the dtv slot info list is allocated. */
144 assert (GL(dl_tls_dtv_slotinfo_list) != NULL);
145 /* There is at this point only one element in the
146 dl_tls_dtv_slotinfo_list list. */
147 assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL);
148
149 struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo;
150
151 /* Determining the offset of the various parts of the static TLS
152 block has several dependencies. In addition we have to work
153 around bugs in some toolchains.
154
155 Each TLS block from the objects available at link time has a size
156 and an alignment requirement. The GNU ld computes the alignment
157 requirements for the data at the positions *in the file*, though.
158 I.e, it is not simply possible to allocate a block with the size
159 of the TLS program header entry. The data is layed out assuming
160 that the first byte of the TLS block fulfills
161
162 p_vaddr mod p_align == &TLS_BLOCK mod p_align
163
164 This means we have to add artificial padding at the beginning of
165 the TLS block. These bytes are never used for the TLS data in
166 this module but the first byte allocated must be aligned
167 according to mod p_align == 0 so that the first byte of the TLS
168 block is aligned according to p_vaddr mod p_align. This is ugly
169 and the linker can help by computing the offsets in the TLS block
170 assuming the first byte of the TLS block is aligned according to
171 p_align.
172
173 The extra space which might be allocated before the first byte of
174 the TLS block need not go unused. The code below tries to use
175 that memory for the next TLS block. This can work if the total
176 memory requirement for the next TLS block is smaller than the
177 gap. */
178
179#if TLS_TCB_AT_TP
180 /* We simply start with zero. */
181 size_t offset = 0;
182
183 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
184 {
185 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
186
187 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
188 & (slotinfo[cnt].map->l_tls_align - 1));
189 size_t off;
190 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
191
192 if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize)
193 {
194 off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize
195 - firstbyte, slotinfo[cnt].map->l_tls_align)
196 + firstbyte;
197 if (off <= freebottom)
198 {
199 freetop = off;
200
201 /* XXX For some architectures we perhaps should store the
202 negative offset. */
203 slotinfo[cnt].map->l_tls_offset = off;
204 continue;
205 }
206 }
207
208 off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte,
209 slotinfo[cnt].map->l_tls_align) + firstbyte;
210 if (off > offset + slotinfo[cnt].map->l_tls_blocksize
211 + (freebottom - freetop))
212 {
213 freetop = offset;
214 freebottom = off - slotinfo[cnt].map->l_tls_blocksize;
215 }
216 offset = off;
217
218 /* XXX For some architectures we perhaps should store the
219 negative offset. */
220 slotinfo[cnt].map->l_tls_offset = off;
221 }
222
223 GL(dl_tls_static_used) = offset;
224 GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align)
225 + TLS_TCB_SIZE);
226#elif TLS_DTV_AT_TP
227 /* The TLS blocks start right after the TCB. */
228 size_t offset = TLS_TCB_SIZE;
229
230 for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt)
231 {
232 assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len);
233
234 size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset
235 & (slotinfo[cnt].map->l_tls_align - 1));
236 size_t off;
237 max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align);
238
239 if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom)
240 {
241 off = roundup (freebottom, slotinfo[cnt].map->l_tls_align);
242 if (off - freebottom < firstbyte)
243 off += slotinfo[cnt].map->l_tls_align;
244 if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop)
245 {
246 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
247 freebottom = (off + slotinfo[cnt].map->l_tls_blocksize
248 - firstbyte);
249 continue;
250 }
251 }
252
253 off = roundup (offset, slotinfo[cnt].map->l_tls_align);
254 if (off - offset < firstbyte)
255 off += slotinfo[cnt].map->l_tls_align;
256
257 slotinfo[cnt].map->l_tls_offset = off - firstbyte;
258 if (off - firstbyte - offset > freetop - freebottom)
259 {
260 freebottom = offset;
261 freetop = off - firstbyte;
262 }
263
264 offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte;
265 }
266
267 GL(dl_tls_static_used) = offset;
268 GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS,
269 TLS_TCB_ALIGN);
270#else
271# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
272#endif
273
274 /* The alignment requirement for the static TLS block. */
275 GL(dl_tls_static_align) = max_align;
276}
277#endif /* SHARED */
278
279static void *
280internal_function
281allocate_dtv (void *result)
282{
283 dtv_t *dtv;
284 size_t dtv_length;
285
286 /* We allocate a few more elements in the dtv than are needed for the
287 initial set of modules. This should avoid in most cases expansions
288 of the dtv. */
289 dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS;
290 dtv = calloc (dtv_length + 2, sizeof (dtv_t));
291 if (dtv != NULL)
292 {
293 /* This is the initial length of the dtv. */
294 dtv[0].counter = dtv_length;
295
296 /* The rest of the dtv (including the generation counter) is
297 Initialize with zero to indicate nothing there. */
298
299 /* Add the dtv to the thread data structures. */
300 INSTALL_DTV (result, dtv);
301 }
302 else
303 result = NULL;
304
305 return result;
306}
307
308
309/* Get size and alignment requirements of the static TLS block. */
310void
311internal_function
312_dl_get_tls_static_info (size_t *sizep, size_t *alignp)
313{
314 *sizep = GL(dl_tls_static_size);
315 *alignp = GL(dl_tls_static_align);
316}
317
318/* Derive the location of the pointer to the start of the original
319 allocation (before alignment) from the pointer to the TCB. */
320static inline void **
321tcb_to_pointer_to_free_location (void *tcb)
322{
323#if TLS_TCB_AT_TP
324 /* The TCB follows the TLS blocks, and the pointer to the front
325 follows the TCB. */
326 void **original_pointer_location = tcb + TLS_TCB_SIZE;
327#elif TLS_DTV_AT_TP
328 /* The TCB comes first, preceded by the pre-TCB, and the pointer is
329 before that. */
330 void **original_pointer_location = tcb - TLS_PRE_TCB_SIZE - sizeof (void *);
331#endif
332 return original_pointer_location;
333}
334
335void *
336internal_function
337_dl_allocate_tls_storage (void)
338{
339 void *result;
340 size_t size = GL(dl_tls_static_size);
341
342#if TLS_DTV_AT_TP
343 /* Memory layout is:
344 [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ]
345 ^ This should be returned. */
346 size += TLS_PRE_TCB_SIZE;
347#endif
348
349 /* Perform the allocation. Reserve space for the required alignment
350 and the pointer to the original allocation. */
351 size_t alignment = GL(dl_tls_static_align);
352 void *allocated = malloc (size + alignment + sizeof (void *));
353 if (__glibc_unlikely (allocated == NULL))
354 return NULL;
355
356 /* Perform alignment and allocate the DTV. */
357#if TLS_TCB_AT_TP
358 /* The TCB follows the TLS blocks, which determine the alignment.
359 (TCB alignment requirements have been taken into account when
360 calculating GL(dl_tls_static_align).) */
361 void *aligned = (void *) roundup ((uintptr_t) allocated, alignment);
362 result = aligned + size - TLS_TCB_SIZE;
363
364 /* Clear the TCB data structure. We can't ask the caller (i.e.
365 libpthread) to do it, because we will initialize the DTV et al. */
366 memset (result, '\0', TLS_TCB_SIZE);
367#elif TLS_DTV_AT_TP
368 /* Pre-TCB and TCB come before the TLS blocks. The layout computed
369 in _dl_determine_tlsoffset assumes that the TCB is aligned to the
370 TLS block alignment, and not just the TLS blocks after it. This
371 can leave an unused alignment gap between the TCB and the TLS
372 blocks. */
373 result = (void *) roundup
374 (sizeof (void *) + TLS_PRE_TCB_SIZE + (uintptr_t) allocated,
375 alignment);
376
377 /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before
378 it. We can't ask the caller (i.e. libpthread) to do it, because
379 we will initialize the DTV et al. */
380 memset (result - TLS_PRE_TCB_SIZE, '\0', TLS_PRE_TCB_SIZE + TLS_TCB_SIZE);
381#endif
382
383 /* Record the value of the original pointer for later
384 deallocation. */
385 *tcb_to_pointer_to_free_location (result) = allocated;
386
387 result = allocate_dtv (result);
388 if (result == NULL)
389 free (allocated);
390 return result;
391}
392
393
394#ifndef SHARED
395extern dtv_t _dl_static_dtv[];
396# define _dl_initial_dtv (&_dl_static_dtv[1])
397#endif
398
399static dtv_t *
400_dl_resize_dtv (dtv_t *dtv)
401{
402 /* Resize the dtv. */
403 dtv_t *newp;
404 /* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by
405 other threads concurrently. */
406 size_t newsize
407 = atomic_load_acquire (&GL(dl_tls_max_dtv_idx)) + DTV_SURPLUS;
408 size_t oldsize = dtv[-1].counter;
409
410 if (dtv == GL(dl_initial_dtv))
411 {
412 /* This is the initial dtv that was either statically allocated in
413 __libc_setup_tls or allocated during rtld startup using the
414 dl-minimal.c malloc instead of the real malloc. We can't free
415 it, we have to abandon the old storage. */
416
417 newp = malloc ((2 + newsize) * sizeof (dtv_t));
418 if (newp == NULL)
419 oom ();
420 memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t));
421 }
422 else
423 {
424 newp = realloc (&dtv[-1],
425 (2 + newsize) * sizeof (dtv_t));
426 if (newp == NULL)
427 oom ();
428 }
429
430 newp[0].counter = newsize;
431
432 /* Clear the newly allocated part. */
433 memset (newp + 2 + oldsize, '\0',
434 (newsize - oldsize) * sizeof (dtv_t));
435
436 /* Return the generation counter. */
437 return &newp[1];
438}
439
440
441void *
442internal_function
443_dl_allocate_tls_init (void *result)
444{
445 if (result == NULL)
446 /* The memory allocation failed. */
447 return NULL;
448
449 dtv_t *dtv = GET_DTV (result);
450 struct dtv_slotinfo_list *listp;
451 size_t total = 0;
452 size_t maxgen = 0;
453
454 /* Check if the current dtv is big enough. */
455 if (dtv[-1].counter < GL(dl_tls_max_dtv_idx))
456 {
457 /* Resize the dtv. */
458 dtv = _dl_resize_dtv (dtv);
459
460 /* Install this new dtv in the thread data structures. */
461 INSTALL_DTV (result, &dtv[-1]);
462 }
463
464 /* We have to prepare the dtv for all currently loaded modules using
465 TLS. For those which are dynamically loaded we add the values
466 indicating deferred allocation. */
467 listp = GL(dl_tls_dtv_slotinfo_list);
468 while (1)
469 {
470 size_t cnt;
471
472 for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
473 {
474 struct link_map *map;
475 void *dest;
476
477 /* Check for the total number of used slots. */
478 if (total + cnt > GL(dl_tls_max_dtv_idx))
479 break;
480
481 map = listp->slotinfo[cnt].map;
482 if (map == NULL)
483 /* Unused entry. */
484 continue;
485
486 /* Keep track of the maximum generation number. This might
487 not be the generation counter. */
488 assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation));
489 maxgen = MAX (maxgen, listp->slotinfo[cnt].gen);
490
491 dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED;
492 dtv[map->l_tls_modid].pointer.to_free = NULL;
493
494 if (map->l_tls_offset == NO_TLS_OFFSET
495 || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET)
496 continue;
497
498 assert (map->l_tls_modid == total + cnt);
499 assert (map->l_tls_blocksize >= map->l_tls_initimage_size);
500#if TLS_TCB_AT_TP
501 assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize);
502 dest = (char *) result - map->l_tls_offset;
503#elif TLS_DTV_AT_TP
504 dest = (char *) result + map->l_tls_offset;
505#else
506# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
507#endif
508
509 /* Set up the DTV entry. The simplified __tls_get_addr that
510 some platforms use in static programs requires it. */
511 dtv[map->l_tls_modid].pointer.val = dest;
512
513 /* Copy the initialization image and clear the BSS part. */
514 memset (__mempcpy (dest, map->l_tls_initimage,
515 map->l_tls_initimage_size), '\0',
516 map->l_tls_blocksize - map->l_tls_initimage_size);
517 }
518
519 total += cnt;
520 if (total >= GL(dl_tls_max_dtv_idx))
521 break;
522
523 listp = listp->next;
524 assert (listp != NULL);
525 }
526
527 /* The DTV version is up-to-date now. */
528 dtv[0].counter = maxgen;
529
530 return result;
531}
532rtld_hidden_def (_dl_allocate_tls_init)
533
534void *
535internal_function
536_dl_allocate_tls (void *mem)
537{
538 return _dl_allocate_tls_init (mem == NULL
539 ? _dl_allocate_tls_storage ()
540 : allocate_dtv (mem));
541}
542rtld_hidden_def (_dl_allocate_tls)
543
544
545void
546internal_function
547_dl_deallocate_tls (void *tcb, bool dealloc_tcb)
548{
549 dtv_t *dtv = GET_DTV (tcb);
550
551 /* We need to free the memory allocated for non-static TLS. */
552 for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt)
553 free (dtv[1 + cnt].pointer.to_free);
554
555 /* The array starts with dtv[-1]. */
556 if (dtv != GL(dl_initial_dtv))
557 free (dtv - 1);
558
559 if (dealloc_tcb)
560 free (*tcb_to_pointer_to_free_location (tcb));
561}
562rtld_hidden_def (_dl_deallocate_tls)
563
564
565#ifdef SHARED
566/* The __tls_get_addr function has two basic forms which differ in the
567 arguments. The IA-64 form takes two parameters, the module ID and
568 offset. The form used, among others, on IA-32 takes a reference to
569 a special structure which contain the same information. The second
570 form seems to be more often used (in the moment) so we default to
571 it. Users of the IA-64 form have to provide adequate definitions
572 of the following macros. */
573# ifndef GET_ADDR_ARGS
574# define GET_ADDR_ARGS tls_index *ti
575# define GET_ADDR_PARAM ti
576# endif
577# ifndef GET_ADDR_MODULE
578# define GET_ADDR_MODULE ti->ti_module
579# endif
580# ifndef GET_ADDR_OFFSET
581# define GET_ADDR_OFFSET ti->ti_offset
582# endif
583
584/* Allocate one DTV entry. */
585static struct dtv_pointer
586allocate_dtv_entry (size_t alignment, size_t size)
587{
588 if (powerof2 (alignment) && alignment <= _Alignof (max_align_t))
589 {
590 /* The alignment is supported by malloc. */
591 void *ptr = malloc (size);
592 return (struct dtv_pointer) { ptr, ptr };
593 }
594
595 /* Emulate memalign to by manually aligning a pointer returned by
596 malloc. First compute the size with an overflow check. */
597 size_t alloc_size = size + alignment;
598 if (alloc_size < size)
599 return (struct dtv_pointer) {};
600
601 /* Perform the allocation. This is the pointer we need to free
602 later. */
603 void *start = malloc (alloc_size);
604 if (start == NULL)
605 return (struct dtv_pointer) {};
606
607 /* Find the aligned position within the larger allocation. */
608 void *aligned = (void *) roundup ((uintptr_t) start, alignment);
609
610 return (struct dtv_pointer) { .val = aligned, .to_free = start };
611}
612
613static struct dtv_pointer
614allocate_and_init (struct link_map *map)
615{
616 struct dtv_pointer result = allocate_dtv_entry
617 (map->l_tls_align, map->l_tls_blocksize);
618 if (result.val == NULL)
619 oom ();
620
621 /* Initialize the memory. */
622 memset (__mempcpy (result.val, map->l_tls_initimage,
623 map->l_tls_initimage_size),
624 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
625
626 return result;
627}
628
629
630struct link_map *
631_dl_update_slotinfo (unsigned long int req_modid)
632{
633 struct link_map *the_map = NULL;
634 dtv_t *dtv = THREAD_DTV ();
635
636 /* The global dl_tls_dtv_slotinfo array contains for each module
637 index the generation counter current when the entry was created.
638 This array never shrinks so that all module indices which were
639 valid at some time can be used to access it. Before the first
640 use of a new module index in this function the array was extended
641 appropriately. Access also does not have to be guarded against
642 modifications of the array. It is assumed that pointer-size
643 values can be read atomically even in SMP environments. It is
644 possible that other threads at the same time dynamically load
645 code and therefore add to the slotinfo list. This is a problem
646 since we must not pick up any information about incomplete work.
647 The solution to this is to ignore all dtv slots which were
648 created after the one we are currently interested. We know that
649 dynamic loading for this module is completed and this is the last
650 load operation we know finished. */
651 unsigned long int idx = req_modid;
652 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
653
654 while (idx >= listp->len)
655 {
656 idx -= listp->len;
657 listp = listp->next;
658 }
659
660 if (dtv[0].counter < listp->slotinfo[idx].gen)
661 {
662 /* The generation counter for the slot is higher than what the
663 current dtv implements. We have to update the whole dtv but
664 only those entries with a generation counter <= the one for
665 the entry we need. */
666 size_t new_gen = listp->slotinfo[idx].gen;
667 size_t total = 0;
668
669 /* We have to look through the entire dtv slotinfo list. */
670 listp = GL(dl_tls_dtv_slotinfo_list);
671 do
672 {
673 for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt)
674 {
675 size_t gen = listp->slotinfo[cnt].gen;
676
677 if (gen > new_gen)
678 /* This is a slot for a generation younger than the
679 one we are handling now. It might be incompletely
680 set up so ignore it. */
681 continue;
682
683 /* If the entry is older than the current dtv layout we
684 know we don't have to handle it. */
685 if (gen <= dtv[0].counter)
686 continue;
687
688 /* If there is no map this means the entry is empty. */
689 struct link_map *map = listp->slotinfo[cnt].map;
690 if (map == NULL)
691 {
692 if (dtv[-1].counter >= total + cnt)
693 {
694 /* If this modid was used at some point the memory
695 might still be allocated. */
696 free (dtv[total + cnt].pointer.to_free);
697 dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED;
698 dtv[total + cnt].pointer.to_free = NULL;
699 }
700
701 continue;
702 }
703
704 /* Check whether the current dtv array is large enough. */
705 size_t modid = map->l_tls_modid;
706 assert (total + cnt == modid);
707 if (dtv[-1].counter < modid)
708 {
709 /* Resize the dtv. */
710 dtv = _dl_resize_dtv (dtv);
711
712 assert (modid <= dtv[-1].counter);
713
714 /* Install this new dtv in the thread data
715 structures. */
716 INSTALL_NEW_DTV (dtv);
717 }
718
719 /* If there is currently memory allocate for this
720 dtv entry free it. */
721 /* XXX Ideally we will at some point create a memory
722 pool. */
723 free (dtv[modid].pointer.to_free);
724 dtv[modid].pointer.val = TLS_DTV_UNALLOCATED;
725 dtv[modid].pointer.to_free = NULL;
726
727 if (modid == req_modid)
728 the_map = map;
729 }
730
731 total += listp->len;
732 }
733 while ((listp = listp->next) != NULL);
734
735 /* This will be the new maximum generation counter. */
736 dtv[0].counter = new_gen;
737 }
738
739 return the_map;
740}
741
742
743static void *
744__attribute_noinline__
745tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map)
746{
747 /* The allocation was deferred. Do it now. */
748 if (the_map == NULL)
749 {
750 /* Find the link map for this module. */
751 size_t idx = GET_ADDR_MODULE;
752 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
753
754 while (idx >= listp->len)
755 {
756 idx -= listp->len;
757 listp = listp->next;
758 }
759
760 the_map = listp->slotinfo[idx].map;
761 }
762
763 /* Make sure that, if a dlopen running in parallel forces the
764 variable into static storage, we'll wait until the address in the
765 static TLS block is set up, and use that. If we're undecided
766 yet, make sure we make the decision holding the lock as well. */
767 if (__glibc_unlikely (the_map->l_tls_offset
768 != FORCED_DYNAMIC_TLS_OFFSET))
769 {
770 __rtld_lock_lock_recursive (GL(dl_load_lock));
771 if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET))
772 {
773 the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET;
774 __rtld_lock_unlock_recursive (GL(dl_load_lock));
775 }
776 else if (__glibc_likely (the_map->l_tls_offset
777 != FORCED_DYNAMIC_TLS_OFFSET))
778 {
779#if TLS_TCB_AT_TP
780 void *p = (char *) THREAD_SELF - the_map->l_tls_offset;
781#elif TLS_DTV_AT_TP
782 void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE;
783#else
784# error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
785#endif
786 __rtld_lock_unlock_recursive (GL(dl_load_lock));
787
788 dtv[GET_ADDR_MODULE].pointer.to_free = NULL;
789 dtv[GET_ADDR_MODULE].pointer.val = p;
790
791 return (char *) p + GET_ADDR_OFFSET;
792 }
793 else
794 __rtld_lock_unlock_recursive (GL(dl_load_lock));
795 }
796 struct dtv_pointer result = allocate_and_init (the_map);
797 dtv[GET_ADDR_MODULE].pointer = result;
798 assert (result.to_free != NULL);
799
800 return (char *) result.val + GET_ADDR_OFFSET;
801}
802
803
804static struct link_map *
805__attribute_noinline__
806update_get_addr (GET_ADDR_ARGS)
807{
808 struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE);
809 dtv_t *dtv = THREAD_DTV ();
810
811 void *p = dtv[GET_ADDR_MODULE].pointer.val;
812
813 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
814 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map);
815
816 return (void *) p + GET_ADDR_OFFSET;
817}
818
819/* For all machines that have a non-macro version of __tls_get_addr, we
820 want to use rtld_hidden_proto/rtld_hidden_def in order to call the
821 internal alias for __tls_get_addr from ld.so. This avoids a PLT entry
822 in ld.so for __tls_get_addr. */
823
824#ifndef __tls_get_addr
825extern void * __tls_get_addr (GET_ADDR_ARGS);
826rtld_hidden_proto (__tls_get_addr)
827rtld_hidden_def (__tls_get_addr)
828#endif
829
830/* The generic dynamic and local dynamic model cannot be used in
831 statically linked applications. */
832void *
833__tls_get_addr (GET_ADDR_ARGS)
834{
835 dtv_t *dtv = THREAD_DTV ();
836
837 if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
838 return update_get_addr (GET_ADDR_PARAM);
839
840 void *p = dtv[GET_ADDR_MODULE].pointer.val;
841
842 if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED))
843 return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL);
844
845 return (char *) p + GET_ADDR_OFFSET;
846}
847#endif
848
849
850/* Look up the module's TLS block as for __tls_get_addr,
851 but never touch anything. Return null if it's not allocated yet. */
852void *
853_dl_tls_get_addr_soft (struct link_map *l)
854{
855 if (__glibc_unlikely (l->l_tls_modid == 0))
856 /* This module has no TLS segment. */
857 return NULL;
858
859 dtv_t *dtv = THREAD_DTV ();
860 if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation)))
861 {
862 /* This thread's DTV is not completely current,
863 but it might already cover this module. */
864
865 if (l->l_tls_modid >= dtv[-1].counter)
866 /* Nope. */
867 return NULL;
868
869 size_t idx = l->l_tls_modid;
870 struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list);
871 while (idx >= listp->len)
872 {
873 idx -= listp->len;
874 listp = listp->next;
875 }
876
877 /* We've reached the slot for this module.
878 If its generation counter is higher than the DTV's,
879 this thread does not know about this module yet. */
880 if (dtv[0].counter < listp->slotinfo[idx].gen)
881 return NULL;
882 }
883
884 void *data = dtv[l->l_tls_modid].pointer.val;
885 if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED))
886 /* The DTV is current, but this thread has not yet needed
887 to allocate this module's segment. */
888 data = NULL;
889
890 return data;
891}
892
893
894void
895_dl_add_to_slotinfo (struct link_map *l)
896{
897 /* Now that we know the object is loaded successfully add
898 modules containing TLS data to the dtv info table. We
899 might have to increase its size. */
900 struct dtv_slotinfo_list *listp;
901 struct dtv_slotinfo_list *prevp;
902 size_t idx = l->l_tls_modid;
903
904 /* Find the place in the dtv slotinfo list. */
905 listp = GL(dl_tls_dtv_slotinfo_list);
906 prevp = NULL; /* Needed to shut up gcc. */
907 do
908 {
909 /* Does it fit in the array of this list element? */
910 if (idx < listp->len)
911 break;
912 idx -= listp->len;
913 prevp = listp;
914 listp = listp->next;
915 }
916 while (listp != NULL);
917
918 if (listp == NULL)
919 {
920 /* When we come here it means we have to add a new element
921 to the slotinfo list. And the new module must be in
922 the first slot. */
923 assert (idx == 0);
924
925 listp = prevp->next = (struct dtv_slotinfo_list *)
926 malloc (sizeof (struct dtv_slotinfo_list)
927 + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
928 if (listp == NULL)
929 {
930 /* We ran out of memory. We will simply fail this
931 call but don't undo anything we did so far. The
932 application will crash or be terminated anyway very
933 soon. */
934
935 /* We have to do this since some entries in the dtv
936 slotinfo array might already point to this
937 generation. */
938 ++GL(dl_tls_generation);
939
940 _dl_signal_error (ENOMEM, "dlopen", NULL, N_("\
941cannot create TLS data structures"));
942 }
943
944 listp->len = TLS_SLOTINFO_SURPLUS;
945 listp->next = NULL;
946 memset (listp->slotinfo, '\0',
947 TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo));
948 }
949
950 /* Add the information into the slotinfo data structure. */
951 listp->slotinfo[idx].map = l;
952 listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
953}
954