1 | /* Thread-local storage handling in the ELF dynamic linker. Generic version. |
2 | Copyright (C) 2002-2017 Free Software Foundation, Inc. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ |
18 | |
19 | #include <assert.h> |
20 | #include <errno.h> |
21 | #include <libintl.h> |
22 | #include <signal.h> |
23 | #include <stdlib.h> |
24 | #include <unistd.h> |
25 | #include <sys/param.h> |
26 | #include <atomic.h> |
27 | |
28 | #include <tls.h> |
29 | #include <dl-tls.h> |
30 | #include <ldsodefs.h> |
31 | |
32 | /* Amount of excess space to allocate in the static TLS area |
33 | to allow dynamic loading of modules defining IE-model TLS data. */ |
34 | #define TLS_STATIC_SURPLUS 64 + DL_NNS * 100 |
35 | |
36 | |
37 | /* Out-of-memory handler. */ |
38 | static void |
39 | __attribute__ ((__noreturn__)) |
40 | oom (void) |
41 | { |
42 | _dl_fatal_printf ("cannot allocate memory for thread-local data: ABORT\n" ); |
43 | } |
44 | |
45 | |
46 | size_t |
47 | internal_function |
48 | _dl_next_tls_modid (void) |
49 | { |
50 | size_t result; |
51 | |
52 | if (__builtin_expect (GL(dl_tls_dtv_gaps), false)) |
53 | { |
54 | size_t disp = 0; |
55 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); |
56 | |
57 | /* Note that this branch will never be executed during program |
58 | start since there are no gaps at that time. Therefore it |
59 | does not matter that the dl_tls_dtv_slotinfo is not allocated |
60 | yet when the function is called for the first times. |
61 | |
62 | NB: the offset +1 is due to the fact that DTV[0] is used |
63 | for something else. */ |
64 | result = GL(dl_tls_static_nelem) + 1; |
65 | if (result <= GL(dl_tls_max_dtv_idx)) |
66 | do |
67 | { |
68 | while (result - disp < runp->len) |
69 | { |
70 | if (runp->slotinfo[result - disp].map == NULL) |
71 | break; |
72 | |
73 | ++result; |
74 | assert (result <= GL(dl_tls_max_dtv_idx) + 1); |
75 | } |
76 | |
77 | if (result - disp < runp->len) |
78 | break; |
79 | |
80 | disp += runp->len; |
81 | } |
82 | while ((runp = runp->next) != NULL); |
83 | |
84 | if (result > GL(dl_tls_max_dtv_idx)) |
85 | { |
86 | /* The new index must indeed be exactly one higher than the |
87 | previous high. */ |
88 | assert (result == GL(dl_tls_max_dtv_idx) + 1); |
89 | /* There is no gap anymore. */ |
90 | GL(dl_tls_dtv_gaps) = false; |
91 | |
92 | goto nogaps; |
93 | } |
94 | } |
95 | else |
96 | { |
97 | /* No gaps, allocate a new entry. */ |
98 | nogaps: |
99 | |
100 | result = ++GL(dl_tls_max_dtv_idx); |
101 | } |
102 | |
103 | return result; |
104 | } |
105 | |
106 | |
107 | size_t |
108 | internal_function |
109 | _dl_count_modids (void) |
110 | { |
111 | /* It is rare that we have gaps; see elf/dl-open.c (_dl_open) where |
112 | we fail to load a module and unload it leaving a gap. If we don't |
113 | have gaps then the number of modids is the current maximum so |
114 | return that. */ |
115 | if (__glibc_likely (!GL(dl_tls_dtv_gaps))) |
116 | return GL(dl_tls_max_dtv_idx); |
117 | |
118 | /* We have gaps and are forced to count the non-NULL entries. */ |
119 | size_t n = 0; |
120 | struct dtv_slotinfo_list *runp = GL(dl_tls_dtv_slotinfo_list); |
121 | while (runp != NULL) |
122 | { |
123 | for (size_t i = 0; i < runp->len; ++i) |
124 | if (runp->slotinfo[i].map != NULL) |
125 | ++n; |
126 | |
127 | runp = runp->next; |
128 | } |
129 | |
130 | return n; |
131 | } |
132 | |
133 | |
134 | #ifdef SHARED |
135 | void |
136 | internal_function |
137 | _dl_determine_tlsoffset (void) |
138 | { |
139 | size_t max_align = TLS_TCB_ALIGN; |
140 | size_t freetop = 0; |
141 | size_t freebottom = 0; |
142 | |
143 | /* The first element of the dtv slot info list is allocated. */ |
144 | assert (GL(dl_tls_dtv_slotinfo_list) != NULL); |
145 | /* There is at this point only one element in the |
146 | dl_tls_dtv_slotinfo_list list. */ |
147 | assert (GL(dl_tls_dtv_slotinfo_list)->next == NULL); |
148 | |
149 | struct dtv_slotinfo *slotinfo = GL(dl_tls_dtv_slotinfo_list)->slotinfo; |
150 | |
151 | /* Determining the offset of the various parts of the static TLS |
152 | block has several dependencies. In addition we have to work |
153 | around bugs in some toolchains. |
154 | |
155 | Each TLS block from the objects available at link time has a size |
156 | and an alignment requirement. The GNU ld computes the alignment |
157 | requirements for the data at the positions *in the file*, though. |
158 | I.e, it is not simply possible to allocate a block with the size |
159 | of the TLS program header entry. The data is layed out assuming |
160 | that the first byte of the TLS block fulfills |
161 | |
162 | p_vaddr mod p_align == &TLS_BLOCK mod p_align |
163 | |
164 | This means we have to add artificial padding at the beginning of |
165 | the TLS block. These bytes are never used for the TLS data in |
166 | this module but the first byte allocated must be aligned |
167 | according to mod p_align == 0 so that the first byte of the TLS |
168 | block is aligned according to p_vaddr mod p_align. This is ugly |
169 | and the linker can help by computing the offsets in the TLS block |
170 | assuming the first byte of the TLS block is aligned according to |
171 | p_align. |
172 | |
173 | The extra space which might be allocated before the first byte of |
174 | the TLS block need not go unused. The code below tries to use |
175 | that memory for the next TLS block. This can work if the total |
176 | memory requirement for the next TLS block is smaller than the |
177 | gap. */ |
178 | |
179 | #if TLS_TCB_AT_TP |
180 | /* We simply start with zero. */ |
181 | size_t offset = 0; |
182 | |
183 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) |
184 | { |
185 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); |
186 | |
187 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset |
188 | & (slotinfo[cnt].map->l_tls_align - 1)); |
189 | size_t off; |
190 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); |
191 | |
192 | if (freebottom - freetop >= slotinfo[cnt].map->l_tls_blocksize) |
193 | { |
194 | off = roundup (freetop + slotinfo[cnt].map->l_tls_blocksize |
195 | - firstbyte, slotinfo[cnt].map->l_tls_align) |
196 | + firstbyte; |
197 | if (off <= freebottom) |
198 | { |
199 | freetop = off; |
200 | |
201 | /* XXX For some architectures we perhaps should store the |
202 | negative offset. */ |
203 | slotinfo[cnt].map->l_tls_offset = off; |
204 | continue; |
205 | } |
206 | } |
207 | |
208 | off = roundup (offset + slotinfo[cnt].map->l_tls_blocksize - firstbyte, |
209 | slotinfo[cnt].map->l_tls_align) + firstbyte; |
210 | if (off > offset + slotinfo[cnt].map->l_tls_blocksize |
211 | + (freebottom - freetop)) |
212 | { |
213 | freetop = offset; |
214 | freebottom = off - slotinfo[cnt].map->l_tls_blocksize; |
215 | } |
216 | offset = off; |
217 | |
218 | /* XXX For some architectures we perhaps should store the |
219 | negative offset. */ |
220 | slotinfo[cnt].map->l_tls_offset = off; |
221 | } |
222 | |
223 | GL(dl_tls_static_used) = offset; |
224 | GL(dl_tls_static_size) = (roundup (offset + TLS_STATIC_SURPLUS, max_align) |
225 | + TLS_TCB_SIZE); |
226 | #elif TLS_DTV_AT_TP |
227 | /* The TLS blocks start right after the TCB. */ |
228 | size_t offset = TLS_TCB_SIZE; |
229 | |
230 | for (size_t cnt = 0; slotinfo[cnt].map != NULL; ++cnt) |
231 | { |
232 | assert (cnt < GL(dl_tls_dtv_slotinfo_list)->len); |
233 | |
234 | size_t firstbyte = (-slotinfo[cnt].map->l_tls_firstbyte_offset |
235 | & (slotinfo[cnt].map->l_tls_align - 1)); |
236 | size_t off; |
237 | max_align = MAX (max_align, slotinfo[cnt].map->l_tls_align); |
238 | |
239 | if (slotinfo[cnt].map->l_tls_blocksize <= freetop - freebottom) |
240 | { |
241 | off = roundup (freebottom, slotinfo[cnt].map->l_tls_align); |
242 | if (off - freebottom < firstbyte) |
243 | off += slotinfo[cnt].map->l_tls_align; |
244 | if (off + slotinfo[cnt].map->l_tls_blocksize - firstbyte <= freetop) |
245 | { |
246 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; |
247 | freebottom = (off + slotinfo[cnt].map->l_tls_blocksize |
248 | - firstbyte); |
249 | continue; |
250 | } |
251 | } |
252 | |
253 | off = roundup (offset, slotinfo[cnt].map->l_tls_align); |
254 | if (off - offset < firstbyte) |
255 | off += slotinfo[cnt].map->l_tls_align; |
256 | |
257 | slotinfo[cnt].map->l_tls_offset = off - firstbyte; |
258 | if (off - firstbyte - offset > freetop - freebottom) |
259 | { |
260 | freebottom = offset; |
261 | freetop = off - firstbyte; |
262 | } |
263 | |
264 | offset = off + slotinfo[cnt].map->l_tls_blocksize - firstbyte; |
265 | } |
266 | |
267 | GL(dl_tls_static_used) = offset; |
268 | GL(dl_tls_static_size) = roundup (offset + TLS_STATIC_SURPLUS, |
269 | TLS_TCB_ALIGN); |
270 | #else |
271 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" |
272 | #endif |
273 | |
274 | /* The alignment requirement for the static TLS block. */ |
275 | GL(dl_tls_static_align) = max_align; |
276 | } |
277 | #endif /* SHARED */ |
278 | |
279 | static void * |
280 | internal_function |
281 | allocate_dtv (void *result) |
282 | { |
283 | dtv_t *dtv; |
284 | size_t dtv_length; |
285 | |
286 | /* We allocate a few more elements in the dtv than are needed for the |
287 | initial set of modules. This should avoid in most cases expansions |
288 | of the dtv. */ |
289 | dtv_length = GL(dl_tls_max_dtv_idx) + DTV_SURPLUS; |
290 | dtv = calloc (dtv_length + 2, sizeof (dtv_t)); |
291 | if (dtv != NULL) |
292 | { |
293 | /* This is the initial length of the dtv. */ |
294 | dtv[0].counter = dtv_length; |
295 | |
296 | /* The rest of the dtv (including the generation counter) is |
297 | Initialize with zero to indicate nothing there. */ |
298 | |
299 | /* Add the dtv to the thread data structures. */ |
300 | INSTALL_DTV (result, dtv); |
301 | } |
302 | else |
303 | result = NULL; |
304 | |
305 | return result; |
306 | } |
307 | |
308 | |
309 | /* Get size and alignment requirements of the static TLS block. */ |
310 | void |
311 | internal_function |
312 | _dl_get_tls_static_info (size_t *sizep, size_t *alignp) |
313 | { |
314 | *sizep = GL(dl_tls_static_size); |
315 | *alignp = GL(dl_tls_static_align); |
316 | } |
317 | |
318 | /* Derive the location of the pointer to the start of the original |
319 | allocation (before alignment) from the pointer to the TCB. */ |
320 | static inline void ** |
321 | tcb_to_pointer_to_free_location (void *tcb) |
322 | { |
323 | #if TLS_TCB_AT_TP |
324 | /* The TCB follows the TLS blocks, and the pointer to the front |
325 | follows the TCB. */ |
326 | void **original_pointer_location = tcb + TLS_TCB_SIZE; |
327 | #elif TLS_DTV_AT_TP |
328 | /* The TCB comes first, preceded by the pre-TCB, and the pointer is |
329 | before that. */ |
330 | void **original_pointer_location = tcb - TLS_PRE_TCB_SIZE - sizeof (void *); |
331 | #endif |
332 | return original_pointer_location; |
333 | } |
334 | |
335 | void * |
336 | internal_function |
337 | _dl_allocate_tls_storage (void) |
338 | { |
339 | void *result; |
340 | size_t size = GL(dl_tls_static_size); |
341 | |
342 | #if TLS_DTV_AT_TP |
343 | /* Memory layout is: |
344 | [ TLS_PRE_TCB_SIZE ] [ TLS_TCB_SIZE ] [ TLS blocks ] |
345 | ^ This should be returned. */ |
346 | size += TLS_PRE_TCB_SIZE; |
347 | #endif |
348 | |
349 | /* Perform the allocation. Reserve space for the required alignment |
350 | and the pointer to the original allocation. */ |
351 | size_t alignment = GL(dl_tls_static_align); |
352 | void *allocated = malloc (size + alignment + sizeof (void *)); |
353 | if (__glibc_unlikely (allocated == NULL)) |
354 | return NULL; |
355 | |
356 | /* Perform alignment and allocate the DTV. */ |
357 | #if TLS_TCB_AT_TP |
358 | /* The TCB follows the TLS blocks, which determine the alignment. |
359 | (TCB alignment requirements have been taken into account when |
360 | calculating GL(dl_tls_static_align).) */ |
361 | void *aligned = (void *) roundup ((uintptr_t) allocated, alignment); |
362 | result = aligned + size - TLS_TCB_SIZE; |
363 | |
364 | /* Clear the TCB data structure. We can't ask the caller (i.e. |
365 | libpthread) to do it, because we will initialize the DTV et al. */ |
366 | memset (result, '\0', TLS_TCB_SIZE); |
367 | #elif TLS_DTV_AT_TP |
368 | /* Pre-TCB and TCB come before the TLS blocks. The layout computed |
369 | in _dl_determine_tlsoffset assumes that the TCB is aligned to the |
370 | TLS block alignment, and not just the TLS blocks after it. This |
371 | can leave an unused alignment gap between the TCB and the TLS |
372 | blocks. */ |
373 | result = (void *) roundup |
374 | (sizeof (void *) + TLS_PRE_TCB_SIZE + (uintptr_t) allocated, |
375 | alignment); |
376 | |
377 | /* Clear the TCB data structure and TLS_PRE_TCB_SIZE bytes before |
378 | it. We can't ask the caller (i.e. libpthread) to do it, because |
379 | we will initialize the DTV et al. */ |
380 | memset (result - TLS_PRE_TCB_SIZE, '\0', TLS_PRE_TCB_SIZE + TLS_TCB_SIZE); |
381 | #endif |
382 | |
383 | /* Record the value of the original pointer for later |
384 | deallocation. */ |
385 | *tcb_to_pointer_to_free_location (result) = allocated; |
386 | |
387 | result = allocate_dtv (result); |
388 | if (result == NULL) |
389 | free (allocated); |
390 | return result; |
391 | } |
392 | |
393 | |
394 | #ifndef SHARED |
395 | extern dtv_t _dl_static_dtv[]; |
396 | # define _dl_initial_dtv (&_dl_static_dtv[1]) |
397 | #endif |
398 | |
399 | static dtv_t * |
400 | _dl_resize_dtv (dtv_t *dtv) |
401 | { |
402 | /* Resize the dtv. */ |
403 | dtv_t *newp; |
404 | /* Load GL(dl_tls_max_dtv_idx) atomically since it may be written to by |
405 | other threads concurrently. */ |
406 | size_t newsize |
407 | = atomic_load_acquire (&GL(dl_tls_max_dtv_idx)) + DTV_SURPLUS; |
408 | size_t oldsize = dtv[-1].counter; |
409 | |
410 | if (dtv == GL(dl_initial_dtv)) |
411 | { |
412 | /* This is the initial dtv that was either statically allocated in |
413 | __libc_setup_tls or allocated during rtld startup using the |
414 | dl-minimal.c malloc instead of the real malloc. We can't free |
415 | it, we have to abandon the old storage. */ |
416 | |
417 | newp = malloc ((2 + newsize) * sizeof (dtv_t)); |
418 | if (newp == NULL) |
419 | oom (); |
420 | memcpy (newp, &dtv[-1], (2 + oldsize) * sizeof (dtv_t)); |
421 | } |
422 | else |
423 | { |
424 | newp = realloc (&dtv[-1], |
425 | (2 + newsize) * sizeof (dtv_t)); |
426 | if (newp == NULL) |
427 | oom (); |
428 | } |
429 | |
430 | newp[0].counter = newsize; |
431 | |
432 | /* Clear the newly allocated part. */ |
433 | memset (newp + 2 + oldsize, '\0', |
434 | (newsize - oldsize) * sizeof (dtv_t)); |
435 | |
436 | /* Return the generation counter. */ |
437 | return &newp[1]; |
438 | } |
439 | |
440 | |
441 | void * |
442 | internal_function |
443 | _dl_allocate_tls_init (void *result) |
444 | { |
445 | if (result == NULL) |
446 | /* The memory allocation failed. */ |
447 | return NULL; |
448 | |
449 | dtv_t *dtv = GET_DTV (result); |
450 | struct dtv_slotinfo_list *listp; |
451 | size_t total = 0; |
452 | size_t maxgen = 0; |
453 | |
454 | /* Check if the current dtv is big enough. */ |
455 | if (dtv[-1].counter < GL(dl_tls_max_dtv_idx)) |
456 | { |
457 | /* Resize the dtv. */ |
458 | dtv = _dl_resize_dtv (dtv); |
459 | |
460 | /* Install this new dtv in the thread data structures. */ |
461 | INSTALL_DTV (result, &dtv[-1]); |
462 | } |
463 | |
464 | /* We have to prepare the dtv for all currently loaded modules using |
465 | TLS. For those which are dynamically loaded we add the values |
466 | indicating deferred allocation. */ |
467 | listp = GL(dl_tls_dtv_slotinfo_list); |
468 | while (1) |
469 | { |
470 | size_t cnt; |
471 | |
472 | for (cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) |
473 | { |
474 | struct link_map *map; |
475 | void *dest; |
476 | |
477 | /* Check for the total number of used slots. */ |
478 | if (total + cnt > GL(dl_tls_max_dtv_idx)) |
479 | break; |
480 | |
481 | map = listp->slotinfo[cnt].map; |
482 | if (map == NULL) |
483 | /* Unused entry. */ |
484 | continue; |
485 | |
486 | /* Keep track of the maximum generation number. This might |
487 | not be the generation counter. */ |
488 | assert (listp->slotinfo[cnt].gen <= GL(dl_tls_generation)); |
489 | maxgen = MAX (maxgen, listp->slotinfo[cnt].gen); |
490 | |
491 | dtv[map->l_tls_modid].pointer.val = TLS_DTV_UNALLOCATED; |
492 | dtv[map->l_tls_modid].pointer.to_free = NULL; |
493 | |
494 | if (map->l_tls_offset == NO_TLS_OFFSET |
495 | || map->l_tls_offset == FORCED_DYNAMIC_TLS_OFFSET) |
496 | continue; |
497 | |
498 | assert (map->l_tls_modid == total + cnt); |
499 | assert (map->l_tls_blocksize >= map->l_tls_initimage_size); |
500 | #if TLS_TCB_AT_TP |
501 | assert ((size_t) map->l_tls_offset >= map->l_tls_blocksize); |
502 | dest = (char *) result - map->l_tls_offset; |
503 | #elif TLS_DTV_AT_TP |
504 | dest = (char *) result + map->l_tls_offset; |
505 | #else |
506 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" |
507 | #endif |
508 | |
509 | /* Set up the DTV entry. The simplified __tls_get_addr that |
510 | some platforms use in static programs requires it. */ |
511 | dtv[map->l_tls_modid].pointer.val = dest; |
512 | |
513 | /* Copy the initialization image and clear the BSS part. */ |
514 | memset (__mempcpy (dest, map->l_tls_initimage, |
515 | map->l_tls_initimage_size), '\0', |
516 | map->l_tls_blocksize - map->l_tls_initimage_size); |
517 | } |
518 | |
519 | total += cnt; |
520 | if (total >= GL(dl_tls_max_dtv_idx)) |
521 | break; |
522 | |
523 | listp = listp->next; |
524 | assert (listp != NULL); |
525 | } |
526 | |
527 | /* The DTV version is up-to-date now. */ |
528 | dtv[0].counter = maxgen; |
529 | |
530 | return result; |
531 | } |
532 | rtld_hidden_def (_dl_allocate_tls_init) |
533 | |
534 | void * |
535 | internal_function |
536 | _dl_allocate_tls (void *mem) |
537 | { |
538 | return _dl_allocate_tls_init (mem == NULL |
539 | ? _dl_allocate_tls_storage () |
540 | : allocate_dtv (mem)); |
541 | } |
542 | rtld_hidden_def (_dl_allocate_tls) |
543 | |
544 | |
545 | void |
546 | internal_function |
547 | _dl_deallocate_tls (void *tcb, bool dealloc_tcb) |
548 | { |
549 | dtv_t *dtv = GET_DTV (tcb); |
550 | |
551 | /* We need to free the memory allocated for non-static TLS. */ |
552 | for (size_t cnt = 0; cnt < dtv[-1].counter; ++cnt) |
553 | free (dtv[1 + cnt].pointer.to_free); |
554 | |
555 | /* The array starts with dtv[-1]. */ |
556 | if (dtv != GL(dl_initial_dtv)) |
557 | free (dtv - 1); |
558 | |
559 | if (dealloc_tcb) |
560 | free (*tcb_to_pointer_to_free_location (tcb)); |
561 | } |
562 | rtld_hidden_def (_dl_deallocate_tls) |
563 | |
564 | |
565 | #ifdef SHARED |
566 | /* The __tls_get_addr function has two basic forms which differ in the |
567 | arguments. The IA-64 form takes two parameters, the module ID and |
568 | offset. The form used, among others, on IA-32 takes a reference to |
569 | a special structure which contain the same information. The second |
570 | form seems to be more often used (in the moment) so we default to |
571 | it. Users of the IA-64 form have to provide adequate definitions |
572 | of the following macros. */ |
573 | # ifndef GET_ADDR_ARGS |
574 | # define GET_ADDR_ARGS tls_index *ti |
575 | # define GET_ADDR_PARAM ti |
576 | # endif |
577 | # ifndef GET_ADDR_MODULE |
578 | # define GET_ADDR_MODULE ti->ti_module |
579 | # endif |
580 | # ifndef GET_ADDR_OFFSET |
581 | # define GET_ADDR_OFFSET ti->ti_offset |
582 | # endif |
583 | |
584 | /* Allocate one DTV entry. */ |
585 | static struct dtv_pointer |
586 | allocate_dtv_entry (size_t alignment, size_t size) |
587 | { |
588 | if (powerof2 (alignment) && alignment <= _Alignof (max_align_t)) |
589 | { |
590 | /* The alignment is supported by malloc. */ |
591 | void *ptr = malloc (size); |
592 | return (struct dtv_pointer) { ptr, ptr }; |
593 | } |
594 | |
595 | /* Emulate memalign to by manually aligning a pointer returned by |
596 | malloc. First compute the size with an overflow check. */ |
597 | size_t alloc_size = size + alignment; |
598 | if (alloc_size < size) |
599 | return (struct dtv_pointer) {}; |
600 | |
601 | /* Perform the allocation. This is the pointer we need to free |
602 | later. */ |
603 | void *start = malloc (alloc_size); |
604 | if (start == NULL) |
605 | return (struct dtv_pointer) {}; |
606 | |
607 | /* Find the aligned position within the larger allocation. */ |
608 | void *aligned = (void *) roundup ((uintptr_t) start, alignment); |
609 | |
610 | return (struct dtv_pointer) { .val = aligned, .to_free = start }; |
611 | } |
612 | |
613 | static struct dtv_pointer |
614 | allocate_and_init (struct link_map *map) |
615 | { |
616 | struct dtv_pointer result = allocate_dtv_entry |
617 | (map->l_tls_align, map->l_tls_blocksize); |
618 | if (result.val == NULL) |
619 | oom (); |
620 | |
621 | /* Initialize the memory. */ |
622 | memset (__mempcpy (result.val, map->l_tls_initimage, |
623 | map->l_tls_initimage_size), |
624 | '\0', map->l_tls_blocksize - map->l_tls_initimage_size); |
625 | |
626 | return result; |
627 | } |
628 | |
629 | |
630 | struct link_map * |
631 | _dl_update_slotinfo (unsigned long int req_modid) |
632 | { |
633 | struct link_map *the_map = NULL; |
634 | dtv_t *dtv = THREAD_DTV (); |
635 | |
636 | /* The global dl_tls_dtv_slotinfo array contains for each module |
637 | index the generation counter current when the entry was created. |
638 | This array never shrinks so that all module indices which were |
639 | valid at some time can be used to access it. Before the first |
640 | use of a new module index in this function the array was extended |
641 | appropriately. Access also does not have to be guarded against |
642 | modifications of the array. It is assumed that pointer-size |
643 | values can be read atomically even in SMP environments. It is |
644 | possible that other threads at the same time dynamically load |
645 | code and therefore add to the slotinfo list. This is a problem |
646 | since we must not pick up any information about incomplete work. |
647 | The solution to this is to ignore all dtv slots which were |
648 | created after the one we are currently interested. We know that |
649 | dynamic loading for this module is completed and this is the last |
650 | load operation we know finished. */ |
651 | unsigned long int idx = req_modid; |
652 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
653 | |
654 | while (idx >= listp->len) |
655 | { |
656 | idx -= listp->len; |
657 | listp = listp->next; |
658 | } |
659 | |
660 | if (dtv[0].counter < listp->slotinfo[idx].gen) |
661 | { |
662 | /* The generation counter for the slot is higher than what the |
663 | current dtv implements. We have to update the whole dtv but |
664 | only those entries with a generation counter <= the one for |
665 | the entry we need. */ |
666 | size_t new_gen = listp->slotinfo[idx].gen; |
667 | size_t total = 0; |
668 | |
669 | /* We have to look through the entire dtv slotinfo list. */ |
670 | listp = GL(dl_tls_dtv_slotinfo_list); |
671 | do |
672 | { |
673 | for (size_t cnt = total == 0 ? 1 : 0; cnt < listp->len; ++cnt) |
674 | { |
675 | size_t gen = listp->slotinfo[cnt].gen; |
676 | |
677 | if (gen > new_gen) |
678 | /* This is a slot for a generation younger than the |
679 | one we are handling now. It might be incompletely |
680 | set up so ignore it. */ |
681 | continue; |
682 | |
683 | /* If the entry is older than the current dtv layout we |
684 | know we don't have to handle it. */ |
685 | if (gen <= dtv[0].counter) |
686 | continue; |
687 | |
688 | /* If there is no map this means the entry is empty. */ |
689 | struct link_map *map = listp->slotinfo[cnt].map; |
690 | if (map == NULL) |
691 | { |
692 | if (dtv[-1].counter >= total + cnt) |
693 | { |
694 | /* If this modid was used at some point the memory |
695 | might still be allocated. */ |
696 | free (dtv[total + cnt].pointer.to_free); |
697 | dtv[total + cnt].pointer.val = TLS_DTV_UNALLOCATED; |
698 | dtv[total + cnt].pointer.to_free = NULL; |
699 | } |
700 | |
701 | continue; |
702 | } |
703 | |
704 | /* Check whether the current dtv array is large enough. */ |
705 | size_t modid = map->l_tls_modid; |
706 | assert (total + cnt == modid); |
707 | if (dtv[-1].counter < modid) |
708 | { |
709 | /* Resize the dtv. */ |
710 | dtv = _dl_resize_dtv (dtv); |
711 | |
712 | assert (modid <= dtv[-1].counter); |
713 | |
714 | /* Install this new dtv in the thread data |
715 | structures. */ |
716 | INSTALL_NEW_DTV (dtv); |
717 | } |
718 | |
719 | /* If there is currently memory allocate for this |
720 | dtv entry free it. */ |
721 | /* XXX Ideally we will at some point create a memory |
722 | pool. */ |
723 | free (dtv[modid].pointer.to_free); |
724 | dtv[modid].pointer.val = TLS_DTV_UNALLOCATED; |
725 | dtv[modid].pointer.to_free = NULL; |
726 | |
727 | if (modid == req_modid) |
728 | the_map = map; |
729 | } |
730 | |
731 | total += listp->len; |
732 | } |
733 | while ((listp = listp->next) != NULL); |
734 | |
735 | /* This will be the new maximum generation counter. */ |
736 | dtv[0].counter = new_gen; |
737 | } |
738 | |
739 | return the_map; |
740 | } |
741 | |
742 | |
743 | static void * |
744 | __attribute_noinline__ |
745 | tls_get_addr_tail (GET_ADDR_ARGS, dtv_t *dtv, struct link_map *the_map) |
746 | { |
747 | /* The allocation was deferred. Do it now. */ |
748 | if (the_map == NULL) |
749 | { |
750 | /* Find the link map for this module. */ |
751 | size_t idx = GET_ADDR_MODULE; |
752 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
753 | |
754 | while (idx >= listp->len) |
755 | { |
756 | idx -= listp->len; |
757 | listp = listp->next; |
758 | } |
759 | |
760 | the_map = listp->slotinfo[idx].map; |
761 | } |
762 | |
763 | /* Make sure that, if a dlopen running in parallel forces the |
764 | variable into static storage, we'll wait until the address in the |
765 | static TLS block is set up, and use that. If we're undecided |
766 | yet, make sure we make the decision holding the lock as well. */ |
767 | if (__glibc_unlikely (the_map->l_tls_offset |
768 | != FORCED_DYNAMIC_TLS_OFFSET)) |
769 | { |
770 | __rtld_lock_lock_recursive (GL(dl_load_lock)); |
771 | if (__glibc_likely (the_map->l_tls_offset == NO_TLS_OFFSET)) |
772 | { |
773 | the_map->l_tls_offset = FORCED_DYNAMIC_TLS_OFFSET; |
774 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); |
775 | } |
776 | else if (__glibc_likely (the_map->l_tls_offset |
777 | != FORCED_DYNAMIC_TLS_OFFSET)) |
778 | { |
779 | #if TLS_TCB_AT_TP |
780 | void *p = (char *) THREAD_SELF - the_map->l_tls_offset; |
781 | #elif TLS_DTV_AT_TP |
782 | void *p = (char *) THREAD_SELF + the_map->l_tls_offset + TLS_PRE_TCB_SIZE; |
783 | #else |
784 | # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined" |
785 | #endif |
786 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); |
787 | |
788 | dtv[GET_ADDR_MODULE].pointer.to_free = NULL; |
789 | dtv[GET_ADDR_MODULE].pointer.val = p; |
790 | |
791 | return (char *) p + GET_ADDR_OFFSET; |
792 | } |
793 | else |
794 | __rtld_lock_unlock_recursive (GL(dl_load_lock)); |
795 | } |
796 | struct dtv_pointer result = allocate_and_init (the_map); |
797 | dtv[GET_ADDR_MODULE].pointer = result; |
798 | assert (result.to_free != NULL); |
799 | |
800 | return (char *) result.val + GET_ADDR_OFFSET; |
801 | } |
802 | |
803 | |
804 | static struct link_map * |
805 | __attribute_noinline__ |
806 | update_get_addr (GET_ADDR_ARGS) |
807 | { |
808 | struct link_map *the_map = _dl_update_slotinfo (GET_ADDR_MODULE); |
809 | dtv_t *dtv = THREAD_DTV (); |
810 | |
811 | void *p = dtv[GET_ADDR_MODULE].pointer.val; |
812 | |
813 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
814 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, the_map); |
815 | |
816 | return (void *) p + GET_ADDR_OFFSET; |
817 | } |
818 | |
819 | /* For all machines that have a non-macro version of __tls_get_addr, we |
820 | want to use rtld_hidden_proto/rtld_hidden_def in order to call the |
821 | internal alias for __tls_get_addr from ld.so. This avoids a PLT entry |
822 | in ld.so for __tls_get_addr. */ |
823 | |
824 | #ifndef __tls_get_addr |
825 | extern void * __tls_get_addr (GET_ADDR_ARGS); |
826 | rtld_hidden_proto (__tls_get_addr) |
827 | rtld_hidden_def (__tls_get_addr) |
828 | #endif |
829 | |
830 | /* The generic dynamic and local dynamic model cannot be used in |
831 | statically linked applications. */ |
832 | void * |
833 | __tls_get_addr (GET_ADDR_ARGS) |
834 | { |
835 | dtv_t *dtv = THREAD_DTV (); |
836 | |
837 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
838 | return update_get_addr (GET_ADDR_PARAM); |
839 | |
840 | void *p = dtv[GET_ADDR_MODULE].pointer.val; |
841 | |
842 | if (__glibc_unlikely (p == TLS_DTV_UNALLOCATED)) |
843 | return tls_get_addr_tail (GET_ADDR_PARAM, dtv, NULL); |
844 | |
845 | return (char *) p + GET_ADDR_OFFSET; |
846 | } |
847 | #endif |
848 | |
849 | |
850 | /* Look up the module's TLS block as for __tls_get_addr, |
851 | but never touch anything. Return null if it's not allocated yet. */ |
852 | void * |
853 | _dl_tls_get_addr_soft (struct link_map *l) |
854 | { |
855 | if (__glibc_unlikely (l->l_tls_modid == 0)) |
856 | /* This module has no TLS segment. */ |
857 | return NULL; |
858 | |
859 | dtv_t *dtv = THREAD_DTV (); |
860 | if (__glibc_unlikely (dtv[0].counter != GL(dl_tls_generation))) |
861 | { |
862 | /* This thread's DTV is not completely current, |
863 | but it might already cover this module. */ |
864 | |
865 | if (l->l_tls_modid >= dtv[-1].counter) |
866 | /* Nope. */ |
867 | return NULL; |
868 | |
869 | size_t idx = l->l_tls_modid; |
870 | struct dtv_slotinfo_list *listp = GL(dl_tls_dtv_slotinfo_list); |
871 | while (idx >= listp->len) |
872 | { |
873 | idx -= listp->len; |
874 | listp = listp->next; |
875 | } |
876 | |
877 | /* We've reached the slot for this module. |
878 | If its generation counter is higher than the DTV's, |
879 | this thread does not know about this module yet. */ |
880 | if (dtv[0].counter < listp->slotinfo[idx].gen) |
881 | return NULL; |
882 | } |
883 | |
884 | void *data = dtv[l->l_tls_modid].pointer.val; |
885 | if (__glibc_unlikely (data == TLS_DTV_UNALLOCATED)) |
886 | /* The DTV is current, but this thread has not yet needed |
887 | to allocate this module's segment. */ |
888 | data = NULL; |
889 | |
890 | return data; |
891 | } |
892 | |
893 | |
894 | void |
895 | _dl_add_to_slotinfo (struct link_map *l) |
896 | { |
897 | /* Now that we know the object is loaded successfully add |
898 | modules containing TLS data to the dtv info table. We |
899 | might have to increase its size. */ |
900 | struct dtv_slotinfo_list *listp; |
901 | struct dtv_slotinfo_list *prevp; |
902 | size_t idx = l->l_tls_modid; |
903 | |
904 | /* Find the place in the dtv slotinfo list. */ |
905 | listp = GL(dl_tls_dtv_slotinfo_list); |
906 | prevp = NULL; /* Needed to shut up gcc. */ |
907 | do |
908 | { |
909 | /* Does it fit in the array of this list element? */ |
910 | if (idx < listp->len) |
911 | break; |
912 | idx -= listp->len; |
913 | prevp = listp; |
914 | listp = listp->next; |
915 | } |
916 | while (listp != NULL); |
917 | |
918 | if (listp == NULL) |
919 | { |
920 | /* When we come here it means we have to add a new element |
921 | to the slotinfo list. And the new module must be in |
922 | the first slot. */ |
923 | assert (idx == 0); |
924 | |
925 | listp = prevp->next = (struct dtv_slotinfo_list *) |
926 | malloc (sizeof (struct dtv_slotinfo_list) |
927 | + TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); |
928 | if (listp == NULL) |
929 | { |
930 | /* We ran out of memory. We will simply fail this |
931 | call but don't undo anything we did so far. The |
932 | application will crash or be terminated anyway very |
933 | soon. */ |
934 | |
935 | /* We have to do this since some entries in the dtv |
936 | slotinfo array might already point to this |
937 | generation. */ |
938 | ++GL(dl_tls_generation); |
939 | |
940 | _dl_signal_error (ENOMEM, "dlopen" , NULL, N_("\ |
941 | cannot create TLS data structures" )); |
942 | } |
943 | |
944 | listp->len = TLS_SLOTINFO_SURPLUS; |
945 | listp->next = NULL; |
946 | memset (listp->slotinfo, '\0', |
947 | TLS_SLOTINFO_SURPLUS * sizeof (struct dtv_slotinfo)); |
948 | } |
949 | |
950 | /* Add the information into the slotinfo data structure. */ |
951 | listp->slotinfo[idx].map = l; |
952 | listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1; |
953 | } |
954 | |