1 | /* Repeating a memory blob, with alias mapping optimization. |
2 | Copyright (C) 2018-2020 Free Software Foundation, Inc. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <https://www.gnu.org/licenses/>. */ |
18 | |
19 | #include <errno.h> |
20 | #include <fcntl.h> |
21 | #include <stdbool.h> |
22 | #include <stdlib.h> |
23 | #include <string.h> |
24 | #include <support/blob_repeat.h> |
25 | #include <support/check.h> |
26 | #include <support/test-driver.h> |
27 | #include <support/support.h> |
28 | #include <support/xunistd.h> |
29 | #include <sys/mman.h> |
30 | #include <unistd.h> |
31 | #include <wchar.h> |
32 | |
33 | /* Small allocations should use malloc directly instead of the mmap |
34 | optimization because mappings carry a lot of overhead. */ |
35 | static const size_t maximum_small_size = 4 * 1024 * 1024; |
36 | |
37 | /* Internal helper for fill. */ |
38 | static void |
39 | fill0 (char *target, const char *element, size_t element_size, |
40 | size_t count) |
41 | { |
42 | while (count > 0) |
43 | { |
44 | memcpy (target, element, element_size); |
45 | target += element_size; |
46 | --count; |
47 | } |
48 | } |
49 | |
50 | /* Fill the buffer at TARGET with COUNT copies of the ELEMENT_SIZE |
51 | bytes starting at ELEMENT. */ |
52 | static void |
53 | fill (char *target, const char *element, size_t element_size, |
54 | size_t count) |
55 | { |
56 | if (element_size == 0 || count == 0) |
57 | return; |
58 | else if (element_size == 1) |
59 | memset (target, element[0], count); |
60 | else if (element_size == sizeof (wchar_t)) |
61 | { |
62 | wchar_t wc; |
63 | memcpy (&wc, element, sizeof (wc)); |
64 | wmemset ((wchar_t *) target, wc, count); |
65 | } |
66 | else if (element_size < 1024 && count > 4096) |
67 | { |
68 | /* Use larger copies for really small element sizes. */ |
69 | char buffer[8192]; |
70 | size_t buffer_count = sizeof (buffer) / element_size; |
71 | fill0 (buffer, element, element_size, buffer_count); |
72 | while (count > 0) |
73 | { |
74 | size_t copy_count = buffer_count; |
75 | if (copy_count > count) |
76 | copy_count = count; |
77 | size_t copy_bytes = copy_count * element_size; |
78 | memcpy (target, buffer, copy_bytes); |
79 | target += copy_bytes; |
80 | count -= copy_count; |
81 | } |
82 | } |
83 | else |
84 | fill0 (target, element, element_size, count); |
85 | } |
86 | |
87 | /* Use malloc instead of mmap for small allocations and unusual size |
88 | combinations. */ |
89 | static struct support_blob_repeat |
90 | allocate_malloc (size_t total_size, const void *element, size_t element_size, |
91 | size_t count) |
92 | { |
93 | void *buffer = malloc (total_size); |
94 | if (buffer == NULL) |
95 | return (struct support_blob_repeat) { 0 }; |
96 | fill (buffer, element, element_size, count); |
97 | return (struct support_blob_repeat) |
98 | { |
99 | .start = buffer, |
100 | .size = total_size, |
101 | .use_malloc = true |
102 | }; |
103 | } |
104 | |
105 | /* Return the least common multiple of PAGE_SIZE and ELEMENT_SIZE, |
106 | avoiding overflow. This assumes that PAGE_SIZE is a power of |
107 | two. */ |
108 | static size_t |
109 | minimum_stride_size (size_t page_size, size_t element_size) |
110 | { |
111 | TEST_VERIFY_EXIT (page_size > 0); |
112 | TEST_VERIFY_EXIT (element_size > 0); |
113 | |
114 | /* Compute the number of trailing zeros common to both sizes. */ |
115 | unsigned int common_zeros = __builtin_ctzll (page_size | element_size); |
116 | |
117 | /* In the product, this power of two appears twice, but in the least |
118 | common multiple, it appears only once. Therefore, shift one |
119 | factor. */ |
120 | size_t multiple; |
121 | if (__builtin_mul_overflow (page_size >> common_zeros, element_size, |
122 | &multiple)) |
123 | return 0; |
124 | return multiple; |
125 | } |
126 | |
127 | /* Allocations larger than maximum_small_size potentially use mmap |
128 | with alias mappings. */ |
129 | static struct support_blob_repeat |
130 | allocate_big (size_t total_size, const void *element, size_t element_size, |
131 | size_t count) |
132 | { |
133 | unsigned long page_size = xsysconf (_SC_PAGESIZE); |
134 | size_t stride_size = minimum_stride_size (page_size, element_size); |
135 | if (stride_size == 0) |
136 | { |
137 | errno = EOVERFLOW; |
138 | return (struct support_blob_repeat) { 0 }; |
139 | } |
140 | |
141 | /* Ensure that the stride size is at least maximum_small_size. This |
142 | is necessary to reduce the number of distinct mappings. */ |
143 | if (stride_size < maximum_small_size) |
144 | stride_size |
145 | = ((maximum_small_size + stride_size - 1) / stride_size) * stride_size; |
146 | |
147 | if (stride_size > total_size) |
148 | /* The mmap optimization would not save anything. */ |
149 | return allocate_malloc (total_size, element, element_size, count); |
150 | |
151 | /* Reserve the memory region. If we cannot create the mapping, |
152 | there is no reason to set up the backing file. */ |
153 | void *target = mmap (NULL, total_size, PROT_NONE, |
154 | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); |
155 | if (target == MAP_FAILED) |
156 | return (struct support_blob_repeat) { 0 }; |
157 | |
158 | /* Create the backing file for the repeated mapping. Call mkstemp |
159 | directly to remove the resources backing the temporary file |
160 | immediately, once support_blob_repeat_free is called. Using |
161 | create_temp_file would result in a warning during post-test |
162 | cleanup. */ |
163 | int fd; |
164 | { |
165 | char *temppath = xasprintf ("%s/support_blob_repeat-XXXXXX" , test_dir); |
166 | fd = mkstemp (temppath); |
167 | if (fd < 0) |
168 | FAIL_EXIT1 ("mkstemp (\"%s\"): %m" , temppath); |
169 | xunlink (temppath); |
170 | free (temppath); |
171 | } |
172 | |
173 | /* Make sure that there is backing storage, so that the fill |
174 | operation will not fault. */ |
175 | if (posix_fallocate (fd, 0, stride_size) != 0) |
176 | FAIL_EXIT1 ("posix_fallocate (%zu): %m" , stride_size); |
177 | |
178 | /* The stride size must still be a multiple of the page size and |
179 | element size. */ |
180 | TEST_VERIFY_EXIT ((stride_size % page_size) == 0); |
181 | TEST_VERIFY_EXIT ((stride_size % element_size) == 0); |
182 | |
183 | /* Fill the backing store. */ |
184 | { |
185 | void *ptr = mmap (target, stride_size, PROT_READ | PROT_WRITE, |
186 | MAP_FIXED | MAP_FILE | MAP_SHARED, fd, 0); |
187 | if (ptr == MAP_FAILED) |
188 | { |
189 | int saved_errno = errno; |
190 | xmunmap (target, total_size); |
191 | xclose (fd); |
192 | errno = saved_errno; |
193 | return (struct support_blob_repeat) { 0 }; |
194 | } |
195 | if (ptr != target) |
196 | FAIL_EXIT1 ("mapping of %zu bytes moved from %p to %p" , |
197 | stride_size, target, ptr); |
198 | |
199 | /* Write the repeating data. */ |
200 | fill (target, element, element_size, stride_size / element_size); |
201 | |
202 | /* Return to a PROT_NONE mapping, just to be on the safe side. */ |
203 | ptr = mmap (target, stride_size, PROT_NONE, |
204 | MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); |
205 | if (ptr == MAP_FAILED) |
206 | FAIL_EXIT1 ("Failed to reinstate PROT_NONE mapping: %m" ); |
207 | if (ptr != target) |
208 | FAIL_EXIT1 ("PROT_NONE mapping of %zu bytes moved from %p to %p" , |
209 | stride_size, target, ptr); |
210 | } |
211 | |
212 | /* Create the alias mappings. */ |
213 | { |
214 | size_t remaining_size = total_size; |
215 | char *current = target; |
216 | int flags = MAP_FIXED | MAP_FILE | MAP_PRIVATE; |
217 | #ifdef MAP_NORESERVE |
218 | flags |= MAP_NORESERVE; |
219 | #endif |
220 | while (remaining_size > 0) |
221 | { |
222 | size_t to_map = stride_size; |
223 | if (to_map > remaining_size) |
224 | to_map = remaining_size; |
225 | void *ptr = mmap (current, to_map, PROT_READ | PROT_WRITE, |
226 | flags, fd, 0); |
227 | if (ptr == MAP_FAILED) |
228 | { |
229 | int saved_errno = errno; |
230 | xmunmap (target, total_size); |
231 | xclose (fd); |
232 | errno = saved_errno; |
233 | return (struct support_blob_repeat) { 0 }; |
234 | } |
235 | if (ptr != current) |
236 | FAIL_EXIT1 ("MAP_PRIVATE mapping of %zu bytes moved from %p to %p" , |
237 | to_map, target, ptr); |
238 | remaining_size -= to_map; |
239 | current += to_map; |
240 | } |
241 | } |
242 | |
243 | xclose (fd); |
244 | |
245 | return (struct support_blob_repeat) |
246 | { |
247 | .start = target, |
248 | .size = total_size, |
249 | .use_malloc = false |
250 | }; |
251 | } |
252 | |
253 | struct support_blob_repeat |
254 | support_blob_repeat_allocate (const void *element, size_t element_size, |
255 | size_t count) |
256 | { |
257 | size_t total_size; |
258 | if (__builtin_mul_overflow (element_size, count, &total_size)) |
259 | { |
260 | errno = EOVERFLOW; |
261 | return (struct support_blob_repeat) { 0 }; |
262 | } |
263 | if (total_size <= maximum_small_size) |
264 | return allocate_malloc (total_size, element, element_size, count); |
265 | else |
266 | return allocate_big (total_size, element, element_size, count); |
267 | } |
268 | |
269 | void |
270 | support_blob_repeat_free (struct support_blob_repeat *blob) |
271 | { |
272 | if (blob->size > 0) |
273 | { |
274 | int saved_errno = errno; |
275 | if (blob->use_malloc) |
276 | free (blob->start); |
277 | else |
278 | xmunmap (blob->start, blob->size); |
279 | errno = saved_errno; |
280 | } |
281 | *blob = (struct support_blob_repeat) { 0 }; |
282 | } |
283 | |