1 | /* This file is part of the GNU C Library. |
2 | Copyright (C) 2008-2016 Free Software Foundation, Inc. |
3 | |
4 | The GNU C Library is free software; you can redistribute it and/or |
5 | modify it under the terms of the GNU Lesser General Public |
6 | License as published by the Free Software Foundation; either |
7 | version 2.1 of the License, or (at your option) any later version. |
8 | |
9 | The GNU C Library is distributed in the hope that it will be useful, |
10 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
12 | Lesser General Public License for more details. |
13 | |
14 | You should have received a copy of the GNU Lesser General Public |
15 | License along with the GNU C Library; if not, see |
16 | <http://www.gnu.org/licenses/>. */ |
17 | |
18 | #ifndef cpu_features_h |
19 | #define cpu_features_h |
20 | |
21 | #define bit_Fast_Rep_String (1 << 0) |
22 | #define bit_Fast_Copy_Backward (1 << 1) |
23 | #define bit_Slow_BSF (1 << 2) |
24 | #define bit_Fast_Unaligned_Load (1 << 4) |
25 | #define bit_Prefer_PMINUB_for_stringop (1 << 5) |
26 | #define bit_AVX_Usable (1 << 6) |
27 | #define bit_FMA_Usable (1 << 7) |
28 | #define bit_FMA4_Usable (1 << 8) |
29 | #define bit_Slow_SSE4_2 (1 << 9) |
30 | #define bit_AVX2_Usable (1 << 10) |
31 | #define bit_AVX_Fast_Unaligned_Load (1 << 11) |
32 | #define bit_AVX512F_Usable (1 << 12) |
33 | #define bit_AVX512DQ_Usable (1 << 13) |
34 | #define bit_I586 (1 << 14) |
35 | #define bit_I686 (1 << 15) |
36 | #define bit_Prefer_MAP_32BIT_EXEC (1 << 16) |
37 | #define bit_Prefer_No_VZEROUPPER (1 << 17) |
38 | #define bit_XSAVEC_Usable (1 << 18) |
39 | |
40 | /* CPUID Feature flags. */ |
41 | |
42 | /* COMMON_CPUID_INDEX_1. */ |
43 | #define bit_CX8 (1 << 8) |
44 | #define bit_CMOV (1 << 15) |
45 | #define bit_SSE2 (1 << 26) |
46 | #define bit_SSSE3 (1 << 9) |
47 | #define bit_SSE4_1 (1 << 19) |
48 | #define bit_SSE4_2 (1 << 20) |
49 | #define bit_OSXSAVE (1 << 27) |
50 | #define bit_AVX (1 << 28) |
51 | #define bit_POPCOUNT (1 << 23) |
52 | #define bit_FMA (1 << 12) |
53 | #define bit_FMA4 (1 << 16) |
54 | |
55 | /* COMMON_CPUID_INDEX_7. */ |
56 | #define bit_RTM (1 << 11) |
57 | #define bit_AVX2 (1 << 5) |
58 | #define bit_AVX512F (1 << 16) |
59 | #define bit_AVX512DQ (1 << 17) |
60 | |
61 | /* XCR0 Feature flags. */ |
62 | #define bit_XMM_state (1 << 1) |
63 | #define bit_YMM_state (2 << 1) |
64 | #define bit_Opmask_state (1 << 5) |
65 | #define bit_ZMM0_15_state (1 << 6) |
66 | #define bit_ZMM16_31_state (1 << 7) |
67 | |
68 | /* The integer bit array index for the first set of internal feature bits. */ |
69 | #define FEATURE_INDEX_1 0 |
70 | |
71 | /* The current maximum size of the feature integer bit array. */ |
72 | #define FEATURE_INDEX_MAX 1 |
73 | |
74 | /* Offset for fxsave/xsave area used by _dl_runtime_resolve. Also need |
75 | space to preserve RCX, RDX, RSI, RDI, R8, R9 and RAX. It must be |
76 | aligned to 16 bytes for fxsave and 64 bytes for xsave. */ |
77 | #define STATE_SAVE_OFFSET (8 * 7 + 8) |
78 | |
79 | /* Save SSE, AVX, AVX512, mask and bound registers. */ |
80 | #define STATE_SAVE_MASK \ |
81 | ((1 << 1) | (1 << 2) | (1 << 3) | (1 << 5) | (1 << 6) | (1 << 7)) |
82 | |
83 | #ifdef __ASSEMBLER__ |
84 | |
85 | # include <ifunc-defines.h> |
86 | # include <rtld-global-offsets.h> |
87 | # include <cpu-features-offsets.h> |
88 | |
89 | # define index_CX8 COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_EDX_OFFSET |
90 | # define index_CMOV COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_EDX_OFFSET |
91 | # define index_SSE2 COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_EDX_OFFSET |
92 | # define index_SSSE3 COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_ECX_OFFSET |
93 | # define index_SSE4_1 COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_ECX_OFFSET |
94 | # define index_SSE4_2 COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_ECX_OFFSET |
95 | # define index_AVX COMMON_CPUID_INDEX_1*CPUID_SIZE+CPUID_ECX_OFFSET |
96 | # define index_AVX2 COMMON_CPUID_INDEX_7*CPUID_SIZE+CPUID_EBX_OFFSET |
97 | |
98 | # define index_Fast_Rep_String FEATURE_INDEX_1*FEATURE_SIZE |
99 | # define index_Fast_Copy_Backward FEATURE_INDEX_1*FEATURE_SIZE |
100 | # define index_Slow_BSF FEATURE_INDEX_1*FEATURE_SIZE |
101 | # define index_Fast_Unaligned_Load FEATURE_INDEX_1*FEATURE_SIZE |
102 | # define index_Prefer_PMINUB_for_stringop FEATURE_INDEX_1*FEATURE_SIZE |
103 | # define index_AVX_Usable FEATURE_INDEX_1*FEATURE_SIZE |
104 | # define index_FMA_Usable FEATURE_INDEX_1*FEATURE_SIZE |
105 | # define index_FMA4_Usable FEATURE_INDEX_1*FEATURE_SIZE |
106 | # define index_Slow_SSE4_2 FEATURE_INDEX_1*FEATURE_SIZE |
107 | # define index_AVX2_Usable FEATURE_INDEX_1*FEATURE_SIZE |
108 | # define index_AVX_Fast_Unaligned_Load FEATURE_INDEX_1*FEATURE_SIZE |
109 | # define index_AVX512F_Usable FEATURE_INDEX_1*FEATURE_SIZE |
110 | # define index_AVX512DQ_Usable FEATURE_INDEX_1*FEATURE_SIZE |
111 | # define index_I586 FEATURE_INDEX_1*FEATURE_SIZE |
112 | # define index_I686 FEATURE_INDEX_1*FEATURE_SIZE |
113 | # define index_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1*FEATURE_SIZE |
114 | # define index_Prefer_No_VZEROUPPER FEATURE_INDEX_1*FEATURE_SIZE |
115 | |
116 | |
117 | # if defined (_LIBC) && !IS_IN (nonlib) |
118 | # ifdef __x86_64__ |
119 | # ifdef SHARED |
120 | # if IS_IN (rtld) |
121 | # define LOAD_RTLD_GLOBAL_RO_RDX |
122 | # define HAS_FEATURE(offset, name) \ |
123 | testl $(bit_##name), _rtld_local_ro+offset+(index_##name)(%rip) |
124 | # else |
125 | # define LOAD_RTLD_GLOBAL_RO_RDX \ |
126 | mov _rtld_global_ro@GOTPCREL(%rip), %RDX_LP |
127 | # define HAS_FEATURE(offset, name) \ |
128 | testl $(bit_##name), \ |
129 | RTLD_GLOBAL_RO_DL_X86_CPU_FEATURES_OFFSET+offset+(index_##name)(%rdx) |
130 | # endif |
131 | # else /* SHARED */ |
132 | # define LOAD_RTLD_GLOBAL_RO_RDX |
133 | # define HAS_FEATURE(offset, name) \ |
134 | testl $(bit_##name), _dl_x86_cpu_features+offset+(index_##name)(%rip) |
135 | # endif /* !SHARED */ |
136 | # else /* __x86_64__ */ |
137 | # ifdef SHARED |
138 | # define LOAD_FUNC_GOT_EAX(func) \ |
139 | leal func@GOTOFF(%edx), %eax |
140 | # if IS_IN (rtld) |
141 | # define LOAD_GOT_AND_RTLD_GLOBAL_RO \ |
142 | LOAD_PIC_REG(dx) |
143 | # define HAS_FEATURE(offset, name) \ |
144 | testl $(bit_##name), offset+(index_##name)+_rtld_local_ro@GOTOFF(%edx) |
145 | # else |
146 | # define LOAD_GOT_AND_RTLD_GLOBAL_RO \ |
147 | LOAD_PIC_REG(dx); \ |
148 | mov _rtld_global_ro@GOT(%edx), %ecx |
149 | # define HAS_FEATURE(offset, name) \ |
150 | testl $(bit_##name), \ |
151 | RTLD_GLOBAL_RO_DL_X86_CPU_FEATURES_OFFSET+offset+(index_##name)(%ecx) |
152 | # endif |
153 | # else /* SHARED */ |
154 | # define LOAD_FUNC_GOT_EAX(func) \ |
155 | leal func, %eax |
156 | # define LOAD_GOT_AND_RTLD_GLOBAL_RO |
157 | # define HAS_FEATURE(offset, name) \ |
158 | testl $(bit_##name), _dl_x86_cpu_features+offset+(index_##name) |
159 | # endif /* !SHARED */ |
160 | # endif /* !__x86_64__ */ |
161 | # else /* _LIBC && !nonlib */ |
162 | # error "Sorry, <cpu-features.h> is unimplemented for assembler" |
163 | # endif /* !_LIBC || nonlib */ |
164 | |
165 | /* HAS_* evaluates to true if we may use the feature at runtime. */ |
166 | # define HAS_CPU_FEATURE(name) HAS_FEATURE (CPUID_OFFSET, name) |
167 | # define HAS_ARCH_FEATURE(name) HAS_FEATURE (FEATURE_OFFSET, name) |
168 | |
169 | #else /* __ASSEMBLER__ */ |
170 | |
171 | enum |
172 | { |
173 | COMMON_CPUID_INDEX_1 = 0, |
174 | COMMON_CPUID_INDEX_7, |
175 | COMMON_CPUID_INDEX_80000001, /* for AMD */ |
176 | /* Keep the following line at the end. */ |
177 | COMMON_CPUID_INDEX_MAX |
178 | }; |
179 | |
180 | struct cpu_features |
181 | { |
182 | enum cpu_features_kind |
183 | { |
184 | arch_kind_unknown = 0, |
185 | arch_kind_intel, |
186 | arch_kind_amd, |
187 | arch_kind_other |
188 | } kind; |
189 | int max_cpuid; |
190 | struct cpuid_registers |
191 | { |
192 | unsigned int eax; |
193 | unsigned int ebx; |
194 | unsigned int ecx; |
195 | unsigned int edx; |
196 | } cpuid[COMMON_CPUID_INDEX_MAX]; |
197 | unsigned int family; |
198 | unsigned int model; |
199 | /* The type must be unsigned long int so that we use |
200 | |
201 | sub xsave_state_size_offset(%rip) %RSP_LP |
202 | |
203 | in _dl_runtime_resolve. */ |
204 | unsigned long int xsave_state_size; |
205 | unsigned int feature[FEATURE_INDEX_MAX]; |
206 | }; |
207 | |
208 | /* Used from outside of glibc to get access to the CPU features |
209 | structure. */ |
210 | extern const struct cpu_features *__get_cpu_features (void) |
211 | __attribute__ ((const)); |
212 | |
213 | # if defined (_LIBC) && !IS_IN (nonlib) |
214 | /* Unused for x86. */ |
215 | # define INIT_ARCH() |
216 | # define __get_cpu_features() (&GLRO(dl_x86_cpu_features)) |
217 | # endif |
218 | |
219 | |
220 | /* HAS_* evaluates to true if we may use the feature at runtime. */ |
221 | # define HAS_CPU_FEATURE(name) \ |
222 | ((__get_cpu_features ()->cpuid[index_##name].reg_##name & (bit_##name)) != 0) |
223 | # define HAS_ARCH_FEATURE(name) \ |
224 | ((__get_cpu_features ()->feature[index_##name] & (bit_##name)) != 0) |
225 | |
226 | # define index_CX8 COMMON_CPUID_INDEX_1 |
227 | # define index_CMOV COMMON_CPUID_INDEX_1 |
228 | # define index_SSE2 COMMON_CPUID_INDEX_1 |
229 | # define index_SSSE3 COMMON_CPUID_INDEX_1 |
230 | # define index_SSE4_1 COMMON_CPUID_INDEX_1 |
231 | # define index_SSE4_2 COMMON_CPUID_INDEX_1 |
232 | # define index_AVX COMMON_CPUID_INDEX_1 |
233 | # define index_AVX2 COMMON_CPUID_INDEX_7 |
234 | # define index_AVX512F COMMON_CPUID_INDEX_7 |
235 | # define index_AVX512DQ COMMON_CPUID_INDEX_7 |
236 | # define index_RTM COMMON_CPUID_INDEX_7 |
237 | # define index_FMA COMMON_CPUID_INDEX_1 |
238 | # define index_FMA4 COMMON_CPUID_INDEX_80000001 |
239 | # define index_POPCOUNT COMMON_CPUID_INDEX_1 |
240 | # define index_OSXSAVE COMMON_CPUID_INDEX_1 |
241 | |
242 | # define reg_CX8 edx |
243 | # define reg_CMOV edx |
244 | # define reg_SSE2 edx |
245 | # define reg_SSSE3 ecx |
246 | # define reg_SSE4_1 ecx |
247 | # define reg_SSE4_2 ecx |
248 | # define reg_AVX ecx |
249 | # define reg_AVX2 ebx |
250 | # define reg_AVX512F ebx |
251 | # define reg_AVX512DQ ebx |
252 | # define reg_RTM ebx |
253 | # define reg_FMA ecx |
254 | # define reg_FMA4 ecx |
255 | # define reg_POPCOUNT ecx |
256 | # define reg_OSXSAVE ecx |
257 | |
258 | # define index_Fast_Rep_String FEATURE_INDEX_1 |
259 | # define index_Fast_Copy_Backward FEATURE_INDEX_1 |
260 | # define index_Slow_BSF FEATURE_INDEX_1 |
261 | # define index_Fast_Unaligned_Load FEATURE_INDEX_1 |
262 | # define index_Prefer_PMINUB_for_stringop FEATURE_INDEX_1 |
263 | # define index_AVX_Usable FEATURE_INDEX_1 |
264 | # define index_FMA_Usable FEATURE_INDEX_1 |
265 | # define index_FMA4_Usable FEATURE_INDEX_1 |
266 | # define index_Slow_SSE4_2 FEATURE_INDEX_1 |
267 | # define index_AVX2_Usable FEATURE_INDEX_1 |
268 | # define index_AVX_Fast_Unaligned_Load FEATURE_INDEX_1 |
269 | # define index_AVX512F_Usable FEATURE_INDEX_1 |
270 | # define index_AVX512DQ_Usable FEATURE_INDEX_1 |
271 | # define index_I586 FEATURE_INDEX_1 |
272 | # define index_I686 FEATURE_INDEX_1 |
273 | # define index_Prefer_MAP_32BIT_EXEC FEATURE_INDEX_1 |
274 | # define index_Prefer_No_VZEROUPPER FEATURE_INDEX_1 |
275 | # define index_XSAVEC_Usable FEATURE_INDEX_1 |
276 | |
277 | #endif /* !__ASSEMBLER__ */ |
278 | |
279 | #ifdef __x86_64__ |
280 | # define HAS_CPUID 1 |
281 | #elif defined __i586__ || defined __pentium__ |
282 | # define HAS_CPUID 1 |
283 | # define HAS_I586 1 |
284 | # define HAS_I686 HAS_ARCH_FEATURE (I686) |
285 | #elif (defined __i686__ || defined __pentiumpro__ \ |
286 | || defined __pentium4__ || defined __nocona__ \ |
287 | || defined __atom__ || defined __core2__ \ |
288 | || defined __corei7__ || defined __corei7_avx__ \ |
289 | || defined __core_avx2__ || defined __nehalem__ \ |
290 | || defined __sandybridge__ || defined __haswell__ \ |
291 | || defined __knl__ || defined __bonnell__ \ |
292 | || defined __silvermont__ \ |
293 | || defined __k6__ || defined __k8__ \ |
294 | || defined __athlon__ || defined __amdfam10__ \ |
295 | || defined __bdver1__ || defined __bdver2__ \ |
296 | || defined __bdver3__ || defined __bdver4__ \ |
297 | || defined __btver1__ || defined __btver2__) |
298 | # define HAS_CPUID 1 |
299 | # define HAS_I586 1 |
300 | # define HAS_I686 1 |
301 | #else |
302 | # define HAS_CPUID 0 |
303 | # define HAS_I586 HAS_ARCH_FEATURE (I586) |
304 | # define HAS_I686 HAS_ARCH_FEATURE (I686) |
305 | #endif |
306 | |
307 | #endif /* cpu_features_h */ |
308 | |