1 | /* Copyright (C) 2011-2017 Free Software Foundation, Inc. |
2 | Contributed by Intel Corporation. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ |
18 | |
19 | #include <sysdep.h> |
20 | |
21 | /* fast SSE2 version with using pmaxub and 64 byte loop */ |
22 | |
23 | .text |
24 | ENTRY(memchr) |
25 | movd %rsi, %xmm1 |
26 | mov %rdi, %rcx |
27 | #ifdef __ILP32__ |
28 | /* Clear the upper 32 bits. */ |
29 | movl %edx, %edx |
30 | #endif |
31 | punpcklbw %xmm1, %xmm1 |
32 | test %RDX_LP, %RDX_LP |
33 | jz L(return_null) |
34 | punpcklbw %xmm1, %xmm1 |
35 | |
36 | and $63, %rcx |
37 | pshufd $0, %xmm1, %xmm1 |
38 | |
39 | cmp $48, %rcx |
40 | ja L(crosscache) |
41 | |
42 | movdqu (%rdi), %xmm0 |
43 | pcmpeqb %xmm1, %xmm0 |
44 | pmovmskb %xmm0, %eax |
45 | test %eax, %eax |
46 | |
47 | jnz L(matches_1) |
48 | sub $16, %rdx |
49 | jbe L(return_null) |
50 | add $16, %rdi |
51 | and $15, %rcx |
52 | and $-16, %rdi |
53 | add %rcx, %rdx |
54 | sub $64, %rdx |
55 | jbe L(exit_loop) |
56 | jmp L(loop_prolog) |
57 | |
58 | .p2align 4 |
59 | L(crosscache): |
60 | and $15, %rcx |
61 | and $-16, %rdi |
62 | movdqa (%rdi), %xmm0 |
63 | |
64 | pcmpeqb %xmm1, %xmm0 |
65 | /* Check if there is a match. */ |
66 | pmovmskb %xmm0, %eax |
67 | /* Remove the leading bytes. */ |
68 | sar %cl, %eax |
69 | test %eax, %eax |
70 | je L(unaligned_no_match) |
71 | /* Check which byte is a match. */ |
72 | bsf %eax, %eax |
73 | |
74 | sub %rax, %rdx |
75 | jbe L(return_null) |
76 | add %rdi, %rax |
77 | add %rcx, %rax |
78 | ret |
79 | |
80 | .p2align 4 |
81 | L(unaligned_no_match): |
82 | /* Calculate the last acceptable address and check for possible |
83 | addition overflow by using satured math: |
84 | rdx = rcx + rdx |
85 | rdx |= -(rdx < rcx) */ |
86 | add %rcx, %rdx |
87 | sbb %rax, %rax |
88 | or %rax, %rdx |
89 | sub $16, %rdx |
90 | jbe L(return_null) |
91 | add $16, %rdi |
92 | sub $64, %rdx |
93 | jbe L(exit_loop) |
94 | |
95 | .p2align 4 |
96 | L(loop_prolog): |
97 | movdqa (%rdi), %xmm0 |
98 | pcmpeqb %xmm1, %xmm0 |
99 | pmovmskb %xmm0, %eax |
100 | test %eax, %eax |
101 | jnz L(matches) |
102 | |
103 | movdqa 16(%rdi), %xmm2 |
104 | pcmpeqb %xmm1, %xmm2 |
105 | pmovmskb %xmm2, %eax |
106 | test %eax, %eax |
107 | jnz L(matches16) |
108 | |
109 | movdqa 32(%rdi), %xmm3 |
110 | pcmpeqb %xmm1, %xmm3 |
111 | pmovmskb %xmm3, %eax |
112 | test %eax, %eax |
113 | jnz L(matches32) |
114 | |
115 | movdqa 48(%rdi), %xmm4 |
116 | pcmpeqb %xmm1, %xmm4 |
117 | add $64, %rdi |
118 | pmovmskb %xmm4, %eax |
119 | test %eax, %eax |
120 | jnz L(matches0) |
121 | |
122 | test $0x3f, %rdi |
123 | jz L(align64_loop) |
124 | |
125 | sub $64, %rdx |
126 | jbe L(exit_loop) |
127 | |
128 | movdqa (%rdi), %xmm0 |
129 | pcmpeqb %xmm1, %xmm0 |
130 | pmovmskb %xmm0, %eax |
131 | test %eax, %eax |
132 | jnz L(matches) |
133 | |
134 | movdqa 16(%rdi), %xmm2 |
135 | pcmpeqb %xmm1, %xmm2 |
136 | pmovmskb %xmm2, %eax |
137 | test %eax, %eax |
138 | jnz L(matches16) |
139 | |
140 | movdqa 32(%rdi), %xmm3 |
141 | pcmpeqb %xmm1, %xmm3 |
142 | pmovmskb %xmm3, %eax |
143 | test %eax, %eax |
144 | jnz L(matches32) |
145 | |
146 | movdqa 48(%rdi), %xmm3 |
147 | pcmpeqb %xmm1, %xmm3 |
148 | pmovmskb %xmm3, %eax |
149 | |
150 | add $64, %rdi |
151 | test %eax, %eax |
152 | jnz L(matches0) |
153 | |
154 | mov %rdi, %rcx |
155 | and $-64, %rdi |
156 | and $63, %rcx |
157 | add %rcx, %rdx |
158 | |
159 | .p2align 4 |
160 | L(align64_loop): |
161 | sub $64, %rdx |
162 | jbe L(exit_loop) |
163 | movdqa (%rdi), %xmm0 |
164 | movdqa 16(%rdi), %xmm2 |
165 | movdqa 32(%rdi), %xmm3 |
166 | movdqa 48(%rdi), %xmm4 |
167 | |
168 | pcmpeqb %xmm1, %xmm0 |
169 | pcmpeqb %xmm1, %xmm2 |
170 | pcmpeqb %xmm1, %xmm3 |
171 | pcmpeqb %xmm1, %xmm4 |
172 | |
173 | pmaxub %xmm0, %xmm3 |
174 | pmaxub %xmm2, %xmm4 |
175 | pmaxub %xmm3, %xmm4 |
176 | pmovmskb %xmm4, %eax |
177 | |
178 | add $64, %rdi |
179 | |
180 | test %eax, %eax |
181 | jz L(align64_loop) |
182 | |
183 | sub $64, %rdi |
184 | |
185 | pmovmskb %xmm0, %eax |
186 | test %eax, %eax |
187 | jnz L(matches) |
188 | |
189 | pmovmskb %xmm2, %eax |
190 | test %eax, %eax |
191 | jnz L(matches16) |
192 | |
193 | movdqa 32(%rdi), %xmm3 |
194 | pcmpeqb %xmm1, %xmm3 |
195 | |
196 | pcmpeqb 48(%rdi), %xmm1 |
197 | pmovmskb %xmm3, %eax |
198 | test %eax, %eax |
199 | jnz L(matches32) |
200 | |
201 | pmovmskb %xmm1, %eax |
202 | bsf %eax, %eax |
203 | lea 48(%rdi, %rax), %rax |
204 | ret |
205 | |
206 | .p2align 4 |
207 | L(exit_loop): |
208 | add $32, %rdx |
209 | jle L(exit_loop_32) |
210 | |
211 | movdqa (%rdi), %xmm0 |
212 | pcmpeqb %xmm1, %xmm0 |
213 | pmovmskb %xmm0, %eax |
214 | test %eax, %eax |
215 | jnz L(matches) |
216 | |
217 | movdqa 16(%rdi), %xmm2 |
218 | pcmpeqb %xmm1, %xmm2 |
219 | pmovmskb %xmm2, %eax |
220 | test %eax, %eax |
221 | jnz L(matches16) |
222 | |
223 | movdqa 32(%rdi), %xmm3 |
224 | pcmpeqb %xmm1, %xmm3 |
225 | pmovmskb %xmm3, %eax |
226 | test %eax, %eax |
227 | jnz L(matches32_1) |
228 | sub $16, %rdx |
229 | jle L(return_null) |
230 | |
231 | pcmpeqb 48(%rdi), %xmm1 |
232 | pmovmskb %xmm1, %eax |
233 | test %eax, %eax |
234 | jnz L(matches48_1) |
235 | xor %rax, %rax |
236 | ret |
237 | |
238 | .p2align 4 |
239 | L(exit_loop_32): |
240 | add $32, %rdx |
241 | movdqa (%rdi), %xmm0 |
242 | pcmpeqb %xmm1, %xmm0 |
243 | pmovmskb %xmm0, %eax |
244 | test %eax, %eax |
245 | jnz L(matches_1) |
246 | sub $16, %rdx |
247 | jbe L(return_null) |
248 | |
249 | pcmpeqb 16(%rdi), %xmm1 |
250 | pmovmskb %xmm1, %eax |
251 | test %eax, %eax |
252 | jnz L(matches16_1) |
253 | xor %rax, %rax |
254 | ret |
255 | |
256 | .p2align 4 |
257 | L(matches0): |
258 | bsf %eax, %eax |
259 | lea -16(%rax, %rdi), %rax |
260 | ret |
261 | |
262 | .p2align 4 |
263 | L(matches): |
264 | bsf %eax, %eax |
265 | add %rdi, %rax |
266 | ret |
267 | |
268 | .p2align 4 |
269 | L(matches16): |
270 | bsf %eax, %eax |
271 | lea 16(%rax, %rdi), %rax |
272 | ret |
273 | |
274 | .p2align 4 |
275 | L(matches32): |
276 | bsf %eax, %eax |
277 | lea 32(%rax, %rdi), %rax |
278 | ret |
279 | |
280 | .p2align 4 |
281 | L(matches_1): |
282 | bsf %eax, %eax |
283 | sub %rax, %rdx |
284 | jbe L(return_null) |
285 | add %rdi, %rax |
286 | ret |
287 | |
288 | .p2align 4 |
289 | L(matches16_1): |
290 | bsf %eax, %eax |
291 | sub %rax, %rdx |
292 | jbe L(return_null) |
293 | lea 16(%rdi, %rax), %rax |
294 | ret |
295 | |
296 | .p2align 4 |
297 | L(matches32_1): |
298 | bsf %eax, %eax |
299 | sub %rax, %rdx |
300 | jbe L(return_null) |
301 | lea 32(%rdi, %rax), %rax |
302 | ret |
303 | |
304 | .p2align 4 |
305 | L(matches48_1): |
306 | bsf %eax, %eax |
307 | sub %rax, %rdx |
308 | jbe L(return_null) |
309 | lea 48(%rdi, %rax), %rax |
310 | ret |
311 | |
312 | .p2align 4 |
313 | L(return_null): |
314 | xor %rax, %rax |
315 | ret |
316 | END(memchr) |
317 | |
318 | strong_alias (memchr, __memchr) |
319 | |
320 | libc_hidden_builtin_def(memchr) |
321 | |