1 | /* Copyright (C) 2011-2016 Free Software Foundation, Inc. |
2 | Contributed by Intel Corporation. |
3 | This file is part of the GNU C Library. |
4 | |
5 | The GNU C Library is free software; you can redistribute it and/or |
6 | modify it under the terms of the GNU Lesser General Public |
7 | License as published by the Free Software Foundation; either |
8 | version 2.1 of the License, or (at your option) any later version. |
9 | |
10 | The GNU C Library is distributed in the hope that it will be useful, |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | Lesser General Public License for more details. |
14 | |
15 | You should have received a copy of the GNU Lesser General Public |
16 | License along with the GNU C Library; if not, see |
17 | <http://www.gnu.org/licenses/>. */ |
18 | |
19 | #include <sysdep.h> |
20 | |
21 | /* fast SSE2 version with using pmaxub and 64 byte loop */ |
22 | |
23 | .text |
24 | ENTRY(memchr) |
25 | movd %rsi, %xmm1 |
26 | mov %rdi, %rcx |
27 | #ifdef __ILP32__ |
28 | /* Clear the upper 32 bits. */ |
29 | movl %edx, %edx |
30 | #endif |
31 | punpcklbw %xmm1, %xmm1 |
32 | test %RDX_LP, %RDX_LP |
33 | jz L(return_null) |
34 | punpcklbw %xmm1, %xmm1 |
35 | |
36 | and $63, %rcx |
37 | pshufd $0, %xmm1, %xmm1 |
38 | |
39 | cmp $48, %rcx |
40 | ja L(crosscache) |
41 | |
42 | movdqu (%rdi), %xmm0 |
43 | pcmpeqb %xmm1, %xmm0 |
44 | pmovmskb %xmm0, %eax |
45 | test %eax, %eax |
46 | |
47 | jnz L(matches_1) |
48 | sub $16, %rdx |
49 | jbe L(return_null) |
50 | add $16, %rdi |
51 | and $15, %rcx |
52 | and $-16, %rdi |
53 | add %rcx, %rdx |
54 | sub $64, %rdx |
55 | jbe L(exit_loop) |
56 | jmp L(loop_prolog) |
57 | |
58 | .p2align 4 |
59 | L(crosscache): |
60 | and $15, %rcx |
61 | and $-16, %rdi |
62 | movdqa (%rdi), %xmm0 |
63 | |
64 | pcmpeqb %xmm1, %xmm0 |
65 | /* Check if there is a match. */ |
66 | pmovmskb %xmm0, %eax |
67 | /* Remove the leading bytes. */ |
68 | sar %cl, %eax |
69 | test %eax, %eax |
70 | je L(unaligned_no_match) |
71 | /* Check which byte is a match. */ |
72 | bsf %eax, %eax |
73 | |
74 | sub %rax, %rdx |
75 | jbe L(return_null) |
76 | add %rdi, %rax |
77 | add %rcx, %rax |
78 | ret |
79 | |
80 | .p2align 4 |
81 | L(unaligned_no_match): |
82 | add %rcx, %rdx |
83 | sub $16, %rdx |
84 | jbe L(return_null) |
85 | add $16, %rdi |
86 | sub $64, %rdx |
87 | jbe L(exit_loop) |
88 | |
89 | .p2align 4 |
90 | L(loop_prolog): |
91 | movdqa (%rdi), %xmm0 |
92 | pcmpeqb %xmm1, %xmm0 |
93 | pmovmskb %xmm0, %eax |
94 | test %eax, %eax |
95 | jnz L(matches) |
96 | |
97 | movdqa 16(%rdi), %xmm2 |
98 | pcmpeqb %xmm1, %xmm2 |
99 | pmovmskb %xmm2, %eax |
100 | test %eax, %eax |
101 | jnz L(matches16) |
102 | |
103 | movdqa 32(%rdi), %xmm3 |
104 | pcmpeqb %xmm1, %xmm3 |
105 | pmovmskb %xmm3, %eax |
106 | test %eax, %eax |
107 | jnz L(matches32) |
108 | |
109 | movdqa 48(%rdi), %xmm4 |
110 | pcmpeqb %xmm1, %xmm4 |
111 | add $64, %rdi |
112 | pmovmskb %xmm4, %eax |
113 | test %eax, %eax |
114 | jnz L(matches0) |
115 | |
116 | test $0x3f, %rdi |
117 | jz L(align64_loop) |
118 | |
119 | sub $64, %rdx |
120 | jbe L(exit_loop) |
121 | |
122 | movdqa (%rdi), %xmm0 |
123 | pcmpeqb %xmm1, %xmm0 |
124 | pmovmskb %xmm0, %eax |
125 | test %eax, %eax |
126 | jnz L(matches) |
127 | |
128 | movdqa 16(%rdi), %xmm2 |
129 | pcmpeqb %xmm1, %xmm2 |
130 | pmovmskb %xmm2, %eax |
131 | test %eax, %eax |
132 | jnz L(matches16) |
133 | |
134 | movdqa 32(%rdi), %xmm3 |
135 | pcmpeqb %xmm1, %xmm3 |
136 | pmovmskb %xmm3, %eax |
137 | test %eax, %eax |
138 | jnz L(matches32) |
139 | |
140 | movdqa 48(%rdi), %xmm3 |
141 | pcmpeqb %xmm1, %xmm3 |
142 | pmovmskb %xmm3, %eax |
143 | |
144 | add $64, %rdi |
145 | test %eax, %eax |
146 | jnz L(matches0) |
147 | |
148 | mov %rdi, %rcx |
149 | and $-64, %rdi |
150 | and $63, %rcx |
151 | add %rcx, %rdx |
152 | |
153 | .p2align 4 |
154 | L(align64_loop): |
155 | sub $64, %rdx |
156 | jbe L(exit_loop) |
157 | movdqa (%rdi), %xmm0 |
158 | movdqa 16(%rdi), %xmm2 |
159 | movdqa 32(%rdi), %xmm3 |
160 | movdqa 48(%rdi), %xmm4 |
161 | |
162 | pcmpeqb %xmm1, %xmm0 |
163 | pcmpeqb %xmm1, %xmm2 |
164 | pcmpeqb %xmm1, %xmm3 |
165 | pcmpeqb %xmm1, %xmm4 |
166 | |
167 | pmaxub %xmm0, %xmm3 |
168 | pmaxub %xmm2, %xmm4 |
169 | pmaxub %xmm3, %xmm4 |
170 | pmovmskb %xmm4, %eax |
171 | |
172 | add $64, %rdi |
173 | |
174 | test %eax, %eax |
175 | jz L(align64_loop) |
176 | |
177 | sub $64, %rdi |
178 | |
179 | pmovmskb %xmm0, %eax |
180 | test %eax, %eax |
181 | jnz L(matches) |
182 | |
183 | pmovmskb %xmm2, %eax |
184 | test %eax, %eax |
185 | jnz L(matches16) |
186 | |
187 | movdqa 32(%rdi), %xmm3 |
188 | pcmpeqb %xmm1, %xmm3 |
189 | |
190 | pcmpeqb 48(%rdi), %xmm1 |
191 | pmovmskb %xmm3, %eax |
192 | test %eax, %eax |
193 | jnz L(matches32) |
194 | |
195 | pmovmskb %xmm1, %eax |
196 | bsf %eax, %eax |
197 | lea 48(%rdi, %rax), %rax |
198 | ret |
199 | |
200 | .p2align 4 |
201 | L(exit_loop): |
202 | add $32, %rdx |
203 | jle L(exit_loop_32) |
204 | |
205 | movdqa (%rdi), %xmm0 |
206 | pcmpeqb %xmm1, %xmm0 |
207 | pmovmskb %xmm0, %eax |
208 | test %eax, %eax |
209 | jnz L(matches) |
210 | |
211 | movdqa 16(%rdi), %xmm2 |
212 | pcmpeqb %xmm1, %xmm2 |
213 | pmovmskb %xmm2, %eax |
214 | test %eax, %eax |
215 | jnz L(matches16) |
216 | |
217 | movdqa 32(%rdi), %xmm3 |
218 | pcmpeqb %xmm1, %xmm3 |
219 | pmovmskb %xmm3, %eax |
220 | test %eax, %eax |
221 | jnz L(matches32_1) |
222 | sub $16, %rdx |
223 | jle L(return_null) |
224 | |
225 | pcmpeqb 48(%rdi), %xmm1 |
226 | pmovmskb %xmm1, %eax |
227 | test %eax, %eax |
228 | jnz L(matches48_1) |
229 | xor %rax, %rax |
230 | ret |
231 | |
232 | .p2align 4 |
233 | L(exit_loop_32): |
234 | add $32, %rdx |
235 | movdqa (%rdi), %xmm0 |
236 | pcmpeqb %xmm1, %xmm0 |
237 | pmovmskb %xmm0, %eax |
238 | test %eax, %eax |
239 | jnz L(matches_1) |
240 | sub $16, %rdx |
241 | jbe L(return_null) |
242 | |
243 | pcmpeqb 16(%rdi), %xmm1 |
244 | pmovmskb %xmm1, %eax |
245 | test %eax, %eax |
246 | jnz L(matches16_1) |
247 | xor %rax, %rax |
248 | ret |
249 | |
250 | .p2align 4 |
251 | L(matches0): |
252 | bsf %eax, %eax |
253 | lea -16(%rax, %rdi), %rax |
254 | ret |
255 | |
256 | .p2align 4 |
257 | L(matches): |
258 | bsf %eax, %eax |
259 | add %rdi, %rax |
260 | ret |
261 | |
262 | .p2align 4 |
263 | L(matches16): |
264 | bsf %eax, %eax |
265 | lea 16(%rax, %rdi), %rax |
266 | ret |
267 | |
268 | .p2align 4 |
269 | L(matches32): |
270 | bsf %eax, %eax |
271 | lea 32(%rax, %rdi), %rax |
272 | ret |
273 | |
274 | .p2align 4 |
275 | L(matches_1): |
276 | bsf %eax, %eax |
277 | sub %rax, %rdx |
278 | jbe L(return_null) |
279 | add %rdi, %rax |
280 | ret |
281 | |
282 | .p2align 4 |
283 | L(matches16_1): |
284 | bsf %eax, %eax |
285 | sub %rax, %rdx |
286 | jbe L(return_null) |
287 | lea 16(%rdi, %rax), %rax |
288 | ret |
289 | |
290 | .p2align 4 |
291 | L(matches32_1): |
292 | bsf %eax, %eax |
293 | sub %rax, %rdx |
294 | jbe L(return_null) |
295 | lea 32(%rdi, %rax), %rax |
296 | ret |
297 | |
298 | .p2align 4 |
299 | L(matches48_1): |
300 | bsf %eax, %eax |
301 | sub %rax, %rdx |
302 | jbe L(return_null) |
303 | lea 48(%rdi, %rax), %rax |
304 | ret |
305 | |
306 | .p2align 4 |
307 | L(return_null): |
308 | xor %rax, %rax |
309 | ret |
310 | END(memchr) |
311 | |
312 | strong_alias (memchr, __memchr) |
313 | |
314 | libc_hidden_builtin_def(memchr) |
315 | |