1/* Copyright (C) 2011-2016 Free Software Foundation, Inc.
2 Contributed by Intel Corporation.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <sysdep.h>
20
21/* fast SSE2 version with using pmaxub and 64 byte loop */
22
23 .text
24ENTRY(memchr)
25 movd %rsi, %xmm1
26 mov %rdi, %rcx
27#ifdef __ILP32__
28 /* Clear the upper 32 bits. */
29 movl %edx, %edx
30#endif
31 punpcklbw %xmm1, %xmm1
32 test %RDX_LP, %RDX_LP
33 jz L(return_null)
34 punpcklbw %xmm1, %xmm1
35
36 and $63, %rcx
37 pshufd $0, %xmm1, %xmm1
38
39 cmp $48, %rcx
40 ja L(crosscache)
41
42 movdqu (%rdi), %xmm0
43 pcmpeqb %xmm1, %xmm0
44 pmovmskb %xmm0, %eax
45 test %eax, %eax
46
47 jnz L(matches_1)
48 sub $16, %rdx
49 jbe L(return_null)
50 add $16, %rdi
51 and $15, %rcx
52 and $-16, %rdi
53 add %rcx, %rdx
54 sub $64, %rdx
55 jbe L(exit_loop)
56 jmp L(loop_prolog)
57
58 .p2align 4
59L(crosscache):
60 and $15, %rcx
61 and $-16, %rdi
62 movdqa (%rdi), %xmm0
63
64 pcmpeqb %xmm1, %xmm0
65/* Check if there is a match. */
66 pmovmskb %xmm0, %eax
67/* Remove the leading bytes. */
68 sar %cl, %eax
69 test %eax, %eax
70 je L(unaligned_no_match)
71/* Check which byte is a match. */
72 bsf %eax, %eax
73
74 sub %rax, %rdx
75 jbe L(return_null)
76 add %rdi, %rax
77 add %rcx, %rax
78 ret
79
80 .p2align 4
81L(unaligned_no_match):
82 add %rcx, %rdx
83 sub $16, %rdx
84 jbe L(return_null)
85 add $16, %rdi
86 sub $64, %rdx
87 jbe L(exit_loop)
88
89 .p2align 4
90L(loop_prolog):
91 movdqa (%rdi), %xmm0
92 pcmpeqb %xmm1, %xmm0
93 pmovmskb %xmm0, %eax
94 test %eax, %eax
95 jnz L(matches)
96
97 movdqa 16(%rdi), %xmm2
98 pcmpeqb %xmm1, %xmm2
99 pmovmskb %xmm2, %eax
100 test %eax, %eax
101 jnz L(matches16)
102
103 movdqa 32(%rdi), %xmm3
104 pcmpeqb %xmm1, %xmm3
105 pmovmskb %xmm3, %eax
106 test %eax, %eax
107 jnz L(matches32)
108
109 movdqa 48(%rdi), %xmm4
110 pcmpeqb %xmm1, %xmm4
111 add $64, %rdi
112 pmovmskb %xmm4, %eax
113 test %eax, %eax
114 jnz L(matches0)
115
116 test $0x3f, %rdi
117 jz L(align64_loop)
118
119 sub $64, %rdx
120 jbe L(exit_loop)
121
122 movdqa (%rdi), %xmm0
123 pcmpeqb %xmm1, %xmm0
124 pmovmskb %xmm0, %eax
125 test %eax, %eax
126 jnz L(matches)
127
128 movdqa 16(%rdi), %xmm2
129 pcmpeqb %xmm1, %xmm2
130 pmovmskb %xmm2, %eax
131 test %eax, %eax
132 jnz L(matches16)
133
134 movdqa 32(%rdi), %xmm3
135 pcmpeqb %xmm1, %xmm3
136 pmovmskb %xmm3, %eax
137 test %eax, %eax
138 jnz L(matches32)
139
140 movdqa 48(%rdi), %xmm3
141 pcmpeqb %xmm1, %xmm3
142 pmovmskb %xmm3, %eax
143
144 add $64, %rdi
145 test %eax, %eax
146 jnz L(matches0)
147
148 mov %rdi, %rcx
149 and $-64, %rdi
150 and $63, %rcx
151 add %rcx, %rdx
152
153 .p2align 4
154L(align64_loop):
155 sub $64, %rdx
156 jbe L(exit_loop)
157 movdqa (%rdi), %xmm0
158 movdqa 16(%rdi), %xmm2
159 movdqa 32(%rdi), %xmm3
160 movdqa 48(%rdi), %xmm4
161
162 pcmpeqb %xmm1, %xmm0
163 pcmpeqb %xmm1, %xmm2
164 pcmpeqb %xmm1, %xmm3
165 pcmpeqb %xmm1, %xmm4
166
167 pmaxub %xmm0, %xmm3
168 pmaxub %xmm2, %xmm4
169 pmaxub %xmm3, %xmm4
170 pmovmskb %xmm4, %eax
171
172 add $64, %rdi
173
174 test %eax, %eax
175 jz L(align64_loop)
176
177 sub $64, %rdi
178
179 pmovmskb %xmm0, %eax
180 test %eax, %eax
181 jnz L(matches)
182
183 pmovmskb %xmm2, %eax
184 test %eax, %eax
185 jnz L(matches16)
186
187 movdqa 32(%rdi), %xmm3
188 pcmpeqb %xmm1, %xmm3
189
190 pcmpeqb 48(%rdi), %xmm1
191 pmovmskb %xmm3, %eax
192 test %eax, %eax
193 jnz L(matches32)
194
195 pmovmskb %xmm1, %eax
196 bsf %eax, %eax
197 lea 48(%rdi, %rax), %rax
198 ret
199
200 .p2align 4
201L(exit_loop):
202 add $32, %rdx
203 jle L(exit_loop_32)
204
205 movdqa (%rdi), %xmm0
206 pcmpeqb %xmm1, %xmm0
207 pmovmskb %xmm0, %eax
208 test %eax, %eax
209 jnz L(matches)
210
211 movdqa 16(%rdi), %xmm2
212 pcmpeqb %xmm1, %xmm2
213 pmovmskb %xmm2, %eax
214 test %eax, %eax
215 jnz L(matches16)
216
217 movdqa 32(%rdi), %xmm3
218 pcmpeqb %xmm1, %xmm3
219 pmovmskb %xmm3, %eax
220 test %eax, %eax
221 jnz L(matches32_1)
222 sub $16, %rdx
223 jle L(return_null)
224
225 pcmpeqb 48(%rdi), %xmm1
226 pmovmskb %xmm1, %eax
227 test %eax, %eax
228 jnz L(matches48_1)
229 xor %rax, %rax
230 ret
231
232 .p2align 4
233L(exit_loop_32):
234 add $32, %rdx
235 movdqa (%rdi), %xmm0
236 pcmpeqb %xmm1, %xmm0
237 pmovmskb %xmm0, %eax
238 test %eax, %eax
239 jnz L(matches_1)
240 sub $16, %rdx
241 jbe L(return_null)
242
243 pcmpeqb 16(%rdi), %xmm1
244 pmovmskb %xmm1, %eax
245 test %eax, %eax
246 jnz L(matches16_1)
247 xor %rax, %rax
248 ret
249
250 .p2align 4
251L(matches0):
252 bsf %eax, %eax
253 lea -16(%rax, %rdi), %rax
254 ret
255
256 .p2align 4
257L(matches):
258 bsf %eax, %eax
259 add %rdi, %rax
260 ret
261
262 .p2align 4
263L(matches16):
264 bsf %eax, %eax
265 lea 16(%rax, %rdi), %rax
266 ret
267
268 .p2align 4
269L(matches32):
270 bsf %eax, %eax
271 lea 32(%rax, %rdi), %rax
272 ret
273
274 .p2align 4
275L(matches_1):
276 bsf %eax, %eax
277 sub %rax, %rdx
278 jbe L(return_null)
279 add %rdi, %rax
280 ret
281
282 .p2align 4
283L(matches16_1):
284 bsf %eax, %eax
285 sub %rax, %rdx
286 jbe L(return_null)
287 lea 16(%rdi, %rax), %rax
288 ret
289
290 .p2align 4
291L(matches32_1):
292 bsf %eax, %eax
293 sub %rax, %rdx
294 jbe L(return_null)
295 lea 32(%rdi, %rax), %rax
296 ret
297
298 .p2align 4
299L(matches48_1):
300 bsf %eax, %eax
301 sub %rax, %rdx
302 jbe L(return_null)
303 lea 48(%rdi, %rax), %rax
304 ret
305
306 .p2align 4
307L(return_null):
308 xor %rax, %rax
309 ret
310END(memchr)
311
312strong_alias (memchr, __memchr)
313
314libc_hidden_builtin_def(memchr)
315