xref: /llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-bit-counting.ll (revision 61510b51c33464a6bc15e4cf5b1ee07e2e0ec1c9) 
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
3; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s
4; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
5
6target triple = "aarch64-unknown-linux-gnu"
7
8;
9; CLZ
10;
11
12define <4 x i8> @ctlz_v4i8(<4 x i8> %op) {
13; CHECK-LABEL: ctlz_v4i8:
14; CHECK:       // %bb.0:
15; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
16; CHECK-NEXT:    ptrue p0.h, vl4
17; CHECK-NEXT:    and z0.h, z0.h, #0xff
18; CHECK-NEXT:    clz z0.h, p0/m, z0.h
19; CHECK-NEXT:    sub z0.h, z0.h, #8 // =0x8
20; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
21; CHECK-NEXT:    ret
22;
23; NONEON-NOSVE-LABEL: ctlz_v4i8:
24; NONEON-NOSVE:       // %bb.0:
25; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
26; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
27; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]
28; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
29; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #2]
30; NONEON-NOSVE-NEXT:    ldrb w11, [sp]
31; NONEON-NOSVE-NEXT:    clz w8, w8
32; NONEON-NOSVE-NEXT:    clz w9, w9
33; NONEON-NOSVE-NEXT:    clz w10, w10
34; NONEON-NOSVE-NEXT:    sub w8, w8, #24
35; NONEON-NOSVE-NEXT:    sub w9, w9, #24
36; NONEON-NOSVE-NEXT:    sub w10, w10, #24
37; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]
38; NONEON-NOSVE-NEXT:    clz w8, w11
39; NONEON-NOSVE-NEXT:    sub w8, w8, #24
40; NONEON-NOSVE-NEXT:    strh w9, [sp, #12]
41; NONEON-NOSVE-NEXT:    strh w10, [sp, #10]
42; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]
43; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
44; NONEON-NOSVE-NEXT:    add sp, sp, #16
45; NONEON-NOSVE-NEXT:    ret
46  %res = call <4 x i8> @llvm.ctlz.v4i8(<4 x i8> %op)
47  ret <4 x i8> %res
48}
49
50define <8 x i8> @ctlz_v8i8(<8 x i8> %op) {
51; CHECK-LABEL: ctlz_v8i8:
52; CHECK:       // %bb.0:
53; CHECK-NEXT:    ptrue p0.b, vl8
54; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
55; CHECK-NEXT:    clz z0.b, p0/m, z0.b
56; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
57; CHECK-NEXT:    ret
58;
59; NONEON-NOSVE-LABEL: ctlz_v8i8:
60; NONEON-NOSVE:       // %bb.0:
61; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
62; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
63; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]
64; NONEON-NOSVE-NEXT:    clz w8, w8
65; NONEON-NOSVE-NEXT:    sub w8, w8, #24
66; NONEON-NOSVE-NEXT:    strb w8, [sp, #15]
67; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]
68; NONEON-NOSVE-NEXT:    clz w8, w8
69; NONEON-NOSVE-NEXT:    sub w8, w8, #24
70; NONEON-NOSVE-NEXT:    strb w8, [sp, #14]
71; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]
72; NONEON-NOSVE-NEXT:    clz w8, w8
73; NONEON-NOSVE-NEXT:    sub w8, w8, #24
74; NONEON-NOSVE-NEXT:    strb w8, [sp, #13]
75; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]
76; NONEON-NOSVE-NEXT:    clz w8, w8
77; NONEON-NOSVE-NEXT:    sub w8, w8, #24
78; NONEON-NOSVE-NEXT:    strb w8, [sp, #12]
79; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]
80; NONEON-NOSVE-NEXT:    clz w8, w8
81; NONEON-NOSVE-NEXT:    sub w8, w8, #24
82; NONEON-NOSVE-NEXT:    strb w8, [sp, #11]
83; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
84; NONEON-NOSVE-NEXT:    clz w8, w8
85; NONEON-NOSVE-NEXT:    sub w8, w8, #24
86; NONEON-NOSVE-NEXT:    strb w8, [sp, #10]
87; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]
88; NONEON-NOSVE-NEXT:    clz w8, w8
89; NONEON-NOSVE-NEXT:    sub w8, w8, #24
90; NONEON-NOSVE-NEXT:    strb w8, [sp, #9]
91; NONEON-NOSVE-NEXT:    ldrb w8, [sp]
92; NONEON-NOSVE-NEXT:    clz w8, w8
93; NONEON-NOSVE-NEXT:    sub w8, w8, #24
94; NONEON-NOSVE-NEXT:    strb w8, [sp, #8]
95; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
96; NONEON-NOSVE-NEXT:    add sp, sp, #16
97; NONEON-NOSVE-NEXT:    ret
98  %res = call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %op)
99  ret <8 x i8> %res
100}
101
102define <16 x i8> @ctlz_v16i8(<16 x i8> %op) {
103; CHECK-LABEL: ctlz_v16i8:
104; CHECK:       // %bb.0:
105; CHECK-NEXT:    ptrue p0.b, vl16
106; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
107; CHECK-NEXT:    clz z0.b, p0/m, z0.b
108; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
109; CHECK-NEXT:    ret
110;
111; NONEON-NOSVE-LABEL: ctlz_v16i8:
112; NONEON-NOSVE:       // %bb.0:
113; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
114; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
115; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]
116; NONEON-NOSVE-NEXT:    clz w8, w8
117; NONEON-NOSVE-NEXT:    sub w8, w8, #24
118; NONEON-NOSVE-NEXT:    strb w8, [sp, #31]
119; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
120; NONEON-NOSVE-NEXT:    clz w8, w8
121; NONEON-NOSVE-NEXT:    sub w8, w8, #24
122; NONEON-NOSVE-NEXT:    strb w8, [sp, #30]
123; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]
124; NONEON-NOSVE-NEXT:    clz w8, w8
125; NONEON-NOSVE-NEXT:    sub w8, w8, #24
126; NONEON-NOSVE-NEXT:    strb w8, [sp, #29]
127; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
128; NONEON-NOSVE-NEXT:    clz w8, w8
129; NONEON-NOSVE-NEXT:    sub w8, w8, #24
130; NONEON-NOSVE-NEXT:    strb w8, [sp, #28]
131; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]
132; NONEON-NOSVE-NEXT:    clz w8, w8
133; NONEON-NOSVE-NEXT:    sub w8, w8, #24
134; NONEON-NOSVE-NEXT:    strb w8, [sp, #27]
135; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
136; NONEON-NOSVE-NEXT:    clz w8, w8
137; NONEON-NOSVE-NEXT:    sub w8, w8, #24
138; NONEON-NOSVE-NEXT:    strb w8, [sp, #26]
139; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]
140; NONEON-NOSVE-NEXT:    clz w8, w8
141; NONEON-NOSVE-NEXT:    sub w8, w8, #24
142; NONEON-NOSVE-NEXT:    strb w8, [sp, #25]
143; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
144; NONEON-NOSVE-NEXT:    clz w8, w8
145; NONEON-NOSVE-NEXT:    sub w8, w8, #24
146; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]
147; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]
148; NONEON-NOSVE-NEXT:    clz w8, w8
149; NONEON-NOSVE-NEXT:    sub w8, w8, #24
150; NONEON-NOSVE-NEXT:    strb w8, [sp, #23]
151; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]
152; NONEON-NOSVE-NEXT:    clz w8, w8
153; NONEON-NOSVE-NEXT:    sub w8, w8, #24
154; NONEON-NOSVE-NEXT:    strb w8, [sp, #22]
155; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]
156; NONEON-NOSVE-NEXT:    clz w8, w8
157; NONEON-NOSVE-NEXT:    sub w8, w8, #24
158; NONEON-NOSVE-NEXT:    strb w8, [sp, #21]
159; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]
160; NONEON-NOSVE-NEXT:    clz w8, w8
161; NONEON-NOSVE-NEXT:    sub w8, w8, #24
162; NONEON-NOSVE-NEXT:    strb w8, [sp, #20]
163; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]
164; NONEON-NOSVE-NEXT:    clz w8, w8
165; NONEON-NOSVE-NEXT:    sub w8, w8, #24
166; NONEON-NOSVE-NEXT:    strb w8, [sp, #19]
167; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
168; NONEON-NOSVE-NEXT:    clz w8, w8
169; NONEON-NOSVE-NEXT:    sub w8, w8, #24
170; NONEON-NOSVE-NEXT:    strb w8, [sp, #18]
171; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]
172; NONEON-NOSVE-NEXT:    clz w8, w8
173; NONEON-NOSVE-NEXT:    sub w8, w8, #24
174; NONEON-NOSVE-NEXT:    strb w8, [sp, #17]
175; NONEON-NOSVE-NEXT:    ldrb w8, [sp]
176; NONEON-NOSVE-NEXT:    clz w8, w8
177; NONEON-NOSVE-NEXT:    sub w8, w8, #24
178; NONEON-NOSVE-NEXT:    strb w8, [sp, #16]
179; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
180; NONEON-NOSVE-NEXT:    add sp, sp, #32
181; NONEON-NOSVE-NEXT:    ret
182  %res = call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %op)
183  ret <16 x i8> %res
184}
185
186define void @ctlz_v32i8(ptr %a) {
187; CHECK-LABEL: ctlz_v32i8:
188; CHECK:       // %bb.0:
189; CHECK-NEXT:    ldp q0, q1, [x0]
190; CHECK-NEXT:    ptrue p0.b, vl16
191; CHECK-NEXT:    clz z0.b, p0/m, z0.b
192; CHECK-NEXT:    clz z1.b, p0/m, z1.b
193; CHECK-NEXT:    stp q0, q1, [x0]
194; CHECK-NEXT:    ret
195;
196; NONEON-NOSVE-LABEL: ctlz_v32i8:
197; NONEON-NOSVE:       // %bb.0:
198; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
199; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
200; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
201; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
202; NONEON-NOSVE-NEXT:    clz w8, w8
203; NONEON-NOSVE-NEXT:    sub w8, w8, #24
204; NONEON-NOSVE-NEXT:    strb w8, [sp, #63]
205; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
206; NONEON-NOSVE-NEXT:    clz w8, w8
207; NONEON-NOSVE-NEXT:    sub w8, w8, #24
208; NONEON-NOSVE-NEXT:    strb w8, [sp, #62]
209; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
210; NONEON-NOSVE-NEXT:    clz w8, w8
211; NONEON-NOSVE-NEXT:    sub w8, w8, #24
212; NONEON-NOSVE-NEXT:    strb w8, [sp, #61]
213; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
214; NONEON-NOSVE-NEXT:    clz w8, w8
215; NONEON-NOSVE-NEXT:    sub w8, w8, #24
216; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
217; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
218; NONEON-NOSVE-NEXT:    clz w8, w8
219; NONEON-NOSVE-NEXT:    sub w8, w8, #24
220; NONEON-NOSVE-NEXT:    strb w8, [sp, #59]
221; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
222; NONEON-NOSVE-NEXT:    clz w8, w8
223; NONEON-NOSVE-NEXT:    sub w8, w8, #24
224; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
225; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
226; NONEON-NOSVE-NEXT:    clz w8, w8
227; NONEON-NOSVE-NEXT:    sub w8, w8, #24
228; NONEON-NOSVE-NEXT:    strb w8, [sp, #57]
229; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
230; NONEON-NOSVE-NEXT:    clz w8, w8
231; NONEON-NOSVE-NEXT:    sub w8, w8, #24
232; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
233; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
234; NONEON-NOSVE-NEXT:    clz w8, w8
235; NONEON-NOSVE-NEXT:    sub w8, w8, #24
236; NONEON-NOSVE-NEXT:    strb w8, [sp, #55]
237; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
238; NONEON-NOSVE-NEXT:    clz w8, w8
239; NONEON-NOSVE-NEXT:    sub w8, w8, #24
240; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
241; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
242; NONEON-NOSVE-NEXT:    clz w8, w8
243; NONEON-NOSVE-NEXT:    sub w8, w8, #24
244; NONEON-NOSVE-NEXT:    strb w8, [sp, #53]
245; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
246; NONEON-NOSVE-NEXT:    clz w8, w8
247; NONEON-NOSVE-NEXT:    sub w8, w8, #24
248; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
249; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
250; NONEON-NOSVE-NEXT:    clz w8, w8
251; NONEON-NOSVE-NEXT:    sub w8, w8, #24
252; NONEON-NOSVE-NEXT:    strb w8, [sp, #51]
253; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
254; NONEON-NOSVE-NEXT:    clz w8, w8
255; NONEON-NOSVE-NEXT:    sub w8, w8, #24
256; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
257; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
258; NONEON-NOSVE-NEXT:    clz w8, w8
259; NONEON-NOSVE-NEXT:    sub w8, w8, #24
260; NONEON-NOSVE-NEXT:    strb w8, [sp, #49]
261; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
262; NONEON-NOSVE-NEXT:    clz w8, w8
263; NONEON-NOSVE-NEXT:    sub w8, w8, #24
264; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
265; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]
266; NONEON-NOSVE-NEXT:    clz w8, w8
267; NONEON-NOSVE-NEXT:    sub w8, w8, #24
268; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]
269; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
270; NONEON-NOSVE-NEXT:    clz w8, w8
271; NONEON-NOSVE-NEXT:    sub w8, w8, #24
272; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]
273; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]
274; NONEON-NOSVE-NEXT:    clz w8, w8
275; NONEON-NOSVE-NEXT:    sub w8, w8, #24
276; NONEON-NOSVE-NEXT:    strb w8, [sp, #45]
277; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
278; NONEON-NOSVE-NEXT:    clz w8, w8
279; NONEON-NOSVE-NEXT:    sub w8, w8, #24
280; NONEON-NOSVE-NEXT:    strb w8, [sp, #44]
281; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]
282; NONEON-NOSVE-NEXT:    clz w8, w8
283; NONEON-NOSVE-NEXT:    sub w8, w8, #24
284; NONEON-NOSVE-NEXT:    strb w8, [sp, #43]
285; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
286; NONEON-NOSVE-NEXT:    clz w8, w8
287; NONEON-NOSVE-NEXT:    sub w8, w8, #24
288; NONEON-NOSVE-NEXT:    strb w8, [sp, #42]
289; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]
290; NONEON-NOSVE-NEXT:    clz w8, w8
291; NONEON-NOSVE-NEXT:    sub w8, w8, #24
292; NONEON-NOSVE-NEXT:    strb w8, [sp, #41]
293; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
294; NONEON-NOSVE-NEXT:    clz w8, w8
295; NONEON-NOSVE-NEXT:    sub w8, w8, #24
296; NONEON-NOSVE-NEXT:    strb w8, [sp, #40]
297; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]
298; NONEON-NOSVE-NEXT:    clz w8, w8
299; NONEON-NOSVE-NEXT:    sub w8, w8, #24
300; NONEON-NOSVE-NEXT:    strb w8, [sp, #39]
301; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]
302; NONEON-NOSVE-NEXT:    clz w8, w8
303; NONEON-NOSVE-NEXT:    sub w8, w8, #24
304; NONEON-NOSVE-NEXT:    strb w8, [sp, #38]
305; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]
306; NONEON-NOSVE-NEXT:    clz w8, w8
307; NONEON-NOSVE-NEXT:    sub w8, w8, #24
308; NONEON-NOSVE-NEXT:    strb w8, [sp, #37]
309; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]
310; NONEON-NOSVE-NEXT:    clz w8, w8
311; NONEON-NOSVE-NEXT:    sub w8, w8, #24
312; NONEON-NOSVE-NEXT:    strb w8, [sp, #36]
313; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]
314; NONEON-NOSVE-NEXT:    clz w8, w8
315; NONEON-NOSVE-NEXT:    sub w8, w8, #24
316; NONEON-NOSVE-NEXT:    strb w8, [sp, #35]
317; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
318; NONEON-NOSVE-NEXT:    clz w8, w8
319; NONEON-NOSVE-NEXT:    sub w8, w8, #24
320; NONEON-NOSVE-NEXT:    strb w8, [sp, #34]
321; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]
322; NONEON-NOSVE-NEXT:    clz w8, w8
323; NONEON-NOSVE-NEXT:    sub w8, w8, #24
324; NONEON-NOSVE-NEXT:    strb w8, [sp, #33]
325; NONEON-NOSVE-NEXT:    ldrb w8, [sp]
326; NONEON-NOSVE-NEXT:    clz w8, w8
327; NONEON-NOSVE-NEXT:    sub w8, w8, #24
328; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]
329; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
330; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
331; NONEON-NOSVE-NEXT:    add sp, sp, #64
332; NONEON-NOSVE-NEXT:    ret
333  %op = load <32 x i8>, ptr %a
334  %res = call <32 x i8> @llvm.ctlz.v32i8(<32 x i8> %op)
335  store <32 x i8> %res, ptr %a
336  ret void
337}
338
339define <2 x i16> @ctlz_v2i16(<2 x i16> %op) {
340; CHECK-LABEL: ctlz_v2i16:
341; CHECK:       // %bb.0:
342; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
343; CHECK-NEXT:    ptrue p0.s, vl2
344; CHECK-NEXT:    and z0.s, z0.s, #0xffff
345; CHECK-NEXT:    clz z0.s, p0/m, z0.s
346; CHECK-NEXT:    sub z0.s, z0.s, #16 // =0x10
347; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
348; CHECK-NEXT:    ret
349;
350; NONEON-NOSVE-LABEL: ctlz_v2i16:
351; NONEON-NOSVE:       // %bb.0:
352; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
353; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
354; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]
355; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
356; NONEON-NOSVE-NEXT:    clz w8, w8
357; NONEON-NOSVE-NEXT:    clz w9, w9
358; NONEON-NOSVE-NEXT:    sub w8, w8, #16
359; NONEON-NOSVE-NEXT:    sub w9, w9, #16
360; NONEON-NOSVE-NEXT:    stp w9, w8, [sp, #8]
361; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
362; NONEON-NOSVE-NEXT:    add sp, sp, #16
363; NONEON-NOSVE-NEXT:    ret
364  %res = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %op)
365  ret <2 x i16> %res
366}
367
368define <4 x i16> @ctlz_v4i16(<4 x i16> %op) {
369; CHECK-LABEL: ctlz_v4i16:
370; CHECK:       // %bb.0:
371; CHECK-NEXT:    ptrue p0.h, vl4
372; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
373; CHECK-NEXT:    clz z0.h, p0/m, z0.h
374; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
375; CHECK-NEXT:    ret
376;
377; NONEON-NOSVE-LABEL: ctlz_v4i16:
378; NONEON-NOSVE:       // %bb.0:
379; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
380; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
381; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]
382; NONEON-NOSVE-NEXT:    clz w8, w8
383; NONEON-NOSVE-NEXT:    sub w8, w8, #16
384; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]
385; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]
386; NONEON-NOSVE-NEXT:    clz w8, w8
387; NONEON-NOSVE-NEXT:    sub w8, w8, #16
388; NONEON-NOSVE-NEXT:    strh w8, [sp, #12]
389; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
390; NONEON-NOSVE-NEXT:    clz w8, w8
391; NONEON-NOSVE-NEXT:    sub w8, w8, #16
392; NONEON-NOSVE-NEXT:    strh w8, [sp, #10]
393; NONEON-NOSVE-NEXT:    ldrh w8, [sp]
394; NONEON-NOSVE-NEXT:    clz w8, w8
395; NONEON-NOSVE-NEXT:    sub w8, w8, #16
396; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]
397; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
398; NONEON-NOSVE-NEXT:    add sp, sp, #16
399; NONEON-NOSVE-NEXT:    ret
400  %res = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %op)
401  ret <4 x i16> %res
402}
403
404define <8 x i16> @ctlz_v8i16(<8 x i16> %op) {
405; CHECK-LABEL: ctlz_v8i16:
406; CHECK:       // %bb.0:
407; CHECK-NEXT:    ptrue p0.h, vl8
408; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
409; CHECK-NEXT:    clz z0.h, p0/m, z0.h
410; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
411; CHECK-NEXT:    ret
412;
413; NONEON-NOSVE-LABEL: ctlz_v8i16:
414; NONEON-NOSVE:       // %bb.0:
415; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
416; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
417; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]
418; NONEON-NOSVE-NEXT:    clz w8, w8
419; NONEON-NOSVE-NEXT:    sub w8, w8, #16
420; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
421; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
422; NONEON-NOSVE-NEXT:    clz w8, w8
423; NONEON-NOSVE-NEXT:    sub w8, w8, #16
424; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
425; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]
426; NONEON-NOSVE-NEXT:    clz w8, w8
427; NONEON-NOSVE-NEXT:    sub w8, w8, #16
428; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
429; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]
430; NONEON-NOSVE-NEXT:    clz w8, w8
431; NONEON-NOSVE-NEXT:    sub w8, w8, #16
432; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
433; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]
434; NONEON-NOSVE-NEXT:    clz w8, w8
435; NONEON-NOSVE-NEXT:    sub w8, w8, #16
436; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]
437; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]
438; NONEON-NOSVE-NEXT:    clz w8, w8
439; NONEON-NOSVE-NEXT:    sub w8, w8, #16
440; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]
441; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
442; NONEON-NOSVE-NEXT:    clz w8, w8
443; NONEON-NOSVE-NEXT:    sub w8, w8, #16
444; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]
445; NONEON-NOSVE-NEXT:    ldrh w8, [sp]
446; NONEON-NOSVE-NEXT:    clz w8, w8
447; NONEON-NOSVE-NEXT:    sub w8, w8, #16
448; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
449; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
450; NONEON-NOSVE-NEXT:    add sp, sp, #32
451; NONEON-NOSVE-NEXT:    ret
452  %res = call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %op)
453  ret <8 x i16> %res
454}
455
456define void @ctlz_v16i16(ptr %a) {
457; CHECK-LABEL: ctlz_v16i16:
458; CHECK:       // %bb.0:
459; CHECK-NEXT:    ldp q0, q1, [x0]
460; CHECK-NEXT:    ptrue p0.h, vl8
461; CHECK-NEXT:    clz z0.h, p0/m, z0.h
462; CHECK-NEXT:    clz z1.h, p0/m, z1.h
463; CHECK-NEXT:    stp q0, q1, [x0]
464; CHECK-NEXT:    ret
465;
466; NONEON-NOSVE-LABEL: ctlz_v16i16:
467; NONEON-NOSVE:       // %bb.0:
468; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
469; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
470; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
471; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
472; NONEON-NOSVE-NEXT:    clz w8, w8
473; NONEON-NOSVE-NEXT:    sub w8, w8, #16
474; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
475; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
476; NONEON-NOSVE-NEXT:    clz w8, w8
477; NONEON-NOSVE-NEXT:    sub w8, w8, #16
478; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
479; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
480; NONEON-NOSVE-NEXT:    clz w8, w8
481; NONEON-NOSVE-NEXT:    sub w8, w8, #16
482; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
483; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
484; NONEON-NOSVE-NEXT:    clz w8, w8
485; NONEON-NOSVE-NEXT:    sub w8, w8, #16
486; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
487; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
488; NONEON-NOSVE-NEXT:    clz w8, w8
489; NONEON-NOSVE-NEXT:    sub w8, w8, #16
490; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
491; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
492; NONEON-NOSVE-NEXT:    clz w8, w8
493; NONEON-NOSVE-NEXT:    sub w8, w8, #16
494; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
495; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
496; NONEON-NOSVE-NEXT:    clz w8, w8
497; NONEON-NOSVE-NEXT:    sub w8, w8, #16
498; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
499; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
500; NONEON-NOSVE-NEXT:    clz w8, w8
501; NONEON-NOSVE-NEXT:    sub w8, w8, #16
502; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
503; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]
504; NONEON-NOSVE-NEXT:    clz w8, w8
505; NONEON-NOSVE-NEXT:    sub w8, w8, #16
506; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
507; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
508; NONEON-NOSVE-NEXT:    clz w8, w8
509; NONEON-NOSVE-NEXT:    sub w8, w8, #16
510; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
511; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]
512; NONEON-NOSVE-NEXT:    clz w8, w8
513; NONEON-NOSVE-NEXT:    sub w8, w8, #16
514; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
515; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]
516; NONEON-NOSVE-NEXT:    clz w8, w8
517; NONEON-NOSVE-NEXT:    sub w8, w8, #16
518; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
519; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]
520; NONEON-NOSVE-NEXT:    clz w8, w8
521; NONEON-NOSVE-NEXT:    sub w8, w8, #16
522; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
523; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]
524; NONEON-NOSVE-NEXT:    clz w8, w8
525; NONEON-NOSVE-NEXT:    sub w8, w8, #16
526; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
527; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
528; NONEON-NOSVE-NEXT:    clz w8, w8
529; NONEON-NOSVE-NEXT:    sub w8, w8, #16
530; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
531; NONEON-NOSVE-NEXT:    ldrh w8, [sp]
532; NONEON-NOSVE-NEXT:    clz w8, w8
533; NONEON-NOSVE-NEXT:    sub w8, w8, #16
534; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
535; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
536; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
537; NONEON-NOSVE-NEXT:    add sp, sp, #64
538; NONEON-NOSVE-NEXT:    ret
539  %op = load <16 x i16>, ptr %a
540  %res = call <16 x i16> @llvm.ctlz.v16i16(<16 x i16> %op)
541  store <16 x i16> %res, ptr %a
542  ret void
543}
544
545define <2 x i32> @ctlz_v2i32(<2 x i32> %op) {
546; CHECK-LABEL: ctlz_v2i32:
547; CHECK:       // %bb.0:
548; CHECK-NEXT:    ptrue p0.s, vl2
549; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
550; CHECK-NEXT:    clz z0.s, p0/m, z0.s
551; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
552; CHECK-NEXT:    ret
553;
554; NONEON-NOSVE-LABEL: ctlz_v2i32:
555; NONEON-NOSVE:       // %bb.0:
556; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
557; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
558; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
559; NONEON-NOSVE-NEXT:    clz w9, w8
560; NONEON-NOSVE-NEXT:    ldr w8, [sp]
561; NONEON-NOSVE-NEXT:    clz w8, w8
562; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8]
563; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
564; NONEON-NOSVE-NEXT:    add sp, sp, #16
565; NONEON-NOSVE-NEXT:    ret
566  %res = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %op)
567  ret <2 x i32> %res
568}
569
570define <4 x i32> @ctlz_v4i32(<4 x i32> %op) {
571; CHECK-LABEL: ctlz_v4i32:
572; CHECK:       // %bb.0:
573; CHECK-NEXT:    ptrue p0.s, vl4
574; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
575; CHECK-NEXT:    clz z0.s, p0/m, z0.s
576; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
577; CHECK-NEXT:    ret
578;
579; NONEON-NOSVE-LABEL: ctlz_v4i32:
580; NONEON-NOSVE:       // %bb.0:
581; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
582; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
583; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]
584; NONEON-NOSVE-NEXT:    clz w9, w8
585; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
586; NONEON-NOSVE-NEXT:    clz w8, w8
587; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
588; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
589; NONEON-NOSVE-NEXT:    clz w9, w8
590; NONEON-NOSVE-NEXT:    ldr w8, [sp]
591; NONEON-NOSVE-NEXT:    clz w8, w8
592; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
593; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
594; NONEON-NOSVE-NEXT:    add sp, sp, #32
595; NONEON-NOSVE-NEXT:    ret
596  %res = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %op)
597  ret <4 x i32> %res
598}
599
600define void @ctlz_v8i32(ptr %a) {
601; CHECK-LABEL: ctlz_v8i32:
602; CHECK:       // %bb.0:
603; CHECK-NEXT:    ldp q0, q1, [x0]
604; CHECK-NEXT:    ptrue p0.s, vl4
605; CHECK-NEXT:    clz z0.s, p0/m, z0.s
606; CHECK-NEXT:    clz z1.s, p0/m, z1.s
607; CHECK-NEXT:    stp q0, q1, [x0]
608; CHECK-NEXT:    ret
609;
610; NONEON-NOSVE-LABEL: ctlz_v8i32:
611; NONEON-NOSVE:       // %bb.0:
612; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
613; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
614; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
615; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]
616; NONEON-NOSVE-NEXT:    clz w9, w8
617; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
618; NONEON-NOSVE-NEXT:    clz w8, w8
619; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
620; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
621; NONEON-NOSVE-NEXT:    clz w9, w8
622; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
623; NONEON-NOSVE-NEXT:    clz w8, w8
624; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
625; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]
626; NONEON-NOSVE-NEXT:    clz w9, w8
627; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
628; NONEON-NOSVE-NEXT:    clz w8, w8
629; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
630; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
631; NONEON-NOSVE-NEXT:    clz w9, w8
632; NONEON-NOSVE-NEXT:    ldr w8, [sp]
633; NONEON-NOSVE-NEXT:    clz w8, w8
634; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
635; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
636; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
637; NONEON-NOSVE-NEXT:    add sp, sp, #64
638; NONEON-NOSVE-NEXT:    ret
639  %op = load <8 x i32>, ptr %a
640  %res = call <8 x i32> @llvm.ctlz.v8i32(<8 x i32> %op)
641  store <8 x i32> %res, ptr %a
642  ret void
643}
644
645define <1 x i64> @ctlz_v1i64(<1 x i64> %op) {
646; CHECK-LABEL: ctlz_v1i64:
647; CHECK:       // %bb.0:
648; CHECK-NEXT:    ptrue p0.d, vl1
649; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
650; CHECK-NEXT:    clz z0.d, p0/m, z0.d
651; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
652; CHECK-NEXT:    ret
653;
654; NONEON-NOSVE-LABEL: ctlz_v1i64:
655; NONEON-NOSVE:       // %bb.0:
656; NONEON-NOSVE-NEXT:    sub sp, sp, #16
657; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
658; NONEON-NOSVE-NEXT:    fmov x8, d0
659; NONEON-NOSVE-NEXT:    clz x8, x8
660; NONEON-NOSVE-NEXT:    str x8, [sp, #8]
661; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
662; NONEON-NOSVE-NEXT:    add sp, sp, #16
663; NONEON-NOSVE-NEXT:    ret
664  %res = call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %op)
665  ret <1 x i64> %res
666}
667
668define <2 x i64> @ctlz_v2i64(<2 x i64> %op) {
669; CHECK-LABEL: ctlz_v2i64:
670; CHECK:       // %bb.0:
671; CHECK-NEXT:    ptrue p0.d, vl2
672; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
673; CHECK-NEXT:    clz z0.d, p0/m, z0.d
674; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
675; CHECK-NEXT:    ret
676;
677; NONEON-NOSVE-LABEL: ctlz_v2i64:
678; NONEON-NOSVE:       // %bb.0:
679; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
680; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
681; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]
682; NONEON-NOSVE-NEXT:    clz x9, x8
683; NONEON-NOSVE-NEXT:    ldr x8, [sp]
684; NONEON-NOSVE-NEXT:    clz x8, x8
685; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #16]
686; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
687; NONEON-NOSVE-NEXT:    add sp, sp, #32
688; NONEON-NOSVE-NEXT:    ret
689  %res = call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %op)
690  ret <2 x i64> %res
691}
692
693define void @ctlz_v4i64(ptr %a) {
694; CHECK-LABEL: ctlz_v4i64:
695; CHECK:       // %bb.0:
696; CHECK-NEXT:    ldp q0, q1, [x0]
697; CHECK-NEXT:    ptrue p0.d, vl2
698; CHECK-NEXT:    clz z0.d, p0/m, z0.d
699; CHECK-NEXT:    clz z1.d, p0/m, z1.d
700; CHECK-NEXT:    stp q0, q1, [x0]
701; CHECK-NEXT:    ret
702;
703; NONEON-NOSVE-LABEL: ctlz_v4i64:
704; NONEON-NOSVE:       // %bb.0:
705; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
706; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
707; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
708; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]
709; NONEON-NOSVE-NEXT:    clz x9, x8
710; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]
711; NONEON-NOSVE-NEXT:    clz x8, x8
712; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
713; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]
714; NONEON-NOSVE-NEXT:    clz x9, x8
715; NONEON-NOSVE-NEXT:    ldr x8, [sp]
716; NONEON-NOSVE-NEXT:    clz x8, x8
717; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]
718; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
719; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
720; NONEON-NOSVE-NEXT:    add sp, sp, #64
721; NONEON-NOSVE-NEXT:    ret
722  %op = load <4 x i64>, ptr %a
723  %res = call <4 x i64> @llvm.ctlz.v4i64(<4 x i64> %op)
724  store <4 x i64> %res, ptr %a
725  ret void
726}
727
728;
729; CNT
730;
731
732define <4 x i8> @ctpop_v4i8(<4 x i8> %op) {
733; CHECK-LABEL: ctpop_v4i8:
734; CHECK:       // %bb.0:
735; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
736; CHECK-NEXT:    ptrue p0.h, vl4
737; CHECK-NEXT:    and z0.h, z0.h, #0xff
738; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
739; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
740; CHECK-NEXT:    ret
741;
742; NONEON-NOSVE-LABEL: ctpop_v4i8:
743; NONEON-NOSVE:       // %bb.0:
744; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
745; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
746; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
747; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #4]
748; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
749; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #2]
750; NONEON-NOSVE-NEXT:    ldrb w12, [sp]
751; NONEON-NOSVE-NEXT:    lsr w13, w9, #1
752; NONEON-NOSVE-NEXT:    lsr w14, w11, #1
753; NONEON-NOSVE-NEXT:    lsr w15, w10, #1
754; NONEON-NOSVE-NEXT:    lsr w16, w12, #1
755; NONEON-NOSVE-NEXT:    and w13, w13, #0x55555555
756; NONEON-NOSVE-NEXT:    sub w9, w9, w13
757; NONEON-NOSVE-NEXT:    and w13, w14, #0x55555555
758; NONEON-NOSVE-NEXT:    and w14, w15, #0x55555555
759; NONEON-NOSVE-NEXT:    sub w11, w11, w13
760; NONEON-NOSVE-NEXT:    lsr w13, w9, #2
761; NONEON-NOSVE-NEXT:    and w15, w16, #0x55555555
762; NONEON-NOSVE-NEXT:    sub w10, w10, w14
763; NONEON-NOSVE-NEXT:    sub w12, w12, w15
764; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
765; NONEON-NOSVE-NEXT:    and w13, w13, #0x33333333
766; NONEON-NOSVE-NEXT:    lsr w14, w11, #2
767; NONEON-NOSVE-NEXT:    lsr w15, w10, #2
768; NONEON-NOSVE-NEXT:    add w9, w9, w13
769; NONEON-NOSVE-NEXT:    lsr w13, w12, #2
770; NONEON-NOSVE-NEXT:    and w11, w11, #0x33333333
771; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
772; NONEON-NOSVE-NEXT:    and w14, w14, #0x33333333
773; NONEON-NOSVE-NEXT:    and w15, w15, #0x33333333
774; NONEON-NOSVE-NEXT:    and w12, w12, #0x33333333
775; NONEON-NOSVE-NEXT:    and w13, w13, #0x33333333
776; NONEON-NOSVE-NEXT:    add w11, w11, w14
777; NONEON-NOSVE-NEXT:    add w10, w10, w15
778; NONEON-NOSVE-NEXT:    add w12, w12, w13
779; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
780; NONEON-NOSVE-NEXT:    add w11, w11, w11, lsr #4
781; NONEON-NOSVE-NEXT:    add w10, w10, w10, lsr #4
782; NONEON-NOSVE-NEXT:    add w12, w12, w12, lsr #4
783; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
784; NONEON-NOSVE-NEXT:    and w11, w11, #0xf0f0f0f
785; NONEON-NOSVE-NEXT:    and w10, w10, #0xf0f0f0f
786; NONEON-NOSVE-NEXT:    and w12, w12, #0xf0f0f0f
787; NONEON-NOSVE-NEXT:    mul w9, w9, w8
788; NONEON-NOSVE-NEXT:    mul w11, w11, w8
789; NONEON-NOSVE-NEXT:    mul w10, w10, w8
790; NONEON-NOSVE-NEXT:    mul w8, w12, w8
791; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
792; NONEON-NOSVE-NEXT:    lsr w11, w11, #24
793; NONEON-NOSVE-NEXT:    lsr w10, w10, #24
794; NONEON-NOSVE-NEXT:    strh w9, [sp, #14]
795; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
796; NONEON-NOSVE-NEXT:    strh w11, [sp, #12]
797; NONEON-NOSVE-NEXT:    strh w10, [sp, #10]
798; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]
799; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
800; NONEON-NOSVE-NEXT:    add sp, sp, #16
801; NONEON-NOSVE-NEXT:    ret
802  %res = call <4 x i8> @llvm.ctpop.v4i8(<4 x i8> %op)
803  ret <4 x i8> %res
804}
805
806define <8 x i8> @ctpop_v8i8(<8 x i8> %op) {
807; CHECK-LABEL: ctpop_v8i8:
808; CHECK:       // %bb.0:
809; CHECK-NEXT:    ptrue p0.b, vl8
810; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
811; CHECK-NEXT:    cnt z0.b, p0/m, z0.b
812; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
813; CHECK-NEXT:    ret
814;
815; NONEON-NOSVE-LABEL: ctpop_v8i8:
816; NONEON-NOSVE:       // %bb.0:
817; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
818; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
819; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
820; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
821; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
822; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
823; NONEON-NOSVE-NEXT:    sub w9, w9, w10
824; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
825; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
826; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
827; NONEON-NOSVE-NEXT:    add w9, w9, w10
828; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
829; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
830; NONEON-NOSVE-NEXT:    mul w9, w9, w8
831; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
832; NONEON-NOSVE-NEXT:    strb w9, [sp, #15]
833; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
834; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
835; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
836; NONEON-NOSVE-NEXT:    sub w9, w9, w10
837; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
838; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
839; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
840; NONEON-NOSVE-NEXT:    add w9, w9, w10
841; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
842; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
843; NONEON-NOSVE-NEXT:    mul w9, w9, w8
844; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
845; NONEON-NOSVE-NEXT:    strb w9, [sp, #14]
846; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]
847; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
848; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
849; NONEON-NOSVE-NEXT:    sub w9, w9, w10
850; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
851; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
852; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
853; NONEON-NOSVE-NEXT:    add w9, w9, w10
854; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
855; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
856; NONEON-NOSVE-NEXT:    mul w9, w9, w8
857; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
858; NONEON-NOSVE-NEXT:    strb w9, [sp, #13]
859; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
860; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
861; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
862; NONEON-NOSVE-NEXT:    sub w9, w9, w10
863; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
864; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
865; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
866; NONEON-NOSVE-NEXT:    add w9, w9, w10
867; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
868; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
869; NONEON-NOSVE-NEXT:    mul w9, w9, w8
870; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
871; NONEON-NOSVE-NEXT:    strb w9, [sp, #12]
872; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
873; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
874; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
875; NONEON-NOSVE-NEXT:    sub w9, w9, w10
876; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
877; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
878; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
879; NONEON-NOSVE-NEXT:    add w9, w9, w10
880; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
881; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
882; NONEON-NOSVE-NEXT:    mul w9, w9, w8
883; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
884; NONEON-NOSVE-NEXT:    strb w9, [sp, #11]
885; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]
886; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
887; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
888; NONEON-NOSVE-NEXT:    sub w9, w9, w10
889; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
890; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
891; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
892; NONEON-NOSVE-NEXT:    add w9, w9, w10
893; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
894; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
895; NONEON-NOSVE-NEXT:    mul w9, w9, w8
896; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
897; NONEON-NOSVE-NEXT:    strb w9, [sp, #10]
898; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]
899; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
900; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
901; NONEON-NOSVE-NEXT:    sub w9, w9, w10
902; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
903; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
904; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
905; NONEON-NOSVE-NEXT:    add w9, w9, w10
906; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
907; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
908; NONEON-NOSVE-NEXT:    mul w9, w9, w8
909; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
910; NONEON-NOSVE-NEXT:    strb w9, [sp, #9]
911; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
912; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
913; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
914; NONEON-NOSVE-NEXT:    sub w9, w9, w10
915; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
916; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
917; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
918; NONEON-NOSVE-NEXT:    add w9, w9, w10
919; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
920; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
921; NONEON-NOSVE-NEXT:    mul w8, w9, w8
922; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
923; NONEON-NOSVE-NEXT:    strb w8, [sp, #8]
924; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
925; NONEON-NOSVE-NEXT:    add sp, sp, #16
926; NONEON-NOSVE-NEXT:    ret
927  %res = call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> %op)
928  ret <8 x i8> %res
929}
930
931define <16 x i8> @ctpop_v16i8(<16 x i8> %op) {
932; CHECK-LABEL: ctpop_v16i8:
933; CHECK:       // %bb.0:
934; CHECK-NEXT:    ptrue p0.b, vl16
935; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
936; CHECK-NEXT:    cnt z0.b, p0/m, z0.b
937; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
938; CHECK-NEXT:    ret
939;
940; NONEON-NOSVE-LABEL: ctpop_v16i8:
941; NONEON-NOSVE:       // %bb.0:
942; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
943; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
944; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
945; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
946; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
947; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
948; NONEON-NOSVE-NEXT:    sub w9, w9, w10
949; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
950; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
951; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
952; NONEON-NOSVE-NEXT:    add w9, w9, w10
953; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
954; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
955; NONEON-NOSVE-NEXT:    mul w9, w9, w8
956; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
957; NONEON-NOSVE-NEXT:    strb w9, [sp, #31]
958; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
959; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
960; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
961; NONEON-NOSVE-NEXT:    sub w9, w9, w10
962; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
963; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
964; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
965; NONEON-NOSVE-NEXT:    add w9, w9, w10
966; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
967; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
968; NONEON-NOSVE-NEXT:    mul w9, w9, w8
969; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
970; NONEON-NOSVE-NEXT:    strb w9, [sp, #30]
971; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
972; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
973; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
974; NONEON-NOSVE-NEXT:    sub w9, w9, w10
975; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
976; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
977; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
978; NONEON-NOSVE-NEXT:    add w9, w9, w10
979; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
980; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
981; NONEON-NOSVE-NEXT:    mul w9, w9, w8
982; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
983; NONEON-NOSVE-NEXT:    strb w9, [sp, #29]
984; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
985; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
986; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
987; NONEON-NOSVE-NEXT:    sub w9, w9, w10
988; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
989; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
990; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
991; NONEON-NOSVE-NEXT:    add w9, w9, w10
992; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
993; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
994; NONEON-NOSVE-NEXT:    mul w9, w9, w8
995; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
996; NONEON-NOSVE-NEXT:    strb w9, [sp, #28]
997; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
998; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
999; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1000; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1001; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1002; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1003; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1004; NONEON-NOSVE-NEXT:    add w9, w9, w10
1005; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1006; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1007; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1008; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1009; NONEON-NOSVE-NEXT:    strb w9, [sp, #27]
1010; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
1011; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1012; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1013; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1014; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1015; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1016; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1017; NONEON-NOSVE-NEXT:    add w9, w9, w10
1018; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1019; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1020; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1021; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1022; NONEON-NOSVE-NEXT:    strb w9, [sp, #26]
1023; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
1024; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1025; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1026; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1027; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1028; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1029; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1030; NONEON-NOSVE-NEXT:    add w9, w9, w10
1031; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1032; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1033; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1034; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1035; NONEON-NOSVE-NEXT:    strb w9, [sp, #25]
1036; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
1037; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1038; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1039; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1040; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1041; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1042; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1043; NONEON-NOSVE-NEXT:    add w9, w9, w10
1044; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1045; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1046; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1047; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1048; NONEON-NOSVE-NEXT:    strb w9, [sp, #24]
1049; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
1050; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1051; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1052; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1053; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1054; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1055; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1056; NONEON-NOSVE-NEXT:    add w9, w9, w10
1057; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1058; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1059; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1060; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1061; NONEON-NOSVE-NEXT:    strb w9, [sp, #23]
1062; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
1063; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1064; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1065; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1066; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1067; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1068; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1069; NONEON-NOSVE-NEXT:    add w9, w9, w10
1070; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1071; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1072; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1073; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1074; NONEON-NOSVE-NEXT:    strb w9, [sp, #22]
1075; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]
1076; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1077; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1078; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1079; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1080; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1081; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1082; NONEON-NOSVE-NEXT:    add w9, w9, w10
1083; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1084; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1085; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1086; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1087; NONEON-NOSVE-NEXT:    strb w9, [sp, #21]
1088; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
1089; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1090; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1091; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1092; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1093; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1094; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1095; NONEON-NOSVE-NEXT:    add w9, w9, w10
1096; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1097; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1098; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1099; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1100; NONEON-NOSVE-NEXT:    strb w9, [sp, #20]
1101; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
1102; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1103; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1104; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1105; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1106; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1107; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1108; NONEON-NOSVE-NEXT:    add w9, w9, w10
1109; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1110; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1111; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1112; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1113; NONEON-NOSVE-NEXT:    strb w9, [sp, #19]
1114; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]
1115; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1116; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1117; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1118; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1119; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1120; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1121; NONEON-NOSVE-NEXT:    add w9, w9, w10
1122; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1123; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1124; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1125; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1126; NONEON-NOSVE-NEXT:    strb w9, [sp, #18]
1127; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]
1128; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1129; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1130; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1131; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1132; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1133; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1134; NONEON-NOSVE-NEXT:    add w9, w9, w10
1135; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1136; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1137; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1138; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1139; NONEON-NOSVE-NEXT:    strb w9, [sp, #17]
1140; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
1141; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1142; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1143; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1144; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1145; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1146; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1147; NONEON-NOSVE-NEXT:    add w9, w9, w10
1148; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1149; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1150; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1151; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
1152; NONEON-NOSVE-NEXT:    strb w8, [sp, #16]
1153; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
1154; NONEON-NOSVE-NEXT:    add sp, sp, #32
1155; NONEON-NOSVE-NEXT:    ret
1156  %res = call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %op)
1157  ret <16 x i8> %res
1158}
1159
1160define void @ctpop_v32i8(ptr %a) {
1161; CHECK-LABEL: ctpop_v32i8:
1162; CHECK:       // %bb.0:
1163; CHECK-NEXT:    ldp q0, q1, [x0]
1164; CHECK-NEXT:    ptrue p0.b, vl16
1165; CHECK-NEXT:    cnt z0.b, p0/m, z0.b
1166; CHECK-NEXT:    cnt z1.b, p0/m, z1.b
1167; CHECK-NEXT:    stp q0, q1, [x0]
1168; CHECK-NEXT:    ret
1169;
1170; NONEON-NOSVE-LABEL: ctpop_v32i8:
1171; NONEON-NOSVE:       // %bb.0:
1172; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
1173; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
1174; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
1175; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
1176; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #31]
1177; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1178; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1179; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1180; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1181; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1182; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1183; NONEON-NOSVE-NEXT:    add w9, w9, w10
1184; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1185; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1186; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1187; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1188; NONEON-NOSVE-NEXT:    strb w9, [sp, #63]
1189; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #30]
1190; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1191; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1192; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1193; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1194; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1195; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1196; NONEON-NOSVE-NEXT:    add w9, w9, w10
1197; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1198; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1199; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1200; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1201; NONEON-NOSVE-NEXT:    strb w9, [sp, #62]
1202; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #29]
1203; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1204; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1205; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1206; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1207; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1208; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1209; NONEON-NOSVE-NEXT:    add w9, w9, w10
1210; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1211; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1212; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1213; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1214; NONEON-NOSVE-NEXT:    strb w9, [sp, #61]
1215; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #28]
1216; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1217; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1218; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1219; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1220; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1221; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1222; NONEON-NOSVE-NEXT:    add w9, w9, w10
1223; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1224; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1225; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1226; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1227; NONEON-NOSVE-NEXT:    strb w9, [sp, #60]
1228; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #27]
1229; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1230; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1231; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1232; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1233; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1234; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1235; NONEON-NOSVE-NEXT:    add w9, w9, w10
1236; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1237; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1238; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1239; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1240; NONEON-NOSVE-NEXT:    strb w9, [sp, #59]
1241; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #26]
1242; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1243; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1244; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1245; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1246; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1247; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1248; NONEON-NOSVE-NEXT:    add w9, w9, w10
1249; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1250; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1251; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1252; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1253; NONEON-NOSVE-NEXT:    strb w9, [sp, #58]
1254; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #25]
1255; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1256; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1257; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1258; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1259; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1260; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1261; NONEON-NOSVE-NEXT:    add w9, w9, w10
1262; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1263; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1264; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1265; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1266; NONEON-NOSVE-NEXT:    strb w9, [sp, #57]
1267; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #24]
1268; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1269; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1270; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1271; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1272; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1273; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1274; NONEON-NOSVE-NEXT:    add w9, w9, w10
1275; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1276; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1277; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1278; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1279; NONEON-NOSVE-NEXT:    strb w9, [sp, #56]
1280; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #23]
1281; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1282; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1283; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1284; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1285; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1286; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1287; NONEON-NOSVE-NEXT:    add w9, w9, w10
1288; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1289; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1290; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1291; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1292; NONEON-NOSVE-NEXT:    strb w9, [sp, #55]
1293; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #22]
1294; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1295; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1296; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1297; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1298; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1299; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1300; NONEON-NOSVE-NEXT:    add w9, w9, w10
1301; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1302; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1303; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1304; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1305; NONEON-NOSVE-NEXT:    strb w9, [sp, #54]
1306; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #21]
1307; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1308; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1309; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1310; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1311; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1312; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1313; NONEON-NOSVE-NEXT:    add w9, w9, w10
1314; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1315; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1316; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1317; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1318; NONEON-NOSVE-NEXT:    strb w9, [sp, #53]
1319; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #20]
1320; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1321; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1322; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1323; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1324; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1325; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1326; NONEON-NOSVE-NEXT:    add w9, w9, w10
1327; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1328; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1329; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1330; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1331; NONEON-NOSVE-NEXT:    strb w9, [sp, #52]
1332; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #19]
1333; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1334; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1335; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1336; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1337; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1338; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1339; NONEON-NOSVE-NEXT:    add w9, w9, w10
1340; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1341; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1342; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1343; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1344; NONEON-NOSVE-NEXT:    strb w9, [sp, #51]
1345; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #18]
1346; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1347; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1348; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1349; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1350; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1351; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1352; NONEON-NOSVE-NEXT:    add w9, w9, w10
1353; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1354; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1355; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1356; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1357; NONEON-NOSVE-NEXT:    strb w9, [sp, #50]
1358; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #17]
1359; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1360; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1361; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1362; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1363; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1364; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1365; NONEON-NOSVE-NEXT:    add w9, w9, w10
1366; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1367; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1368; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1369; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1370; NONEON-NOSVE-NEXT:    strb w9, [sp, #49]
1371; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #16]
1372; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1373; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1374; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1375; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1376; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1377; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1378; NONEON-NOSVE-NEXT:    add w9, w9, w10
1379; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1380; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1381; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1382; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1383; NONEON-NOSVE-NEXT:    strb w9, [sp, #48]
1384; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #15]
1385; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1386; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1387; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1388; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1389; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1390; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1391; NONEON-NOSVE-NEXT:    add w9, w9, w10
1392; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1393; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1394; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1395; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1396; NONEON-NOSVE-NEXT:    strb w9, [sp, #47]
1397; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #14]
1398; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1399; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1400; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1401; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1402; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1403; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1404; NONEON-NOSVE-NEXT:    add w9, w9, w10
1405; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1406; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1407; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1408; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1409; NONEON-NOSVE-NEXT:    strb w9, [sp, #46]
1410; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #13]
1411; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1412; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1413; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1414; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1415; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1416; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1417; NONEON-NOSVE-NEXT:    add w9, w9, w10
1418; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1419; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1420; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1421; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1422; NONEON-NOSVE-NEXT:    strb w9, [sp, #45]
1423; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #12]
1424; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1425; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1426; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1427; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1428; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1429; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1430; NONEON-NOSVE-NEXT:    add w9, w9, w10
1431; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1432; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1433; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1434; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1435; NONEON-NOSVE-NEXT:    strb w9, [sp, #44]
1436; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #11]
1437; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1438; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1439; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1440; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1441; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1442; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1443; NONEON-NOSVE-NEXT:    add w9, w9, w10
1444; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1445; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1446; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1447; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1448; NONEON-NOSVE-NEXT:    strb w9, [sp, #43]
1449; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #10]
1450; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1451; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1452; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1453; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1454; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1455; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1456; NONEON-NOSVE-NEXT:    add w9, w9, w10
1457; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1458; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1459; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1460; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1461; NONEON-NOSVE-NEXT:    strb w9, [sp, #42]
1462; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #9]
1463; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1464; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1465; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1466; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1467; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1468; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1469; NONEON-NOSVE-NEXT:    add w9, w9, w10
1470; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1471; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1472; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1473; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1474; NONEON-NOSVE-NEXT:    strb w9, [sp, #41]
1475; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #8]
1476; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1477; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1478; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1479; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1480; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1481; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1482; NONEON-NOSVE-NEXT:    add w9, w9, w10
1483; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1484; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1485; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1486; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1487; NONEON-NOSVE-NEXT:    strb w9, [sp, #40]
1488; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
1489; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1490; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1491; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1492; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1493; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1494; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1495; NONEON-NOSVE-NEXT:    add w9, w9, w10
1496; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1497; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1498; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1499; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1500; NONEON-NOSVE-NEXT:    strb w9, [sp, #39]
1501; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #6]
1502; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1503; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1504; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1505; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1506; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1507; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1508; NONEON-NOSVE-NEXT:    add w9, w9, w10
1509; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1510; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1511; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1512; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1513; NONEON-NOSVE-NEXT:    strb w9, [sp, #38]
1514; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #5]
1515; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1516; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1517; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1518; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1519; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1520; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1521; NONEON-NOSVE-NEXT:    add w9, w9, w10
1522; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1523; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1524; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1525; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1526; NONEON-NOSVE-NEXT:    strb w9, [sp, #37]
1527; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #4]
1528; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1529; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1530; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1531; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1532; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1533; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1534; NONEON-NOSVE-NEXT:    add w9, w9, w10
1535; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1536; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1537; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1538; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1539; NONEON-NOSVE-NEXT:    strb w9, [sp, #36]
1540; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
1541; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1542; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1543; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1544; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1545; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1546; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1547; NONEON-NOSVE-NEXT:    add w9, w9, w10
1548; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1549; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1550; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1551; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1552; NONEON-NOSVE-NEXT:    strb w9, [sp, #35]
1553; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #2]
1554; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1555; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1556; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1557; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1558; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1559; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1560; NONEON-NOSVE-NEXT:    add w9, w9, w10
1561; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1562; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1563; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1564; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1565; NONEON-NOSVE-NEXT:    strb w9, [sp, #34]
1566; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1]
1567; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1568; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1569; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1570; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1571; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1572; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1573; NONEON-NOSVE-NEXT:    add w9, w9, w10
1574; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1575; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1576; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1577; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1578; NONEON-NOSVE-NEXT:    strb w9, [sp, #33]
1579; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
1580; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1581; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1582; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1583; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1584; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1585; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1586; NONEON-NOSVE-NEXT:    add w9, w9, w10
1587; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1588; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1589; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1590; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
1591; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]
1592; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
1593; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
1594; NONEON-NOSVE-NEXT:    add sp, sp, #64
1595; NONEON-NOSVE-NEXT:    ret
1596  %op = load <32 x i8>, ptr %a
1597  %res = call <32 x i8> @llvm.ctpop.v32i8(<32 x i8> %op)
1598  store <32 x i8> %res, ptr %a
1599  ret void
1600}
1601
1602define <2 x i16> @ctpop_v2i16(<2 x i16> %op) {
1603; CHECK-LABEL: ctpop_v2i16:
1604; CHECK:       // %bb.0:
1605; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1606; CHECK-NEXT:    ptrue p0.s, vl2
1607; CHECK-NEXT:    and z0.s, z0.s, #0xffff
1608; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
1609; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
1610; CHECK-NEXT:    ret
1611;
1612; NONEON-NOSVE-LABEL: ctpop_v2i16:
1613; NONEON-NOSVE:       // %bb.0:
1614; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
1615; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
1616; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
1617; NONEON-NOSVE-NEXT:    ldrh w10, [sp]
1618; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
1619; NONEON-NOSVE-NEXT:    lsr w11, w9, #1
1620; NONEON-NOSVE-NEXT:    lsr w12, w10, #1
1621; NONEON-NOSVE-NEXT:    and w11, w11, #0x55555555
1622; NONEON-NOSVE-NEXT:    and w12, w12, #0x55555555
1623; NONEON-NOSVE-NEXT:    sub w9, w9, w11
1624; NONEON-NOSVE-NEXT:    sub w10, w10, w12
1625; NONEON-NOSVE-NEXT:    lsr w11, w9, #2
1626; NONEON-NOSVE-NEXT:    lsr w12, w10, #2
1627; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1628; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1629; NONEON-NOSVE-NEXT:    and w11, w11, #0x33333333
1630; NONEON-NOSVE-NEXT:    add w9, w9, w11
1631; NONEON-NOSVE-NEXT:    and w11, w12, #0x33333333
1632; NONEON-NOSVE-NEXT:    add w10, w10, w11
1633; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1634; NONEON-NOSVE-NEXT:    add w10, w10, w10, lsr #4
1635; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1636; NONEON-NOSVE-NEXT:    and w10, w10, #0xf0f0f0f
1637; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1638; NONEON-NOSVE-NEXT:    mul w8, w10, w8
1639; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1640; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
1641; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8]
1642; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
1643; NONEON-NOSVE-NEXT:    add sp, sp, #16
1644; NONEON-NOSVE-NEXT:    ret
1645  %res = call <2 x i16> @llvm.ctpop.v2i16(<2 x i16> %op)
1646  ret <2 x i16> %res
1647}
1648
1649define <4 x i16> @ctpop_v4i16(<4 x i16> %op) {
1650; CHECK-LABEL: ctpop_v4i16:
1651; CHECK:       // %bb.0:
1652; CHECK-NEXT:    ptrue p0.h, vl4
1653; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1654; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
1655; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
1656; CHECK-NEXT:    ret
1657;
1658; NONEON-NOSVE-LABEL: ctpop_v4i16:
1659; NONEON-NOSVE:       // %bb.0:
1660; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
1661; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
1662; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]
1663; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
1664; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1665; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1666; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1667; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1668; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1669; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1670; NONEON-NOSVE-NEXT:    add w9, w9, w10
1671; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1672; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1673; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1674; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1675; NONEON-NOSVE-NEXT:    strh w9, [sp, #14]
1676; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
1677; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1678; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1679; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1680; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1681; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1682; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1683; NONEON-NOSVE-NEXT:    add w9, w9, w10
1684; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1685; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1686; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1687; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1688; NONEON-NOSVE-NEXT:    strh w9, [sp, #12]
1689; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]
1690; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1691; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1692; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1693; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1694; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1695; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1696; NONEON-NOSVE-NEXT:    add w9, w9, w10
1697; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1698; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1699; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1700; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1701; NONEON-NOSVE-NEXT:    strh w9, [sp, #10]
1702; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
1703; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1704; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1705; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1706; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1707; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1708; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1709; NONEON-NOSVE-NEXT:    add w9, w9, w10
1710; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1711; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1712; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1713; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
1714; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]
1715; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
1716; NONEON-NOSVE-NEXT:    add sp, sp, #16
1717; NONEON-NOSVE-NEXT:    ret
1718  %res = call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %op)
1719  ret <4 x i16> %res
1720}
1721
1722define <8 x i16> @ctpop_v8i16(<8 x i16> %op) {
1723; CHECK-LABEL: ctpop_v8i16:
1724; CHECK:       // %bb.0:
1725; CHECK-NEXT:    ptrue p0.h, vl8
1726; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
1727; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
1728; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
1729; CHECK-NEXT:    ret
1730;
1731; NONEON-NOSVE-LABEL: ctpop_v8i16:
1732; NONEON-NOSVE:       // %bb.0:
1733; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
1734; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
1735; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
1736; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
1737; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1738; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1739; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1740; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1741; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1742; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1743; NONEON-NOSVE-NEXT:    add w9, w9, w10
1744; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1745; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1746; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1747; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1748; NONEON-NOSVE-NEXT:    strh w9, [sp, #30]
1749; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
1750; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1751; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1752; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1753; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1754; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1755; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1756; NONEON-NOSVE-NEXT:    add w9, w9, w10
1757; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1758; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1759; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1760; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1761; NONEON-NOSVE-NEXT:    strh w9, [sp, #28]
1762; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
1763; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1764; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1765; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1766; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1767; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1768; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1769; NONEON-NOSVE-NEXT:    add w9, w9, w10
1770; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1771; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1772; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1773; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1774; NONEON-NOSVE-NEXT:    strh w9, [sp, #26]
1775; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
1776; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1777; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1778; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1779; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1780; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1781; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1782; NONEON-NOSVE-NEXT:    add w9, w9, w10
1783; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1784; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1785; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1786; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1787; NONEON-NOSVE-NEXT:    strh w9, [sp, #24]
1788; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]
1789; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1790; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1791; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1792; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1793; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1794; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1795; NONEON-NOSVE-NEXT:    add w9, w9, w10
1796; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1797; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1798; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1799; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1800; NONEON-NOSVE-NEXT:    strh w9, [sp, #22]
1801; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
1802; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1803; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1804; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1805; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1806; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1807; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1808; NONEON-NOSVE-NEXT:    add w9, w9, w10
1809; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1810; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1811; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1812; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1813; NONEON-NOSVE-NEXT:    strh w9, [sp, #20]
1814; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]
1815; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1816; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1817; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1818; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1819; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1820; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1821; NONEON-NOSVE-NEXT:    add w9, w9, w10
1822; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1823; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1824; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1825; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1826; NONEON-NOSVE-NEXT:    strh w9, [sp, #18]
1827; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
1828; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1829; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1830; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1831; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1832; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1833; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1834; NONEON-NOSVE-NEXT:    add w9, w9, w10
1835; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1836; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1837; NONEON-NOSVE-NEXT:    mul w8, w9, w8
1838; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
1839; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
1840; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
1841; NONEON-NOSVE-NEXT:    add sp, sp, #32
1842; NONEON-NOSVE-NEXT:    ret
1843  %res = call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %op)
1844  ret <8 x i16> %res
1845}
1846
1847define void @ctpop_v16i16(ptr %a) {
1848; CHECK-LABEL: ctpop_v16i16:
1849; CHECK:       // %bb.0:
1850; CHECK-NEXT:    ldp q0, q1, [x0]
1851; CHECK-NEXT:    ptrue p0.h, vl8
1852; CHECK-NEXT:    cnt z0.h, p0/m, z0.h
1853; CHECK-NEXT:    cnt z1.h, p0/m, z1.h
1854; CHECK-NEXT:    stp q0, q1, [x0]
1855; CHECK-NEXT:    ret
1856;
1857; NONEON-NOSVE-LABEL: ctpop_v16i16:
1858; NONEON-NOSVE:       // %bb.0:
1859; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
1860; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
1861; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
1862; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
1863; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #30]
1864; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1865; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1866; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1867; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1868; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1869; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1870; NONEON-NOSVE-NEXT:    add w9, w9, w10
1871; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1872; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1873; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1874; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1875; NONEON-NOSVE-NEXT:    strh w9, [sp, #62]
1876; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #28]
1877; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1878; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1879; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1880; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1881; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1882; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1883; NONEON-NOSVE-NEXT:    add w9, w9, w10
1884; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1885; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1886; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1887; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1888; NONEON-NOSVE-NEXT:    strh w9, [sp, #60]
1889; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #26]
1890; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1891; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1892; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1893; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1894; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1895; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1896; NONEON-NOSVE-NEXT:    add w9, w9, w10
1897; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1898; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1899; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1900; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1901; NONEON-NOSVE-NEXT:    strh w9, [sp, #58]
1902; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #24]
1903; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1904; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1905; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1906; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1907; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1908; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1909; NONEON-NOSVE-NEXT:    add w9, w9, w10
1910; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1911; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1912; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1913; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1914; NONEON-NOSVE-NEXT:    strh w9, [sp, #56]
1915; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #22]
1916; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1917; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1918; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1919; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1920; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1921; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1922; NONEON-NOSVE-NEXT:    add w9, w9, w10
1923; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1924; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1925; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1926; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1927; NONEON-NOSVE-NEXT:    strh w9, [sp, #54]
1928; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #20]
1929; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1930; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1931; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1932; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1933; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1934; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1935; NONEON-NOSVE-NEXT:    add w9, w9, w10
1936; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1937; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1938; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1939; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1940; NONEON-NOSVE-NEXT:    strh w9, [sp, #52]
1941; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #18]
1942; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1943; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1944; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1945; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1946; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1947; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1948; NONEON-NOSVE-NEXT:    add w9, w9, w10
1949; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1950; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1951; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1952; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1953; NONEON-NOSVE-NEXT:    strh w9, [sp, #50]
1954; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #16]
1955; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1956; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1957; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1958; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1959; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1960; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1961; NONEON-NOSVE-NEXT:    add w9, w9, w10
1962; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1963; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1964; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1965; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1966; NONEON-NOSVE-NEXT:    strh w9, [sp, #48]
1967; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
1968; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1969; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1970; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1971; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1972; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1973; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1974; NONEON-NOSVE-NEXT:    add w9, w9, w10
1975; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1976; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1977; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1978; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1979; NONEON-NOSVE-NEXT:    strh w9, [sp, #46]
1980; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #12]
1981; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1982; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1983; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1984; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1985; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1986; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
1987; NONEON-NOSVE-NEXT:    add w9, w9, w10
1988; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
1989; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
1990; NONEON-NOSVE-NEXT:    mul w9, w9, w8
1991; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
1992; NONEON-NOSVE-NEXT:    strh w9, [sp, #44]
1993; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
1994; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
1995; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
1996; NONEON-NOSVE-NEXT:    sub w9, w9, w10
1997; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
1998; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
1999; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2000; NONEON-NOSVE-NEXT:    add w9, w9, w10
2001; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2002; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2003; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2004; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
2005; NONEON-NOSVE-NEXT:    strh w9, [sp, #42]
2006; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #8]
2007; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2008; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2009; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2010; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2011; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2012; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2013; NONEON-NOSVE-NEXT:    add w9, w9, w10
2014; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2015; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2016; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2017; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
2018; NONEON-NOSVE-NEXT:    strh w9, [sp, #40]
2019; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #6]
2020; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2021; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2022; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2023; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2024; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2025; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2026; NONEON-NOSVE-NEXT:    add w9, w9, w10
2027; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2028; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2029; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2030; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
2031; NONEON-NOSVE-NEXT:    strh w9, [sp, #38]
2032; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
2033; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2034; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2035; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2036; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2037; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2038; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2039; NONEON-NOSVE-NEXT:    add w9, w9, w10
2040; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2041; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2042; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2043; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
2044; NONEON-NOSVE-NEXT:    strh w9, [sp, #36]
2045; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #2]
2046; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2047; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2048; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2049; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2050; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2051; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2052; NONEON-NOSVE-NEXT:    add w9, w9, w10
2053; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2054; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2055; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2056; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
2057; NONEON-NOSVE-NEXT:    strh w9, [sp, #34]
2058; NONEON-NOSVE-NEXT:    ldrh w9, [sp]
2059; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2060; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2061; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2062; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2063; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2064; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2065; NONEON-NOSVE-NEXT:    add w9, w9, w10
2066; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2067; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2068; NONEON-NOSVE-NEXT:    mul w8, w9, w8
2069; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
2070; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
2071; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
2072; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
2073; NONEON-NOSVE-NEXT:    add sp, sp, #64
2074; NONEON-NOSVE-NEXT:    ret
2075  %op = load <16 x i16>, ptr %a
2076  %res = call <16 x i16> @llvm.ctpop.v16i16(<16 x i16> %op)
2077  store <16 x i16> %res, ptr %a
2078  ret void
2079}
2080
2081define <2 x i32> @ctpop_v2i32(<2 x i32> %op) {
2082; CHECK-LABEL: ctpop_v2i32:
2083; CHECK:       // %bb.0:
2084; CHECK-NEXT:    ptrue p0.s, vl2
2085; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2086; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
2087; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2088; CHECK-NEXT:    ret
2089;
2090; NONEON-NOSVE-LABEL: ctpop_v2i32:
2091; NONEON-NOSVE:       // %bb.0:
2092; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2093; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2094; NONEON-NOSVE-NEXT:    ldr w9, [sp, #4]
2095; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
2096; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2097; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2098; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2099; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2100; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2101; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2102; NONEON-NOSVE-NEXT:    add w9, w9, w10
2103; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2104; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2105; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2106; NONEON-NOSVE-NEXT:    lsr w11, w9, #24
2107; NONEON-NOSVE-NEXT:    ldr w9, [sp]
2108; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2109; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2110; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2111; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2112; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2113; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2114; NONEON-NOSVE-NEXT:    add w9, w9, w10
2115; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2116; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2117; NONEON-NOSVE-NEXT:    mul w8, w9, w8
2118; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
2119; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #8]
2120; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2121; NONEON-NOSVE-NEXT:    add sp, sp, #16
2122; NONEON-NOSVE-NEXT:    ret
2123  %res = call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %op)
2124  ret <2 x i32> %res
2125}
2126
2127define <4 x i32> @ctpop_v4i32(<4 x i32> %op) {
2128; CHECK-LABEL: ctpop_v4i32:
2129; CHECK:       // %bb.0:
2130; CHECK-NEXT:    ptrue p0.s, vl4
2131; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2132; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
2133; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2134; CHECK-NEXT:    ret
2135;
2136; NONEON-NOSVE-LABEL: ctpop_v4i32:
2137; NONEON-NOSVE:       // %bb.0:
2138; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
2139; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
2140; NONEON-NOSVE-NEXT:    ldr w9, [sp, #12]
2141; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
2142; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2143; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2144; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2145; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2146; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2147; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2148; NONEON-NOSVE-NEXT:    add w9, w9, w10
2149; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2150; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2151; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2152; NONEON-NOSVE-NEXT:    lsr w11, w9, #24
2153; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8]
2154; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2155; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2156; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2157; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2158; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2159; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2160; NONEON-NOSVE-NEXT:    add w9, w9, w10
2161; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2162; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2163; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2164; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
2165; NONEON-NOSVE-NEXT:    stp w9, w11, [sp, #24]
2166; NONEON-NOSVE-NEXT:    ldr w9, [sp, #4]
2167; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2168; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2169; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2170; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2171; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2172; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2173; NONEON-NOSVE-NEXT:    add w9, w9, w10
2174; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2175; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2176; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2177; NONEON-NOSVE-NEXT:    lsr w11, w9, #24
2178; NONEON-NOSVE-NEXT:    ldr w9, [sp]
2179; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2180; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2181; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2182; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2183; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2184; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2185; NONEON-NOSVE-NEXT:    add w9, w9, w10
2186; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2187; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2188; NONEON-NOSVE-NEXT:    mul w8, w9, w8
2189; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
2190; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #16]
2191; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
2192; NONEON-NOSVE-NEXT:    add sp, sp, #32
2193; NONEON-NOSVE-NEXT:    ret
2194  %res = call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %op)
2195  ret <4 x i32> %res
2196}
2197
2198define void @ctpop_v8i32(ptr %a) {
2199; CHECK-LABEL: ctpop_v8i32:
2200; CHECK:       // %bb.0:
2201; CHECK-NEXT:    ldp q0, q1, [x0]
2202; CHECK-NEXT:    ptrue p0.s, vl4
2203; CHECK-NEXT:    cnt z0.s, p0/m, z0.s
2204; CHECK-NEXT:    cnt z1.s, p0/m, z1.s
2205; CHECK-NEXT:    stp q0, q1, [x0]
2206; CHECK-NEXT:    ret
2207;
2208; NONEON-NOSVE-LABEL: ctpop_v8i32:
2209; NONEON-NOSVE:       // %bb.0:
2210; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
2211; NONEON-NOSVE-NEXT:    mov w8, #16843009 // =0x1010101
2212; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
2213; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
2214; NONEON-NOSVE-NEXT:    ldr w9, [sp, #28]
2215; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2216; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2217; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2218; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2219; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2220; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2221; NONEON-NOSVE-NEXT:    add w9, w9, w10
2222; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2223; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2224; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2225; NONEON-NOSVE-NEXT:    lsr w11, w9, #24
2226; NONEON-NOSVE-NEXT:    ldr w9, [sp, #24]
2227; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2228; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2229; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2230; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2231; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2232; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2233; NONEON-NOSVE-NEXT:    add w9, w9, w10
2234; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2235; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2236; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2237; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
2238; NONEON-NOSVE-NEXT:    stp w9, w11, [sp, #56]
2239; NONEON-NOSVE-NEXT:    ldr w9, [sp, #20]
2240; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2241; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2242; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2243; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2244; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2245; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2246; NONEON-NOSVE-NEXT:    add w9, w9, w10
2247; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2248; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2249; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2250; NONEON-NOSVE-NEXT:    lsr w11, w9, #24
2251; NONEON-NOSVE-NEXT:    ldr w9, [sp, #16]
2252; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2253; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2254; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2255; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2256; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2257; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2258; NONEON-NOSVE-NEXT:    add w9, w9, w10
2259; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2260; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2261; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2262; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
2263; NONEON-NOSVE-NEXT:    stp w9, w11, [sp, #48]
2264; NONEON-NOSVE-NEXT:    ldr w9, [sp, #12]
2265; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2266; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2267; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2268; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2269; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2270; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2271; NONEON-NOSVE-NEXT:    add w9, w9, w10
2272; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2273; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2274; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2275; NONEON-NOSVE-NEXT:    lsr w11, w9, #24
2276; NONEON-NOSVE-NEXT:    ldr w9, [sp, #8]
2277; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2278; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2279; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2280; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2281; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2282; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2283; NONEON-NOSVE-NEXT:    add w9, w9, w10
2284; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2285; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2286; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2287; NONEON-NOSVE-NEXT:    lsr w9, w9, #24
2288; NONEON-NOSVE-NEXT:    stp w9, w11, [sp, #40]
2289; NONEON-NOSVE-NEXT:    ldr w9, [sp, #4]
2290; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2291; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2292; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2293; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2294; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2295; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2296; NONEON-NOSVE-NEXT:    add w9, w9, w10
2297; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2298; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2299; NONEON-NOSVE-NEXT:    mul w9, w9, w8
2300; NONEON-NOSVE-NEXT:    lsr w11, w9, #24
2301; NONEON-NOSVE-NEXT:    ldr w9, [sp]
2302; NONEON-NOSVE-NEXT:    lsr w10, w9, #1
2303; NONEON-NOSVE-NEXT:    and w10, w10, #0x55555555
2304; NONEON-NOSVE-NEXT:    sub w9, w9, w10
2305; NONEON-NOSVE-NEXT:    lsr w10, w9, #2
2306; NONEON-NOSVE-NEXT:    and w9, w9, #0x33333333
2307; NONEON-NOSVE-NEXT:    and w10, w10, #0x33333333
2308; NONEON-NOSVE-NEXT:    add w9, w9, w10
2309; NONEON-NOSVE-NEXT:    add w9, w9, w9, lsr #4
2310; NONEON-NOSVE-NEXT:    and w9, w9, #0xf0f0f0f
2311; NONEON-NOSVE-NEXT:    mul w8, w9, w8
2312; NONEON-NOSVE-NEXT:    lsr w8, w8, #24
2313; NONEON-NOSVE-NEXT:    stp w8, w11, [sp, #32]
2314; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
2315; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
2316; NONEON-NOSVE-NEXT:    add sp, sp, #64
2317; NONEON-NOSVE-NEXT:    ret
2318  %op = load <8 x i32>, ptr %a
2319  %res = call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %op)
2320  store <8 x i32> %res, ptr %a
2321  ret void
2322}
2323
2324define <1 x i64> @ctpop_v1i64(<1 x i64> %op) {
2325; CHECK-LABEL: ctpop_v1i64:
2326; CHECK:       // %bb.0:
2327; CHECK-NEXT:    ptrue p0.d, vl1
2328; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2329; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
2330; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2331; CHECK-NEXT:    ret
2332;
2333; NONEON-NOSVE-LABEL: ctpop_v1i64:
2334; NONEON-NOSVE:       // %bb.0:
2335; NONEON-NOSVE-NEXT:    sub sp, sp, #16
2336; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2337; NONEON-NOSVE-NEXT:    fmov x9, d0
2338; NONEON-NOSVE-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
2339; NONEON-NOSVE-NEXT:    lsr x10, x9, #1
2340; NONEON-NOSVE-NEXT:    and x10, x10, #0x5555555555555555
2341; NONEON-NOSVE-NEXT:    sub x9, x9, x10
2342; NONEON-NOSVE-NEXT:    lsr x10, x9, #2
2343; NONEON-NOSVE-NEXT:    and x9, x9, #0x3333333333333333
2344; NONEON-NOSVE-NEXT:    and x10, x10, #0x3333333333333333
2345; NONEON-NOSVE-NEXT:    add x9, x9, x10
2346; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #4
2347; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
2348; NONEON-NOSVE-NEXT:    mul x8, x9, x8
2349; NONEON-NOSVE-NEXT:    lsr x8, x8, #56
2350; NONEON-NOSVE-NEXT:    str x8, [sp, #8]
2351; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2352; NONEON-NOSVE-NEXT:    add sp, sp, #16
2353; NONEON-NOSVE-NEXT:    ret
2354  %res = call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %op)
2355  ret <1 x i64> %res
2356}
2357
2358define <2 x i64> @ctpop_v2i64(<2 x i64> %op) {
2359; CHECK-LABEL: ctpop_v2i64:
2360; CHECK:       // %bb.0:
2361; CHECK-NEXT:    ptrue p0.d, vl2
2362; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2363; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
2364; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2365; CHECK-NEXT:    ret
2366;
2367; NONEON-NOSVE-LABEL: ctpop_v2i64:
2368; NONEON-NOSVE:       // %bb.0:
2369; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
2370; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
2371; NONEON-NOSVE-NEXT:    ldr x9, [sp, #8]
2372; NONEON-NOSVE-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
2373; NONEON-NOSVE-NEXT:    lsr x10, x9, #1
2374; NONEON-NOSVE-NEXT:    and x10, x10, #0x5555555555555555
2375; NONEON-NOSVE-NEXT:    sub x9, x9, x10
2376; NONEON-NOSVE-NEXT:    lsr x10, x9, #2
2377; NONEON-NOSVE-NEXT:    and x9, x9, #0x3333333333333333
2378; NONEON-NOSVE-NEXT:    and x10, x10, #0x3333333333333333
2379; NONEON-NOSVE-NEXT:    add x9, x9, x10
2380; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #4
2381; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
2382; NONEON-NOSVE-NEXT:    mul x9, x9, x8
2383; NONEON-NOSVE-NEXT:    lsr x11, x9, #56
2384; NONEON-NOSVE-NEXT:    ldr x9, [sp]
2385; NONEON-NOSVE-NEXT:    lsr x10, x9, #1
2386; NONEON-NOSVE-NEXT:    and x10, x10, #0x5555555555555555
2387; NONEON-NOSVE-NEXT:    sub x9, x9, x10
2388; NONEON-NOSVE-NEXT:    lsr x10, x9, #2
2389; NONEON-NOSVE-NEXT:    and x9, x9, #0x3333333333333333
2390; NONEON-NOSVE-NEXT:    and x10, x10, #0x3333333333333333
2391; NONEON-NOSVE-NEXT:    add x9, x9, x10
2392; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #4
2393; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
2394; NONEON-NOSVE-NEXT:    mul x8, x9, x8
2395; NONEON-NOSVE-NEXT:    lsr x8, x8, #56
2396; NONEON-NOSVE-NEXT:    stp x8, x11, [sp, #16]
2397; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
2398; NONEON-NOSVE-NEXT:    add sp, sp, #32
2399; NONEON-NOSVE-NEXT:    ret
2400  %res = call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %op)
2401  ret <2 x i64> %res
2402}
2403
2404define void @ctpop_v4i64(ptr %a) {
2405; CHECK-LABEL: ctpop_v4i64:
2406; CHECK:       // %bb.0:
2407; CHECK-NEXT:    ldp q0, q1, [x0]
2408; CHECK-NEXT:    ptrue p0.d, vl2
2409; CHECK-NEXT:    cnt z0.d, p0/m, z0.d
2410; CHECK-NEXT:    cnt z1.d, p0/m, z1.d
2411; CHECK-NEXT:    stp q0, q1, [x0]
2412; CHECK-NEXT:    ret
2413;
2414; NONEON-NOSVE-LABEL: ctpop_v4i64:
2415; NONEON-NOSVE:       // %bb.0:
2416; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
2417; NONEON-NOSVE-NEXT:    mov x8, #72340172838076673 // =0x101010101010101
2418; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
2419; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
2420; NONEON-NOSVE-NEXT:    ldr x9, [sp, #24]
2421; NONEON-NOSVE-NEXT:    lsr x10, x9, #1
2422; NONEON-NOSVE-NEXT:    and x10, x10, #0x5555555555555555
2423; NONEON-NOSVE-NEXT:    sub x9, x9, x10
2424; NONEON-NOSVE-NEXT:    lsr x10, x9, #2
2425; NONEON-NOSVE-NEXT:    and x9, x9, #0x3333333333333333
2426; NONEON-NOSVE-NEXT:    and x10, x10, #0x3333333333333333
2427; NONEON-NOSVE-NEXT:    add x9, x9, x10
2428; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #4
2429; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
2430; NONEON-NOSVE-NEXT:    mul x9, x9, x8
2431; NONEON-NOSVE-NEXT:    lsr x11, x9, #56
2432; NONEON-NOSVE-NEXT:    ldr x9, [sp, #16]
2433; NONEON-NOSVE-NEXT:    lsr x10, x9, #1
2434; NONEON-NOSVE-NEXT:    and x10, x10, #0x5555555555555555
2435; NONEON-NOSVE-NEXT:    sub x9, x9, x10
2436; NONEON-NOSVE-NEXT:    lsr x10, x9, #2
2437; NONEON-NOSVE-NEXT:    and x9, x9, #0x3333333333333333
2438; NONEON-NOSVE-NEXT:    and x10, x10, #0x3333333333333333
2439; NONEON-NOSVE-NEXT:    add x9, x9, x10
2440; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #4
2441; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
2442; NONEON-NOSVE-NEXT:    mul x9, x9, x8
2443; NONEON-NOSVE-NEXT:    lsr x9, x9, #56
2444; NONEON-NOSVE-NEXT:    stp x9, x11, [sp, #48]
2445; NONEON-NOSVE-NEXT:    ldr x9, [sp, #8]
2446; NONEON-NOSVE-NEXT:    lsr x10, x9, #1
2447; NONEON-NOSVE-NEXT:    and x10, x10, #0x5555555555555555
2448; NONEON-NOSVE-NEXT:    sub x9, x9, x10
2449; NONEON-NOSVE-NEXT:    lsr x10, x9, #2
2450; NONEON-NOSVE-NEXT:    and x9, x9, #0x3333333333333333
2451; NONEON-NOSVE-NEXT:    and x10, x10, #0x3333333333333333
2452; NONEON-NOSVE-NEXT:    add x9, x9, x10
2453; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #4
2454; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
2455; NONEON-NOSVE-NEXT:    mul x9, x9, x8
2456; NONEON-NOSVE-NEXT:    lsr x11, x9, #56
2457; NONEON-NOSVE-NEXT:    ldr x9, [sp]
2458; NONEON-NOSVE-NEXT:    lsr x10, x9, #1
2459; NONEON-NOSVE-NEXT:    and x10, x10, #0x5555555555555555
2460; NONEON-NOSVE-NEXT:    sub x9, x9, x10
2461; NONEON-NOSVE-NEXT:    lsr x10, x9, #2
2462; NONEON-NOSVE-NEXT:    and x9, x9, #0x3333333333333333
2463; NONEON-NOSVE-NEXT:    and x10, x10, #0x3333333333333333
2464; NONEON-NOSVE-NEXT:    add x9, x9, x10
2465; NONEON-NOSVE-NEXT:    add x9, x9, x9, lsr #4
2466; NONEON-NOSVE-NEXT:    and x9, x9, #0xf0f0f0f0f0f0f0f
2467; NONEON-NOSVE-NEXT:    mul x8, x9, x8
2468; NONEON-NOSVE-NEXT:    lsr x8, x8, #56
2469; NONEON-NOSVE-NEXT:    stp x8, x11, [sp, #32]
2470; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
2471; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
2472; NONEON-NOSVE-NEXT:    add sp, sp, #64
2473; NONEON-NOSVE-NEXT:    ret
2474  %op = load <4 x i64>, ptr %a
2475  %res = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %op)
2476  store <4 x i64> %res, ptr %a
2477  ret void
2478}
2479
2480;
2481; Count trailing zeros
2482;
2483
2484define <4 x i8> @cttz_v4i8(<4 x i8> %op) {
2485; CHECK-LABEL: cttz_v4i8:
2486; CHECK:       // %bb.0:
2487; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2488; CHECK-NEXT:    ptrue p0.h, vl4
2489; CHECK-NEXT:    orr z0.h, z0.h, #0x100
2490; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
2491; CHECK-NEXT:    clz z0.h, p0/m, z0.h
2492; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2493; CHECK-NEXT:    ret
2494;
2495; NONEON-NOSVE-LABEL: cttz_v4i8:
2496; NONEON-NOSVE:       // %bb.0:
2497; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2498; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2499; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]
2500; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2501; NONEON-NOSVE-NEXT:    rbit w8, w8
2502; NONEON-NOSVE-NEXT:    clz w8, w8
2503; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]
2504; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]
2505; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2506; NONEON-NOSVE-NEXT:    rbit w8, w8
2507; NONEON-NOSVE-NEXT:    clz w8, w8
2508; NONEON-NOSVE-NEXT:    strh w8, [sp, #12]
2509; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
2510; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2511; NONEON-NOSVE-NEXT:    rbit w8, w8
2512; NONEON-NOSVE-NEXT:    clz w8, w8
2513; NONEON-NOSVE-NEXT:    strh w8, [sp, #10]
2514; NONEON-NOSVE-NEXT:    ldrh w8, [sp]
2515; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2516; NONEON-NOSVE-NEXT:    rbit w8, w8
2517; NONEON-NOSVE-NEXT:    clz w8, w8
2518; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]
2519; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2520; NONEON-NOSVE-NEXT:    add sp, sp, #16
2521; NONEON-NOSVE-NEXT:    ret
2522  %res = call <4 x i8> @llvm.cttz.v4i8(<4 x i8> %op)
2523  ret <4 x i8> %res
2524}
2525
2526define <8 x i8> @cttz_v8i8(<8 x i8> %op) {
2527; CHECK-LABEL: cttz_v8i8:
2528; CHECK:       // %bb.0:
2529; CHECK-NEXT:    ptrue p0.b, vl8
2530; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2531; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
2532; CHECK-NEXT:    clz z0.b, p0/m, z0.b
2533; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2534; CHECK-NEXT:    ret
2535;
2536; NONEON-NOSVE-LABEL: cttz_v8i8:
2537; NONEON-NOSVE:       // %bb.0:
2538; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2539; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2540; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]
2541; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2542; NONEON-NOSVE-NEXT:    rbit w8, w8
2543; NONEON-NOSVE-NEXT:    clz w8, w8
2544; NONEON-NOSVE-NEXT:    strb w8, [sp, #15]
2545; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]
2546; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2547; NONEON-NOSVE-NEXT:    rbit w8, w8
2548; NONEON-NOSVE-NEXT:    clz w8, w8
2549; NONEON-NOSVE-NEXT:    strb w8, [sp, #14]
2550; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]
2551; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2552; NONEON-NOSVE-NEXT:    rbit w8, w8
2553; NONEON-NOSVE-NEXT:    clz w8, w8
2554; NONEON-NOSVE-NEXT:    strb w8, [sp, #13]
2555; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]
2556; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2557; NONEON-NOSVE-NEXT:    rbit w8, w8
2558; NONEON-NOSVE-NEXT:    clz w8, w8
2559; NONEON-NOSVE-NEXT:    strb w8, [sp, #12]
2560; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]
2561; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2562; NONEON-NOSVE-NEXT:    rbit w8, w8
2563; NONEON-NOSVE-NEXT:    clz w8, w8
2564; NONEON-NOSVE-NEXT:    strb w8, [sp, #11]
2565; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
2566; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2567; NONEON-NOSVE-NEXT:    rbit w8, w8
2568; NONEON-NOSVE-NEXT:    clz w8, w8
2569; NONEON-NOSVE-NEXT:    strb w8, [sp, #10]
2570; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]
2571; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2572; NONEON-NOSVE-NEXT:    rbit w8, w8
2573; NONEON-NOSVE-NEXT:    clz w8, w8
2574; NONEON-NOSVE-NEXT:    strb w8, [sp, #9]
2575; NONEON-NOSVE-NEXT:    ldrb w8, [sp]
2576; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2577; NONEON-NOSVE-NEXT:    rbit w8, w8
2578; NONEON-NOSVE-NEXT:    clz w8, w8
2579; NONEON-NOSVE-NEXT:    strb w8, [sp, #8]
2580; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2581; NONEON-NOSVE-NEXT:    add sp, sp, #16
2582; NONEON-NOSVE-NEXT:    ret
2583  %res = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %op)
2584  ret <8 x i8> %res
2585}
2586
2587define <16 x i8> @cttz_v16i8(<16 x i8> %op) {
2588; CHECK-LABEL: cttz_v16i8:
2589; CHECK:       // %bb.0:
2590; CHECK-NEXT:    ptrue p0.b, vl16
2591; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2592; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
2593; CHECK-NEXT:    clz z0.b, p0/m, z0.b
2594; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2595; CHECK-NEXT:    ret
2596;
2597; NONEON-NOSVE-LABEL: cttz_v16i8:
2598; NONEON-NOSVE:       // %bb.0:
2599; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
2600; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
2601; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]
2602; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2603; NONEON-NOSVE-NEXT:    rbit w8, w8
2604; NONEON-NOSVE-NEXT:    clz w8, w8
2605; NONEON-NOSVE-NEXT:    strb w8, [sp, #31]
2606; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
2607; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2608; NONEON-NOSVE-NEXT:    rbit w8, w8
2609; NONEON-NOSVE-NEXT:    clz w8, w8
2610; NONEON-NOSVE-NEXT:    strb w8, [sp, #30]
2611; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]
2612; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2613; NONEON-NOSVE-NEXT:    rbit w8, w8
2614; NONEON-NOSVE-NEXT:    clz w8, w8
2615; NONEON-NOSVE-NEXT:    strb w8, [sp, #29]
2616; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
2617; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2618; NONEON-NOSVE-NEXT:    rbit w8, w8
2619; NONEON-NOSVE-NEXT:    clz w8, w8
2620; NONEON-NOSVE-NEXT:    strb w8, [sp, #28]
2621; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]
2622; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2623; NONEON-NOSVE-NEXT:    rbit w8, w8
2624; NONEON-NOSVE-NEXT:    clz w8, w8
2625; NONEON-NOSVE-NEXT:    strb w8, [sp, #27]
2626; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
2627; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2628; NONEON-NOSVE-NEXT:    rbit w8, w8
2629; NONEON-NOSVE-NEXT:    clz w8, w8
2630; NONEON-NOSVE-NEXT:    strb w8, [sp, #26]
2631; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]
2632; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2633; NONEON-NOSVE-NEXT:    rbit w8, w8
2634; NONEON-NOSVE-NEXT:    clz w8, w8
2635; NONEON-NOSVE-NEXT:    strb w8, [sp, #25]
2636; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
2637; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2638; NONEON-NOSVE-NEXT:    rbit w8, w8
2639; NONEON-NOSVE-NEXT:    clz w8, w8
2640; NONEON-NOSVE-NEXT:    strb w8, [sp, #24]
2641; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]
2642; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2643; NONEON-NOSVE-NEXT:    rbit w8, w8
2644; NONEON-NOSVE-NEXT:    clz w8, w8
2645; NONEON-NOSVE-NEXT:    strb w8, [sp, #23]
2646; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]
2647; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2648; NONEON-NOSVE-NEXT:    rbit w8, w8
2649; NONEON-NOSVE-NEXT:    clz w8, w8
2650; NONEON-NOSVE-NEXT:    strb w8, [sp, #22]
2651; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]
2652; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2653; NONEON-NOSVE-NEXT:    rbit w8, w8
2654; NONEON-NOSVE-NEXT:    clz w8, w8
2655; NONEON-NOSVE-NEXT:    strb w8, [sp, #21]
2656; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]
2657; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2658; NONEON-NOSVE-NEXT:    rbit w8, w8
2659; NONEON-NOSVE-NEXT:    clz w8, w8
2660; NONEON-NOSVE-NEXT:    strb w8, [sp, #20]
2661; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]
2662; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2663; NONEON-NOSVE-NEXT:    rbit w8, w8
2664; NONEON-NOSVE-NEXT:    clz w8, w8
2665; NONEON-NOSVE-NEXT:    strb w8, [sp, #19]
2666; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
2667; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2668; NONEON-NOSVE-NEXT:    rbit w8, w8
2669; NONEON-NOSVE-NEXT:    clz w8, w8
2670; NONEON-NOSVE-NEXT:    strb w8, [sp, #18]
2671; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]
2672; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2673; NONEON-NOSVE-NEXT:    rbit w8, w8
2674; NONEON-NOSVE-NEXT:    clz w8, w8
2675; NONEON-NOSVE-NEXT:    strb w8, [sp, #17]
2676; NONEON-NOSVE-NEXT:    ldrb w8, [sp]
2677; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2678; NONEON-NOSVE-NEXT:    rbit w8, w8
2679; NONEON-NOSVE-NEXT:    clz w8, w8
2680; NONEON-NOSVE-NEXT:    strb w8, [sp, #16]
2681; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
2682; NONEON-NOSVE-NEXT:    add sp, sp, #32
2683; NONEON-NOSVE-NEXT:    ret
2684  %res = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %op)
2685  ret <16 x i8> %res
2686}
2687
2688define void @cttz_v32i8(ptr %a) {
2689; CHECK-LABEL: cttz_v32i8:
2690; CHECK:       // %bb.0:
2691; CHECK-NEXT:    ldp q0, q1, [x0]
2692; CHECK-NEXT:    ptrue p0.b, vl16
2693; CHECK-NEXT:    rbit z0.b, p0/m, z0.b
2694; CHECK-NEXT:    rbit z1.b, p0/m, z1.b
2695; CHECK-NEXT:    clz z0.b, p0/m, z0.b
2696; CHECK-NEXT:    clz z1.b, p0/m, z1.b
2697; CHECK-NEXT:    stp q0, q1, [x0]
2698; CHECK-NEXT:    ret
2699;
2700; NONEON-NOSVE-LABEL: cttz_v32i8:
2701; NONEON-NOSVE:       // %bb.0:
2702; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
2703; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
2704; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
2705; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #31]
2706; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2707; NONEON-NOSVE-NEXT:    rbit w8, w8
2708; NONEON-NOSVE-NEXT:    clz w8, w8
2709; NONEON-NOSVE-NEXT:    strb w8, [sp, #63]
2710; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #30]
2711; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2712; NONEON-NOSVE-NEXT:    rbit w8, w8
2713; NONEON-NOSVE-NEXT:    clz w8, w8
2714; NONEON-NOSVE-NEXT:    strb w8, [sp, #62]
2715; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #29]
2716; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2717; NONEON-NOSVE-NEXT:    rbit w8, w8
2718; NONEON-NOSVE-NEXT:    clz w8, w8
2719; NONEON-NOSVE-NEXT:    strb w8, [sp, #61]
2720; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #28]
2721; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2722; NONEON-NOSVE-NEXT:    rbit w8, w8
2723; NONEON-NOSVE-NEXT:    clz w8, w8
2724; NONEON-NOSVE-NEXT:    strb w8, [sp, #60]
2725; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #27]
2726; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2727; NONEON-NOSVE-NEXT:    rbit w8, w8
2728; NONEON-NOSVE-NEXT:    clz w8, w8
2729; NONEON-NOSVE-NEXT:    strb w8, [sp, #59]
2730; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #26]
2731; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2732; NONEON-NOSVE-NEXT:    rbit w8, w8
2733; NONEON-NOSVE-NEXT:    clz w8, w8
2734; NONEON-NOSVE-NEXT:    strb w8, [sp, #58]
2735; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #25]
2736; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2737; NONEON-NOSVE-NEXT:    rbit w8, w8
2738; NONEON-NOSVE-NEXT:    clz w8, w8
2739; NONEON-NOSVE-NEXT:    strb w8, [sp, #57]
2740; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #24]
2741; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2742; NONEON-NOSVE-NEXT:    rbit w8, w8
2743; NONEON-NOSVE-NEXT:    clz w8, w8
2744; NONEON-NOSVE-NEXT:    strb w8, [sp, #56]
2745; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #23]
2746; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2747; NONEON-NOSVE-NEXT:    rbit w8, w8
2748; NONEON-NOSVE-NEXT:    clz w8, w8
2749; NONEON-NOSVE-NEXT:    strb w8, [sp, #55]
2750; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #22]
2751; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2752; NONEON-NOSVE-NEXT:    rbit w8, w8
2753; NONEON-NOSVE-NEXT:    clz w8, w8
2754; NONEON-NOSVE-NEXT:    strb w8, [sp, #54]
2755; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #21]
2756; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2757; NONEON-NOSVE-NEXT:    rbit w8, w8
2758; NONEON-NOSVE-NEXT:    clz w8, w8
2759; NONEON-NOSVE-NEXT:    strb w8, [sp, #53]
2760; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #20]
2761; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2762; NONEON-NOSVE-NEXT:    rbit w8, w8
2763; NONEON-NOSVE-NEXT:    clz w8, w8
2764; NONEON-NOSVE-NEXT:    strb w8, [sp, #52]
2765; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #19]
2766; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2767; NONEON-NOSVE-NEXT:    rbit w8, w8
2768; NONEON-NOSVE-NEXT:    clz w8, w8
2769; NONEON-NOSVE-NEXT:    strb w8, [sp, #51]
2770; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #18]
2771; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2772; NONEON-NOSVE-NEXT:    rbit w8, w8
2773; NONEON-NOSVE-NEXT:    clz w8, w8
2774; NONEON-NOSVE-NEXT:    strb w8, [sp, #50]
2775; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #17]
2776; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2777; NONEON-NOSVE-NEXT:    rbit w8, w8
2778; NONEON-NOSVE-NEXT:    clz w8, w8
2779; NONEON-NOSVE-NEXT:    strb w8, [sp, #49]
2780; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
2781; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2782; NONEON-NOSVE-NEXT:    rbit w8, w8
2783; NONEON-NOSVE-NEXT:    clz w8, w8
2784; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
2785; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #15]
2786; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2787; NONEON-NOSVE-NEXT:    rbit w8, w8
2788; NONEON-NOSVE-NEXT:    clz w8, w8
2789; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]
2790; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
2791; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2792; NONEON-NOSVE-NEXT:    rbit w8, w8
2793; NONEON-NOSVE-NEXT:    clz w8, w8
2794; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]
2795; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #13]
2796; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2797; NONEON-NOSVE-NEXT:    rbit w8, w8
2798; NONEON-NOSVE-NEXT:    clz w8, w8
2799; NONEON-NOSVE-NEXT:    strb w8, [sp, #45]
2800; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #12]
2801; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2802; NONEON-NOSVE-NEXT:    rbit w8, w8
2803; NONEON-NOSVE-NEXT:    clz w8, w8
2804; NONEON-NOSVE-NEXT:    strb w8, [sp, #44]
2805; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #11]
2806; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2807; NONEON-NOSVE-NEXT:    rbit w8, w8
2808; NONEON-NOSVE-NEXT:    clz w8, w8
2809; NONEON-NOSVE-NEXT:    strb w8, [sp, #43]
2810; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #10]
2811; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2812; NONEON-NOSVE-NEXT:    rbit w8, w8
2813; NONEON-NOSVE-NEXT:    clz w8, w8
2814; NONEON-NOSVE-NEXT:    strb w8, [sp, #42]
2815; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #9]
2816; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2817; NONEON-NOSVE-NEXT:    rbit w8, w8
2818; NONEON-NOSVE-NEXT:    clz w8, w8
2819; NONEON-NOSVE-NEXT:    strb w8, [sp, #41]
2820; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #8]
2821; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2822; NONEON-NOSVE-NEXT:    rbit w8, w8
2823; NONEON-NOSVE-NEXT:    clz w8, w8
2824; NONEON-NOSVE-NEXT:    strb w8, [sp, #40]
2825; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #7]
2826; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2827; NONEON-NOSVE-NEXT:    rbit w8, w8
2828; NONEON-NOSVE-NEXT:    clz w8, w8
2829; NONEON-NOSVE-NEXT:    strb w8, [sp, #39]
2830; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #6]
2831; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2832; NONEON-NOSVE-NEXT:    rbit w8, w8
2833; NONEON-NOSVE-NEXT:    clz w8, w8
2834; NONEON-NOSVE-NEXT:    strb w8, [sp, #38]
2835; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #5]
2836; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2837; NONEON-NOSVE-NEXT:    rbit w8, w8
2838; NONEON-NOSVE-NEXT:    clz w8, w8
2839; NONEON-NOSVE-NEXT:    strb w8, [sp, #37]
2840; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #4]
2841; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2842; NONEON-NOSVE-NEXT:    rbit w8, w8
2843; NONEON-NOSVE-NEXT:    clz w8, w8
2844; NONEON-NOSVE-NEXT:    strb w8, [sp, #36]
2845; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #3]
2846; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2847; NONEON-NOSVE-NEXT:    rbit w8, w8
2848; NONEON-NOSVE-NEXT:    clz w8, w8
2849; NONEON-NOSVE-NEXT:    strb w8, [sp, #35]
2850; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
2851; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2852; NONEON-NOSVE-NEXT:    rbit w8, w8
2853; NONEON-NOSVE-NEXT:    clz w8, w8
2854; NONEON-NOSVE-NEXT:    strb w8, [sp, #34]
2855; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #1]
2856; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2857; NONEON-NOSVE-NEXT:    rbit w8, w8
2858; NONEON-NOSVE-NEXT:    clz w8, w8
2859; NONEON-NOSVE-NEXT:    strb w8, [sp, #33]
2860; NONEON-NOSVE-NEXT:    ldrb w8, [sp]
2861; NONEON-NOSVE-NEXT:    orr w8, w8, #0x100
2862; NONEON-NOSVE-NEXT:    rbit w8, w8
2863; NONEON-NOSVE-NEXT:    clz w8, w8
2864; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]
2865; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
2866; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
2867; NONEON-NOSVE-NEXT:    add sp, sp, #64
2868; NONEON-NOSVE-NEXT:    ret
2869  %op = load <32 x i8>, ptr %a
2870  %res = call <32 x i8> @llvm.cttz.v32i8(<32 x i8> %op)
2871  store <32 x i8> %res, ptr %a
2872  ret void
2873}
2874
2875define <2 x i16> @cttz_v2i16(<2 x i16> %op) {
2876; CHECK-LABEL: cttz_v2i16:
2877; CHECK:       // %bb.0:
2878; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2879; CHECK-NEXT:    ptrue p0.s, vl2
2880; CHECK-NEXT:    orr z0.s, z0.s, #0x10000
2881; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
2882; CHECK-NEXT:    clz z0.s, p0/m, z0.s
2883; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2884; CHECK-NEXT:    ret
2885;
2886; NONEON-NOSVE-LABEL: cttz_v2i16:
2887; NONEON-NOSVE:       // %bb.0:
2888; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2889; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2890; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
2891; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2892; NONEON-NOSVE-NEXT:    rbit w8, w8
2893; NONEON-NOSVE-NEXT:    clz w9, w8
2894; NONEON-NOSVE-NEXT:    ldr w8, [sp]
2895; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2896; NONEON-NOSVE-NEXT:    rbit w8, w8
2897; NONEON-NOSVE-NEXT:    clz w8, w8
2898; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8]
2899; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2900; NONEON-NOSVE-NEXT:    add sp, sp, #16
2901; NONEON-NOSVE-NEXT:    ret
2902  %res = call <2 x i16> @llvm.cttz.v2i16(<2 x i16> %op)
2903  ret <2 x i16> %res
2904}
2905
2906define <4 x i16> @cttz_v4i16(<4 x i16> %op) {
2907; CHECK-LABEL: cttz_v4i16:
2908; CHECK:       // %bb.0:
2909; CHECK-NEXT:    ptrue p0.h, vl4
2910; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2911; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
2912; CHECK-NEXT:    clz z0.h, p0/m, z0.h
2913; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2914; CHECK-NEXT:    ret
2915;
2916; NONEON-NOSVE-LABEL: cttz_v4i16:
2917; NONEON-NOSVE:       // %bb.0:
2918; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
2919; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
2920; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]
2921; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2922; NONEON-NOSVE-NEXT:    rbit w8, w8
2923; NONEON-NOSVE-NEXT:    clz w8, w8
2924; NONEON-NOSVE-NEXT:    strh w8, [sp, #14]
2925; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]
2926; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2927; NONEON-NOSVE-NEXT:    rbit w8, w8
2928; NONEON-NOSVE-NEXT:    clz w8, w8
2929; NONEON-NOSVE-NEXT:    strh w8, [sp, #12]
2930; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
2931; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2932; NONEON-NOSVE-NEXT:    rbit w8, w8
2933; NONEON-NOSVE-NEXT:    clz w8, w8
2934; NONEON-NOSVE-NEXT:    strh w8, [sp, #10]
2935; NONEON-NOSVE-NEXT:    ldrh w8, [sp]
2936; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2937; NONEON-NOSVE-NEXT:    rbit w8, w8
2938; NONEON-NOSVE-NEXT:    clz w8, w8
2939; NONEON-NOSVE-NEXT:    strh w8, [sp, #8]
2940; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
2941; NONEON-NOSVE-NEXT:    add sp, sp, #16
2942; NONEON-NOSVE-NEXT:    ret
2943  %res = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %op)
2944  ret <4 x i16> %res
2945}
2946
2947define <8 x i16> @cttz_v8i16(<8 x i16> %op) {
2948; CHECK-LABEL: cttz_v8i16:
2949; CHECK:       // %bb.0:
2950; CHECK-NEXT:    ptrue p0.h, vl8
2951; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
2952; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
2953; CHECK-NEXT:    clz z0.h, p0/m, z0.h
2954; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2955; CHECK-NEXT:    ret
2956;
2957; NONEON-NOSVE-LABEL: cttz_v8i16:
2958; NONEON-NOSVE:       // %bb.0:
2959; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
2960; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
2961; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]
2962; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2963; NONEON-NOSVE-NEXT:    rbit w8, w8
2964; NONEON-NOSVE-NEXT:    clz w8, w8
2965; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
2966; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
2967; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2968; NONEON-NOSVE-NEXT:    rbit w8, w8
2969; NONEON-NOSVE-NEXT:    clz w8, w8
2970; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
2971; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]
2972; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2973; NONEON-NOSVE-NEXT:    rbit w8, w8
2974; NONEON-NOSVE-NEXT:    clz w8, w8
2975; NONEON-NOSVE-NEXT:    strh w8, [sp, #26]
2976; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]
2977; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2978; NONEON-NOSVE-NEXT:    rbit w8, w8
2979; NONEON-NOSVE-NEXT:    clz w8, w8
2980; NONEON-NOSVE-NEXT:    strh w8, [sp, #24]
2981; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]
2982; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2983; NONEON-NOSVE-NEXT:    rbit w8, w8
2984; NONEON-NOSVE-NEXT:    clz w8, w8
2985; NONEON-NOSVE-NEXT:    strh w8, [sp, #22]
2986; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]
2987; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2988; NONEON-NOSVE-NEXT:    rbit w8, w8
2989; NONEON-NOSVE-NEXT:    clz w8, w8
2990; NONEON-NOSVE-NEXT:    strh w8, [sp, #20]
2991; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
2992; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2993; NONEON-NOSVE-NEXT:    rbit w8, w8
2994; NONEON-NOSVE-NEXT:    clz w8, w8
2995; NONEON-NOSVE-NEXT:    strh w8, [sp, #18]
2996; NONEON-NOSVE-NEXT:    ldrh w8, [sp]
2997; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
2998; NONEON-NOSVE-NEXT:    rbit w8, w8
2999; NONEON-NOSVE-NEXT:    clz w8, w8
3000; NONEON-NOSVE-NEXT:    strh w8, [sp, #16]
3001; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
3002; NONEON-NOSVE-NEXT:    add sp, sp, #32
3003; NONEON-NOSVE-NEXT:    ret
3004  %res = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %op)
3005  ret <8 x i16> %res
3006}
3007
3008define void @cttz_v16i16(ptr %a) {
3009; CHECK-LABEL: cttz_v16i16:
3010; CHECK:       // %bb.0:
3011; CHECK-NEXT:    ldp q0, q1, [x0]
3012; CHECK-NEXT:    ptrue p0.h, vl8
3013; CHECK-NEXT:    rbit z0.h, p0/m, z0.h
3014; CHECK-NEXT:    rbit z1.h, p0/m, z1.h
3015; CHECK-NEXT:    clz z0.h, p0/m, z0.h
3016; CHECK-NEXT:    clz z1.h, p0/m, z1.h
3017; CHECK-NEXT:    stp q0, q1, [x0]
3018; CHECK-NEXT:    ret
3019;
3020; NONEON-NOSVE-LABEL: cttz_v16i16:
3021; NONEON-NOSVE:       // %bb.0:
3022; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
3023; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
3024; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
3025; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #30]
3026; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3027; NONEON-NOSVE-NEXT:    rbit w8, w8
3028; NONEON-NOSVE-NEXT:    clz w8, w8
3029; NONEON-NOSVE-NEXT:    strh w8, [sp, #62]
3030; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #28]
3031; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3032; NONEON-NOSVE-NEXT:    rbit w8, w8
3033; NONEON-NOSVE-NEXT:    clz w8, w8
3034; NONEON-NOSVE-NEXT:    strh w8, [sp, #60]
3035; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #26]
3036; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3037; NONEON-NOSVE-NEXT:    rbit w8, w8
3038; NONEON-NOSVE-NEXT:    clz w8, w8
3039; NONEON-NOSVE-NEXT:    strh w8, [sp, #58]
3040; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #24]
3041; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3042; NONEON-NOSVE-NEXT:    rbit w8, w8
3043; NONEON-NOSVE-NEXT:    clz w8, w8
3044; NONEON-NOSVE-NEXT:    strh w8, [sp, #56]
3045; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #22]
3046; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3047; NONEON-NOSVE-NEXT:    rbit w8, w8
3048; NONEON-NOSVE-NEXT:    clz w8, w8
3049; NONEON-NOSVE-NEXT:    strh w8, [sp, #54]
3050; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #20]
3051; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3052; NONEON-NOSVE-NEXT:    rbit w8, w8
3053; NONEON-NOSVE-NEXT:    clz w8, w8
3054; NONEON-NOSVE-NEXT:    strh w8, [sp, #52]
3055; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #18]
3056; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3057; NONEON-NOSVE-NEXT:    rbit w8, w8
3058; NONEON-NOSVE-NEXT:    clz w8, w8
3059; NONEON-NOSVE-NEXT:    strh w8, [sp, #50]
3060; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
3061; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3062; NONEON-NOSVE-NEXT:    rbit w8, w8
3063; NONEON-NOSVE-NEXT:    clz w8, w8
3064; NONEON-NOSVE-NEXT:    strh w8, [sp, #48]
3065; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #14]
3066; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3067; NONEON-NOSVE-NEXT:    rbit w8, w8
3068; NONEON-NOSVE-NEXT:    clz w8, w8
3069; NONEON-NOSVE-NEXT:    strh w8, [sp, #46]
3070; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
3071; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3072; NONEON-NOSVE-NEXT:    rbit w8, w8
3073; NONEON-NOSVE-NEXT:    clz w8, w8
3074; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
3075; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #10]
3076; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3077; NONEON-NOSVE-NEXT:    rbit w8, w8
3078; NONEON-NOSVE-NEXT:    clz w8, w8
3079; NONEON-NOSVE-NEXT:    strh w8, [sp, #42]
3080; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]
3081; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3082; NONEON-NOSVE-NEXT:    rbit w8, w8
3083; NONEON-NOSVE-NEXT:    clz w8, w8
3084; NONEON-NOSVE-NEXT:    strh w8, [sp, #40]
3085; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #6]
3086; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3087; NONEON-NOSVE-NEXT:    rbit w8, w8
3088; NONEON-NOSVE-NEXT:    clz w8, w8
3089; NONEON-NOSVE-NEXT:    strh w8, [sp, #38]
3090; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #4]
3091; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3092; NONEON-NOSVE-NEXT:    rbit w8, w8
3093; NONEON-NOSVE-NEXT:    clz w8, w8
3094; NONEON-NOSVE-NEXT:    strh w8, [sp, #36]
3095; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
3096; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3097; NONEON-NOSVE-NEXT:    rbit w8, w8
3098; NONEON-NOSVE-NEXT:    clz w8, w8
3099; NONEON-NOSVE-NEXT:    strh w8, [sp, #34]
3100; NONEON-NOSVE-NEXT:    ldrh w8, [sp]
3101; NONEON-NOSVE-NEXT:    orr w8, w8, #0x10000
3102; NONEON-NOSVE-NEXT:    rbit w8, w8
3103; NONEON-NOSVE-NEXT:    clz w8, w8
3104; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
3105; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
3106; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
3107; NONEON-NOSVE-NEXT:    add sp, sp, #64
3108; NONEON-NOSVE-NEXT:    ret
3109  %op = load <16 x i16>, ptr %a
3110  %res = call <16 x i16> @llvm.cttz.v16i16(<16 x i16> %op)
3111  store <16 x i16> %res, ptr %a
3112  ret void
3113}
3114
3115define <2 x i32> @cttz_v2i32(<2 x i32> %op) {
3116; CHECK-LABEL: cttz_v2i32:
3117; CHECK:       // %bb.0:
3118; CHECK-NEXT:    ptrue p0.s, vl2
3119; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
3120; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
3121; CHECK-NEXT:    clz z0.s, p0/m, z0.s
3122; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
3123; CHECK-NEXT:    ret
3124;
3125; NONEON-NOSVE-LABEL: cttz_v2i32:
3126; NONEON-NOSVE:       // %bb.0:
3127; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
3128; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
3129; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
3130; NONEON-NOSVE-NEXT:    rbit w8, w8
3131; NONEON-NOSVE-NEXT:    clz w9, w8
3132; NONEON-NOSVE-NEXT:    ldr w8, [sp]
3133; NONEON-NOSVE-NEXT:    rbit w8, w8
3134; NONEON-NOSVE-NEXT:    clz w8, w8
3135; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #8]
3136; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
3137; NONEON-NOSVE-NEXT:    add sp, sp, #16
3138; NONEON-NOSVE-NEXT:    ret
3139  %res = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %op)
3140  ret <2 x i32> %res
3141}
3142
3143define <4 x i32> @cttz_v4i32(<4 x i32> %op) {
3144; CHECK-LABEL: cttz_v4i32:
3145; CHECK:       // %bb.0:
3146; CHECK-NEXT:    ptrue p0.s, vl4
3147; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
3148; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
3149; CHECK-NEXT:    clz z0.s, p0/m, z0.s
3150; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
3151; CHECK-NEXT:    ret
3152;
3153; NONEON-NOSVE-LABEL: cttz_v4i32:
3154; NONEON-NOSVE:       // %bb.0:
3155; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
3156; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
3157; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]
3158; NONEON-NOSVE-NEXT:    rbit w8, w8
3159; NONEON-NOSVE-NEXT:    clz w9, w8
3160; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
3161; NONEON-NOSVE-NEXT:    rbit w8, w8
3162; NONEON-NOSVE-NEXT:    clz w8, w8
3163; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
3164; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
3165; NONEON-NOSVE-NEXT:    rbit w8, w8
3166; NONEON-NOSVE-NEXT:    clz w9, w8
3167; NONEON-NOSVE-NEXT:    ldr w8, [sp]
3168; NONEON-NOSVE-NEXT:    rbit w8, w8
3169; NONEON-NOSVE-NEXT:    clz w8, w8
3170; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
3171; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
3172; NONEON-NOSVE-NEXT:    add sp, sp, #32
3173; NONEON-NOSVE-NEXT:    ret
3174  %res = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %op)
3175  ret <4 x i32> %res
3176}
3177
3178define void @cttz_v8i32(ptr %a) {
3179; CHECK-LABEL: cttz_v8i32:
3180; CHECK:       // %bb.0:
3181; CHECK-NEXT:    ldp q0, q1, [x0]
3182; CHECK-NEXT:    ptrue p0.s, vl4
3183; CHECK-NEXT:    rbit z0.s, p0/m, z0.s
3184; CHECK-NEXT:    rbit z1.s, p0/m, z1.s
3185; CHECK-NEXT:    clz z0.s, p0/m, z0.s
3186; CHECK-NEXT:    clz z1.s, p0/m, z1.s
3187; CHECK-NEXT:    stp q0, q1, [x0]
3188; CHECK-NEXT:    ret
3189;
3190; NONEON-NOSVE-LABEL: cttz_v8i32:
3191; NONEON-NOSVE:       // %bb.0:
3192; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
3193; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
3194; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
3195; NONEON-NOSVE-NEXT:    ldr w8, [sp, #28]
3196; NONEON-NOSVE-NEXT:    rbit w8, w8
3197; NONEON-NOSVE-NEXT:    clz w9, w8
3198; NONEON-NOSVE-NEXT:    ldr w8, [sp, #24]
3199; NONEON-NOSVE-NEXT:    rbit w8, w8
3200; NONEON-NOSVE-NEXT:    clz w8, w8
3201; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #56]
3202; NONEON-NOSVE-NEXT:    ldr w8, [sp, #20]
3203; NONEON-NOSVE-NEXT:    rbit w8, w8
3204; NONEON-NOSVE-NEXT:    clz w9, w8
3205; NONEON-NOSVE-NEXT:    ldr w8, [sp, #16]
3206; NONEON-NOSVE-NEXT:    rbit w8, w8
3207; NONEON-NOSVE-NEXT:    clz w8, w8
3208; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #48]
3209; NONEON-NOSVE-NEXT:    ldr w8, [sp, #12]
3210; NONEON-NOSVE-NEXT:    rbit w8, w8
3211; NONEON-NOSVE-NEXT:    clz w9, w8
3212; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
3213; NONEON-NOSVE-NEXT:    rbit w8, w8
3214; NONEON-NOSVE-NEXT:    clz w8, w8
3215; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #40]
3216; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
3217; NONEON-NOSVE-NEXT:    rbit w8, w8
3218; NONEON-NOSVE-NEXT:    clz w9, w8
3219; NONEON-NOSVE-NEXT:    ldr w8, [sp]
3220; NONEON-NOSVE-NEXT:    rbit w8, w8
3221; NONEON-NOSVE-NEXT:    clz w8, w8
3222; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #32]
3223; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
3224; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
3225; NONEON-NOSVE-NEXT:    add sp, sp, #64
3226; NONEON-NOSVE-NEXT:    ret
3227  %op = load <8 x i32>, ptr %a
3228  %res = call <8 x i32> @llvm.cttz.v8i32(<8 x i32> %op)
3229  store <8 x i32> %res, ptr %a
3230  ret void
3231}
3232
3233define <1 x i64> @cttz_v1i64(<1 x i64> %op) {
3234; CHECK-LABEL: cttz_v1i64:
3235; CHECK:       // %bb.0:
3236; CHECK-NEXT:    ptrue p0.d, vl1
3237; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
3238; CHECK-NEXT:    rbit z0.d, p0/m, z0.d
3239; CHECK-NEXT:    clz z0.d, p0/m, z0.d
3240; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
3241; CHECK-NEXT:    ret
3242;
3243; NONEON-NOSVE-LABEL: cttz_v1i64:
3244; NONEON-NOSVE:       // %bb.0:
3245; NONEON-NOSVE-NEXT:    sub sp, sp, #16
3246; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
3247; NONEON-NOSVE-NEXT:    fmov x8, d0
3248; NONEON-NOSVE-NEXT:    rbit x8, x8
3249; NONEON-NOSVE-NEXT:    clz x8, x8
3250; NONEON-NOSVE-NEXT:    str x8, [sp, #8]
3251; NONEON-NOSVE-NEXT:    ldr d0, [sp, #8]
3252; NONEON-NOSVE-NEXT:    add sp, sp, #16
3253; NONEON-NOSVE-NEXT:    ret
3254  %res = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %op)
3255  ret <1 x i64> %res
3256}
3257
3258define <2 x i64> @cttz_v2i64(<2 x i64> %op) {
3259; CHECK-LABEL: cttz_v2i64:
3260; CHECK:       // %bb.0:
3261; CHECK-NEXT:    ptrue p0.d, vl2
3262; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
3263; CHECK-NEXT:    rbit z0.d, p0/m, z0.d
3264; CHECK-NEXT:    clz z0.d, p0/m, z0.d
3265; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
3266; CHECK-NEXT:    ret
3267;
3268; NONEON-NOSVE-LABEL: cttz_v2i64:
3269; NONEON-NOSVE:       // %bb.0:
3270; NONEON-NOSVE-NEXT:    str q0, [sp, #-32]!
3271; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 32
3272; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]
3273; NONEON-NOSVE-NEXT:    rbit x8, x8
3274; NONEON-NOSVE-NEXT:    clz x9, x8
3275; NONEON-NOSVE-NEXT:    ldr x8, [sp]
3276; NONEON-NOSVE-NEXT:    rbit x8, x8
3277; NONEON-NOSVE-NEXT:    clz x8, x8
3278; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #16]
3279; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
3280; NONEON-NOSVE-NEXT:    add sp, sp, #32
3281; NONEON-NOSVE-NEXT:    ret
3282  %res = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %op)
3283  ret <2 x i64> %res
3284}
3285
3286define void @cttz_v4i64(ptr %a) {
3287; CHECK-LABEL: cttz_v4i64:
3288; CHECK:       // %bb.0:
3289; CHECK-NEXT:    ldp q0, q1, [x0]
3290; CHECK-NEXT:    ptrue p0.d, vl2
3291; CHECK-NEXT:    rbit z0.d, p0/m, z0.d
3292; CHECK-NEXT:    rbit z1.d, p0/m, z1.d
3293; CHECK-NEXT:    clz z0.d, p0/m, z0.d
3294; CHECK-NEXT:    clz z1.d, p0/m, z1.d
3295; CHECK-NEXT:    stp q0, q1, [x0]
3296; CHECK-NEXT:    ret
3297;
3298; NONEON-NOSVE-LABEL: cttz_v4i64:
3299; NONEON-NOSVE:       // %bb.0:
3300; NONEON-NOSVE-NEXT:    ldp q1, q0, [x0]
3301; NONEON-NOSVE-NEXT:    stp q1, q0, [sp, #-64]!
3302; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
3303; NONEON-NOSVE-NEXT:    ldr x8, [sp, #24]
3304; NONEON-NOSVE-NEXT:    rbit x8, x8
3305; NONEON-NOSVE-NEXT:    clz x9, x8
3306; NONEON-NOSVE-NEXT:    ldr x8, [sp, #16]
3307; NONEON-NOSVE-NEXT:    rbit x8, x8
3308; NONEON-NOSVE-NEXT:    clz x8, x8
3309; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #48]
3310; NONEON-NOSVE-NEXT:    ldr x8, [sp, #8]
3311; NONEON-NOSVE-NEXT:    rbit x8, x8
3312; NONEON-NOSVE-NEXT:    clz x9, x8
3313; NONEON-NOSVE-NEXT:    ldr x8, [sp]
3314; NONEON-NOSVE-NEXT:    rbit x8, x8
3315; NONEON-NOSVE-NEXT:    clz x8, x8
3316; NONEON-NOSVE-NEXT:    stp x8, x9, [sp, #32]
3317; NONEON-NOSVE-NEXT:    ldp q0, q1, [sp, #32]
3318; NONEON-NOSVE-NEXT:    stp q0, q1, [x0]
3319; NONEON-NOSVE-NEXT:    add sp, sp, #64
3320; NONEON-NOSVE-NEXT:    ret
3321  %op = load <4 x i64>, ptr %a
3322  %res = call <4 x i64> @llvm.cttz.v4i64(<4 x i64> %op)
3323  store <4 x i64> %res, ptr %a
3324  ret void
3325}
3326
3327
3328declare <4 x i8> @llvm.ctlz.v4i8(<4 x i8>)
3329declare <8 x i8> @llvm.ctlz.v8i8(<8 x i8>)
3330declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>)
3331declare <32 x i8> @llvm.ctlz.v32i8(<32 x i8>)
3332declare <2 x i16> @llvm.ctlz.v2i16(<2 x i16>)
3333declare <4 x i16> @llvm.ctlz.v4i16(<4 x i16>)
3334declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>)
3335declare <16 x i16> @llvm.ctlz.v16i16(<16 x i16>)
3336declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>)
3337declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>)
3338declare <8 x i32> @llvm.ctlz.v8i32(<8 x i32>)
3339declare <1 x i64> @llvm.ctlz.v1i64(<1 x i64>)
3340declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>)
3341declare <4 x i64> @llvm.ctlz.v4i64(<4 x i64>)
3342
3343declare <4 x i8> @llvm.ctpop.v4i8(<4 x i8>)
3344declare <8 x i8> @llvm.ctpop.v8i8(<8 x i8>)
3345declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>)
3346declare <32 x i8> @llvm.ctpop.v32i8(<32 x i8>)
3347declare <2 x i16> @llvm.ctpop.v2i16(<2 x i16>)
3348declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)
3349declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
3350declare <16 x i16> @llvm.ctpop.v16i16(<16 x i16>)
3351declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
3352declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
3353declare <8 x i32> @llvm.ctpop.v8i32(<8 x i32>)
3354declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>)
3355declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
3356declare <4 x i64> @llvm.ctpop.v4i64(<4 x i64>)
3357
3358declare <4 x i8> @llvm.cttz.v4i8(<4 x i8>)
3359declare <8 x i8> @llvm.cttz.v8i8(<8 x i8>)
3360declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>)
3361declare <32 x i8> @llvm.cttz.v32i8(<32 x i8>)
3362declare <2 x i16> @llvm.cttz.v2i16(<2 x i16>)
3363declare <4 x i16> @llvm.cttz.v4i16(<4 x i16>)
3364declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>)
3365declare <16 x i16> @llvm.cttz.v16i16(<16 x i16>)
3366declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>)
3367declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>)
3368declare <8 x i32> @llvm.cttz.v8i32(<8 x i32>)
3369declare <1 x i64> @llvm.cttz.v1i64(<1 x i64>)
3370declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>)
3371declare <4 x i64> @llvm.cttz.v4i64(<4 x i64>)
3372