xref: /llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-store.ll (revision 61510b51c33464a6bc15e4cf5b1ee07e2e0ec1c9) 
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
3; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
4
5
6target triple = "aarch64-unknown-linux-gnu"
7
8;
9; Masked Store
10;
11
12define void @masked_store_v4i8(ptr %dst, <4 x i1> %mask) {
13; CHECK-LABEL: masked_store_v4i8:
14; CHECK:       // %bb.0:
15; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
16; CHECK-NEXT:    ptrue p0.h, vl4
17; CHECK-NEXT:    lsl z0.h, z0.h, #15
18; CHECK-NEXT:    asr z0.h, z0.h, #15
19; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
20; CHECK-NEXT:    mov z0.h, #0 // =0x0
21; CHECK-NEXT:    st1b { z0.h }, p0, [x0]
22; CHECK-NEXT:    ret
23;
24; NONEON-NOSVE-LABEL: masked_store_v4i8:
25; NONEON-NOSVE:       // %bb.0:
26; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
27; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
28; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
29; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
30; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #6]
31; NONEON-NOSVE-NEXT:    ldrh w11, [sp]
32; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
33; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
34; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
35; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
36; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
37; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
38; NONEON-NOSVE-NEXT:    bfxil w8, w11, #0, #1
39; NONEON-NOSVE-NEXT:    orr w9, w9, w10
40; NONEON-NOSVE-NEXT:    orr w8, w8, w9
41; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB0_5
42; NONEON-NOSVE-NEXT:  // %bb.1: // %else
43; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB0_6
44; NONEON-NOSVE-NEXT:  .LBB0_2: // %else2
45; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB0_7
46; NONEON-NOSVE-NEXT:  .LBB0_3: // %else4
47; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB0_8
48; NONEON-NOSVE-NEXT:  .LBB0_4: // %else6
49; NONEON-NOSVE-NEXT:    add sp, sp, #16
50; NONEON-NOSVE-NEXT:    ret
51; NONEON-NOSVE-NEXT:  .LBB0_5: // %cond.store
52; NONEON-NOSVE-NEXT:    strb wzr, [x0]
53; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB0_2
54; NONEON-NOSVE-NEXT:  .LBB0_6: // %cond.store1
55; NONEON-NOSVE-NEXT:    strb wzr, [x0, #1]
56; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB0_3
57; NONEON-NOSVE-NEXT:  .LBB0_7: // %cond.store3
58; NONEON-NOSVE-NEXT:    strb wzr, [x0, #2]
59; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB0_4
60; NONEON-NOSVE-NEXT:  .LBB0_8: // %cond.store5
61; NONEON-NOSVE-NEXT:    strb wzr, [x0, #3]
62; NONEON-NOSVE-NEXT:    add sp, sp, #16
63; NONEON-NOSVE-NEXT:    ret
64  call void @llvm.masked.store.v4i8(<4 x i8> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
65  ret void
66}
67
68define void @masked_store_v8i8(ptr %dst, <8 x i1> %mask) {
69; CHECK-LABEL: masked_store_v8i8:
70; CHECK:       // %bb.0:
71; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
72; CHECK-NEXT:    ptrue p0.b, vl8
73; CHECK-NEXT:    lsl z0.b, z0.b, #7
74; CHECK-NEXT:    asr z0.b, z0.b, #7
75; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, #0
76; CHECK-NEXT:    mov z0.b, #0 // =0x0
77; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
78; CHECK-NEXT:    ret
79;
80; NONEON-NOSVE-LABEL: masked_store_v8i8:
81; NONEON-NOSVE:       // %bb.0:
82; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
83; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
84; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
85; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
86; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1]
87; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #4]
88; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #5]
89; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #6]
90; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
91; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
92; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
93; NONEON-NOSVE-NEXT:    ldrb w11, [sp]
94; NONEON-NOSVE-NEXT:    sbfx w12, w12, #0, #1
95; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
96; NONEON-NOSVE-NEXT:    and w8, w8, #0x4
97; NONEON-NOSVE-NEXT:    and w9, w9, #0x8
98; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
99; NONEON-NOSVE-NEXT:    orr w8, w8, w9
100; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
101; NONEON-NOSVE-NEXT:    and w10, w10, #0x2
102; NONEON-NOSVE-NEXT:    and w12, w12, #0x10
103; NONEON-NOSVE-NEXT:    bfxil w10, w11, #0, #1
104; NONEON-NOSVE-NEXT:    and w11, w13, #0x20
105; NONEON-NOSVE-NEXT:    orr w8, w8, w12
106; NONEON-NOSVE-NEXT:    and w12, w14, #0x40
107; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
108; NONEON-NOSVE-NEXT:    orr w8, w10, w8
109; NONEON-NOSVE-NEXT:    orr w10, w11, w12
110; NONEON-NOSVE-NEXT:    orr w8, w8, w10
111; NONEON-NOSVE-NEXT:    and w9, w9, #0x80
112; NONEON-NOSVE-NEXT:    add w9, w8, w9
113; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
114; NONEON-NOSVE-NEXT:    tbnz w9, #0, .LBB1_9
115; NONEON-NOSVE-NEXT:  // %bb.1: // %else
116; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB1_10
117; NONEON-NOSVE-NEXT:  .LBB1_2: // %else2
118; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB1_11
119; NONEON-NOSVE-NEXT:  .LBB1_3: // %else4
120; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB1_12
121; NONEON-NOSVE-NEXT:  .LBB1_4: // %else6
122; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB1_13
123; NONEON-NOSVE-NEXT:  .LBB1_5: // %else8
124; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB1_14
125; NONEON-NOSVE-NEXT:  .LBB1_6: // %else10
126; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB1_15
127; NONEON-NOSVE-NEXT:  .LBB1_7: // %else12
128; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB1_16
129; NONEON-NOSVE-NEXT:  .LBB1_8: // %else14
130; NONEON-NOSVE-NEXT:    add sp, sp, #16
131; NONEON-NOSVE-NEXT:    ret
132; NONEON-NOSVE-NEXT:  .LBB1_9: // %cond.store
133; NONEON-NOSVE-NEXT:    strb wzr, [x0]
134; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB1_2
135; NONEON-NOSVE-NEXT:  .LBB1_10: // %cond.store1
136; NONEON-NOSVE-NEXT:    strb wzr, [x0, #1]
137; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB1_3
138; NONEON-NOSVE-NEXT:  .LBB1_11: // %cond.store3
139; NONEON-NOSVE-NEXT:    strb wzr, [x0, #2]
140; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB1_4
141; NONEON-NOSVE-NEXT:  .LBB1_12: // %cond.store5
142; NONEON-NOSVE-NEXT:    strb wzr, [x0, #3]
143; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB1_5
144; NONEON-NOSVE-NEXT:  .LBB1_13: // %cond.store7
145; NONEON-NOSVE-NEXT:    strb wzr, [x0, #4]
146; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB1_6
147; NONEON-NOSVE-NEXT:  .LBB1_14: // %cond.store9
148; NONEON-NOSVE-NEXT:    strb wzr, [x0, #5]
149; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB1_7
150; NONEON-NOSVE-NEXT:  .LBB1_15: // %cond.store11
151; NONEON-NOSVE-NEXT:    strb wzr, [x0, #6]
152; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB1_8
153; NONEON-NOSVE-NEXT:  .LBB1_16: // %cond.store13
154; NONEON-NOSVE-NEXT:    strb wzr, [x0, #7]
155; NONEON-NOSVE-NEXT:    add sp, sp, #16
156; NONEON-NOSVE-NEXT:    ret
157  call void @llvm.masked.store.v8i8(<8 x i8> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
158  ret void
159}
160
161define void @masked_store_v16i8(ptr %dst, <16 x i1> %mask) {
162; CHECK-LABEL: masked_store_v16i8:
163; CHECK:       // %bb.0:
164; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
165; CHECK-NEXT:    ptrue p0.b, vl16
166; CHECK-NEXT:    lsl z0.b, z0.b, #7
167; CHECK-NEXT:    asr z0.b, z0.b, #7
168; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, #0
169; CHECK-NEXT:    mov z0.b, #0 // =0x0
170; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
171; CHECK-NEXT:    ret
172;
173; NONEON-NOSVE-LABEL: masked_store_v16i8:
174; NONEON-NOSVE:       // %bb.0:
175; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
176; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
177; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
178; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
179; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1]
180; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #4]
181; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #5]
182; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #6]
183; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
184; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
185; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
186; NONEON-NOSVE-NEXT:    ldrb w11, [sp]
187; NONEON-NOSVE-NEXT:    sbfx w12, w12, #0, #1
188; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
189; NONEON-NOSVE-NEXT:    and w8, w8, #0x4
190; NONEON-NOSVE-NEXT:    and w9, w9, #0x8
191; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
192; NONEON-NOSVE-NEXT:    orr w8, w8, w9
193; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
194; NONEON-NOSVE-NEXT:    and w10, w10, #0x2
195; NONEON-NOSVE-NEXT:    and w12, w12, #0x10
196; NONEON-NOSVE-NEXT:    bfxil w10, w11, #0, #1
197; NONEON-NOSVE-NEXT:    and w11, w13, #0x20
198; NONEON-NOSVE-NEXT:    orr w8, w8, w12
199; NONEON-NOSVE-NEXT:    and w12, w14, #0x40
200; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
201; NONEON-NOSVE-NEXT:    orr w8, w10, w8
202; NONEON-NOSVE-NEXT:    orr w10, w11, w12
203; NONEON-NOSVE-NEXT:    orr w8, w8, w10
204; NONEON-NOSVE-NEXT:    and w9, w9, #0xffffff80
205; NONEON-NOSVE-NEXT:    add w8, w8, w9
206; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB2_17
207; NONEON-NOSVE-NEXT:  // %bb.1: // %else
208; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB2_18
209; NONEON-NOSVE-NEXT:  .LBB2_2: // %else2
210; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB2_19
211; NONEON-NOSVE-NEXT:  .LBB2_3: // %else4
212; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB2_20
213; NONEON-NOSVE-NEXT:  .LBB2_4: // %else6
214; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB2_21
215; NONEON-NOSVE-NEXT:  .LBB2_5: // %else8
216; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB2_22
217; NONEON-NOSVE-NEXT:  .LBB2_6: // %else10
218; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB2_23
219; NONEON-NOSVE-NEXT:  .LBB2_7: // %else12
220; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB2_24
221; NONEON-NOSVE-NEXT:  .LBB2_8: // %else14
222; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB2_25
223; NONEON-NOSVE-NEXT:  .LBB2_9: // %else16
224; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB2_26
225; NONEON-NOSVE-NEXT:  .LBB2_10: // %else18
226; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB2_27
227; NONEON-NOSVE-NEXT:  .LBB2_11: // %else20
228; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB2_28
229; NONEON-NOSVE-NEXT:  .LBB2_12: // %else22
230; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB2_29
231; NONEON-NOSVE-NEXT:  .LBB2_13: // %else24
232; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB2_30
233; NONEON-NOSVE-NEXT:  .LBB2_14: // %else26
234; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB2_31
235; NONEON-NOSVE-NEXT:  .LBB2_15: // %else28
236; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB2_32
237; NONEON-NOSVE-NEXT:  .LBB2_16: // %else30
238; NONEON-NOSVE-NEXT:    add sp, sp, #16
239; NONEON-NOSVE-NEXT:    ret
240; NONEON-NOSVE-NEXT:  .LBB2_17: // %cond.store
241; NONEON-NOSVE-NEXT:    strb wzr, [x0]
242; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB2_2
243; NONEON-NOSVE-NEXT:  .LBB2_18: // %cond.store1
244; NONEON-NOSVE-NEXT:    strb wzr, [x0, #1]
245; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB2_3
246; NONEON-NOSVE-NEXT:  .LBB2_19: // %cond.store3
247; NONEON-NOSVE-NEXT:    strb wzr, [x0, #2]
248; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB2_4
249; NONEON-NOSVE-NEXT:  .LBB2_20: // %cond.store5
250; NONEON-NOSVE-NEXT:    strb wzr, [x0, #3]
251; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB2_5
252; NONEON-NOSVE-NEXT:  .LBB2_21: // %cond.store7
253; NONEON-NOSVE-NEXT:    strb wzr, [x0, #4]
254; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB2_6
255; NONEON-NOSVE-NEXT:  .LBB2_22: // %cond.store9
256; NONEON-NOSVE-NEXT:    strb wzr, [x0, #5]
257; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB2_7
258; NONEON-NOSVE-NEXT:  .LBB2_23: // %cond.store11
259; NONEON-NOSVE-NEXT:    strb wzr, [x0, #6]
260; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB2_8
261; NONEON-NOSVE-NEXT:  .LBB2_24: // %cond.store13
262; NONEON-NOSVE-NEXT:    strb wzr, [x0, #7]
263; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB2_9
264; NONEON-NOSVE-NEXT:  .LBB2_25: // %cond.store15
265; NONEON-NOSVE-NEXT:    strb wzr, [x0, #8]
266; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB2_10
267; NONEON-NOSVE-NEXT:  .LBB2_26: // %cond.store17
268; NONEON-NOSVE-NEXT:    strb wzr, [x0, #9]
269; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB2_11
270; NONEON-NOSVE-NEXT:  .LBB2_27: // %cond.store19
271; NONEON-NOSVE-NEXT:    strb wzr, [x0, #10]
272; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB2_12
273; NONEON-NOSVE-NEXT:  .LBB2_28: // %cond.store21
274; NONEON-NOSVE-NEXT:    strb wzr, [x0, #11]
275; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB2_13
276; NONEON-NOSVE-NEXT:  .LBB2_29: // %cond.store23
277; NONEON-NOSVE-NEXT:    strb wzr, [x0, #12]
278; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB2_14
279; NONEON-NOSVE-NEXT:  .LBB2_30: // %cond.store25
280; NONEON-NOSVE-NEXT:    strb wzr, [x0, #13]
281; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB2_15
282; NONEON-NOSVE-NEXT:  .LBB2_31: // %cond.store27
283; NONEON-NOSVE-NEXT:    strb wzr, [x0, #14]
284; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB2_16
285; NONEON-NOSVE-NEXT:  .LBB2_32: // %cond.store29
286; NONEON-NOSVE-NEXT:    strb wzr, [x0, #15]
287; NONEON-NOSVE-NEXT:    add sp, sp, #16
288; NONEON-NOSVE-NEXT:    ret
289  call void @llvm.masked.store.v16i8(<16 x i8> zeroinitializer, ptr %dst, i32 8, <16 x i1> %mask)
290  ret void
291}
292
293define void @masked_store_v32i8(ptr %dst, <32 x i1> %mask) {
294; CHECK-LABEL: masked_store_v32i8:
295; CHECK:       // %bb.0:
296; CHECK-NEXT:    sub sp, sp, #32
297; CHECK-NEXT:    .cfi_def_cfa_offset 32
298; CHECK-NEXT:    ldr w8, [sp, #96]
299; CHECK-NEXT:    ldr w9, [sp, #88]
300; CHECK-NEXT:    ptrue p0.b, vl16
301; CHECK-NEXT:    ldr w10, [sp, #120]
302; CHECK-NEXT:    strb w7, [sp, #6]
303; CHECK-NEXT:    strb w8, [sp, #15]
304; CHECK-NEXT:    ldr w8, [sp, #80]
305; CHECK-NEXT:    strb w9, [sp, #14]
306; CHECK-NEXT:    ldr w9, [sp, #72]
307; CHECK-NEXT:    strb w8, [sp, #13]
308; CHECK-NEXT:    ldr w8, [sp, #64]
309; CHECK-NEXT:    strb w9, [sp, #12]
310; CHECK-NEXT:    ldr w9, [sp, #56]
311; CHECK-NEXT:    strb w8, [sp, #11]
312; CHECK-NEXT:    ldr w8, [sp, #48]
313; CHECK-NEXT:    strb w9, [sp, #10]
314; CHECK-NEXT:    ldr w9, [sp, #40]
315; CHECK-NEXT:    strb w8, [sp, #9]
316; CHECK-NEXT:    ldr w8, [sp, #32]
317; CHECK-NEXT:    strb w9, [sp, #8]
318; CHECK-NEXT:    ldr w9, [sp, #216]
319; CHECK-NEXT:    strb w8, [sp, #7]
320; CHECK-NEXT:    ldr w8, [sp, #224]
321; CHECK-NEXT:    strb w9, [sp, #30]
322; CHECK-NEXT:    ldr w9, [sp, #200]
323; CHECK-NEXT:    strb w8, [sp, #31]
324; CHECK-NEXT:    ldr w8, [sp, #208]
325; CHECK-NEXT:    strb w9, [sp, #28]
326; CHECK-NEXT:    ldr w9, [sp, #184]
327; CHECK-NEXT:    strb w8, [sp, #29]
328; CHECK-NEXT:    ldr w8, [sp, #192]
329; CHECK-NEXT:    strb w9, [sp, #26]
330; CHECK-NEXT:    ldr w9, [sp, #168]
331; CHECK-NEXT:    strb w8, [sp, #27]
332; CHECK-NEXT:    ldr w8, [sp, #176]
333; CHECK-NEXT:    strb w9, [sp, #24]
334; CHECK-NEXT:    ldr w9, [sp, #152]
335; CHECK-NEXT:    strb w8, [sp, #25]
336; CHECK-NEXT:    ldr w8, [sp, #160]
337; CHECK-NEXT:    strb w9, [sp, #22]
338; CHECK-NEXT:    ldr w9, [sp, #136]
339; CHECK-NEXT:    strb w8, [sp, #23]
340; CHECK-NEXT:    ldr w8, [sp, #144]
341; CHECK-NEXT:    strb w9, [sp, #20]
342; CHECK-NEXT:    ldr w9, [sp, #112]
343; CHECK-NEXT:    strb w8, [sp, #21]
344; CHECK-NEXT:    ldr w8, [sp, #128]
345; CHECK-NEXT:    strb w6, [sp, #5]
346; CHECK-NEXT:    strb w8, [sp, #19]
347; CHECK-NEXT:    ldr w8, [sp, #104]
348; CHECK-NEXT:    strb w5, [sp, #4]
349; CHECK-NEXT:    strb w4, [sp, #3]
350; CHECK-NEXT:    strb w3, [sp, #2]
351; CHECK-NEXT:    strb w2, [sp, #1]
352; CHECK-NEXT:    strb w1, [sp]
353; CHECK-NEXT:    strb w10, [sp, #18]
354; CHECK-NEXT:    strb w9, [sp, #17]
355; CHECK-NEXT:    strb w8, [sp, #16]
356; CHECK-NEXT:    mov w8, #16 // =0x10
357; CHECK-NEXT:    ldp q1, q0, [sp]
358; CHECK-NEXT:    lsl z0.b, z0.b, #7
359; CHECK-NEXT:    lsl z1.b, z1.b, #7
360; CHECK-NEXT:    asr z0.b, z0.b, #7
361; CHECK-NEXT:    asr z1.b, z1.b, #7
362; CHECK-NEXT:    cmpne p1.b, p0/z, z0.b, #0
363; CHECK-NEXT:    cmpne p0.b, p0/z, z1.b, #0
364; CHECK-NEXT:    mov z0.b, #0 // =0x0
365; CHECK-NEXT:    st1b { z0.b }, p1, [x0, x8]
366; CHECK-NEXT:    st1b { z0.b }, p0, [x0]
367; CHECK-NEXT:    add sp, sp, #32
368; CHECK-NEXT:    ret
369;
370; NONEON-NOSVE-LABEL: masked_store_v32i8:
371; NONEON-NOSVE:       // %bb.0:
372; NONEON-NOSVE-NEXT:    ldr w8, [sp, #80]
373; NONEON-NOSVE-NEXT:    ldr w9, [sp, #88]
374; NONEON-NOSVE-NEXT:    sbfx w15, w7, #0, #1
375; NONEON-NOSVE-NEXT:    ldr w10, [sp, #96]
376; NONEON-NOSVE-NEXT:    ldr w12, [sp, #104]
377; NONEON-NOSVE-NEXT:    ldr w11, [sp, #72]
378; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
379; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
380; NONEON-NOSVE-NEXT:    ldr w13, [sp, #120]
381; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
382; NONEON-NOSVE-NEXT:    sbfx w12, w12, #0, #1
383; NONEON-NOSVE-NEXT:    ldr w14, [sp, #128]
384; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
385; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
386; NONEON-NOSVE-NEXT:    ldr w16, [sp]
387; NONEON-NOSVE-NEXT:    bfxil w8, w11, #0, #1
388; NONEON-NOSVE-NEXT:    ldr w11, [sp, #112]
389; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
390; NONEON-NOSVE-NEXT:    orr w9, w9, w10
391; NONEON-NOSVE-NEXT:    and w10, w12, #0x10
392; NONEON-NOSVE-NEXT:    sbfx w12, w4, #0, #1
393; NONEON-NOSVE-NEXT:    sbfx w11, w11, #0, #1
394; NONEON-NOSVE-NEXT:    orr w9, w9, w10
395; NONEON-NOSVE-NEXT:    sbfx w10, w13, #0, #1
396; NONEON-NOSVE-NEXT:    orr w8, w8, w9
397; NONEON-NOSVE-NEXT:    sbfx w13, w5, #0, #1
398; NONEON-NOSVE-NEXT:    and w12, w12, #0x8
399; NONEON-NOSVE-NEXT:    and w9, w11, #0x20
400; NONEON-NOSVE-NEXT:    and w10, w10, #0x40
401; NONEON-NOSVE-NEXT:    sbfx w11, w3, #0, #1
402; NONEON-NOSVE-NEXT:    orr w9, w9, w10
403; NONEON-NOSVE-NEXT:    sbfx w10, w2, #0, #1
404; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
405; NONEON-NOSVE-NEXT:    and w11, w11, #0x4
406; NONEON-NOSVE-NEXT:    orr w8, w8, w9
407; NONEON-NOSVE-NEXT:    and w10, w10, #0x2
408; NONEON-NOSVE-NEXT:    orr w11, w11, w12
409; NONEON-NOSVE-NEXT:    and w12, w13, #0x10
410; NONEON-NOSVE-NEXT:    sbfx w13, w6, #0, #1
411; NONEON-NOSVE-NEXT:    bfxil w10, w1, #0, #1
412; NONEON-NOSVE-NEXT:    orr w11, w11, w12
413; NONEON-NOSVE-NEXT:    and w12, w13, #0x20
414; NONEON-NOSVE-NEXT:    and w13, w15, #0x40
415; NONEON-NOSVE-NEXT:    sbfx w15, w16, #0, #1
416; NONEON-NOSVE-NEXT:    orr w9, w10, w11
417; NONEON-NOSVE-NEXT:    orr w10, w12, w13
418; NONEON-NOSVE-NEXT:    and w11, w14, #0xff80
419; NONEON-NOSVE-NEXT:    orr w9, w9, w10
420; NONEON-NOSVE-NEXT:    and w10, w15, #0xff80
421; NONEON-NOSVE-NEXT:    add w11, w8, w11
422; NONEON-NOSVE-NEXT:    add w8, w9, w10
423; NONEON-NOSVE-NEXT:    bfi w8, w11, #16, #16
424; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB3_33
425; NONEON-NOSVE-NEXT:  // %bb.1: // %else
426; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB3_34
427; NONEON-NOSVE-NEXT:  .LBB3_2: // %else2
428; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB3_35
429; NONEON-NOSVE-NEXT:  .LBB3_3: // %else4
430; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB3_36
431; NONEON-NOSVE-NEXT:  .LBB3_4: // %else6
432; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB3_37
433; NONEON-NOSVE-NEXT:  .LBB3_5: // %else8
434; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB3_38
435; NONEON-NOSVE-NEXT:  .LBB3_6: // %else10
436; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB3_39
437; NONEON-NOSVE-NEXT:  .LBB3_7: // %else12
438; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB3_40
439; NONEON-NOSVE-NEXT:  .LBB3_8: // %else14
440; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB3_41
441; NONEON-NOSVE-NEXT:  .LBB3_9: // %else16
442; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB3_42
443; NONEON-NOSVE-NEXT:  .LBB3_10: // %else18
444; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB3_43
445; NONEON-NOSVE-NEXT:  .LBB3_11: // %else20
446; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB3_44
447; NONEON-NOSVE-NEXT:  .LBB3_12: // %else22
448; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB3_45
449; NONEON-NOSVE-NEXT:  .LBB3_13: // %else24
450; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB3_46
451; NONEON-NOSVE-NEXT:  .LBB3_14: // %else26
452; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB3_47
453; NONEON-NOSVE-NEXT:  .LBB3_15: // %else28
454; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB3_48
455; NONEON-NOSVE-NEXT:  .LBB3_16: // %else30
456; NONEON-NOSVE-NEXT:    tbnz w8, #16, .LBB3_49
457; NONEON-NOSVE-NEXT:  .LBB3_17: // %else32
458; NONEON-NOSVE-NEXT:    tbnz w8, #17, .LBB3_50
459; NONEON-NOSVE-NEXT:  .LBB3_18: // %else34
460; NONEON-NOSVE-NEXT:    tbnz w8, #18, .LBB3_51
461; NONEON-NOSVE-NEXT:  .LBB3_19: // %else36
462; NONEON-NOSVE-NEXT:    tbnz w8, #19, .LBB3_52
463; NONEON-NOSVE-NEXT:  .LBB3_20: // %else38
464; NONEON-NOSVE-NEXT:    tbnz w8, #20, .LBB3_53
465; NONEON-NOSVE-NEXT:  .LBB3_21: // %else40
466; NONEON-NOSVE-NEXT:    tbnz w8, #21, .LBB3_54
467; NONEON-NOSVE-NEXT:  .LBB3_22: // %else42
468; NONEON-NOSVE-NEXT:    tbnz w8, #22, .LBB3_55
469; NONEON-NOSVE-NEXT:  .LBB3_23: // %else44
470; NONEON-NOSVE-NEXT:    tbnz w8, #23, .LBB3_56
471; NONEON-NOSVE-NEXT:  .LBB3_24: // %else46
472; NONEON-NOSVE-NEXT:    tbnz w8, #24, .LBB3_57
473; NONEON-NOSVE-NEXT:  .LBB3_25: // %else48
474; NONEON-NOSVE-NEXT:    tbnz w8, #25, .LBB3_58
475; NONEON-NOSVE-NEXT:  .LBB3_26: // %else50
476; NONEON-NOSVE-NEXT:    tbnz w8, #26, .LBB3_59
477; NONEON-NOSVE-NEXT:  .LBB3_27: // %else52
478; NONEON-NOSVE-NEXT:    tbnz w8, #27, .LBB3_60
479; NONEON-NOSVE-NEXT:  .LBB3_28: // %else54
480; NONEON-NOSVE-NEXT:    tbnz w8, #28, .LBB3_61
481; NONEON-NOSVE-NEXT:  .LBB3_29: // %else56
482; NONEON-NOSVE-NEXT:    tbnz w8, #29, .LBB3_62
483; NONEON-NOSVE-NEXT:  .LBB3_30: // %else58
484; NONEON-NOSVE-NEXT:    tbnz w8, #30, .LBB3_63
485; NONEON-NOSVE-NEXT:  .LBB3_31: // %else60
486; NONEON-NOSVE-NEXT:    tbnz w8, #31, .LBB3_64
487; NONEON-NOSVE-NEXT:  .LBB3_32: // %else62
488; NONEON-NOSVE-NEXT:    ret
489; NONEON-NOSVE-NEXT:  .LBB3_33: // %cond.store
490; NONEON-NOSVE-NEXT:    strb wzr, [x0]
491; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB3_2
492; NONEON-NOSVE-NEXT:  .LBB3_34: // %cond.store1
493; NONEON-NOSVE-NEXT:    strb wzr, [x0, #1]
494; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB3_3
495; NONEON-NOSVE-NEXT:  .LBB3_35: // %cond.store3
496; NONEON-NOSVE-NEXT:    strb wzr, [x0, #2]
497; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB3_4
498; NONEON-NOSVE-NEXT:  .LBB3_36: // %cond.store5
499; NONEON-NOSVE-NEXT:    strb wzr, [x0, #3]
500; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB3_5
501; NONEON-NOSVE-NEXT:  .LBB3_37: // %cond.store7
502; NONEON-NOSVE-NEXT:    strb wzr, [x0, #4]
503; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB3_6
504; NONEON-NOSVE-NEXT:  .LBB3_38: // %cond.store9
505; NONEON-NOSVE-NEXT:    strb wzr, [x0, #5]
506; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB3_7
507; NONEON-NOSVE-NEXT:  .LBB3_39: // %cond.store11
508; NONEON-NOSVE-NEXT:    strb wzr, [x0, #6]
509; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB3_8
510; NONEON-NOSVE-NEXT:  .LBB3_40: // %cond.store13
511; NONEON-NOSVE-NEXT:    strb wzr, [x0, #7]
512; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB3_9
513; NONEON-NOSVE-NEXT:  .LBB3_41: // %cond.store15
514; NONEON-NOSVE-NEXT:    strb wzr, [x0, #8]
515; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB3_10
516; NONEON-NOSVE-NEXT:  .LBB3_42: // %cond.store17
517; NONEON-NOSVE-NEXT:    strb wzr, [x0, #9]
518; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB3_11
519; NONEON-NOSVE-NEXT:  .LBB3_43: // %cond.store19
520; NONEON-NOSVE-NEXT:    strb wzr, [x0, #10]
521; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB3_12
522; NONEON-NOSVE-NEXT:  .LBB3_44: // %cond.store21
523; NONEON-NOSVE-NEXT:    strb wzr, [x0, #11]
524; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB3_13
525; NONEON-NOSVE-NEXT:  .LBB3_45: // %cond.store23
526; NONEON-NOSVE-NEXT:    strb wzr, [x0, #12]
527; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB3_14
528; NONEON-NOSVE-NEXT:  .LBB3_46: // %cond.store25
529; NONEON-NOSVE-NEXT:    strb wzr, [x0, #13]
530; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB3_15
531; NONEON-NOSVE-NEXT:  .LBB3_47: // %cond.store27
532; NONEON-NOSVE-NEXT:    strb wzr, [x0, #14]
533; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB3_16
534; NONEON-NOSVE-NEXT:  .LBB3_48: // %cond.store29
535; NONEON-NOSVE-NEXT:    strb wzr, [x0, #15]
536; NONEON-NOSVE-NEXT:    tbz w8, #16, .LBB3_17
537; NONEON-NOSVE-NEXT:  .LBB3_49: // %cond.store31
538; NONEON-NOSVE-NEXT:    strb wzr, [x0, #16]
539; NONEON-NOSVE-NEXT:    tbz w8, #17, .LBB3_18
540; NONEON-NOSVE-NEXT:  .LBB3_50: // %cond.store33
541; NONEON-NOSVE-NEXT:    strb wzr, [x0, #17]
542; NONEON-NOSVE-NEXT:    tbz w8, #18, .LBB3_19
543; NONEON-NOSVE-NEXT:  .LBB3_51: // %cond.store35
544; NONEON-NOSVE-NEXT:    strb wzr, [x0, #18]
545; NONEON-NOSVE-NEXT:    tbz w8, #19, .LBB3_20
546; NONEON-NOSVE-NEXT:  .LBB3_52: // %cond.store37
547; NONEON-NOSVE-NEXT:    strb wzr, [x0, #19]
548; NONEON-NOSVE-NEXT:    tbz w8, #20, .LBB3_21
549; NONEON-NOSVE-NEXT:  .LBB3_53: // %cond.store39
550; NONEON-NOSVE-NEXT:    strb wzr, [x0, #20]
551; NONEON-NOSVE-NEXT:    tbz w8, #21, .LBB3_22
552; NONEON-NOSVE-NEXT:  .LBB3_54: // %cond.store41
553; NONEON-NOSVE-NEXT:    strb wzr, [x0, #21]
554; NONEON-NOSVE-NEXT:    tbz w8, #22, .LBB3_23
555; NONEON-NOSVE-NEXT:  .LBB3_55: // %cond.store43
556; NONEON-NOSVE-NEXT:    strb wzr, [x0, #22]
557; NONEON-NOSVE-NEXT:    tbz w8, #23, .LBB3_24
558; NONEON-NOSVE-NEXT:  .LBB3_56: // %cond.store45
559; NONEON-NOSVE-NEXT:    strb wzr, [x0, #23]
560; NONEON-NOSVE-NEXT:    tbz w8, #24, .LBB3_25
561; NONEON-NOSVE-NEXT:  .LBB3_57: // %cond.store47
562; NONEON-NOSVE-NEXT:    strb wzr, [x0, #24]
563; NONEON-NOSVE-NEXT:    tbz w8, #25, .LBB3_26
564; NONEON-NOSVE-NEXT:  .LBB3_58: // %cond.store49
565; NONEON-NOSVE-NEXT:    strb wzr, [x0, #25]
566; NONEON-NOSVE-NEXT:    tbz w8, #26, .LBB3_27
567; NONEON-NOSVE-NEXT:  .LBB3_59: // %cond.store51
568; NONEON-NOSVE-NEXT:    strb wzr, [x0, #26]
569; NONEON-NOSVE-NEXT:    tbz w8, #27, .LBB3_28
570; NONEON-NOSVE-NEXT:  .LBB3_60: // %cond.store53
571; NONEON-NOSVE-NEXT:    strb wzr, [x0, #27]
572; NONEON-NOSVE-NEXT:    tbz w8, #28, .LBB3_29
573; NONEON-NOSVE-NEXT:  .LBB3_61: // %cond.store55
574; NONEON-NOSVE-NEXT:    strb wzr, [x0, #28]
575; NONEON-NOSVE-NEXT:    tbz w8, #29, .LBB3_30
576; NONEON-NOSVE-NEXT:  .LBB3_62: // %cond.store57
577; NONEON-NOSVE-NEXT:    strb wzr, [x0, #29]
578; NONEON-NOSVE-NEXT:    tbz w8, #30, .LBB3_31
579; NONEON-NOSVE-NEXT:  .LBB3_63: // %cond.store59
580; NONEON-NOSVE-NEXT:    strb wzr, [x0, #30]
581; NONEON-NOSVE-NEXT:    tbz w8, #31, .LBB3_32
582; NONEON-NOSVE-NEXT:  .LBB3_64: // %cond.store61
583; NONEON-NOSVE-NEXT:    strb wzr, [x0, #31]
584; NONEON-NOSVE-NEXT:    ret
585  call void @llvm.masked.store.v32i8(<32 x i8> zeroinitializer, ptr %dst, i32 8, <32 x i1> %mask)
586  ret void
587}
588
589define void @masked_store_v2f16(ptr %dst, <2 x i1> %mask) {
590; CHECK-LABEL: masked_store_v2f16:
591; CHECK:       // %bb.0:
592; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
593; CHECK-NEXT:    fmov s1, wzr
594; CHECK-NEXT:    mov z2.s, z0.s[1]
595; CHECK-NEXT:    ptrue p0.h, vl4
596; CHECK-NEXT:    zip1 z0.h, z0.h, z2.h
597; CHECK-NEXT:    zip1 z1.h, z1.h, z1.h
598; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
599; CHECK-NEXT:    lsl z0.h, z0.h, #15
600; CHECK-NEXT:    asr z0.h, z0.h, #15
601; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
602; CHECK-NEXT:    mov z0.h, #0 // =0x0
603; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
604; CHECK-NEXT:    ret
605;
606; NONEON-NOSVE-LABEL: masked_store_v2f16:
607; NONEON-NOSVE:       // %bb.0:
608; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
609; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
610; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
611; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
612; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
613; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
614; NONEON-NOSVE-NEXT:    bfxil w8, w9, #0, #1
615; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB4_3
616; NONEON-NOSVE-NEXT:  // %bb.1: // %else
617; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB4_4
618; NONEON-NOSVE-NEXT:  .LBB4_2: // %else2
619; NONEON-NOSVE-NEXT:    add sp, sp, #16
620; NONEON-NOSVE-NEXT:    ret
621; NONEON-NOSVE-NEXT:  .LBB4_3: // %cond.store
622; NONEON-NOSVE-NEXT:    fmov s0, wzr
623; NONEON-NOSVE-NEXT:    str h0, [x0]
624; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB4_2
625; NONEON-NOSVE-NEXT:  .LBB4_4: // %cond.store1
626; NONEON-NOSVE-NEXT:    fmov s0, wzr
627; NONEON-NOSVE-NEXT:    str h0, [x0, #2]
628; NONEON-NOSVE-NEXT:    add sp, sp, #16
629; NONEON-NOSVE-NEXT:    ret
630  call void @llvm.masked.store.v2f16(<2 x half> zeroinitializer, ptr %dst, i32 8, <2 x i1> %mask)
631  ret void
632}
633
634define void @masked_store_v4f16(ptr %dst, <4 x i1> %mask) {
635; CHECK-LABEL: masked_store_v4f16:
636; CHECK:       // %bb.0:
637; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
638; CHECK-NEXT:    ptrue p0.h, vl4
639; CHECK-NEXT:    lsl z0.h, z0.h, #15
640; CHECK-NEXT:    asr z0.h, z0.h, #15
641; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
642; CHECK-NEXT:    mov z0.h, #0 // =0x0
643; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
644; CHECK-NEXT:    ret
645;
646; NONEON-NOSVE-LABEL: masked_store_v4f16:
647; NONEON-NOSVE:       // %bb.0:
648; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
649; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
650; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
651; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
652; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #6]
653; NONEON-NOSVE-NEXT:    ldrh w11, [sp]
654; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
655; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
656; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
657; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
658; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
659; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
660; NONEON-NOSVE-NEXT:    bfxil w8, w11, #0, #1
661; NONEON-NOSVE-NEXT:    orr w9, w9, w10
662; NONEON-NOSVE-NEXT:    orr w8, w8, w9
663; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB5_5
664; NONEON-NOSVE-NEXT:  // %bb.1: // %else
665; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB5_6
666; NONEON-NOSVE-NEXT:  .LBB5_2: // %else2
667; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB5_7
668; NONEON-NOSVE-NEXT:  .LBB5_3: // %else4
669; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB5_8
670; NONEON-NOSVE-NEXT:  .LBB5_4: // %else6
671; NONEON-NOSVE-NEXT:    add sp, sp, #16
672; NONEON-NOSVE-NEXT:    ret
673; NONEON-NOSVE-NEXT:  .LBB5_5: // %cond.store
674; NONEON-NOSVE-NEXT:    fmov s0, wzr
675; NONEON-NOSVE-NEXT:    str h0, [x0]
676; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB5_2
677; NONEON-NOSVE-NEXT:  .LBB5_6: // %cond.store1
678; NONEON-NOSVE-NEXT:    fmov s0, wzr
679; NONEON-NOSVE-NEXT:    str h0, [x0, #2]
680; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB5_3
681; NONEON-NOSVE-NEXT:  .LBB5_7: // %cond.store3
682; NONEON-NOSVE-NEXT:    fmov s0, wzr
683; NONEON-NOSVE-NEXT:    str h0, [x0, #4]
684; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB5_4
685; NONEON-NOSVE-NEXT:  .LBB5_8: // %cond.store5
686; NONEON-NOSVE-NEXT:    fmov s0, wzr
687; NONEON-NOSVE-NEXT:    str h0, [x0, #6]
688; NONEON-NOSVE-NEXT:    add sp, sp, #16
689; NONEON-NOSVE-NEXT:    ret
690  call void @llvm.masked.store.v4f16(<4 x half> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
691  ret void
692}
693
694define void @masked_store_v8f16(ptr %dst, <8 x i1> %mask) {
695; CHECK-LABEL: masked_store_v8f16:
696; CHECK:       // %bb.0:
697; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
698; CHECK-NEXT:    ptrue p0.h, vl8
699; CHECK-NEXT:    uunpklo z0.h, z0.b
700; CHECK-NEXT:    lsl z0.h, z0.h, #15
701; CHECK-NEXT:    asr z0.h, z0.h, #15
702; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
703; CHECK-NEXT:    mov z0.h, #0 // =0x0
704; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
705; CHECK-NEXT:    ret
706;
707; NONEON-NOSVE-LABEL: masked_store_v8f16:
708; NONEON-NOSVE:       // %bb.0:
709; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
710; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
711; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
712; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
713; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1]
714; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #4]
715; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #5]
716; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #6]
717; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
718; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
719; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
720; NONEON-NOSVE-NEXT:    ldrb w11, [sp]
721; NONEON-NOSVE-NEXT:    sbfx w12, w12, #0, #1
722; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
723; NONEON-NOSVE-NEXT:    and w8, w8, #0x4
724; NONEON-NOSVE-NEXT:    and w9, w9, #0x8
725; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
726; NONEON-NOSVE-NEXT:    orr w8, w8, w9
727; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
728; NONEON-NOSVE-NEXT:    and w10, w10, #0x2
729; NONEON-NOSVE-NEXT:    and w12, w12, #0x10
730; NONEON-NOSVE-NEXT:    bfxil w10, w11, #0, #1
731; NONEON-NOSVE-NEXT:    and w11, w13, #0x20
732; NONEON-NOSVE-NEXT:    orr w8, w8, w12
733; NONEON-NOSVE-NEXT:    and w12, w14, #0x40
734; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
735; NONEON-NOSVE-NEXT:    orr w8, w10, w8
736; NONEON-NOSVE-NEXT:    orr w10, w11, w12
737; NONEON-NOSVE-NEXT:    orr w8, w8, w10
738; NONEON-NOSVE-NEXT:    and w9, w9, #0x80
739; NONEON-NOSVE-NEXT:    add w9, w8, w9
740; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
741; NONEON-NOSVE-NEXT:    tbnz w9, #0, .LBB6_9
742; NONEON-NOSVE-NEXT:  // %bb.1: // %else
743; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB6_10
744; NONEON-NOSVE-NEXT:  .LBB6_2: // %else2
745; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB6_11
746; NONEON-NOSVE-NEXT:  .LBB6_3: // %else4
747; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB6_12
748; NONEON-NOSVE-NEXT:  .LBB6_4: // %else6
749; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB6_13
750; NONEON-NOSVE-NEXT:  .LBB6_5: // %else8
751; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB6_14
752; NONEON-NOSVE-NEXT:  .LBB6_6: // %else10
753; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB6_15
754; NONEON-NOSVE-NEXT:  .LBB6_7: // %else12
755; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB6_16
756; NONEON-NOSVE-NEXT:  .LBB6_8: // %else14
757; NONEON-NOSVE-NEXT:    add sp, sp, #16
758; NONEON-NOSVE-NEXT:    ret
759; NONEON-NOSVE-NEXT:  .LBB6_9: // %cond.store
760; NONEON-NOSVE-NEXT:    fmov s0, wzr
761; NONEON-NOSVE-NEXT:    str h0, [x0]
762; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB6_2
763; NONEON-NOSVE-NEXT:  .LBB6_10: // %cond.store1
764; NONEON-NOSVE-NEXT:    fmov s0, wzr
765; NONEON-NOSVE-NEXT:    str h0, [x0, #2]
766; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB6_3
767; NONEON-NOSVE-NEXT:  .LBB6_11: // %cond.store3
768; NONEON-NOSVE-NEXT:    fmov s0, wzr
769; NONEON-NOSVE-NEXT:    str h0, [x0, #4]
770; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB6_4
771; NONEON-NOSVE-NEXT:  .LBB6_12: // %cond.store5
772; NONEON-NOSVE-NEXT:    fmov s0, wzr
773; NONEON-NOSVE-NEXT:    str h0, [x0, #6]
774; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB6_5
775; NONEON-NOSVE-NEXT:  .LBB6_13: // %cond.store7
776; NONEON-NOSVE-NEXT:    fmov s0, wzr
777; NONEON-NOSVE-NEXT:    str h0, [x0, #8]
778; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB6_6
779; NONEON-NOSVE-NEXT:  .LBB6_14: // %cond.store9
780; NONEON-NOSVE-NEXT:    fmov s0, wzr
781; NONEON-NOSVE-NEXT:    str h0, [x0, #10]
782; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB6_7
783; NONEON-NOSVE-NEXT:  .LBB6_15: // %cond.store11
784; NONEON-NOSVE-NEXT:    fmov s0, wzr
785; NONEON-NOSVE-NEXT:    str h0, [x0, #12]
786; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB6_8
787; NONEON-NOSVE-NEXT:  .LBB6_16: // %cond.store13
788; NONEON-NOSVE-NEXT:    fmov s0, wzr
789; NONEON-NOSVE-NEXT:    str h0, [x0, #14]
790; NONEON-NOSVE-NEXT:    add sp, sp, #16
791; NONEON-NOSVE-NEXT:    ret
792  call void @llvm.masked.store.v8f16(<8 x half> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
793  ret void
794}
795
796define void @masked_store_v16f16(ptr %dst, <16 x i1> %mask) {
797; CHECK-LABEL: masked_store_v16f16:
798; CHECK:       // %bb.0:
799; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
800; CHECK-NEXT:    uunpklo z1.h, z0.b
801; CHECK-NEXT:    ptrue p0.h, vl8
802; CHECK-NEXT:    mov x8, #8 // =0x8
803; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
804; CHECK-NEXT:    uunpklo z0.h, z0.b
805; CHECK-NEXT:    lsl z1.h, z1.h, #15
806; CHECK-NEXT:    asr z1.h, z1.h, #15
807; CHECK-NEXT:    lsl z0.h, z0.h, #15
808; CHECK-NEXT:    asr z0.h, z0.h, #15
809; CHECK-NEXT:    cmpne p1.h, p0/z, z0.h, #0
810; CHECK-NEXT:    cmpne p0.h, p0/z, z1.h, #0
811; CHECK-NEXT:    mov z0.h, #0 // =0x0
812; CHECK-NEXT:    st1h { z0.h }, p1, [x0, x8, lsl #1]
813; CHECK-NEXT:    st1h { z0.h }, p0, [x0]
814; CHECK-NEXT:    ret
815;
816; NONEON-NOSVE-LABEL: masked_store_v16f16:
817; NONEON-NOSVE:       // %bb.0:
818; NONEON-NOSVE-NEXT:    str q0, [sp, #-16]!
819; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
820; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
821; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
822; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1]
823; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #4]
824; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #5]
825; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #6]
826; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
827; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
828; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
829; NONEON-NOSVE-NEXT:    ldrb w11, [sp]
830; NONEON-NOSVE-NEXT:    sbfx w12, w12, #0, #1
831; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
832; NONEON-NOSVE-NEXT:    and w8, w8, #0x4
833; NONEON-NOSVE-NEXT:    and w9, w9, #0x8
834; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
835; NONEON-NOSVE-NEXT:    orr w8, w8, w9
836; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
837; NONEON-NOSVE-NEXT:    and w10, w10, #0x2
838; NONEON-NOSVE-NEXT:    and w12, w12, #0x10
839; NONEON-NOSVE-NEXT:    bfxil w10, w11, #0, #1
840; NONEON-NOSVE-NEXT:    and w11, w13, #0x20
841; NONEON-NOSVE-NEXT:    orr w8, w8, w12
842; NONEON-NOSVE-NEXT:    and w12, w14, #0x40
843; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
844; NONEON-NOSVE-NEXT:    orr w8, w10, w8
845; NONEON-NOSVE-NEXT:    orr w10, w11, w12
846; NONEON-NOSVE-NEXT:    orr w8, w8, w10
847; NONEON-NOSVE-NEXT:    and w9, w9, #0xffffff80
848; NONEON-NOSVE-NEXT:    add w8, w8, w9
849; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB7_17
850; NONEON-NOSVE-NEXT:  // %bb.1: // %else
851; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB7_18
852; NONEON-NOSVE-NEXT:  .LBB7_2: // %else2
853; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB7_19
854; NONEON-NOSVE-NEXT:  .LBB7_3: // %else4
855; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB7_20
856; NONEON-NOSVE-NEXT:  .LBB7_4: // %else6
857; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB7_21
858; NONEON-NOSVE-NEXT:  .LBB7_5: // %else8
859; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB7_22
860; NONEON-NOSVE-NEXT:  .LBB7_6: // %else10
861; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB7_23
862; NONEON-NOSVE-NEXT:  .LBB7_7: // %else12
863; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB7_24
864; NONEON-NOSVE-NEXT:  .LBB7_8: // %else14
865; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB7_25
866; NONEON-NOSVE-NEXT:  .LBB7_9: // %else16
867; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB7_26
868; NONEON-NOSVE-NEXT:  .LBB7_10: // %else18
869; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB7_27
870; NONEON-NOSVE-NEXT:  .LBB7_11: // %else20
871; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB7_28
872; NONEON-NOSVE-NEXT:  .LBB7_12: // %else22
873; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB7_29
874; NONEON-NOSVE-NEXT:  .LBB7_13: // %else24
875; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB7_30
876; NONEON-NOSVE-NEXT:  .LBB7_14: // %else26
877; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB7_31
878; NONEON-NOSVE-NEXT:  .LBB7_15: // %else28
879; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB7_32
880; NONEON-NOSVE-NEXT:  .LBB7_16: // %else30
881; NONEON-NOSVE-NEXT:    add sp, sp, #16
882; NONEON-NOSVE-NEXT:    ret
883; NONEON-NOSVE-NEXT:  .LBB7_17: // %cond.store
884; NONEON-NOSVE-NEXT:    fmov s0, wzr
885; NONEON-NOSVE-NEXT:    str h0, [x0]
886; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB7_2
887; NONEON-NOSVE-NEXT:  .LBB7_18: // %cond.store1
888; NONEON-NOSVE-NEXT:    fmov s0, wzr
889; NONEON-NOSVE-NEXT:    str h0, [x0, #2]
890; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB7_3
891; NONEON-NOSVE-NEXT:  .LBB7_19: // %cond.store3
892; NONEON-NOSVE-NEXT:    fmov s0, wzr
893; NONEON-NOSVE-NEXT:    str h0, [x0, #4]
894; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB7_4
895; NONEON-NOSVE-NEXT:  .LBB7_20: // %cond.store5
896; NONEON-NOSVE-NEXT:    fmov s0, wzr
897; NONEON-NOSVE-NEXT:    str h0, [x0, #6]
898; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB7_5
899; NONEON-NOSVE-NEXT:  .LBB7_21: // %cond.store7
900; NONEON-NOSVE-NEXT:    fmov s0, wzr
901; NONEON-NOSVE-NEXT:    str h0, [x0, #8]
902; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB7_6
903; NONEON-NOSVE-NEXT:  .LBB7_22: // %cond.store9
904; NONEON-NOSVE-NEXT:    fmov s0, wzr
905; NONEON-NOSVE-NEXT:    str h0, [x0, #10]
906; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB7_7
907; NONEON-NOSVE-NEXT:  .LBB7_23: // %cond.store11
908; NONEON-NOSVE-NEXT:    fmov s0, wzr
909; NONEON-NOSVE-NEXT:    str h0, [x0, #12]
910; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB7_8
911; NONEON-NOSVE-NEXT:  .LBB7_24: // %cond.store13
912; NONEON-NOSVE-NEXT:    fmov s0, wzr
913; NONEON-NOSVE-NEXT:    str h0, [x0, #14]
914; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB7_9
915; NONEON-NOSVE-NEXT:  .LBB7_25: // %cond.store15
916; NONEON-NOSVE-NEXT:    fmov s0, wzr
917; NONEON-NOSVE-NEXT:    str h0, [x0, #16]
918; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB7_10
919; NONEON-NOSVE-NEXT:  .LBB7_26: // %cond.store17
920; NONEON-NOSVE-NEXT:    fmov s0, wzr
921; NONEON-NOSVE-NEXT:    str h0, [x0, #18]
922; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB7_11
923; NONEON-NOSVE-NEXT:  .LBB7_27: // %cond.store19
924; NONEON-NOSVE-NEXT:    fmov s0, wzr
925; NONEON-NOSVE-NEXT:    str h0, [x0, #20]
926; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB7_12
927; NONEON-NOSVE-NEXT:  .LBB7_28: // %cond.store21
928; NONEON-NOSVE-NEXT:    fmov s0, wzr
929; NONEON-NOSVE-NEXT:    str h0, [x0, #22]
930; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB7_13
931; NONEON-NOSVE-NEXT:  .LBB7_29: // %cond.store23
932; NONEON-NOSVE-NEXT:    fmov s0, wzr
933; NONEON-NOSVE-NEXT:    str h0, [x0, #24]
934; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB7_14
935; NONEON-NOSVE-NEXT:  .LBB7_30: // %cond.store25
936; NONEON-NOSVE-NEXT:    fmov s0, wzr
937; NONEON-NOSVE-NEXT:    str h0, [x0, #26]
938; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB7_15
939; NONEON-NOSVE-NEXT:  .LBB7_31: // %cond.store27
940; NONEON-NOSVE-NEXT:    fmov s0, wzr
941; NONEON-NOSVE-NEXT:    str h0, [x0, #28]
942; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB7_16
943; NONEON-NOSVE-NEXT:  .LBB7_32: // %cond.store29
944; NONEON-NOSVE-NEXT:    fmov s0, wzr
945; NONEON-NOSVE-NEXT:    str h0, [x0, #30]
946; NONEON-NOSVE-NEXT:    add sp, sp, #16
947; NONEON-NOSVE-NEXT:    ret
948  call void @llvm.masked.store.v16f16(<16 x half> zeroinitializer, ptr %dst, i32 8, <16 x i1> %mask)
949  ret void
950}
951
952define void @masked_store_v4f32(ptr %dst, <4 x i1> %mask) {
953; CHECK-LABEL: masked_store_v4f32:
954; CHECK:       // %bb.0:
955; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
956; CHECK-NEXT:    ptrue p0.s, vl4
957; CHECK-NEXT:    uunpklo z0.s, z0.h
958; CHECK-NEXT:    lsl z0.s, z0.s, #31
959; CHECK-NEXT:    asr z0.s, z0.s, #31
960; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #0
961; CHECK-NEXT:    mov z0.s, #0 // =0x0
962; CHECK-NEXT:    st1w { z0.s }, p0, [x0]
963; CHECK-NEXT:    ret
964;
965; NONEON-NOSVE-LABEL: masked_store_v4f32:
966; NONEON-NOSVE:       // %bb.0:
967; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
968; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
969; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
970; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
971; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #6]
972; NONEON-NOSVE-NEXT:    ldrh w11, [sp]
973; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
974; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
975; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
976; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
977; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
978; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
979; NONEON-NOSVE-NEXT:    bfxil w8, w11, #0, #1
980; NONEON-NOSVE-NEXT:    orr w9, w9, w10
981; NONEON-NOSVE-NEXT:    orr w8, w8, w9
982; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB8_5
983; NONEON-NOSVE-NEXT:  // %bb.1: // %else
984; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB8_6
985; NONEON-NOSVE-NEXT:  .LBB8_2: // %else2
986; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB8_7
987; NONEON-NOSVE-NEXT:  .LBB8_3: // %else4
988; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB8_8
989; NONEON-NOSVE-NEXT:  .LBB8_4: // %else6
990; NONEON-NOSVE-NEXT:    add sp, sp, #16
991; NONEON-NOSVE-NEXT:    ret
992; NONEON-NOSVE-NEXT:  .LBB8_5: // %cond.store
993; NONEON-NOSVE-NEXT:    str wzr, [x0]
994; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB8_2
995; NONEON-NOSVE-NEXT:  .LBB8_6: // %cond.store1
996; NONEON-NOSVE-NEXT:    str wzr, [x0, #4]
997; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB8_3
998; NONEON-NOSVE-NEXT:  .LBB8_7: // %cond.store3
999; NONEON-NOSVE-NEXT:    str wzr, [x0, #8]
1000; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB8_4
1001; NONEON-NOSVE-NEXT:  .LBB8_8: // %cond.store5
1002; NONEON-NOSVE-NEXT:    str wzr, [x0, #12]
1003; NONEON-NOSVE-NEXT:    add sp, sp, #16
1004; NONEON-NOSVE-NEXT:    ret
1005  call void @llvm.masked.store.v4f32(<4 x float> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
1006  ret void
1007}
1008
1009define void @masked_store_v8f32(ptr %dst, <8 x i1> %mask) {
1010; CHECK-LABEL: masked_store_v8f32:
1011; CHECK:       // %bb.0:
1012; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1013; CHECK-NEXT:    mov z1.b, z0.b[7]
1014; CHECK-NEXT:    mov z2.b, z0.b[6]
1015; CHECK-NEXT:    mov x8, #4 // =0x4
1016; CHECK-NEXT:    mov z3.b, z0.b[5]
1017; CHECK-NEXT:    mov z4.b, z0.b[4]
1018; CHECK-NEXT:    mov z5.b, z0.b[3]
1019; CHECK-NEXT:    mov z6.b, z0.b[2]
1020; CHECK-NEXT:    mov z7.b, z0.b[1]
1021; CHECK-NEXT:    ptrue p0.s, vl4
1022; CHECK-NEXT:    zip1 z1.h, z2.h, z1.h
1023; CHECK-NEXT:    zip1 z2.h, z4.h, z3.h
1024; CHECK-NEXT:    zip1 z3.h, z6.h, z5.h
1025; CHECK-NEXT:    zip1 z0.h, z0.h, z7.h
1026; CHECK-NEXT:    zip1 z1.s, z2.s, z1.s
1027; CHECK-NEXT:    zip1 z0.s, z0.s, z3.s
1028; CHECK-NEXT:    uunpklo z1.s, z1.h
1029; CHECK-NEXT:    uunpklo z0.s, z0.h
1030; CHECK-NEXT:    lsl z1.s, z1.s, #31
1031; CHECK-NEXT:    lsl z0.s, z0.s, #31
1032; CHECK-NEXT:    asr z1.s, z1.s, #31
1033; CHECK-NEXT:    asr z0.s, z0.s, #31
1034; CHECK-NEXT:    cmpne p1.s, p0/z, z1.s, #0
1035; CHECK-NEXT:    mov z1.s, #0 // =0x0
1036; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #0
1037; CHECK-NEXT:    st1w { z1.s }, p1, [x0, x8, lsl #2]
1038; CHECK-NEXT:    st1w { z1.s }, p0, [x0]
1039; CHECK-NEXT:    ret
1040;
1041; NONEON-NOSVE-LABEL: masked_store_v8f32:
1042; NONEON-NOSVE:       // %bb.0:
1043; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
1044; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
1045; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #2]
1046; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #3]
1047; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1]
1048; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #4]
1049; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #5]
1050; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #6]
1051; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
1052; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
1053; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
1054; NONEON-NOSVE-NEXT:    ldrb w11, [sp]
1055; NONEON-NOSVE-NEXT:    sbfx w12, w12, #0, #1
1056; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
1057; NONEON-NOSVE-NEXT:    and w8, w8, #0x4
1058; NONEON-NOSVE-NEXT:    and w9, w9, #0x8
1059; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
1060; NONEON-NOSVE-NEXT:    orr w8, w8, w9
1061; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #7]
1062; NONEON-NOSVE-NEXT:    and w10, w10, #0x2
1063; NONEON-NOSVE-NEXT:    and w12, w12, #0x10
1064; NONEON-NOSVE-NEXT:    bfxil w10, w11, #0, #1
1065; NONEON-NOSVE-NEXT:    and w11, w13, #0x20
1066; NONEON-NOSVE-NEXT:    orr w8, w8, w12
1067; NONEON-NOSVE-NEXT:    and w12, w14, #0x40
1068; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
1069; NONEON-NOSVE-NEXT:    orr w8, w10, w8
1070; NONEON-NOSVE-NEXT:    orr w10, w11, w12
1071; NONEON-NOSVE-NEXT:    orr w8, w8, w10
1072; NONEON-NOSVE-NEXT:    and w9, w9, #0x80
1073; NONEON-NOSVE-NEXT:    add w9, w8, w9
1074; NONEON-NOSVE-NEXT:    and w8, w9, #0xff
1075; NONEON-NOSVE-NEXT:    tbnz w9, #0, .LBB9_9
1076; NONEON-NOSVE-NEXT:  // %bb.1: // %else
1077; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB9_10
1078; NONEON-NOSVE-NEXT:  .LBB9_2: // %else2
1079; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB9_11
1080; NONEON-NOSVE-NEXT:  .LBB9_3: // %else4
1081; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB9_12
1082; NONEON-NOSVE-NEXT:  .LBB9_4: // %else6
1083; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB9_13
1084; NONEON-NOSVE-NEXT:  .LBB9_5: // %else8
1085; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB9_14
1086; NONEON-NOSVE-NEXT:  .LBB9_6: // %else10
1087; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB9_15
1088; NONEON-NOSVE-NEXT:  .LBB9_7: // %else12
1089; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB9_16
1090; NONEON-NOSVE-NEXT:  .LBB9_8: // %else14
1091; NONEON-NOSVE-NEXT:    add sp, sp, #16
1092; NONEON-NOSVE-NEXT:    ret
1093; NONEON-NOSVE-NEXT:  .LBB9_9: // %cond.store
1094; NONEON-NOSVE-NEXT:    str wzr, [x0]
1095; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB9_2
1096; NONEON-NOSVE-NEXT:  .LBB9_10: // %cond.store1
1097; NONEON-NOSVE-NEXT:    str wzr, [x0, #4]
1098; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB9_3
1099; NONEON-NOSVE-NEXT:  .LBB9_11: // %cond.store3
1100; NONEON-NOSVE-NEXT:    str wzr, [x0, #8]
1101; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB9_4
1102; NONEON-NOSVE-NEXT:  .LBB9_12: // %cond.store5
1103; NONEON-NOSVE-NEXT:    str wzr, [x0, #12]
1104; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB9_5
1105; NONEON-NOSVE-NEXT:  .LBB9_13: // %cond.store7
1106; NONEON-NOSVE-NEXT:    str wzr, [x0, #16]
1107; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB9_6
1108; NONEON-NOSVE-NEXT:  .LBB9_14: // %cond.store9
1109; NONEON-NOSVE-NEXT:    str wzr, [x0, #20]
1110; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB9_7
1111; NONEON-NOSVE-NEXT:  .LBB9_15: // %cond.store11
1112; NONEON-NOSVE-NEXT:    str wzr, [x0, #24]
1113; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB9_8
1114; NONEON-NOSVE-NEXT:  .LBB9_16: // %cond.store13
1115; NONEON-NOSVE-NEXT:    str wzr, [x0, #28]
1116; NONEON-NOSVE-NEXT:    add sp, sp, #16
1117; NONEON-NOSVE-NEXT:    ret
1118  call void @llvm.masked.store.v8f32(<8 x float> zeroinitializer, ptr %dst, i32 8, <8 x i1> %mask)
1119  ret void
1120}
1121
1122define void @masked_store_v2f64(ptr %dst, <2 x i1> %mask) {
1123; CHECK-LABEL: masked_store_v2f64:
1124; CHECK:       // %bb.0:
1125; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1126; CHECK-NEXT:    ptrue p0.d, vl2
1127; CHECK-NEXT:    uunpklo z0.d, z0.s
1128; CHECK-NEXT:    lsl z0.d, z0.d, #63
1129; CHECK-NEXT:    asr z0.d, z0.d, #63
1130; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
1131; CHECK-NEXT:    mov z0.d, #0 // =0x0
1132; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
1133; CHECK-NEXT:    ret
1134;
1135; NONEON-NOSVE-LABEL: masked_store_v2f64:
1136; NONEON-NOSVE:       // %bb.0:
1137; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
1138; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
1139; NONEON-NOSVE-NEXT:    ldr w8, [sp, #4]
1140; NONEON-NOSVE-NEXT:    ldrb w9, [sp]
1141; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
1142; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
1143; NONEON-NOSVE-NEXT:    bfxil w8, w9, #0, #1
1144; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB10_3
1145; NONEON-NOSVE-NEXT:  // %bb.1: // %else
1146; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB10_4
1147; NONEON-NOSVE-NEXT:  .LBB10_2: // %else2
1148; NONEON-NOSVE-NEXT:    add sp, sp, #16
1149; NONEON-NOSVE-NEXT:    ret
1150; NONEON-NOSVE-NEXT:  .LBB10_3: // %cond.store
1151; NONEON-NOSVE-NEXT:    str xzr, [x0]
1152; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB10_2
1153; NONEON-NOSVE-NEXT:  .LBB10_4: // %cond.store1
1154; NONEON-NOSVE-NEXT:    str xzr, [x0, #8]
1155; NONEON-NOSVE-NEXT:    add sp, sp, #16
1156; NONEON-NOSVE-NEXT:    ret
1157  call void @llvm.masked.store.v2f64(<2 x double> zeroinitializer, ptr %dst, i32 8, <2 x i1> %mask)
1158  ret void
1159}
1160
1161define void @masked_store_v4f64(ptr %dst, <4 x i1> %mask) {
1162; CHECK-LABEL: masked_store_v4f64:
1163; CHECK:       // %bb.0:
1164; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1165; CHECK-NEXT:    ptrue p0.d, vl2
1166; CHECK-NEXT:    mov x8, #2 // =0x2
1167; CHECK-NEXT:    uunpklo z0.s, z0.h
1168; CHECK-NEXT:    uunpklo z1.d, z0.s
1169; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
1170; CHECK-NEXT:    uunpklo z0.d, z0.s
1171; CHECK-NEXT:    lsl z1.d, z1.d, #63
1172; CHECK-NEXT:    lsl z0.d, z0.d, #63
1173; CHECK-NEXT:    asr z1.d, z1.d, #63
1174; CHECK-NEXT:    asr z0.d, z0.d, #63
1175; CHECK-NEXT:    cmpne p1.d, p0/z, z0.d, #0
1176; CHECK-NEXT:    cmpne p0.d, p0/z, z1.d, #0
1177; CHECK-NEXT:    mov z0.d, #0 // =0x0
1178; CHECK-NEXT:    st1d { z0.d }, p1, [x0, x8, lsl #3]
1179; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
1180; CHECK-NEXT:    ret
1181;
1182; NONEON-NOSVE-LABEL: masked_store_v4f64:
1183; NONEON-NOSVE:       // %bb.0:
1184; NONEON-NOSVE-NEXT:    str d0, [sp, #-16]!
1185; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 16
1186; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #2]
1187; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #4]
1188; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #6]
1189; NONEON-NOSVE-NEXT:    ldrh w11, [sp]
1190; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
1191; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
1192; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
1193; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
1194; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
1195; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
1196; NONEON-NOSVE-NEXT:    bfxil w8, w11, #0, #1
1197; NONEON-NOSVE-NEXT:    orr w9, w9, w10
1198; NONEON-NOSVE-NEXT:    orr w8, w8, w9
1199; NONEON-NOSVE-NEXT:    tbnz w8, #0, .LBB11_5
1200; NONEON-NOSVE-NEXT:  // %bb.1: // %else
1201; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB11_6
1202; NONEON-NOSVE-NEXT:  .LBB11_2: // %else2
1203; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB11_7
1204; NONEON-NOSVE-NEXT:  .LBB11_3: // %else4
1205; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB11_8
1206; NONEON-NOSVE-NEXT:  .LBB11_4: // %else6
1207; NONEON-NOSVE-NEXT:    add sp, sp, #16
1208; NONEON-NOSVE-NEXT:    ret
1209; NONEON-NOSVE-NEXT:  .LBB11_5: // %cond.store
1210; NONEON-NOSVE-NEXT:    str xzr, [x0]
1211; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB11_2
1212; NONEON-NOSVE-NEXT:  .LBB11_6: // %cond.store1
1213; NONEON-NOSVE-NEXT:    str xzr, [x0, #8]
1214; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB11_3
1215; NONEON-NOSVE-NEXT:  .LBB11_7: // %cond.store3
1216; NONEON-NOSVE-NEXT:    str xzr, [x0, #16]
1217; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB11_4
1218; NONEON-NOSVE-NEXT:  .LBB11_8: // %cond.store5
1219; NONEON-NOSVE-NEXT:    str xzr, [x0, #24]
1220; NONEON-NOSVE-NEXT:    add sp, sp, #16
1221; NONEON-NOSVE-NEXT:    ret
1222  call void @llvm.masked.store.v4f64(<4 x double> zeroinitializer, ptr %dst, i32 8, <4 x i1> %mask)
1223  ret void
1224}
1225
1226declare void @llvm.masked.store.v4i8(<4 x i8>, ptr, i32, <4 x i1>)
1227declare void @llvm.masked.store.v8i8(<8 x i8>, ptr, i32, <8 x i1>)
1228declare void @llvm.masked.store.v16i8(<16 x i8>, ptr, i32, <16 x i1>)
1229declare void @llvm.masked.store.v32i8(<32 x i8>, ptr, i32, <32 x i1>)
1230declare void @llvm.masked.store.v2f16(<2 x half>, ptr, i32, <2 x i1>)
1231declare void @llvm.masked.store.v4f16(<4 x half>, ptr, i32, <4 x i1>)
1232declare void @llvm.masked.store.v8f16(<8 x half>, ptr, i32, <8 x i1>)
1233declare void @llvm.masked.store.v16f16(<16 x half>, ptr, i32, <16 x i1>)
1234declare void @llvm.masked.store.v4f32(<4 x float>, ptr, i32, <4 x i1>)
1235declare void @llvm.masked.store.v8f32(<8 x float>, ptr, i32, <8 x i1>)
1236declare void @llvm.masked.store.v2f64(<2 x double>, ptr, i32, <2 x i1>)
1237declare void @llvm.masked.store.v4f64(<4 x double>, ptr, i32, <4 x i1>)
1238