xref: /llvm-project/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-masked-load.ll (revision 61510b51c33464a6bc15e4cf5b1ee07e2e0ec1c9) 
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
3; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
4
5
6target triple = "aarch64-unknown-linux-gnu"
7
8;
9; Masked Load
10;
11
12define <4 x i8> @masked_load_v4i8(ptr %src, <4 x i1> %mask) {
13; CHECK-LABEL: masked_load_v4i8:
14; CHECK:       // %bb.0:
15; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
16; CHECK-NEXT:    ptrue p0.h, vl4
17; CHECK-NEXT:    lsl z0.h, z0.h, #15
18; CHECK-NEXT:    asr z0.h, z0.h, #15
19; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
20; CHECK-NEXT:    ld1b { z0.h }, p0/z, [x0]
21; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
22; CHECK-NEXT:    ret
23;
24; NONEON-NOSVE-LABEL: masked_load_v4i8:
25; NONEON-NOSVE:       // %bb.0:
26; NONEON-NOSVE-NEXT:    sub sp, sp, #128
27; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 128
28; NONEON-NOSVE-NEXT:    str d0, [sp, #112]
29; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #114]
30; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #116]
31; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #118]
32; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #112]
33; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
34; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
35; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
36; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
37; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
38; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
39; NONEON-NOSVE-NEXT:    bfxil w8, w11, #0, #1
40; NONEON-NOSVE-NEXT:    orr w9, w9, w10
41; NONEON-NOSVE-NEXT:    orr w8, w8, w9
42; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB0_2
43; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
44; NONEON-NOSVE-NEXT:    ldrb w9, [x0]
45; NONEON-NOSVE-NEXT:    strh wzr, [sp, #110]
46; NONEON-NOSVE-NEXT:    stur wzr, [sp, #106]
47; NONEON-NOSVE-NEXT:    strh w9, [sp, #104]
48; NONEON-NOSVE-NEXT:    ldr d0, [sp, #104]
49; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB0_3
50; NONEON-NOSVE-NEXT:    b .LBB0_4
51; NONEON-NOSVE-NEXT:  .LBB0_2:
52; NONEON-NOSVE-NEXT:    adrp x9, .LCPI0_0
53; NONEON-NOSVE-NEXT:    ldr d0, [x9, :lo12:.LCPI0_0]
54; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB0_4
55; NONEON-NOSVE-NEXT:  .LBB0_3: // %cond.load1
56; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #1]
57; NONEON-NOSVE-NEXT:    str d0, [sp, #80]
58; NONEON-NOSVE-NEXT:    strh w9, [sp, #96]
59; NONEON-NOSVE-NEXT:    ldr w9, [sp, #84]
60; NONEON-NOSVE-NEXT:    ldr d0, [sp, #96]
61; NONEON-NOSVE-NEXT:    str w9, [sp, #92]
62; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #80]
63; NONEON-NOSVE-NEXT:    str d0, [sp, #72]
64; NONEON-NOSVE-NEXT:    strh w9, [sp, #88]
65; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #72]
66; NONEON-NOSVE-NEXT:    strh w9, [sp, #90]
67; NONEON-NOSVE-NEXT:    ldr d0, [sp, #88]
68; NONEON-NOSVE-NEXT:  .LBB0_4: // %else2
69; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB0_7
70; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
71; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB0_8
72; NONEON-NOSVE-NEXT:  .LBB0_6: // %else8
73; NONEON-NOSVE-NEXT:    add sp, sp, #128
74; NONEON-NOSVE-NEXT:    ret
75; NONEON-NOSVE-NEXT:  .LBB0_7: // %cond.load4
76; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #2]
77; NONEON-NOSVE-NEXT:    str d0, [sp, #48]
78; NONEON-NOSVE-NEXT:    strh w9, [sp, #64]
79; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #54]
80; NONEON-NOSVE-NEXT:    ldr d0, [sp, #64]
81; NONEON-NOSVE-NEXT:    strh w9, [sp, #62]
82; NONEON-NOSVE-NEXT:    ldr w9, [sp, #48]
83; NONEON-NOSVE-NEXT:    str d0, [sp, #40]
84; NONEON-NOSVE-NEXT:    str w9, [sp, #56]
85; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #40]
86; NONEON-NOSVE-NEXT:    strh w9, [sp, #60]
87; NONEON-NOSVE-NEXT:    ldr d0, [sp, #56]
88; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB0_6
89; NONEON-NOSVE-NEXT:  .LBB0_8: // %cond.load7
90; NONEON-NOSVE-NEXT:    ldrb w8, [x0, #3]
91; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
92; NONEON-NOSVE-NEXT:    strh w8, [sp, #32]
93; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
94; NONEON-NOSVE-NEXT:    ldr d0, [sp, #32]
95; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
96; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
97; NONEON-NOSVE-NEXT:    str d0, [sp, #16]
98; NONEON-NOSVE-NEXT:    str w8, [sp, #24]
99; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #16]
100; NONEON-NOSVE-NEXT:    strh w8, [sp, #30]
101; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
102; NONEON-NOSVE-NEXT:    add sp, sp, #128
103; NONEON-NOSVE-NEXT:    ret
104  %load = call <4 x i8> @llvm.masked.load.v4i8(ptr %src, i32 8, <4 x i1> %mask, <4 x i8> zeroinitializer)
105  ret <4 x i8> %load
106}
107
108define <8 x i8> @masked_load_v8i8(ptr %src, <8 x i1> %mask) {
109; CHECK-LABEL: masked_load_v8i8:
110; CHECK:       // %bb.0:
111; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
112; CHECK-NEXT:    ptrue p0.b, vl8
113; CHECK-NEXT:    lsl z0.b, z0.b, #7
114; CHECK-NEXT:    asr z0.b, z0.b, #7
115; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, #0
116; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
117; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
118; CHECK-NEXT:    ret
119;
120; NONEON-NOSVE-LABEL: masked_load_v8i8:
121; NONEON-NOSVE:       // %bb.0:
122; NONEON-NOSVE-NEXT:    sub sp, sp, #272
123; NONEON-NOSVE-NEXT:    str x29, [sp, #256] // 8-byte Folded Spill
124; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 272
125; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
126; NONEON-NOSVE-NEXT:    str d0, [sp, #240]
127; NONEON-NOSVE-NEXT:    add x9, sp, #176
128; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #242]
129; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #243]
130; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #241]
131; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #244]
132; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #245]
133; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #246]
134; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
135; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
136; NONEON-NOSVE-NEXT:    sbfx w11, w11, #0, #1
137; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #240]
138; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
139; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
140; NONEON-NOSVE-NEXT:    and w8, w8, #0x4
141; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
142; NONEON-NOSVE-NEXT:    sbfx w15, w15, #0, #1
143; NONEON-NOSVE-NEXT:    orr w8, w8, w10
144; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #247]
145; NONEON-NOSVE-NEXT:    and w11, w11, #0x2
146; NONEON-NOSVE-NEXT:    and w13, w13, #0x10
147; NONEON-NOSVE-NEXT:    bfxil w11, w12, #0, #1
148; NONEON-NOSVE-NEXT:    and w12, w14, #0x20
149; NONEON-NOSVE-NEXT:    orr w8, w8, w13
150; NONEON-NOSVE-NEXT:    and w13, w15, #0x40
151; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
152; NONEON-NOSVE-NEXT:    orr w8, w11, w8
153; NONEON-NOSVE-NEXT:    orr w11, w12, w13
154; NONEON-NOSVE-NEXT:    orr w8, w8, w11
155; NONEON-NOSVE-NEXT:    and w10, w10, #0x80
156; NONEON-NOSVE-NEXT:    add w10, w8, w10
157; NONEON-NOSVE-NEXT:    and w8, w10, #0xff
158; NONEON-NOSVE-NEXT:    tbz w10, #0, .LBB1_2
159; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
160; NONEON-NOSVE-NEXT:    ldrb w10, [x0]
161; NONEON-NOSVE-NEXT:    strb wzr, [sp, #239]
162; NONEON-NOSVE-NEXT:    sturh wzr, [x9, #61]
163; NONEON-NOSVE-NEXT:    stur wzr, [x9, #57]
164; NONEON-NOSVE-NEXT:    strb w10, [sp, #232]
165; NONEON-NOSVE-NEXT:    ldr d0, [sp, #232]
166; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB1_3
167; NONEON-NOSVE-NEXT:    b .LBB1_4
168; NONEON-NOSVE-NEXT:  .LBB1_2:
169; NONEON-NOSVE-NEXT:    adrp x10, .LCPI1_0
170; NONEON-NOSVE-NEXT:    ldr d0, [x10, :lo12:.LCPI1_0]
171; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB1_4
172; NONEON-NOSVE-NEXT:  .LBB1_3: // %cond.load1
173; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #1]
174; NONEON-NOSVE-NEXT:    str d0, [sp, #208]
175; NONEON-NOSVE-NEXT:    strb w10, [sp, #224]
176; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #214]
177; NONEON-NOSVE-NEXT:    ldr d0, [sp, #224]
178; NONEON-NOSVE-NEXT:    strh w10, [sp, #222]
179; NONEON-NOSVE-NEXT:    str d0, [sp, #200]
180; NONEON-NOSVE-NEXT:    ldur w10, [x9, #34]
181; NONEON-NOSVE-NEXT:    stur w10, [x9, #42]
182; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #208]
183; NONEON-NOSVE-NEXT:    strb w10, [sp, #216]
184; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #200]
185; NONEON-NOSVE-NEXT:    strb w10, [sp, #217]
186; NONEON-NOSVE-NEXT:    ldr d0, [sp, #216]
187; NONEON-NOSVE-NEXT:  .LBB1_4: // %else2
188; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB1_12
189; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
190; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB1_13
191; NONEON-NOSVE-NEXT:  .LBB1_6: // %else8
192; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB1_14
193; NONEON-NOSVE-NEXT:  .LBB1_7: // %else11
194; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB1_15
195; NONEON-NOSVE-NEXT:  .LBB1_8: // %else14
196; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB1_16
197; NONEON-NOSVE-NEXT:  .LBB1_9: // %else17
198; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB1_11
199; NONEON-NOSVE-NEXT:  .LBB1_10: // %cond.load19
200; NONEON-NOSVE-NEXT:    ldrb w8, [x0, #7]
201; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
202; NONEON-NOSVE-NEXT:    strb w8, [sp, #32]
203; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
204; NONEON-NOSVE-NEXT:    ldr d0, [sp, #32]
205; NONEON-NOSVE-NEXT:    strb w8, [sp, #30]
206; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
207; NONEON-NOSVE-NEXT:    str d0, [sp, #16]
208; NONEON-NOSVE-NEXT:    strh w8, [sp, #28]
209; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
210; NONEON-NOSVE-NEXT:    str w8, [sp, #24]
211; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
212; NONEON-NOSVE-NEXT:    strb w8, [sp, #31]
213; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
214; NONEON-NOSVE-NEXT:  .LBB1_11: // %else20
215; NONEON-NOSVE-NEXT:    ldr x29, [sp, #256] // 8-byte Folded Reload
216; NONEON-NOSVE-NEXT:    add sp, sp, #272
217; NONEON-NOSVE-NEXT:    ret
218; NONEON-NOSVE-NEXT:  .LBB1_12: // %cond.load4
219; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #2]
220; NONEON-NOSVE-NEXT:    str d0, [sp, #176]
221; NONEON-NOSVE-NEXT:    strb w10, [sp, #192]
222; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #183]
223; NONEON-NOSVE-NEXT:    ldr d0, [sp, #192]
224; NONEON-NOSVE-NEXT:    strb w10, [sp, #191]
225; NONEON-NOSVE-NEXT:    str d0, [sp, #168]
226; NONEON-NOSVE-NEXT:    ldur w10, [x9, #3]
227; NONEON-NOSVE-NEXT:    stur w10, [x9, #11]
228; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #176]
229; NONEON-NOSVE-NEXT:    strh w9, [sp, #184]
230; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #168]
231; NONEON-NOSVE-NEXT:    strb w9, [sp, #186]
232; NONEON-NOSVE-NEXT:    ldr d0, [sp, #184]
233; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB1_6
234; NONEON-NOSVE-NEXT:  .LBB1_13: // %cond.load7
235; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #3]
236; NONEON-NOSVE-NEXT:    str d0, [sp, #144]
237; NONEON-NOSVE-NEXT:    strb w9, [sp, #160]
238; NONEON-NOSVE-NEXT:    ldr w9, [sp, #148]
239; NONEON-NOSVE-NEXT:    ldr d0, [sp, #160]
240; NONEON-NOSVE-NEXT:    str w9, [sp, #156]
241; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #146]
242; NONEON-NOSVE-NEXT:    str d0, [sp, #136]
243; NONEON-NOSVE-NEXT:    strb w9, [sp, #154]
244; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #144]
245; NONEON-NOSVE-NEXT:    strh w9, [sp, #152]
246; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #136]
247; NONEON-NOSVE-NEXT:    strb w9, [sp, #155]
248; NONEON-NOSVE-NEXT:    ldr d0, [sp, #152]
249; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB1_7
250; NONEON-NOSVE-NEXT:  .LBB1_14: // %cond.load10
251; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #4]
252; NONEON-NOSVE-NEXT:    str d0, [sp, #112]
253; NONEON-NOSVE-NEXT:    strb w9, [sp, #128]
254; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #119]
255; NONEON-NOSVE-NEXT:    ldr d0, [sp, #128]
256; NONEON-NOSVE-NEXT:    strb w9, [sp, #127]
257; NONEON-NOSVE-NEXT:    ldurh w9, [sp, #117]
258; NONEON-NOSVE-NEXT:    str d0, [sp, #104]
259; NONEON-NOSVE-NEXT:    sturh w9, [sp, #125]
260; NONEON-NOSVE-NEXT:    ldr w9, [sp, #112]
261; NONEON-NOSVE-NEXT:    str w9, [sp, #120]
262; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #104]
263; NONEON-NOSVE-NEXT:    strb w9, [sp, #124]
264; NONEON-NOSVE-NEXT:    ldr d0, [sp, #120]
265; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB1_8
266; NONEON-NOSVE-NEXT:  .LBB1_15: // %cond.load13
267; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #5]
268; NONEON-NOSVE-NEXT:    str d0, [sp, #80]
269; NONEON-NOSVE-NEXT:    strb w9, [sp, #96]
270; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #86]
271; NONEON-NOSVE-NEXT:    ldr d0, [sp, #96]
272; NONEON-NOSVE-NEXT:    strh w9, [sp, #94]
273; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #84]
274; NONEON-NOSVE-NEXT:    str d0, [sp, #72]
275; NONEON-NOSVE-NEXT:    strb w9, [sp, #92]
276; NONEON-NOSVE-NEXT:    ldr w9, [sp, #80]
277; NONEON-NOSVE-NEXT:    str w9, [sp, #88]
278; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #72]
279; NONEON-NOSVE-NEXT:    strb w9, [sp, #93]
280; NONEON-NOSVE-NEXT:    ldr d0, [sp, #88]
281; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB1_9
282; NONEON-NOSVE-NEXT:  .LBB1_16: // %cond.load16
283; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #6]
284; NONEON-NOSVE-NEXT:    str d0, [sp, #48]
285; NONEON-NOSVE-NEXT:    strb w9, [sp, #64]
286; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #55]
287; NONEON-NOSVE-NEXT:    ldr d0, [sp, #64]
288; NONEON-NOSVE-NEXT:    strb w9, [sp, #63]
289; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #52]
290; NONEON-NOSVE-NEXT:    str d0, [sp, #40]
291; NONEON-NOSVE-NEXT:    strh w9, [sp, #60]
292; NONEON-NOSVE-NEXT:    ldr w9, [sp, #48]
293; NONEON-NOSVE-NEXT:    str w9, [sp, #56]
294; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #40]
295; NONEON-NOSVE-NEXT:    strb w9, [sp, #62]
296; NONEON-NOSVE-NEXT:    ldr d0, [sp, #56]
297; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB1_10
298; NONEON-NOSVE-NEXT:    b .LBB1_11
299  %load = call <8 x i8> @llvm.masked.load.v8i8(ptr %src, i32 8, <8 x i1> %mask, <8 x i8> zeroinitializer)
300  ret <8 x i8> %load
301}
302
303define <16 x i8> @masked_load_v16i8(ptr %src, <16 x i1> %mask) {
304; CHECK-LABEL: masked_load_v16i8:
305; CHECK:       // %bb.0:
306; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
307; CHECK-NEXT:    ptrue p0.b, vl16
308; CHECK-NEXT:    lsl z0.b, z0.b, #7
309; CHECK-NEXT:    asr z0.b, z0.b, #7
310; CHECK-NEXT:    cmpne p0.b, p0/z, z0.b, #0
311; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
312; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
313; CHECK-NEXT:    ret
314;
315; NONEON-NOSVE-LABEL: masked_load_v16i8:
316; NONEON-NOSVE:       // %bb.0:
317; NONEON-NOSVE-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
318; NONEON-NOSVE-NEXT:    sub sp, sp, #992
319; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 1008
320; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
321; NONEON-NOSVE-NEXT:    str q0, [sp, #976]
322; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #978]
323; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #979]
324; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #977]
325; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #980]
326; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #981]
327; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #982]
328; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
329; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
330; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
331; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #976]
332; NONEON-NOSVE-NEXT:    sbfx w12, w12, #0, #1
333; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
334; NONEON-NOSVE-NEXT:    and w8, w8, #0x4
335; NONEON-NOSVE-NEXT:    and w9, w9, #0x8
336; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
337; NONEON-NOSVE-NEXT:    orr w8, w8, w9
338; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #983]
339; NONEON-NOSVE-NEXT:    and w10, w10, #0x2
340; NONEON-NOSVE-NEXT:    and w12, w12, #0x10
341; NONEON-NOSVE-NEXT:    bfxil w10, w11, #0, #1
342; NONEON-NOSVE-NEXT:    and w11, w13, #0x20
343; NONEON-NOSVE-NEXT:    orr w8, w8, w12
344; NONEON-NOSVE-NEXT:    and w12, w14, #0x40
345; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
346; NONEON-NOSVE-NEXT:    orr w8, w10, w8
347; NONEON-NOSVE-NEXT:    orr w10, w11, w12
348; NONEON-NOSVE-NEXT:    orr w8, w8, w10
349; NONEON-NOSVE-NEXT:    and w9, w9, #0xffffff80
350; NONEON-NOSVE-NEXT:    add w8, w8, w9
351; NONEON-NOSVE-NEXT:    add x9, sp, #720
352; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB2_2
353; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
354; NONEON-NOSVE-NEXT:    ldrb w10, [x0]
355; NONEON-NOSVE-NEXT:    strb wzr, [sp, #975]
356; NONEON-NOSVE-NEXT:    sturh wzr, [x9, #253]
357; NONEON-NOSVE-NEXT:    stur wzr, [x9, #249]
358; NONEON-NOSVE-NEXT:    stur xzr, [x9, #241]
359; NONEON-NOSVE-NEXT:    strb w10, [sp, #960]
360; NONEON-NOSVE-NEXT:    ldr q0, [sp, #960]
361; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB2_3
362; NONEON-NOSVE-NEXT:    b .LBB2_4
363; NONEON-NOSVE-NEXT:  .LBB2_2:
364; NONEON-NOSVE-NEXT:    adrp x10, .LCPI2_0
365; NONEON-NOSVE-NEXT:    ldr q0, [x10, :lo12:.LCPI2_0]
366; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB2_4
367; NONEON-NOSVE-NEXT:  .LBB2_3: // %cond.load1
368; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #1]
369; NONEON-NOSVE-NEXT:    str q0, [sp, #912]
370; NONEON-NOSVE-NEXT:    strb w10, [sp, #944]
371; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #926]
372; NONEON-NOSVE-NEXT:    ldr q0, [sp, #944]
373; NONEON-NOSVE-NEXT:    strh w10, [sp, #942]
374; NONEON-NOSVE-NEXT:    str q0, [sp, #896]
375; NONEON-NOSVE-NEXT:    ldur w10, [x9, #202]
376; NONEON-NOSVE-NEXT:    ldur x11, [x9, #194]
377; NONEON-NOSVE-NEXT:    stur w10, [x9, #218]
378; NONEON-NOSVE-NEXT:    stur x11, [x9, #210]
379; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #912]
380; NONEON-NOSVE-NEXT:    strb w10, [sp, #928]
381; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #896]
382; NONEON-NOSVE-NEXT:    strb w10, [sp, #929]
383; NONEON-NOSVE-NEXT:    ldr q0, [sp, #928]
384; NONEON-NOSVE-NEXT:  .LBB2_4: // %else2
385; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB2_20
386; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
387; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB2_21
388; NONEON-NOSVE-NEXT:  .LBB2_6: // %else8
389; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB2_22
390; NONEON-NOSVE-NEXT:  .LBB2_7: // %else11
391; NONEON-NOSVE-NEXT:    add x9, sp, #464
392; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB2_23
393; NONEON-NOSVE-NEXT:  .LBB2_8: // %else14
394; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB2_24
395; NONEON-NOSVE-NEXT:  .LBB2_9: // %else17
396; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB2_25
397; NONEON-NOSVE-NEXT:  .LBB2_10: // %else20
398; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB2_26
399; NONEON-NOSVE-NEXT:  .LBB2_11: // %else23
400; NONEON-NOSVE-NEXT:    add x9, sp, #208
401; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB2_27
402; NONEON-NOSVE-NEXT:  .LBB2_12: // %else26
403; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB2_28
404; NONEON-NOSVE-NEXT:  .LBB2_13: // %else29
405; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB2_29
406; NONEON-NOSVE-NEXT:  .LBB2_14: // %else32
407; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB2_30
408; NONEON-NOSVE-NEXT:  .LBB2_15: // %else35
409; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB2_31
410; NONEON-NOSVE-NEXT:  .LBB2_16: // %else38
411; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB2_32
412; NONEON-NOSVE-NEXT:  .LBB2_17: // %else41
413; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB2_19
414; NONEON-NOSVE-NEXT:  .LBB2_18: // %cond.load43
415; NONEON-NOSVE-NEXT:    ldrb w8, [x0, #15]
416; NONEON-NOSVE-NEXT:    str q0, [sp]
417; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
418; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
419; NONEON-NOSVE-NEXT:    ldr q0, [sp, #48]
420; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]
421; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
422; NONEON-NOSVE-NEXT:    str q0, [sp, #16]
423; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
424; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
425; NONEON-NOSVE-NEXT:    str w8, [sp, #40]
426; NONEON-NOSVE-NEXT:    ldr x8, [sp]
427; NONEON-NOSVE-NEXT:    str x8, [sp, #32]
428; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
429; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]
430; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
431; NONEON-NOSVE-NEXT:  .LBB2_19: // %else44
432; NONEON-NOSVE-NEXT:    add sp, sp, #992
433; NONEON-NOSVE-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
434; NONEON-NOSVE-NEXT:    ret
435; NONEON-NOSVE-NEXT:  .LBB2_20: // %cond.load4
436; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #2]
437; NONEON-NOSVE-NEXT:    str q0, [sp, #848]
438; NONEON-NOSVE-NEXT:    strb w10, [sp, #880]
439; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #863]
440; NONEON-NOSVE-NEXT:    ldr q0, [sp, #880]
441; NONEON-NOSVE-NEXT:    strb w10, [sp, #879]
442; NONEON-NOSVE-NEXT:    str q0, [sp, #832]
443; NONEON-NOSVE-NEXT:    ldur w10, [x9, #139]
444; NONEON-NOSVE-NEXT:    ldur x11, [x9, #131]
445; NONEON-NOSVE-NEXT:    stur w10, [x9, #155]
446; NONEON-NOSVE-NEXT:    stur x11, [x9, #147]
447; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #848]
448; NONEON-NOSVE-NEXT:    strh w10, [sp, #864]
449; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #832]
450; NONEON-NOSVE-NEXT:    strb w10, [sp, #866]
451; NONEON-NOSVE-NEXT:    ldr q0, [sp, #864]
452; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB2_6
453; NONEON-NOSVE-NEXT:  .LBB2_21: // %cond.load7
454; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #3]
455; NONEON-NOSVE-NEXT:    str q0, [sp, #784]
456; NONEON-NOSVE-NEXT:    strb w10, [sp, #816]
457; NONEON-NOSVE-NEXT:    ldr w10, [sp, #796]
458; NONEON-NOSVE-NEXT:    ldr q0, [sp, #816]
459; NONEON-NOSVE-NEXT:    str w10, [sp, #812]
460; NONEON-NOSVE-NEXT:    str q0, [sp, #768]
461; NONEON-NOSVE-NEXT:    ldur x10, [x9, #68]
462; NONEON-NOSVE-NEXT:    stur x10, [x9, #84]
463; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #786]
464; NONEON-NOSVE-NEXT:    strb w10, [sp, #802]
465; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #784]
466; NONEON-NOSVE-NEXT:    strh w10, [sp, #800]
467; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #768]
468; NONEON-NOSVE-NEXT:    strb w10, [sp, #803]
469; NONEON-NOSVE-NEXT:    ldr q0, [sp, #800]
470; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB2_7
471; NONEON-NOSVE-NEXT:  .LBB2_22: // %cond.load10
472; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #4]
473; NONEON-NOSVE-NEXT:    str q0, [sp, #720]
474; NONEON-NOSVE-NEXT:    strb w10, [sp, #752]
475; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #735]
476; NONEON-NOSVE-NEXT:    ldr q0, [sp, #752]
477; NONEON-NOSVE-NEXT:    strb w10, [sp, #751]
478; NONEON-NOSVE-NEXT:    str q0, [sp, #704]
479; NONEON-NOSVE-NEXT:    ldurh w10, [x9, #13]
480; NONEON-NOSVE-NEXT:    ldur x11, [x9, #5]
481; NONEON-NOSVE-NEXT:    sturh w10, [x9, #29]
482; NONEON-NOSVE-NEXT:    stur x11, [x9, #21]
483; NONEON-NOSVE-NEXT:    ldr w9, [sp, #720]
484; NONEON-NOSVE-NEXT:    str w9, [sp, #736]
485; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #704]
486; NONEON-NOSVE-NEXT:    strb w9, [sp, #740]
487; NONEON-NOSVE-NEXT:    ldr q0, [sp, #736]
488; NONEON-NOSVE-NEXT:    add x9, sp, #464
489; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB2_8
490; NONEON-NOSVE-NEXT:  .LBB2_23: // %cond.load13
491; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #5]
492; NONEON-NOSVE-NEXT:    str q0, [sp, #656]
493; NONEON-NOSVE-NEXT:    strb w10, [sp, #688]
494; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #670]
495; NONEON-NOSVE-NEXT:    ldr q0, [sp, #688]
496; NONEON-NOSVE-NEXT:    strh w10, [sp, #686]
497; NONEON-NOSVE-NEXT:    str q0, [sp, #640]
498; NONEON-NOSVE-NEXT:    ldur x10, [x9, #198]
499; NONEON-NOSVE-NEXT:    stur x10, [x9, #214]
500; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #660]
501; NONEON-NOSVE-NEXT:    strb w10, [sp, #676]
502; NONEON-NOSVE-NEXT:    ldr w10, [sp, #656]
503; NONEON-NOSVE-NEXT:    str w10, [sp, #672]
504; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #640]
505; NONEON-NOSVE-NEXT:    strb w10, [sp, #677]
506; NONEON-NOSVE-NEXT:    ldr q0, [sp, #672]
507; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB2_9
508; NONEON-NOSVE-NEXT:  .LBB2_24: // %cond.load16
509; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #6]
510; NONEON-NOSVE-NEXT:    str q0, [sp, #592]
511; NONEON-NOSVE-NEXT:    strb w10, [sp, #624]
512; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #607]
513; NONEON-NOSVE-NEXT:    ldr q0, [sp, #624]
514; NONEON-NOSVE-NEXT:    strb w10, [sp, #623]
515; NONEON-NOSVE-NEXT:    str q0, [sp, #576]
516; NONEON-NOSVE-NEXT:    ldur x10, [x9, #135]
517; NONEON-NOSVE-NEXT:    stur x10, [x9, #151]
518; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #596]
519; NONEON-NOSVE-NEXT:    strh w10, [sp, #612]
520; NONEON-NOSVE-NEXT:    ldr w10, [sp, #592]
521; NONEON-NOSVE-NEXT:    str w10, [sp, #608]
522; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #576]
523; NONEON-NOSVE-NEXT:    strb w10, [sp, #614]
524; NONEON-NOSVE-NEXT:    ldr q0, [sp, #608]
525; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB2_10
526; NONEON-NOSVE-NEXT:  .LBB2_25: // %cond.load19
527; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #7]
528; NONEON-NOSVE-NEXT:    str q0, [sp, #528]
529; NONEON-NOSVE-NEXT:    strb w10, [sp, #560]
530; NONEON-NOSVE-NEXT:    ldr x10, [sp, #536]
531; NONEON-NOSVE-NEXT:    ldr q0, [sp, #560]
532; NONEON-NOSVE-NEXT:    str x10, [sp, #552]
533; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #534]
534; NONEON-NOSVE-NEXT:    str q0, [sp, #512]
535; NONEON-NOSVE-NEXT:    strb w10, [sp, #550]
536; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #532]
537; NONEON-NOSVE-NEXT:    strh w10, [sp, #548]
538; NONEON-NOSVE-NEXT:    ldr w10, [sp, #528]
539; NONEON-NOSVE-NEXT:    str w10, [sp, #544]
540; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #512]
541; NONEON-NOSVE-NEXT:    strb w10, [sp, #551]
542; NONEON-NOSVE-NEXT:    ldr q0, [sp, #544]
543; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB2_11
544; NONEON-NOSVE-NEXT:  .LBB2_26: // %cond.load22
545; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #8]
546; NONEON-NOSVE-NEXT:    str q0, [sp, #464]
547; NONEON-NOSVE-NEXT:    strb w10, [sp, #496]
548; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #479]
549; NONEON-NOSVE-NEXT:    ldr q0, [sp, #496]
550; NONEON-NOSVE-NEXT:    strb w10, [sp, #495]
551; NONEON-NOSVE-NEXT:    str q0, [sp, #448]
552; NONEON-NOSVE-NEXT:    ldurh w10, [x9, #13]
553; NONEON-NOSVE-NEXT:    ldur w11, [x9, #9]
554; NONEON-NOSVE-NEXT:    sturh w10, [x9, #29]
555; NONEON-NOSVE-NEXT:    stur w11, [x9, #25]
556; NONEON-NOSVE-NEXT:    ldr x9, [sp, #464]
557; NONEON-NOSVE-NEXT:    str x9, [sp, #480]
558; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #448]
559; NONEON-NOSVE-NEXT:    strb w9, [sp, #488]
560; NONEON-NOSVE-NEXT:    ldr q0, [sp, #480]
561; NONEON-NOSVE-NEXT:    add x9, sp, #208
562; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB2_12
563; NONEON-NOSVE-NEXT:  .LBB2_27: // %cond.load25
564; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #9]
565; NONEON-NOSVE-NEXT:    str q0, [sp, #400]
566; NONEON-NOSVE-NEXT:    strb w10, [sp, #432]
567; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #414]
568; NONEON-NOSVE-NEXT:    ldr q0, [sp, #432]
569; NONEON-NOSVE-NEXT:    strh w10, [sp, #430]
570; NONEON-NOSVE-NEXT:    str q0, [sp, #384]
571; NONEON-NOSVE-NEXT:    ldur w10, [x9, #202]
572; NONEON-NOSVE-NEXT:    stur w10, [x9, #218]
573; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #408]
574; NONEON-NOSVE-NEXT:    strb w10, [sp, #424]
575; NONEON-NOSVE-NEXT:    ldr x10, [sp, #400]
576; NONEON-NOSVE-NEXT:    str x10, [sp, #416]
577; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #384]
578; NONEON-NOSVE-NEXT:    strb w10, [sp, #425]
579; NONEON-NOSVE-NEXT:    ldr q0, [sp, #416]
580; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB2_13
581; NONEON-NOSVE-NEXT:  .LBB2_28: // %cond.load28
582; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #10]
583; NONEON-NOSVE-NEXT:    str q0, [sp, #336]
584; NONEON-NOSVE-NEXT:    strb w10, [sp, #368]
585; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #351]
586; NONEON-NOSVE-NEXT:    ldr q0, [sp, #368]
587; NONEON-NOSVE-NEXT:    strb w10, [sp, #367]
588; NONEON-NOSVE-NEXT:    str q0, [sp, #320]
589; NONEON-NOSVE-NEXT:    ldur w10, [x9, #139]
590; NONEON-NOSVE-NEXT:    stur w10, [x9, #155]
591; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #344]
592; NONEON-NOSVE-NEXT:    strh w10, [sp, #360]
593; NONEON-NOSVE-NEXT:    ldr x10, [sp, #336]
594; NONEON-NOSVE-NEXT:    str x10, [sp, #352]
595; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #320]
596; NONEON-NOSVE-NEXT:    strb w10, [sp, #362]
597; NONEON-NOSVE-NEXT:    ldr q0, [sp, #352]
598; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB2_14
599; NONEON-NOSVE-NEXT:  .LBB2_29: // %cond.load31
600; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #11]
601; NONEON-NOSVE-NEXT:    str q0, [sp, #272]
602; NONEON-NOSVE-NEXT:    strb w10, [sp, #304]
603; NONEON-NOSVE-NEXT:    ldr w10, [sp, #284]
604; NONEON-NOSVE-NEXT:    ldr q0, [sp, #304]
605; NONEON-NOSVE-NEXT:    str w10, [sp, #300]
606; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #282]
607; NONEON-NOSVE-NEXT:    str q0, [sp, #256]
608; NONEON-NOSVE-NEXT:    strb w10, [sp, #298]
609; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #280]
610; NONEON-NOSVE-NEXT:    strh w10, [sp, #296]
611; NONEON-NOSVE-NEXT:    ldr x10, [sp, #272]
612; NONEON-NOSVE-NEXT:    str x10, [sp, #288]
613; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #256]
614; NONEON-NOSVE-NEXT:    strb w10, [sp, #299]
615; NONEON-NOSVE-NEXT:    ldr q0, [sp, #288]
616; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB2_15
617; NONEON-NOSVE-NEXT:  .LBB2_30: // %cond.load34
618; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #12]
619; NONEON-NOSVE-NEXT:    str q0, [sp, #208]
620; NONEON-NOSVE-NEXT:    strb w10, [sp, #240]
621; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #223]
622; NONEON-NOSVE-NEXT:    ldr q0, [sp, #240]
623; NONEON-NOSVE-NEXT:    strb w10, [sp, #239]
624; NONEON-NOSVE-NEXT:    str q0, [sp, #192]
625; NONEON-NOSVE-NEXT:    ldurh w10, [x9, #13]
626; NONEON-NOSVE-NEXT:    sturh w10, [x9, #29]
627; NONEON-NOSVE-NEXT:    ldr w9, [sp, #216]
628; NONEON-NOSVE-NEXT:    str w9, [sp, #232]
629; NONEON-NOSVE-NEXT:    ldr x9, [sp, #208]
630; NONEON-NOSVE-NEXT:    str x9, [sp, #224]
631; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #192]
632; NONEON-NOSVE-NEXT:    strb w9, [sp, #236]
633; NONEON-NOSVE-NEXT:    ldr q0, [sp, #224]
634; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB2_16
635; NONEON-NOSVE-NEXT:  .LBB2_31: // %cond.load37
636; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #13]
637; NONEON-NOSVE-NEXT:    str q0, [sp, #144]
638; NONEON-NOSVE-NEXT:    strb w9, [sp, #176]
639; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #158]
640; NONEON-NOSVE-NEXT:    ldr q0, [sp, #176]
641; NONEON-NOSVE-NEXT:    strh w9, [sp, #174]
642; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #156]
643; NONEON-NOSVE-NEXT:    str q0, [sp, #128]
644; NONEON-NOSVE-NEXT:    strb w9, [sp, #172]
645; NONEON-NOSVE-NEXT:    ldr w9, [sp, #152]
646; NONEON-NOSVE-NEXT:    str w9, [sp, #168]
647; NONEON-NOSVE-NEXT:    ldr x9, [sp, #144]
648; NONEON-NOSVE-NEXT:    str x9, [sp, #160]
649; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #128]
650; NONEON-NOSVE-NEXT:    strb w9, [sp, #173]
651; NONEON-NOSVE-NEXT:    ldr q0, [sp, #160]
652; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB2_17
653; NONEON-NOSVE-NEXT:  .LBB2_32: // %cond.load40
654; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #14]
655; NONEON-NOSVE-NEXT:    str q0, [sp, #80]
656; NONEON-NOSVE-NEXT:    strb w9, [sp, #112]
657; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #95]
658; NONEON-NOSVE-NEXT:    ldr q0, [sp, #112]
659; NONEON-NOSVE-NEXT:    strb w9, [sp, #111]
660; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #92]
661; NONEON-NOSVE-NEXT:    str q0, [sp, #64]
662; NONEON-NOSVE-NEXT:    strh w9, [sp, #108]
663; NONEON-NOSVE-NEXT:    ldr w9, [sp, #88]
664; NONEON-NOSVE-NEXT:    str w9, [sp, #104]
665; NONEON-NOSVE-NEXT:    ldr x9, [sp, #80]
666; NONEON-NOSVE-NEXT:    str x9, [sp, #96]
667; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #64]
668; NONEON-NOSVE-NEXT:    strb w9, [sp, #110]
669; NONEON-NOSVE-NEXT:    ldr q0, [sp, #96]
670; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB2_18
671; NONEON-NOSVE-NEXT:    b .LBB2_19
672  %load = call <16 x i8> @llvm.masked.load.v16i8(ptr %src, i32 8, <16 x i1> %mask, <16 x i8> zeroinitializer)
673  ret <16 x i8> %load
674}
675
676define <32 x i8> @masked_load_v32i8(ptr %src, <32 x i1> %mask) {
677; CHECK-LABEL: masked_load_v32i8:
678; CHECK:       // %bb.0:
679; CHECK-NEXT:    sub sp, sp, #32
680; CHECK-NEXT:    .cfi_def_cfa_offset 32
681; CHECK-NEXT:    ldr w8, [sp, #224]
682; CHECK-NEXT:    ldr w9, [sp, #216]
683; CHECK-NEXT:    ptrue p0.b, vl16
684; CHECK-NEXT:    strb w7, [sp, #6]
685; CHECK-NEXT:    strb w8, [sp, #31]
686; CHECK-NEXT:    ldr w8, [sp, #208]
687; CHECK-NEXT:    strb w9, [sp, #30]
688; CHECK-NEXT:    ldr w9, [sp, #200]
689; CHECK-NEXT:    strb w8, [sp, #29]
690; CHECK-NEXT:    ldr w8, [sp, #192]
691; CHECK-NEXT:    strb w9, [sp, #28]
692; CHECK-NEXT:    ldr w9, [sp, #184]
693; CHECK-NEXT:    strb w8, [sp, #27]
694; CHECK-NEXT:    ldr w8, [sp, #176]
695; CHECK-NEXT:    strb w9, [sp, #26]
696; CHECK-NEXT:    ldr w9, [sp, #168]
697; CHECK-NEXT:    strb w8, [sp, #25]
698; CHECK-NEXT:    ldr w8, [sp, #160]
699; CHECK-NEXT:    strb w9, [sp, #24]
700; CHECK-NEXT:    ldr w9, [sp, #152]
701; CHECK-NEXT:    strb w8, [sp, #23]
702; CHECK-NEXT:    ldr w8, [sp, #144]
703; CHECK-NEXT:    strb w9, [sp, #22]
704; CHECK-NEXT:    ldr w9, [sp, #136]
705; CHECK-NEXT:    strb w8, [sp, #21]
706; CHECK-NEXT:    ldr w8, [sp, #128]
707; CHECK-NEXT:    strb w9, [sp, #20]
708; CHECK-NEXT:    ldr w9, [sp, #120]
709; CHECK-NEXT:    strb w8, [sp, #19]
710; CHECK-NEXT:    ldr w8, [sp, #112]
711; CHECK-NEXT:    strb w9, [sp, #18]
712; CHECK-NEXT:    ldr w9, [sp, #104]
713; CHECK-NEXT:    strb w8, [sp, #17]
714; CHECK-NEXT:    ldr w8, [sp, #96]
715; CHECK-NEXT:    strb w9, [sp, #16]
716; CHECK-NEXT:    ldr w9, [sp, #88]
717; CHECK-NEXT:    strb w8, [sp, #15]
718; CHECK-NEXT:    ldr w8, [sp, #80]
719; CHECK-NEXT:    strb w9, [sp, #14]
720; CHECK-NEXT:    ldr w9, [sp, #72]
721; CHECK-NEXT:    strb w8, [sp, #13]
722; CHECK-NEXT:    ldr w8, [sp, #64]
723; CHECK-NEXT:    strb w9, [sp, #12]
724; CHECK-NEXT:    ldr w9, [sp, #56]
725; CHECK-NEXT:    strb w8, [sp, #11]
726; CHECK-NEXT:    ldr w8, [sp, #48]
727; CHECK-NEXT:    strb w9, [sp, #10]
728; CHECK-NEXT:    ldr w9, [sp, #40]
729; CHECK-NEXT:    strb w8, [sp, #9]
730; CHECK-NEXT:    ldr w8, [sp, #32]
731; CHECK-NEXT:    strb w9, [sp, #8]
732; CHECK-NEXT:    strb w8, [sp, #7]
733; CHECK-NEXT:    mov w8, #16 // =0x10
734; CHECK-NEXT:    strb w6, [sp, #5]
735; CHECK-NEXT:    strb w5, [sp, #4]
736; CHECK-NEXT:    strb w4, [sp, #3]
737; CHECK-NEXT:    strb w3, [sp, #2]
738; CHECK-NEXT:    strb w2, [sp, #1]
739; CHECK-NEXT:    strb w1, [sp]
740; CHECK-NEXT:    ldp q1, q0, [sp]
741; CHECK-NEXT:    lsl z0.b, z0.b, #7
742; CHECK-NEXT:    lsl z1.b, z1.b, #7
743; CHECK-NEXT:    asr z0.b, z0.b, #7
744; CHECK-NEXT:    asr z1.b, z1.b, #7
745; CHECK-NEXT:    cmpne p1.b, p0/z, z0.b, #0
746; CHECK-NEXT:    cmpne p0.b, p0/z, z1.b, #0
747; CHECK-NEXT:    ld1b { z0.b }, p0/z, [x0]
748; CHECK-NEXT:    ld1b { z1.b }, p1/z, [x0, x8]
749; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
750; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
751; CHECK-NEXT:    add sp, sp, #32
752; CHECK-NEXT:    ret
753;
754; NONEON-NOSVE-LABEL: masked_load_v32i8:
755; NONEON-NOSVE:       // %bb.0:
756; NONEON-NOSVE-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
757; NONEON-NOSVE-NEXT:    sub sp, sp, #2000
758; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 2016
759; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
760; NONEON-NOSVE-NEXT:    ldr w8, [sp, #2096]
761; NONEON-NOSVE-NEXT:    ldr w9, [sp, #2104]
762; NONEON-NOSVE-NEXT:    sbfx w15, w7, #0, #1
763; NONEON-NOSVE-NEXT:    ldr w10, [sp, #2112]
764; NONEON-NOSVE-NEXT:    ldr w11, [sp, #2088]
765; NONEON-NOSVE-NEXT:    ldr w12, [sp, #2120]
766; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
767; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
768; NONEON-NOSVE-NEXT:    ldr w13, [sp, #2136]
769; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
770; NONEON-NOSVE-NEXT:    ldr w14, [sp, #2144]
771; NONEON-NOSVE-NEXT:    ldr w16, [sp, #2016]
772; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
773; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
774; NONEON-NOSVE-NEXT:    bfxil w8, w11, #0, #1
775; NONEON-NOSVE-NEXT:    sbfx w11, w12, #0, #1
776; NONEON-NOSVE-NEXT:    ldr w12, [sp, #2128]
777; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
778; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
779; NONEON-NOSVE-NEXT:    orr w9, w9, w10
780; NONEON-NOSVE-NEXT:    and w10, w11, #0x10
781; NONEON-NOSVE-NEXT:    sbfx w11, w12, #0, #1
782; NONEON-NOSVE-NEXT:    orr w9, w9, w10
783; NONEON-NOSVE-NEXT:    sbfx w10, w13, #0, #1
784; NONEON-NOSVE-NEXT:    sbfx w12, w4, #0, #1
785; NONEON-NOSVE-NEXT:    orr w8, w8, w9
786; NONEON-NOSVE-NEXT:    and w9, w11, #0x20
787; NONEON-NOSVE-NEXT:    sbfx w11, w3, #0, #1
788; NONEON-NOSVE-NEXT:    and w10, w10, #0x40
789; NONEON-NOSVE-NEXT:    sbfx w13, w5, #0, #1
790; NONEON-NOSVE-NEXT:    and w12, w12, #0x8
791; NONEON-NOSVE-NEXT:    orr w9, w9, w10
792; NONEON-NOSVE-NEXT:    sbfx w10, w2, #0, #1
793; NONEON-NOSVE-NEXT:    and w11, w11, #0x4
794; NONEON-NOSVE-NEXT:    orr w11, w11, w12
795; NONEON-NOSVE-NEXT:    and w12, w13, #0x10
796; NONEON-NOSVE-NEXT:    sbfx w13, w6, #0, #1
797; NONEON-NOSVE-NEXT:    and w10, w10, #0x2
798; NONEON-NOSVE-NEXT:    orr w11, w11, w12
799; NONEON-NOSVE-NEXT:    orr w8, w8, w9
800; NONEON-NOSVE-NEXT:    bfxil w10, w1, #0, #1
801; NONEON-NOSVE-NEXT:    and w12, w13, #0x20
802; NONEON-NOSVE-NEXT:    and w13, w15, #0x40
803; NONEON-NOSVE-NEXT:    sbfx w15, w16, #0, #1
804; NONEON-NOSVE-NEXT:    orr w9, w10, w11
805; NONEON-NOSVE-NEXT:    orr w10, w12, w13
806; NONEON-NOSVE-NEXT:    and w11, w14, #0xff80
807; NONEON-NOSVE-NEXT:    orr w9, w9, w10
808; NONEON-NOSVE-NEXT:    and w10, w15, #0xff80
809; NONEON-NOSVE-NEXT:    add w11, w8, w11
810; NONEON-NOSVE-NEXT:    add w8, w9, w10
811; NONEON-NOSVE-NEXT:    adrp x9, .LCPI3_0
812; NONEON-NOSVE-NEXT:    bfi w8, w11, #16, #16
813; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI3_0]
814; NONEON-NOSVE-NEXT:    add x9, sp, #1744
815; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB3_2
816; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
817; NONEON-NOSVE-NEXT:    ldrb w10, [x0]
818; NONEON-NOSVE-NEXT:    strb wzr, [sp, #1999]
819; NONEON-NOSVE-NEXT:    sturh wzr, [x9, #253]
820; NONEON-NOSVE-NEXT:    stur wzr, [x9, #249]
821; NONEON-NOSVE-NEXT:    stur xzr, [x9, #241]
822; NONEON-NOSVE-NEXT:    strb w10, [sp, #1984]
823; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1984]
824; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB3_3
825; NONEON-NOSVE-NEXT:    b .LBB3_4
826; NONEON-NOSVE-NEXT:  .LBB3_2:
827; NONEON-NOSVE-NEXT:    str q1, [sp, #-16]!
828; NONEON-NOSVE-NEXT:    ldr q0, [sp], #16
829; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB3_4
830; NONEON-NOSVE-NEXT:  .LBB3_3: // %cond.load1
831; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #1]
832; NONEON-NOSVE-NEXT:    str q0, [sp, #1936]
833; NONEON-NOSVE-NEXT:    strb w10, [sp, #1968]
834; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #1950]
835; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1968]
836; NONEON-NOSVE-NEXT:    strh w10, [sp, #1966]
837; NONEON-NOSVE-NEXT:    str q0, [sp, #1920]
838; NONEON-NOSVE-NEXT:    ldur w10, [x9, #202]
839; NONEON-NOSVE-NEXT:    ldur x11, [x9, #194]
840; NONEON-NOSVE-NEXT:    stur w10, [x9, #218]
841; NONEON-NOSVE-NEXT:    stur x11, [x9, #210]
842; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1936]
843; NONEON-NOSVE-NEXT:    strb w10, [sp, #1952]
844; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1920]
845; NONEON-NOSVE-NEXT:    strb w10, [sp, #1953]
846; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1952]
847; NONEON-NOSVE-NEXT:  .LBB3_4: // %else2
848; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB3_36
849; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
850; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB3_37
851; NONEON-NOSVE-NEXT:  .LBB3_6: // %else8
852; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB3_38
853; NONEON-NOSVE-NEXT:  .LBB3_7: // %else11
854; NONEON-NOSVE-NEXT:    add x9, sp, #1488
855; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB3_39
856; NONEON-NOSVE-NEXT:  .LBB3_8: // %else14
857; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB3_40
858; NONEON-NOSVE-NEXT:  .LBB3_9: // %else17
859; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB3_41
860; NONEON-NOSVE-NEXT:  .LBB3_10: // %else20
861; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB3_42
862; NONEON-NOSVE-NEXT:  .LBB3_11: // %else23
863; NONEON-NOSVE-NEXT:    add x9, sp, #1232
864; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB3_43
865; NONEON-NOSVE-NEXT:  .LBB3_12: // %else26
866; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB3_44
867; NONEON-NOSVE-NEXT:  .LBB3_13: // %else29
868; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB3_45
869; NONEON-NOSVE-NEXT:  .LBB3_14: // %else32
870; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB3_46
871; NONEON-NOSVE-NEXT:  .LBB3_15: // %else35
872; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB3_47
873; NONEON-NOSVE-NEXT:  .LBB3_16: // %else38
874; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB3_48
875; NONEON-NOSVE-NEXT:  .LBB3_17: // %else41
876; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB3_49
877; NONEON-NOSVE-NEXT:  .LBB3_18: // %else44
878; NONEON-NOSVE-NEXT:    tbnz w8, #16, .LBB3_50
879; NONEON-NOSVE-NEXT:  .LBB3_19: // %else47
880; NONEON-NOSVE-NEXT:    add x9, sp, #720
881; NONEON-NOSVE-NEXT:    tbnz w8, #17, .LBB3_51
882; NONEON-NOSVE-NEXT:  .LBB3_20: // %else50
883; NONEON-NOSVE-NEXT:    tbnz w8, #18, .LBB3_52
884; NONEON-NOSVE-NEXT:  .LBB3_21: // %else53
885; NONEON-NOSVE-NEXT:    tbnz w8, #19, .LBB3_53
886; NONEON-NOSVE-NEXT:  .LBB3_22: // %else56
887; NONEON-NOSVE-NEXT:    tbnz w8, #20, .LBB3_54
888; NONEON-NOSVE-NEXT:  .LBB3_23: // %else59
889; NONEON-NOSVE-NEXT:    add x9, sp, #464
890; NONEON-NOSVE-NEXT:    tbnz w8, #21, .LBB3_55
891; NONEON-NOSVE-NEXT:  .LBB3_24: // %else62
892; NONEON-NOSVE-NEXT:    tbnz w8, #22, .LBB3_56
893; NONEON-NOSVE-NEXT:  .LBB3_25: // %else65
894; NONEON-NOSVE-NEXT:    tbnz w8, #23, .LBB3_57
895; NONEON-NOSVE-NEXT:  .LBB3_26: // %else68
896; NONEON-NOSVE-NEXT:    tbnz w8, #24, .LBB3_58
897; NONEON-NOSVE-NEXT:  .LBB3_27: // %else71
898; NONEON-NOSVE-NEXT:    add x9, sp, #208
899; NONEON-NOSVE-NEXT:    tbnz w8, #25, .LBB3_59
900; NONEON-NOSVE-NEXT:  .LBB3_28: // %else74
901; NONEON-NOSVE-NEXT:    tbnz w8, #26, .LBB3_60
902; NONEON-NOSVE-NEXT:  .LBB3_29: // %else77
903; NONEON-NOSVE-NEXT:    tbnz w8, #27, .LBB3_61
904; NONEON-NOSVE-NEXT:  .LBB3_30: // %else80
905; NONEON-NOSVE-NEXT:    tbnz w8, #28, .LBB3_62
906; NONEON-NOSVE-NEXT:  .LBB3_31: // %else83
907; NONEON-NOSVE-NEXT:    tbnz w8, #29, .LBB3_63
908; NONEON-NOSVE-NEXT:  .LBB3_32: // %else86
909; NONEON-NOSVE-NEXT:    tbnz w8, #30, .LBB3_64
910; NONEON-NOSVE-NEXT:  .LBB3_33: // %else89
911; NONEON-NOSVE-NEXT:    tbz w8, #31, .LBB3_35
912; NONEON-NOSVE-NEXT:  .LBB3_34: // %cond.load91
913; NONEON-NOSVE-NEXT:    ldrb w8, [x0, #31]
914; NONEON-NOSVE-NEXT:    str q1, [sp]
915; NONEON-NOSVE-NEXT:    strb w8, [sp, #48]
916; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #14]
917; NONEON-NOSVE-NEXT:    ldr q1, [sp, #48]
918; NONEON-NOSVE-NEXT:    strb w8, [sp, #46]
919; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
920; NONEON-NOSVE-NEXT:    str q1, [sp, #16]
921; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
922; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
923; NONEON-NOSVE-NEXT:    str w8, [sp, #40]
924; NONEON-NOSVE-NEXT:    ldr x8, [sp]
925; NONEON-NOSVE-NEXT:    str x8, [sp, #32]
926; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #16]
927; NONEON-NOSVE-NEXT:    strb w8, [sp, #47]
928; NONEON-NOSVE-NEXT:    ldr q1, [sp, #32]
929; NONEON-NOSVE-NEXT:  .LBB3_35: // %else92
930; NONEON-NOSVE-NEXT:    add sp, sp, #2000
931; NONEON-NOSVE-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
932; NONEON-NOSVE-NEXT:    ret
933; NONEON-NOSVE-NEXT:  .LBB3_36: // %cond.load4
934; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #2]
935; NONEON-NOSVE-NEXT:    str q0, [sp, #1872]
936; NONEON-NOSVE-NEXT:    strb w10, [sp, #1904]
937; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1887]
938; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1904]
939; NONEON-NOSVE-NEXT:    strb w10, [sp, #1903]
940; NONEON-NOSVE-NEXT:    str q0, [sp, #1856]
941; NONEON-NOSVE-NEXT:    ldur w10, [x9, #139]
942; NONEON-NOSVE-NEXT:    ldur x11, [x9, #131]
943; NONEON-NOSVE-NEXT:    stur w10, [x9, #155]
944; NONEON-NOSVE-NEXT:    stur x11, [x9, #147]
945; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #1872]
946; NONEON-NOSVE-NEXT:    strh w10, [sp, #1888]
947; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1856]
948; NONEON-NOSVE-NEXT:    strb w10, [sp, #1890]
949; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1888]
950; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB3_6
951; NONEON-NOSVE-NEXT:  .LBB3_37: // %cond.load7
952; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #3]
953; NONEON-NOSVE-NEXT:    str q0, [sp, #1808]
954; NONEON-NOSVE-NEXT:    strb w10, [sp, #1840]
955; NONEON-NOSVE-NEXT:    ldr w10, [sp, #1820]
956; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1840]
957; NONEON-NOSVE-NEXT:    str w10, [sp, #1836]
958; NONEON-NOSVE-NEXT:    str q0, [sp, #1792]
959; NONEON-NOSVE-NEXT:    ldur x10, [x9, #68]
960; NONEON-NOSVE-NEXT:    stur x10, [x9, #84]
961; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1810]
962; NONEON-NOSVE-NEXT:    strb w10, [sp, #1826]
963; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #1808]
964; NONEON-NOSVE-NEXT:    strh w10, [sp, #1824]
965; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1792]
966; NONEON-NOSVE-NEXT:    strb w10, [sp, #1827]
967; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1824]
968; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB3_7
969; NONEON-NOSVE-NEXT:  .LBB3_38: // %cond.load10
970; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #4]
971; NONEON-NOSVE-NEXT:    str q0, [sp, #1744]
972; NONEON-NOSVE-NEXT:    strb w10, [sp, #1776]
973; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1759]
974; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1776]
975; NONEON-NOSVE-NEXT:    strb w10, [sp, #1775]
976; NONEON-NOSVE-NEXT:    str q0, [sp, #1728]
977; NONEON-NOSVE-NEXT:    ldurh w10, [x9, #13]
978; NONEON-NOSVE-NEXT:    ldur x11, [x9, #5]
979; NONEON-NOSVE-NEXT:    sturh w10, [x9, #29]
980; NONEON-NOSVE-NEXT:    stur x11, [x9, #21]
981; NONEON-NOSVE-NEXT:    ldr w9, [sp, #1744]
982; NONEON-NOSVE-NEXT:    str w9, [sp, #1760]
983; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1728]
984; NONEON-NOSVE-NEXT:    strb w9, [sp, #1764]
985; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1760]
986; NONEON-NOSVE-NEXT:    add x9, sp, #1488
987; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB3_8
988; NONEON-NOSVE-NEXT:  .LBB3_39: // %cond.load13
989; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #5]
990; NONEON-NOSVE-NEXT:    str q0, [sp, #1680]
991; NONEON-NOSVE-NEXT:    strb w10, [sp, #1712]
992; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #1694]
993; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1712]
994; NONEON-NOSVE-NEXT:    strh w10, [sp, #1710]
995; NONEON-NOSVE-NEXT:    str q0, [sp, #1664]
996; NONEON-NOSVE-NEXT:    ldur x10, [x9, #198]
997; NONEON-NOSVE-NEXT:    stur x10, [x9, #214]
998; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1684]
999; NONEON-NOSVE-NEXT:    strb w10, [sp, #1700]
1000; NONEON-NOSVE-NEXT:    ldr w10, [sp, #1680]
1001; NONEON-NOSVE-NEXT:    str w10, [sp, #1696]
1002; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1664]
1003; NONEON-NOSVE-NEXT:    strb w10, [sp, #1701]
1004; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1696]
1005; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB3_9
1006; NONEON-NOSVE-NEXT:  .LBB3_40: // %cond.load16
1007; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #6]
1008; NONEON-NOSVE-NEXT:    str q0, [sp, #1616]
1009; NONEON-NOSVE-NEXT:    strb w10, [sp, #1648]
1010; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1631]
1011; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1648]
1012; NONEON-NOSVE-NEXT:    strb w10, [sp, #1647]
1013; NONEON-NOSVE-NEXT:    str q0, [sp, #1600]
1014; NONEON-NOSVE-NEXT:    ldur x10, [x9, #135]
1015; NONEON-NOSVE-NEXT:    stur x10, [x9, #151]
1016; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #1620]
1017; NONEON-NOSVE-NEXT:    strh w10, [sp, #1636]
1018; NONEON-NOSVE-NEXT:    ldr w10, [sp, #1616]
1019; NONEON-NOSVE-NEXT:    str w10, [sp, #1632]
1020; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1600]
1021; NONEON-NOSVE-NEXT:    strb w10, [sp, #1638]
1022; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1632]
1023; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB3_10
1024; NONEON-NOSVE-NEXT:  .LBB3_41: // %cond.load19
1025; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #7]
1026; NONEON-NOSVE-NEXT:    str q0, [sp, #1552]
1027; NONEON-NOSVE-NEXT:    strb w10, [sp, #1584]
1028; NONEON-NOSVE-NEXT:    ldr x10, [sp, #1560]
1029; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1584]
1030; NONEON-NOSVE-NEXT:    str x10, [sp, #1576]
1031; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1558]
1032; NONEON-NOSVE-NEXT:    str q0, [sp, #1536]
1033; NONEON-NOSVE-NEXT:    strb w10, [sp, #1574]
1034; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #1556]
1035; NONEON-NOSVE-NEXT:    strh w10, [sp, #1572]
1036; NONEON-NOSVE-NEXT:    ldr w10, [sp, #1552]
1037; NONEON-NOSVE-NEXT:    str w10, [sp, #1568]
1038; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1536]
1039; NONEON-NOSVE-NEXT:    strb w10, [sp, #1575]
1040; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1568]
1041; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB3_11
1042; NONEON-NOSVE-NEXT:  .LBB3_42: // %cond.load22
1043; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #8]
1044; NONEON-NOSVE-NEXT:    str q0, [sp, #1488]
1045; NONEON-NOSVE-NEXT:    strb w10, [sp, #1520]
1046; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1503]
1047; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1520]
1048; NONEON-NOSVE-NEXT:    strb w10, [sp, #1519]
1049; NONEON-NOSVE-NEXT:    str q0, [sp, #1472]
1050; NONEON-NOSVE-NEXT:    ldurh w10, [x9, #13]
1051; NONEON-NOSVE-NEXT:    ldur w11, [x9, #9]
1052; NONEON-NOSVE-NEXT:    sturh w10, [x9, #29]
1053; NONEON-NOSVE-NEXT:    stur w11, [x9, #25]
1054; NONEON-NOSVE-NEXT:    ldr x9, [sp, #1488]
1055; NONEON-NOSVE-NEXT:    str x9, [sp, #1504]
1056; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1472]
1057; NONEON-NOSVE-NEXT:    strb w9, [sp, #1512]
1058; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1504]
1059; NONEON-NOSVE-NEXT:    add x9, sp, #1232
1060; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB3_12
1061; NONEON-NOSVE-NEXT:  .LBB3_43: // %cond.load25
1062; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #9]
1063; NONEON-NOSVE-NEXT:    str q0, [sp, #1424]
1064; NONEON-NOSVE-NEXT:    strb w10, [sp, #1456]
1065; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #1438]
1066; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1456]
1067; NONEON-NOSVE-NEXT:    strh w10, [sp, #1454]
1068; NONEON-NOSVE-NEXT:    str q0, [sp, #1408]
1069; NONEON-NOSVE-NEXT:    ldur w10, [x9, #202]
1070; NONEON-NOSVE-NEXT:    stur w10, [x9, #218]
1071; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1432]
1072; NONEON-NOSVE-NEXT:    strb w10, [sp, #1448]
1073; NONEON-NOSVE-NEXT:    ldr x10, [sp, #1424]
1074; NONEON-NOSVE-NEXT:    str x10, [sp, #1440]
1075; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1408]
1076; NONEON-NOSVE-NEXT:    strb w10, [sp, #1449]
1077; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1440]
1078; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB3_13
1079; NONEON-NOSVE-NEXT:  .LBB3_44: // %cond.load28
1080; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #10]
1081; NONEON-NOSVE-NEXT:    str q0, [sp, #1360]
1082; NONEON-NOSVE-NEXT:    strb w10, [sp, #1392]
1083; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1375]
1084; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1392]
1085; NONEON-NOSVE-NEXT:    strb w10, [sp, #1391]
1086; NONEON-NOSVE-NEXT:    str q0, [sp, #1344]
1087; NONEON-NOSVE-NEXT:    ldur w10, [x9, #139]
1088; NONEON-NOSVE-NEXT:    stur w10, [x9, #155]
1089; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #1368]
1090; NONEON-NOSVE-NEXT:    strh w10, [sp, #1384]
1091; NONEON-NOSVE-NEXT:    ldr x10, [sp, #1360]
1092; NONEON-NOSVE-NEXT:    str x10, [sp, #1376]
1093; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1344]
1094; NONEON-NOSVE-NEXT:    strb w10, [sp, #1386]
1095; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1376]
1096; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB3_14
1097; NONEON-NOSVE-NEXT:  .LBB3_45: // %cond.load31
1098; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #11]
1099; NONEON-NOSVE-NEXT:    str q0, [sp, #1296]
1100; NONEON-NOSVE-NEXT:    strb w10, [sp, #1328]
1101; NONEON-NOSVE-NEXT:    ldr w10, [sp, #1308]
1102; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1328]
1103; NONEON-NOSVE-NEXT:    str w10, [sp, #1324]
1104; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1306]
1105; NONEON-NOSVE-NEXT:    str q0, [sp, #1280]
1106; NONEON-NOSVE-NEXT:    strb w10, [sp, #1322]
1107; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #1304]
1108; NONEON-NOSVE-NEXT:    strh w10, [sp, #1320]
1109; NONEON-NOSVE-NEXT:    ldr x10, [sp, #1296]
1110; NONEON-NOSVE-NEXT:    str x10, [sp, #1312]
1111; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1280]
1112; NONEON-NOSVE-NEXT:    strb w10, [sp, #1323]
1113; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1312]
1114; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB3_15
1115; NONEON-NOSVE-NEXT:  .LBB3_46: // %cond.load34
1116; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #12]
1117; NONEON-NOSVE-NEXT:    str q0, [sp, #1232]
1118; NONEON-NOSVE-NEXT:    strb w10, [sp, #1264]
1119; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #1247]
1120; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1264]
1121; NONEON-NOSVE-NEXT:    strb w10, [sp, #1263]
1122; NONEON-NOSVE-NEXT:    str q0, [sp, #1216]
1123; NONEON-NOSVE-NEXT:    ldurh w10, [x9, #13]
1124; NONEON-NOSVE-NEXT:    sturh w10, [x9, #29]
1125; NONEON-NOSVE-NEXT:    ldr w9, [sp, #1240]
1126; NONEON-NOSVE-NEXT:    str w9, [sp, #1256]
1127; NONEON-NOSVE-NEXT:    ldr x9, [sp, #1232]
1128; NONEON-NOSVE-NEXT:    str x9, [sp, #1248]
1129; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1216]
1130; NONEON-NOSVE-NEXT:    strb w9, [sp, #1260]
1131; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1248]
1132; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB3_16
1133; NONEON-NOSVE-NEXT:  .LBB3_47: // %cond.load37
1134; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #13]
1135; NONEON-NOSVE-NEXT:    str q0, [sp, #1168]
1136; NONEON-NOSVE-NEXT:    strb w9, [sp, #1200]
1137; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #1182]
1138; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1200]
1139; NONEON-NOSVE-NEXT:    strh w9, [sp, #1198]
1140; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1180]
1141; NONEON-NOSVE-NEXT:    str q0, [sp, #1152]
1142; NONEON-NOSVE-NEXT:    strb w9, [sp, #1196]
1143; NONEON-NOSVE-NEXT:    ldr w9, [sp, #1176]
1144; NONEON-NOSVE-NEXT:    str w9, [sp, #1192]
1145; NONEON-NOSVE-NEXT:    ldr x9, [sp, #1168]
1146; NONEON-NOSVE-NEXT:    str x9, [sp, #1184]
1147; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1152]
1148; NONEON-NOSVE-NEXT:    strb w9, [sp, #1197]
1149; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1184]
1150; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB3_17
1151; NONEON-NOSVE-NEXT:  .LBB3_48: // %cond.load40
1152; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #14]
1153; NONEON-NOSVE-NEXT:    str q0, [sp, #1104]
1154; NONEON-NOSVE-NEXT:    strb w9, [sp, #1136]
1155; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1119]
1156; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1136]
1157; NONEON-NOSVE-NEXT:    strb w9, [sp, #1135]
1158; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #1116]
1159; NONEON-NOSVE-NEXT:    str q0, [sp, #1088]
1160; NONEON-NOSVE-NEXT:    strh w9, [sp, #1132]
1161; NONEON-NOSVE-NEXT:    ldr w9, [sp, #1112]
1162; NONEON-NOSVE-NEXT:    str w9, [sp, #1128]
1163; NONEON-NOSVE-NEXT:    ldr x9, [sp, #1104]
1164; NONEON-NOSVE-NEXT:    str x9, [sp, #1120]
1165; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1088]
1166; NONEON-NOSVE-NEXT:    strb w9, [sp, #1134]
1167; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1120]
1168; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB3_18
1169; NONEON-NOSVE-NEXT:  .LBB3_49: // %cond.load43
1170; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #15]
1171; NONEON-NOSVE-NEXT:    str q0, [sp, #1024]
1172; NONEON-NOSVE-NEXT:    strb w9, [sp, #1072]
1173; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1038]
1174; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1072]
1175; NONEON-NOSVE-NEXT:    strb w9, [sp, #1070]
1176; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #1036]
1177; NONEON-NOSVE-NEXT:    str q0, [sp, #1040]
1178; NONEON-NOSVE-NEXT:    strh w9, [sp, #1068]
1179; NONEON-NOSVE-NEXT:    ldr w9, [sp, #1032]
1180; NONEON-NOSVE-NEXT:    str w9, [sp, #1064]
1181; NONEON-NOSVE-NEXT:    ldr x9, [sp, #1024]
1182; NONEON-NOSVE-NEXT:    str x9, [sp, #1056]
1183; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #1040]
1184; NONEON-NOSVE-NEXT:    strb w9, [sp, #1071]
1185; NONEON-NOSVE-NEXT:    ldr q0, [sp, #1056]
1186; NONEON-NOSVE-NEXT:    tbz w8, #16, .LBB3_19
1187; NONEON-NOSVE-NEXT:  .LBB3_50: // %cond.load46
1188; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #16]
1189; NONEON-NOSVE-NEXT:    str q1, [sp, #976]
1190; NONEON-NOSVE-NEXT:    add x10, sp, #976
1191; NONEON-NOSVE-NEXT:    strb w9, [sp, #1008]
1192; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #991]
1193; NONEON-NOSVE-NEXT:    ldr q1, [sp, #1008]
1194; NONEON-NOSVE-NEXT:    strb w9, [sp, #1007]
1195; NONEON-NOSVE-NEXT:    str q1, [sp, #960]
1196; NONEON-NOSVE-NEXT:    ldurh w9, [x10, #13]
1197; NONEON-NOSVE-NEXT:    ldur w11, [x10, #9]
1198; NONEON-NOSVE-NEXT:    sturh w9, [x10, #29]
1199; NONEON-NOSVE-NEXT:    ldur x9, [x10, #1]
1200; NONEON-NOSVE-NEXT:    stur w11, [x10, #25]
1201; NONEON-NOSVE-NEXT:    stur x9, [x10, #17]
1202; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #960]
1203; NONEON-NOSVE-NEXT:    strb w9, [sp, #992]
1204; NONEON-NOSVE-NEXT:    ldr q1, [sp, #992]
1205; NONEON-NOSVE-NEXT:    add x9, sp, #720
1206; NONEON-NOSVE-NEXT:    tbz w8, #17, .LBB3_20
1207; NONEON-NOSVE-NEXT:  .LBB3_51: // %cond.load49
1208; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #17]
1209; NONEON-NOSVE-NEXT:    str q1, [sp, #912]
1210; NONEON-NOSVE-NEXT:    strb w10, [sp, #944]
1211; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #926]
1212; NONEON-NOSVE-NEXT:    ldr q1, [sp, #944]
1213; NONEON-NOSVE-NEXT:    strh w10, [sp, #942]
1214; NONEON-NOSVE-NEXT:    str q1, [sp, #896]
1215; NONEON-NOSVE-NEXT:    ldur w10, [x9, #202]
1216; NONEON-NOSVE-NEXT:    ldur x11, [x9, #194]
1217; NONEON-NOSVE-NEXT:    stur w10, [x9, #218]
1218; NONEON-NOSVE-NEXT:    stur x11, [x9, #210]
1219; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #912]
1220; NONEON-NOSVE-NEXT:    strb w10, [sp, #928]
1221; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #896]
1222; NONEON-NOSVE-NEXT:    strb w10, [sp, #929]
1223; NONEON-NOSVE-NEXT:    ldr q1, [sp, #928]
1224; NONEON-NOSVE-NEXT:    tbz w8, #18, .LBB3_21
1225; NONEON-NOSVE-NEXT:  .LBB3_52: // %cond.load52
1226; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #18]
1227; NONEON-NOSVE-NEXT:    str q1, [sp, #848]
1228; NONEON-NOSVE-NEXT:    strb w10, [sp, #880]
1229; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #863]
1230; NONEON-NOSVE-NEXT:    ldr q1, [sp, #880]
1231; NONEON-NOSVE-NEXT:    strb w10, [sp, #879]
1232; NONEON-NOSVE-NEXT:    str q1, [sp, #832]
1233; NONEON-NOSVE-NEXT:    ldur w10, [x9, #139]
1234; NONEON-NOSVE-NEXT:    ldur x11, [x9, #131]
1235; NONEON-NOSVE-NEXT:    stur w10, [x9, #155]
1236; NONEON-NOSVE-NEXT:    stur x11, [x9, #147]
1237; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #848]
1238; NONEON-NOSVE-NEXT:    strh w10, [sp, #864]
1239; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #832]
1240; NONEON-NOSVE-NEXT:    strb w10, [sp, #866]
1241; NONEON-NOSVE-NEXT:    ldr q1, [sp, #864]
1242; NONEON-NOSVE-NEXT:    tbz w8, #19, .LBB3_22
1243; NONEON-NOSVE-NEXT:  .LBB3_53: // %cond.load55
1244; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #19]
1245; NONEON-NOSVE-NEXT:    str q1, [sp, #784]
1246; NONEON-NOSVE-NEXT:    strb w10, [sp, #816]
1247; NONEON-NOSVE-NEXT:    ldr w10, [sp, #796]
1248; NONEON-NOSVE-NEXT:    ldr q1, [sp, #816]
1249; NONEON-NOSVE-NEXT:    str w10, [sp, #812]
1250; NONEON-NOSVE-NEXT:    str q1, [sp, #768]
1251; NONEON-NOSVE-NEXT:    ldur x10, [x9, #68]
1252; NONEON-NOSVE-NEXT:    stur x10, [x9, #84]
1253; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #786]
1254; NONEON-NOSVE-NEXT:    strb w10, [sp, #802]
1255; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #784]
1256; NONEON-NOSVE-NEXT:    strh w10, [sp, #800]
1257; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #768]
1258; NONEON-NOSVE-NEXT:    strb w10, [sp, #803]
1259; NONEON-NOSVE-NEXT:    ldr q1, [sp, #800]
1260; NONEON-NOSVE-NEXT:    tbz w8, #20, .LBB3_23
1261; NONEON-NOSVE-NEXT:  .LBB3_54: // %cond.load58
1262; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #20]
1263; NONEON-NOSVE-NEXT:    str q1, [sp, #720]
1264; NONEON-NOSVE-NEXT:    strb w10, [sp, #752]
1265; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #735]
1266; NONEON-NOSVE-NEXT:    ldr q1, [sp, #752]
1267; NONEON-NOSVE-NEXT:    strb w10, [sp, #751]
1268; NONEON-NOSVE-NEXT:    str q1, [sp, #704]
1269; NONEON-NOSVE-NEXT:    ldurh w10, [x9, #13]
1270; NONEON-NOSVE-NEXT:    ldur x11, [x9, #5]
1271; NONEON-NOSVE-NEXT:    sturh w10, [x9, #29]
1272; NONEON-NOSVE-NEXT:    stur x11, [x9, #21]
1273; NONEON-NOSVE-NEXT:    ldr w9, [sp, #720]
1274; NONEON-NOSVE-NEXT:    str w9, [sp, #736]
1275; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #704]
1276; NONEON-NOSVE-NEXT:    strb w9, [sp, #740]
1277; NONEON-NOSVE-NEXT:    ldr q1, [sp, #736]
1278; NONEON-NOSVE-NEXT:    add x9, sp, #464
1279; NONEON-NOSVE-NEXT:    tbz w8, #21, .LBB3_24
1280; NONEON-NOSVE-NEXT:  .LBB3_55: // %cond.load61
1281; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #21]
1282; NONEON-NOSVE-NEXT:    str q1, [sp, #656]
1283; NONEON-NOSVE-NEXT:    strb w10, [sp, #688]
1284; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #670]
1285; NONEON-NOSVE-NEXT:    ldr q1, [sp, #688]
1286; NONEON-NOSVE-NEXT:    strh w10, [sp, #686]
1287; NONEON-NOSVE-NEXT:    str q1, [sp, #640]
1288; NONEON-NOSVE-NEXT:    ldur x10, [x9, #198]
1289; NONEON-NOSVE-NEXT:    stur x10, [x9, #214]
1290; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #660]
1291; NONEON-NOSVE-NEXT:    strb w10, [sp, #676]
1292; NONEON-NOSVE-NEXT:    ldr w10, [sp, #656]
1293; NONEON-NOSVE-NEXT:    str w10, [sp, #672]
1294; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #640]
1295; NONEON-NOSVE-NEXT:    strb w10, [sp, #677]
1296; NONEON-NOSVE-NEXT:    ldr q1, [sp, #672]
1297; NONEON-NOSVE-NEXT:    tbz w8, #22, .LBB3_25
1298; NONEON-NOSVE-NEXT:  .LBB3_56: // %cond.load64
1299; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #22]
1300; NONEON-NOSVE-NEXT:    str q1, [sp, #592]
1301; NONEON-NOSVE-NEXT:    strb w10, [sp, #624]
1302; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #607]
1303; NONEON-NOSVE-NEXT:    ldr q1, [sp, #624]
1304; NONEON-NOSVE-NEXT:    strb w10, [sp, #623]
1305; NONEON-NOSVE-NEXT:    str q1, [sp, #576]
1306; NONEON-NOSVE-NEXT:    ldur x10, [x9, #135]
1307; NONEON-NOSVE-NEXT:    stur x10, [x9, #151]
1308; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #596]
1309; NONEON-NOSVE-NEXT:    strh w10, [sp, #612]
1310; NONEON-NOSVE-NEXT:    ldr w10, [sp, #592]
1311; NONEON-NOSVE-NEXT:    str w10, [sp, #608]
1312; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #576]
1313; NONEON-NOSVE-NEXT:    strb w10, [sp, #614]
1314; NONEON-NOSVE-NEXT:    ldr q1, [sp, #608]
1315; NONEON-NOSVE-NEXT:    tbz w8, #23, .LBB3_26
1316; NONEON-NOSVE-NEXT:  .LBB3_57: // %cond.load67
1317; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #23]
1318; NONEON-NOSVE-NEXT:    str q1, [sp, #528]
1319; NONEON-NOSVE-NEXT:    strb w10, [sp, #560]
1320; NONEON-NOSVE-NEXT:    ldr x10, [sp, #536]
1321; NONEON-NOSVE-NEXT:    ldr q1, [sp, #560]
1322; NONEON-NOSVE-NEXT:    str x10, [sp, #552]
1323; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #534]
1324; NONEON-NOSVE-NEXT:    str q1, [sp, #512]
1325; NONEON-NOSVE-NEXT:    strb w10, [sp, #550]
1326; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #532]
1327; NONEON-NOSVE-NEXT:    strh w10, [sp, #548]
1328; NONEON-NOSVE-NEXT:    ldr w10, [sp, #528]
1329; NONEON-NOSVE-NEXT:    str w10, [sp, #544]
1330; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #512]
1331; NONEON-NOSVE-NEXT:    strb w10, [sp, #551]
1332; NONEON-NOSVE-NEXT:    ldr q1, [sp, #544]
1333; NONEON-NOSVE-NEXT:    tbz w8, #24, .LBB3_27
1334; NONEON-NOSVE-NEXT:  .LBB3_58: // %cond.load70
1335; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #24]
1336; NONEON-NOSVE-NEXT:    str q1, [sp, #464]
1337; NONEON-NOSVE-NEXT:    strb w10, [sp, #496]
1338; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #479]
1339; NONEON-NOSVE-NEXT:    ldr q1, [sp, #496]
1340; NONEON-NOSVE-NEXT:    strb w10, [sp, #495]
1341; NONEON-NOSVE-NEXT:    str q1, [sp, #448]
1342; NONEON-NOSVE-NEXT:    ldurh w10, [x9, #13]
1343; NONEON-NOSVE-NEXT:    ldur w11, [x9, #9]
1344; NONEON-NOSVE-NEXT:    sturh w10, [x9, #29]
1345; NONEON-NOSVE-NEXT:    stur w11, [x9, #25]
1346; NONEON-NOSVE-NEXT:    ldr x9, [sp, #464]
1347; NONEON-NOSVE-NEXT:    str x9, [sp, #480]
1348; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #448]
1349; NONEON-NOSVE-NEXT:    strb w9, [sp, #488]
1350; NONEON-NOSVE-NEXT:    ldr q1, [sp, #480]
1351; NONEON-NOSVE-NEXT:    add x9, sp, #208
1352; NONEON-NOSVE-NEXT:    tbz w8, #25, .LBB3_28
1353; NONEON-NOSVE-NEXT:  .LBB3_59: // %cond.load73
1354; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #25]
1355; NONEON-NOSVE-NEXT:    str q1, [sp, #400]
1356; NONEON-NOSVE-NEXT:    strb w10, [sp, #432]
1357; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #414]
1358; NONEON-NOSVE-NEXT:    ldr q1, [sp, #432]
1359; NONEON-NOSVE-NEXT:    strh w10, [sp, #430]
1360; NONEON-NOSVE-NEXT:    str q1, [sp, #384]
1361; NONEON-NOSVE-NEXT:    ldur w10, [x9, #202]
1362; NONEON-NOSVE-NEXT:    stur w10, [x9, #218]
1363; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #408]
1364; NONEON-NOSVE-NEXT:    strb w10, [sp, #424]
1365; NONEON-NOSVE-NEXT:    ldr x10, [sp, #400]
1366; NONEON-NOSVE-NEXT:    str x10, [sp, #416]
1367; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #384]
1368; NONEON-NOSVE-NEXT:    strb w10, [sp, #425]
1369; NONEON-NOSVE-NEXT:    ldr q1, [sp, #416]
1370; NONEON-NOSVE-NEXT:    tbz w8, #26, .LBB3_29
1371; NONEON-NOSVE-NEXT:  .LBB3_60: // %cond.load76
1372; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #26]
1373; NONEON-NOSVE-NEXT:    str q1, [sp, #336]
1374; NONEON-NOSVE-NEXT:    strb w10, [sp, #368]
1375; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #351]
1376; NONEON-NOSVE-NEXT:    ldr q1, [sp, #368]
1377; NONEON-NOSVE-NEXT:    strb w10, [sp, #367]
1378; NONEON-NOSVE-NEXT:    str q1, [sp, #320]
1379; NONEON-NOSVE-NEXT:    ldur w10, [x9, #139]
1380; NONEON-NOSVE-NEXT:    stur w10, [x9, #155]
1381; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #344]
1382; NONEON-NOSVE-NEXT:    strh w10, [sp, #360]
1383; NONEON-NOSVE-NEXT:    ldr x10, [sp, #336]
1384; NONEON-NOSVE-NEXT:    str x10, [sp, #352]
1385; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #320]
1386; NONEON-NOSVE-NEXT:    strb w10, [sp, #362]
1387; NONEON-NOSVE-NEXT:    ldr q1, [sp, #352]
1388; NONEON-NOSVE-NEXT:    tbz w8, #27, .LBB3_30
1389; NONEON-NOSVE-NEXT:  .LBB3_61: // %cond.load79
1390; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #27]
1391; NONEON-NOSVE-NEXT:    str q1, [sp, #272]
1392; NONEON-NOSVE-NEXT:    strb w10, [sp, #304]
1393; NONEON-NOSVE-NEXT:    ldr w10, [sp, #284]
1394; NONEON-NOSVE-NEXT:    ldr q1, [sp, #304]
1395; NONEON-NOSVE-NEXT:    str w10, [sp, #300]
1396; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #282]
1397; NONEON-NOSVE-NEXT:    str q1, [sp, #256]
1398; NONEON-NOSVE-NEXT:    strb w10, [sp, #298]
1399; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #280]
1400; NONEON-NOSVE-NEXT:    strh w10, [sp, #296]
1401; NONEON-NOSVE-NEXT:    ldr x10, [sp, #272]
1402; NONEON-NOSVE-NEXT:    str x10, [sp, #288]
1403; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #256]
1404; NONEON-NOSVE-NEXT:    strb w10, [sp, #299]
1405; NONEON-NOSVE-NEXT:    ldr q1, [sp, #288]
1406; NONEON-NOSVE-NEXT:    tbz w8, #28, .LBB3_31
1407; NONEON-NOSVE-NEXT:  .LBB3_62: // %cond.load82
1408; NONEON-NOSVE-NEXT:    ldrb w10, [x0, #28]
1409; NONEON-NOSVE-NEXT:    str q1, [sp, #208]
1410; NONEON-NOSVE-NEXT:    strb w10, [sp, #240]
1411; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #223]
1412; NONEON-NOSVE-NEXT:    ldr q1, [sp, #240]
1413; NONEON-NOSVE-NEXT:    strb w10, [sp, #239]
1414; NONEON-NOSVE-NEXT:    str q1, [sp, #192]
1415; NONEON-NOSVE-NEXT:    ldurh w10, [x9, #13]
1416; NONEON-NOSVE-NEXT:    sturh w10, [x9, #29]
1417; NONEON-NOSVE-NEXT:    ldr w9, [sp, #216]
1418; NONEON-NOSVE-NEXT:    str w9, [sp, #232]
1419; NONEON-NOSVE-NEXT:    ldr x9, [sp, #208]
1420; NONEON-NOSVE-NEXT:    str x9, [sp, #224]
1421; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #192]
1422; NONEON-NOSVE-NEXT:    strb w9, [sp, #236]
1423; NONEON-NOSVE-NEXT:    ldr q1, [sp, #224]
1424; NONEON-NOSVE-NEXT:    tbz w8, #29, .LBB3_32
1425; NONEON-NOSVE-NEXT:  .LBB3_63: // %cond.load85
1426; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #29]
1427; NONEON-NOSVE-NEXT:    str q1, [sp, #144]
1428; NONEON-NOSVE-NEXT:    strb w9, [sp, #176]
1429; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #158]
1430; NONEON-NOSVE-NEXT:    ldr q1, [sp, #176]
1431; NONEON-NOSVE-NEXT:    strh w9, [sp, #174]
1432; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #156]
1433; NONEON-NOSVE-NEXT:    str q1, [sp, #128]
1434; NONEON-NOSVE-NEXT:    strb w9, [sp, #172]
1435; NONEON-NOSVE-NEXT:    ldr w9, [sp, #152]
1436; NONEON-NOSVE-NEXT:    str w9, [sp, #168]
1437; NONEON-NOSVE-NEXT:    ldr x9, [sp, #144]
1438; NONEON-NOSVE-NEXT:    str x9, [sp, #160]
1439; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #128]
1440; NONEON-NOSVE-NEXT:    strb w9, [sp, #173]
1441; NONEON-NOSVE-NEXT:    ldr q1, [sp, #160]
1442; NONEON-NOSVE-NEXT:    tbz w8, #30, .LBB3_33
1443; NONEON-NOSVE-NEXT:  .LBB3_64: // %cond.load88
1444; NONEON-NOSVE-NEXT:    ldrb w9, [x0, #30]
1445; NONEON-NOSVE-NEXT:    str q1, [sp, #80]
1446; NONEON-NOSVE-NEXT:    strb w9, [sp, #112]
1447; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #95]
1448; NONEON-NOSVE-NEXT:    ldr q1, [sp, #112]
1449; NONEON-NOSVE-NEXT:    strb w9, [sp, #111]
1450; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #92]
1451; NONEON-NOSVE-NEXT:    str q1, [sp, #64]
1452; NONEON-NOSVE-NEXT:    strh w9, [sp, #108]
1453; NONEON-NOSVE-NEXT:    ldr w9, [sp, #88]
1454; NONEON-NOSVE-NEXT:    str w9, [sp, #104]
1455; NONEON-NOSVE-NEXT:    ldr x9, [sp, #80]
1456; NONEON-NOSVE-NEXT:    str x9, [sp, #96]
1457; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #64]
1458; NONEON-NOSVE-NEXT:    strb w9, [sp, #110]
1459; NONEON-NOSVE-NEXT:    ldr q1, [sp, #96]
1460; NONEON-NOSVE-NEXT:    tbnz w8, #31, .LBB3_34
1461; NONEON-NOSVE-NEXT:    b .LBB3_35
1462  %load = call <32 x i8> @llvm.masked.load.v32i8(ptr %src, i32 8, <32 x i1> %mask, <32 x i8> zeroinitializer)
1463  ret <32 x i8> %load
1464}
1465
1466define <2 x half> @masked_load_v2f16(ptr %src, <2 x i1> %mask) {
1467; CHECK-LABEL: masked_load_v2f16:
1468; CHECK:       // %bb.0:
1469; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1470; CHECK-NEXT:    fmov s1, wzr
1471; CHECK-NEXT:    mov z2.s, z0.s[1]
1472; CHECK-NEXT:    ptrue p0.h, vl4
1473; CHECK-NEXT:    zip1 z0.h, z0.h, z2.h
1474; CHECK-NEXT:    zip1 z1.h, z1.h, z1.h
1475; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
1476; CHECK-NEXT:    lsl z0.h, z0.h, #15
1477; CHECK-NEXT:    asr z0.h, z0.h, #15
1478; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
1479; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
1480; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
1481; CHECK-NEXT:    ret
1482;
1483; NONEON-NOSVE-LABEL: masked_load_v2f16:
1484; NONEON-NOSVE:       // %bb.0:
1485; NONEON-NOSVE-NEXT:    sub sp, sp, #48
1486; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 48
1487; NONEON-NOSVE-NEXT:    str d0, [sp, #32]
1488; NONEON-NOSVE-NEXT:    ldr w8, [sp, #36]
1489; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #32]
1490; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
1491; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
1492; NONEON-NOSVE-NEXT:    bfxil w8, w9, #0, #1
1493; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB4_2
1494; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
1495; NONEON-NOSVE-NEXT:    fmov s0, wzr
1496; NONEON-NOSVE-NEXT:    ldr h1, [x0]
1497; NONEON-NOSVE-NEXT:    str h1, [sp, #24]
1498; NONEON-NOSVE-NEXT:    str h0, [sp, #26]
1499; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
1500; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB4_3
1501; NONEON-NOSVE-NEXT:    b .LBB4_4
1502; NONEON-NOSVE-NEXT:  .LBB4_2:
1503; NONEON-NOSVE-NEXT:    adrp x9, .LCPI4_0
1504; NONEON-NOSVE-NEXT:    ldr d0, [x9, :lo12:.LCPI4_0]
1505; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB4_4
1506; NONEON-NOSVE-NEXT:  .LBB4_3: // %cond.load1
1507; NONEON-NOSVE-NEXT:    ldr h1, [x0, #2]
1508; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
1509; NONEON-NOSVE-NEXT:    ldr h0, [sp, #8]
1510; NONEON-NOSVE-NEXT:    str h1, [sp, #18]
1511; NONEON-NOSVE-NEXT:    str h0, [sp, #16]
1512; NONEON-NOSVE-NEXT:    ldr d0, [sp, #16]
1513; NONEON-NOSVE-NEXT:  .LBB4_4: // %else2
1514; NONEON-NOSVE-NEXT:    add sp, sp, #48
1515; NONEON-NOSVE-NEXT:    ret
1516  %load = call <2 x half> @llvm.masked.load.v2f16(ptr %src, i32 8, <2 x i1> %mask, <2 x half> zeroinitializer)
1517  ret <2 x half> %load
1518}
1519
1520define <4 x half> @masked_load_v4f16(ptr %src, <4 x i1> %mask) {
1521; CHECK-LABEL: masked_load_v4f16:
1522; CHECK:       // %bb.0:
1523; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1524; CHECK-NEXT:    ptrue p0.h, vl4
1525; CHECK-NEXT:    lsl z0.h, z0.h, #15
1526; CHECK-NEXT:    asr z0.h, z0.h, #15
1527; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
1528; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
1529; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
1530; CHECK-NEXT:    ret
1531;
1532; NONEON-NOSVE-LABEL: masked_load_v4f16:
1533; NONEON-NOSVE:       // %bb.0:
1534; NONEON-NOSVE-NEXT:    sub sp, sp, #128
1535; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 128
1536; NONEON-NOSVE-NEXT:    str d0, [sp, #112]
1537; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #114]
1538; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #116]
1539; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #118]
1540; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #112]
1541; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
1542; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
1543; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
1544; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
1545; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
1546; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
1547; NONEON-NOSVE-NEXT:    bfxil w8, w11, #0, #1
1548; NONEON-NOSVE-NEXT:    orr w9, w9, w10
1549; NONEON-NOSVE-NEXT:    orr w8, w8, w9
1550; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB5_2
1551; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
1552; NONEON-NOSVE-NEXT:    fmov s0, wzr
1553; NONEON-NOSVE-NEXT:    ldr h1, [x0]
1554; NONEON-NOSVE-NEXT:    stur wzr, [sp, #106]
1555; NONEON-NOSVE-NEXT:    str h1, [sp, #104]
1556; NONEON-NOSVE-NEXT:    str h0, [sp, #110]
1557; NONEON-NOSVE-NEXT:    ldr d0, [sp, #104]
1558; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB5_3
1559; NONEON-NOSVE-NEXT:    b .LBB5_4
1560; NONEON-NOSVE-NEXT:  .LBB5_2:
1561; NONEON-NOSVE-NEXT:    adrp x9, .LCPI5_0
1562; NONEON-NOSVE-NEXT:    ldr d0, [x9, :lo12:.LCPI5_0]
1563; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB5_4
1564; NONEON-NOSVE-NEXT:  .LBB5_3: // %cond.load1
1565; NONEON-NOSVE-NEXT:    ldr h1, [x0, #2]
1566; NONEON-NOSVE-NEXT:    str d0, [sp, #80]
1567; NONEON-NOSVE-NEXT:    ldr w9, [sp, #84]
1568; NONEON-NOSVE-NEXT:    str h1, [sp, #96]
1569; NONEON-NOSVE-NEXT:    ldr d0, [sp, #96]
1570; NONEON-NOSVE-NEXT:    str w9, [sp, #92]
1571; NONEON-NOSVE-NEXT:    str d0, [sp, #72]
1572; NONEON-NOSVE-NEXT:    ldr h0, [sp, #80]
1573; NONEON-NOSVE-NEXT:    str h0, [sp, #88]
1574; NONEON-NOSVE-NEXT:    ldr h0, [sp, #72]
1575; NONEON-NOSVE-NEXT:    str h0, [sp, #90]
1576; NONEON-NOSVE-NEXT:    ldr d0, [sp, #88]
1577; NONEON-NOSVE-NEXT:  .LBB5_4: // %else2
1578; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB5_7
1579; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
1580; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB5_8
1581; NONEON-NOSVE-NEXT:  .LBB5_6: // %else8
1582; NONEON-NOSVE-NEXT:    add sp, sp, #128
1583; NONEON-NOSVE-NEXT:    ret
1584; NONEON-NOSVE-NEXT:  .LBB5_7: // %cond.load4
1585; NONEON-NOSVE-NEXT:    ldr h1, [x0, #4]
1586; NONEON-NOSVE-NEXT:    str d0, [sp, #48]
1587; NONEON-NOSVE-NEXT:    ldr w9, [sp, #48]
1588; NONEON-NOSVE-NEXT:    str h1, [sp, #64]
1589; NONEON-NOSVE-NEXT:    ldr d0, [sp, #64]
1590; NONEON-NOSVE-NEXT:    str w9, [sp, #56]
1591; NONEON-NOSVE-NEXT:    str d0, [sp, #40]
1592; NONEON-NOSVE-NEXT:    ldr h0, [sp, #54]
1593; NONEON-NOSVE-NEXT:    str h0, [sp, #62]
1594; NONEON-NOSVE-NEXT:    ldr h0, [sp, #40]
1595; NONEON-NOSVE-NEXT:    str h0, [sp, #60]
1596; NONEON-NOSVE-NEXT:    ldr d0, [sp, #56]
1597; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB5_6
1598; NONEON-NOSVE-NEXT:  .LBB5_8: // %cond.load7
1599; NONEON-NOSVE-NEXT:    ldr h1, [x0, #6]
1600; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
1601; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
1602; NONEON-NOSVE-NEXT:    str h1, [sp, #32]
1603; NONEON-NOSVE-NEXT:    ldr d0, [sp, #32]
1604; NONEON-NOSVE-NEXT:    str w8, [sp, #24]
1605; NONEON-NOSVE-NEXT:    str d0, [sp, #16]
1606; NONEON-NOSVE-NEXT:    ldr h0, [sp, #12]
1607; NONEON-NOSVE-NEXT:    str h0, [sp, #28]
1608; NONEON-NOSVE-NEXT:    ldr h0, [sp, #16]
1609; NONEON-NOSVE-NEXT:    str h0, [sp, #30]
1610; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
1611; NONEON-NOSVE-NEXT:    add sp, sp, #128
1612; NONEON-NOSVE-NEXT:    ret
1613  %load = call <4 x half> @llvm.masked.load.v4f16(ptr %src, i32 8, <4 x i1> %mask, <4 x half> zeroinitializer)
1614  ret <4 x half> %load
1615}
1616
1617define <8 x half> @masked_load_v8f16(ptr %src, <8 x i1> %mask) {
1618; CHECK-LABEL: masked_load_v8f16:
1619; CHECK:       // %bb.0:
1620; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
1621; CHECK-NEXT:    ptrue p0.h, vl8
1622; CHECK-NEXT:    uunpklo z0.h, z0.b
1623; CHECK-NEXT:    lsl z0.h, z0.h, #15
1624; CHECK-NEXT:    asr z0.h, z0.h, #15
1625; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
1626; CHECK-NEXT:    ld1h { z0.h }, p0/z, [x0]
1627; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
1628; CHECK-NEXT:    ret
1629;
1630; NONEON-NOSVE-LABEL: masked_load_v8f16:
1631; NONEON-NOSVE:       // %bb.0:
1632; NONEON-NOSVE-NEXT:    sub sp, sp, #496
1633; NONEON-NOSVE-NEXT:    str x29, [sp, #480] // 8-byte Folded Spill
1634; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 496
1635; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
1636; NONEON-NOSVE-NEXT:    str d0, [sp, #464]
1637; NONEON-NOSVE-NEXT:    add x9, sp, #208
1638; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #466]
1639; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #467]
1640; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #465]
1641; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #468]
1642; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #469]
1643; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #470]
1644; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
1645; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
1646; NONEON-NOSVE-NEXT:    sbfx w11, w11, #0, #1
1647; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #464]
1648; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
1649; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
1650; NONEON-NOSVE-NEXT:    and w8, w8, #0x4
1651; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
1652; NONEON-NOSVE-NEXT:    sbfx w15, w15, #0, #1
1653; NONEON-NOSVE-NEXT:    orr w8, w8, w10
1654; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #471]
1655; NONEON-NOSVE-NEXT:    and w11, w11, #0x2
1656; NONEON-NOSVE-NEXT:    and w13, w13, #0x10
1657; NONEON-NOSVE-NEXT:    bfxil w11, w12, #0, #1
1658; NONEON-NOSVE-NEXT:    and w12, w14, #0x20
1659; NONEON-NOSVE-NEXT:    orr w8, w8, w13
1660; NONEON-NOSVE-NEXT:    and w13, w15, #0x40
1661; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
1662; NONEON-NOSVE-NEXT:    orr w8, w11, w8
1663; NONEON-NOSVE-NEXT:    orr w11, w12, w13
1664; NONEON-NOSVE-NEXT:    orr w8, w8, w11
1665; NONEON-NOSVE-NEXT:    and w10, w10, #0x80
1666; NONEON-NOSVE-NEXT:    add w10, w8, w10
1667; NONEON-NOSVE-NEXT:    and w8, w10, #0xff
1668; NONEON-NOSVE-NEXT:    tbz w10, #0, .LBB6_2
1669; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
1670; NONEON-NOSVE-NEXT:    fmov s0, wzr
1671; NONEON-NOSVE-NEXT:    ldr h1, [x0]
1672; NONEON-NOSVE-NEXT:    stur wzr, [x9, #250]
1673; NONEON-NOSVE-NEXT:    stur xzr, [x9, #242]
1674; NONEON-NOSVE-NEXT:    str h1, [sp, #448]
1675; NONEON-NOSVE-NEXT:    str h0, [sp, #462]
1676; NONEON-NOSVE-NEXT:    ldr q0, [sp, #448]
1677; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB6_3
1678; NONEON-NOSVE-NEXT:    b .LBB6_4
1679; NONEON-NOSVE-NEXT:  .LBB6_2:
1680; NONEON-NOSVE-NEXT:    adrp x10, .LCPI6_0
1681; NONEON-NOSVE-NEXT:    ldr q0, [x10, :lo12:.LCPI6_0]
1682; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB6_4
1683; NONEON-NOSVE-NEXT:  .LBB6_3: // %cond.load1
1684; NONEON-NOSVE-NEXT:    ldr h1, [x0, #2]
1685; NONEON-NOSVE-NEXT:    str q0, [sp, #400]
1686; NONEON-NOSVE-NEXT:    ldr w10, [sp, #412]
1687; NONEON-NOSVE-NEXT:    str h1, [sp, #432]
1688; NONEON-NOSVE-NEXT:    ldr q0, [sp, #432]
1689; NONEON-NOSVE-NEXT:    str w10, [sp, #428]
1690; NONEON-NOSVE-NEXT:    str q0, [sp, #384]
1691; NONEON-NOSVE-NEXT:    ldur x10, [x9, #196]
1692; NONEON-NOSVE-NEXT:    stur x10, [x9, #212]
1693; NONEON-NOSVE-NEXT:    ldr h0, [sp, #400]
1694; NONEON-NOSVE-NEXT:    str h0, [sp, #416]
1695; NONEON-NOSVE-NEXT:    ldr h0, [sp, #384]
1696; NONEON-NOSVE-NEXT:    str h0, [sp, #418]
1697; NONEON-NOSVE-NEXT:    ldr q0, [sp, #416]
1698; NONEON-NOSVE-NEXT:  .LBB6_4: // %else2
1699; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB6_12
1700; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
1701; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB6_13
1702; NONEON-NOSVE-NEXT:  .LBB6_6: // %else8
1703; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB6_14
1704; NONEON-NOSVE-NEXT:  .LBB6_7: // %else11
1705; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB6_15
1706; NONEON-NOSVE-NEXT:  .LBB6_8: // %else14
1707; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB6_16
1708; NONEON-NOSVE-NEXT:  .LBB6_9: // %else17
1709; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB6_11
1710; NONEON-NOSVE-NEXT:  .LBB6_10: // %cond.load19
1711; NONEON-NOSVE-NEXT:    ldr h1, [x0, #14]
1712; NONEON-NOSVE-NEXT:    str q0, [sp]
1713; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
1714; NONEON-NOSVE-NEXT:    str h1, [sp, #48]
1715; NONEON-NOSVE-NEXT:    ldr q0, [sp, #48]
1716; NONEON-NOSVE-NEXT:    str w8, [sp, #40]
1717; NONEON-NOSVE-NEXT:    ldr x8, [sp]
1718; NONEON-NOSVE-NEXT:    str q0, [sp, #16]
1719; NONEON-NOSVE-NEXT:    ldr h0, [sp, #12]
1720; NONEON-NOSVE-NEXT:    str x8, [sp, #32]
1721; NONEON-NOSVE-NEXT:    str h0, [sp, #44]
1722; NONEON-NOSVE-NEXT:    ldr h0, [sp, #16]
1723; NONEON-NOSVE-NEXT:    str h0, [sp, #46]
1724; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
1725; NONEON-NOSVE-NEXT:  .LBB6_11: // %else20
1726; NONEON-NOSVE-NEXT:    ldr x29, [sp, #480] // 8-byte Folded Reload
1727; NONEON-NOSVE-NEXT:    add sp, sp, #496
1728; NONEON-NOSVE-NEXT:    ret
1729; NONEON-NOSVE-NEXT:  .LBB6_12: // %cond.load4
1730; NONEON-NOSVE-NEXT:    ldr h1, [x0, #4]
1731; NONEON-NOSVE-NEXT:    str q0, [sp, #336]
1732; NONEON-NOSVE-NEXT:    str h1, [sp, #368]
1733; NONEON-NOSVE-NEXT:    ldr q0, [sp, #368]
1734; NONEON-NOSVE-NEXT:    str q0, [sp, #320]
1735; NONEON-NOSVE-NEXT:    ldr h0, [sp, #350]
1736; NONEON-NOSVE-NEXT:    str h0, [sp, #366]
1737; NONEON-NOSVE-NEXT:    ldur x10, [x9, #134]
1738; NONEON-NOSVE-NEXT:    stur x10, [x9, #150]
1739; NONEON-NOSVE-NEXT:    ldr w10, [sp, #336]
1740; NONEON-NOSVE-NEXT:    ldr h0, [sp, #320]
1741; NONEON-NOSVE-NEXT:    str w10, [sp, #352]
1742; NONEON-NOSVE-NEXT:    str h0, [sp, #356]
1743; NONEON-NOSVE-NEXT:    ldr q0, [sp, #352]
1744; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB6_6
1745; NONEON-NOSVE-NEXT:  .LBB6_13: // %cond.load7
1746; NONEON-NOSVE-NEXT:    ldr h1, [x0, #6]
1747; NONEON-NOSVE-NEXT:    str q0, [sp, #272]
1748; NONEON-NOSVE-NEXT:    ldr x10, [sp, #280]
1749; NONEON-NOSVE-NEXT:    str h1, [sp, #304]
1750; NONEON-NOSVE-NEXT:    ldr q0, [sp, #304]
1751; NONEON-NOSVE-NEXT:    str x10, [sp, #296]
1752; NONEON-NOSVE-NEXT:    ldr w10, [sp, #272]
1753; NONEON-NOSVE-NEXT:    str q0, [sp, #256]
1754; NONEON-NOSVE-NEXT:    ldr h0, [sp, #276]
1755; NONEON-NOSVE-NEXT:    str w10, [sp, #288]
1756; NONEON-NOSVE-NEXT:    str h0, [sp, #292]
1757; NONEON-NOSVE-NEXT:    ldr h0, [sp, #256]
1758; NONEON-NOSVE-NEXT:    str h0, [sp, #294]
1759; NONEON-NOSVE-NEXT:    ldr q0, [sp, #288]
1760; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB6_7
1761; NONEON-NOSVE-NEXT:  .LBB6_14: // %cond.load10
1762; NONEON-NOSVE-NEXT:    ldr h1, [x0, #8]
1763; NONEON-NOSVE-NEXT:    str q0, [sp, #208]
1764; NONEON-NOSVE-NEXT:    str h1, [sp, #240]
1765; NONEON-NOSVE-NEXT:    ldr q0, [sp, #240]
1766; NONEON-NOSVE-NEXT:    str q0, [sp, #192]
1767; NONEON-NOSVE-NEXT:    ldr h0, [sp, #222]
1768; NONEON-NOSVE-NEXT:    str h0, [sp, #238]
1769; NONEON-NOSVE-NEXT:    ldur w10, [x9, #10]
1770; NONEON-NOSVE-NEXT:    stur w10, [x9, #26]
1771; NONEON-NOSVE-NEXT:    ldr x9, [sp, #208]
1772; NONEON-NOSVE-NEXT:    ldr h0, [sp, #192]
1773; NONEON-NOSVE-NEXT:    str x9, [sp, #224]
1774; NONEON-NOSVE-NEXT:    str h0, [sp, #232]
1775; NONEON-NOSVE-NEXT:    ldr q0, [sp, #224]
1776; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB6_8
1777; NONEON-NOSVE-NEXT:  .LBB6_15: // %cond.load13
1778; NONEON-NOSVE-NEXT:    ldr h1, [x0, #10]
1779; NONEON-NOSVE-NEXT:    str q0, [sp, #144]
1780; NONEON-NOSVE-NEXT:    ldr w9, [sp, #156]
1781; NONEON-NOSVE-NEXT:    str h1, [sp, #176]
1782; NONEON-NOSVE-NEXT:    ldr q0, [sp, #176]
1783; NONEON-NOSVE-NEXT:    str w9, [sp, #172]
1784; NONEON-NOSVE-NEXT:    ldr x9, [sp, #144]
1785; NONEON-NOSVE-NEXT:    str q0, [sp, #128]
1786; NONEON-NOSVE-NEXT:    ldr h0, [sp, #152]
1787; NONEON-NOSVE-NEXT:    str x9, [sp, #160]
1788; NONEON-NOSVE-NEXT:    str h0, [sp, #168]
1789; NONEON-NOSVE-NEXT:    ldr h0, [sp, #128]
1790; NONEON-NOSVE-NEXT:    str h0, [sp, #170]
1791; NONEON-NOSVE-NEXT:    ldr q0, [sp, #160]
1792; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB6_9
1793; NONEON-NOSVE-NEXT:  .LBB6_16: // %cond.load16
1794; NONEON-NOSVE-NEXT:    ldr h1, [x0, #12]
1795; NONEON-NOSVE-NEXT:    str q0, [sp, #80]
1796; NONEON-NOSVE-NEXT:    ldr w9, [sp, #88]
1797; NONEON-NOSVE-NEXT:    str h1, [sp, #112]
1798; NONEON-NOSVE-NEXT:    ldr q0, [sp, #112]
1799; NONEON-NOSVE-NEXT:    str w9, [sp, #104]
1800; NONEON-NOSVE-NEXT:    ldr x9, [sp, #80]
1801; NONEON-NOSVE-NEXT:    str q0, [sp, #64]
1802; NONEON-NOSVE-NEXT:    ldr h0, [sp, #94]
1803; NONEON-NOSVE-NEXT:    str x9, [sp, #96]
1804; NONEON-NOSVE-NEXT:    str h0, [sp, #110]
1805; NONEON-NOSVE-NEXT:    ldr h0, [sp, #64]
1806; NONEON-NOSVE-NEXT:    str h0, [sp, #108]
1807; NONEON-NOSVE-NEXT:    ldr q0, [sp, #96]
1808; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB6_10
1809; NONEON-NOSVE-NEXT:    b .LBB6_11
1810  %load = call <8 x half> @llvm.masked.load.v8f16(ptr %src, i32 8, <8 x i1> %mask, <8 x half> zeroinitializer)
1811  ret <8 x half> %load
1812}
1813
1814define <16 x half> @masked_load_v16f16(ptr %src, <16 x i1> %mask) {
1815; CHECK-LABEL: masked_load_v16f16:
1816; CHECK:       // %bb.0:
1817; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
1818; CHECK-NEXT:    uunpklo z1.h, z0.b
1819; CHECK-NEXT:    ptrue p0.h, vl8
1820; CHECK-NEXT:    mov x8, #8 // =0x8
1821; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
1822; CHECK-NEXT:    uunpklo z0.h, z0.b
1823; CHECK-NEXT:    lsl z1.h, z1.h, #15
1824; CHECK-NEXT:    asr z1.h, z1.h, #15
1825; CHECK-NEXT:    lsl z0.h, z0.h, #15
1826; CHECK-NEXT:    cmpne p1.h, p0/z, z1.h, #0
1827; CHECK-NEXT:    asr z0.h, z0.h, #15
1828; CHECK-NEXT:    cmpne p0.h, p0/z, z0.h, #0
1829; CHECK-NEXT:    ld1h { z0.h }, p1/z, [x0]
1830; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
1831; CHECK-NEXT:    ld1h { z1.h }, p0/z, [x0, x8, lsl #1]
1832; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
1833; CHECK-NEXT:    ret
1834;
1835; NONEON-NOSVE-LABEL: masked_load_v16f16:
1836; NONEON-NOSVE:       // %bb.0:
1837; NONEON-NOSVE-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
1838; NONEON-NOSVE-NEXT:    sub sp, sp, #992
1839; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 1008
1840; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
1841; NONEON-NOSVE-NEXT:    str q0, [sp, #976]
1842; NONEON-NOSVE-NEXT:    adrp x9, .LCPI7_0
1843; NONEON-NOSVE-NEXT:    ldrb w8, [sp, #978]
1844; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #979]
1845; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #977]
1846; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #980]
1847; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #981]
1848; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #982]
1849; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
1850; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
1851; NONEON-NOSVE-NEXT:    sbfx w11, w11, #0, #1
1852; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #976]
1853; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
1854; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
1855; NONEON-NOSVE-NEXT:    and w8, w8, #0x4
1856; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
1857; NONEON-NOSVE-NEXT:    sbfx w15, w15, #0, #1
1858; NONEON-NOSVE-NEXT:    orr w8, w8, w10
1859; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #983]
1860; NONEON-NOSVE-NEXT:    and w11, w11, #0x2
1861; NONEON-NOSVE-NEXT:    and w13, w13, #0x10
1862; NONEON-NOSVE-NEXT:    bfxil w11, w12, #0, #1
1863; NONEON-NOSVE-NEXT:    and w12, w14, #0x20
1864; NONEON-NOSVE-NEXT:    orr w8, w8, w13
1865; NONEON-NOSVE-NEXT:    and w13, w15, #0x40
1866; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
1867; NONEON-NOSVE-NEXT:    orr w8, w11, w8
1868; NONEON-NOSVE-NEXT:    orr w11, w12, w13
1869; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI7_0]
1870; NONEON-NOSVE-NEXT:    orr w8, w8, w11
1871; NONEON-NOSVE-NEXT:    and w10, w10, #0xffffff80
1872; NONEON-NOSVE-NEXT:    add x9, sp, #720
1873; NONEON-NOSVE-NEXT:    add w8, w8, w10
1874; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB7_2
1875; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
1876; NONEON-NOSVE-NEXT:    fmov s0, wzr
1877; NONEON-NOSVE-NEXT:    ldr h2, [x0]
1878; NONEON-NOSVE-NEXT:    stur wzr, [x9, #250]
1879; NONEON-NOSVE-NEXT:    stur xzr, [x9, #242]
1880; NONEON-NOSVE-NEXT:    str h2, [sp, #960]
1881; NONEON-NOSVE-NEXT:    str h0, [sp, #974]
1882; NONEON-NOSVE-NEXT:    ldr q0, [sp, #960]
1883; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB7_3
1884; NONEON-NOSVE-NEXT:    b .LBB7_4
1885; NONEON-NOSVE-NEXT:  .LBB7_2:
1886; NONEON-NOSVE-NEXT:    str q1, [sp, #-16]!
1887; NONEON-NOSVE-NEXT:    ldr q0, [sp], #16
1888; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB7_4
1889; NONEON-NOSVE-NEXT:  .LBB7_3: // %cond.load1
1890; NONEON-NOSVE-NEXT:    ldr h2, [x0, #2]
1891; NONEON-NOSVE-NEXT:    str q0, [sp, #912]
1892; NONEON-NOSVE-NEXT:    ldr w10, [sp, #924]
1893; NONEON-NOSVE-NEXT:    str h2, [sp, #944]
1894; NONEON-NOSVE-NEXT:    ldr q0, [sp, #944]
1895; NONEON-NOSVE-NEXT:    str w10, [sp, #940]
1896; NONEON-NOSVE-NEXT:    str q0, [sp, #896]
1897; NONEON-NOSVE-NEXT:    ldur x10, [x9, #196]
1898; NONEON-NOSVE-NEXT:    stur x10, [x9, #212]
1899; NONEON-NOSVE-NEXT:    ldr h0, [sp, #912]
1900; NONEON-NOSVE-NEXT:    str h0, [sp, #928]
1901; NONEON-NOSVE-NEXT:    ldr h0, [sp, #896]
1902; NONEON-NOSVE-NEXT:    str h0, [sp, #930]
1903; NONEON-NOSVE-NEXT:    ldr q0, [sp, #928]
1904; NONEON-NOSVE-NEXT:  .LBB7_4: // %else2
1905; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB7_20
1906; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
1907; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB7_21
1908; NONEON-NOSVE-NEXT:  .LBB7_6: // %else8
1909; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB7_22
1910; NONEON-NOSVE-NEXT:  .LBB7_7: // %else11
1911; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB7_23
1912; NONEON-NOSVE-NEXT:  .LBB7_8: // %else14
1913; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB7_24
1914; NONEON-NOSVE-NEXT:  .LBB7_9: // %else17
1915; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB7_25
1916; NONEON-NOSVE-NEXT:  .LBB7_10: // %else20
1917; NONEON-NOSVE-NEXT:    tbnz w8, #8, .LBB7_26
1918; NONEON-NOSVE-NEXT:  .LBB7_11: // %else23
1919; NONEON-NOSVE-NEXT:    add x9, sp, #208
1920; NONEON-NOSVE-NEXT:    tbnz w8, #9, .LBB7_27
1921; NONEON-NOSVE-NEXT:  .LBB7_12: // %else26
1922; NONEON-NOSVE-NEXT:    tbnz w8, #10, .LBB7_28
1923; NONEON-NOSVE-NEXT:  .LBB7_13: // %else29
1924; NONEON-NOSVE-NEXT:    tbnz w8, #11, .LBB7_29
1925; NONEON-NOSVE-NEXT:  .LBB7_14: // %else32
1926; NONEON-NOSVE-NEXT:    tbnz w8, #12, .LBB7_30
1927; NONEON-NOSVE-NEXT:  .LBB7_15: // %else35
1928; NONEON-NOSVE-NEXT:    tbnz w8, #13, .LBB7_31
1929; NONEON-NOSVE-NEXT:  .LBB7_16: // %else38
1930; NONEON-NOSVE-NEXT:    tbnz w8, #14, .LBB7_32
1931; NONEON-NOSVE-NEXT:  .LBB7_17: // %else41
1932; NONEON-NOSVE-NEXT:    tbz w8, #15, .LBB7_19
1933; NONEON-NOSVE-NEXT:  .LBB7_18: // %cond.load43
1934; NONEON-NOSVE-NEXT:    ldr h2, [x0, #30]
1935; NONEON-NOSVE-NEXT:    str q1, [sp]
1936; NONEON-NOSVE-NEXT:    ldr w8, [sp, #8]
1937; NONEON-NOSVE-NEXT:    str h2, [sp, #48]
1938; NONEON-NOSVE-NEXT:    ldr q1, [sp, #48]
1939; NONEON-NOSVE-NEXT:    str w8, [sp, #40]
1940; NONEON-NOSVE-NEXT:    ldr x8, [sp]
1941; NONEON-NOSVE-NEXT:    str q1, [sp, #16]
1942; NONEON-NOSVE-NEXT:    ldr h1, [sp, #12]
1943; NONEON-NOSVE-NEXT:    str x8, [sp, #32]
1944; NONEON-NOSVE-NEXT:    str h1, [sp, #44]
1945; NONEON-NOSVE-NEXT:    ldr h1, [sp, #16]
1946; NONEON-NOSVE-NEXT:    str h1, [sp, #46]
1947; NONEON-NOSVE-NEXT:    ldr q1, [sp, #32]
1948; NONEON-NOSVE-NEXT:  .LBB7_19: // %else44
1949; NONEON-NOSVE-NEXT:    add sp, sp, #992
1950; NONEON-NOSVE-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
1951; NONEON-NOSVE-NEXT:    ret
1952; NONEON-NOSVE-NEXT:  .LBB7_20: // %cond.load4
1953; NONEON-NOSVE-NEXT:    ldr h2, [x0, #4]
1954; NONEON-NOSVE-NEXT:    str q0, [sp, #848]
1955; NONEON-NOSVE-NEXT:    str h2, [sp, #880]
1956; NONEON-NOSVE-NEXT:    ldr q0, [sp, #880]
1957; NONEON-NOSVE-NEXT:    str q0, [sp, #832]
1958; NONEON-NOSVE-NEXT:    ldr h0, [sp, #862]
1959; NONEON-NOSVE-NEXT:    str h0, [sp, #878]
1960; NONEON-NOSVE-NEXT:    ldur x10, [x9, #134]
1961; NONEON-NOSVE-NEXT:    stur x10, [x9, #150]
1962; NONEON-NOSVE-NEXT:    ldr w10, [sp, #848]
1963; NONEON-NOSVE-NEXT:    ldr h0, [sp, #832]
1964; NONEON-NOSVE-NEXT:    str w10, [sp, #864]
1965; NONEON-NOSVE-NEXT:    str h0, [sp, #868]
1966; NONEON-NOSVE-NEXT:    ldr q0, [sp, #864]
1967; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB7_6
1968; NONEON-NOSVE-NEXT:  .LBB7_21: // %cond.load7
1969; NONEON-NOSVE-NEXT:    ldr h2, [x0, #6]
1970; NONEON-NOSVE-NEXT:    str q0, [sp, #784]
1971; NONEON-NOSVE-NEXT:    ldr x10, [sp, #792]
1972; NONEON-NOSVE-NEXT:    str h2, [sp, #816]
1973; NONEON-NOSVE-NEXT:    ldr q0, [sp, #816]
1974; NONEON-NOSVE-NEXT:    str x10, [sp, #808]
1975; NONEON-NOSVE-NEXT:    ldr w10, [sp, #784]
1976; NONEON-NOSVE-NEXT:    str q0, [sp, #768]
1977; NONEON-NOSVE-NEXT:    ldr h0, [sp, #788]
1978; NONEON-NOSVE-NEXT:    str w10, [sp, #800]
1979; NONEON-NOSVE-NEXT:    str h0, [sp, #804]
1980; NONEON-NOSVE-NEXT:    ldr h0, [sp, #768]
1981; NONEON-NOSVE-NEXT:    str h0, [sp, #806]
1982; NONEON-NOSVE-NEXT:    ldr q0, [sp, #800]
1983; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB7_7
1984; NONEON-NOSVE-NEXT:  .LBB7_22: // %cond.load10
1985; NONEON-NOSVE-NEXT:    ldr h2, [x0, #8]
1986; NONEON-NOSVE-NEXT:    str q0, [sp, #720]
1987; NONEON-NOSVE-NEXT:    str h2, [sp, #752]
1988; NONEON-NOSVE-NEXT:    ldr q0, [sp, #752]
1989; NONEON-NOSVE-NEXT:    str q0, [sp, #704]
1990; NONEON-NOSVE-NEXT:    ldr h0, [sp, #734]
1991; NONEON-NOSVE-NEXT:    str h0, [sp, #750]
1992; NONEON-NOSVE-NEXT:    ldur w10, [x9, #10]
1993; NONEON-NOSVE-NEXT:    stur w10, [x9, #26]
1994; NONEON-NOSVE-NEXT:    ldr x9, [sp, #720]
1995; NONEON-NOSVE-NEXT:    ldr h0, [sp, #704]
1996; NONEON-NOSVE-NEXT:    str x9, [sp, #736]
1997; NONEON-NOSVE-NEXT:    str h0, [sp, #744]
1998; NONEON-NOSVE-NEXT:    ldr q0, [sp, #736]
1999; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB7_8
2000; NONEON-NOSVE-NEXT:  .LBB7_23: // %cond.load13
2001; NONEON-NOSVE-NEXT:    ldr h2, [x0, #10]
2002; NONEON-NOSVE-NEXT:    str q0, [sp, #656]
2003; NONEON-NOSVE-NEXT:    ldr w9, [sp, #668]
2004; NONEON-NOSVE-NEXT:    str h2, [sp, #688]
2005; NONEON-NOSVE-NEXT:    ldr q0, [sp, #688]
2006; NONEON-NOSVE-NEXT:    str w9, [sp, #684]
2007; NONEON-NOSVE-NEXT:    ldr x9, [sp, #656]
2008; NONEON-NOSVE-NEXT:    str q0, [sp, #640]
2009; NONEON-NOSVE-NEXT:    ldr h0, [sp, #664]
2010; NONEON-NOSVE-NEXT:    str x9, [sp, #672]
2011; NONEON-NOSVE-NEXT:    str h0, [sp, #680]
2012; NONEON-NOSVE-NEXT:    ldr h0, [sp, #640]
2013; NONEON-NOSVE-NEXT:    str h0, [sp, #682]
2014; NONEON-NOSVE-NEXT:    ldr q0, [sp, #672]
2015; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB7_9
2016; NONEON-NOSVE-NEXT:  .LBB7_24: // %cond.load16
2017; NONEON-NOSVE-NEXT:    ldr h2, [x0, #12]
2018; NONEON-NOSVE-NEXT:    str q0, [sp, #592]
2019; NONEON-NOSVE-NEXT:    ldr w9, [sp, #600]
2020; NONEON-NOSVE-NEXT:    str h2, [sp, #624]
2021; NONEON-NOSVE-NEXT:    ldr q0, [sp, #624]
2022; NONEON-NOSVE-NEXT:    str w9, [sp, #616]
2023; NONEON-NOSVE-NEXT:    ldr x9, [sp, #592]
2024; NONEON-NOSVE-NEXT:    str q0, [sp, #576]
2025; NONEON-NOSVE-NEXT:    ldr h0, [sp, #606]
2026; NONEON-NOSVE-NEXT:    str x9, [sp, #608]
2027; NONEON-NOSVE-NEXT:    str h0, [sp, #622]
2028; NONEON-NOSVE-NEXT:    ldr h0, [sp, #576]
2029; NONEON-NOSVE-NEXT:    str h0, [sp, #620]
2030; NONEON-NOSVE-NEXT:    ldr q0, [sp, #608]
2031; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB7_10
2032; NONEON-NOSVE-NEXT:  .LBB7_25: // %cond.load19
2033; NONEON-NOSVE-NEXT:    ldr h2, [x0, #14]
2034; NONEON-NOSVE-NEXT:    str q0, [sp, #512]
2035; NONEON-NOSVE-NEXT:    ldr w9, [sp, #520]
2036; NONEON-NOSVE-NEXT:    str h2, [sp, #560]
2037; NONEON-NOSVE-NEXT:    ldr q0, [sp, #560]
2038; NONEON-NOSVE-NEXT:    str w9, [sp, #552]
2039; NONEON-NOSVE-NEXT:    ldr x9, [sp, #512]
2040; NONEON-NOSVE-NEXT:    str q0, [sp, #528]
2041; NONEON-NOSVE-NEXT:    ldr h0, [sp, #524]
2042; NONEON-NOSVE-NEXT:    str x9, [sp, #544]
2043; NONEON-NOSVE-NEXT:    str h0, [sp, #556]
2044; NONEON-NOSVE-NEXT:    ldr h0, [sp, #528]
2045; NONEON-NOSVE-NEXT:    str h0, [sp, #558]
2046; NONEON-NOSVE-NEXT:    ldr q0, [sp, #544]
2047; NONEON-NOSVE-NEXT:    tbz w8, #8, .LBB7_11
2048; NONEON-NOSVE-NEXT:  .LBB7_26: // %cond.load22
2049; NONEON-NOSVE-NEXT:    ldr h2, [x0, #16]
2050; NONEON-NOSVE-NEXT:    str q1, [sp, #464]
2051; NONEON-NOSVE-NEXT:    add x9, sp, #464
2052; NONEON-NOSVE-NEXT:    str h2, [sp, #496]
2053; NONEON-NOSVE-NEXT:    ldr q1, [sp, #496]
2054; NONEON-NOSVE-NEXT:    str q1, [sp, #448]
2055; NONEON-NOSVE-NEXT:    ldr h1, [sp, #478]
2056; NONEON-NOSVE-NEXT:    str h1, [sp, #494]
2057; NONEON-NOSVE-NEXT:    ldur w10, [x9, #10]
2058; NONEON-NOSVE-NEXT:    ldur x11, [x9, #2]
2059; NONEON-NOSVE-NEXT:    stur w10, [x9, #26]
2060; NONEON-NOSVE-NEXT:    stur x11, [x9, #18]
2061; NONEON-NOSVE-NEXT:    ldr h1, [sp, #448]
2062; NONEON-NOSVE-NEXT:    str h1, [sp, #480]
2063; NONEON-NOSVE-NEXT:    ldr q1, [sp, #480]
2064; NONEON-NOSVE-NEXT:    add x9, sp, #208
2065; NONEON-NOSVE-NEXT:    tbz w8, #9, .LBB7_12
2066; NONEON-NOSVE-NEXT:  .LBB7_27: // %cond.load25
2067; NONEON-NOSVE-NEXT:    ldr h2, [x0, #18]
2068; NONEON-NOSVE-NEXT:    str q1, [sp, #400]
2069; NONEON-NOSVE-NEXT:    ldr w10, [sp, #412]
2070; NONEON-NOSVE-NEXT:    str h2, [sp, #432]
2071; NONEON-NOSVE-NEXT:    ldr q1, [sp, #432]
2072; NONEON-NOSVE-NEXT:    str w10, [sp, #428]
2073; NONEON-NOSVE-NEXT:    str q1, [sp, #384]
2074; NONEON-NOSVE-NEXT:    ldur x10, [x9, #196]
2075; NONEON-NOSVE-NEXT:    stur x10, [x9, #212]
2076; NONEON-NOSVE-NEXT:    ldr h1, [sp, #400]
2077; NONEON-NOSVE-NEXT:    str h1, [sp, #416]
2078; NONEON-NOSVE-NEXT:    ldr h1, [sp, #384]
2079; NONEON-NOSVE-NEXT:    str h1, [sp, #418]
2080; NONEON-NOSVE-NEXT:    ldr q1, [sp, #416]
2081; NONEON-NOSVE-NEXT:    tbz w8, #10, .LBB7_13
2082; NONEON-NOSVE-NEXT:  .LBB7_28: // %cond.load28
2083; NONEON-NOSVE-NEXT:    ldr h2, [x0, #20]
2084; NONEON-NOSVE-NEXT:    str q1, [sp, #336]
2085; NONEON-NOSVE-NEXT:    str h2, [sp, #368]
2086; NONEON-NOSVE-NEXT:    ldr q1, [sp, #368]
2087; NONEON-NOSVE-NEXT:    str q1, [sp, #320]
2088; NONEON-NOSVE-NEXT:    ldr h1, [sp, #350]
2089; NONEON-NOSVE-NEXT:    str h1, [sp, #366]
2090; NONEON-NOSVE-NEXT:    ldur x10, [x9, #134]
2091; NONEON-NOSVE-NEXT:    stur x10, [x9, #150]
2092; NONEON-NOSVE-NEXT:    ldr w10, [sp, #336]
2093; NONEON-NOSVE-NEXT:    ldr h1, [sp, #320]
2094; NONEON-NOSVE-NEXT:    str w10, [sp, #352]
2095; NONEON-NOSVE-NEXT:    str h1, [sp, #356]
2096; NONEON-NOSVE-NEXT:    ldr q1, [sp, #352]
2097; NONEON-NOSVE-NEXT:    tbz w8, #11, .LBB7_14
2098; NONEON-NOSVE-NEXT:  .LBB7_29: // %cond.load31
2099; NONEON-NOSVE-NEXT:    ldr h2, [x0, #22]
2100; NONEON-NOSVE-NEXT:    str q1, [sp, #272]
2101; NONEON-NOSVE-NEXT:    ldr x10, [sp, #280]
2102; NONEON-NOSVE-NEXT:    str h2, [sp, #304]
2103; NONEON-NOSVE-NEXT:    ldr q1, [sp, #304]
2104; NONEON-NOSVE-NEXT:    str x10, [sp, #296]
2105; NONEON-NOSVE-NEXT:    ldr w10, [sp, #272]
2106; NONEON-NOSVE-NEXT:    str q1, [sp, #256]
2107; NONEON-NOSVE-NEXT:    ldr h1, [sp, #276]
2108; NONEON-NOSVE-NEXT:    str w10, [sp, #288]
2109; NONEON-NOSVE-NEXT:    str h1, [sp, #292]
2110; NONEON-NOSVE-NEXT:    ldr h1, [sp, #256]
2111; NONEON-NOSVE-NEXT:    str h1, [sp, #294]
2112; NONEON-NOSVE-NEXT:    ldr q1, [sp, #288]
2113; NONEON-NOSVE-NEXT:    tbz w8, #12, .LBB7_15
2114; NONEON-NOSVE-NEXT:  .LBB7_30: // %cond.load34
2115; NONEON-NOSVE-NEXT:    ldr h2, [x0, #24]
2116; NONEON-NOSVE-NEXT:    str q1, [sp, #208]
2117; NONEON-NOSVE-NEXT:    str h2, [sp, #240]
2118; NONEON-NOSVE-NEXT:    ldr q1, [sp, #240]
2119; NONEON-NOSVE-NEXT:    str q1, [sp, #192]
2120; NONEON-NOSVE-NEXT:    ldr h1, [sp, #222]
2121; NONEON-NOSVE-NEXT:    str h1, [sp, #238]
2122; NONEON-NOSVE-NEXT:    ldur w10, [x9, #10]
2123; NONEON-NOSVE-NEXT:    stur w10, [x9, #26]
2124; NONEON-NOSVE-NEXT:    ldr x9, [sp, #208]
2125; NONEON-NOSVE-NEXT:    ldr h1, [sp, #192]
2126; NONEON-NOSVE-NEXT:    str x9, [sp, #224]
2127; NONEON-NOSVE-NEXT:    str h1, [sp, #232]
2128; NONEON-NOSVE-NEXT:    ldr q1, [sp, #224]
2129; NONEON-NOSVE-NEXT:    tbz w8, #13, .LBB7_16
2130; NONEON-NOSVE-NEXT:  .LBB7_31: // %cond.load37
2131; NONEON-NOSVE-NEXT:    ldr h2, [x0, #26]
2132; NONEON-NOSVE-NEXT:    str q1, [sp, #144]
2133; NONEON-NOSVE-NEXT:    ldr w9, [sp, #156]
2134; NONEON-NOSVE-NEXT:    str h2, [sp, #176]
2135; NONEON-NOSVE-NEXT:    ldr q1, [sp, #176]
2136; NONEON-NOSVE-NEXT:    str w9, [sp, #172]
2137; NONEON-NOSVE-NEXT:    ldr x9, [sp, #144]
2138; NONEON-NOSVE-NEXT:    str q1, [sp, #128]
2139; NONEON-NOSVE-NEXT:    ldr h1, [sp, #152]
2140; NONEON-NOSVE-NEXT:    str x9, [sp, #160]
2141; NONEON-NOSVE-NEXT:    str h1, [sp, #168]
2142; NONEON-NOSVE-NEXT:    ldr h1, [sp, #128]
2143; NONEON-NOSVE-NEXT:    str h1, [sp, #170]
2144; NONEON-NOSVE-NEXT:    ldr q1, [sp, #160]
2145; NONEON-NOSVE-NEXT:    tbz w8, #14, .LBB7_17
2146; NONEON-NOSVE-NEXT:  .LBB7_32: // %cond.load40
2147; NONEON-NOSVE-NEXT:    ldr h2, [x0, #28]
2148; NONEON-NOSVE-NEXT:    str q1, [sp, #80]
2149; NONEON-NOSVE-NEXT:    ldr w9, [sp, #88]
2150; NONEON-NOSVE-NEXT:    str h2, [sp, #112]
2151; NONEON-NOSVE-NEXT:    ldr q1, [sp, #112]
2152; NONEON-NOSVE-NEXT:    str w9, [sp, #104]
2153; NONEON-NOSVE-NEXT:    ldr x9, [sp, #80]
2154; NONEON-NOSVE-NEXT:    str q1, [sp, #64]
2155; NONEON-NOSVE-NEXT:    ldr h1, [sp, #94]
2156; NONEON-NOSVE-NEXT:    str x9, [sp, #96]
2157; NONEON-NOSVE-NEXT:    str h1, [sp, #110]
2158; NONEON-NOSVE-NEXT:    ldr h1, [sp, #64]
2159; NONEON-NOSVE-NEXT:    str h1, [sp, #108]
2160; NONEON-NOSVE-NEXT:    ldr q1, [sp, #96]
2161; NONEON-NOSVE-NEXT:    tbnz w8, #15, .LBB7_18
2162; NONEON-NOSVE-NEXT:    b .LBB7_19
2163  %load = call <16 x half> @llvm.masked.load.v16f16(ptr %src, i32 8, <16 x i1> %mask, <16 x half> zeroinitializer)
2164  ret <16 x half> %load
2165}
2166
2167define <2 x float> @masked_load_v2f32(ptr %src, <2 x i1> %mask) {
2168; CHECK-LABEL: masked_load_v2f32:
2169; CHECK:       // %bb.0:
2170; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2171; CHECK-NEXT:    ptrue p0.s, vl2
2172; CHECK-NEXT:    lsl z0.s, z0.s, #31
2173; CHECK-NEXT:    asr z0.s, z0.s, #31
2174; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #0
2175; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
2176; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $z0
2177; CHECK-NEXT:    ret
2178;
2179; NONEON-NOSVE-LABEL: masked_load_v2f32:
2180; NONEON-NOSVE:       // %bb.0:
2181; NONEON-NOSVE-NEXT:    sub sp, sp, #64
2182; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 64
2183; NONEON-NOSVE-NEXT:    str d0, [sp, #48]
2184; NONEON-NOSVE-NEXT:    ldr w8, [sp, #52]
2185; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #48]
2186; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
2187; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
2188; NONEON-NOSVE-NEXT:    bfxil w8, w9, #0, #1
2189; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB8_2
2190; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
2191; NONEON-NOSVE-NEXT:    ldr s0, [x0]
2192; NONEON-NOSVE-NEXT:    str wzr, [sp, #44]
2193; NONEON-NOSVE-NEXT:    str s0, [sp, #40]
2194; NONEON-NOSVE-NEXT:    ldr d0, [sp, #40]
2195; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB8_3
2196; NONEON-NOSVE-NEXT:    b .LBB8_4
2197; NONEON-NOSVE-NEXT:  .LBB8_2:
2198; NONEON-NOSVE-NEXT:    adrp x9, .LCPI8_0
2199; NONEON-NOSVE-NEXT:    ldr d0, [x9, :lo12:.LCPI8_0]
2200; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB8_4
2201; NONEON-NOSVE-NEXT:  .LBB8_3: // %cond.load1
2202; NONEON-NOSVE-NEXT:    ldr s1, [x0, #4]
2203; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
2204; NONEON-NOSVE-NEXT:    str s1, [sp, #32]
2205; NONEON-NOSVE-NEXT:    ldr s1, [sp, #8]
2206; NONEON-NOSVE-NEXT:    ldr d0, [sp, #32]
2207; NONEON-NOSVE-NEXT:    str d0, [sp, #16]
2208; NONEON-NOSVE-NEXT:    ldr s0, [sp, #16]
2209; NONEON-NOSVE-NEXT:    stp s1, s0, [sp, #24]
2210; NONEON-NOSVE-NEXT:    ldr d0, [sp, #24]
2211; NONEON-NOSVE-NEXT:  .LBB8_4: // %else2
2212; NONEON-NOSVE-NEXT:    add sp, sp, #64
2213; NONEON-NOSVE-NEXT:    ret
2214  %load = call <2 x float> @llvm.masked.load.v2f32(ptr %src, i32 8, <2 x i1> %mask, <2 x float> zeroinitializer)
2215  ret <2 x float> %load
2216}
2217
2218define <4 x float> @masked_load_v4f32(ptr %src, <4 x i1> %mask) {
2219; CHECK-LABEL: masked_load_v4f32:
2220; CHECK:       // %bb.0:
2221; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2222; CHECK-NEXT:    ptrue p0.s, vl4
2223; CHECK-NEXT:    uunpklo z0.s, z0.h
2224; CHECK-NEXT:    lsl z0.s, z0.s, #31
2225; CHECK-NEXT:    asr z0.s, z0.s, #31
2226; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #0
2227; CHECK-NEXT:    ld1w { z0.s }, p0/z, [x0]
2228; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2229; CHECK-NEXT:    ret
2230;
2231; NONEON-NOSVE-LABEL: masked_load_v4f32:
2232; NONEON-NOSVE:       // %bb.0:
2233; NONEON-NOSVE-NEXT:    sub sp, sp, #224
2234; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 224
2235; NONEON-NOSVE-NEXT:    str d0, [sp, #208]
2236; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #210]
2237; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #212]
2238; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #214]
2239; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #208]
2240; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
2241; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
2242; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
2243; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
2244; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
2245; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
2246; NONEON-NOSVE-NEXT:    bfxil w8, w11, #0, #1
2247; NONEON-NOSVE-NEXT:    orr w9, w9, w10
2248; NONEON-NOSVE-NEXT:    orr w8, w8, w9
2249; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB9_2
2250; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
2251; NONEON-NOSVE-NEXT:    ldr s0, [x0]
2252; NONEON-NOSVE-NEXT:    str wzr, [sp, #204]
2253; NONEON-NOSVE-NEXT:    stur xzr, [sp, #196]
2254; NONEON-NOSVE-NEXT:    str s0, [sp, #192]
2255; NONEON-NOSVE-NEXT:    ldr q0, [sp, #192]
2256; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB9_3
2257; NONEON-NOSVE-NEXT:    b .LBB9_4
2258; NONEON-NOSVE-NEXT:  .LBB9_2:
2259; NONEON-NOSVE-NEXT:    adrp x9, .LCPI9_0
2260; NONEON-NOSVE-NEXT:    ldr q0, [x9, :lo12:.LCPI9_0]
2261; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB9_4
2262; NONEON-NOSVE-NEXT:  .LBB9_3: // %cond.load1
2263; NONEON-NOSVE-NEXT:    ldr s1, [x0, #4]
2264; NONEON-NOSVE-NEXT:    str q0, [sp, #144]
2265; NONEON-NOSVE-NEXT:    str s1, [sp, #176]
2266; NONEON-NOSVE-NEXT:    ldr q0, [sp, #176]
2267; NONEON-NOSVE-NEXT:    str q0, [sp, #128]
2268; NONEON-NOSVE-NEXT:    ldp s0, s1, [sp, #152]
2269; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #168]
2270; NONEON-NOSVE-NEXT:    ldr s1, [sp, #144]
2271; NONEON-NOSVE-NEXT:    ldr s0, [sp, #128]
2272; NONEON-NOSVE-NEXT:    stp s1, s0, [sp, #160]
2273; NONEON-NOSVE-NEXT:    ldr q0, [sp, #160]
2274; NONEON-NOSVE-NEXT:  .LBB9_4: // %else2
2275; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB9_7
2276; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
2277; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB9_8
2278; NONEON-NOSVE-NEXT:  .LBB9_6: // %else8
2279; NONEON-NOSVE-NEXT:    add sp, sp, #224
2280; NONEON-NOSVE-NEXT:    ret
2281; NONEON-NOSVE-NEXT:  .LBB9_7: // %cond.load4
2282; NONEON-NOSVE-NEXT:    ldr s1, [x0, #8]
2283; NONEON-NOSVE-NEXT:    str q0, [sp, #80]
2284; NONEON-NOSVE-NEXT:    str s1, [sp, #112]
2285; NONEON-NOSVE-NEXT:    ldr s1, [sp, #92]
2286; NONEON-NOSVE-NEXT:    ldr q0, [sp, #112]
2287; NONEON-NOSVE-NEXT:    str q0, [sp, #64]
2288; NONEON-NOSVE-NEXT:    ldp s0, s2, [sp, #80]
2289; NONEON-NOSVE-NEXT:    stp s0, s2, [sp, #96]
2290; NONEON-NOSVE-NEXT:    ldr s0, [sp, #64]
2291; NONEON-NOSVE-NEXT:    stp s0, s1, [sp, #104]
2292; NONEON-NOSVE-NEXT:    ldr q0, [sp, #96]
2293; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB9_6
2294; NONEON-NOSVE-NEXT:  .LBB9_8: // %cond.load7
2295; NONEON-NOSVE-NEXT:    ldr s1, [x0, #12]
2296; NONEON-NOSVE-NEXT:    str q0, [sp]
2297; NONEON-NOSVE-NEXT:    ldr x8, [sp]
2298; NONEON-NOSVE-NEXT:    str s1, [sp, #48]
2299; NONEON-NOSVE-NEXT:    ldr s1, [sp, #8]
2300; NONEON-NOSVE-NEXT:    ldr q0, [sp, #48]
2301; NONEON-NOSVE-NEXT:    str x8, [sp, #32]
2302; NONEON-NOSVE-NEXT:    str q0, [sp, #16]
2303; NONEON-NOSVE-NEXT:    ldr s0, [sp, #16]
2304; NONEON-NOSVE-NEXT:    stp s1, s0, [sp, #40]
2305; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
2306; NONEON-NOSVE-NEXT:    add sp, sp, #224
2307; NONEON-NOSVE-NEXT:    ret
2308  %load = call <4 x float> @llvm.masked.load.v4f32(ptr %src, i32 8, <4 x i1> %mask, <4 x float> zeroinitializer)
2309  ret <4 x float> %load
2310}
2311
2312define <8 x float> @masked_load_v8f32(ptr %src, <8 x i1> %mask) {
2313; CHECK-LABEL: masked_load_v8f32:
2314; CHECK:       // %bb.0:
2315; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2316; CHECK-NEXT:    mov z1.b, z0.b[3]
2317; CHECK-NEXT:    mov z2.b, z0.b[2]
2318; CHECK-NEXT:    mov x8, #4 // =0x4
2319; CHECK-NEXT:    mov z3.b, z0.b[1]
2320; CHECK-NEXT:    mov z4.b, z0.b[7]
2321; CHECK-NEXT:    mov z5.b, z0.b[6]
2322; CHECK-NEXT:    mov z6.b, z0.b[5]
2323; CHECK-NEXT:    mov z7.b, z0.b[4]
2324; CHECK-NEXT:    ptrue p0.s, vl4
2325; CHECK-NEXT:    zip1 z1.h, z2.h, z1.h
2326; CHECK-NEXT:    zip1 z0.h, z0.h, z3.h
2327; CHECK-NEXT:    zip1 z2.h, z5.h, z4.h
2328; CHECK-NEXT:    zip1 z3.h, z7.h, z6.h
2329; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
2330; CHECK-NEXT:    zip1 z1.s, z3.s, z2.s
2331; CHECK-NEXT:    uunpklo z0.s, z0.h
2332; CHECK-NEXT:    uunpklo z1.s, z1.h
2333; CHECK-NEXT:    lsl z0.s, z0.s, #31
2334; CHECK-NEXT:    lsl z1.s, z1.s, #31
2335; CHECK-NEXT:    asr z0.s, z0.s, #31
2336; CHECK-NEXT:    asr z1.s, z1.s, #31
2337; CHECK-NEXT:    cmpne p1.s, p0/z, z0.s, #0
2338; CHECK-NEXT:    cmpne p0.s, p0/z, z1.s, #0
2339; CHECK-NEXT:    ld1w { z0.s }, p1/z, [x0]
2340; CHECK-NEXT:    ld1w { z1.s }, p0/z, [x0, x8, lsl #2]
2341; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2342; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
2343; CHECK-NEXT:    ret
2344;
2345; NONEON-NOSVE-LABEL: masked_load_v8f32:
2346; NONEON-NOSVE:       // %bb.0:
2347; NONEON-NOSVE-NEXT:    sub sp, sp, #496
2348; NONEON-NOSVE-NEXT:    str x29, [sp, #480] // 8-byte Folded Spill
2349; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 496
2350; NONEON-NOSVE-NEXT:    .cfi_offset w29, -16
2351; NONEON-NOSVE-NEXT:    str d0, [sp, #464]
2352; NONEON-NOSVE-NEXT:    adrp x8, .LCPI10_0
2353; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #466]
2354; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #467]
2355; NONEON-NOSVE-NEXT:    ldrb w11, [sp, #465]
2356; NONEON-NOSVE-NEXT:    ldrb w13, [sp, #468]
2357; NONEON-NOSVE-NEXT:    ldrb w14, [sp, #469]
2358; NONEON-NOSVE-NEXT:    ldrb w15, [sp, #470]
2359; NONEON-NOSVE-NEXT:    sbfx w9, w9, #0, #1
2360; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
2361; NONEON-NOSVE-NEXT:    sbfx w11, w11, #0, #1
2362; NONEON-NOSVE-NEXT:    ldrb w12, [sp, #464]
2363; NONEON-NOSVE-NEXT:    sbfx w13, w13, #0, #1
2364; NONEON-NOSVE-NEXT:    sbfx w14, w14, #0, #1
2365; NONEON-NOSVE-NEXT:    and w9, w9, #0x4
2366; NONEON-NOSVE-NEXT:    and w10, w10, #0x8
2367; NONEON-NOSVE-NEXT:    sbfx w15, w15, #0, #1
2368; NONEON-NOSVE-NEXT:    orr w9, w9, w10
2369; NONEON-NOSVE-NEXT:    ldrb w10, [sp, #471]
2370; NONEON-NOSVE-NEXT:    and w11, w11, #0x2
2371; NONEON-NOSVE-NEXT:    and w13, w13, #0x10
2372; NONEON-NOSVE-NEXT:    bfxil w11, w12, #0, #1
2373; NONEON-NOSVE-NEXT:    and w12, w14, #0x20
2374; NONEON-NOSVE-NEXT:    orr w9, w9, w13
2375; NONEON-NOSVE-NEXT:    and w13, w15, #0x40
2376; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
2377; NONEON-NOSVE-NEXT:    orr w9, w11, w9
2378; NONEON-NOSVE-NEXT:    orr w11, w12, w13
2379; NONEON-NOSVE-NEXT:    ldr q1, [x8, :lo12:.LCPI10_0]
2380; NONEON-NOSVE-NEXT:    orr w9, w9, w11
2381; NONEON-NOSVE-NEXT:    and w10, w10, #0x80
2382; NONEON-NOSVE-NEXT:    add w10, w9, w10
2383; NONEON-NOSVE-NEXT:    add x9, sp, #208
2384; NONEON-NOSVE-NEXT:    and w8, w10, #0xff
2385; NONEON-NOSVE-NEXT:    tbz w10, #0, .LBB10_2
2386; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
2387; NONEON-NOSVE-NEXT:    ldr s0, [x0]
2388; NONEON-NOSVE-NEXT:    str wzr, [sp, #460]
2389; NONEON-NOSVE-NEXT:    stur xzr, [x9, #244]
2390; NONEON-NOSVE-NEXT:    str s0, [sp, #448]
2391; NONEON-NOSVE-NEXT:    ldr q0, [sp, #448]
2392; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB10_3
2393; NONEON-NOSVE-NEXT:    b .LBB10_4
2394; NONEON-NOSVE-NEXT:  .LBB10_2:
2395; NONEON-NOSVE-NEXT:    str q1, [sp, #-16]!
2396; NONEON-NOSVE-NEXT:    ldr q0, [sp], #16
2397; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB10_4
2398; NONEON-NOSVE-NEXT:  .LBB10_3: // %cond.load1
2399; NONEON-NOSVE-NEXT:    ldr s2, [x0, #4]
2400; NONEON-NOSVE-NEXT:    str q0, [sp, #400]
2401; NONEON-NOSVE-NEXT:    str s2, [sp, #432]
2402; NONEON-NOSVE-NEXT:    ldr q0, [sp, #432]
2403; NONEON-NOSVE-NEXT:    str q0, [sp, #384]
2404; NONEON-NOSVE-NEXT:    ldr s0, [sp, #412]
2405; NONEON-NOSVE-NEXT:    str s0, [sp, #428]
2406; NONEON-NOSVE-NEXT:    ldr s0, [sp, #408]
2407; NONEON-NOSVE-NEXT:    str s0, [sp, #424]
2408; NONEON-NOSVE-NEXT:    ldr s0, [sp, #400]
2409; NONEON-NOSVE-NEXT:    str s0, [sp, #416]
2410; NONEON-NOSVE-NEXT:    ldr s0, [sp, #384]
2411; NONEON-NOSVE-NEXT:    str s0, [sp, #420]
2412; NONEON-NOSVE-NEXT:    ldr q0, [sp, #416]
2413; NONEON-NOSVE-NEXT:  .LBB10_4: // %else2
2414; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB10_12
2415; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
2416; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB10_13
2417; NONEON-NOSVE-NEXT:  .LBB10_6: // %else8
2418; NONEON-NOSVE-NEXT:    tbnz w8, #4, .LBB10_14
2419; NONEON-NOSVE-NEXT:  .LBB10_7: // %else11
2420; NONEON-NOSVE-NEXT:    tbnz w8, #5, .LBB10_15
2421; NONEON-NOSVE-NEXT:  .LBB10_8: // %else14
2422; NONEON-NOSVE-NEXT:    tbnz w8, #6, .LBB10_16
2423; NONEON-NOSVE-NEXT:  .LBB10_9: // %else17
2424; NONEON-NOSVE-NEXT:    tbz w8, #7, .LBB10_11
2425; NONEON-NOSVE-NEXT:  .LBB10_10: // %cond.load19
2426; NONEON-NOSVE-NEXT:    ldr s2, [x0, #28]
2427; NONEON-NOSVE-NEXT:    str q1, [sp]
2428; NONEON-NOSVE-NEXT:    ldr x8, [sp]
2429; NONEON-NOSVE-NEXT:    str s2, [sp, #48]
2430; NONEON-NOSVE-NEXT:    ldr s2, [sp, #8]
2431; NONEON-NOSVE-NEXT:    ldr q1, [sp, #48]
2432; NONEON-NOSVE-NEXT:    str x8, [sp, #32]
2433; NONEON-NOSVE-NEXT:    str q1, [sp, #16]
2434; NONEON-NOSVE-NEXT:    ldr s1, [sp, #16]
2435; NONEON-NOSVE-NEXT:    stp s2, s1, [sp, #40]
2436; NONEON-NOSVE-NEXT:    ldr q1, [sp, #32]
2437; NONEON-NOSVE-NEXT:  .LBB10_11: // %else20
2438; NONEON-NOSVE-NEXT:    ldr x29, [sp, #480] // 8-byte Folded Reload
2439; NONEON-NOSVE-NEXT:    add sp, sp, #496
2440; NONEON-NOSVE-NEXT:    ret
2441; NONEON-NOSVE-NEXT:  .LBB10_12: // %cond.load4
2442; NONEON-NOSVE-NEXT:    ldr s2, [x0, #8]
2443; NONEON-NOSVE-NEXT:    str q0, [sp, #336]
2444; NONEON-NOSVE-NEXT:    str s2, [sp, #368]
2445; NONEON-NOSVE-NEXT:    ldr q0, [sp, #368]
2446; NONEON-NOSVE-NEXT:    str q0, [sp, #320]
2447; NONEON-NOSVE-NEXT:    ldr s0, [sp, #348]
2448; NONEON-NOSVE-NEXT:    str s0, [sp, #364]
2449; NONEON-NOSVE-NEXT:    ldr s0, [sp, #340]
2450; NONEON-NOSVE-NEXT:    str s0, [sp, #356]
2451; NONEON-NOSVE-NEXT:    ldr s0, [sp, #336]
2452; NONEON-NOSVE-NEXT:    str s0, [sp, #352]
2453; NONEON-NOSVE-NEXT:    ldr s0, [sp, #320]
2454; NONEON-NOSVE-NEXT:    str s0, [sp, #360]
2455; NONEON-NOSVE-NEXT:    ldr q0, [sp, #352]
2456; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB10_6
2457; NONEON-NOSVE-NEXT:  .LBB10_13: // %cond.load7
2458; NONEON-NOSVE-NEXT:    ldr s2, [x0, #12]
2459; NONEON-NOSVE-NEXT:    str q0, [sp, #256]
2460; NONEON-NOSVE-NEXT:    ldr x10, [sp, #256]
2461; NONEON-NOSVE-NEXT:    str s2, [sp, #304]
2462; NONEON-NOSVE-NEXT:    ldr q0, [sp, #304]
2463; NONEON-NOSVE-NEXT:    str x10, [sp, #288]
2464; NONEON-NOSVE-NEXT:    str q0, [sp, #272]
2465; NONEON-NOSVE-NEXT:    ldr s0, [sp, #264]
2466; NONEON-NOSVE-NEXT:    str s0, [sp, #296]
2467; NONEON-NOSVE-NEXT:    ldr s0, [sp, #272]
2468; NONEON-NOSVE-NEXT:    str s0, [sp, #300]
2469; NONEON-NOSVE-NEXT:    ldr q0, [sp, #288]
2470; NONEON-NOSVE-NEXT:    tbz w8, #4, .LBB10_7
2471; NONEON-NOSVE-NEXT:  .LBB10_14: // %cond.load10
2472; NONEON-NOSVE-NEXT:    ldr s2, [x0, #16]
2473; NONEON-NOSVE-NEXT:    str q1, [sp, #208]
2474; NONEON-NOSVE-NEXT:    str s2, [sp, #240]
2475; NONEON-NOSVE-NEXT:    ldr q1, [sp, #240]
2476; NONEON-NOSVE-NEXT:    str q1, [sp, #192]
2477; NONEON-NOSVE-NEXT:    ldr s1, [sp, #220]
2478; NONEON-NOSVE-NEXT:    str s1, [sp, #236]
2479; NONEON-NOSVE-NEXT:    ldur x10, [x9, #4]
2480; NONEON-NOSVE-NEXT:    stur x10, [x9, #20]
2481; NONEON-NOSVE-NEXT:    ldr s1, [sp, #192]
2482; NONEON-NOSVE-NEXT:    str s1, [sp, #224]
2483; NONEON-NOSVE-NEXT:    ldr q1, [sp, #224]
2484; NONEON-NOSVE-NEXT:    tbz w8, #5, .LBB10_8
2485; NONEON-NOSVE-NEXT:  .LBB10_15: // %cond.load13
2486; NONEON-NOSVE-NEXT:    ldr s2, [x0, #20]
2487; NONEON-NOSVE-NEXT:    str q1, [sp, #144]
2488; NONEON-NOSVE-NEXT:    str s2, [sp, #176]
2489; NONEON-NOSVE-NEXT:    ldr q1, [sp, #176]
2490; NONEON-NOSVE-NEXT:    str q1, [sp, #128]
2491; NONEON-NOSVE-NEXT:    ldp s1, s2, [sp, #152]
2492; NONEON-NOSVE-NEXT:    stp s1, s2, [sp, #168]
2493; NONEON-NOSVE-NEXT:    ldr s2, [sp, #144]
2494; NONEON-NOSVE-NEXT:    ldr s1, [sp, #128]
2495; NONEON-NOSVE-NEXT:    stp s2, s1, [sp, #160]
2496; NONEON-NOSVE-NEXT:    ldr q1, [sp, #160]
2497; NONEON-NOSVE-NEXT:    tbz w8, #6, .LBB10_9
2498; NONEON-NOSVE-NEXT:  .LBB10_16: // %cond.load16
2499; NONEON-NOSVE-NEXT:    ldr s2, [x0, #24]
2500; NONEON-NOSVE-NEXT:    str q1, [sp, #80]
2501; NONEON-NOSVE-NEXT:    str s2, [sp, #112]
2502; NONEON-NOSVE-NEXT:    ldr s2, [sp, #92]
2503; NONEON-NOSVE-NEXT:    ldr q1, [sp, #112]
2504; NONEON-NOSVE-NEXT:    str q1, [sp, #64]
2505; NONEON-NOSVE-NEXT:    ldp s1, s3, [sp, #80]
2506; NONEON-NOSVE-NEXT:    stp s1, s3, [sp, #96]
2507; NONEON-NOSVE-NEXT:    ldr s1, [sp, #64]
2508; NONEON-NOSVE-NEXT:    stp s1, s2, [sp, #104]
2509; NONEON-NOSVE-NEXT:    ldr q1, [sp, #96]
2510; NONEON-NOSVE-NEXT:    tbnz w8, #7, .LBB10_10
2511; NONEON-NOSVE-NEXT:    b .LBB10_11
2512  %load = call <8 x float> @llvm.masked.load.v8f32(ptr %src, i32 8, <8 x i1> %mask, <8 x float> zeroinitializer)
2513  ret <8 x float> %load
2514}
2515
2516define <2 x double> @masked_load_v2f64(ptr %src, <2 x i1> %mask) {
2517; CHECK-LABEL: masked_load_v2f64:
2518; CHECK:       // %bb.0:
2519; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2520; CHECK-NEXT:    ptrue p0.d, vl2
2521; CHECK-NEXT:    uunpklo z0.d, z0.s
2522; CHECK-NEXT:    lsl z0.d, z0.d, #63
2523; CHECK-NEXT:    asr z0.d, z0.d, #63
2524; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
2525; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
2526; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2527; CHECK-NEXT:    ret
2528;
2529; NONEON-NOSVE-LABEL: masked_load_v2f64:
2530; NONEON-NOSVE:       // %bb.0:
2531; NONEON-NOSVE-NEXT:    sub sp, sp, #96
2532; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 96
2533; NONEON-NOSVE-NEXT:    str d0, [sp, #80]
2534; NONEON-NOSVE-NEXT:    ldr w8, [sp, #84]
2535; NONEON-NOSVE-NEXT:    ldrb w9, [sp, #80]
2536; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
2537; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
2538; NONEON-NOSVE-NEXT:    bfxil w8, w9, #0, #1
2539; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB11_2
2540; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
2541; NONEON-NOSVE-NEXT:    ldr d0, [x0]
2542; NONEON-NOSVE-NEXT:    str xzr, [sp, #72]
2543; NONEON-NOSVE-NEXT:    str d0, [sp, #64]
2544; NONEON-NOSVE-NEXT:    ldr q0, [sp, #64]
2545; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB11_3
2546; NONEON-NOSVE-NEXT:    b .LBB11_4
2547; NONEON-NOSVE-NEXT:  .LBB11_2:
2548; NONEON-NOSVE-NEXT:    adrp x9, .LCPI11_0
2549; NONEON-NOSVE-NEXT:    ldr q0, [x9, :lo12:.LCPI11_0]
2550; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB11_4
2551; NONEON-NOSVE-NEXT:  .LBB11_3: // %cond.load1
2552; NONEON-NOSVE-NEXT:    ldr d1, [x0, #8]
2553; NONEON-NOSVE-NEXT:    str q0, [sp]
2554; NONEON-NOSVE-NEXT:    str d1, [sp, #48]
2555; NONEON-NOSVE-NEXT:    ldr d1, [sp]
2556; NONEON-NOSVE-NEXT:    ldr q0, [sp, #48]
2557; NONEON-NOSVE-NEXT:    str q0, [sp, #16]
2558; NONEON-NOSVE-NEXT:    ldr d0, [sp, #16]
2559; NONEON-NOSVE-NEXT:    stp d1, d0, [sp, #32]
2560; NONEON-NOSVE-NEXT:    ldr q0, [sp, #32]
2561; NONEON-NOSVE-NEXT:  .LBB11_4: // %else2
2562; NONEON-NOSVE-NEXT:    add sp, sp, #96
2563; NONEON-NOSVE-NEXT:    ret
2564  %load = call <2 x double> @llvm.masked.load.v2f64(ptr %src, i32 8, <2 x i1> %mask, <2 x double> zeroinitializer)
2565  ret <2 x double> %load
2566}
2567
2568define <4 x double> @masked_load_v4f64(ptr %src, <4 x i1> %mask) {
2569; CHECK-LABEL: masked_load_v4f64:
2570; CHECK:       // %bb.0:
2571; CHECK-NEXT:    // kill: def $d0 killed $d0 def $z0
2572; CHECK-NEXT:    ptrue p0.d, vl2
2573; CHECK-NEXT:    mov x8, #2 // =0x2
2574; CHECK-NEXT:    uunpklo z0.s, z0.h
2575; CHECK-NEXT:    uunpklo z1.d, z0.s
2576; CHECK-NEXT:    ext z0.b, z0.b, z0.b, #8
2577; CHECK-NEXT:    uunpklo z0.d, z0.s
2578; CHECK-NEXT:    lsl z1.d, z1.d, #63
2579; CHECK-NEXT:    lsl z0.d, z0.d, #63
2580; CHECK-NEXT:    asr z1.d, z1.d, #63
2581; CHECK-NEXT:    asr z0.d, z0.d, #63
2582; CHECK-NEXT:    cmpne p1.d, p0/z, z1.d, #0
2583; CHECK-NEXT:    cmpne p0.d, p0/z, z0.d, #0
2584; CHECK-NEXT:    ld1d { z0.d }, p1/z, [x0]
2585; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2586; CHECK-NEXT:    ld1d { z1.d }, p0/z, [x0, x8, lsl #3]
2587; CHECK-NEXT:    // kill: def $q1 killed $q1 killed $z1
2588; CHECK-NEXT:    ret
2589;
2590; NONEON-NOSVE-LABEL: masked_load_v4f64:
2591; NONEON-NOSVE:       // %bb.0:
2592; NONEON-NOSVE-NEXT:    sub sp, sp, #224
2593; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 224
2594; NONEON-NOSVE-NEXT:    str d0, [sp, #208]
2595; NONEON-NOSVE-NEXT:    adrp x9, .LCPI12_0
2596; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #210]
2597; NONEON-NOSVE-NEXT:    ldrh w10, [sp, #212]
2598; NONEON-NOSVE-NEXT:    ldrh w11, [sp, #214]
2599; NONEON-NOSVE-NEXT:    ldrh w12, [sp, #208]
2600; NONEON-NOSVE-NEXT:    ldr q1, [x9, :lo12:.LCPI12_0]
2601; NONEON-NOSVE-NEXT:    sbfx w8, w8, #0, #1
2602; NONEON-NOSVE-NEXT:    sbfx w10, w10, #0, #1
2603; NONEON-NOSVE-NEXT:    sbfx w11, w11, #0, #1
2604; NONEON-NOSVE-NEXT:    and w8, w8, #0x2
2605; NONEON-NOSVE-NEXT:    and w10, w10, #0x4
2606; NONEON-NOSVE-NEXT:    and w11, w11, #0x8
2607; NONEON-NOSVE-NEXT:    bfxil w8, w12, #0, #1
2608; NONEON-NOSVE-NEXT:    orr w10, w10, w11
2609; NONEON-NOSVE-NEXT:    orr w8, w8, w10
2610; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB12_2
2611; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
2612; NONEON-NOSVE-NEXT:    ldr d0, [x0]
2613; NONEON-NOSVE-NEXT:    str xzr, [sp, #200]
2614; NONEON-NOSVE-NEXT:    str d0, [sp, #192]
2615; NONEON-NOSVE-NEXT:    ldr q0, [sp, #192]
2616; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB12_3
2617; NONEON-NOSVE-NEXT:    b .LBB12_4
2618; NONEON-NOSVE-NEXT:  .LBB12_2:
2619; NONEON-NOSVE-NEXT:    str q1, [sp, #-16]!
2620; NONEON-NOSVE-NEXT:    ldr q0, [sp], #16
2621; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB12_4
2622; NONEON-NOSVE-NEXT:  .LBB12_3: // %cond.load1
2623; NONEON-NOSVE-NEXT:    ldr d2, [x0, #8]
2624; NONEON-NOSVE-NEXT:    str q0, [sp, #128]
2625; NONEON-NOSVE-NEXT:    str d2, [sp, #176]
2626; NONEON-NOSVE-NEXT:    ldr d2, [sp, #128]
2627; NONEON-NOSVE-NEXT:    ldr q0, [sp, #176]
2628; NONEON-NOSVE-NEXT:    str q0, [sp, #144]
2629; NONEON-NOSVE-NEXT:    ldr d0, [sp, #144]
2630; NONEON-NOSVE-NEXT:    stp d2, d0, [sp, #160]
2631; NONEON-NOSVE-NEXT:    ldr q0, [sp, #160]
2632; NONEON-NOSVE-NEXT:  .LBB12_4: // %else2
2633; NONEON-NOSVE-NEXT:    tbnz w8, #2, .LBB12_7
2634; NONEON-NOSVE-NEXT:  // %bb.5: // %else5
2635; NONEON-NOSVE-NEXT:    tbnz w8, #3, .LBB12_8
2636; NONEON-NOSVE-NEXT:  .LBB12_6: // %else8
2637; NONEON-NOSVE-NEXT:    add sp, sp, #224
2638; NONEON-NOSVE-NEXT:    ret
2639; NONEON-NOSVE-NEXT:  .LBB12_7: // %cond.load4
2640; NONEON-NOSVE-NEXT:    ldr d2, [x0, #16]
2641; NONEON-NOSVE-NEXT:    str q1, [sp, #80]
2642; NONEON-NOSVE-NEXT:    str d2, [sp, #112]
2643; NONEON-NOSVE-NEXT:    ldr d2, [sp, #88]
2644; NONEON-NOSVE-NEXT:    ldr q1, [sp, #112]
2645; NONEON-NOSVE-NEXT:    str q1, [sp, #64]
2646; NONEON-NOSVE-NEXT:    ldr d1, [sp, #64]
2647; NONEON-NOSVE-NEXT:    stp d1, d2, [sp, #96]
2648; NONEON-NOSVE-NEXT:    ldr q1, [sp, #96]
2649; NONEON-NOSVE-NEXT:    tbz w8, #3, .LBB12_6
2650; NONEON-NOSVE-NEXT:  .LBB12_8: // %cond.load7
2651; NONEON-NOSVE-NEXT:    ldr d2, [x0, #24]
2652; NONEON-NOSVE-NEXT:    str q1, [sp]
2653; NONEON-NOSVE-NEXT:    str d2, [sp, #48]
2654; NONEON-NOSVE-NEXT:    ldr d2, [sp]
2655; NONEON-NOSVE-NEXT:    ldr q1, [sp, #48]
2656; NONEON-NOSVE-NEXT:    str q1, [sp, #16]
2657; NONEON-NOSVE-NEXT:    ldr d1, [sp, #16]
2658; NONEON-NOSVE-NEXT:    stp d2, d1, [sp, #32]
2659; NONEON-NOSVE-NEXT:    ldr q1, [sp, #32]
2660; NONEON-NOSVE-NEXT:    add sp, sp, #224
2661; NONEON-NOSVE-NEXT:    ret
2662  %load = call <4 x double> @llvm.masked.load.v4f64(ptr %src, i32 8, <4 x i1> %mask, <4 x double> zeroinitializer)
2663  ret <4 x double> %load
2664}
2665
2666define <3 x i32> @masked_load_zext_v3i32(ptr %load_ptr, <3 x i1> %pm) {
2667; CHECK-LABEL: masked_load_zext_v3i32:
2668; CHECK:       // %bb.0:
2669; CHECK-NEXT:    fmov s0, w2
2670; CHECK-NEXT:    fmov s1, w1
2671; CHECK-NEXT:    adrp x8, .LCPI13_0
2672; CHECK-NEXT:    ptrue p0.s, vl4
2673; CHECK-NEXT:    zip1 z0.h, z1.h, z0.h
2674; CHECK-NEXT:    fmov s1, w3
2675; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
2676; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI13_0]
2677; CHECK-NEXT:    and z0.d, z0.d, z1.d
2678; CHECK-NEXT:    lsl z0.h, z0.h, #15
2679; CHECK-NEXT:    asr z0.h, z0.h, #15
2680; CHECK-NEXT:    sunpklo z0.s, z0.h
2681; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #0
2682; CHECK-NEXT:    ld1h { z0.s }, p0/z, [x0]
2683; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2684; CHECK-NEXT:    ret
2685;
2686; NONEON-NOSVE-LABEL: masked_load_zext_v3i32:
2687; NONEON-NOSVE:       // %bb.0:
2688; NONEON-NOSVE-NEXT:    sub sp, sp, #80
2689; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
2690; NONEON-NOSVE-NEXT:    and w8, w1, #0x1
2691; NONEON-NOSVE-NEXT:    bfi w8, w2, #1, #1
2692; NONEON-NOSVE-NEXT:    bfi w8, w3, #2, #1
2693; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB13_2
2694; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
2695; NONEON-NOSVE-NEXT:    ldrh w9, [x0]
2696; NONEON-NOSVE-NEXT:    stur wzr, [sp, #66]
2697; NONEON-NOSVE-NEXT:    strh w9, [sp, #64]
2698; NONEON-NOSVE-NEXT:    ldr d0, [sp, #64]
2699; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB13_3
2700; NONEON-NOSVE-NEXT:    b .LBB13_4
2701; NONEON-NOSVE-NEXT:  .LBB13_2:
2702; NONEON-NOSVE-NEXT:    adrp x9, .LCPI13_0
2703; NONEON-NOSVE-NEXT:    ldr d0, [x9, :lo12:.LCPI13_0]
2704; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB13_4
2705; NONEON-NOSVE-NEXT:  .LBB13_3: // %cond.load1
2706; NONEON-NOSVE-NEXT:    ldrh w9, [x0, #2]
2707; NONEON-NOSVE-NEXT:    str d0, [sp, #48]
2708; NONEON-NOSVE-NEXT:    strh w9, [sp, #58]
2709; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #52]
2710; NONEON-NOSVE-NEXT:    strh w9, [sp, #60]
2711; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #48]
2712; NONEON-NOSVE-NEXT:    strh w9, [sp, #56]
2713; NONEON-NOSVE-NEXT:    ldr d0, [sp, #56]
2714; NONEON-NOSVE-NEXT:  .LBB13_4: // %else2
2715; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB13_6
2716; NONEON-NOSVE-NEXT:  // %bb.5: // %cond.load4
2717; NONEON-NOSVE-NEXT:    ldrh w8, [x0, #4]
2718; NONEON-NOSVE-NEXT:    str d0, [sp, #32]
2719; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
2720; NONEON-NOSVE-NEXT:    ldr w8, [sp, #32]
2721; NONEON-NOSVE-NEXT:    str w8, [sp, #40]
2722; NONEON-NOSVE-NEXT:    ldr d0, [sp, #40]
2723; NONEON-NOSVE-NEXT:  .LBB13_6: // %else5
2724; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
2725; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #14]
2726; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #12]
2727; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
2728; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #10]
2729; NONEON-NOSVE-NEXT:    ldrh w8, [sp, #8]
2730; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
2731; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
2732; NONEON-NOSVE-NEXT:    add sp, sp, #80
2733; NONEON-NOSVE-NEXT:    ret
2734  %load_value = tail call <3 x i16> @llvm.masked.load.v3i16.p0(ptr %load_ptr, i32 4, <3 x i1> %pm, <3 x i16> zeroinitializer)
2735  %extend = zext <3 x i16> %load_value to <3 x i32>
2736  ret <3 x i32> %extend;
2737}
2738
2739define <3 x i32> @masked_load_sext_v3i32(ptr %load_ptr, <3 x i1> %pm) {
2740; CHECK-LABEL: masked_load_sext_v3i32:
2741; CHECK:       // %bb.0:
2742; CHECK-NEXT:    fmov s0, w2
2743; CHECK-NEXT:    fmov s1, w1
2744; CHECK-NEXT:    adrp x8, .LCPI14_0
2745; CHECK-NEXT:    ptrue p0.s, vl4
2746; CHECK-NEXT:    zip1 z0.h, z1.h, z0.h
2747; CHECK-NEXT:    fmov s1, w3
2748; CHECK-NEXT:    zip1 z0.s, z0.s, z1.s
2749; CHECK-NEXT:    ldr d1, [x8, :lo12:.LCPI14_0]
2750; CHECK-NEXT:    and z0.d, z0.d, z1.d
2751; CHECK-NEXT:    lsl z0.h, z0.h, #15
2752; CHECK-NEXT:    asr z0.h, z0.h, #15
2753; CHECK-NEXT:    sunpklo z0.s, z0.h
2754; CHECK-NEXT:    cmpne p0.s, p0/z, z0.s, #0
2755; CHECK-NEXT:    ld1sh { z0.s }, p0/z, [x0]
2756; CHECK-NEXT:    // kill: def $q0 killed $q0 killed $z0
2757; CHECK-NEXT:    ret
2758;
2759; NONEON-NOSVE-LABEL: masked_load_sext_v3i32:
2760; NONEON-NOSVE:       // %bb.0:
2761; NONEON-NOSVE-NEXT:    sub sp, sp, #80
2762; NONEON-NOSVE-NEXT:    .cfi_def_cfa_offset 80
2763; NONEON-NOSVE-NEXT:    and w8, w1, #0x1
2764; NONEON-NOSVE-NEXT:    bfi w8, w2, #1, #1
2765; NONEON-NOSVE-NEXT:    bfi w8, w3, #2, #1
2766; NONEON-NOSVE-NEXT:    tbz w8, #0, .LBB14_2
2767; NONEON-NOSVE-NEXT:  // %bb.1: // %cond.load
2768; NONEON-NOSVE-NEXT:    ldrh w9, [x0]
2769; NONEON-NOSVE-NEXT:    stur wzr, [sp, #66]
2770; NONEON-NOSVE-NEXT:    strh w9, [sp, #64]
2771; NONEON-NOSVE-NEXT:    ldr d0, [sp, #64]
2772; NONEON-NOSVE-NEXT:    tbnz w8, #1, .LBB14_3
2773; NONEON-NOSVE-NEXT:    b .LBB14_4
2774; NONEON-NOSVE-NEXT:  .LBB14_2:
2775; NONEON-NOSVE-NEXT:    adrp x9, .LCPI14_0
2776; NONEON-NOSVE-NEXT:    ldr d0, [x9, :lo12:.LCPI14_0]
2777; NONEON-NOSVE-NEXT:    tbz w8, #1, .LBB14_4
2778; NONEON-NOSVE-NEXT:  .LBB14_3: // %cond.load1
2779; NONEON-NOSVE-NEXT:    ldrh w9, [x0, #2]
2780; NONEON-NOSVE-NEXT:    str d0, [sp, #48]
2781; NONEON-NOSVE-NEXT:    strh w9, [sp, #58]
2782; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #52]
2783; NONEON-NOSVE-NEXT:    strh w9, [sp, #60]
2784; NONEON-NOSVE-NEXT:    ldrh w9, [sp, #48]
2785; NONEON-NOSVE-NEXT:    strh w9, [sp, #56]
2786; NONEON-NOSVE-NEXT:    ldr d0, [sp, #56]
2787; NONEON-NOSVE-NEXT:  .LBB14_4: // %else2
2788; NONEON-NOSVE-NEXT:    tbz w8, #2, .LBB14_6
2789; NONEON-NOSVE-NEXT:  // %bb.5: // %cond.load4
2790; NONEON-NOSVE-NEXT:    ldrh w8, [x0, #4]
2791; NONEON-NOSVE-NEXT:    str d0, [sp, #32]
2792; NONEON-NOSVE-NEXT:    strh w8, [sp, #44]
2793; NONEON-NOSVE-NEXT:    ldr w8, [sp, #32]
2794; NONEON-NOSVE-NEXT:    str w8, [sp, #40]
2795; NONEON-NOSVE-NEXT:    ldr d0, [sp, #40]
2796; NONEON-NOSVE-NEXT:  .LBB14_6: // %else5
2797; NONEON-NOSVE-NEXT:    str d0, [sp, #8]
2798; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #14]
2799; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #12]
2800; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #24]
2801; NONEON-NOSVE-NEXT:    ldrsh w9, [sp, #10]
2802; NONEON-NOSVE-NEXT:    ldrsh w8, [sp, #8]
2803; NONEON-NOSVE-NEXT:    stp w8, w9, [sp, #16]
2804; NONEON-NOSVE-NEXT:    ldr q0, [sp, #16]
2805; NONEON-NOSVE-NEXT:    add sp, sp, #80
2806; NONEON-NOSVE-NEXT:    ret
2807  %load_value = tail call <3 x i16> @llvm.masked.load.v3i16.p0(ptr %load_ptr, i32 4, <3 x i1> %pm, <3 x i16> zeroinitializer)
2808  %extend = sext <3 x i16> %load_value to <3 x i32>
2809  ret <3 x i32> %extend;
2810}
2811
2812declare <4 x i8> @llvm.masked.load.v4i8(ptr, i32, <4 x i1>, <4 x i8>)
2813declare <8 x i8> @llvm.masked.load.v8i8(ptr, i32, <8 x i1>, <8 x i8>)
2814declare <16 x i8> @llvm.masked.load.v16i8(ptr, i32, <16 x i1>, <16 x i8>)
2815declare <32 x i8> @llvm.masked.load.v32i8(ptr, i32, <32 x i1>, <32 x i8>)
2816
2817declare <2 x half> @llvm.masked.load.v2f16(ptr, i32, <2 x i1>, <2 x half>)
2818declare <4 x half> @llvm.masked.load.v4f16(ptr, i32, <4 x i1>, <4 x half>)
2819declare <8 x half> @llvm.masked.load.v8f16(ptr, i32, <8 x i1>, <8 x half>)
2820declare <16 x half> @llvm.masked.load.v16f16(ptr, i32, <16 x i1>, <16 x half>)
2821
2822declare <2 x float> @llvm.masked.load.v2f32(ptr, i32, <2 x i1>, <2 x float>)
2823declare <4 x float> @llvm.masked.load.v4f32(ptr, i32, <4 x i1>, <4 x float>)
2824declare <8 x float> @llvm.masked.load.v8f32(ptr, i32, <8 x i1>, <8 x float>)
2825
2826declare <2 x double> @llvm.masked.load.v2f64(ptr, i32, <2 x i1>, <2 x double>)
2827declare <4 x double> @llvm.masked.load.v4f64(ptr, i32, <4 x i1>, <4 x double>)
2828
2829declare <3 x i16> @llvm.masked.load.v3i16.p0(ptr, i32, <3 x i1>, <3 x i16>)
2830