xref: /llvm-project/llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll (revision b5b663aac17415625340eb29c8010832bfc4c21c) 
1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
2; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -tail-predication=enabled --verify-machineinstrs %s -o - | FileCheck %s
3
4define dso_local i32 @vpsel_mul_reduce_add(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture readonly %c, i32 %N) {
5; CHECK-LABEL: vpsel_mul_reduce_add:
6; CHECK:       @ %bb.0: @ %entry
7; CHECK-NEXT:    cmp r3, #0
8; CHECK-NEXT:    itt eq
9; CHECK-NEXT:    moveq r0, #0
10; CHECK-NEXT:    bxeq lr
11; CHECK-NEXT:  .LBB0_1: @ %vector.ph
12; CHECK-NEXT:    push {r4, lr}
13; CHECK-NEXT:    add.w r12, r3, #3
14; CHECK-NEXT:    mov.w lr, #1
15; CHECK-NEXT:    bic r12, r12, #3
16; CHECK-NEXT:    vmov.i32 q1, #0x0
17; CHECK-NEXT:    sub.w r12, r12, #4
18; CHECK-NEXT:    add.w lr, lr, r12, lsr #2
19; CHECK-NEXT:    mov.w r12, #0
20; CHECK-NEXT:  .LBB0_2: @ %vector.body
21; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
22; CHECK-NEXT:    and r4, r12, #15
23; CHECK-NEXT:    vctp.32 r3
24; CHECK-NEXT:    vmov q0, q1
25; CHECK-NEXT:    vpstt
26; CHECK-NEXT:    vldrwt.u32 q1, [r2], #16
27; CHECK-NEXT:    vldrwt.u32 q2, [r1], #16
28; CHECK-NEXT:    vdup.32 q3, r4
29; CHECK-NEXT:    add.w r12, r12, #4
30; CHECK-NEXT:    vpt.i32 eq, q3, zr
31; CHECK-NEXT:    vmovt q1, q2
32; CHECK-NEXT:    vctp.32 r3
33; CHECK-NEXT:    vpst
34; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
35; CHECK-NEXT:    subs r3, #4
36; CHECK-NEXT:    vmul.i32 q1, q1, q2
37; CHECK-NEXT:    vadd.i32 q1, q1, q0
38; CHECK-NEXT:    le lr, .LBB0_2
39; CHECK-NEXT:  @ %bb.3: @ %middle.block
40; CHECK-NEXT:    vpsel q0, q1, q0
41; CHECK-NEXT:    vaddv.u32 r0, q0
42; CHECK-NEXT:    pop {r4, pc}
43entry:
44  %cmp8 = icmp eq i32 %N, 0
45  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
46
47vector.ph:                                        ; preds = %entry
48  %n.rnd.up = add i32 %N, 3
49  %n.vec = and i32 %n.rnd.up, -4
50  br label %vector.body
51
52vector.body:                                      ; preds = %vector.body, %vector.ph
53  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
54  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
55  %tmp = getelementptr inbounds i32, ptr %a, i32 %index
56  %tmp1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)
57  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
58  %tmp3 = getelementptr inbounds i32, ptr %b, i32 %index
59  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp3, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
60  %tmp5 = getelementptr inbounds i32, ptr %c, i32 %index
61  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp5, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
62  %rem = urem i32 %index, 16
63  %rem.broadcast.splatinsert = insertelement <4 x i32> undef, i32 %rem, i32 0
64  %rem.broadcast.splat = shufflevector <4 x i32> %rem.broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
65  %cmp = icmp eq <4 x i32> %rem.broadcast.splat, <i32 0, i32 0, i32 0, i32 0>
66  %wide.masked.load = select <4 x i1> %cmp, <4 x i32> %wide.masked.load.b, <4 x i32> %wide.masked.load.c
67  %mul = mul nsw <4 x i32> %wide.masked.load, %wide.masked.load.a
68  %add = add nsw <4 x i32> %mul, %vec.phi
69  %index.next = add i32 %index, 4
70  %tmp7 = icmp eq i32 %index.next, %n.vec
71  br i1 %tmp7, label %middle.block, label %vector.body
72
73middle.block:                                     ; preds = %vector.body
74  %tmp8 = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
75  %tmp9 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %tmp8)
76  br label %for.cond.cleanup
77
78for.cond.cleanup:                                 ; preds = %middle.block, %entry
79  %res.0.lcssa = phi i32 [ 0, %entry ], [ %tmp9, %middle.block ]
80  ret i32 %res.0.lcssa
81}
82
83define dso_local i32 @vpsel_mul_reduce_add_2(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b,
84; CHECK-LABEL: vpsel_mul_reduce_add_2:
85; CHECK:       @ %bb.0: @ %entry
86; CHECK-NEXT:    push {r4, r5, r7, lr}
87; CHECK-NEXT:    vpush {d8, d9}
88; CHECK-NEXT:    ldr.w r12, [sp, #32]
89; CHECK-NEXT:    cmp.w r12, #0
90; CHECK-NEXT:    beq .LBB1_4
91; CHECK-NEXT:  @ %bb.1: @ %vector.ph
92; CHECK-NEXT:    add.w r4, r12, #3
93; CHECK-NEXT:    vmov.i32 q1, #0x0
94; CHECK-NEXT:    bic r4, r4, #3
95; CHECK-NEXT:    sub.w lr, r4, #4
96; CHECK-NEXT:    movs r4, #1
97; CHECK-NEXT:    add.w lr, r4, lr, lsr #2
98; CHECK-NEXT:    movs r4, #0
99; CHECK-NEXT:  .LBB1_2: @ %vector.body
100; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
101; CHECK-NEXT:    vctp.32 r12
102; CHECK-NEXT:    and r5, r4, #15
103; CHECK-NEXT:    vmov q0, q1
104; CHECK-NEXT:    vpsttt
105; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
106; CHECK-NEXT:    vldrwt.u32 q2, [r3], #16
107; CHECK-NEXT:    vldrwt.u32 q3, [r2], #16
108; CHECK-NEXT:    vdup.32 q4, r5
109; CHECK-NEXT:    adds r4, #4
110; CHECK-NEXT:    vpt.i32 eq, q4, zr
111; CHECK-NEXT:    vsubt.i32 q1, q3, q2
112; CHECK-NEXT:    vctp.32 r12
113; CHECK-NEXT:    vpst
114; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
115; CHECK-NEXT:    sub.w r12, r12, #4
116; CHECK-NEXT:    vmul.i32 q1, q1, q2
117; CHECK-NEXT:    vadd.i32 q1, q1, q0
118; CHECK-NEXT:    le lr, .LBB1_2
119; CHECK-NEXT:  @ %bb.3: @ %middle.block
120; CHECK-NEXT:    vpsel q0, q1, q0
121; CHECK-NEXT:    vaddv.u32 r0, q0
122; CHECK-NEXT:    vpop {d8, d9}
123; CHECK-NEXT:    pop {r4, r5, r7, pc}
124; CHECK-NEXT:  .LBB1_4:
125; CHECK-NEXT:    movs r0, #0
126; CHECK-NEXT:    vpop {d8, d9}
127; CHECK-NEXT:    pop {r4, r5, r7, pc}
128                                         ptr noalias nocapture readonly %c, ptr noalias nocapture readonly %d, i32 %N) {
129entry:
130  %cmp8 = icmp eq i32 %N, 0
131  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
132
133vector.ph:                                        ; preds = %entry
134  %n.rnd.up = add i32 %N, 3
135  %n.vec = and i32 %n.rnd.up, -4
136  br label %vector.body
137
138vector.body:                                      ; preds = %vector.body, %vector.ph
139  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
140  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
141  %tmp = getelementptr inbounds i32, ptr %a, i32 %index
142  %tmp1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)
143  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
144  %tmp3 = getelementptr inbounds i32, ptr %b, i32 %index
145  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp3, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
146  %tmp5 = getelementptr inbounds i32, ptr %c, i32 %index
147  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp5, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
148  %tmp7 = getelementptr inbounds i32, ptr %d, i32 %index
149  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp7, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
150  %sub = sub <4 x i32> %wide.masked.load.c, %wide.masked.load.d
151  %rem = urem i32 %index, 16
152  %rem.broadcast.splatinsert = insertelement <4 x i32> undef, i32 %rem, i32 0
153  %rem.broadcast.splat = shufflevector <4 x i32> %rem.broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer
154  %cmp = icmp eq <4 x i32> %rem.broadcast.splat, <i32 0, i32 0, i32 0, i32 0>
155  %sel = select <4 x i1> %cmp, <4 x i32> %sub, <4 x i32> %wide.masked.load.b
156  %mul = mul  <4 x i32> %sel, %wide.masked.load.a
157  %add = add  <4 x i32> %mul, %vec.phi
158  %index.next = add i32 %index, 4
159  %cmp.exit = icmp eq i32 %index.next, %n.vec
160  br i1 %cmp.exit, label %middle.block, label %vector.body
161
162middle.block:                                     ; preds = %vector.body
163  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
164  %reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc)
165  br label %for.cond.cleanup
166
167for.cond.cleanup:                                 ; preds = %middle.block, %entry
168  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
169  ret i32 %res.0.lcssa
170}
171
172define dso_local i32 @and_mul_reduce_add(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b,
173; CHECK-LABEL: and_mul_reduce_add:
174; CHECK:       @ %bb.0: @ %entry
175; CHECK-NEXT:    push {r4, lr}
176; CHECK-NEXT:    sub sp, #4
177; CHECK-NEXT:    ldr.w r12, [sp, #12]
178; CHECK-NEXT:    cmp.w r12, #0
179; CHECK-NEXT:    beq .LBB2_4
180; CHECK-NEXT:  @ %bb.1: @ %vector.ph
181; CHECK-NEXT:    add.w lr, r12, #3
182; CHECK-NEXT:    movs r4, #1
183; CHECK-NEXT:    bic lr, lr, #3
184; CHECK-NEXT:    vmov.i32 q1, #0x0
185; CHECK-NEXT:    sub.w lr, lr, #4
186; CHECK-NEXT:    add.w r4, r4, lr, lsr #2
187; CHECK-NEXT:    dls lr, r4
188; CHECK-NEXT:  .LBB2_2: @ %vector.body
189; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
190; CHECK-NEXT:    vctp.32 r12
191; CHECK-NEXT:    vmov q0, q1
192; CHECK-NEXT:    vpstt
193; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
194; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
195; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
196; CHECK-NEXT:    sub.w r12, r12, #4
197; CHECK-NEXT:    vsub.i32 q1, q2, q1
198; CHECK-NEXT:    vpsttt
199; CHECK-NEXT:    vcmpt.i32 eq, q1, zr
200; CHECK-NEXT:    vldrwt.u32 q1, [r3], #16
201; CHECK-NEXT:    vldrwt.u32 q2, [r2], #16
202; CHECK-NEXT:    vmul.i32 q1, q2, q1
203; CHECK-NEXT:    vadd.i32 q1, q1, q0
204; CHECK-NEXT:    le lr, .LBB2_2
205; CHECK-NEXT:  @ %bb.3: @ %middle.block
206; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
207; CHECK-NEXT:    vpsel q0, q1, q0
208; CHECK-NEXT:    vaddv.u32 r0, q0
209; CHECK-NEXT:    add sp, #4
210; CHECK-NEXT:    pop {r4, pc}
211; CHECK-NEXT:  .LBB2_4:
212; CHECK-NEXT:    movs r0, #0
213; CHECK-NEXT:    add sp, #4
214; CHECK-NEXT:    pop {r4, pc}
215                                         ptr noalias nocapture readonly %c, ptr noalias nocapture readonly %d, i32 %N) {
216entry:
217  %cmp8 = icmp eq i32 %N, 0
218  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
219
220vector.ph:                                        ; preds = %entry
221  %n.rnd.up = add i32 %N, 3
222  %n.vec = and i32 %n.rnd.up, -4
223  br label %vector.body
224
225vector.body:                                      ; preds = %vector.body, %vector.ph
226  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
227  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
228  %tmp = getelementptr inbounds i32, ptr %a, i32 %index
229  %tmp1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)
230  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
231  %tmp3 = getelementptr inbounds i32, ptr %b, i32 %index
232  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp3, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
233  %sub = sub <4 x i32> %wide.masked.load.a, %wide.masked.load.b
234  %cmp = icmp eq <4 x i32> %sub, <i32 0, i32 0, i32 0, i32 0>
235  %mask = and <4 x i1> %cmp, %tmp1
236  %tmp5 = getelementptr inbounds i32, ptr %c, i32 %index
237  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp5, i32 4, <4 x i1> %mask, <4 x i32> undef)
238  %tmp7 = getelementptr inbounds i32, ptr %d, i32 %index
239  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp7, i32 4, <4 x i1> %mask, <4 x i32> undef)
240  %mul = mul  <4 x i32> %wide.masked.load.c, %wide.masked.load.d
241  %add = add  <4 x i32> %mul, %vec.phi
242  %index.next = add i32 %index, 4
243  %cmp.exit = icmp eq i32 %index.next, %n.vec
244  br i1 %cmp.exit, label %middle.block, label %vector.body
245
246middle.block:                                     ; preds = %vector.body
247  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
248  %reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc)
249  br label %for.cond.cleanup
250
251for.cond.cleanup:                                 ; preds = %middle.block, %entry
252  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
253  ret i32 %res.0.lcssa
254}
255
256define dso_local i32 @or_mul_reduce_add(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture readonly %c, ptr noalias nocapture readonly %d, i32 %N) {
257; CHECK-LABEL: or_mul_reduce_add:
258; CHECK:       @ %bb.0: @ %entry
259; CHECK-NEXT:    push {r4, lr}
260; CHECK-NEXT:    sub sp, #4
261; CHECK-NEXT:    ldr.w r12, [sp, #12]
262; CHECK-NEXT:    cmp.w r12, #0
263; CHECK-NEXT:    beq .LBB3_4
264; CHECK-NEXT:  @ %bb.1: @ %vector.ph
265; CHECK-NEXT:    add.w lr, r12, #3
266; CHECK-NEXT:    movs r4, #1
267; CHECK-NEXT:    bic lr, lr, #3
268; CHECK-NEXT:    vmov.i32 q1, #0x0
269; CHECK-NEXT:    sub.w lr, lr, #4
270; CHECK-NEXT:    add.w r4, r4, lr, lsr #2
271; CHECK-NEXT:    dls lr, r4
272; CHECK-NEXT:  .LBB3_2: @ %vector.body
273; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
274; CHECK-NEXT:    vctp.32 r12
275; CHECK-NEXT:    vmov q0, q1
276; CHECK-NEXT:    vpstt
277; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
278; CHECK-NEXT:    vldrwt.u32 q2, [r0], #16
279; CHECK-NEXT:    vstr p0, [sp] @ 4-byte Spill
280; CHECK-NEXT:    vpnot
281; CHECK-NEXT:    vsub.i32 q1, q2, q1
282; CHECK-NEXT:    sub.w r12, r12, #4
283; CHECK-NEXT:    vpstee
284; CHECK-NEXT:    vcmpt.i32 ne, q1, zr
285; CHECK-NEXT:    vldrwe.u32 q1, [r3], #16
286; CHECK-NEXT:    vldrwe.u32 q2, [r2], #16
287; CHECK-NEXT:    vmul.i32 q1, q2, q1
288; CHECK-NEXT:    vadd.i32 q1, q1, q0
289; CHECK-NEXT:    le lr, .LBB3_2
290; CHECK-NEXT:  @ %bb.3: @ %middle.block
291; CHECK-NEXT:    vldr p0, [sp] @ 4-byte Reload
292; CHECK-NEXT:    vpsel q0, q1, q0
293; CHECK-NEXT:    vaddv.u32 r0, q0
294; CHECK-NEXT:    add sp, #4
295; CHECK-NEXT:    pop {r4, pc}
296; CHECK-NEXT:  .LBB3_4:
297; CHECK-NEXT:    movs r0, #0
298; CHECK-NEXT:    add sp, #4
299; CHECK-NEXT:    pop {r4, pc}
300entry:
301  %cmp8 = icmp eq i32 %N, 0
302  br i1 %cmp8, label %for.cond.cleanup, label %vector.ph
303
304vector.ph:                                        ; preds = %entry
305  %n.rnd.up = add i32 %N, 3
306  %n.vec = and i32 %n.rnd.up, -4
307  br label %vector.body
308
309vector.body:                                      ; preds = %vector.body, %vector.ph
310  %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
311  %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %add, %vector.body ]
312  %tmp = getelementptr inbounds i32, ptr %a, i32 %index
313  %tmp1 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %index, i32 %N)
314  %wide.masked.load.a = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
315  %tmp3 = getelementptr inbounds i32, ptr %b, i32 %index
316  %wide.masked.load.b = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp3, i32 4, <4 x i1> %tmp1, <4 x i32> undef)
317  %sub = sub <4 x i32> %wide.masked.load.a, %wide.masked.load.b
318  %cmp = icmp eq <4 x i32> %sub, <i32 0, i32 0, i32 0, i32 0>
319  %mask = or <4 x i1> %cmp, %tmp1
320  %tmp5 = getelementptr inbounds i32, ptr %c, i32 %index
321  %wide.masked.load.c = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp5, i32 4, <4 x i1> %mask, <4 x i32> undef)
322  %tmp7 = getelementptr inbounds i32, ptr %d, i32 %index
323  %wide.masked.load.d = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp7, i32 4, <4 x i1> %mask, <4 x i32> undef)
324  %mul = mul  <4 x i32> %wide.masked.load.c, %wide.masked.load.d
325  %add = add  <4 x i32> %mul, %vec.phi
326  %index.next = add i32 %index, 4
327  %cmp.exit = icmp eq i32 %index.next, %n.vec
328  br i1 %cmp.exit, label %middle.block, label %vector.body
329
330middle.block:                                     ; preds = %vector.body
331  %acc = select <4 x i1> %tmp1, <4 x i32> %add, <4 x i32> %vec.phi
332  %reduce = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %acc)
333  br label %for.cond.cleanup
334
335for.cond.cleanup:                                 ; preds = %middle.block, %entry
336  %res.0.lcssa = phi i32 [ 0, %entry ], [ %reduce, %middle.block ]
337  ret i32 %res.0.lcssa
338}
339
340define dso_local void @continue_on_zero(ptr noalias nocapture %arg, ptr noalias nocapture readonly %arg1, i32 %arg2) {
341; CHECK-LABEL: continue_on_zero:
342; CHECK:       @ %bb.0: @ %bb
343; CHECK-NEXT:    push {r7, lr}
344; CHECK-NEXT:    cmp r2, #0
345; CHECK-NEXT:    it eq
346; CHECK-NEXT:    popeq {r7, pc}
347; CHECK-NEXT:  .LBB4_1: @ %bb3
348; CHECK-NEXT:    dlstp.32 lr, r2
349; CHECK-NEXT:  .LBB4_2: @ %bb9
350; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
351; CHECK-NEXT:    vldrw.u32 q0, [r1], #16
352; CHECK-NEXT:    vpt.i32 ne, q0, zr
353; CHECK-NEXT:    vldrwt.u32 q1, [r0]
354; CHECK-NEXT:    vmul.i32 q0, q1, q0
355; CHECK-NEXT:    vpst
356; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
357; CHECK-NEXT:    letp lr, .LBB4_2
358; CHECK-NEXT:  @ %bb.3: @ %bb27
359; CHECK-NEXT:    pop {r7, pc}
360bb:
361  %tmp = icmp eq i32 %arg2, 0
362  br i1 %tmp, label %bb27, label %bb3
363
364bb3:                                              ; preds = %bb
365  %tmp4 = add i32 %arg2, 3
366  %tmp5 = and i32 %tmp4, -4
367  br label %bb9
368
369bb9:                                              ; preds = %bb9, %bb3
370  %tmp10 = phi i32 [ 0, %bb3 ], [ %tmp25, %bb9 ]
371  %tmp14 = getelementptr inbounds i32, ptr %arg1, i32 %tmp10
372  %tmp15 = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %tmp10, i32 %arg2)
373  %tmp17 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp14, i32 4, <4 x i1> %tmp15, <4 x i32> undef)
374  %tmp18 = icmp ne <4 x i32> %tmp17, zeroinitializer
375  %tmp19 = getelementptr inbounds i32, ptr %arg, i32 %tmp10
376  %tmp20 = and <4 x i1> %tmp18, %tmp15
377  %tmp22 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp19, i32 4, <4 x i1> %tmp20, <4 x i32> undef)
378  %tmp23 = mul nsw <4 x i32> %tmp22, %tmp17
379  call void @llvm.masked.store.v4i32.p0(<4 x i32> %tmp23, ptr %tmp19, i32 4, <4 x i1> %tmp20)
380  %tmp25 = add i32 %tmp10, 4
381  %tmp26 = icmp eq i32 %tmp25, %tmp5
382  br i1 %tmp26, label %bb27, label %bb9
383
384bb27:                                             ; preds = %bb9, %bb
385  ret void
386}
387
388define dso_local arm_aapcs_vfpcc void @range_test(ptr noalias nocapture %arg, ptr noalias nocapture readonly %arg1, i32 %arg2, i32 %arg3) {
389; CHECK-LABEL: range_test:
390; CHECK:       @ %bb.0: @ %bb
391; CHECK-NEXT:    push {r7, lr}
392; CHECK-NEXT:    cmp r3, #0
393; CHECK-NEXT:    it eq
394; CHECK-NEXT:    popeq {r7, pc}
395; CHECK-NEXT:  .LBB5_1: @ %bb4
396; CHECK-NEXT:    dlstp.32 lr, r3
397; CHECK-NEXT:  .LBB5_2: @ %bb12
398; CHECK-NEXT:    @ =>This Inner Loop Header: Depth=1
399; CHECK-NEXT:    vldrw.u32 q0, [r0]
400; CHECK-NEXT:    vpt.i32 ne, q0, zr
401; CHECK-NEXT:    vcmpt.s32 le, q0, r2
402; CHECK-NEXT:    vpst
403; CHECK-NEXT:    vldrwt.u32 q1, [r1], #16
404; CHECK-NEXT:    vmul.i32 q0, q1, q0
405; CHECK-NEXT:    vpst
406; CHECK-NEXT:    vstrwt.32 q0, [r0], #16
407; CHECK-NEXT:    letp lr, .LBB5_2
408; CHECK-NEXT:  @ %bb.3: @ %bb32
409; CHECK-NEXT:    pop {r7, pc}
410bb:
411  %tmp = icmp eq i32 %arg3, 0
412  br i1 %tmp, label %bb32, label %bb4
413
414bb4:                                              ; preds = %bb
415  %tmp5 = add i32 %arg3, 3
416  %tmp6 = and i32 %tmp5, -4
417  %tmp10 = insertelement <4 x i32> undef, i32 %arg2, i32 0
418  %tmp11 = shufflevector <4 x i32> %tmp10, <4 x i32> undef, <4 x i32> zeroinitializer
419  br label %bb12
420
421bb12:                                             ; preds = %bb12, %bb4
422  %tmp13 = phi i32 [ 0, %bb4 ], [ %tmp30, %bb12 ]
423  %tmp17 = getelementptr inbounds i32, ptr %arg, i32 %tmp13
424  %tmp18= call <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32 %tmp13, i32 %arg3)
425  %tmp20 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp17, i32 4, <4 x i1> %tmp18, <4 x i32> undef)
426  %tmp21 = icmp ne <4 x i32> %tmp20, zeroinitializer
427  %tmp22 = icmp sle <4 x i32> %tmp20, %tmp11
428  %tmp23 = getelementptr inbounds i32, ptr %arg1, i32 %tmp13
429  %tmp24 = and <4 x i1> %tmp22, %tmp21
430  %tmp25 = and <4 x i1> %tmp24, %tmp18
431  %tmp27 = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr %tmp23, i32 4, <4 x i1> %tmp25, <4 x i32> undef)
432  %tmp28 = mul nsw <4 x i32> %tmp27, %tmp20
433  call void @llvm.masked.store.v4i32.p0(<4 x i32> %tmp28, ptr %tmp17, i32 4, <4 x i1> %tmp25)
434  %tmp30 = add i32 %tmp13, 4
435  %tmp31 = icmp eq i32 %tmp30, %tmp6
436  br i1 %tmp31, label %bb32, label %bb12
437
438bb32:                                             ; preds = %bb12, %bb
439  ret void
440}
441
442; Function Attrs: argmemonly nounwind readonly willreturn
443declare <4 x i32> @llvm.masked.load.v4i32.p0(ptr, i32 immarg, <4 x i1>, <4 x i32>)
444declare void @llvm.masked.store.v4i32.p0(<4 x i32>, ptr, i32, <4 x i1>)
445
446; Function Attrs: nounwind readnone willreturn
447declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
448
449declare <4 x i1> @llvm.get.active.lane.mask.v4i1.i32(i32, i32)
450