1; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py 2 3; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp --arm-memtransfer-tploop=allow %s -o - | FileCheck %s 4 5!0 = !{i32 1, !"wchar_size", i32 4} 6!1 = !{i32 1, !"min_enum_size", i32 4} 7!2 = !{!"clang version 11.0.0 (git@github.com:llvm/llvm-project.git 26f04d01a39a33d73fd23165c208b215bf5c350d)"} 8!3 = !{!4, !4, i64 0} 9!4 = !{!"int", !5, i64 0} 10!5 = !{!"omnipotent char", !6, i64 0} 11!6 = !{!"Simple C/C++ TBAA"} 12!7 = distinct !{!7, !8} 13!8 = !{!"llvm.loop.isvectorized", i32 1} 14!9 = distinct !{!9, !10, !8} 15!10 = !{!"llvm.loop.unroll.runtime.disable"} 16 17 18 19define arm_aapcs_vfpcc void @push_out_mul_gather(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) { 20; CHECK-LABEL: push_out_mul_gather: 21; CHECK: @ %bb.0: @ %vector.ph 22; CHECK-NEXT: adr r3, .LCPI0_0 23; CHECK-NEXT: vldrw.u32 q0, [r3] 24; CHECK-NEXT: vadd.i32 q0, q0, r0 25; CHECK-NEXT: .LBB0_1: @ %vector.body 26; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 27; CHECK-NEXT: vldrw.u32 q1, [q0, #96]! 28; CHECK-NEXT: subs r2, #4 29; CHECK-NEXT: vstrb.8 q1, [r1], #16 30; CHECK-NEXT: bne .LBB0_1 31; CHECK-NEXT: @ %bb.2: @ %end 32; CHECK-NEXT: bx lr 33; CHECK-NEXT: .p2align 4 34; CHECK-NEXT: @ %bb.3: 35; CHECK-NEXT: .LCPI0_0: 36; CHECK-NEXT: .long 4294967200 @ 0xffffffa0 37; CHECK-NEXT: .long 4294967224 @ 0xffffffb8 38; CHECK-NEXT: .long 4294967248 @ 0xffffffd0 39; CHECK-NEXT: .long 4294967272 @ 0xffffffe8 40 41vector.ph: ; preds = %for.body.preheader 42 br label %vector.body 43 44vector.body: ; preds = %vector.body, %vector.ph 45 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 46 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ] 47 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3> 48 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0 49 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 50 %2 = getelementptr inbounds i32, ptr %dst, i32 %index 51 %3 = bitcast ptr %2 to ptr 52 store <4 x i32> %wide.masked.gather, ptr %3, align 4 53 %index.next = add i32 %index, 4 54 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 55 %4 = icmp eq i32 %index.next, %n.vec 56 br i1 %4, label %end, label %vector.body 57 58end: 59 ret void; 60} 61 62define arm_aapcs_vfpcc void @push_out_add_gather(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) { 63; CHECK-LABEL: push_out_add_gather: 64; CHECK: @ %bb.0: @ %vector.ph 65; CHECK-NEXT: adr r3, .LCPI1_0 66; CHECK-NEXT: vldrw.u32 q0, [r3] 67; CHECK-NEXT: vadd.i32 q0, q0, r0 68; CHECK-NEXT: .LBB1_1: @ %vector.body 69; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 70; CHECK-NEXT: vldrw.u32 q1, [q0, #32]! 71; CHECK-NEXT: subs r2, #4 72; CHECK-NEXT: vstrb.8 q1, [r1], #16 73; CHECK-NEXT: bne .LBB1_1 74; CHECK-NEXT: @ %bb.2: @ %end 75; CHECK-NEXT: bx lr 76; CHECK-NEXT: .p2align 4 77; CHECK-NEXT: @ %bb.3: 78; CHECK-NEXT: .LCPI1_0: 79; CHECK-NEXT: .long 4294967288 @ 0xfffffff8 80; CHECK-NEXT: .long 0 @ 0x0 81; CHECK-NEXT: .long 8 @ 0x8 82; CHECK-NEXT: .long 16 @ 0x10 83 84vector.ph: ; preds = %for.body.preheader 85 br label %vector.body 86 87vector.body: ; preds = %vector.body, %vector.ph 88 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 89 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ] 90 %0 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6> 91 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0 92 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 93 %2 = getelementptr inbounds i32, ptr %dst, i32 %index 94 %3 = bitcast ptr %2 to ptr 95 store <4 x i32> %wide.masked.gather, ptr %3, align 4 96 %index.next = add i32 %index, 4 97 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 98 %4 = icmp eq i32 %index.next, %n.vec 99 br i1 %4, label %end, label %vector.body 100 101end: 102 ret void; 103} 104 105define arm_aapcs_vfpcc void @push_out_mul_add_gather(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) { 106; CHECK-LABEL: push_out_mul_add_gather: 107; CHECK: @ %bb.0: @ %vector.ph 108; CHECK-NEXT: adr r3, .LCPI2_0 109; CHECK-NEXT: vldrw.u32 q0, [r3] 110; CHECK-NEXT: vadd.i32 q0, q0, r0 111; CHECK-NEXT: .LBB2_1: @ %vector.body 112; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 113; CHECK-NEXT: vldrw.u32 q1, [q0, #96]! 114; CHECK-NEXT: subs r2, #4 115; CHECK-NEXT: vstrb.8 q1, [r1], #16 116; CHECK-NEXT: bne .LBB2_1 117; CHECK-NEXT: @ %bb.2: @ %end 118; CHECK-NEXT: bx lr 119; CHECK-NEXT: .p2align 4 120; CHECK-NEXT: @ %bb.3: 121; CHECK-NEXT: .LCPI2_0: 122; CHECK-NEXT: .long 4294967224 @ 0xffffffb8 123; CHECK-NEXT: .long 4294967248 @ 0xffffffd0 124; CHECK-NEXT: .long 4294967272 @ 0xffffffe8 125; CHECK-NEXT: .long 0 @ 0x0 126 127vector.ph: ; preds = %for.body.preheader 128 br label %vector.body 129 130vector.body: ; preds = %vector.body, %vector.ph 131 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 132 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ] 133 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3> 134 %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6> 135 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %1 136 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 137 %3 = getelementptr inbounds i32, ptr %dst, i32 %index 138 %4 = bitcast ptr %3 to ptr 139 store <4 x i32> %wide.masked.gather, ptr %4, align 4 140 %index.next = add i32 %index, 4 141 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 142 %5 = icmp eq i32 %index.next, %n.vec 143 br i1 %5, label %end, label %vector.body 144 145end: 146 ret void; 147} 148 149define arm_aapcs_vfpcc void @push_out_mul_scatter(ptr noalias nocapture readonly %data, 150; CHECK-LABEL: push_out_mul_scatter: 151; CHECK: @ %bb.0: @ %vector.ph 152; CHECK-NEXT: adr r1, .LCPI3_0 153; CHECK-NEXT: vldrw.u32 q1, [r1] 154; CHECK-NEXT: vadd.i32 q1, q1, r0 155; CHECK-NEXT: .LBB3_1: @ %vector.body 156; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 157; CHECK-NEXT: subs r2, #4 158; CHECK-NEXT: vstrw.32 q0, [q1, #96]! 159; CHECK-NEXT: bne .LBB3_1 160; CHECK-NEXT: @ %bb.2: @ %end 161; CHECK-NEXT: bx lr 162; CHECK-NEXT: .p2align 4 163; CHECK-NEXT: @ %bb.3: 164; CHECK-NEXT: .LCPI3_0: 165; CHECK-NEXT: .long 4294967200 @ 0xffffffa0 166; CHECK-NEXT: .long 4294967224 @ 0xffffffb8 167; CHECK-NEXT: .long 4294967248 @ 0xffffffd0 168; CHECK-NEXT: .long 4294967272 @ 0xffffffe8 169 ptr noalias nocapture %dst, i32 %n.vec, 170 <4 x i32> %to.store) { 171 172vector.ph: ; preds = %for.body.preheader 173 br label %vector.body 174 175vector.body: ; preds = %vector.body, %vector.ph 176 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 177 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ] 178 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3> 179 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0 180 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %to.store, <4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>) 181 %index.next = add i32 %index, 4 182 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 183 %2 = icmp eq i32 %index.next, %n.vec 184 br i1 %2, label %end, label %vector.body 185 186end: 187 ret void; 188} 189 190define arm_aapcs_vfpcc void @push_out_add_scatter(ptr noalias nocapture readonly %data, 191; CHECK-LABEL: push_out_add_scatter: 192; CHECK: @ %bb.0: @ %vector.ph 193; CHECK-NEXT: adr r1, .LCPI4_0 194; CHECK-NEXT: vldrw.u32 q1, [r1] 195; CHECK-NEXT: vadd.i32 q1, q1, r0 196; CHECK-NEXT: .LBB4_1: @ %vector.body 197; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 198; CHECK-NEXT: subs r2, #4 199; CHECK-NEXT: vstrw.32 q0, [q1, #32]! 200; CHECK-NEXT: bne .LBB4_1 201; CHECK-NEXT: @ %bb.2: @ %end 202; CHECK-NEXT: bx lr 203; CHECK-NEXT: .p2align 4 204; CHECK-NEXT: @ %bb.3: 205; CHECK-NEXT: .LCPI4_0: 206; CHECK-NEXT: .long 4294967288 @ 0xfffffff8 207; CHECK-NEXT: .long 0 @ 0x0 208; CHECK-NEXT: .long 8 @ 0x8 209; CHECK-NEXT: .long 16 @ 0x10 210 ptr noalias nocapture %dst, i32 %n.vec, 211 <4 x i32> %to.store) { 212 213vector.ph: ; preds = %for.body.preheader 214 br label %vector.body 215 216vector.body: ; preds = %vector.body, %vector.ph 217 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 218 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ] 219 %0 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6> 220 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0 221 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %to.store, <4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>) 222 %index.next = add i32 %index, 4 223 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 224 %2 = icmp eq i32 %index.next, %n.vec 225 br i1 %2, label %end, label %vector.body 226 227end: 228 ret void; 229} 230 231define arm_aapcs_vfpcc void @push_out_mul_gather_scatter(ptr noalias nocapture readonly %data, 232; CHECK-LABEL: push_out_mul_gather_scatter: 233; CHECK: @ %bb.0: @ %vector.ph 234; CHECK-NEXT: adr r1, .LCPI5_0 235; CHECK-NEXT: vmov.i32 q0, #0x18 236; CHECK-NEXT: vldrw.u32 q1, [r1] 237; CHECK-NEXT: .LBB5_1: @ %vector.body 238; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 239; CHECK-NEXT: vldrw.u32 q2, [r0, q1, uxtw #2] 240; CHECK-NEXT: vadd.i32 q3, q1, q0 241; CHECK-NEXT: subs r2, #4 242; CHECK-NEXT: vstrw.32 q2, [r0, q1, uxtw #2] 243; CHECK-NEXT: vmov q1, q3 244; CHECK-NEXT: bne .LBB5_1 245; CHECK-NEXT: @ %bb.2: @ %end 246; CHECK-NEXT: bx lr 247; CHECK-NEXT: .p2align 4 248; CHECK-NEXT: @ %bb.3: 249; CHECK-NEXT: .LCPI5_0: 250; CHECK-NEXT: .long 0 @ 0x0 251; CHECK-NEXT: .long 6 @ 0x6 252; CHECK-NEXT: .long 12 @ 0xc 253; CHECK-NEXT: .long 18 @ 0x12 254 ptr noalias nocapture %dst, i32 %n.vec) { 255 256vector.ph: ; preds = %for.body.preheader 257 br label %vector.body 258 259vector.body: ; preds = %vector.body, %vector.ph 260 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 261 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ] 262 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3> 263 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0 264 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 265 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %wide.masked.gather, <4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>) 266 %index.next = add i32 %index, 4 267 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 268 %2 = icmp eq i32 %index.next, %n.vec 269 br i1 %2, label %end, label %vector.body 270 271end: 272 ret void; 273} 274 275define arm_aapcs_vfpcc void @push_out_add_sub_block(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec) { 276; CHECK-LABEL: push_out_add_sub_block: 277; CHECK: @ %bb.0: @ %vector.ph 278; CHECK-NEXT: adr r3, .LCPI6_0 279; CHECK-NEXT: vldrw.u32 q0, [r3] 280; CHECK-NEXT: vadd.i32 q0, q0, r0 281; CHECK-NEXT: .LBB6_1: @ %vector.body 282; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 283; CHECK-NEXT: vldrw.u32 q1, [q0, #32]! 284; CHECK-NEXT: subs r2, #4 285; CHECK-NEXT: vstrb.8 q1, [r1], #16 286; CHECK-NEXT: bne .LBB6_1 287; CHECK-NEXT: @ %bb.2: @ %end 288; CHECK-NEXT: bx lr 289; CHECK-NEXT: .p2align 4 290; CHECK-NEXT: @ %bb.3: 291; CHECK-NEXT: .LCPI6_0: 292; CHECK-NEXT: .long 4294967288 @ 0xfffffff8 293; CHECK-NEXT: .long 0 @ 0x0 294; CHECK-NEXT: .long 8 @ 0x8 295; CHECK-NEXT: .long 16 @ 0x10 296 297vector.ph: ; preds = %for.body.preheader 298 br label %vector.body 299 300vector.body: ; preds = %vector.body, %vector.ph 301 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body.end ] 302 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body.end ] 303 br label %lower.block; 304 305lower.block: ; preds = %vector.body 306 %0 = add <4 x i32> %vec.ind, <i32 6, i32 6, i32 6, i32 6> 307 %1 = getelementptr inbounds i32, ptr %data, <4 x i32> %0 308 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %1, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 309 %2 = getelementptr inbounds i32, ptr %dst, i32 %index 310 %3 = bitcast ptr %2 to ptr 311 store <4 x i32> %wide.masked.gather, ptr %3, align 4 312 %index.next = add i32 %index, 4 313 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 314 br label %vector.body.end 315 316vector.body.end: ; preds = %lower.block 317 %4 = icmp eq i32 %index.next, %n.vec 318 br i1 %4, label %end, label %vector.body 319 320end: 321 ret void; 322} 323 324define arm_aapcs_vfpcc void @non_gatscat_use1(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec, ptr %x) { 325; CHECK-LABEL: non_gatscat_use1: 326; CHECK: @ %bb.0: @ %vector.ph 327; CHECK-NEXT: .save {r4, lr} 328; CHECK-NEXT: push {r4, lr} 329; CHECK-NEXT: .vsave {d8, d9} 330; CHECK-NEXT: vpush {d8, d9} 331; CHECK-NEXT: adr r4, .LCPI7_0 332; CHECK-NEXT: mov.w r12, #9 333; CHECK-NEXT: vldrw.u32 q1, [r4] 334; CHECK-NEXT: mov.w lr, #12 335; CHECK-NEXT: movs r4, #8 336; CHECK-NEXT: vdup.32 q0, r0 337; CHECK-NEXT: .LBB7_1: @ %vector.body 338; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 339; CHECK-NEXT: vmov q3, q0 340; CHECK-NEXT: vadd.i32 q2, q1, r4 341; CHECK-NEXT: vmla.i32 q3, q1, lr 342; CHECK-NEXT: vmul.i32 q1, q1, r12 343; CHECK-NEXT: vldrw.u32 q4, [q3, #24] 344; CHECK-NEXT: subs r2, #4 345; CHECK-NEXT: vstrw.32 q1, [r3] 346; CHECK-NEXT: vmov q1, q2 347; CHECK-NEXT: vstrb.8 q4, [r1], #16 348; CHECK-NEXT: bne .LBB7_1 349; CHECK-NEXT: @ %bb.2: @ %end 350; CHECK-NEXT: vpop {d8, d9} 351; CHECK-NEXT: pop {r4, pc} 352; CHECK-NEXT: .p2align 4 353; CHECK-NEXT: @ %bb.3: 354; CHECK-NEXT: .LCPI7_0: 355; CHECK-NEXT: .long 0 @ 0x0 356; CHECK-NEXT: .long 2 @ 0x2 357; CHECK-NEXT: .long 4 @ 0x4 358; CHECK-NEXT: .long 6 @ 0x6 359 360vector.ph: ; preds = %for.body.preheader 361 br label %vector.body 362 363vector.body: ; preds = %vector.body, %vector.ph 364 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 365 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ] 366 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3> 367 %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6> 368 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %1 369 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 370 %3 = getelementptr inbounds i32, ptr %dst, i32 %index 371 %4 = bitcast ptr %3 to ptr 372 store <4 x i32> %wide.masked.gather, ptr %4, align 4 373 %non_gatscat_use = mul <4 x i32> %0, <i32 3, i32 3, i32 3, i32 3> 374 store <4 x i32> %non_gatscat_use, ptr %x, align 4 375 %index.next = add i32 %index, 4 376 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 377 %5 = icmp eq i32 %index.next, %n.vec 378 br i1 %5, label %end, label %vector.body 379 380end: 381 ret void; 382} 383 384define arm_aapcs_vfpcc void @non_gatscat_use2(ptr noalias nocapture readonly %data, ptr noalias nocapture %dst, i32 %n.vec, ptr %x) { 385; CHECK-LABEL: non_gatscat_use2: 386; CHECK: @ %bb.0: @ %vector.ph 387; CHECK-NEXT: .save {r4, r5, r7, lr} 388; CHECK-NEXT: push {r4, r5, r7, lr} 389; CHECK-NEXT: .vsave {d8, d9, d10, d11} 390; CHECK-NEXT: vpush {d8, d9, d10, d11} 391; CHECK-NEXT: adr r4, .LCPI8_0 392; CHECK-NEXT: movs r5, #18 393; CHECK-NEXT: vldrw.u32 q2, [r4] 394; CHECK-NEXT: mov.w r12, #9 395; CHECK-NEXT: mov.w lr, #12 396; CHECK-NEXT: movs r4, #8 397; CHECK-NEXT: vdup.32 q0, r0 398; CHECK-NEXT: vdup.32 q1, r5 399; CHECK-NEXT: .LBB8_1: @ %vector.body 400; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1 401; CHECK-NEXT: vmov q4, q0 402; CHECK-NEXT: vadd.i32 q3, q2, r4 403; CHECK-NEXT: vmla.i32 q4, q2, lr 404; CHECK-NEXT: subs r2, #4 405; CHECK-NEXT: vldrw.u32 q5, [q4, #24] 406; CHECK-NEXT: vmov q4, q1 407; CHECK-NEXT: vmla.i32 q4, q2, r12 408; CHECK-NEXT: vmov q2, q3 409; CHECK-NEXT: vstrb.8 q5, [r1], #16 410; CHECK-NEXT: vstrw.32 q4, [r3] 411; CHECK-NEXT: bne .LBB8_1 412; CHECK-NEXT: @ %bb.2: @ %end 413; CHECK-NEXT: vpop {d8, d9, d10, d11} 414; CHECK-NEXT: pop {r4, r5, r7, pc} 415; CHECK-NEXT: .p2align 4 416; CHECK-NEXT: @ %bb.3: 417; CHECK-NEXT: .LCPI8_0: 418; CHECK-NEXT: .long 0 @ 0x0 419; CHECK-NEXT: .long 2 @ 0x2 420; CHECK-NEXT: .long 4 @ 0x4 421; CHECK-NEXT: .long 6 @ 0x6 422 423vector.ph: ; preds = %for.body.preheader 424 br label %vector.body 425 426vector.body: ; preds = %vector.body, %vector.ph 427 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 428 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ] 429 %0 = mul <4 x i32> %vec.ind, <i32 3, i32 3, i32 3, i32 3> 430 %1 = add <4 x i32> %0, <i32 6, i32 6, i32 6, i32 6> 431 %2 = getelementptr inbounds i32, ptr %data, <4 x i32> %1 432 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %2, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 433 %3 = getelementptr inbounds i32, ptr %dst, i32 %index 434 %4 = bitcast ptr %3 to ptr 435 store <4 x i32> %wide.masked.gather, ptr %4, align 4 436 %non_gatscat_use = mul <4 x i32> %1, <i32 3, i32 3, i32 3, i32 3> 437 store <4 x i32> %non_gatscat_use, ptr %x, align 4 438 %index.next = add i32 %index, 4 439 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 440 %5 = icmp eq i32 %index.next, %n.vec 441 br i1 %5, label %end, label %vector.body 442 443end: 444 ret void; 445} 446 447define dso_local void @arm_mat_mult_q31(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, ptr noalias nocapture %C, i32 %n, i32 %m, i32 %l) local_unnamed_addr #0 { 448; CHECK-LABEL: arm_mat_mult_q31: 449; CHECK: @ %bb.0: @ %for.cond8.preheader.us.us.preheader.preheader 450; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, lr} 451; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, lr} 452; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15} 453; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15} 454; CHECK-NEXT: .pad #32 455; CHECK-NEXT: sub sp, #32 456; CHECK-NEXT: ldrd r9, r12, [sp, #128] 457; CHECK-NEXT: sub.w r7, r12, #1 458; CHECK-NEXT: movs r6, #1 459; CHECK-NEXT: mov.w r8, #0 460; CHECK-NEXT: add.w r7, r6, r7, lsr #1 461; CHECK-NEXT: bic r7, r7, #3 462; CHECK-NEXT: subs r7, #4 463; CHECK-NEXT: add.w r10, r6, r7, lsr #2 464; CHECK-NEXT: adr r6, .LCPI9_1 465; CHECK-NEXT: vldrw.u32 q0, [r6] 466; CHECK-NEXT: adr r7, .LCPI9_0 467; CHECK-NEXT: vldrw.u32 q1, [r7] 468; CHECK-NEXT: vstrw.32 q0, [sp] @ 16-byte Spill 469; CHECK-NEXT: vdup.32 q0, r9 470; CHECK-NEXT: vmov q2, q0 471; CHECK-NEXT: vshl.i32 q3, q0, #3 472; CHECK-NEXT: vstrw.32 q1, [sp, #16] @ 16-byte Spill 473; CHECK-NEXT: .LBB9_1: @ %for.cond8.preheader.us.us.preheader 474; CHECK-NEXT: @ =>This Loop Header: Depth=1 475; CHECK-NEXT: @ Child Loop BB9_2 Depth 2 476; CHECK-NEXT: @ Child Loop BB9_3 Depth 3 477; CHECK-NEXT: mul lr, r8, r12 478; CHECK-NEXT: vldrw.u32 q0, [sp] @ 16-byte Reload 479; CHECK-NEXT: movs r7, #0 480; CHECK-NEXT: mul r6, r8, r9 481; CHECK-NEXT: vdup.32 q4, lr 482; CHECK-NEXT: vshl.i32 q4, q4, #2 483; CHECK-NEXT: vadd.i32 q4, q4, r0 484; CHECK-NEXT: vadd.i32 q4, q4, q0 485; CHECK-NEXT: .LBB9_2: @ %vector.ph 486; CHECK-NEXT: @ Parent Loop BB9_1 Depth=1 487; CHECK-NEXT: @ => This Loop Header: Depth=2 488; CHECK-NEXT: @ Child Loop BB9_3 Depth 3 489; CHECK-NEXT: vldrw.u32 q0, [sp, #16] @ 16-byte Reload 490; CHECK-NEXT: vmov q7, q2 491; CHECK-NEXT: dls lr, r10 492; CHECK-NEXT: vmov.i32 q5, #0x0 493; CHECK-NEXT: vmlas.i32 q7, q0, r7 494; CHECK-NEXT: vmov q6, q4 495; CHECK-NEXT: .LBB9_3: @ %vector.body 496; CHECK-NEXT: @ Parent Loop BB9_1 Depth=1 497; CHECK-NEXT: @ Parent Loop BB9_2 Depth=2 498; CHECK-NEXT: @ => This Inner Loop Header: Depth=3 499; CHECK-NEXT: vadd.i32 q0, q7, q3 500; CHECK-NEXT: vldrw.u32 q1, [r1, q7, uxtw #2] 501; CHECK-NEXT: vldrw.u32 q7, [q6, #32]! 502; CHECK-NEXT: vmul.i32 q1, q1, q7 503; CHECK-NEXT: vmov q7, q0 504; CHECK-NEXT: vadd.i32 q5, q1, q5 505; CHECK-NEXT: le lr, .LBB9_3 506; CHECK-NEXT: @ %bb.4: @ %middle.block 507; CHECK-NEXT: @ in Loop: Header=BB9_2 Depth=2 508; CHECK-NEXT: adds r5, r7, r6 509; CHECK-NEXT: adds r7, #1 510; CHECK-NEXT: vaddv.u32 r4, q5 511; CHECK-NEXT: cmp r7, r9 512; CHECK-NEXT: str.w r4, [r2, r5, lsl #2] 513; CHECK-NEXT: bne .LBB9_2 514; CHECK-NEXT: @ %bb.5: @ %for.cond4.for.cond.cleanup6_crit_edge.us 515; CHECK-NEXT: @ in Loop: Header=BB9_1 Depth=1 516; CHECK-NEXT: add.w r8, r8, #1 517; CHECK-NEXT: cmp r8, r3 518; CHECK-NEXT: bne .LBB9_1 519; CHECK-NEXT: @ %bb.6: @ %for.end25 520; CHECK-NEXT: add sp, #32 521; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} 522; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, pc} 523; CHECK-NEXT: .p2align 4 524; CHECK-NEXT: @ %bb.7: 525; CHECK-NEXT: .LCPI9_0: 526; CHECK-NEXT: .long 0 @ 0x0 527; CHECK-NEXT: .long 2 @ 0x2 528; CHECK-NEXT: .long 4 @ 0x4 529; CHECK-NEXT: .long 6 @ 0x6 530; CHECK-NEXT: .LCPI9_1: 531; CHECK-NEXT: .long 4294967264 @ 0xffffffe0 532; CHECK-NEXT: .long 4294967272 @ 0xffffffe8 533; CHECK-NEXT: .long 4294967280 @ 0xfffffff0 534; CHECK-NEXT: .long 4294967288 @ 0xfffffff8 535 536for.cond8.preheader.us.us.preheader.preheader: ; preds = %entry 537 %0 = add i32 %l, -1 538 %1 = lshr i32 %0, 1 539 %2 = add nuw i32 %1, 1 540 %min.iters.check = icmp ult i32 %0, 6 541 %n.vec = and i32 %2, -4 542 %broadcast.splatinsert86 = insertelement <4 x i32> undef, i32 %m, i32 0 543 %broadcast.splat87 = shufflevector <4 x i32> %broadcast.splatinsert86, <4 x i32> undef, <4 x i32> zeroinitializer 544 %cmp.n = icmp eq i32 %2, %n.vec 545 br label %for.cond8.preheader.us.us.preheader 546 547for.cond8.preheader.us.us.preheader: ; preds = %for.cond8.preheader.us.us.preheader.preheader, %for.cond4.for.cond.cleanup6_crit_edge.us 548 %i.054.us = phi i32 [ %inc24.us, %for.cond4.for.cond.cleanup6_crit_edge.us ], [ 0, %for.cond8.preheader.us.us.preheader.preheader ] 549 %mul.us = mul i32 %i.054.us, %l 550 %mul18.us = mul i32 %i.054.us, %m 551 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %mul.us, i32 0 552 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer 553 br label %vector.ph 554 555for.cond4.for.cond.cleanup6_crit_edge.us: ; preds = %for.cond8.for.cond.cleanup10_crit_edge.us.us 556 %inc24.us = add nuw nsw i32 %i.054.us, 1 557 %exitcond85 = icmp eq i32 %inc24.us, %n 558 br i1 %exitcond85, label %for.end25, label %for.cond8.preheader.us.us.preheader 559 560vector.ph: ; preds = %middle.block, %for.cond8.preheader.us.us.preheader 561 %j.051.us.us = phi i32 [ %inc.us.us, %middle.block ], [ 0, %for.cond8.preheader.us.us.preheader ] 562 %broadcast.splatinsert88 = insertelement <4 x i32> undef, i32 %j.051.us.us, i32 0 563 %broadcast.splat89 = shufflevector <4 x i32> %broadcast.splatinsert88, <4 x i32> undef, <4 x i32> zeroinitializer 564 br label %vector.body 565 566vector.body: ; preds = %vector.body, %vector.ph 567 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 568 %vec.ind = phi <4 x i32> [ <i32 0, i32 2, i32 4, i32 6>, %vector.ph ], [ %vec.ind.next, %vector.body ] 569 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %9, %vector.body ] 570 %3 = add <4 x i32> %vec.ind, %broadcast.splat 571 %4 = getelementptr inbounds i32, ptr %A, <4 x i32> %3 572 %wide.masked.gather = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %4, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), !tbaa !3 573 %5 = mul <4 x i32> %vec.ind, %broadcast.splat87 574 %6 = add <4 x i32> %5, %broadcast.splat89 575 %7 = getelementptr inbounds i32, ptr %B, <4 x i32> %6 576 %wide.masked.gather90 = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %7, i32 4, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef), !tbaa !3 577 %8 = mul nsw <4 x i32> %wide.masked.gather90, %wide.masked.gather 578 %9 = add <4 x i32> %8, %vec.phi 579 %index.next = add i32 %index, 4 580 %vec.ind.next = add <4 x i32> %vec.ind, <i32 8, i32 8, i32 8, i32 8> 581 %10 = icmp eq i32 %index.next, %n.vec 582 br i1 %10, label %middle.block, label %vector.body, !llvm.loop !7 583 584middle.block: ; preds = %vector.body 585 %11 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %9) 586;for.cond8.for.cond.cleanup10_crit_edge.us.us: ; preds = %for.body11.us.us, %middle.block 587 %add19.us.us = add i32 %j.051.us.us, %mul18.us 588 %arrayidx20.us.us = getelementptr inbounds i32, ptr %C, i32 %add19.us.us 589 store i32 %11, ptr %arrayidx20.us.us, align 4, !tbaa !3 590 %inc.us.us = add nuw nsw i32 %j.051.us.us, 1 591 %exitcond = icmp eq i32 %inc.us.us, %m 592 br i1 %exitcond, label %for.cond4.for.cond.cleanup6_crit_edge.us, label %vector.ph 593 594for.end25: ; preds = %for.cond4.for.cond.cleanup6_crit_edge.us, %entry 595 ret void 596} 597 598define dso_local void @arm_mat_mult_q15(ptr noalias nocapture readonly %A, ptr noalias nocapture readonly %B, ptr noalias nocapture %C, i32 %n, i32 %m, i32 %l) local_unnamed_addr #0 { 599; CHECK-LABEL: arm_mat_mult_q15: 600; CHECK: @ %bb.0: @ %entry 601; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} 602; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} 603; CHECK-NEXT: .pad #4 604; CHECK-NEXT: sub sp, #4 605; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15} 606; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15} 607; CHECK-NEXT: .pad #32 608; CHECK-NEXT: sub sp, #32 609; CHECK-NEXT: strd r0, r2, [sp, #24] @ 8-byte Folded Spill 610; CHECK-NEXT: cmp r3, #0 611; CHECK-NEXT: str r3, [sp, #8] @ 4-byte Spill 612; CHECK-NEXT: itt ne 613; CHECK-NEXT: ldrne r0, [sp, #136] 614; CHECK-NEXT: cmpne r0, #0 615; CHECK-NEXT: bne .LBB10_2 616; CHECK-NEXT: .LBB10_1: @ %for.cond.cleanup 617; CHECK-NEXT: add sp, #32 618; CHECK-NEXT: vpop {d8, d9, d10, d11, d12, d13, d14, d15} 619; CHECK-NEXT: add sp, #4 620; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, r10, r11, pc} 621; CHECK-NEXT: .LBB10_2: @ %for.cond1.preheader.us.preheader 622; CHECK-NEXT: ldr.w r12, [sp, #140] 623; CHECK-NEXT: movs r7, #1 624; CHECK-NEXT: mov.w r11, #0 625; CHECK-NEXT: vmov.i32 q0, #0x0 626; CHECK-NEXT: bic r2, r12, #3 627; CHECK-NEXT: subs r3, r2, #4 628; CHECK-NEXT: add.w r0, r7, r3, lsr #2 629; CHECK-NEXT: ldr r7, [sp, #136] 630; CHECK-NEXT: adr r3, .LCPI10_0 631; CHECK-NEXT: str r0, [sp, #16] @ 4-byte Spill 632; CHECK-NEXT: lsl.w r0, r12, #1 633; CHECK-NEXT: vdup.32 q1, r7 634; CHECK-NEXT: vldrw.u32 q2, [r3] 635; CHECK-NEXT: str r0, [sp, #4] @ 4-byte Spill 636; CHECK-NEXT: ldr r0, [sp, #24] @ 4-byte Reload 637; CHECK-NEXT: lsls r6, r7, #1 638; CHECK-NEXT: vshl.i32 q3, q1, #2 639; CHECK-NEXT: movs r3, #0 640; CHECK-NEXT: str r0, [sp, #20] @ 4-byte Spill 641; CHECK-NEXT: b .LBB10_5 642; CHECK-NEXT: .LBB10_3: @ %for.cond5.preheader.us73.preheader 643; CHECK-NEXT: @ in Loop: Header=BB10_5 Depth=1 644; CHECK-NEXT: ldr r0, [sp, #28] @ 4-byte Reload 645; CHECK-NEXT: add.w r3, r0, r5, lsl #1 646; CHECK-NEXT: wlstp.8 lr, r6, .LBB10_4 647; CHECK-NEXT: b .LBB10_15 648; CHECK-NEXT: .LBB10_4: @ %for.cond1.for.cond.cleanup3_crit_edge.us 649; CHECK-NEXT: @ in Loop: Header=BB10_5 Depth=1 650; CHECK-NEXT: ldr r0, [sp, #4] @ 4-byte Reload 651; CHECK-NEXT: add r11, r12 652; CHECK-NEXT: ldr r3, [sp, #20] @ 4-byte Reload 653; CHECK-NEXT: add r3, r0 654; CHECK-NEXT: str r3, [sp, #20] @ 4-byte Spill 655; CHECK-NEXT: ldr r3, [sp, #12] @ 4-byte Reload 656; CHECK-NEXT: ldr r0, [sp, #8] @ 4-byte Reload 657; CHECK-NEXT: adds r3, #1 658; CHECK-NEXT: cmp r3, r0 659; CHECK-NEXT: beq .LBB10_1 660; CHECK-NEXT: .LBB10_5: @ %for.cond1.preheader.us 661; CHECK-NEXT: @ =>This Loop Header: Depth=1 662; CHECK-NEXT: @ Child Loop BB10_8 Depth 2 663; CHECK-NEXT: @ Child Loop BB10_11 Depth 3 664; CHECK-NEXT: @ Child Loop BB10_14 Depth 3 665; CHECK-NEXT: @ Child Loop BB10_15 Depth 2 666; CHECK-NEXT: mul r5, r3, r7 667; CHECK-NEXT: cmp.w r12, #0 668; CHECK-NEXT: str r3, [sp, #12] @ 4-byte Spill 669; CHECK-NEXT: beq .LBB10_3 670; CHECK-NEXT: @ %bb.6: @ %for.cond5.preheader.us.us.preheader 671; CHECK-NEXT: @ in Loop: Header=BB10_5 Depth=1 672; CHECK-NEXT: mov.w r8, #0 673; CHECK-NEXT: b .LBB10_8 674; CHECK-NEXT: .LBB10_7: @ %for.cond5.for.cond.cleanup7_crit_edge.us.us 675; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2 676; CHECK-NEXT: ldr r3, [sp, #28] @ 4-byte Reload 677; CHECK-NEXT: add.w r0, r8, r5 678; CHECK-NEXT: add.w r8, r8, #1 679; CHECK-NEXT: cmp r8, r7 680; CHECK-NEXT: strh.w r10, [r3, r0, lsl #1] 681; CHECK-NEXT: beq .LBB10_4 682; CHECK-NEXT: .LBB10_8: @ %for.cond5.preheader.us.us 683; CHECK-NEXT: @ Parent Loop BB10_5 Depth=1 684; CHECK-NEXT: @ => This Loop Header: Depth=2 685; CHECK-NEXT: @ Child Loop BB10_11 Depth 3 686; CHECK-NEXT: @ Child Loop BB10_14 Depth 3 687; CHECK-NEXT: cmp.w r12, #3 688; CHECK-NEXT: bhi .LBB10_10 689; CHECK-NEXT: @ %bb.9: @ in Loop: Header=BB10_8 Depth=2 690; CHECK-NEXT: movs r4, #0 691; CHECK-NEXT: mov.w r10, #0 692; CHECK-NEXT: b .LBB10_13 693; CHECK-NEXT: .LBB10_10: @ %vector.ph 694; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2 695; CHECK-NEXT: ldr r0, [sp, #16] @ 4-byte Reload 696; CHECK-NEXT: vmov q5, q1 697; CHECK-NEXT: vmov.i32 q4, #0x0 698; CHECK-NEXT: vmlas.i32 q5, q2, r8 699; CHECK-NEXT: dls lr, r0 700; CHECK-NEXT: ldr r3, [sp, #20] @ 4-byte Reload 701; CHECK-NEXT: .LBB10_11: @ %vector.body 702; CHECK-NEXT: @ Parent Loop BB10_5 Depth=1 703; CHECK-NEXT: @ Parent Loop BB10_8 Depth=2 704; CHECK-NEXT: @ => This Inner Loop Header: Depth=3 705; CHECK-NEXT: vadd.i32 q6, q5, q3 706; CHECK-NEXT: vldrh.s32 q7, [r1, q5, uxtw #1] 707; CHECK-NEXT: vldrh.s32 q5, [r3], #8 708; CHECK-NEXT: vmul.i32 q5, q7, q5 709; CHECK-NEXT: vadd.i32 q4, q5, q4 710; CHECK-NEXT: vmov q5, q6 711; CHECK-NEXT: le lr, .LBB10_11 712; CHECK-NEXT: @ %bb.12: @ %middle.block 713; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2 714; CHECK-NEXT: vaddv.u32 r10, q4 715; CHECK-NEXT: cmp r2, r12 716; CHECK-NEXT: mov r4, r2 717; CHECK-NEXT: beq .LBB10_7 718; CHECK-NEXT: .LBB10_13: @ %for.body8.us.us.preheader 719; CHECK-NEXT: @ in Loop: Header=BB10_8 Depth=2 720; CHECK-NEXT: mla r3, r7, r4, r8 721; CHECK-NEXT: add.w r0, r11, r4 722; CHECK-NEXT: ldr r7, [sp, #24] @ 4-byte Reload 723; CHECK-NEXT: sub.w lr, r12, r4 724; CHECK-NEXT: add.w r9, r7, r0, lsl #1 725; CHECK-NEXT: ldr r7, [sp, #136] 726; CHECK-NEXT: add.w r3, r1, r3, lsl #1 727; CHECK-NEXT: .LBB10_14: @ %for.body8.us.us 728; CHECK-NEXT: @ Parent Loop BB10_5 Depth=1 729; CHECK-NEXT: @ Parent Loop BB10_8 Depth=2 730; CHECK-NEXT: @ => This Inner Loop Header: Depth=3 731; CHECK-NEXT: ldrsh.w r4, [r3] 732; CHECK-NEXT: add r3, r6 733; CHECK-NEXT: ldrsh r0, [r9], #2 734; CHECK-NEXT: smlabb r10, r4, r0, r10 735; CHECK-NEXT: le lr, .LBB10_14 736; CHECK-NEXT: b .LBB10_7 737; CHECK-NEXT: .LBB10_15: @ Parent Loop BB10_5 Depth=1 738; CHECK-NEXT: @ => This Inner Loop Header: Depth=2 739; CHECK-NEXT: vstrb.8 q0, [r3], #16 740; CHECK-NEXT: letp lr, .LBB10_15 741; CHECK-NEXT: b .LBB10_4 742; CHECK-NEXT: .p2align 4 743; CHECK-NEXT: @ %bb.16: 744; CHECK-NEXT: .LCPI10_0: 745; CHECK-NEXT: .long 0 @ 0x0 746; CHECK-NEXT: .long 1 @ 0x1 747; CHECK-NEXT: .long 2 @ 0x2 748; CHECK-NEXT: .long 3 @ 0x3 749entry: 750 %cmp48 = icmp eq i32 %n, 0 751 br i1 %cmp48, label %for.cond.cleanup, label %for.cond1.preheader.lr.ph 752 753for.cond1.preheader.lr.ph: ; preds = %entry 754 %cmp245 = icmp eq i32 %m, 0 755 %cmp642 = icmp eq i32 %l, 0 756 br i1 %cmp245, label %for.cond.cleanup, label %for.cond1.preheader.us.preheader 757 758for.cond1.preheader.us.preheader: ; preds = %for.cond1.preheader.lr.ph 759 %0 = shl nuw i32 %m, 1 760 %min.iters.check = icmp ult i32 %l, 4 761 %n.vec = and i32 %l, -4 762 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %m, i32 0 763 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer 764 %cmp.n = icmp eq i32 %n.vec, %l 765 br label %for.cond1.preheader.us 766 767for.cond1.preheader.us: ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.us.preheader 768 %i.049.us = phi i32 [ %inc23.us, %for.cond1.for.cond.cleanup3_crit_edge.us ], [ 0, %for.cond1.preheader.us.preheader ] 769 %1 = mul i32 %i.049.us, %m 770 %mul.us = mul i32 %i.049.us, %l 771 br i1 %cmp642, label %for.cond5.preheader.us73.preheader, label %for.cond5.preheader.us.us 772 773for.cond5.preheader.us73.preheader: ; preds = %for.cond1.preheader.us 774 %scevgep = getelementptr i16, ptr %C, i32 %1 775 %scevgep82 = bitcast ptr %scevgep to ptr 776 call void @llvm.memset.p0.i32(ptr align 2 %scevgep82, i8 0, i32 %0, i1 false) 777 br label %for.cond1.for.cond.cleanup3_crit_edge.us 778 779for.cond1.for.cond.cleanup3_crit_edge.us: ; preds = %for.cond5.for.cond.cleanup7_crit_edge.us.us, %for.cond5.preheader.us73.preheader 780 %inc23.us = add nuw nsw i32 %i.049.us, 1 781 %exitcond84 = icmp eq i32 %inc23.us, %n 782 br i1 %exitcond84, label %for.cond.cleanup, label %for.cond1.preheader.us 783 784for.cond5.preheader.us.us: ; preds = %for.cond1.preheader.us, %for.cond5.for.cond.cleanup7_crit_edge.us.us 785 %j.046.us.us = phi i32 [ %inc20.us.us, %for.cond5.for.cond.cleanup7_crit_edge.us.us ], [ 0, %for.cond1.preheader.us ] 786 br i1 %min.iters.check, label %for.body8.us.us.preheader, label %vector.ph 787 788for.body8.us.us.preheader: ; preds = %middle.block, %for.cond5.preheader.us.us 789 %k.044.us.us.ph = phi i32 [ 0, %for.cond5.preheader.us.us ], [ %n.vec, %middle.block ] 790 %sum.043.us.us.ph = phi i32 [ 0, %for.cond5.preheader.us.us ], [ %13, %middle.block ] 791 br label %for.body8.us.us 792 793vector.ph: ; preds = %for.cond5.preheader.us.us 794 %broadcast.splatinsert85 = insertelement <4 x i32> undef, i32 %j.046.us.us, i32 0 795 %broadcast.splat86 = shufflevector <4 x i32> %broadcast.splatinsert85, <4 x i32> undef, <4 x i32> zeroinitializer 796 br label %vector.body 797 798vector.body: ; preds = %vector.body, %vector.ph 799 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 800 %vec.ind = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %vector.ph ], [ %vec.ind.next, %vector.body ] 801 %vec.phi = phi <4 x i32> [ zeroinitializer, %vector.ph ], [ %11, %vector.body ] 802 %2 = add i32 %index, %mul.us 803 %3 = getelementptr inbounds i16, ptr %A, i32 %2 804 %4 = bitcast ptr %3 to ptr 805 %wide.load = load <4 x i16>, ptr %4, align 2, !tbaa !3 806 %5 = sext <4 x i16> %wide.load to <4 x i32> 807 %6 = mul <4 x i32> %vec.ind, %broadcast.splat 808 %7 = add <4 x i32> %6, %broadcast.splat86 809 %8 = getelementptr inbounds i16, ptr %B, <4 x i32> %7 810 %wide.masked.gather = call <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr> %8, i32 2, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i16> undef), !tbaa !3 811 %9 = sext <4 x i16> %wide.masked.gather to <4 x i32> 812 %10 = mul nsw <4 x i32> %9, %5 813 %11 = add <4 x i32> %10, %vec.phi 814 %index.next = add i32 %index, 4 815 %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4> 816 %12 = icmp eq i32 %index.next, %n.vec 817 br i1 %12, label %middle.block, label %vector.body, !llvm.loop !7 818 819middle.block: ; preds = %vector.body 820 %13 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %11) 821 br i1 %cmp.n, label %for.cond5.for.cond.cleanup7_crit_edge.us.us, label %for.body8.us.us.preheader 822 823for.cond5.for.cond.cleanup7_crit_edge.us.us: ; preds = %for.body8.us.us, %middle.block 824 %add14.us.us.lcssa = phi i32 [ %13, %middle.block ], [ %add14.us.us, %for.body8.us.us ] 825 %conv15.us.us = trunc i32 %add14.us.us.lcssa to i16 826 %add17.us.us = add i32 %j.046.us.us, %1 827 %arrayidx18.us.us = getelementptr inbounds i16, ptr %C, i32 %add17.us.us 828 store i16 %conv15.us.us, ptr %arrayidx18.us.us, align 2, !tbaa !3 829 %inc20.us.us = add nuw nsw i32 %j.046.us.us, 1 830 %exitcond83 = icmp eq i32 %inc20.us.us, %m 831 br i1 %exitcond83, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.cond5.preheader.us.us 832 833for.body8.us.us: ; preds = %for.body8.us.us.preheader, %for.body8.us.us 834 %k.044.us.us = phi i32 [ %inc.us.us, %for.body8.us.us ], [ %k.044.us.us.ph, %for.body8.us.us.preheader ] 835 %sum.043.us.us = phi i32 [ %add14.us.us, %for.body8.us.us ], [ %sum.043.us.us.ph, %for.body8.us.us.preheader ] 836 %add.us.us = add i32 %k.044.us.us, %mul.us 837 %arrayidx.us.us = getelementptr inbounds i16, ptr %A, i32 %add.us.us 838 %14 = load i16, ptr %arrayidx.us.us, align 2, !tbaa !3 839 %conv.us.us = sext i16 %14 to i32 840 %mul9.us.us = mul i32 %k.044.us.us, %m 841 %add10.us.us = add i32 %mul9.us.us, %j.046.us.us 842 %arrayidx11.us.us = getelementptr inbounds i16, ptr %B, i32 %add10.us.us 843 %15 = load i16, ptr %arrayidx11.us.us, align 2, !tbaa !3 844 %conv12.us.us = sext i16 %15 to i32 845 %mul13.us.us = mul nsw i32 %conv12.us.us, %conv.us.us 846 %add14.us.us = add nsw i32 %mul13.us.us, %sum.043.us.us 847 %inc.us.us = add nuw nsw i32 %k.044.us.us, 1 848 %exitcond = icmp eq i32 %inc.us.us, %l 849 br i1 %exitcond, label %for.cond5.for.cond.cleanup7_crit_edge.us.us, label %for.body8.us.us, !llvm.loop !9 850 851for.cond.cleanup: ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %for.cond1.preheader.lr.ph, %entry 852 ret void 853} 854 855define hidden arm_aapcs_vfpcc i32 @arm_depthwise_conv_s8(ptr nocapture readonly %input, i16 zeroext %input_x, i16 zeroext %input_y, i16 zeroext %input_ch, ptr nocapture readonly %kernel, i16 zeroext %output_ch, i16 zeroext %ch_mult, i16 zeroext %kernel_x, i16 zeroext %kernel_y, i16 zeroext %pad_x, i16 zeroext %pad_y, i16 zeroext %stride_x, i16 zeroext %stride_y, ptr nocapture readonly %bias, ptr nocapture %output, ptr nocapture readonly %output_shift, ptr nocapture readonly %output_mult, i16 zeroext %output_x, i16 zeroext %output_y, i32 %output_offset, i32 %input_offset, i32 %output_activation_min, i32 %output_activation_max, i16 zeroext %dilation_x, i16 zeroext %dilation_y, ptr nocapture readnone %buffer_a) local_unnamed_addr #0 { 856; CHECK-LABEL: arm_depthwise_conv_s8: 857; CHECK: @ %bb.0: @ %entry 858; CHECK-NEXT: .save {r4, r5, r6, r7, r8, r9, r10, r11, lr} 859; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr} 860; CHECK-NEXT: .pad #4 861; CHECK-NEXT: sub sp, #4 862; CHECK-NEXT: .vsave {d8, d9, d10, d11, d12, d13, d14, d15} 863; CHECK-NEXT: vpush {d8, d9, d10, d11, d12, d13, d14, d15} 864; CHECK-NEXT: .pad #24 865; CHECK-NEXT: sub sp, #24 866; CHECK-NEXT: ldrd r2, r7, [sp, #136] 867; CHECK-NEXT: add.w r8, r7, #10 868; CHECK-NEXT: adr r7, .LCPI11_0 869; CHECK-NEXT: ldr r1, [sp, #128] 870; CHECK-NEXT: vdup.32 q0, r2 871; CHECK-NEXT: vldrw.u32 q1, [r7] 872; CHECK-NEXT: movs r4, #0 873; CHECK-NEXT: mov.w r10, #6 874; CHECK-NEXT: movs r6, #11 875; CHECK-NEXT: vshl.i32 q0, q0, #2 876; CHECK-NEXT: movs r5, #0 877; CHECK-NEXT: .LBB11_1: @ %for.body10.i 878; CHECK-NEXT: @ =>This Loop Header: Depth=1 879; CHECK-NEXT: @ Child Loop BB11_2 Depth 2 880; CHECK-NEXT: @ Child Loop BB11_3 Depth 3 881; CHECK-NEXT: @ Child Loop BB11_4 Depth 4 882; CHECK-NEXT: @ Child Loop BB11_5 Depth 5 883; CHECK-NEXT: mov.w r9, #0 884; CHECK-NEXT: str r5, [sp, #4] @ 4-byte Spill 885; CHECK-NEXT: .LBB11_2: @ %for.cond22.preheader.i 886; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1 887; CHECK-NEXT: @ => This Loop Header: Depth=2 888; CHECK-NEXT: @ Child Loop BB11_3 Depth 3 889; CHECK-NEXT: @ Child Loop BB11_4 Depth 4 890; CHECK-NEXT: @ Child Loop BB11_5 Depth 5 891; CHECK-NEXT: movs r7, #0 892; CHECK-NEXT: vdup.32 q2, r9 893; CHECK-NEXT: vstrw.32 q2, [sp, #8] @ 16-byte Spill 894; CHECK-NEXT: .LBB11_3: @ %for.body27.i 895; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1 896; CHECK-NEXT: @ Parent Loop BB11_2 Depth=2 897; CHECK-NEXT: @ => This Loop Header: Depth=3 898; CHECK-NEXT: @ Child Loop BB11_4 Depth 4 899; CHECK-NEXT: @ Child Loop BB11_5 Depth 5 900; CHECK-NEXT: dls lr, r10 901; CHECK-NEXT: mov.w r12, #0 902; CHECK-NEXT: mov.w r11, #4 903; CHECK-NEXT: vdup.32 q3, r7 904; CHECK-NEXT: .LBB11_4: @ %for.body78.us.i 905; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1 906; CHECK-NEXT: @ Parent Loop BB11_2 Depth=2 907; CHECK-NEXT: @ Parent Loop BB11_3 Depth=3 908; CHECK-NEXT: @ => This Loop Header: Depth=4 909; CHECK-NEXT: @ Child Loop BB11_5 Depth 5 910; CHECK-NEXT: mul r5, r11, r6 911; CHECK-NEXT: vmov q4, q3 912; CHECK-NEXT: vadd.i32 q5, q1, r5 913; CHECK-NEXT: vmla.i32 q4, q5, r2 914; CHECK-NEXT: vldrw.u32 q5, [sp, #8] @ 16-byte Reload 915; CHECK-NEXT: adds r5, #113 916; CHECK-NEXT: vadd.i32 q6, q1, r5 917; CHECK-NEXT: mov r5, r8 918; CHECK-NEXT: vmla.i32 q5, q6, r2 919; CHECK-NEXT: .LBB11_5: @ %vector.body 920; CHECK-NEXT: @ Parent Loop BB11_1 Depth=1 921; CHECK-NEXT: @ Parent Loop BB11_2 Depth=2 922; CHECK-NEXT: @ Parent Loop BB11_3 Depth=3 923; CHECK-NEXT: @ Parent Loop BB11_4 Depth=4 924; CHECK-NEXT: @ => This Inner Loop Header: Depth=5 925; CHECK-NEXT: vldrb.s32 q2, [r0, q5] 926; CHECK-NEXT: vadd.i32 q7, q5, q0 927; CHECK-NEXT: vldrb.s32 q5, [r1, q4] 928; CHECK-NEXT: vadd.i32 q6, q4, q0 929; CHECK-NEXT: vadd.i32 q2, q2, r2 930; CHECK-NEXT: subs r5, #4 931; CHECK-NEXT: vmlava.u32 r12, q2, q5 932; CHECK-NEXT: vmov q5, q7 933; CHECK-NEXT: vmov q4, q6 934; CHECK-NEXT: bne .LBB11_5 935; CHECK-NEXT: @ %bb.6: @ %middle.block 936; CHECK-NEXT: @ in Loop: Header=BB11_4 Depth=4 937; CHECK-NEXT: add.w r11, r11, #1 938; CHECK-NEXT: le lr, .LBB11_4 939; CHECK-NEXT: @ %bb.7: @ %for.cond.cleanup77.i 940; CHECK-NEXT: @ in Loop: Header=BB11_3 Depth=3 941; CHECK-NEXT: adds r7, #1 942; CHECK-NEXT: adds r4, #1 943; CHECK-NEXT: cmp r7, r2 944; CHECK-NEXT: bne .LBB11_3 945; CHECK-NEXT: @ %bb.8: @ %for.cond.cleanup26.i 946; CHECK-NEXT: @ in Loop: Header=BB11_2 Depth=2 947; CHECK-NEXT: add.w r9, r9, #1 948; CHECK-NEXT: cmp r9, r3 949; CHECK-NEXT: bne .LBB11_2 950; CHECK-NEXT: @ %bb.9: @ %for.cond.cleanup20.i 951; CHECK-NEXT: @ in Loop: Header=BB11_1 Depth=1 952; CHECK-NEXT: ldr r5, [sp, #4] @ 4-byte Reload 953; CHECK-NEXT: ldr r7, [sp, #180] 954; CHECK-NEXT: adds r5, #1 955; CHECK-NEXT: cmp r5, r7 956; CHECK-NEXT: it eq 957; CHECK-NEXT: moveq r5, #0 958; CHECK-NEXT: b .LBB11_1 959; CHECK-NEXT: .p2align 4 960; CHECK-NEXT: @ %bb.10: 961; CHECK-NEXT: .LCPI11_0: 962; CHECK-NEXT: .long 0 @ 0x0 963; CHECK-NEXT: .long 1 @ 0x1 964; CHECK-NEXT: .long 2 @ 0x2 965; CHECK-NEXT: .long 3 @ 0x3 966entry: 967 %conv = zext i16 %ch_mult to i32 968 %conv6.i = zext i16 %output_x to i32 969 %conv17.i = zext i16 %input_ch to i32 970 %conv60.i = zext i16 %kernel_x to i32 971 %broadcast.splatinsert63 = insertelement <4 x i32> undef, i32 %conv, i32 0 972 %broadcast.splat64 = shufflevector <4 x i32> %broadcast.splatinsert63, <4 x i32> undef, <4 x i32> zeroinitializer 973 %broadcast.splatinsert69 = insertelement <4 x i32> undef, i32 %conv, i32 0 974 %broadcast.splat70 = shufflevector <4 x i32> %broadcast.splatinsert69, <4 x i32> undef, <4 x i32> zeroinitializer 975 %broadcast.splatinsert73 = insertelement <4 x i32> undef, i32 %conv, i32 0 976 %broadcast.splat74 = shufflevector <4 x i32> %broadcast.splatinsert73, <4 x i32> undef, <4 x i32> zeroinitializer 977 %unroll_iter = and i32 %conv, 65534 978 br label %for.body.i38 979 980for.body.i38: ; preds = %for.cond.cleanup9.i, %entry 981 %i_out.024.i = phi i32 [ 0, %entry ], [ %i_out.1.lcssa.i, %for.cond.cleanup9.i ] 982 %i_out_y.023.i = phi i32 [ 0, %entry ], [ %inc140.i, %for.cond.cleanup9.i ] 983 br label %for.body10.i 984 985for.cond.cleanup9.i: ; preds = %for.cond.cleanup20.i, %for.body.i38 986 %i_out.1.lcssa.i = phi i32 [ %i_out.2.lcssa.i, %for.cond.cleanup20.i ] 987 %inc140.i = add nuw nsw i32 %i_out_y.023.i, 1 988 br i1 0, label %if.end, label %for.body.i38 989 990for.body10.i: ; preds = %for.cond.cleanup20.i, %for.body.i38 991 %i_out.120.i = phi i32 [ %i_out.024.i, %for.body.i38 ], [ %i_out.2.lcssa.i, %for.cond.cleanup20.i ] 992 %i_out_x.019.i = phi i32 [ 0, %for.body.i38 ], [ %inc137.i, %for.cond.cleanup20.i ] 993 %n.vec = add nsw i32 %conv60.i, 10 994 br i1 0, label %for.cond.cleanup20.i, label %for.cond22.preheader.lr.ph.i 995 996for.cond22.preheader.lr.ph.i: ; preds = %for.body10.i 997 %.splatinsert = insertelement <4 x i32> undef, i32 0, i32 0 998 %.splat = shufflevector <4 x i32> %.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer 999 %induction = add <4 x i32> %.splat, <i32 0, i32 1, i32 2, i32 3> 1000 %cmp.n = icmp eq i32 10, %n.vec 1001 br label %for.cond22.preheader.i 1002 1003for.cond22.preheader.i: ; preds = %for.cond.cleanup26.i, %for.cond22.preheader.lr.ph.i 1004 %i_out.216.i = phi i32 [ %i_out.120.i, %for.cond22.preheader.lr.ph.i ], [ %i_out.3.lcssa.i, %for.cond.cleanup26.i ] 1005 %i_input_ch.014.i = phi i32 [ 0, %for.cond22.preheader.lr.ph.i ], [ %inc134.i, %for.cond.cleanup26.i ] 1006 br i1 0, label %for.cond.cleanup26.i, label %for.body27.lr.ph.i 1007 1008for.body27.lr.ph.i: ; preds = %for.cond22.preheader.i 1009 br i1 0, label %for.body27.i.us.preheader, label %for.body27.i.preheader 1010 1011for.body27.i.preheader: ; preds = %for.body27.lr.ph.i 1012 %broadcast.splatinsert65 = insertelement <4 x i32> undef, i32 %i_input_ch.014.i, i32 0 1013 %broadcast.splat66 = shufflevector <4 x i32> %broadcast.splatinsert65, <4 x i32> undef, <4 x i32> zeroinitializer 1014 br label %for.body27.i 1015 1016for.body27.i.us.preheader: ; preds = %for.body27.lr.ph.i 1017 br i1 0, label %for.cond.cleanup26.i.loopexit.unr-lcssa, label %for.body27.i.us 1018 1019for.body27.i.us: ; preds = %for.body27.i.us, %for.body27.i.us.preheader 1020 %i_out.311.i.us = phi i32 [ %inc128.i.us.1, %for.body27.i.us ], [ %i_out.216.i, %for.body27.i.us.preheader ] 1021 %i_ch_mult.010.i.us = phi i32 [ %inc131.i.us.1, %for.body27.i.us ], [ 0, %for.body27.i.us.preheader ] 1022 %niter = phi i32 [ 0, %for.body27.i.us ], [ %unroll_iter, %for.body27.i.us.preheader ] 1023 %inc128.i.us.1 = add nsw i32 %i_out.311.i.us, 2 1024 %inc131.i.us.1 = add nuw nsw i32 %i_ch_mult.010.i.us, 2 1025 br i1 0, label %for.cond.cleanup26.i.loopexit.unr-lcssa, label %for.body27.i.us 1026 1027for.cond.cleanup20.i: ; preds = %for.cond.cleanup26.i, %for.body10.i 1028 %i_out.2.lcssa.i = phi i32 [ %i_out.120.i, %for.body10.i ], [ %i_out.3.lcssa.i, %for.cond.cleanup26.i ] 1029 %inc137.i = add nuw nsw i32 %i_out_x.019.i, 1 1030 %exitcond27.i = icmp eq i32 %inc137.i, %conv6.i 1031 br i1 %exitcond27.i, label %for.cond.cleanup9.i, label %for.body10.i 1032 1033for.cond.cleanup26.i.loopexit.unr-lcssa: ; preds = %for.body27.i.us, %for.body27.i.us.preheader 1034 %inc128.i.us.lcssa.ph = phi i32 [ undef, %for.body27.i.us.preheader ], [ %inc128.i.us.1, %for.body27.i.us ] 1035 br label %for.cond.cleanup26.i 1036 1037for.cond.cleanup26.i: ; preds = %for.cond.cleanup77.i, %for.cond.cleanup26.i.loopexit.unr-lcssa, %for.cond22.preheader.i 1038 %i_out.3.lcssa.i = phi i32 [ %i_out.216.i, %for.cond22.preheader.i ], [ %inc128.i.us.lcssa.ph, %for.cond.cleanup26.i.loopexit.unr-lcssa ], [ %inc128.i, %for.cond.cleanup77.i ] 1039 %inc134.i = add nuw nsw i32 %i_input_ch.014.i, 1 1040 %exitcond26.i = icmp eq i32 %inc134.i, %conv17.i 1041 br i1 %exitcond26.i, label %for.cond.cleanup20.i, label %for.cond22.preheader.i 1042 1043for.body27.i: ; preds = %for.cond.cleanup77.i, %for.body27.i.preheader 1044 %i_out.311.i = phi i32 [ %inc128.i, %for.cond.cleanup77.i ], [ %i_out.216.i, %for.body27.i.preheader ] 1045 %i_ch_mult.010.i = phi i32 [ %inc131.i, %for.cond.cleanup77.i ], [ 0, %for.body27.i.preheader ] 1046 %broadcast.splatinsert71 = insertelement <4 x i32> undef, i32 %i_ch_mult.010.i, i32 0 1047 %broadcast.splat72 = shufflevector <4 x i32> %broadcast.splatinsert71, <4 x i32> undef, <4 x i32> zeroinitializer 1048 br label %for.body78.us.i 1049 1050for.body78.us.i: ; preds = %middle.block, %for.body27.i 1051 %i_ker_y.06.us.i = phi i32 [ %inc110.us.i, %middle.block ], [ 4, %for.body27.i ] 1052 %acc_0.05.us.i = phi i32 [ %tmp89, %middle.block ], [ 0, %for.body27.i ] 1053 %add80.us.i43 = add nsw i32 %i_ker_y.06.us.i, 10 1054 %mul89.us.i = mul nsw i32 %add80.us.i43, 11 1055 %add87.us.i44 = add i32 %mul89.us.i, 3 1056 %mul95.us.i = mul nsw i32 %i_ker_y.06.us.i, 11 1057 br label %vector.ph 1058 1059vector.ph: ; preds = %for.body78.us.i 1060 %broadcast.splatinsert = insertelement <4 x i32> undef, i32 %add87.us.i44, i32 0 1061 %broadcast.splat = shufflevector <4 x i32> %broadcast.splatinsert, <4 x i32> undef, <4 x i32> zeroinitializer 1062 %broadcast.splatinsert67 = insertelement <4 x i32> undef, i32 %mul95.us.i, i32 0 1063 %broadcast.splat68 = shufflevector <4 x i32> %broadcast.splatinsert67, <4 x i32> undef, <4 x i32> zeroinitializer 1064 br label %vector.body 1065 1066vector.body: ; preds = %vector.body, %vector.ph 1067 %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] 1068 %vec.ind = phi <4 x i32> [ %induction, %vector.ph ], [ %vec.ind.next, %vector.body ] 1069 %vec.phi = phi i32 [ %acc_0.05.us.i, %vector.ph ], [ %tmp89, %vector.body ] 1070 %tmp76 = add <4 x i32> %broadcast.splat, %vec.ind 1071 %tmp77 = mul nsw <4 x i32> %tmp76, %broadcast.splat64 1072 %tmp78 = add nsw <4 x i32> %tmp77, %broadcast.splat66 1073 %tmp79 = add nsw <4 x i32> %vec.ind, %broadcast.splat68 1074 %tmp80 = mul nsw <4 x i32> %broadcast.splat70, %tmp79 1075 %tmp81 = add nsw <4 x i32> %tmp80, %broadcast.splat72 1076 %tmp82 = getelementptr inbounds i8, ptr %input, <4 x i32> %tmp78 1077 %wide.masked.gather = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %tmp82, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef) 1078 %tmp83 = sext <4 x i8> %wide.masked.gather to <4 x i32> 1079 %tmp84 = add nsw <4 x i32> %broadcast.splat74, %tmp83 1080 %tmp85 = getelementptr inbounds i8, ptr %kernel, <4 x i32> %tmp81 1081 %wide.masked.gather75 = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %tmp85, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef) 1082 %tmp86 = sext <4 x i8> %wide.masked.gather75 to <4 x i32> 1083 %tmp87 = mul nsw <4 x i32> %tmp84, %tmp86 1084 %tmp88 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %tmp87) 1085 %tmp89 = add i32 %tmp88, %vec.phi 1086 %index.next = add i32 %index, 4 1087 %vec.ind.next = add <4 x i32> %vec.ind, <i32 4, i32 4, i32 4, i32 4> 1088 %tmp90 = icmp eq i32 %index.next, %n.vec 1089 br i1 %tmp90, label %middle.block, label %vector.body 1090 1091middle.block: ; preds = %vector.body 1092 %inc110.us.i = add nsw i32 %i_ker_y.06.us.i, 1 1093 %cmp75.us.i = icmp slt i32 %inc110.us.i, 10 1094 br i1 %cmp75.us.i, label %for.body78.us.i, label %for.cond.cleanup77.i 1095 1096for.cond.cleanup77.i: ; preds = %middle.block 1097 %inc128.i = add nsw i32 %i_out.311.i, 1 1098 %inc131.i = add nuw nsw i32 %i_ch_mult.010.i, 1 1099 %exitcond.i50 = icmp eq i32 %inc131.i, %conv 1100 br i1 %exitcond.i50, label %for.cond.cleanup26.i, label %for.body27.i 1101 1102if.end: ; preds = %for.cond.cleanup9.i, %entry, %for.cond.cleanup7.i, %if.then 1103 ret i32 0 1104} 1105 1106declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i32>) 1107declare <4 x i16> @llvm.masked.gather.v4i16.v4p0(<4 x ptr>, i32, <4 x i1>, <4 x i16>) 1108declare <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr>, i32 immarg, <4 x i1>, <4 x i8>) #3 1109 1110declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>) 1111declare void @llvm.memset.p0.i32(ptr align 2, i8, i32, i1) 1112 1113declare void @llvm.masked.scatter.v4i32.v4p0(<4 x i32>, <4 x ptr>, i32, <4 x i1>) 1114