1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 2; RUN: opt -S -mtriple aarch64 -mattr=+sve -passes=loop-vectorize -enable-vplan-native-path < %s | FileCheck %s 3 4@A = external local_unnamed_addr global [1024 x float], align 4 5@B = external local_unnamed_addr global [512 x float], align 4 6 7; Test if the vplan-native-path successfully vectorizes a loop using scalable vectors if the target preferes scalable vectors. 8define void @foo() { 9; CHECK-LABEL: define void @foo 10; CHECK-SAME: () #[[ATTR0:[0-9]+]] { 11; CHECK-NEXT: entry: 12; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() 13; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[TMP0]], 4 14; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]] 15; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] 16; CHECK: vector.ph: 17; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() 18; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 19; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] 20; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] 21; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() 22; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 23; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64() 24; CHECK-NEXT: [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 1) 25; CHECK-NEXT: [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP7]] 26; CHECK-NEXT: [[TMP8:%.*]] = mul i64 1, [[TMP5]] 27; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP8]], i64 0 28; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer 29; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] 30; CHECK: vector.body: 31; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_LATCH:%.*]] ] 32; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_LATCH]] ] 33; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, <vscale x 4 x i64> [[VEC_IND]] 34; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP9]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison) 35; CHECK-NEXT: br label [[INNER_LOOP1:%.*]] 36; CHECK: inner_loop1: 37; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <vscale x 4 x i64> [ zeroinitializer, [[VECTOR_BODY]] ], [ [[TMP12:%.*]], [[INNER_LOOP1]] ] 38; CHECK-NEXT: [[VEC_PHI2:%.*]] = phi <vscale x 4 x float> [ [[WIDE_MASKED_GATHER]], [[VECTOR_BODY]] ], [ [[TMP11:%.*]], [[INNER_LOOP1]] ] 39; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, <vscale x 4 x i64> [[VEC_PHI]] 40; CHECK-NEXT: [[WIDE_MASKED_GATHER3:%.*]] = call <vscale x 4 x float> @llvm.masked.gather.nxv4f32.nxv4p0(<vscale x 4 x ptr> [[TMP10]], i32 4, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x float> poison) 41; CHECK-NEXT: [[TMP11]] = fmul <vscale x 4 x float> [[VEC_PHI2]], [[WIDE_MASKED_GATHER3]] 42; CHECK-NEXT: [[TMP12]] = add nuw nsw <vscale x 4 x i64> [[VEC_PHI]], splat (i64 1) 43; CHECK-NEXT: [[TMP13:%.*]] = icmp eq <vscale x 4 x i64> [[TMP12]], splat (i64 512) 44; CHECK-NEXT: [[TMP14:%.*]] = extractelement <vscale x 4 x i1> [[TMP13]], i32 0 45; CHECK-NEXT: br i1 [[TMP14]], label [[VECTOR_LATCH]], label [[INNER_LOOP1]] 46; CHECK: vector.latch: 47; CHECK-NEXT: [[VEC_PHI4:%.*]] = phi <vscale x 4 x float> [ [[TMP11]], [[INNER_LOOP1]] ] 48; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float> [[VEC_PHI4]], <vscale x 4 x ptr> [[TMP9]], i32 4, <vscale x 4 x i1> splat (i1 true)) 49; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] 50; CHECK-NEXT: [[VEC_IND_NEXT]] = add <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]] 51; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] 52; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] 53; CHECK: middle.block: 54; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]] 55; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] 56; CHECK: scalar.ph: 57; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] 58; CHECK-NEXT: br label [[OUTER_LOOP:%.*]] 59; CHECK: outer_loop: 60; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[OUTER_LOOP_LATCH:%.*]] ] 61; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 [[I]] 62; CHECK-NEXT: [[X_START:%.*]] = load float, ptr [[ARRAYIDX1]], align 4 63; CHECK-NEXT: br label [[INNER_LOOP:%.*]] 64; CHECK: inner_loop: 65; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, [[OUTER_LOOP]] ], [ [[J_NEXT:%.*]], [[INNER_LOOP]] ] 66; CHECK-NEXT: [[X:%.*]] = phi float [ [[X_START]], [[OUTER_LOOP]] ], [ [[X_NEXT:%.*]], [[INNER_LOOP]] ] 67; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 [[J]] 68; CHECK-NEXT: [[B:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 69; CHECK-NEXT: [[X_NEXT]] = fmul float [[X]], [[B]] 70; CHECK-NEXT: [[J_NEXT]] = add nuw nsw i64 [[J]], 1 71; CHECK-NEXT: [[INNER_EXITCOND:%.*]] = icmp eq i64 [[J_NEXT]], 512 72; CHECK-NEXT: br i1 [[INNER_EXITCOND]], label [[OUTER_LOOP_LATCH]], label [[INNER_LOOP]] 73; CHECK: outer_loop_latch: 74; CHECK-NEXT: [[X_NEXT_LCSSA:%.*]] = phi float [ [[X_NEXT]], [[INNER_LOOP]] ] 75; CHECK-NEXT: store float [[X_NEXT_LCSSA]], ptr [[ARRAYIDX1]], align 4 76; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1 77; CHECK-NEXT: [[OUTER_EXITCOND:%.*]] = icmp eq i64 [[I_NEXT]], 1024 78; CHECK-NEXT: br i1 [[OUTER_EXITCOND]], label [[EXIT]], label [[OUTER_LOOP]], !llvm.loop [[LOOP3:![0-9]+]] 79; CHECK: exit: 80; CHECK-NEXT: ret void 81; 82entry: 83 br label %outer_loop 84 85outer_loop: 86 %i = phi i64 [ 0, %entry ], [ %i.next, %outer_loop_latch ] 87 %arrayidx1 = getelementptr inbounds [1024 x float], ptr @A, i64 0, i64 %i 88 %x.start = load float, ptr %arrayidx1, align 4 89 br label %inner_loop 90 91inner_loop: 92 %j = phi i64 [ 0, %outer_loop ], [ %j.next, %inner_loop ] 93 %x = phi float [ %x.start, %outer_loop ], [ %x.next, %inner_loop ] 94 %arrayidx2 = getelementptr inbounds [512 x float], ptr @B, i64 0, i64 %j 95 %b = load float, ptr %arrayidx2, align 4 96 %x.next = fmul float %x, %b 97 %j.next = add nuw nsw i64 %j, 1 98 %inner_exitcond = icmp eq i64 %j.next, 512 99 br i1 %inner_exitcond, label %outer_loop_latch, label %inner_loop 100 101outer_loop_latch: 102 store float %x.next, ptr %arrayidx1, align 4 103 %i.next = add nuw nsw i64 %i, 1 104 %outer_exitcond = icmp eq i64 %i.next, 1024 105 br i1 %outer_exitcond, label %exit, label %outer_loop, !llvm.loop !1 106 107exit: 108 ret void 109} 110 111!1 = distinct !{!1, !2} 112!2 = !{!"llvm.loop.vectorize.enable", i1 true} 113