1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt -passes=loop-vectorize -S -o - < %s | FileCheck %s 3; RUN: opt -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S -o - < %s | FileCheck %s 4 5target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" 6target triple = "aarch64" 7 8; This test is not vectorized on AArch64 due to requiring predicated loads. 9; It should also not be interleaved as the predicated interleaving will just 10; create less efficient code. 11 12define void @arm_correlate_f16(ptr nocapture noundef readonly %pSrcA, i32 noundef %srcALen, ptr nocapture noundef readonly %pSrcB, i32 noundef %srcBLen, ptr nocapture noundef writeonly %pDst) { 13; CHECK-LABEL: @arm_correlate_f16( 14; CHECK-NEXT: entry: 15; CHECK-NEXT: [[SUB:%.*]] = add i32 [[SRCBLEN:%.*]], -1 16; CHECK-NEXT: [[IDX_EXT:%.*]] = zext i32 [[SUB]] to i64 17; CHECK-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds half, ptr [[PSRCB:%.*]], i64 [[IDX_EXT]] 18; CHECK-NEXT: [[ADD:%.*]] = add i32 [[SRCALEN:%.*]], -2 19; CHECK-NEXT: [[SUB1:%.*]] = add i32 [[ADD]], [[SRCBLEN]] 20; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i32 [[SRCALEN]], [[SRCBLEN]] 21; CHECK-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] 22; CHECK: if.then: 23; CHECK-NEXT: [[SUB2:%.*]] = sub i32 [[SRCALEN]], [[SRCBLEN]] 24; CHECK-NEXT: [[IDX_EXT3:%.*]] = zext i32 [[SUB2]] to i64 25; CHECK-NEXT: [[ADD_PTR4:%.*]] = getelementptr inbounds half, ptr [[PDST:%.*]], i64 [[IDX_EXT3]] 26; CHECK-NEXT: br label [[IF_END12:%.*]] 27; CHECK: if.else: 28; CHECK-NEXT: [[CMP5:%.*]] = icmp ult i32 [[SRCALEN]], [[SRCBLEN]] 29; CHECK-NEXT: br i1 [[CMP5]], label [[IF_THEN6:%.*]], label [[IF_END12]] 30; CHECK: if.then6: 31; CHECK-NEXT: [[SUB7:%.*]] = add i32 [[SRCALEN]], -1 32; CHECK-NEXT: [[IDX_EXT8:%.*]] = zext i32 [[SUB7]] to i64 33; CHECK-NEXT: [[ADD_PTR9:%.*]] = getelementptr inbounds half, ptr [[PSRCA:%.*]], i64 [[IDX_EXT8]] 34; CHECK-NEXT: [[IDX_EXT10:%.*]] = zext i32 [[SUB1]] to i64 35; CHECK-NEXT: [[ADD_PTR11:%.*]] = getelementptr inbounds half, ptr [[PDST]], i64 [[IDX_EXT10]] 36; CHECK-NEXT: br label [[IF_END12]] 37; CHECK: if.end12: 38; CHECK-NEXT: [[SRCALEN_ADDR_0:%.*]] = phi i32 [ [[SRCALEN]], [[IF_THEN]] ], [ [[SRCBLEN]], [[IF_THEN6]] ], [ [[SRCALEN]], [[IF_ELSE]] ] 39; CHECK-NEXT: [[SRCBLEN_ADDR_0:%.*]] = phi i32 [ [[SRCBLEN]], [[IF_THEN]] ], [ [[SRCALEN]], [[IF_THEN6]] ], [ [[SRCBLEN]], [[IF_ELSE]] ] 40; CHECK-NEXT: [[PDST_ADDR_0:%.*]] = phi ptr [ [[ADD_PTR4]], [[IF_THEN]] ], [ [[ADD_PTR11]], [[IF_THEN6]] ], [ [[PDST]], [[IF_ELSE]] ] 41; CHECK-NEXT: [[PIN1_0:%.*]] = phi ptr [ [[PSRCA]], [[IF_THEN]] ], [ [[PSRCB]], [[IF_THEN6]] ], [ [[PSRCA]], [[IF_ELSE]] ] 42; CHECK-NEXT: [[PIN2_0:%.*]] = phi ptr [ [[ADD_PTR]], [[IF_THEN]] ], [ [[ADD_PTR9]], [[IF_THEN6]] ], [ [[ADD_PTR]], [[IF_ELSE]] ] 43; CHECK-NEXT: [[CMP27:%.*]] = phi i64 [ 1, [[IF_THEN]] ], [ -1, [[IF_THEN6]] ], [ 1, [[IF_ELSE]] ] 44; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[SRCBLEN]], [[SRCALEN]] 45; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[TMP0]], -1 46; CHECK-NEXT: br label [[FOR_COND14_PREHEADER:%.*]] 47; CHECK: for.cond14.preheader: 48; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i32 [ 1, [[IF_END12]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_END:%.*]] ] 49; CHECK-NEXT: [[I_077:%.*]] = phi i32 [ 0, [[IF_END12]] ], [ [[INC33:%.*]], [[FOR_END]] ] 50; CHECK-NEXT: [[PDST_ADDR_176:%.*]] = phi ptr [ [[PDST_ADDR_0]], [[IF_END12]] ], [ [[PDST_ADDR_2:%.*]], [[FOR_END]] ] 51; CHECK-NEXT: br label [[FOR_BODY16:%.*]] 52; CHECK: for.body16: 53; CHECK-NEXT: [[J_074:%.*]] = phi i32 [ 0, [[FOR_COND14_PREHEADER]] ], [ [[INC:%.*]], [[FOR_INC:%.*]] ] 54; CHECK-NEXT: [[SUM_073:%.*]] = phi half [ 0xH0000, [[FOR_COND14_PREHEADER]] ], [ [[SUM_1:%.*]], [[FOR_INC]] ] 55; CHECK-NEXT: [[SUB17:%.*]] = sub i32 [[I_077]], [[J_074]] 56; CHECK-NEXT: [[CMP18:%.*]] = icmp ult i32 [[SUB17]], [[SRCBLEN_ADDR_0]] 57; CHECK-NEXT: [[CMP19:%.*]] = icmp ult i32 [[J_074]], [[SRCALEN_ADDR_0]] 58; CHECK-NEXT: [[OR_COND:%.*]] = and i1 [[CMP19]], [[CMP18]] 59; CHECK-NEXT: br i1 [[OR_COND]], label [[IF_THEN20:%.*]], label [[FOR_INC]] 60; CHECK: if.then20: 61; CHECK-NEXT: [[IDXPROM:%.*]] = zext i32 [[J_074]] to i64 62; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds half, ptr [[PIN1_0]], i64 [[IDXPROM]] 63; CHECK-NEXT: [[TMP2:%.*]] = load half, ptr [[ARRAYIDX]], align 2 64; CHECK-NEXT: [[SUB22:%.*]] = sub nsw i32 0, [[SUB17]] 65; CHECK-NEXT: [[IDXPROM23:%.*]] = sext i32 [[SUB22]] to i64 66; CHECK-NEXT: [[ARRAYIDX24:%.*]] = getelementptr inbounds half, ptr [[PIN2_0]], i64 [[IDXPROM23]] 67; CHECK-NEXT: [[TMP3:%.*]] = load half, ptr [[ARRAYIDX24]], align 2 68; CHECK-NEXT: [[MUL:%.*]] = fmul fast half [[TMP3]], [[TMP2]] 69; CHECK-NEXT: [[ADD25:%.*]] = fadd fast half [[MUL]], [[SUM_073]] 70; CHECK-NEXT: br label [[FOR_INC]] 71; CHECK: for.inc: 72; CHECK-NEXT: [[SUM_1]] = phi half [ [[ADD25]], [[IF_THEN20]] ], [ [[SUM_073]], [[FOR_BODY16]] ] 73; CHECK-NEXT: [[INC]] = add nuw i32 [[J_074]], 1 74; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[INDVARS_IV]] 75; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_END]], label [[FOR_BODY16]] 76; CHECK: for.end: 77; CHECK-NEXT: [[SUM_1_LCSSA:%.*]] = phi half [ [[SUM_1]], [[FOR_INC]] ] 78; CHECK-NEXT: [[PDST_ADDR_2]] = getelementptr inbounds half, ptr [[PDST_ADDR_176]], i64 [[CMP27]] 79; CHECK-NEXT: store half [[SUM_1_LCSSA]], ptr [[PDST_ADDR_176]], align 2 80; CHECK-NEXT: [[INC33]] = add nuw i32 [[I_077]], 1 81; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i32 [[INDVARS_IV]], 1 82; CHECK-NEXT: [[EXITCOND78_NOT:%.*]] = icmp eq i32 [[INC33]], [[TMP1]] 83; CHECK-NEXT: br i1 [[EXITCOND78_NOT]], label [[FOR_END34:%.*]], label [[FOR_COND14_PREHEADER]] 84; CHECK: for.end34: 85; CHECK-NEXT: ret void 86; 87entry: 88 %sub = add i32 %srcBLen, -1 89 %idx.ext = zext i32 %sub to i64 90 %add.ptr = getelementptr inbounds half, ptr %pSrcB, i64 %idx.ext 91 %add = add i32 %srcALen, -2 92 %sub1 = add i32 %add, %srcBLen 93 %cmp = icmp ugt i32 %srcALen, %srcBLen 94 br i1 %cmp, label %if.then, label %if.else 95 96if.then: ; preds = %entry 97 %sub2 = sub i32 %srcALen, %srcBLen 98 %idx.ext3 = zext i32 %sub2 to i64 99 %add.ptr4 = getelementptr inbounds half, ptr %pDst, i64 %idx.ext3 100 br label %if.end12 101 102if.else: ; preds = %entry 103 %cmp5 = icmp ult i32 %srcALen, %srcBLen 104 br i1 %cmp5, label %if.then6, label %if.end12 105 106if.then6: ; preds = %if.else 107 %sub7 = add i32 %srcALen, -1 108 %idx.ext8 = zext i32 %sub7 to i64 109 %add.ptr9 = getelementptr inbounds half, ptr %pSrcA, i64 %idx.ext8 110 %idx.ext10 = zext i32 %sub1 to i64 111 %add.ptr11 = getelementptr inbounds half, ptr %pDst, i64 %idx.ext10 112 br label %if.end12 113 114if.end12: ; preds = %if.else, %if.then6, %if.then 115 %srcALen.addr.0 = phi i32 [ %srcALen, %if.then ], [ %srcBLen, %if.then6 ], [ %srcALen, %if.else ] 116 %srcBLen.addr.0 = phi i32 [ %srcBLen, %if.then ], [ %srcALen, %if.then6 ], [ %srcBLen, %if.else ] 117 %pDst.addr.0 = phi ptr [ %add.ptr4, %if.then ], [ %add.ptr11, %if.then6 ], [ %pDst, %if.else ] 118 %pIn1.0 = phi ptr [ %pSrcA, %if.then ], [ %pSrcB, %if.then6 ], [ %pSrcA, %if.else ] 119 %pIn2.0 = phi ptr [ %add.ptr, %if.then ], [ %add.ptr9, %if.then6 ], [ %add.ptr, %if.else ] 120 %cmp27 = phi i64 [ 1, %if.then ], [ -1, %if.then6 ], [ 1, %if.else ] 121 %0 = add i32 %srcBLen, %srcALen 122 %1 = add i32 %0, -1 123 br label %for.cond14.preheader 124 125for.cond14.preheader: ; preds = %if.end12, %for.end 126 %indvars.iv = phi i32 [ 1, %if.end12 ], [ %indvars.iv.next, %for.end ] 127 %i.077 = phi i32 [ 0, %if.end12 ], [ %inc33, %for.end ] 128 %pDst.addr.176 = phi ptr [ %pDst.addr.0, %if.end12 ], [ %pDst.addr.2, %for.end ] 129 br label %for.body16 130 131for.body16: ; preds = %for.cond14.preheader, %for.inc 132 %j.074 = phi i32 [ 0, %for.cond14.preheader ], [ %inc, %for.inc ] 133 %sum.073 = phi half [ 0xH0000, %for.cond14.preheader ], [ %sum.1, %for.inc ] 134 %sub17 = sub i32 %i.077, %j.074 135 %cmp18 = icmp ult i32 %sub17, %srcBLen.addr.0 136 %cmp19 = icmp ult i32 %j.074, %srcALen.addr.0 137 %or.cond = and i1 %cmp19, %cmp18 138 br i1 %or.cond, label %if.then20, label %for.inc 139 140if.then20: ; preds = %for.body16 141 %idxprom = zext i32 %j.074 to i64 142 %arrayidx = getelementptr inbounds half, ptr %pIn1.0, i64 %idxprom 143 %2 = load half, ptr %arrayidx, align 2 144 %sub22 = sub nsw i32 0, %sub17 145 %idxprom23 = sext i32 %sub22 to i64 146 %arrayidx24 = getelementptr inbounds half, ptr %pIn2.0, i64 %idxprom23 147 %3 = load half, ptr %arrayidx24, align 2 148 %mul = fmul fast half %3, %2 149 %add25 = fadd fast half %mul, %sum.073 150 br label %for.inc 151 152for.inc: ; preds = %for.body16, %if.then20 153 %sum.1 = phi half [ %add25, %if.then20 ], [ %sum.073, %for.body16 ] 154 %inc = add nuw i32 %j.074, 1 155 %exitcond = icmp eq i32 %inc, %indvars.iv 156 br i1 %exitcond, label %for.end, label %for.body16 157 158for.end: ; preds = %for.inc 159 %sum.1.lcssa = phi half [ %sum.1, %for.inc ] 160 %pDst.addr.2 = getelementptr inbounds half, ptr %pDst.addr.176, i64 %cmp27 161 store half %sum.1.lcssa, ptr %pDst.addr.176, align 2 162 %inc33 = add nuw i32 %i.077, 1 163 %indvars.iv.next = add i32 %indvars.iv, 1 164 %exitcond78.not = icmp eq i32 %inc33, %1 165 br i1 %exitcond78.not, label %for.end34, label %for.cond14.preheader 166 167for.end34: ; preds = %for.end 168 ret void 169} 170