1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py 2; RUN: opt -passes=loop-load-elim -S < %s | \ 3; RUN: FileCheck %s -check-prefix=DEFAULT 4 5; RUN: opt -passes=loop-load-elim -S -enable-mem-access-versioning=0 < %s | \ 6; RUN: FileCheck %s -check-prefix=NO-VERSION 7 8; RUN: opt -passes=loop-load-elim -S -loop-load-elimination-scev-check-threshold=1 < %s | \ 9; RUN: FileCheck %s -check-prefix=THRESHOLD 10 11; Forwarding in the presence of symbolic strides: 12; 13; for (unsigned i = 0; i < 100; i++) 14; A[i + 1] = A[Stride * i] + B[i]; 15 16target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" 17 18define void @f(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, i64 %N, 19; 20; 21; 22; 23; 24; DEFAULT-LABEL: @f( 25; DEFAULT-NEXT: for.body.lver.check: 26; DEFAULT-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1 27; DEFAULT-NEXT: br i1 [[IDENT_CHECK]], label [[FOR_BODY_PH_LVER_ORIG:%.*]], label [[FOR_BODY_PH:%.*]] 28; DEFAULT: for.body.ph.lver.orig: 29; DEFAULT-NEXT: br label [[FOR_BODY_LVER_ORIG:%.*]] 30; DEFAULT: for.body.lver.orig: 31; DEFAULT-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ 0, [[FOR_BODY_PH_LVER_ORIG]] ], [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], [[FOR_BODY_LVER_ORIG]] ] 32; DEFAULT-NEXT: [[MUL_LVER_ORIG:%.*]] = mul i64 [[INDVARS_IV_LVER_ORIG]], [[STRIDE]] 33; DEFAULT-NEXT: [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[MUL_LVER_ORIG]] 34; DEFAULT-NEXT: [[LOAD_LVER_ORIG:%.*]] = load i32, ptr [[ARRAYIDX_LVER_ORIG]], align 4 35; DEFAULT-NEXT: [[ARRAYIDX2_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV_LVER_ORIG]] 36; DEFAULT-NEXT: [[LOAD_1_LVER_ORIG:%.*]] = load i32, ptr [[ARRAYIDX2_LVER_ORIG]], align 4 37; DEFAULT-NEXT: [[ADD_LVER_ORIG:%.*]] = add i32 [[LOAD_1_LVER_ORIG]], [[LOAD_LVER_ORIG]] 38; DEFAULT-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1 39; DEFAULT-NEXT: [[ARRAYIDX_NEXT_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_LVER_ORIG]] 40; DEFAULT-NEXT: store i32 [[ADD_LVER_ORIG]], ptr [[ARRAYIDX_NEXT_LVER_ORIG]], align 4 41; DEFAULT-NEXT: [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[N:%.*]] 42; DEFAULT-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]] 43; DEFAULT: for.body.ph: 44; DEFAULT-NEXT: [[LOAD_INITIAL:%.*]] = load i32, ptr [[A]], align 4 45; DEFAULT-NEXT: br label [[FOR_BODY:%.*]] 46; DEFAULT: for.body: 47; DEFAULT-NEXT: [[STORE_FORWARDED:%.*]] = phi i32 [ [[LOAD_INITIAL]], [[FOR_BODY_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ] 48; DEFAULT-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 49; DEFAULT-NEXT: [[MUL:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE]] 50; DEFAULT-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL]] 51; DEFAULT-NEXT: [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 52; DEFAULT-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]] 53; DEFAULT-NEXT: [[LOAD_1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 54; DEFAULT-NEXT: [[ADD]] = add i32 [[LOAD_1]], [[STORE_FORWARDED]] 55; DEFAULT-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 56; DEFAULT-NEXT: [[ARRAYIDX_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT]] 57; DEFAULT-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX_NEXT]], align 4 58; DEFAULT-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 59; DEFAULT-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT1:%.*]], label [[FOR_BODY]] 60; DEFAULT: for.end.loopexit: 61; DEFAULT-NEXT: br label [[FOR_END:%.*]] 62; DEFAULT: for.end.loopexit1: 63; DEFAULT-NEXT: br label [[FOR_END]] 64; DEFAULT: for.end: 65; DEFAULT-NEXT: ret void 66; 67; NO-VERSION-LABEL: @f( 68; NO-VERSION-NEXT: entry: 69; NO-VERSION-NEXT: br label [[FOR_BODY:%.*]] 70; NO-VERSION: for.body: 71; NO-VERSION-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 72; NO-VERSION-NEXT: [[MUL:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE:%.*]] 73; NO-VERSION-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[MUL]] 74; NO-VERSION-NEXT: [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 75; NO-VERSION-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]] 76; NO-VERSION-NEXT: [[LOAD_1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 77; NO-VERSION-NEXT: [[ADD:%.*]] = add i32 [[LOAD_1]], [[LOAD]] 78; NO-VERSION-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 79; NO-VERSION-NEXT: [[ARRAYIDX_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT]] 80; NO-VERSION-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX_NEXT]], align 4 81; NO-VERSION-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N:%.*]] 82; NO-VERSION-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]] 83; NO-VERSION: for.end: 84; NO-VERSION-NEXT: ret void 85; 86; THRESHOLD-LABEL: @f( 87; THRESHOLD-NEXT: for.body.lver.check: 88; THRESHOLD-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1 89; THRESHOLD-NEXT: br i1 [[IDENT_CHECK]], label [[FOR_BODY_PH_LVER_ORIG:%.*]], label [[FOR_BODY_PH:%.*]] 90; THRESHOLD: for.body.ph.lver.orig: 91; THRESHOLD-NEXT: br label [[FOR_BODY_LVER_ORIG:%.*]] 92; THRESHOLD: for.body.lver.orig: 93; THRESHOLD-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ 0, [[FOR_BODY_PH_LVER_ORIG]] ], [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], [[FOR_BODY_LVER_ORIG]] ] 94; THRESHOLD-NEXT: [[MUL_LVER_ORIG:%.*]] = mul i64 [[INDVARS_IV_LVER_ORIG]], [[STRIDE]] 95; THRESHOLD-NEXT: [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[MUL_LVER_ORIG]] 96; THRESHOLD-NEXT: [[LOAD_LVER_ORIG:%.*]] = load i32, ptr [[ARRAYIDX_LVER_ORIG]], align 4 97; THRESHOLD-NEXT: [[ARRAYIDX2_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV_LVER_ORIG]] 98; THRESHOLD-NEXT: [[LOAD_1_LVER_ORIG:%.*]] = load i32, ptr [[ARRAYIDX2_LVER_ORIG]], align 4 99; THRESHOLD-NEXT: [[ADD_LVER_ORIG:%.*]] = add i32 [[LOAD_1_LVER_ORIG]], [[LOAD_LVER_ORIG]] 100; THRESHOLD-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1 101; THRESHOLD-NEXT: [[ARRAYIDX_NEXT_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_LVER_ORIG]] 102; THRESHOLD-NEXT: store i32 [[ADD_LVER_ORIG]], ptr [[ARRAYIDX_NEXT_LVER_ORIG]], align 4 103; THRESHOLD-NEXT: [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[N:%.*]] 104; THRESHOLD-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]] 105; THRESHOLD: for.body.ph: 106; THRESHOLD-NEXT: [[LOAD_INITIAL:%.*]] = load i32, ptr [[A]], align 4 107; THRESHOLD-NEXT: br label [[FOR_BODY:%.*]] 108; THRESHOLD: for.body: 109; THRESHOLD-NEXT: [[STORE_FORWARDED:%.*]] = phi i32 [ [[LOAD_INITIAL]], [[FOR_BODY_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ] 110; THRESHOLD-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 111; THRESHOLD-NEXT: [[MUL:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE]] 112; THRESHOLD-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL]] 113; THRESHOLD-NEXT: [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 114; THRESHOLD-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]] 115; THRESHOLD-NEXT: [[LOAD_1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 116; THRESHOLD-NEXT: [[ADD]] = add i32 [[LOAD_1]], [[STORE_FORWARDED]] 117; THRESHOLD-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 118; THRESHOLD-NEXT: [[ARRAYIDX_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT]] 119; THRESHOLD-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX_NEXT]], align 4 120; THRESHOLD-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 121; THRESHOLD-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT1:%.*]], label [[FOR_BODY]] 122; THRESHOLD: for.end.loopexit: 123; THRESHOLD-NEXT: br label [[FOR_END:%.*]] 124; THRESHOLD: for.end.loopexit1: 125; THRESHOLD-NEXT: br label [[FOR_END]] 126; THRESHOLD: for.end: 127; THRESHOLD-NEXT: ret void 128; 129 i64 %stride) { 130 131 132entry: 133 br label %for.body 134 135for.body: ; preds = %for.body, %entry 136 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 137 %mul = mul i64 %indvars.iv, %stride 138 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %mul 139 %load = load i32, ptr %arrayidx, align 4 140 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv 141 %load_1 = load i32, ptr %arrayidx2, align 4 142 %add = add i32 %load_1, %load 143 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 144 %arrayidx_next = getelementptr inbounds i32, ptr %A, i64 %indvars.iv.next 145 store i32 %add, ptr %arrayidx_next, align 4 146 %exitcond = icmp eq i64 %indvars.iv.next, %N 147 br i1 %exitcond, label %for.end, label %for.body 148 149for.end: ; preds = %for.body 150 ret void 151} 152 153; Similar to @f(), but with a struct type. 154define void @f_struct(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, i64 %N, 155; 156; 157; 158; 159; 160; DEFAULT-LABEL: @f_struct( 161; DEFAULT-NEXT: for.body.lver.check: 162; DEFAULT-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1 163; DEFAULT-NEXT: br i1 [[IDENT_CHECK]], label [[FOR_BODY_PH_LVER_ORIG:%.*]], label [[FOR_BODY_PH:%.*]] 164; DEFAULT: for.body.ph.lver.orig: 165; DEFAULT-NEXT: br label [[FOR_BODY_LVER_ORIG:%.*]] 166; DEFAULT: for.body.lver.orig: 167; DEFAULT-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ 0, [[FOR_BODY_PH_LVER_ORIG]] ], [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], [[FOR_BODY_LVER_ORIG]] ] 168; DEFAULT-NEXT: [[MUL_LVER_ORIG:%.*]] = mul i64 [[INDVARS_IV_LVER_ORIG]], [[STRIDE]] 169; DEFAULT-NEXT: [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A:%.*]], i64 [[MUL_LVER_ORIG]] 170; DEFAULT-NEXT: [[LOAD_LVER_ORIG:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX_LVER_ORIG]], align 4 171; DEFAULT-NEXT: [[ARRAYIDX2_LVER_ORIG:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[B:%.*]], i64 [[INDVARS_IV_LVER_ORIG]] 172; DEFAULT-NEXT: [[LOAD_1_LVER_ORIG:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX2_LVER_ORIG]], align 4 173; DEFAULT-NEXT: [[V1_LVER_ORIG:%.*]] = extractvalue { i32, i8 } [[LOAD_LVER_ORIG]], 0 174; DEFAULT-NEXT: [[V2_LVER_ORIG:%.*]] = extractvalue { i32, i8 } [[LOAD_1_LVER_ORIG]], 0 175; DEFAULT-NEXT: [[ADD_LVER_ORIG:%.*]] = add i32 [[V1_LVER_ORIG]], [[V2_LVER_ORIG]] 176; DEFAULT-NEXT: [[INS_LVER_ORIG:%.*]] = insertvalue { i32, i8 } undef, i32 [[ADD_LVER_ORIG]], 0 177; DEFAULT-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1 178; DEFAULT-NEXT: [[ARRAYIDX_NEXT_LVER_ORIG:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A]], i64 [[INDVARS_IV_NEXT_LVER_ORIG]] 179; DEFAULT-NEXT: store { i32, i8 } [[INS_LVER_ORIG]], ptr [[ARRAYIDX_NEXT_LVER_ORIG]], align 4 180; DEFAULT-NEXT: [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[N:%.*]] 181; DEFAULT-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]] 182; DEFAULT: for.body.ph: 183; DEFAULT-NEXT: [[LOAD_INITIAL:%.*]] = load { i32, i8 }, ptr [[A]], align 4 184; DEFAULT-NEXT: br label [[FOR_BODY:%.*]] 185; DEFAULT: for.body: 186; DEFAULT-NEXT: [[STORE_FORWARDED:%.*]] = phi { i32, i8 } [ [[LOAD_INITIAL]], [[FOR_BODY_PH]] ], [ [[INS:%.*]], [[FOR_BODY]] ] 187; DEFAULT-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 188; DEFAULT-NEXT: [[MUL:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE]] 189; DEFAULT-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A]], i64 [[MUL]] 190; DEFAULT-NEXT: [[LOAD:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX]], align 4 191; DEFAULT-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[B]], i64 [[INDVARS_IV]] 192; DEFAULT-NEXT: [[LOAD_1:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX2]], align 4 193; DEFAULT-NEXT: [[V1:%.*]] = extractvalue { i32, i8 } [[STORE_FORWARDED]], 0 194; DEFAULT-NEXT: [[V2:%.*]] = extractvalue { i32, i8 } [[LOAD_1]], 0 195; DEFAULT-NEXT: [[ADD:%.*]] = add i32 [[V1]], [[V2]] 196; DEFAULT-NEXT: [[INS]] = insertvalue { i32, i8 } undef, i32 [[ADD]], 0 197; DEFAULT-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 198; DEFAULT-NEXT: [[ARRAYIDX_NEXT:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A]], i64 [[INDVARS_IV_NEXT]] 199; DEFAULT-NEXT: store { i32, i8 } [[INS]], ptr [[ARRAYIDX_NEXT]], align 4 200; DEFAULT-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 201; DEFAULT-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT1:%.*]], label [[FOR_BODY]] 202; DEFAULT: for.end.loopexit: 203; DEFAULT-NEXT: br label [[FOR_END:%.*]] 204; DEFAULT: for.end.loopexit1: 205; DEFAULT-NEXT: br label [[FOR_END]] 206; DEFAULT: for.end: 207; DEFAULT-NEXT: ret void 208; 209; NO-VERSION-LABEL: @f_struct( 210; NO-VERSION-NEXT: entry: 211; NO-VERSION-NEXT: br label [[FOR_BODY:%.*]] 212; NO-VERSION: for.body: 213; NO-VERSION-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 214; NO-VERSION-NEXT: [[MUL:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE:%.*]] 215; NO-VERSION-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A:%.*]], i64 [[MUL]] 216; NO-VERSION-NEXT: [[LOAD:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX]], align 4 217; NO-VERSION-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[B:%.*]], i64 [[INDVARS_IV]] 218; NO-VERSION-NEXT: [[LOAD_1:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX2]], align 4 219; NO-VERSION-NEXT: [[V1:%.*]] = extractvalue { i32, i8 } [[LOAD]], 0 220; NO-VERSION-NEXT: [[V2:%.*]] = extractvalue { i32, i8 } [[LOAD_1]], 0 221; NO-VERSION-NEXT: [[ADD:%.*]] = add i32 [[V1]], [[V2]] 222; NO-VERSION-NEXT: [[INS:%.*]] = insertvalue { i32, i8 } undef, i32 [[ADD]], 0 223; NO-VERSION-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 224; NO-VERSION-NEXT: [[ARRAYIDX_NEXT:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A]], i64 [[INDVARS_IV_NEXT]] 225; NO-VERSION-NEXT: store { i32, i8 } [[INS]], ptr [[ARRAYIDX_NEXT]], align 4 226; NO-VERSION-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N:%.*]] 227; NO-VERSION-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]] 228; NO-VERSION: for.end: 229; NO-VERSION-NEXT: ret void 230; 231; THRESHOLD-LABEL: @f_struct( 232; THRESHOLD-NEXT: for.body.lver.check: 233; THRESHOLD-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE:%.*]], 1 234; THRESHOLD-NEXT: br i1 [[IDENT_CHECK]], label [[FOR_BODY_PH_LVER_ORIG:%.*]], label [[FOR_BODY_PH:%.*]] 235; THRESHOLD: for.body.ph.lver.orig: 236; THRESHOLD-NEXT: br label [[FOR_BODY_LVER_ORIG:%.*]] 237; THRESHOLD: for.body.lver.orig: 238; THRESHOLD-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ 0, [[FOR_BODY_PH_LVER_ORIG]] ], [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], [[FOR_BODY_LVER_ORIG]] ] 239; THRESHOLD-NEXT: [[MUL_LVER_ORIG:%.*]] = mul i64 [[INDVARS_IV_LVER_ORIG]], [[STRIDE]] 240; THRESHOLD-NEXT: [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A:%.*]], i64 [[MUL_LVER_ORIG]] 241; THRESHOLD-NEXT: [[LOAD_LVER_ORIG:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX_LVER_ORIG]], align 4 242; THRESHOLD-NEXT: [[ARRAYIDX2_LVER_ORIG:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[B:%.*]], i64 [[INDVARS_IV_LVER_ORIG]] 243; THRESHOLD-NEXT: [[LOAD_1_LVER_ORIG:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX2_LVER_ORIG]], align 4 244; THRESHOLD-NEXT: [[V1_LVER_ORIG:%.*]] = extractvalue { i32, i8 } [[LOAD_LVER_ORIG]], 0 245; THRESHOLD-NEXT: [[V2_LVER_ORIG:%.*]] = extractvalue { i32, i8 } [[LOAD_1_LVER_ORIG]], 0 246; THRESHOLD-NEXT: [[ADD_LVER_ORIG:%.*]] = add i32 [[V1_LVER_ORIG]], [[V2_LVER_ORIG]] 247; THRESHOLD-NEXT: [[INS_LVER_ORIG:%.*]] = insertvalue { i32, i8 } undef, i32 [[ADD_LVER_ORIG]], 0 248; THRESHOLD-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1 249; THRESHOLD-NEXT: [[ARRAYIDX_NEXT_LVER_ORIG:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A]], i64 [[INDVARS_IV_NEXT_LVER_ORIG]] 250; THRESHOLD-NEXT: store { i32, i8 } [[INS_LVER_ORIG]], ptr [[ARRAYIDX_NEXT_LVER_ORIG]], align 4 251; THRESHOLD-NEXT: [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[N:%.*]] 252; THRESHOLD-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]] 253; THRESHOLD: for.body.ph: 254; THRESHOLD-NEXT: [[LOAD_INITIAL:%.*]] = load { i32, i8 }, ptr [[A]], align 4 255; THRESHOLD-NEXT: br label [[FOR_BODY:%.*]] 256; THRESHOLD: for.body: 257; THRESHOLD-NEXT: [[STORE_FORWARDED:%.*]] = phi { i32, i8 } [ [[LOAD_INITIAL]], [[FOR_BODY_PH]] ], [ [[INS:%.*]], [[FOR_BODY]] ] 258; THRESHOLD-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 259; THRESHOLD-NEXT: [[MUL:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE]] 260; THRESHOLD-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A]], i64 [[MUL]] 261; THRESHOLD-NEXT: [[LOAD:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX]], align 4 262; THRESHOLD-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[B]], i64 [[INDVARS_IV]] 263; THRESHOLD-NEXT: [[LOAD_1:%.*]] = load { i32, i8 }, ptr [[ARRAYIDX2]], align 4 264; THRESHOLD-NEXT: [[V1:%.*]] = extractvalue { i32, i8 } [[STORE_FORWARDED]], 0 265; THRESHOLD-NEXT: [[V2:%.*]] = extractvalue { i32, i8 } [[LOAD_1]], 0 266; THRESHOLD-NEXT: [[ADD:%.*]] = add i32 [[V1]], [[V2]] 267; THRESHOLD-NEXT: [[INS]] = insertvalue { i32, i8 } undef, i32 [[ADD]], 0 268; THRESHOLD-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 269; THRESHOLD-NEXT: [[ARRAYIDX_NEXT:%.*]] = getelementptr inbounds { i32, i8 }, ptr [[A]], i64 [[INDVARS_IV_NEXT]] 270; THRESHOLD-NEXT: store { i32, i8 } [[INS]], ptr [[ARRAYIDX_NEXT]], align 4 271; THRESHOLD-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 272; THRESHOLD-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT1:%.*]], label [[FOR_BODY]] 273; THRESHOLD: for.end.loopexit: 274; THRESHOLD-NEXT: br label [[FOR_END:%.*]] 275; THRESHOLD: for.end.loopexit1: 276; THRESHOLD-NEXT: br label [[FOR_END]] 277; THRESHOLD: for.end: 278; THRESHOLD-NEXT: ret void 279; 280 i64 %stride) { 281 282 283entry: 284 br label %for.body 285 286for.body: ; preds = %for.body, %entry 287 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 288 %mul = mul i64 %indvars.iv, %stride 289 %arrayidx = getelementptr inbounds { i32, i8 }, ptr %A, i64 %mul 290 %load = load { i32, i8 }, ptr %arrayidx, align 4 291 %arrayidx2 = getelementptr inbounds { i32, i8 }, ptr %B, i64 %indvars.iv 292 %load_1 = load { i32, i8 }, ptr %arrayidx2, align 4 293 294 295 %v1 = extractvalue { i32, i8 } %load, 0 296 %v2 = extractvalue { i32, i8} %load_1, 0 297 %add = add i32 %v1, %v2 298 %ins = insertvalue { i32, i8 } undef, i32 %add, 0 299 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 300 %arrayidx_next = getelementptr inbounds { i32, i8 }, ptr %A, i64 %indvars.iv.next 301 store { i32, i8 } %ins, ptr %arrayidx_next, align 4 302 %exitcond = icmp eq i64 %indvars.iv.next, %N 303 br i1 %exitcond, label %for.end, label %for.body 304 305for.end: ; preds = %for.body 306 ret void 307} 308 309; With two symbolic strides: 310; 311; for (unsigned i = 0; i < 100; i++) 312; A[Stride2 * (i + 1)] = A[Stride1 * i] + B[i]; 313 314define void @two_strides(ptr noalias nocapture %A, ptr noalias nocapture readonly %B, i64 %N, 315; 316; 317; 318; 319; 320; 321; DEFAULT-LABEL: @two_strides( 322; DEFAULT-NEXT: for.body.lver.check: 323; DEFAULT-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i64 [[STRIDE_2:%.*]], 1 324; DEFAULT-NEXT: [[IDENT_CHECK1:%.*]] = icmp ne i64 [[STRIDE_1:%.*]], 1 325; DEFAULT-NEXT: [[TMP0:%.*]] = or i1 [[IDENT_CHECK]], [[IDENT_CHECK1]] 326; DEFAULT-NEXT: br i1 [[TMP0]], label [[FOR_BODY_PH_LVER_ORIG:%.*]], label [[FOR_BODY_PH:%.*]] 327; DEFAULT: for.body.ph.lver.orig: 328; DEFAULT-NEXT: br label [[FOR_BODY_LVER_ORIG:%.*]] 329; DEFAULT: for.body.lver.orig: 330; DEFAULT-NEXT: [[INDVARS_IV_LVER_ORIG:%.*]] = phi i64 [ 0, [[FOR_BODY_PH_LVER_ORIG]] ], [ [[INDVARS_IV_NEXT_LVER_ORIG:%.*]], [[FOR_BODY_LVER_ORIG]] ] 331; DEFAULT-NEXT: [[MUL_LVER_ORIG:%.*]] = mul i64 [[INDVARS_IV_LVER_ORIG]], [[STRIDE_1]] 332; DEFAULT-NEXT: [[ARRAYIDX_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[MUL_LVER_ORIG]] 333; DEFAULT-NEXT: [[LOAD_LVER_ORIG:%.*]] = load i32, ptr [[ARRAYIDX_LVER_ORIG]], align 4 334; DEFAULT-NEXT: [[ARRAYIDX2_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV_LVER_ORIG]] 335; DEFAULT-NEXT: [[LOAD_1_LVER_ORIG:%.*]] = load i32, ptr [[ARRAYIDX2_LVER_ORIG]], align 4 336; DEFAULT-NEXT: [[ADD_LVER_ORIG:%.*]] = add i32 [[LOAD_1_LVER_ORIG]], [[LOAD_LVER_ORIG]] 337; DEFAULT-NEXT: [[INDVARS_IV_NEXT_LVER_ORIG]] = add nuw nsw i64 [[INDVARS_IV_LVER_ORIG]], 1 338; DEFAULT-NEXT: [[MUL_2_LVER_ORIG:%.*]] = mul i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[STRIDE_2]] 339; DEFAULT-NEXT: [[ARRAYIDX_NEXT_LVER_ORIG:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_2_LVER_ORIG]] 340; DEFAULT-NEXT: store i32 [[ADD_LVER_ORIG]], ptr [[ARRAYIDX_NEXT_LVER_ORIG]], align 4 341; DEFAULT-NEXT: [[EXITCOND_LVER_ORIG:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_LVER_ORIG]], [[N:%.*]] 342; DEFAULT-NEXT: br i1 [[EXITCOND_LVER_ORIG]], label [[FOR_END_LOOPEXIT:%.*]], label [[FOR_BODY_LVER_ORIG]] 343; DEFAULT: for.body.ph: 344; DEFAULT-NEXT: [[LOAD_INITIAL:%.*]] = load i32, ptr [[A]], align 4 345; DEFAULT-NEXT: br label [[FOR_BODY:%.*]] 346; DEFAULT: for.body: 347; DEFAULT-NEXT: [[STORE_FORWARDED:%.*]] = phi i32 [ [[LOAD_INITIAL]], [[FOR_BODY_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ] 348; DEFAULT-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[FOR_BODY_PH]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 349; DEFAULT-NEXT: [[MUL:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE_1]] 350; DEFAULT-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL]] 351; DEFAULT-NEXT: [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 352; DEFAULT-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDVARS_IV]] 353; DEFAULT-NEXT: [[LOAD_1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 354; DEFAULT-NEXT: [[ADD]] = add i32 [[LOAD_1]], [[STORE_FORWARDED]] 355; DEFAULT-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 356; DEFAULT-NEXT: [[MUL_2:%.*]] = mul i64 [[INDVARS_IV_NEXT]], [[STRIDE_2]] 357; DEFAULT-NEXT: [[ARRAYIDX_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_2]] 358; DEFAULT-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX_NEXT]], align 4 359; DEFAULT-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] 360; DEFAULT-NEXT: br i1 [[EXITCOND]], label [[FOR_END_LOOPEXIT2:%.*]], label [[FOR_BODY]] 361; DEFAULT: for.end.loopexit: 362; DEFAULT-NEXT: br label [[FOR_END:%.*]] 363; DEFAULT: for.end.loopexit2: 364; DEFAULT-NEXT: br label [[FOR_END]] 365; DEFAULT: for.end: 366; DEFAULT-NEXT: ret void 367; 368; NO-VERSION-LABEL: @two_strides( 369; NO-VERSION-NEXT: entry: 370; NO-VERSION-NEXT: br label [[FOR_BODY:%.*]] 371; NO-VERSION: for.body: 372; NO-VERSION-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 373; NO-VERSION-NEXT: [[MUL:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE_1:%.*]] 374; NO-VERSION-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[MUL]] 375; NO-VERSION-NEXT: [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 376; NO-VERSION-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]] 377; NO-VERSION-NEXT: [[LOAD_1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 378; NO-VERSION-NEXT: [[ADD:%.*]] = add i32 [[LOAD_1]], [[LOAD]] 379; NO-VERSION-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 380; NO-VERSION-NEXT: [[MUL_2:%.*]] = mul i64 [[INDVARS_IV_NEXT]], [[STRIDE_2:%.*]] 381; NO-VERSION-NEXT: [[ARRAYIDX_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_2]] 382; NO-VERSION-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX_NEXT]], align 4 383; NO-VERSION-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N:%.*]] 384; NO-VERSION-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]] 385; NO-VERSION: for.end: 386; NO-VERSION-NEXT: ret void 387; 388; THRESHOLD-LABEL: @two_strides( 389; THRESHOLD-NEXT: entry: 390; THRESHOLD-NEXT: br label [[FOR_BODY:%.*]] 391; THRESHOLD: for.body: 392; THRESHOLD-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ] 393; THRESHOLD-NEXT: [[MUL:%.*]] = mul i64 [[INDVARS_IV]], [[STRIDE_1:%.*]] 394; THRESHOLD-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[MUL]] 395; THRESHOLD-NEXT: [[LOAD:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 396; THRESHOLD-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDVARS_IV]] 397; THRESHOLD-NEXT: [[LOAD_1:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 398; THRESHOLD-NEXT: [[ADD:%.*]] = add i32 [[LOAD_1]], [[LOAD]] 399; THRESHOLD-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 400; THRESHOLD-NEXT: [[MUL_2:%.*]] = mul i64 [[INDVARS_IV_NEXT]], [[STRIDE_2:%.*]] 401; THRESHOLD-NEXT: [[ARRAYIDX_NEXT:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[MUL_2]] 402; THRESHOLD-NEXT: store i32 [[ADD]], ptr [[ARRAYIDX_NEXT]], align 4 403; THRESHOLD-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N:%.*]] 404; THRESHOLD-NEXT: br i1 [[EXITCOND]], label [[FOR_END:%.*]], label [[FOR_BODY]] 405; THRESHOLD: for.end: 406; THRESHOLD-NEXT: ret void 407; 408 i64 %stride.1, i64 %stride.2) { 409 410 411entry: 412 br label %for.body 413 414for.body: ; preds = %for.body, %entry 415 %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] 416 %mul = mul i64 %indvars.iv, %stride.1 417 %arrayidx = getelementptr inbounds i32, ptr %A, i64 %mul 418 %load = load i32, ptr %arrayidx, align 4 419 %arrayidx2 = getelementptr inbounds i32, ptr %B, i64 %indvars.iv 420 %load_1 = load i32, ptr %arrayidx2, align 4 421 %add = add i32 %load_1, %load 422 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 423 %mul.2 = mul i64 %indvars.iv.next, %stride.2 424 %arrayidx_next = getelementptr inbounds i32, ptr %A, i64 %mul.2 425 store i32 %add, ptr %arrayidx_next, align 4 426 %exitcond = icmp eq i64 %indvars.iv.next, %N 427 br i1 %exitcond, label %for.end, label %for.body 428 429for.end: ; preds = %for.body 430 ret void 431} 432