1; RUN: opt %loadNPMPolly -polly-pattern-matching-based-opts=true \ 2; RUN: -polly-target-throughput-vector-fma=1 \ 3; RUN: -polly-target-latency-vector-fma=8 \ 4; RUN: -polly-target-1st-cache-level-size=0 \ 5; RUN: -polly-target-vector-register-bitwidth=256 \ 6; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s 7 8; RUN: opt %loadNPMPolly -polly-pattern-matching-based-opts=true \ 9; RUN: -polly-target-throughput-vector-fma=1 \ 10; RUN: -polly-target-latency-vector-fma=8 \ 11; RUN: -polly-target-1st-cache-level-associativity=8 \ 12; RUN: -polly-target-2nd-cache-level-associativity=8 \ 13; RUN: -polly-target-1st-cache-level-size=32768 \ 14; RUN: -polly-target-vector-register-bitwidth=256 \ 15; RUN: -polly-target-2nd-cache-level-size=262144 \ 16; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s 2>&1 | FileCheck %s --check-prefix=EXTRACTION-OF-MACRO-KERNEL 17; 18; /* C := alpha*A*B + beta*C */ 19; for (i = 0; i < _PB_NI; i++) 20; for (j = 0; j < _PB_NJ; j++) 21; { 22; C[i][j] *= beta; 23; for (k = 0; k < _PB_NK; ++k) 24; C[i][j] += alpha * A[i][k] * B[k][j]; 25; } 26; 27; CHECK-LABEL: :: isl ast :: kernel_gemm :: %bb8---%bb32 28; CHECK: { 29; CHECK-NEXT: // 1st level tiling - Tiles 30; CHECK-NEXT: for (int c0 = 0; c0 <= 32; c0 += 1) 31; CHECK-NEXT: for (int c1 = 0; c1 <= 32; c1 += 1) { 32; CHECK-NEXT: // 1st level tiling - Points 33; CHECK-NEXT: for (int c2 = 0; c2 <= 31; c2 += 1) 34; CHECK-NEXT: for (int c3 = 0; c3 <= 31; c3 += 1) 35; CHECK-NEXT: Stmt_bb9(32 * c0 + c2, 32 * c1 + c3); 36; CHECK-NEXT: } 37; CHECK-NEXT: // Register tiling - Tiles 38; CHECK-NEXT: for (int c0 = 0; c0 <= 131; c0 += 1) 39; CHECK-NEXT: for (int c1 = 0; c1 <= 263; c1 += 1) 40; CHECK-NEXT: for (int c2 = 0; c2 <= 1023; c2 += 1) { 41; CHECK-NEXT: // Register tiling - Points 42; CHECK-NEXT: { 43; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0, c2); 44; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 1, c2); 45; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 2, c2); 46; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 3, c2); 47; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 4, c2); 48; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 5, c2); 49; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 6, c2); 50; CHECK-NEXT: Stmt_Copy_0(4 * c1, 8 * c0 + 7, c2); 51; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0, c2); 52; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 1, c2); 53; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 2, c2); 54; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 3, c2); 55; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 4, c2); 56; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 5, c2); 57; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 6, c2); 58; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 1, 8 * c0 + 7, c2); 59; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0, c2); 60; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 1, c2); 61; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 2, c2); 62; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 3, c2); 63; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 4, c2); 64; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 5, c2); 65; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 6, c2); 66; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 2, 8 * c0 + 7, c2); 67; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0, c2); 68; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 1, c2); 69; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 2, c2); 70; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 3, c2); 71; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 4, c2); 72; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 5, c2); 73; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 6, c2); 74; CHECK-NEXT: Stmt_Copy_0(4 * c1 + 3, 8 * c0 + 7, c2); 75; CHECK-NEXT: } 76; CHECK-NEXT: } 77; CHECK-NEXT: } 78; 79; EXTRACTION-OF-MACRO-KERNEL-LABEL: :: isl ast :: kernel_gemm :: %bb8---%bb32 80; EXTRACTION-OF-MACRO-KERNEL: { 81; EXTRACTION-OF-MACRO-KERNEL-NEXT: // 1st level tiling - Tiles 82; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c0 = 0; c0 <= 32; c0 += 1) 83; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c1 = 0; c1 <= 32; c1 += 1) { 84; EXTRACTION-OF-MACRO-KERNEL-NEXT: // 1st level tiling - Points 85; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c2 = 0; c2 <= 31; c2 += 1) 86; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c3 = 0; c3 <= 31; c3 += 1) 87; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_bb9(32 * c0 + c2, 32 * c1 + c3); 88; EXTRACTION-OF-MACRO-KERNEL-NEXT: } 89; EXTRACTION-OF-MACRO-KERNEL-NEXT: // 1st level tiling - Tiles 90; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c1 = 0; c1 <= 3; c1 += 1) { 91; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c3 = 0; c3 <= 1055; c3 += 1) 92; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c4 = 256 * c1; c4 <= 256 * c1 + 255; c4 += 1) 93; EXTRACTION-OF-MACRO-KERNEL-NEXT: CopyStmt_0(0, c3, c4); 94; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c2 = 0; c2 <= 10; c2 += 1) { 95; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c6 = 96 * c2; c6 <= 96 * c2 + 95; c6 += 1) 96; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c7 = 256 * c1; c7 <= 256 * c1 + 255; c7 += 1) 97; EXTRACTION-OF-MACRO-KERNEL-NEXT: CopyStmt_1(0, c1, c2, c6, c7); 98; EXTRACTION-OF-MACRO-KERNEL-NEXT: // 1st level tiling - Points 99; EXTRACTION-OF-MACRO-KERNEL-NEXT: // Register tiling - Tiles 100; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c3 = 0; c3 <= 131; c3 += 1) 101; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c4 = 0; c4 <= 23; c4 += 1) 102; EXTRACTION-OF-MACRO-KERNEL-NEXT: for (int c5 = 0; c5 <= 255; c5 += 1) { 103; EXTRACTION-OF-MACRO-KERNEL-NEXT: // Loop Vectorizer Disabled 104; EXTRACTION-OF-MACRO-KERNEL-NEXT: // Register tiling - Points 105; EXTRACTION-OF-MACRO-KERNEL-NEXT: { 106; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3, 256 * c1 + c5); 107; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 1, 256 * c1 + c5); 108; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 2, 256 * c1 + c5); 109; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 3, 256 * c1 + c5); 110; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 4, 256 * c1 + c5); 111; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 5, 256 * c1 + c5); 112; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 6, 256 * c1 + c5); 113; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4, 8 * c3 + 7, 256 * c1 + c5); 114; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3, 256 * c1 + c5); 115; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 1, 256 * c1 + c5); 116; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 2, 256 * c1 + c5); 117; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 3, 256 * c1 + c5); 118; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 4, 256 * c1 + c5); 119; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 5, 256 * c1 + c5); 120; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 6, 256 * c1 + c5); 121; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 1, 8 * c3 + 7, 256 * c1 + c5); 122; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3, 256 * c1 + c5); 123; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 1, 256 * c1 + c5); 124; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 2, 256 * c1 + c5); 125; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 3, 256 * c1 + c5); 126; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 4, 256 * c1 + c5); 127; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 5, 256 * c1 + c5); 128; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 6, 256 * c1 + c5); 129; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 2, 8 * c3 + 7, 256 * c1 + c5); 130; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3, 256 * c1 + c5); 131; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 1, 256 * c1 + c5); 132; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 2, 256 * c1 + c5); 133; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 3, 256 * c1 + c5); 134; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 4, 256 * c1 + c5); 135; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 5, 256 * c1 + c5); 136; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 6, 256 * c1 + c5); 137; EXTRACTION-OF-MACRO-KERNEL-NEXT: Stmt_Copy_0(96 * c2 + 4 * c4 + 3, 8 * c3 + 7, 256 * c1 + c5); 138; EXTRACTION-OF-MACRO-KERNEL-NEXT: } 139; EXTRACTION-OF-MACRO-KERNEL-NEXT: } 140; EXTRACTION-OF-MACRO-KERNEL-NEXT: } 141; EXTRACTION-OF-MACRO-KERNEL-NEXT: } 142; EXTRACTION-OF-MACRO-KERNEL-NEXT: } 143; 144target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" 145target triple = "x86_64-unknown-unknown" 146 147define internal void @kernel_gemm(i32 %arg, i32 %arg1, i32 %arg2, double %arg3, double %arg4, ptr %arg5, ptr %arg6, ptr %arg7) #0 { 148bb: 149 br label %bb8 150 151bb8: ; preds = %bb29, %bb 152 %tmp = phi i64 [ 0, %bb ], [ %tmp30, %bb29 ] 153 br label %bb9 154 155bb9: ; preds = %bb26, %bb8 156 %tmp10 = phi i64 [ 0, %bb8 ], [ %tmp27, %bb26 ] 157 %tmp11 = getelementptr inbounds [1056 x double], ptr %arg5, i64 %tmp, i64 %tmp10 158 %tmp12 = load double, ptr %tmp11, align 8 159 %tmp13 = fmul double %tmp12, %arg4 160 store double %tmp13, ptr %tmp11, align 8 161 br label %Copy_0 162 163Copy_0: ; preds = %Copy_0, %bb9 164 %tmp15 = phi i64 [ 0, %bb9 ], [ %tmp24, %Copy_0 ] 165 %tmp16 = getelementptr inbounds [1024 x double], ptr %arg6, i64 %tmp, i64 %tmp15 166 %tmp17 = load double, ptr %tmp16, align 8 167 %tmp18 = fmul double %tmp17, %arg3 168 %tmp19 = getelementptr inbounds [1056 x double], ptr %arg7, i64 %tmp15, i64 %tmp10 169 %tmp20 = load double, ptr %tmp19, align 8 170 %tmp21 = fmul double %tmp18, %tmp20 171 %tmp22 = load double, ptr %tmp11, align 8 172 %tmp23 = fadd double %tmp22, %tmp21 173 store double %tmp23, ptr %tmp11, align 8 174 %tmp24 = add nuw nsw i64 %tmp15, 1 175 %tmp25 = icmp ne i64 %tmp24, 1024 176 br i1 %tmp25, label %Copy_0, label %bb26 177 178bb26: ; preds = %Copy_0 179 %tmp27 = add nuw nsw i64 %tmp10, 1 180 %tmp28 = icmp ne i64 %tmp27, 1056 181 br i1 %tmp28, label %bb9, label %bb29 182 183bb29: ; preds = %bb26 184 %tmp30 = add nuw nsw i64 %tmp, 1 185 %tmp31 = icmp ne i64 %tmp30, 1056 186 br i1 %tmp31, label %bb8, label %bb32 187 188bb32: ; preds = %bb29 189 ret void 190} 191