1; RUN: opt %loadNPMPolly -polly-pattern-matching-based-opts=true \ 2; RUN: -polly-target-throughput-vector-fma=1 \ 3; RUN: -polly-target-latency-vector-fma=8 \ 4; RUN: -polly-target-1st-cache-level-associativity=8 \ 5; RUN: -polly-target-2nd-cache-level-associativity=8 \ 6; RUN: -polly-target-1st-cache-level-size=32768 \ 7; RUN: -polly-target-vector-register-bitwidth=256 \ 8; RUN: -polly-target-2nd-cache-level-size=262144 \ 9; RUN: '-passes=polly-opt-isl,print<polly-ast>' -disable-output < %s | FileCheck %s 10; 11; opt %loadNPMPolly -passes=polly-opt-isl -polly-pattern-matching-based-opts=true \ 12; -polly-target-throughput-vector-fma=1 \ 13; -polly-target-latency-vector-fma=8 \ 14; -passes=polly-codegen -polly-target-1st-cache-level-associativity=8 \ 15; -polly-target-2nd-cache-level-associativity=8 \ 16; -polly-target-1st-cache-level-size=32768 \ 17; -polly-target-vector-register-bitwidth=256 \ 18; -polly-target-2nd-cache-level-size=262144 -gvn -licm -slp-vectorizer \ 19; -mcpu=corei7 -stats -S < %s 2>&1 | FileCheck %s --check-prefix=AUTO-VECTORIZATION 20; 21; 22; /* We isolate a set of partial tile prefixes, which contains only partial 23; tile prefixes that have exactly Mr x Nr iterations of the two innermost 24; loops produced by the optimization of the matrix multiplication. Mr and 25; Nr are parameters of the micro-kernel (see getMicroKernelParams and 26; getMacroKernelParams from lib/Transform/ScheduleOptimizer.cpp for 27; details). This test check that in case of parametric bounds it helps to 28; get rid of the conditional expressions of the unrolled innermost loops, 29; which prevents stores and loads of the unrolled loops from being sunk 30; and hoisted. Otherwise, it causes a run-time regression in comparison 31; to the vectorized code with sunk and hoisted memory accesses. */ 32; 33; /* C := A * B + C */ 34; for (i = 0; i < _PB_NI; i++) 35; for (j = 0; j < _PB_NJ; j++) 36; for (k = 0; k < _PB_NK; ++k) 37; C[i][j] += A[i][k] * B[k][j]; 38; 39; CHECK: if (ni >= 1) { 40; CHECK-NEXT: // 1st level tiling - Tiles 41; CHECK-NEXT: for (int c0 = 0; c0 <= floord(nj - 1, 2048); c0 += 1) 42; CHECK-NEXT: for (int c1 = 0; c1 <= floord(nk - 1, 256); c1 += 1) { 43; CHECK-NEXT: for (int c3 = 2048 * c0; c3 <= min(nj - 1, 2048 * c0 + 2047); c3 += 1) 44; CHECK-NEXT: for (int c4 = 256 * c1; c4 <= min(nk - 1, 256 * c1 + 255); c4 += 1) 45; CHECK-NEXT: CopyStmt_0(0, c3, c4); 46; CHECK-NEXT: for (int c2 = 0; c2 <= floord(ni - 1, 96); c2 += 1) { 47; CHECK-NEXT: for (int c6 = 96 * c2; c6 <= min(ni - 1, 96 * c2 + 95); c6 += 1) 48; CHECK-NEXT: for (int c7 = 256 * c1; c7 <= min(nk - 1, 256 * c1 + 255); c7 += 1) 49; CHECK-NEXT: CopyStmt_1(c0, c1, c2, c6, c7); 50; CHECK-NEXT: // 1st level tiling - Points 51; CHECK-NEXT: // Register tiling - Tiles 52; CHECK-NEXT: { 53; CHECK-NEXT: if (ni >= 96 * c2 + 4) 54; CHECK-NEXT: for (int c3 = 0; c3 <= min(255, -256 * c0 + nj / 8 - 1); c3 += 1) { 55; CHECK-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + ni / 4 - 1); c4 += 1) 56; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) { 57; CHECK-NEXT: // Loop Vectorizer Disabled 58; CHECK-NEXT: // Register tiling - Points 59; CHECK-NEXT: { 60; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3, 256 * c1 + c5); 61; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 62; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 63; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 64; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 65; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 66; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 67; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 68; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3, 256 * c1 + c5); 69; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 70; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 71; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 72; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 73; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 74; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 75; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 76; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3, 256 * c1 + c5); 77; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 78; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 79; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 80; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 81; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 82; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 83; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 84; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3, 256 * c1 + c5); 85; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 86; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 87; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 88; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 89; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 90; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 91; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 92; CHECK-NEXT: } 93; CHECK-NEXT: } 94; CHECK-NEXT: if ((ni >= 96 * c2 + 5 && 96 * c2 + 7 >= ni && c3 >= 0) || (ni >= 96 * c2 + 8 && 96 * c2 + 95 >= ni && ni % 4 >= 1)) { 95; CHECK-NEXT: if (96 * c2 + 7 >= ni) { 96; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) { 97; CHECK-NEXT: // Loop Vectorizer Disabled 98; CHECK-NEXT: // Register tiling - Points 99; CHECK-NEXT: { 100; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3, 256 * c1 + c5); 101; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 102; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 103; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 104; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 105; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 106; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 107; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 108; CHECK-NEXT: if (ni >= 96 * c2 + 6) { 109; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3, 256 * c1 + c5); 110; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 111; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 112; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 113; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 114; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 115; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 116; CHECK-NEXT: Stmt_for_body6(96 * c2 + 5, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 117; CHECK-NEXT: if (96 * c2 + 7 == ni) { 118; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5); 119; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 120; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 121; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 122; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 123; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 124; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 125; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 126; CHECK-NEXT: } 127; CHECK-NEXT: } 128; CHECK-NEXT: } 129; CHECK-NEXT: } 130; CHECK-NEXT: } else { 131; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) { 132; CHECK-NEXT: // Loop Vectorizer Disabled 133; CHECK-NEXT: // Register tiling - Points 134; CHECK-NEXT: { 135; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5); 136; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 137; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 138; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 139; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 140; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 141; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 142; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 143; CHECK-NEXT: if (ni % 4 >= 2) { 144; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3, 256 * c1 + c5); 145; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 146; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 147; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 148; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 149; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 150; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 151; CHECK-NEXT: Stmt_for_body6(-((ni - 1) % 4) + ni, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 152; CHECK-NEXT: if ((ni + 1) % 4 == 0) { 153; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5); 154; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 155; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 156; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 157; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 158; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 159; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 160; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 161; CHECK-NEXT: } 162; CHECK-NEXT: } 163; CHECK-NEXT: } 164; CHECK-NEXT: } 165; CHECK-NEXT: } 166; CHECK-NEXT: } 167; CHECK-NEXT: } 168; CHECK-NEXT: if (96 * c2 + 3 >= ni || 2048 * c0 + 7 >= nj || (2048 * c0 + 2047 >= nj && nj % 8 >= 1)) { 169; CHECK-NEXT: if (2048 * c0 + 7 >= nj) { 170; CHECK-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + (ni - 1) / 4); c4 += 1) 171; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) { 172; CHECK-NEXT: // Loop Vectorizer Disabled 173; CHECK-NEXT: // Register tiling - Points 174; CHECK-NEXT: { 175; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0, 256 * c1 + c5); 176; CHECK-NEXT: if (nj >= 2048 * c0 + 2) { 177; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 1, 256 * c1 + c5); 178; CHECK-NEXT: if (nj >= 2048 * c0 + 3) { 179; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 2, 256 * c1 + c5); 180; CHECK-NEXT: if (nj >= 2048 * c0 + 4) { 181; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 3, 256 * c1 + c5); 182; CHECK-NEXT: if (nj >= 2048 * c0 + 5) { 183; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 4, 256 * c1 + c5); 184; CHECK-NEXT: if (nj >= 2048 * c0 + 6) { 185; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, 2048 * c0 + 5, 256 * c1 + c5); 186; CHECK-NEXT: if (2048 * c0 + 7 == nj) 187; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, nj - 1, 256 * c1 + c5); 188; CHECK-NEXT: } 189; CHECK-NEXT: } 190; CHECK-NEXT: } 191; CHECK-NEXT: } 192; CHECK-NEXT: } 193; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 2) { 194; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0, 256 * c1 + c5); 195; CHECK-NEXT: if (nj >= 2048 * c0 + 2) { 196; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 1, 256 * c1 + c5); 197; CHECK-NEXT: if (nj >= 2048 * c0 + 3) { 198; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 2, 256 * c1 + c5); 199; CHECK-NEXT: if (nj >= 2048 * c0 + 4) { 200; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 3, 256 * c1 + c5); 201; CHECK-NEXT: if (nj >= 2048 * c0 + 5) { 202; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 4, 256 * c1 + c5); 203; CHECK-NEXT: if (nj >= 2048 * c0 + 6) { 204; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, 2048 * c0 + 5, 256 * c1 + c5); 205; CHECK-NEXT: if (2048 * c0 + 7 == nj) 206; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, nj - 1, 256 * c1 + c5); 207; CHECK-NEXT: } 208; CHECK-NEXT: } 209; CHECK-NEXT: } 210; CHECK-NEXT: } 211; CHECK-NEXT: } 212; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 3) { 213; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0, 256 * c1 + c5); 214; CHECK-NEXT: if (nj >= 2048 * c0 + 2) { 215; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 1, 256 * c1 + c5); 216; CHECK-NEXT: if (nj >= 2048 * c0 + 3) { 217; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 2, 256 * c1 + c5); 218; CHECK-NEXT: if (nj >= 2048 * c0 + 4) { 219; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 3, 256 * c1 + c5); 220; CHECK-NEXT: if (nj >= 2048 * c0 + 5) { 221; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 4, 256 * c1 + c5); 222; CHECK-NEXT: if (nj >= 2048 * c0 + 6) { 223; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, 2048 * c0 + 5, 256 * c1 + c5); 224; CHECK-NEXT: if (2048 * c0 + 7 == nj) 225; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, nj - 1, 256 * c1 + c5); 226; CHECK-NEXT: } 227; CHECK-NEXT: } 228; CHECK-NEXT: } 229; CHECK-NEXT: } 230; CHECK-NEXT: } 231; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 4) { 232; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0, 256 * c1 + c5); 233; CHECK-NEXT: if (nj >= 2048 * c0 + 2) { 234; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 1, 256 * c1 + c5); 235; CHECK-NEXT: if (nj >= 2048 * c0 + 3) { 236; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 2, 256 * c1 + c5); 237; CHECK-NEXT: if (nj >= 2048 * c0 + 4) { 238; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 3, 256 * c1 + c5); 239; CHECK-NEXT: if (nj >= 2048 * c0 + 5) { 240; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 4, 256 * c1 + c5); 241; CHECK-NEXT: if (nj >= 2048 * c0 + 6) { 242; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, 2048 * c0 + 5, 256 * c1 + c5); 243; CHECK-NEXT: if (2048 * c0 + 7 == nj) 244; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, nj - 1, 256 * c1 + c5); 245; CHECK-NEXT: } 246; CHECK-NEXT: } 247; CHECK-NEXT: } 248; CHECK-NEXT: } 249; CHECK-NEXT: } 250; CHECK-NEXT: } 251; CHECK-NEXT: } 252; CHECK-NEXT: } 253; CHECK-NEXT: } 254; CHECK-NEXT: } 255; CHECK-NEXT: } else if (96 * c2 + 3 >= ni) { 256; CHECK-NEXT: for (int c3 = 0; c3 <= min(255, -256 * c0 + (nj - 1) / 8); c3 += 1) 257; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) { 258; CHECK-NEXT: // Loop Vectorizer Disabled 259; CHECK-NEXT: // Register tiling - Points 260; CHECK-NEXT: { 261; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3, 256 * c1 + c5); 262; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 2) { 263; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 264; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 3) { 265; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 266; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 4) { 267; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 268; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 5) { 269; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 270; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 6) { 271; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 272; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 7) { 273; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 274; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 8) 275; CHECK-NEXT: Stmt_for_body6(96 * c2, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 276; CHECK-NEXT: } 277; CHECK-NEXT: } 278; CHECK-NEXT: } 279; CHECK-NEXT: } 280; CHECK-NEXT: } 281; CHECK-NEXT: } 282; CHECK-NEXT: if (ni >= 96 * c2 + 2) { 283; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3, 256 * c1 + c5); 284; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 2) { 285; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 286; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 3) { 287; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 288; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 4) { 289; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 290; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 5) { 291; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 292; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 6) { 293; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 294; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 7) { 295; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 296; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 8) 297; CHECK-NEXT: Stmt_for_body6(96 * c2 + 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 298; CHECK-NEXT: } 299; CHECK-NEXT: } 300; CHECK-NEXT: } 301; CHECK-NEXT: } 302; CHECK-NEXT: } 303; CHECK-NEXT: } 304; CHECK-NEXT: if (96 * c2 + 3 == ni) { 305; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3, 256 * c1 + c5); 306; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 2) { 307; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 1, 256 * c1 + c5); 308; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 3) { 309; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 2, 256 * c1 + c5); 310; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 4) { 311; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 3, 256 * c1 + c5); 312; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 5) { 313; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 4, 256 * c1 + c5); 314; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 6) { 315; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 5, 256 * c1 + c5); 316; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 7) { 317; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 6, 256 * c1 + c5); 318; CHECK-NEXT: if (nj >= 2048 * c0 + 8 * c3 + 8) 319; CHECK-NEXT: Stmt_for_body6(ni - 1, 2048 * c0 + 8 * c3 + 7, 256 * c1 + c5); 320; CHECK-NEXT: } 321; CHECK-NEXT: } 322; CHECK-NEXT: } 323; CHECK-NEXT: } 324; CHECK-NEXT: } 325; CHECK-NEXT: } 326; CHECK-NEXT: } 327; CHECK-NEXT: } 328; CHECK-NEXT: } 329; CHECK-NEXT: } 330; CHECK-NEXT: } else { 331; CHECK-NEXT: for (int c4 = 0; c4 <= min(23, -24 * c2 + (ni - 1) / 4); c4 += 1) 332; CHECK-NEXT: for (int c5 = 0; c5 <= min(255, nk - 256 * c1 - 1); c5 += 1) { 333; CHECK-NEXT: // Loop Vectorizer Disabled 334; CHECK-NEXT: // Register tiling - Points 335; CHECK-NEXT: { 336; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5); 337; CHECK-NEXT: if (nj % 8 >= 2) { 338; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj, 256 * c1 + c5); 339; CHECK-NEXT: if (nj % 8 >= 3) { 340; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5); 341; CHECK-NEXT: if (nj % 8 >= 4) { 342; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5); 343; CHECK-NEXT: if (nj % 8 >= 5) { 344; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5); 345; CHECK-NEXT: if (nj % 8 >= 6) { 346; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5); 347; CHECK-NEXT: if ((nj + 1) % 8 == 0) 348; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4, nj - 1, 256 * c1 + c5); 349; CHECK-NEXT: } 350; CHECK-NEXT: } 351; CHECK-NEXT: } 352; CHECK-NEXT: } 353; CHECK-NEXT: } 354; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 2) { 355; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5); 356; CHECK-NEXT: if (nj % 8 >= 2) { 357; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj, 256 * c1 + c5); 358; CHECK-NEXT: if (nj % 8 >= 3) { 359; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5); 360; CHECK-NEXT: if (nj % 8 >= 4) { 361; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5); 362; CHECK-NEXT: if (nj % 8 >= 5) { 363; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5); 364; CHECK-NEXT: if (nj % 8 >= 6) { 365; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5); 366; CHECK-NEXT: if ((nj + 1) % 8 == 0) 367; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 1, nj - 1, 256 * c1 + c5); 368; CHECK-NEXT: } 369; CHECK-NEXT: } 370; CHECK-NEXT: } 371; CHECK-NEXT: } 372; CHECK-NEXT: } 373; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 3) { 374; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5); 375; CHECK-NEXT: if (nj % 8 >= 2) { 376; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj, 256 * c1 + c5); 377; CHECK-NEXT: if (nj % 8 >= 3) { 378; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5); 379; CHECK-NEXT: if (nj % 8 >= 4) { 380; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5); 381; CHECK-NEXT: if (nj % 8 >= 5) { 382; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5); 383; CHECK-NEXT: if (nj % 8 >= 6) { 384; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5); 385; CHECK-NEXT: if ((nj + 1) % 8 == 0) 386; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 2, nj - 1, 256 * c1 + c5); 387; CHECK-NEXT: } 388; CHECK-NEXT: } 389; CHECK-NEXT: } 390; CHECK-NEXT: } 391; CHECK-NEXT: } 392; CHECK-NEXT: if (ni >= 96 * c2 + 4 * c4 + 4) { 393; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj - 1, 256 * c1 + c5); 394; CHECK-NEXT: if (nj % 8 >= 2) { 395; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj, 256 * c1 + c5); 396; CHECK-NEXT: if (nj % 8 >= 3) { 397; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 1, 256 * c1 + c5); 398; CHECK-NEXT: if (nj % 8 >= 4) { 399; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 2, 256 * c1 + c5); 400; CHECK-NEXT: if (nj % 8 >= 5) { 401; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 3, 256 * c1 + c5); 402; CHECK-NEXT: if (nj % 8 >= 6) { 403; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, -((nj - 1) % 8) + nj + 4, 256 * c1 + c5); 404; CHECK-NEXT: if ((nj + 1) % 8 == 0) 405; CHECK-NEXT: Stmt_for_body6(96 * c2 + 4 * c4 + 3, nj - 1, 256 * c1 + c5); 406; CHECK-NEXT: } 407; CHECK-NEXT: } 408; CHECK-NEXT: } 409; CHECK-NEXT: } 410; CHECK-NEXT: } 411; CHECK-NEXT: } 412; CHECK-NEXT: } 413; CHECK-NEXT: } 414; CHECK-NEXT: } 415; CHECK-NEXT: } 416; CHECK-NEXT: } 417; CHECK-NEXT: } 418; CHECK-NEXT: } 419; CHECK-NEXT: } 420; CHECK-NEXT: } 421; CHECK-NEXT: } 422; 423 424; AUTO-VECTORIZATION: fmul <4 x double> 425; AUTO-VECTORIZATION: fadd <4 x double> 426 427; AUTO-VECTORIZATION: 36 SLP - Number of vector instructions generated 428; AUTO-VECTORIZATION: 453 licm - Number of instructions hoisted out of loop 429; AUTO-VECTORIZATION: 2 licm - Number of load insts hoisted or sunk 430; AUTO-VECTORIZATION: 32 licm - Number of memory locations promoted to registers 431; 432target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" 433target triple = "x86_64-unknown-unknown" 434 435define internal void @kernel_gemm(i32 %ni, i32 %nj, i32 %nk, double %alpha, double %beta, ptr %C, ptr %A, ptr %B) #0 { 436entry: 437 br label %entry.split 438 439entry.split: ; preds = %entry 440 %cmp39 = icmp sgt i32 %ni, 0 441 br i1 %cmp39, label %for.cond1.preheader.lr.ph, label %for.end22 442 443for.cond1.preheader.lr.ph: ; preds = %entry.split 444 br label %for.cond1.preheader 445 446for.cond1.preheader: ; preds = %for.inc20, %for.cond1.preheader.lr.ph 447 %indvars.iv45 = phi i64 [ 0, %for.cond1.preheader.lr.ph ], [ %indvars.iv.next46, %for.inc20 ] 448 %cmp237 = icmp sgt i32 %nj, 0 449 br i1 %cmp237, label %for.cond4.preheader.lr.ph, label %for.inc20 450 451for.cond4.preheader.lr.ph: ; preds = %for.cond1.preheader 452 br label %for.cond4.preheader 453 454for.cond4.preheader: ; preds = %for.inc17, %for.cond4.preheader.lr.ph 455 %indvars.iv41 = phi i64 [ 0, %for.cond4.preheader.lr.ph ], [ %indvars.iv.next42, %for.inc17 ] 456 %cmp535 = icmp sgt i32 %nk, 0 457 br i1 %cmp535, label %for.body6.lr.ph, label %for.inc17 458 459for.body6.lr.ph: ; preds = %for.cond4.preheader 460 br label %for.body6 461 462for.body6: ; preds = %for.body6, %for.body6.lr.ph 463 %indvars.iv = phi i64 [ 0, %for.body6.lr.ph ], [ %indvars.iv.next, %for.body6 ] 464 %arrayidx8 = getelementptr inbounds [1024 x double], ptr %A, i64 %indvars.iv45, i64 %indvars.iv 465 %tmp = load double, ptr %arrayidx8, align 8 466 %arrayidx12 = getelementptr inbounds [1024 x double], ptr %B, i64 %indvars.iv, i64 %indvars.iv41 467 %tmp1 = load double, ptr %arrayidx12, align 8 468 %mul = fmul double %tmp, %tmp1 469 %arrayidx16 = getelementptr inbounds [1024 x double], ptr %C, i64 %indvars.iv45, i64 %indvars.iv41 470 %tmp2 = load double, ptr %arrayidx16, align 8 471 %add = fadd double %tmp2, %mul 472 store double %add, ptr %arrayidx16, align 8 473 %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 474 %wide.trip.count = zext i32 %nk to i64 475 %exitcond = icmp ne i64 %indvars.iv.next, %wide.trip.count 476 br i1 %exitcond, label %for.body6, label %for.cond4.for.inc17_crit_edge 477 478for.cond4.for.inc17_crit_edge: ; preds = %for.body6 479 br label %for.inc17 480 481for.inc17: ; preds = %for.cond4.for.inc17_crit_edge, %for.cond4.preheader 482 %indvars.iv.next42 = add nuw nsw i64 %indvars.iv41, 1 483 %wide.trip.count43 = zext i32 %nj to i64 484 %exitcond44 = icmp ne i64 %indvars.iv.next42, %wide.trip.count43 485 br i1 %exitcond44, label %for.cond4.preheader, label %for.cond1.for.inc20_crit_edge 486 487for.cond1.for.inc20_crit_edge: ; preds = %for.inc17 488 br label %for.inc20 489 490for.inc20: ; preds = %for.cond1.for.inc20_crit_edge, %for.cond1.preheader 491 %indvars.iv.next46 = add nuw nsw i64 %indvars.iv45, 1 492 %wide.trip.count47 = zext i32 %ni to i64 493 %exitcond48 = icmp ne i64 %indvars.iv.next46, %wide.trip.count47 494 br i1 %exitcond48, label %for.cond1.preheader, label %for.cond.for.end22_crit_edge 495 496for.cond.for.end22_crit_edge: ; preds = %for.inc20 497 br label %for.end22 498 499for.end22: ; preds = %for.cond.for.end22_crit_edge, %entry.split 500 ret void 501} 502 503attributes #0 = { nounwind uwtable "target-cpu"="x86-64" "target-features"="+aes,+avx,+cmov,+cx16,+fxsr,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" } 504