xref: /llvm-project/llvm/test/Transforms/LowerMatrixIntrinsics/bigger-expressions-double.ll (revision 23c21759458014fc4d7cbea45b6fbe7349a0a4fd) 
1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
2; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
3
4define void @transpose_multiply(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
5; CHECK-LABEL: @transpose_multiply(
6; CHECK-NEXT:  entry:
7; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[A_PTR:%.*]], align 8
8; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[A_PTR]], i64 3
9; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
10; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[A_PTR]], i64 6
11; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
12; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[B_PTR:%.*]], align 8
13; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr double, ptr [[B_PTR]], i64 3
14; CHECK-NEXT:    [[COL_LOAD6:%.*]] = load <3 x double>, ptr [[VEC_GEP5]], align 8
15; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr double, ptr [[B_PTR]], i64 6
16; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <3 x double>, ptr [[VEC_GEP7]], align 8
17; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
18; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x double> poison, double [[TMP0]], i64 0
19; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 0
20; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1
21; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 0
22; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2
23; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
24; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <3 x double> poison, double [[TMP6]], i64 0
25; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 1
26; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
27; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 1
28; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
29; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
30; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <3 x double> poison, double [[TMP12]], i64 0
31; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 2
32; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1
33; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 2
34; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
35; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
36; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
37; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP18]], i64 0
38; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
39; CHECK-NEXT:    [[TMP19:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
40; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
41; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
42; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i64 0
43; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
44; CHECK-NEXT:    [[TMP21:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
45; CHECK-NEXT:    [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]]
46; CHECK-NEXT:    [[BLOCK12:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
47; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
48; CHECK-NEXT:    [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> poison, double [[TMP23]], i64 0
49; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
50; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
51; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
52; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
53; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
54; CHECK-NEXT:    [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
55; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
56; CHECK-NEXT:    [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> poison, double [[TMP28]], i64 0
57; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> poison, <1 x i32> zeroinitializer
58; CHECK-NEXT:    [[TMP29:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
59; CHECK-NEXT:    [[BLOCK18:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
60; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
61; CHECK-NEXT:    [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> poison, double [[TMP30]], i64 0
62; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> poison, <1 x i32> zeroinitializer
63; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
64; CHECK-NEXT:    [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
65; CHECK-NEXT:    [[BLOCK21:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
66; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
67; CHECK-NEXT:    [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> poison, double [[TMP33]], i64 0
68; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
69; CHECK-NEXT:    [[TMP34:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
70; CHECK-NEXT:    [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]]
71; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
72; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> <i32 0, i32 3, i32 2>
73; CHECK-NEXT:    [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
74; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
75; CHECK-NEXT:    [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> poison, double [[TMP38]], i64 0
76; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> poison, <1 x i32> zeroinitializer
77; CHECK-NEXT:    [[TMP39:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
78; CHECK-NEXT:    [[BLOCK27:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
79; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
80; CHECK-NEXT:    [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> poison, double [[TMP40]], i64 0
81; CHECK-NEXT:    [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> poison, <1 x i32> zeroinitializer
82; CHECK-NEXT:    [[TMP41:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
83; CHECK-NEXT:    [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]]
84; CHECK-NEXT:    [[BLOCK30:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
85; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
86; CHECK-NEXT:    [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> poison, double [[TMP43]], i64 0
87; CHECK-NEXT:    [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
88; CHECK-NEXT:    [[TMP44:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
89; CHECK-NEXT:    [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]]
90; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
91; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> <i32 0, i32 1, i32 3>
92; CHECK-NEXT:    [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
93; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
94; CHECK-NEXT:    [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> poison, double [[TMP48]], i64 0
95; CHECK-NEXT:    [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> poison, <1 x i32> zeroinitializer
96; CHECK-NEXT:    [[TMP49:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
97; CHECK-NEXT:    [[BLOCK36:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
98; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
99; CHECK-NEXT:    [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> poison, double [[TMP50]], i64 0
100; CHECK-NEXT:    [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> poison, <1 x i32> zeroinitializer
101; CHECK-NEXT:    [[TMP51:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
102; CHECK-NEXT:    [[TMP52:%.*]] = fadd <1 x double> [[TMP49]], [[TMP51]]
103; CHECK-NEXT:    [[BLOCK39:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
104; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
105; CHECK-NEXT:    [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> poison, double [[TMP53]], i64 0
106; CHECK-NEXT:    [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> poison, <1 x i32> zeroinitializer
107; CHECK-NEXT:    [[TMP54:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
108; CHECK-NEXT:    [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]]
109; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
110; CHECK-NEXT:    [[TMP57:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP56]], <3 x i32> <i32 3, i32 1, i32 2>
111; CHECK-NEXT:    [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
112; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
113; CHECK-NEXT:    [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> poison, double [[TMP58]], i64 0
114; CHECK-NEXT:    [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> poison, <1 x i32> zeroinitializer
115; CHECK-NEXT:    [[TMP59:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]]
116; CHECK-NEXT:    [[BLOCK45:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
117; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
118; CHECK-NEXT:    [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> poison, double [[TMP60]], i64 0
119; CHECK-NEXT:    [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> poison, <1 x i32> zeroinitializer
120; CHECK-NEXT:    [[TMP61:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]]
121; CHECK-NEXT:    [[TMP62:%.*]] = fadd <1 x double> [[TMP59]], [[TMP61]]
122; CHECK-NEXT:    [[BLOCK48:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
123; CHECK-NEXT:    [[TMP63:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
124; CHECK-NEXT:    [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> poison, double [[TMP63]], i64 0
125; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> poison, <1 x i32> zeroinitializer
126; CHECK-NEXT:    [[TMP64:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
127; CHECK-NEXT:    [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]]
128; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
129; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> <i32 0, i32 3, i32 2>
130; CHECK-NEXT:    [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
131; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
132; CHECK-NEXT:    [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> poison, double [[TMP68]], i64 0
133; CHECK-NEXT:    [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> poison, <1 x i32> zeroinitializer
134; CHECK-NEXT:    [[TMP69:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]]
135; CHECK-NEXT:    [[BLOCK54:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
136; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
137; CHECK-NEXT:    [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> poison, double [[TMP70]], i64 0
138; CHECK-NEXT:    [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> poison, <1 x i32> zeroinitializer
139; CHECK-NEXT:    [[TMP71:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]]
140; CHECK-NEXT:    [[TMP72:%.*]] = fadd <1 x double> [[TMP69]], [[TMP71]]
141; CHECK-NEXT:    [[BLOCK57:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
142; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
143; CHECK-NEXT:    [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> poison, double [[TMP73]], i64 0
144; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> poison, <1 x i32> zeroinitializer
145; CHECK-NEXT:    [[TMP74:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
146; CHECK-NEXT:    [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]]
147; CHECK-NEXT:    [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
148; CHECK-NEXT:    [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> <i32 0, i32 1, i32 3>
149; CHECK-NEXT:    [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
150; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
151; CHECK-NEXT:    [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> poison, double [[TMP78]], i64 0
152; CHECK-NEXT:    [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> poison, <1 x i32> zeroinitializer
153; CHECK-NEXT:    [[TMP79:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
154; CHECK-NEXT:    [[BLOCK63:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
155; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
156; CHECK-NEXT:    [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> poison, double [[TMP80]], i64 0
157; CHECK-NEXT:    [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> poison, <1 x i32> zeroinitializer
158; CHECK-NEXT:    [[TMP81:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]]
159; CHECK-NEXT:    [[TMP82:%.*]] = fadd <1 x double> [[TMP79]], [[TMP81]]
160; CHECK-NEXT:    [[BLOCK66:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
161; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
162; CHECK-NEXT:    [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> poison, double [[TMP83]], i64 0
163; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> poison, <1 x i32> zeroinitializer
164; CHECK-NEXT:    [[TMP84:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
165; CHECK-NEXT:    [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]]
166; CHECK-NEXT:    [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
167; CHECK-NEXT:    [[TMP87:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP86]], <3 x i32> <i32 3, i32 1, i32 2>
168; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
169; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
170; CHECK-NEXT:    [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> poison, double [[TMP88]], i64 0
171; CHECK-NEXT:    [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> poison, <1 x i32> zeroinitializer
172; CHECK-NEXT:    [[TMP89:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]]
173; CHECK-NEXT:    [[BLOCK72:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
174; CHECK-NEXT:    [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
175; CHECK-NEXT:    [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> poison, double [[TMP90]], i64 0
176; CHECK-NEXT:    [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> poison, <1 x i32> zeroinitializer
177; CHECK-NEXT:    [[TMP91:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]]
178; CHECK-NEXT:    [[TMP92:%.*]] = fadd <1 x double> [[TMP89]], [[TMP91]]
179; CHECK-NEXT:    [[BLOCK75:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
180; CHECK-NEXT:    [[TMP93:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
181; CHECK-NEXT:    [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> poison, double [[TMP93]], i64 0
182; CHECK-NEXT:    [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> poison, <1 x i32> zeroinitializer
183; CHECK-NEXT:    [[TMP94:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
184; CHECK-NEXT:    [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]]
185; CHECK-NEXT:    [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
186; CHECK-NEXT:    [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> <i32 0, i32 3, i32 2>
187; CHECK-NEXT:    [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
188; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
189; CHECK-NEXT:    [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> poison, double [[TMP98]], i64 0
190; CHECK-NEXT:    [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> poison, <1 x i32> zeroinitializer
191; CHECK-NEXT:    [[TMP99:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
192; CHECK-NEXT:    [[BLOCK81:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
193; CHECK-NEXT:    [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
194; CHECK-NEXT:    [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> poison, double [[TMP100]], i64 0
195; CHECK-NEXT:    [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> poison, <1 x i32> zeroinitializer
196; CHECK-NEXT:    [[TMP101:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]]
197; CHECK-NEXT:    [[TMP102:%.*]] = fadd <1 x double> [[TMP99]], [[TMP101]]
198; CHECK-NEXT:    [[BLOCK84:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
199; CHECK-NEXT:    [[TMP103:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
200; CHECK-NEXT:    [[SPLAT_SPLATINSERT85:%.*]] = insertelement <1 x double> poison, double [[TMP103]], i64 0
201; CHECK-NEXT:    [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> poison, <1 x i32> zeroinitializer
202; CHECK-NEXT:    [[TMP104:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
203; CHECK-NEXT:    [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]]
204; CHECK-NEXT:    [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
205; CHECK-NEXT:    [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> <i32 0, i32 1, i32 3>
206; CHECK-NEXT:    store <3 x double> [[TMP47]], ptr [[C_PTR:%.*]], align 8
207; CHECK-NEXT:    [[VEC_GEP87:%.*]] = getelementptr double, ptr [[C_PTR]], i64 3
208; CHECK-NEXT:    store <3 x double> [[TMP77]], ptr [[VEC_GEP87]], align 8
209; CHECK-NEXT:    [[VEC_GEP88:%.*]] = getelementptr double, ptr [[C_PTR]], i64 6
210; CHECK-NEXT:    store <3 x double> [[TMP107]], ptr [[VEC_GEP88]], align 8
211; CHECK-NEXT:    ret void
212;
213
214; Load columns of input matrixes %A and %B.
215
216
217; Transpose %A.
218
219
220; Lower multiply(transpose(%A), %B)
221
222
223; Store result columns.
224
225
226entry:
227  %a = load <9 x double>, ptr %A.Ptr, align 8
228  %b = load <9 x double>, ptr %B.Ptr, align 8
229  %a.trans  = call <9 x double> @llvm.matrix.transpose(<9 x double> %a, i32 3, i32 3)
230  %c = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
231  store <9 x double> %c, ptr %C.Ptr, align 8
232  ret void
233}
234
235declare <9 x double> @llvm.matrix.transpose(<9 x double>, i32, i32)
236declare <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double>, <9 x double>, i32, i32, i32)
237
238define void @transpose_multiply_add(ptr %A.Ptr, ptr %B.Ptr, ptr %C.Ptr) {
239; CHECK-LABEL: @transpose_multiply_add(
240; CHECK-NEXT:  entry:
241; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[A_PTR:%.*]], align 8
242; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[A_PTR]], i64 3
243; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
244; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[A_PTR]], i64 6
245; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
246; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[B_PTR:%.*]], align 8
247; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr double, ptr [[B_PTR]], i64 3
248; CHECK-NEXT:    [[COL_LOAD6:%.*]] = load <3 x double>, ptr [[VEC_GEP5]], align 8
249; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr double, ptr [[B_PTR]], i64 6
250; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <3 x double>, ptr [[VEC_GEP7]], align 8
251; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 0
252; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x double> poison, double [[TMP0]], i64 0
253; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 0
254; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1
255; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 0
256; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2
257; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 1
258; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <3 x double> poison, double [[TMP6]], i64 0
259; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 1
260; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
261; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 1
262; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
263; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[COL_LOAD]], i64 2
264; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <3 x double> poison, double [[TMP12]], i64 0
265; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[COL_LOAD1]], i64 2
266; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1
267; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[COL_LOAD3]], i64 2
268; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
269; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
270; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
271; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP18]], i64 0
272; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
273; CHECK-NEXT:    [[TMP19:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
274; CHECK-NEXT:    [[BLOCK9:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
275; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
276; CHECK-NEXT:    [[SPLAT_SPLATINSERT10:%.*]] = insertelement <1 x double> poison, double [[TMP20]], i64 0
277; CHECK-NEXT:    [[SPLAT_SPLAT11:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT10]], <1 x double> poison, <1 x i32> zeroinitializer
278; CHECK-NEXT:    [[TMP21:%.*]] = fmul <1 x double> [[BLOCK9]], [[SPLAT_SPLAT11]]
279; CHECK-NEXT:    [[TMP22:%.*]] = fadd <1 x double> [[TMP19]], [[TMP21]]
280; CHECK-NEXT:    [[BLOCK12:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
281; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
282; CHECK-NEXT:    [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> poison, double [[TMP23]], i64 0
283; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
284; CHECK-NEXT:    [[TMP24:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
285; CHECK-NEXT:    [[TMP25:%.*]] = fadd <1 x double> [[TMP22]], [[TMP24]]
286; CHECK-NEXT:    [[TMP26:%.*]] = shufflevector <1 x double> [[TMP25]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
287; CHECK-NEXT:    [[TMP27:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP26]], <3 x i32> <i32 3, i32 1, i32 2>
288; CHECK-NEXT:    [[BLOCK15:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
289; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
290; CHECK-NEXT:    [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> poison, double [[TMP28]], i64 0
291; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> poison, <1 x i32> zeroinitializer
292; CHECK-NEXT:    [[TMP29:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
293; CHECK-NEXT:    [[BLOCK18:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
294; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
295; CHECK-NEXT:    [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> poison, double [[TMP30]], i64 0
296; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> poison, <1 x i32> zeroinitializer
297; CHECK-NEXT:    [[TMP31:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
298; CHECK-NEXT:    [[TMP32:%.*]] = fadd <1 x double> [[TMP29]], [[TMP31]]
299; CHECK-NEXT:    [[BLOCK21:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
300; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
301; CHECK-NEXT:    [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> poison, double [[TMP33]], i64 0
302; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
303; CHECK-NEXT:    [[TMP34:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
304; CHECK-NEXT:    [[TMP35:%.*]] = fadd <1 x double> [[TMP32]], [[TMP34]]
305; CHECK-NEXT:    [[TMP36:%.*]] = shufflevector <1 x double> [[TMP35]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
306; CHECK-NEXT:    [[TMP37:%.*]] = shufflevector <3 x double> [[TMP27]], <3 x double> [[TMP36]], <3 x i32> <i32 0, i32 3, i32 2>
307; CHECK-NEXT:    [[BLOCK24:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
308; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 0
309; CHECK-NEXT:    [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> poison, double [[TMP38]], i64 0
310; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> poison, <1 x i32> zeroinitializer
311; CHECK-NEXT:    [[TMP39:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
312; CHECK-NEXT:    [[BLOCK27:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
313; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 1
314; CHECK-NEXT:    [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> poison, double [[TMP40]], i64 0
315; CHECK-NEXT:    [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> poison, <1 x i32> zeroinitializer
316; CHECK-NEXT:    [[TMP41:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
317; CHECK-NEXT:    [[TMP42:%.*]] = fadd <1 x double> [[TMP39]], [[TMP41]]
318; CHECK-NEXT:    [[BLOCK30:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
319; CHECK-NEXT:    [[TMP43:%.*]] = extractelement <3 x double> [[COL_LOAD4]], i64 2
320; CHECK-NEXT:    [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> poison, double [[TMP43]], i64 0
321; CHECK-NEXT:    [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
322; CHECK-NEXT:    [[TMP44:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
323; CHECK-NEXT:    [[TMP45:%.*]] = fadd <1 x double> [[TMP42]], [[TMP44]]
324; CHECK-NEXT:    [[TMP46:%.*]] = shufflevector <1 x double> [[TMP45]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
325; CHECK-NEXT:    [[TMP47:%.*]] = shufflevector <3 x double> [[TMP37]], <3 x double> [[TMP46]], <3 x i32> <i32 0, i32 1, i32 3>
326; CHECK-NEXT:    [[BLOCK33:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
327; CHECK-NEXT:    [[TMP48:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
328; CHECK-NEXT:    [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> poison, double [[TMP48]], i64 0
329; CHECK-NEXT:    [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> poison, <1 x i32> zeroinitializer
330; CHECK-NEXT:    [[TMP49:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
331; CHECK-NEXT:    [[BLOCK36:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
332; CHECK-NEXT:    [[TMP50:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
333; CHECK-NEXT:    [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> poison, double [[TMP50]], i64 0
334; CHECK-NEXT:    [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> poison, <1 x i32> zeroinitializer
335; CHECK-NEXT:    [[TMP51:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
336; CHECK-NEXT:    [[TMP52:%.*]] = fadd <1 x double> [[TMP49]], [[TMP51]]
337; CHECK-NEXT:    [[BLOCK39:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
338; CHECK-NEXT:    [[TMP53:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
339; CHECK-NEXT:    [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> poison, double [[TMP53]], i64 0
340; CHECK-NEXT:    [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> poison, <1 x i32> zeroinitializer
341; CHECK-NEXT:    [[TMP54:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
342; CHECK-NEXT:    [[TMP55:%.*]] = fadd <1 x double> [[TMP52]], [[TMP54]]
343; CHECK-NEXT:    [[TMP56:%.*]] = shufflevector <1 x double> [[TMP55]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
344; CHECK-NEXT:    [[TMP57:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP56]], <3 x i32> <i32 3, i32 1, i32 2>
345; CHECK-NEXT:    [[BLOCK42:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
346; CHECK-NEXT:    [[TMP58:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
347; CHECK-NEXT:    [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> poison, double [[TMP58]], i64 0
348; CHECK-NEXT:    [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> poison, <1 x i32> zeroinitializer
349; CHECK-NEXT:    [[TMP59:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]]
350; CHECK-NEXT:    [[BLOCK45:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
351; CHECK-NEXT:    [[TMP60:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
352; CHECK-NEXT:    [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> poison, double [[TMP60]], i64 0
353; CHECK-NEXT:    [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> poison, <1 x i32> zeroinitializer
354; CHECK-NEXT:    [[TMP61:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]]
355; CHECK-NEXT:    [[TMP62:%.*]] = fadd <1 x double> [[TMP59]], [[TMP61]]
356; CHECK-NEXT:    [[BLOCK48:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
357; CHECK-NEXT:    [[TMP63:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
358; CHECK-NEXT:    [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> poison, double [[TMP63]], i64 0
359; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> poison, <1 x i32> zeroinitializer
360; CHECK-NEXT:    [[TMP64:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
361; CHECK-NEXT:    [[TMP65:%.*]] = fadd <1 x double> [[TMP62]], [[TMP64]]
362; CHECK-NEXT:    [[TMP66:%.*]] = shufflevector <1 x double> [[TMP65]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
363; CHECK-NEXT:    [[TMP67:%.*]] = shufflevector <3 x double> [[TMP57]], <3 x double> [[TMP66]], <3 x i32> <i32 0, i32 3, i32 2>
364; CHECK-NEXT:    [[BLOCK51:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
365; CHECK-NEXT:    [[TMP68:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 0
366; CHECK-NEXT:    [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> poison, double [[TMP68]], i64 0
367; CHECK-NEXT:    [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> poison, <1 x i32> zeroinitializer
368; CHECK-NEXT:    [[TMP69:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]]
369; CHECK-NEXT:    [[BLOCK54:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
370; CHECK-NEXT:    [[TMP70:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 1
371; CHECK-NEXT:    [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> poison, double [[TMP70]], i64 0
372; CHECK-NEXT:    [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> poison, <1 x i32> zeroinitializer
373; CHECK-NEXT:    [[TMP71:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]]
374; CHECK-NEXT:    [[TMP72:%.*]] = fadd <1 x double> [[TMP69]], [[TMP71]]
375; CHECK-NEXT:    [[BLOCK57:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
376; CHECK-NEXT:    [[TMP73:%.*]] = extractelement <3 x double> [[COL_LOAD6]], i64 2
377; CHECK-NEXT:    [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> poison, double [[TMP73]], i64 0
378; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> poison, <1 x i32> zeroinitializer
379; CHECK-NEXT:    [[TMP74:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
380; CHECK-NEXT:    [[TMP75:%.*]] = fadd <1 x double> [[TMP72]], [[TMP74]]
381; CHECK-NEXT:    [[TMP76:%.*]] = shufflevector <1 x double> [[TMP75]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
382; CHECK-NEXT:    [[TMP77:%.*]] = shufflevector <3 x double> [[TMP67]], <3 x double> [[TMP76]], <3 x i32> <i32 0, i32 1, i32 3>
383; CHECK-NEXT:    [[BLOCK60:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> zeroinitializer
384; CHECK-NEXT:    [[TMP78:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
385; CHECK-NEXT:    [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> poison, double [[TMP78]], i64 0
386; CHECK-NEXT:    [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> poison, <1 x i32> zeroinitializer
387; CHECK-NEXT:    [[TMP79:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
388; CHECK-NEXT:    [[BLOCK63:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> zeroinitializer
389; CHECK-NEXT:    [[TMP80:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
390; CHECK-NEXT:    [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> poison, double [[TMP80]], i64 0
391; CHECK-NEXT:    [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> poison, <1 x i32> zeroinitializer
392; CHECK-NEXT:    [[TMP81:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]]
393; CHECK-NEXT:    [[TMP82:%.*]] = fadd <1 x double> [[TMP79]], [[TMP81]]
394; CHECK-NEXT:    [[BLOCK66:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> zeroinitializer
395; CHECK-NEXT:    [[TMP83:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
396; CHECK-NEXT:    [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> poison, double [[TMP83]], i64 0
397; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> poison, <1 x i32> zeroinitializer
398; CHECK-NEXT:    [[TMP84:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
399; CHECK-NEXT:    [[TMP85:%.*]] = fadd <1 x double> [[TMP82]], [[TMP84]]
400; CHECK-NEXT:    [[TMP86:%.*]] = shufflevector <1 x double> [[TMP85]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
401; CHECK-NEXT:    [[TMP87:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP86]], <3 x i32> <i32 3, i32 1, i32 2>
402; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 1>
403; CHECK-NEXT:    [[TMP88:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
404; CHECK-NEXT:    [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> poison, double [[TMP88]], i64 0
405; CHECK-NEXT:    [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> poison, <1 x i32> zeroinitializer
406; CHECK-NEXT:    [[TMP89:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]]
407; CHECK-NEXT:    [[BLOCK72:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 1>
408; CHECK-NEXT:    [[TMP90:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
409; CHECK-NEXT:    [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> poison, double [[TMP90]], i64 0
410; CHECK-NEXT:    [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> poison, <1 x i32> zeroinitializer
411; CHECK-NEXT:    [[TMP91:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]]
412; CHECK-NEXT:    [[TMP92:%.*]] = fadd <1 x double> [[TMP89]], [[TMP91]]
413; CHECK-NEXT:    [[BLOCK75:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 1>
414; CHECK-NEXT:    [[TMP93:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
415; CHECK-NEXT:    [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> poison, double [[TMP93]], i64 0
416; CHECK-NEXT:    [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> poison, <1 x i32> zeroinitializer
417; CHECK-NEXT:    [[TMP94:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
418; CHECK-NEXT:    [[TMP95:%.*]] = fadd <1 x double> [[TMP92]], [[TMP94]]
419; CHECK-NEXT:    [[TMP96:%.*]] = shufflevector <1 x double> [[TMP95]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
420; CHECK-NEXT:    [[TMP97:%.*]] = shufflevector <3 x double> [[TMP87]], <3 x double> [[TMP96]], <3 x i32> <i32 0, i32 3, i32 2>
421; CHECK-NEXT:    [[BLOCK78:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> poison, <1 x i32> <i32 2>
422; CHECK-NEXT:    [[TMP98:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 0
423; CHECK-NEXT:    [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> poison, double [[TMP98]], i64 0
424; CHECK-NEXT:    [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> poison, <1 x i32> zeroinitializer
425; CHECK-NEXT:    [[TMP99:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
426; CHECK-NEXT:    [[BLOCK81:%.*]] = shufflevector <3 x double> [[TMP11]], <3 x double> poison, <1 x i32> <i32 2>
427; CHECK-NEXT:    [[TMP100:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 1
428; CHECK-NEXT:    [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> poison, double [[TMP100]], i64 0
429; CHECK-NEXT:    [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> poison, <1 x i32> zeroinitializer
430; CHECK-NEXT:    [[TMP101:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]]
431; CHECK-NEXT:    [[TMP102:%.*]] = fadd <1 x double> [[TMP99]], [[TMP101]]
432; CHECK-NEXT:    [[BLOCK84:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <1 x i32> <i32 2>
433; CHECK-NEXT:    [[TMP103:%.*]] = extractelement <3 x double> [[COL_LOAD8]], i64 2
434; CHECK-NEXT:    [[SPLAT_SPLATINSERT85:%.*]] = insertelement <1 x double> poison, double [[TMP103]], i64 0
435; CHECK-NEXT:    [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> poison, <1 x i32> zeroinitializer
436; CHECK-NEXT:    [[TMP104:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
437; CHECK-NEXT:    [[TMP105:%.*]] = fadd <1 x double> [[TMP102]], [[TMP104]]
438; CHECK-NEXT:    [[TMP106:%.*]] = shufflevector <1 x double> [[TMP105]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
439; CHECK-NEXT:    [[TMP107:%.*]] = shufflevector <3 x double> [[TMP97]], <3 x double> [[TMP106]], <3 x i32> <i32 0, i32 1, i32 3>
440; CHECK-NEXT:    [[COL_LOAD87:%.*]] = load <3 x double>, ptr [[C_PTR:%.*]], align 8
441; CHECK-NEXT:    [[VEC_GEP88:%.*]] = getelementptr double, ptr [[C_PTR]], i64 3
442; CHECK-NEXT:    [[COL_LOAD89:%.*]] = load <3 x double>, ptr [[VEC_GEP88]], align 8
443; CHECK-NEXT:    [[VEC_GEP90:%.*]] = getelementptr double, ptr [[C_PTR]], i64 6
444; CHECK-NEXT:    [[COL_LOAD91:%.*]] = load <3 x double>, ptr [[VEC_GEP90]], align 8
445; CHECK-NEXT:    [[TMP108:%.*]] = fadd <3 x double> [[COL_LOAD87]], [[TMP47]]
446; CHECK-NEXT:    [[TMP109:%.*]] = fadd <3 x double> [[COL_LOAD89]], [[TMP77]]
447; CHECK-NEXT:    [[TMP110:%.*]] = fadd <3 x double> [[COL_LOAD91]], [[TMP107]]
448; CHECK-NEXT:    store <3 x double> [[TMP108]], ptr [[C_PTR]], align 8
449; CHECK-NEXT:    [[VEC_GEP92:%.*]] = getelementptr double, ptr [[C_PTR]], i64 3
450; CHECK-NEXT:    store <3 x double> [[TMP109]], ptr [[VEC_GEP92]], align 8
451; CHECK-NEXT:    [[VEC_GEP93:%.*]] = getelementptr double, ptr [[C_PTR]], i64 6
452; CHECK-NEXT:    store <3 x double> [[TMP110]], ptr [[VEC_GEP93]], align 8
453; CHECK-NEXT:    ret void
454;
455
456
457; Transpose %A.
458
459
460; Lower multiply(transpose(%A), %B)
461
462
463;  Embed result of multiply into flat vector.
464
465
466; Load %C.
467
468
469; Add column vectors.
470
471
472; Store result columns.
473
474
475entry:
476  %a = load <9 x double>, ptr %A.Ptr, align 8
477  %b = load <9 x double>, ptr %B.Ptr, align 8
478  %a.trans  = call <9 x double> @llvm.matrix.transpose(<9 x double> %a, i32 3, i32 3)
479  %mult = call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %a.trans, <9 x double> %b, i32 3, i32 3, i32 3)
480  %c = load <9 x double>, ptr %C.Ptr, align 8
481  %res = fadd <9 x double> %c, %mult
482
483  store <9 x double> %res, ptr %C.Ptr, align 8
484  ret void
485}
486