1; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2 2; Check memory cost model action for fixed vector SVE and Neon 3; Vector bits size lower than 256 bits end up assuming Neon cost model 4; CHECK-NEON has same performance as CHECK-SVE-128 5 6; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+neon < %s | FileCheck %s --check-prefix=CHECK-NEON 7; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=128 < %s | FileCheck %s --check-prefix=CHECK-SVE-128 8; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s --check-prefix=CHECK-SVE-256 9; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=aarch64--linux-gnu -mattr=+sve -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s --check-prefix=CHECK-SVE-512 10 11target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" 12 13define <16 x i8> @load16(ptr %ptr) { 14; CHECK: function 'load16' 15; CHECK-NEON-LABEL: 'load16' 16; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i8>, ptr %ptr, align 16 17; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %out 18; 19; CHECK-SVE-128-LABEL: 'load16' 20; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i8>, ptr %ptr, align 16 21; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %out 22; 23; CHECK-SVE-256-LABEL: 'load16' 24; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i8>, ptr %ptr, align 16 25; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %out 26; 27; CHECK-SVE-512-LABEL: 'load16' 28; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i8>, ptr %ptr, align 16 29; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i8> %out 30; 31 %out = load <16 x i8>, ptr %ptr 32 ret <16 x i8> %out 33} 34 35define void @store16(ptr %ptr, <16 x i8> %val) { 36; CHECK: function 'store16' 37; CHECK-NEON-LABEL: 'store16' 38; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %val, ptr %ptr, align 16 39; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 40; 41; CHECK-SVE-128-LABEL: 'store16' 42; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %val, ptr %ptr, align 16 43; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 44; 45; CHECK-SVE-256-LABEL: 'store16' 46; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %val, ptr %ptr, align 16 47; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 48; 49; CHECK-SVE-512-LABEL: 'store16' 50; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <16 x i8> %val, ptr %ptr, align 16 51; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 52; 53 store <16 x i8> %val, ptr %ptr 54 ret void 55} 56 57define <8 x i8> @load8(ptr %ptr) { 58; CHECK: function 'load8' 59; CHECK-NEON-LABEL: 'load8' 60; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i8>, ptr %ptr, align 8 61; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %out 62; 63; CHECK-SVE-128-LABEL: 'load8' 64; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i8>, ptr %ptr, align 8 65; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %out 66; 67; CHECK-SVE-256-LABEL: 'load8' 68; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i8>, ptr %ptr, align 8 69; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %out 70; 71; CHECK-SVE-512-LABEL: 'load8' 72; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i8>, ptr %ptr, align 8 73; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i8> %out 74; 75 %out = load <8 x i8>, ptr %ptr 76 ret <8 x i8> %out 77} 78 79define void @store8(ptr %ptr, <8 x i8> %val) { 80; CHECK: function 'store8' 81; CHECK-NEON-LABEL: 'store8' 82; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %val, ptr %ptr, align 8 83; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 84; 85; CHECK-SVE-128-LABEL: 'store8' 86; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %val, ptr %ptr, align 8 87; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 88; 89; CHECK-SVE-256-LABEL: 'store8' 90; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %val, ptr %ptr, align 8 91; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 92; 93; CHECK-SVE-512-LABEL: 'store8' 94; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <8 x i8> %val, ptr %ptr, align 8 95; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 96; 97 store <8 x i8> %val, ptr %ptr 98 ret void 99} 100 101define <4 x i8> @load4(ptr %ptr) { 102; CHECK: function 'load4' 103; CHECK-NEON-LABEL: 'load4' 104; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %out = load <4 x i8>, ptr %ptr, align 4 105; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %out 106; 107; CHECK-SVE-128-LABEL: 'load4' 108; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %out = load <4 x i8>, ptr %ptr, align 4 109; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %out 110; 111; CHECK-SVE-256-LABEL: 'load4' 112; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <4 x i8>, ptr %ptr, align 4 113; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %out 114; 115; CHECK-SVE-512-LABEL: 'load4' 116; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <4 x i8>, ptr %ptr, align 4 117; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %out 118; 119 %out = load <4 x i8>, ptr %ptr 120 ret <4 x i8> %out 121} 122 123define void @store4(ptr %ptr, <4 x i8> %val) { 124; CHECK: function 'store4' 125; CHECK-NEON-LABEL: 'store4' 126; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %val, ptr %ptr, align 4 127; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 128; 129; CHECK-SVE-128-LABEL: 'store4' 130; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i8> %val, ptr %ptr, align 4 131; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 132; 133; CHECK-SVE-256-LABEL: 'store4' 134; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %val, ptr %ptr, align 4 135; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 136; 137; CHECK-SVE-512-LABEL: 'store4' 138; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> %val, ptr %ptr, align 4 139; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 140; 141 store <4 x i8> %val, ptr %ptr 142 ret void 143} 144 145define <16 x i16> @load_256(ptr %ptr) { 146; CHECK: function 'load_256' 147; CHECK-NEON-LABEL: 'load_256' 148; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %out = load <16 x i16>, ptr %ptr, align 32 149; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %out 150; 151; CHECK-SVE-128-LABEL: 'load_256' 152; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %out = load <16 x i16>, ptr %ptr, align 32 153; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %out 154; 155; CHECK-SVE-256-LABEL: 'load_256' 156; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i16>, ptr %ptr, align 32 157; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %out 158; 159; CHECK-SVE-512-LABEL: 'load_256' 160; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <16 x i16>, ptr %ptr, align 32 161; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <16 x i16> %out 162; 163 %out = load <16 x i16>, ptr %ptr 164 ret <16 x i16> %out 165} 166 167define <8 x i64> @load_512(ptr %ptr) { 168; CHECK: function 'load_512' 169; CHECK-NEON-LABEL: 'load_512' 170; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %out = load <8 x i64>, ptr %ptr, align 64 171; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %out 172; 173; CHECK-SVE-128-LABEL: 'load_512' 174; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %out = load <8 x i64>, ptr %ptr, align 64 175; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %out 176; 177; CHECK-SVE-256-LABEL: 'load_512' 178; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %out = load <8 x i64>, ptr %ptr, align 64 179; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %out 180; 181; CHECK-SVE-512-LABEL: 'load_512' 182; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %out = load <8 x i64>, ptr %ptr, align 64 183; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <8 x i64> %out 184; 185 %out = load <8 x i64>, ptr %ptr 186 ret <8 x i64> %out 187} 188 189declare <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr>, i32 immarg, <4 x i1>, <4 x i8>) 190define <4 x i8> @gather_load_4xi8_constant_mask(<4 x ptr> %ptrs) { 191; CHECK: gather_load_4xi8_constant_mask 192; CHECK-NEON-LABEL: 'gather_load_4xi8_constant_mask' 193; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i8> undef) 194; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv 195; 196; CHECK-SVE-128-LABEL: 'gather_load_4xi8_constant_mask' 197; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i8> undef) 198; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv 199; 200; CHECK-SVE-256-LABEL: 'gather_load_4xi8_constant_mask' 201; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i8> undef) 202; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv 203; 204; CHECK-SVE-512-LABEL: 'gather_load_4xi8_constant_mask' 205; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i8> undef) 206; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv 207; 208 %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i8> undef) 209 ret <4 x i8> %lv 210} 211 212define <4 x i8> @gather_load_4xi8_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) { 213; CHECK: gather_load_4xi8_variable_mask 214; CHECK-NEON-LABEL: 'gather_load_4xi8_variable_mask' 215; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef) 216; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv 217; 218; CHECK-SVE-128-LABEL: 'gather_load_4xi8_variable_mask' 219; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef) 220; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv 221; 222; CHECK-SVE-256-LABEL: 'gather_load_4xi8_variable_mask' 223; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef) 224; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv 225; 226; CHECK-SVE-512-LABEL: 'gather_load_4xi8_variable_mask' 227; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef) 228; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i8> %lv 229; 230 %lv = call <4 x i8> @llvm.masked.gather.v4i8.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i8> undef) 231 ret <4 x i8> %lv 232} 233 234declare void @llvm.masked.scatter.v4i8.v4p0(<4 x i8>, <4 x ptr>, i32 immarg, <4 x i1>) 235define void @scatter_store_4xi8_constant_mask(<4 x i8> %val, <4 x ptr> %ptrs) { 236; CHECK: scatter_store_4xi8_constant_mask 237; CHECK-NEON-LABEL: 'scatter_store_4xi8_constant_mask' 238; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true)) 239; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 240; 241; CHECK-SVE-128-LABEL: 'scatter_store_4xi8_constant_mask' 242; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true)) 243; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 244; 245; CHECK-SVE-256-LABEL: 'scatter_store_4xi8_constant_mask' 246; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true)) 247; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 248; 249; CHECK-SVE-512-LABEL: 'scatter_store_4xi8_constant_mask' 250; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true)) 251; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 252; 253 call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>) 254 ret void 255} 256 257define void @scatter_store_4xi8_variable_mask(<4 x i8> %val, <4 x ptr> %ptrs, <4 x i1> %cond) { 258; CHECK: scatter_store_4xi8_variable_mask 259; CHECK-NEON-LABEL: 'scatter_store_4xi8_variable_mask' 260; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 261; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 262; 263; CHECK-SVE-128-LABEL: 'scatter_store_4xi8_variable_mask' 264; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 265; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 266; 267; CHECK-SVE-256-LABEL: 'scatter_store_4xi8_variable_mask' 268; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 269; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 270; 271; CHECK-SVE-512-LABEL: 'scatter_store_4xi8_variable_mask' 272; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 273; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 274; 275 call void @llvm.masked.scatter.v4i8.v4p0(<4 x i8> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 276 ret void 277} 278 279declare <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr>, i32 immarg, <4 x i1>, <4 x i32>) 280define <4 x i32> @gather_load_4xi32_constant_mask(<4 x ptr> %ptrs) { 281; CHECK: gather_load_4xi32_constant_mask 282; CHECK-NEON-LABEL: 'gather_load_4xi32_constant_mask' 283; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i32> undef) 284; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv 285; 286; CHECK-SVE-128-LABEL: 'gather_load_4xi32_constant_mask' 287; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 20 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i32> undef) 288; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv 289; 290; CHECK-SVE-256-LABEL: 'gather_load_4xi32_constant_mask' 291; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i32> undef) 292; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv 293; 294; CHECK-SVE-512-LABEL: 'gather_load_4xi32_constant_mask' 295; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true), <4 x i32> undef) 296; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv 297; 298 %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>, <4 x i32> undef) 299 ret <4 x i32> %lv 300} 301 302define <4 x i32> @gather_load_4xi32_variable_mask(<4 x ptr> %ptrs, <4 x i1> %cond) { 303; CHECK: gather_load_4xi32_variable_mask 304; CHECK-NEON-LABEL: 'gather_load_4xi32_variable_mask' 305; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef) 306; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv 307; 308; CHECK-SVE-128-LABEL: 'gather_load_4xi32_variable_mask' 309; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 28 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef) 310; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv 311; 312; CHECK-SVE-256-LABEL: 'gather_load_4xi32_variable_mask' 313; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef) 314; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv 315; 316; CHECK-SVE-512-LABEL: 'gather_load_4xi32_variable_mask' 317; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 40 for instruction: %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef) 318; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret <4 x i32> %lv 319; 320 %lv = call <4 x i32> @llvm.masked.gather.v4i32.v4p0(<4 x ptr> %ptrs, i32 1, <4 x i1> %cond, <4 x i32> undef) 321 ret <4 x i32> %lv 322} 323 324declare void @llvm.masked.scatter.v4i32.v4p0(<4 x i32>, <4 x ptr>, i32 immarg, <4 x i1>) 325define void @scatter_store_4xi32_constant_mask(<4 x i32> %val, <4 x ptr> %ptrs) { 326; CHECK: scatter_store_4xi32_constant_mask 327; CHECK-NEON-LABEL: 'scatter_store_4xi32_constant_mask' 328; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true)) 329; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 330; 331; CHECK-SVE-128-LABEL: 'scatter_store_4xi32_constant_mask' 332; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 20 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true)) 333; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 334; 335; CHECK-SVE-256-LABEL: 'scatter_store_4xi32_constant_mask' 336; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true)) 337; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 338; 339; CHECK-SVE-512-LABEL: 'scatter_store_4xi32_constant_mask' 340; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> splat (i1 true)) 341; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 342; 343 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> <i1 true, i1 true, i1 true, i1 true>) 344 ret void 345} 346 347define void @scatter_store_4xi32_variable_mask(<4 x i32> %val, <4 x ptr> %ptrs, <4 x i1> %cond) { 348; CHECK: scatter_store_4xi32_variable_mask 349; CHECK-NEON-LABEL: 'scatter_store_4xi32_variable_mask' 350; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 351; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 352; 353; CHECK-SVE-128-LABEL: 'scatter_store_4xi32_variable_mask' 354; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 28 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 355; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 356; 357; CHECK-SVE-256-LABEL: 'scatter_store_4xi32_variable_mask' 358; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 359; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 360; 361; CHECK-SVE-512-LABEL: 'scatter_store_4xi32_variable_mask' 362; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 40 for instruction: call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 363; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 364; 365 call void @llvm.masked.scatter.v4i32.v4p0(<4 x i32> %val, <4 x ptr> %ptrs, i32 1, <4 x i1> %cond) 366 ret void 367} 368 369declare <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr>, i32, <256 x i1>, <256 x i16>) 370define void @sve_gather_vls(<256 x i1> %v256i1mask) { 371; CHECK-LABEL: 'sve_scatter_vls' 372; CHECK-NEON-LABEL: 'sve_gather_vls' 373; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 1792 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer) 374; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 375; 376; CHECK-SVE-128-LABEL: 'sve_gather_vls' 377; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 1792 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer) 378; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 379; 380; CHECK-SVE-256-LABEL: 'sve_gather_vls' 381; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 2560 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer) 382; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 383; 384; CHECK-SVE-512-LABEL: 'sve_gather_vls' 385; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 2560 for instruction: %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer) 386; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 387; 388entry: 389 %res.v256i16 = call <256 x i16> @llvm.masked.gather.v256i16.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x i16> zeroinitializer) 390 ret void 391} 392 393declare <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr>, i32, <256 x i1>, <256 x float>) 394define void @sve_gather_vls_float(<256 x i1> %v256i1mask) { 395; CHECK-LABEL: 'sve_gather_vls_float' 396; CHECK-NEON-LABEL: 'sve_gather_vls_float' 397; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 1664 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer) 398; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 399; 400; CHECK-SVE-128-LABEL: 'sve_gather_vls_float' 401; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 1664 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer) 402; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 403; 404; CHECK-SVE-256-LABEL: 'sve_gather_vls_float' 405; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 2560 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer) 406; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 407; 408; CHECK-SVE-512-LABEL: 'sve_gather_vls_float' 409; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 2560 for instruction: %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer) 410; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 411; 412entry: 413 %res.v256f32 = call <256 x float> @llvm.masked.gather.v256f32.v256p0(<256 x ptr> undef, i32 0, <256 x i1> %v256i1mask, <256 x float> zeroinitializer) 414 ret void 415} 416 417declare void @llvm.masked.scatter.v256i8.v256p0(<256 x i8>, <256 x ptr>, i32, <256 x i1>) 418define void @sve_scatter_vls(<256 x i1> %v256i1mask){ 419; CHECK-LABEL: 'sve_scatter_vls' 420; CHECK-NEON-LABEL: 'sve_scatter_vls' 421; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 1792 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask) 422; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 423; 424; CHECK-SVE-128-LABEL: 'sve_scatter_vls' 425; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 1792 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask) 426; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 427; 428; CHECK-SVE-256-LABEL: 'sve_scatter_vls' 429; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 2560 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask) 430; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 431; 432; CHECK-SVE-512-LABEL: 'sve_scatter_vls' 433; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 2560 for instruction: call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask) 434; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 435; 436entry: 437 call void @llvm.masked.scatter.v256i8.v256p0(<256 x i8> undef, <256 x ptr> undef, i32 0, <256 x i1> %v256i1mask) 438 ret void 439} 440 441declare void @llvm.masked.scatter.v512f16.v512p0(<512 x half>, <512 x ptr>, i32, <512 x i1>) 442define void @sve_scatter_vls_float(<512 x i1> %v512i1mask){ 443; CHECK-LABEL: 'sve_scatter_vls_float' 444; CHECK-NEON-LABEL: 'sve_scatter_vls_float' 445; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 3456 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask) 446; CHECK-NEON-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 447; 448; CHECK-SVE-128-LABEL: 'sve_scatter_vls_float' 449; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 3456 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask) 450; CHECK-SVE-128-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 451; 452; CHECK-SVE-256-LABEL: 'sve_scatter_vls_float' 453; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 5120 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask) 454; CHECK-SVE-256-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 455; 456; CHECK-SVE-512-LABEL: 'sve_scatter_vls_float' 457; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 5120 for instruction: call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask) 458; CHECK-SVE-512-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void 459; 460 call void @llvm.masked.scatter.v512f16.v512p0(<512 x half> undef, <512 x ptr> undef, i32 0, <512 x i1> %v512i1mask) 461 ret void 462} 463