1//===-- IR/VPIntrinsics.def - Describes llvm.vp.* Intrinsics -*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file contains descriptions of the various Vector Predication intrinsics. 10// This is used as a central place for enumerating the different instructions 11// and should eventually be the place to put comments about the instructions. 12// 13//===----------------------------------------------------------------------===// 14 15// NOTE: NO INCLUDE GUARD DESIRED! 16 17// Provide definitions of macros so that users of this file do not have to 18// define everything to use it... 19// 20// Register a VP intrinsic and begin its property scope. 21// All VP intrinsic scopes are top level, ie it is illegal to place a 22// BEGIN_REGISTER_VP_INTRINSIC within a VP intrinsic scope. 23// \p VPID The VP intrinsic id. 24// \p MASKPOS The mask operand position. 25// \p EVLPOS The explicit vector length operand position. 26#ifndef BEGIN_REGISTER_VP_INTRINSIC 27#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) 28#endif 29 30// End the property scope of a VP intrinsic. 31#ifndef END_REGISTER_VP_INTRINSIC 32#define END_REGISTER_VP_INTRINSIC(VPID) 33#endif 34 35// Register a new VP SDNode and begin its property scope. 36// When the SDNode scope is nested within a VP intrinsic scope, it is 37// implicitly registered as the canonical SDNode for this VP intrinsic. There 38// is one VP intrinsic that maps directly to one SDNode that goes by the 39// same name. Since the operands are also the same, we open the property 40// scopes for both the VPIntrinsic and the SDNode at once. 41// \p VPSD The SelectionDAG Node id (eg VP_ADD). 42// \p LEGALPOS The operand position of the SDNode that is used for legalizing. 43// If LEGALPOS < 0, then the return type given by 44// TheNode->getValueType(-1-LEGALPOS) is used. 45// \p TDNAME The name of the TableGen definition of this SDNode. 46// \p MASKPOS The mask operand position. 47// \p EVLPOS The explicit vector length operand position. 48#ifndef BEGIN_REGISTER_VP_SDNODE 49#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) 50#endif 51 52// End the property scope of a new VP SDNode. 53#ifndef END_REGISTER_VP_SDNODE 54#define END_REGISTER_VP_SDNODE(VPSD) 55#endif 56 57// Helper macro to set up the mapping from VP intrinsic to ISD opcode. 58// Note: More than one VP intrinsic may map to one ISD opcode. 59#ifndef HELPER_MAP_VPID_TO_VPSD 60#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) 61#endif 62 63// Helper macros for the common "1:1 - Intrinsic : SDNode" case. 64// 65// There is one VP intrinsic that maps directly to one SDNode that goes by the 66// same name. Since the operands are also the same, we open the property 67// scopes for both the VPIntrinsic and the SDNode at once. 68// 69// \p VPID The canonical name (eg `vp_add`, which at the same time is the 70// name of the intrinsic and the TableGen def of the SDNode). 71// \p MASKPOS The mask operand position. 72// \p EVLPOS The explicit vector length operand position. 73// \p VPSD The SelectionDAG Node id (eg VP_ADD). 74// \p LEGALPOS The operand position of the SDNode that is used for legalizing 75// this SDNode. This can be `-1`, in which case the return type of 76// the SDNode is used. 77#define BEGIN_REGISTER_VP(VPID, MASKPOS, EVLPOS, VPSD, LEGALPOS) \ 78 BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) \ 79 BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, VPID, MASKPOS, EVLPOS) \ 80 HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) 81 82#define END_REGISTER_VP(VPID, VPSD) \ 83 END_REGISTER_VP_INTRINSIC(VPID) \ 84 END_REGISTER_VP_SDNODE(VPSD) 85 86// The following macros attach properties to the scope they are placed in. This 87// assigns the property to the VP Intrinsic and/or SDNode that belongs to the 88// scope. 89// 90// Property Macros { 91 92// The intrinsic and/or SDNode has the same function as this LLVM IR Opcode. 93// \p OPC The opcode of the instruction with the same function. 94#ifndef VP_PROPERTY_FUNCTIONAL_OPC 95#define VP_PROPERTY_FUNCTIONAL_OPC(OPC) 96#endif 97 98// Whether the intrinsic may have a rounding mode or exception behavior operand 99// bundle. 100// \p HASROUND '1' if the intrinsic can have a rounding mode operand bundle, 101// '0' otherwise. 102// \p HASEXCEPT '1' if the intrinsic can have an exception behavior operand 103// bundle, '0' otherwise. 104// \p INTRINID The constrained fp intrinsic this VP intrinsic corresponds to. 105#ifndef VP_PROPERTY_CONSTRAINEDFP 106#define VP_PROPERTY_CONSTRAINEDFP(HASROUND, HASEXCEPT, INTRINID) 107#endif 108 109// Map this VP intrinsic to its canonical functional intrinsic. 110// \p INTRIN The non-VP intrinsics with the same function. 111#ifndef VP_PROPERTY_FUNCTIONAL_INTRINSIC 112#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) 113#endif 114 115// This VP Intrinsic is a memory operation 116// The pointer arg is at POINTERPOS and the data arg is at DATAPOS. 117#ifndef VP_PROPERTY_MEMOP 118#define VP_PROPERTY_MEMOP(POINTERPOS, DATAPOS) 119#endif 120 121// Map this VP reduction intrinsic to its reduction operand positions. 122#ifndef VP_PROPERTY_REDUCTION 123#define VP_PROPERTY_REDUCTION(STARTPOS, VECTORPOS) 124#endif 125 126// A property to infer VP binary-op SDNode opcodes automatically. 127#ifndef VP_PROPERTY_BINARYOP 128#define VP_PROPERTY_BINARYOP 129#endif 130 131// A property to infer VP type casts automatically. 132#ifndef VP_PROPERTY_CASTOP 133#define VP_PROPERTY_CASTOP 134#endif 135 136// This VP Intrinsic is a comparison operation 137// The condition code arg is at CCPOS and accepts floating-point condition 138// codes if ISFP is set, else it accepts integer condition codes. 139#ifndef VP_PROPERTY_CMP 140#define VP_PROPERTY_CMP(CCPOS, ISFP) 141#endif 142 143/// } Property Macros 144 145///// Integer Arithmetic { 146 147// Specialized helper macro for integer binary operators (%x, %y, %mask, %evl). 148#ifdef HELPER_REGISTER_BINARY_INT_VP 149#error \ 150 "The internal helper macro HELPER_REGISTER_BINARY_INT_VP is already defined!" 151#endif 152#define HELPER_REGISTER_BINARY_INT_VP(VPID, VPSD, IROPC) \ 153 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, -1) \ 154 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 155 VP_PROPERTY_BINARYOP \ 156 END_REGISTER_VP(VPID, VPSD) 157 158// llvm.vp.add(x,y,mask,vlen) 159HELPER_REGISTER_BINARY_INT_VP(vp_add, VP_ADD, Add) 160 161// llvm.vp.and(x,y,mask,vlen) 162HELPER_REGISTER_BINARY_INT_VP(vp_and, VP_AND, And) 163 164// llvm.vp.ashr(x,y,mask,vlen) 165HELPER_REGISTER_BINARY_INT_VP(vp_ashr, VP_ASHR, AShr) 166 167// llvm.vp.lshr(x,y,mask,vlen) 168HELPER_REGISTER_BINARY_INT_VP(vp_lshr, VP_LSHR, LShr) 169 170// llvm.vp.mul(x,y,mask,vlen) 171HELPER_REGISTER_BINARY_INT_VP(vp_mul, VP_MUL, Mul) 172 173// llvm.vp.or(x,y,mask,vlen) 174HELPER_REGISTER_BINARY_INT_VP(vp_or, VP_OR, Or) 175 176// llvm.vp.sdiv(x,y,mask,vlen) 177HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv) 178 179// llvm.vp.shl(x,y,mask,vlen) 180HELPER_REGISTER_BINARY_INT_VP(vp_shl, VP_SHL, Shl) 181 182// llvm.vp.srem(x,y,mask,vlen) 183HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem) 184 185// llvm.vp.sub(x,y,mask,vlen) 186HELPER_REGISTER_BINARY_INT_VP(vp_sub, VP_SUB, Sub) 187 188// llvm.vp.udiv(x,y,mask,vlen) 189HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv) 190 191// llvm.vp.urem(x,y,mask,vlen) 192HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem) 193 194// llvm.vp.xor(x,y,mask,vlen) 195HELPER_REGISTER_BINARY_INT_VP(vp_xor, VP_XOR, Xor) 196 197#undef HELPER_REGISTER_BINARY_INT_VP 198 199// llvm.vp.smin(x,y,mask,vlen) 200BEGIN_REGISTER_VP(vp_smin, 2, 3, VP_SMIN, -1) 201VP_PROPERTY_BINARYOP 202END_REGISTER_VP(vp_smin, VP_SMIN) 203 204// llvm.vp.smax(x,y,mask,vlen) 205BEGIN_REGISTER_VP(vp_smax, 2, 3, VP_SMAX, -1) 206VP_PROPERTY_BINARYOP 207END_REGISTER_VP(vp_smax, VP_SMAX) 208 209// llvm.vp.umin(x,y,mask,vlen) 210BEGIN_REGISTER_VP(vp_umin, 2, 3, VP_UMIN, -1) 211VP_PROPERTY_BINARYOP 212END_REGISTER_VP(vp_umin, VP_UMIN) 213 214// llvm.vp.umax(x,y,mask,vlen) 215BEGIN_REGISTER_VP(vp_umax, 2, 3, VP_UMAX, -1) 216VP_PROPERTY_BINARYOP 217END_REGISTER_VP(vp_umax, VP_UMAX) 218 219// llvm.vp.abs(x,mask,vlen,is_int_min_poison) 220BEGIN_REGISTER_VP(vp_abs, 1, 2, VP_ABS, -1) 221END_REGISTER_VP(vp_abs, VP_ABS) 222 223// llvm.vp.bswap(x,mask,vlen) 224BEGIN_REGISTER_VP(vp_bswap, 1, 2, VP_BSWAP, -1) 225END_REGISTER_VP(vp_bswap, VP_BSWAP) 226 227// llvm.vp.bitreverse(x,mask,vlen) 228BEGIN_REGISTER_VP(vp_bitreverse, 1, 2, VP_BITREVERSE, -1) 229END_REGISTER_VP(vp_bitreverse, VP_BITREVERSE) 230 231// llvm.vp.ctpop(x,mask,vlen) 232BEGIN_REGISTER_VP(vp_ctpop, 1, 2, VP_CTPOP, -1) 233END_REGISTER_VP(vp_ctpop, VP_CTPOP) 234 235// llvm.vp.ctlz(x,mask,vlen, is_zero_poison) 236BEGIN_REGISTER_VP_INTRINSIC(vp_ctlz, 1, 2) 237BEGIN_REGISTER_VP_SDNODE(VP_CTLZ, -1, vp_ctlz, 1, 2) 238END_REGISTER_VP_SDNODE(VP_CTLZ) 239BEGIN_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF, -1, vp_ctlz_zero_undef, 1, 2) 240END_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF) 241END_REGISTER_VP_INTRINSIC(vp_ctlz) 242 243// llvm.vp.cttz(x,mask,vlen, is_zero_poison) 244BEGIN_REGISTER_VP_INTRINSIC(vp_cttz, 1, 2) 245BEGIN_REGISTER_VP_SDNODE(VP_CTTZ, -1, vp_cttz, 1, 2) 246END_REGISTER_VP_SDNODE(VP_CTTZ) 247BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF, -1, vp_cttz_zero_undef, 1, 2) 248END_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF) 249END_REGISTER_VP_INTRINSIC(vp_cttz) 250 251// llvm.vp.fshl(x,y,z,mask,vlen) 252BEGIN_REGISTER_VP(vp_fshl, 3, 4, VP_FSHL, -1) 253END_REGISTER_VP(vp_fshl, VP_FSHL) 254 255// llvm.vp.fshr(x,y,z,mask,vlen) 256BEGIN_REGISTER_VP(vp_fshr, 3, 4, VP_FSHR, -1) 257END_REGISTER_VP(vp_fshr, VP_FSHR) 258///// } Integer Arithmetic 259 260///// Floating-Point Arithmetic { 261 262// Specialized helper macro for floating-point binary operators 263// <operation>(%x, %y, %mask, %evl). 264#ifdef HELPER_REGISTER_BINARY_FP_VP 265#error \ 266 "The internal helper macro HELPER_REGISTER_BINARY_FP_VP is already defined!" 267#endif 268#define HELPER_REGISTER_BINARY_FP_VP(OPSUFFIX, VPSD, IROPC) \ 269 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 2, 3, VPSD, -1) \ 270 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 271 VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_##OPSUFFIX) \ 272 VP_PROPERTY_BINARYOP \ 273 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 274 275// llvm.vp.fadd(x,y,mask,vlen) 276HELPER_REGISTER_BINARY_FP_VP(fadd, VP_FADD, FAdd) 277 278// llvm.vp.fsub(x,y,mask,vlen) 279HELPER_REGISTER_BINARY_FP_VP(fsub, VP_FSUB, FSub) 280 281// llvm.vp.fmul(x,y,mask,vlen) 282HELPER_REGISTER_BINARY_FP_VP(fmul, VP_FMUL, FMul) 283 284// llvm.vp.fdiv(x,y,mask,vlen) 285HELPER_REGISTER_BINARY_FP_VP(fdiv, VP_FDIV, FDiv) 286 287// llvm.vp.frem(x,y,mask,vlen) 288HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem) 289 290#undef HELPER_REGISTER_BINARY_FP_VP 291 292// llvm.vp.fneg(x,mask,vlen) 293BEGIN_REGISTER_VP(vp_fneg, 1, 2, VP_FNEG, -1) 294VP_PROPERTY_FUNCTIONAL_OPC(FNeg) 295END_REGISTER_VP(vp_fneg, VP_FNEG) 296 297// llvm.vp.fabs(x,mask,vlen) 298BEGIN_REGISTER_VP(vp_fabs, 1, 2, VP_FABS, -1) 299END_REGISTER_VP(vp_fabs, VP_FABS) 300 301// llvm.vp.sqrt(x,mask,vlen) 302BEGIN_REGISTER_VP(vp_sqrt, 1, 2, VP_SQRT, -1) 303END_REGISTER_VP(vp_sqrt, VP_SQRT) 304 305// llvm.vp.fma(x,y,z,mask,vlen) 306BEGIN_REGISTER_VP(vp_fma, 3, 4, VP_FMA, -1) 307VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_fma) 308END_REGISTER_VP(vp_fma, VP_FMA) 309 310// llvm.vp.fmuladd(x,y,z,mask,vlen) 311BEGIN_REGISTER_VP(vp_fmuladd, 3, 4, VP_FMULADD, -1) 312VP_PROPERTY_CONSTRAINEDFP(1, 1, experimental_constrained_fmuladd) 313END_REGISTER_VP(vp_fmuladd, VP_FMULADD) 314 315// llvm.vp.copysign(x,y,mask,vlen) 316BEGIN_REGISTER_VP(vp_copysign, 2, 3, VP_FCOPYSIGN, -1) 317END_REGISTER_VP(vp_copysign, VP_FCOPYSIGN) 318 319// llvm.vp.minnum(x, y, mask,vlen) 320BEGIN_REGISTER_VP(vp_minnum, 2, 3, VP_FMINNUM, -1) 321VP_PROPERTY_BINARYOP 322END_REGISTER_VP(vp_minnum, VP_FMINNUM) 323 324// llvm.vp.maxnum(x, y, mask,vlen) 325BEGIN_REGISTER_VP(vp_maxnum, 2, 3, VP_FMAXNUM, -1) 326VP_PROPERTY_BINARYOP 327END_REGISTER_VP(vp_maxnum, VP_FMAXNUM) 328 329// llvm.vp.ceil(x,mask,vlen) 330BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1) 331END_REGISTER_VP(vp_ceil, VP_FCEIL) 332 333// llvm.vp.floor(x,mask,vlen) 334BEGIN_REGISTER_VP(vp_floor, 1, 2, VP_FFLOOR, -1) 335END_REGISTER_VP(vp_floor, VP_FFLOOR) 336 337// llvm.vp.round(x,mask,vlen) 338BEGIN_REGISTER_VP(vp_round, 1, 2, VP_FROUND, -1) 339END_REGISTER_VP(vp_round, VP_FROUND) 340 341// llvm.vp.roundeven(x,mask,vlen) 342BEGIN_REGISTER_VP(vp_roundeven, 1, 2, VP_FROUNDEVEN, -1) 343END_REGISTER_VP(vp_roundeven, VP_FROUNDEVEN) 344 345// llvm.vp.roundtozero(x,mask,vlen) 346BEGIN_REGISTER_VP(vp_roundtozero, 1, 2, VP_FROUNDTOZERO, -1) 347END_REGISTER_VP(vp_roundtozero, VP_FROUNDTOZERO) 348 349// llvm.vp.rint(x,mask,vlen) 350BEGIN_REGISTER_VP(vp_rint, 1, 2, VP_FRINT, -1) 351END_REGISTER_VP(vp_rint, VP_FRINT) 352 353// llvm.vp.nearbyint(x,mask,vlen) 354BEGIN_REGISTER_VP(vp_nearbyint, 1, 2, VP_FNEARBYINT, -1) 355END_REGISTER_VP(vp_nearbyint, VP_FNEARBYINT) 356 357///// } Floating-Point Arithmetic 358 359///// Type Casts { 360// Specialized helper macro for type conversions. 361// <operation>(%x, %mask, %evl). 362#ifdef HELPER_REGISTER_FP_CAST_VP 363#error \ 364 "The internal helper macro HELPER_REGISTER_FP_CAST_VP is already defined!" 365#endif 366#define HELPER_REGISTER_FP_CAST_VP(OPSUFFIX, VPSD, IROPC, HASROUND) \ 367 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \ 368 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 369 VP_PROPERTY_CONSTRAINEDFP(HASROUND, 1, experimental_constrained_##OPSUFFIX) \ 370 VP_PROPERTY_CASTOP \ 371 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 372 373// llvm.vp.fptoui(x,mask,vlen) 374HELPER_REGISTER_FP_CAST_VP(fptoui, VP_FP_TO_UINT, FPToUI, 0) 375 376// llvm.vp.fptosi(x,mask,vlen) 377HELPER_REGISTER_FP_CAST_VP(fptosi, VP_FP_TO_SINT, FPToSI, 0) 378 379// llvm.vp.uitofp(x,mask,vlen) 380HELPER_REGISTER_FP_CAST_VP(uitofp, VP_UINT_TO_FP, UIToFP, 1) 381 382// llvm.vp.sitofp(x,mask,vlen) 383HELPER_REGISTER_FP_CAST_VP(sitofp, VP_SINT_TO_FP, SIToFP, 1) 384 385// llvm.vp.fptrunc(x,mask,vlen) 386HELPER_REGISTER_FP_CAST_VP(fptrunc, VP_FP_ROUND, FPTrunc, 1) 387 388// llvm.vp.fpext(x,mask,vlen) 389HELPER_REGISTER_FP_CAST_VP(fpext, VP_FP_EXTEND, FPExt, 0) 390 391#undef HELPER_REGISTER_FP_CAST_VP 392 393// Specialized helper macro for integer type conversions. 394// <operation>(%x, %mask, %evl). 395#ifdef HELPER_REGISTER_INT_CAST_VP 396#error \ 397 "The internal helper macro HELPER_REGISTER_INT_CAST_VP is already defined!" 398#endif 399#define HELPER_REGISTER_INT_CAST_VP(OPSUFFIX, VPSD, IROPC) \ 400 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \ 401 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 402 VP_PROPERTY_CASTOP \ 403 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 404 405// llvm.vp.trunc(x,mask,vlen) 406HELPER_REGISTER_INT_CAST_VP(trunc, VP_TRUNCATE, Trunc) 407 408// llvm.vp.zext(x,mask,vlen) 409HELPER_REGISTER_INT_CAST_VP(zext, VP_ZERO_EXTEND, ZExt) 410 411// llvm.vp.sext(x,mask,vlen) 412HELPER_REGISTER_INT_CAST_VP(sext, VP_SIGN_EXTEND, SExt) 413 414// llvm.vp.ptrtoint(x,mask,vlen) 415HELPER_REGISTER_INT_CAST_VP(ptrtoint, VP_PTRTOINT, PtrToInt) 416 417// llvm.vp.inttoptr(x,mask,vlen) 418HELPER_REGISTER_INT_CAST_VP(inttoptr, VP_INTTOPTR, IntToPtr) 419 420#undef HELPER_REGISTER_INT_CAST_VP 421 422///// } Type Casts 423 424///// Comparisons { 425 426// VP_SETCC (ISel only) 427BEGIN_REGISTER_VP_SDNODE(VP_SETCC, 0, vp_setcc, 3, 4) 428END_REGISTER_VP_SDNODE(VP_SETCC) 429 430// llvm.vp.fcmp(x,y,cc,mask,vlen) 431BEGIN_REGISTER_VP_INTRINSIC(vp_fcmp, 3, 4) 432HELPER_MAP_VPID_TO_VPSD(vp_fcmp, VP_SETCC) 433VP_PROPERTY_FUNCTIONAL_OPC(FCmp) 434VP_PROPERTY_CMP(2, true) 435VP_PROPERTY_CONSTRAINEDFP(0, 1, experimental_constrained_fcmp) 436END_REGISTER_VP_INTRINSIC(vp_fcmp) 437 438// llvm.vp.icmp(x,y,cc,mask,vlen) 439BEGIN_REGISTER_VP_INTRINSIC(vp_icmp, 3, 4) 440HELPER_MAP_VPID_TO_VPSD(vp_icmp, VP_SETCC) 441VP_PROPERTY_FUNCTIONAL_OPC(ICmp) 442VP_PROPERTY_CMP(2, false) 443END_REGISTER_VP_INTRINSIC(vp_icmp) 444 445///// } Comparisons 446 447///// Memory Operations { 448// llvm.vp.store(val,ptr,mask,vlen) 449BEGIN_REGISTER_VP_INTRINSIC(vp_store, 2, 3) 450// chain = VP_STORE chain,val,base,offset,mask,evl 451BEGIN_REGISTER_VP_SDNODE(VP_STORE, 1, vp_store, 4, 5) 452HELPER_MAP_VPID_TO_VPSD(vp_store, VP_STORE) 453VP_PROPERTY_FUNCTIONAL_OPC(Store) 454VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_store) 455VP_PROPERTY_MEMOP(1, 0) 456END_REGISTER_VP(vp_store, VP_STORE) 457 458// llvm.experimental.vp.strided.store(val,ptr,stride,mask,vlen) 459BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_store, 3, 4) 460// chain = EXPERIMENTAL_VP_STRIDED_STORE chain,val,base,offset,stride,mask,evl 461BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_STORE, 1, experimental_vp_strided_store, 5, 6) 462HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE) 463VP_PROPERTY_MEMOP(1, 0) 464END_REGISTER_VP(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE) 465 466// llvm.vp.scatter(ptr,val,mask,vlen) 467BEGIN_REGISTER_VP_INTRINSIC(vp_scatter, 2, 3) 468// chain = VP_SCATTER chain,val,base,indices,scale,mask,evl 469BEGIN_REGISTER_VP_SDNODE(VP_SCATTER, 1, vp_scatter, 5, 6) 470HELPER_MAP_VPID_TO_VPSD(vp_scatter, VP_SCATTER) 471VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_scatter) 472VP_PROPERTY_MEMOP(1, 0) 473END_REGISTER_VP(vp_scatter, VP_SCATTER) 474 475// llvm.vp.load(ptr,mask,vlen) 476BEGIN_REGISTER_VP_INTRINSIC(vp_load, 1, 2) 477// val,chain = VP_LOAD chain,base,offset,mask,evl 478BEGIN_REGISTER_VP_SDNODE(VP_LOAD, -1, vp_load, 3, 4) 479HELPER_MAP_VPID_TO_VPSD(vp_load, VP_LOAD) 480VP_PROPERTY_FUNCTIONAL_OPC(Load) 481VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_load) 482VP_PROPERTY_MEMOP(0, std::nullopt) 483END_REGISTER_VP(vp_load, VP_LOAD) 484 485// llvm.experimental.vp.strided.load(ptr,stride,mask,vlen) 486BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_load, 2, 3) 487// chain = EXPERIMENTAL_VP_STRIDED_LOAD chain,base,offset,stride,mask,evl 488BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_LOAD, -1, experimental_vp_strided_load, 4, 5) 489HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD) 490VP_PROPERTY_MEMOP(0, std::nullopt) 491END_REGISTER_VP(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD) 492 493// llvm.vp.gather(ptr,mask,vlen) 494BEGIN_REGISTER_VP_INTRINSIC(vp_gather, 1, 2) 495// val,chain = VP_GATHER chain,base,indices,scale,mask,evl 496BEGIN_REGISTER_VP_SDNODE(VP_GATHER, -1, vp_gather, 4, 5) 497HELPER_MAP_VPID_TO_VPSD(vp_gather, VP_GATHER) 498VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_gather) 499VP_PROPERTY_MEMOP(0, std::nullopt) 500END_REGISTER_VP(vp_gather, VP_GATHER) 501 502///// } Memory Operations 503 504///// Reductions { 505 506// Specialized helper macro for VP reductions (%start, %x, %mask, %evl). 507#ifdef HELPER_REGISTER_REDUCTION_VP 508#error \ 509 "The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!" 510#endif 511#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN) \ 512 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, 1) \ 513 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \ 514 VP_PROPERTY_REDUCTION(0, 1) \ 515 END_REGISTER_VP(VPID, VPSD) 516 517// llvm.vp.reduce.add(start,x,mask,vlen) 518HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD, 519 experimental_vector_reduce_add) 520 521// llvm.vp.reduce.mul(start,x,mask,vlen) 522HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL, 523 experimental_vector_reduce_mul) 524 525// llvm.vp.reduce.and(start,x,mask,vlen) 526HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND, 527 experimental_vector_reduce_and) 528 529// llvm.vp.reduce.or(start,x,mask,vlen) 530HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR, 531 experimental_vector_reduce_or) 532 533// llvm.vp.reduce.xor(start,x,mask,vlen) 534HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR, 535 experimental_vector_reduce_xor) 536 537// llvm.vp.reduce.smax(start,x,mask,vlen) 538HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX, 539 experimental_vector_reduce_smax) 540 541// llvm.vp.reduce.smin(start,x,mask,vlen) 542HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN, 543 experimental_vector_reduce_smin) 544 545// llvm.vp.reduce.umax(start,x,mask,vlen) 546HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX, 547 experimental_vector_reduce_umax) 548 549// llvm.vp.reduce.umin(start,x,mask,vlen) 550HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN, 551 experimental_vector_reduce_umin) 552 553// llvm.vp.reduce.fmax(start,x,mask,vlen) 554HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX, 555 experimental_vector_reduce_fmax) 556 557// llvm.vp.reduce.fmin(start,x,mask,vlen) 558HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN, 559 experimental_vector_reduce_fmin) 560 561#undef HELPER_REGISTER_REDUCTION_VP 562 563// Specialized helper macro for VP reductions as above but with two forms: 564// sequential and reassociative. These manifest as the presence of 'reassoc' 565// fast-math flags in the IR and as two distinct ISD opcodes in the 566// SelectionDAG. 567// Note we by default map from the VP intrinsic to the SEQ ISD opcode, which 568// can then be relaxed to the non-SEQ ISD opcode if the 'reassoc' flag is set. 569#ifdef HELPER_REGISTER_REDUCTION_SEQ_VP 570#error \ 571 "The internal helper macro HELPER_REGISTER_REDUCTION_SEQ_VP is already defined!" 572#endif 573#define HELPER_REGISTER_REDUCTION_SEQ_VP(VPID, VPSD, SEQ_VPSD, INTRIN) \ 574 BEGIN_REGISTER_VP_INTRINSIC(VPID, 2, 3) \ 575 BEGIN_REGISTER_VP_SDNODE(VPSD, 1, VPID, 2, 3) \ 576 VP_PROPERTY_REDUCTION(0, 1) \ 577 END_REGISTER_VP_SDNODE(VPSD) \ 578 BEGIN_REGISTER_VP_SDNODE(SEQ_VPSD, 1, VPID, 2, 3) \ 579 HELPER_MAP_VPID_TO_VPSD(VPID, SEQ_VPSD) \ 580 VP_PROPERTY_REDUCTION(0, 1) \ 581 END_REGISTER_VP_SDNODE(SEQ_VPSD) \ 582 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \ 583 END_REGISTER_VP_INTRINSIC(VPID) 584 585// llvm.vp.reduce.fadd(start,x,mask,vlen) 586HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fadd, VP_REDUCE_FADD, 587 VP_REDUCE_SEQ_FADD, 588 experimental_vector_reduce_fadd) 589 590// llvm.vp.reduce.fmul(start,x,mask,vlen) 591HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL, 592 VP_REDUCE_SEQ_FMUL, 593 experimental_vector_reduce_fmul) 594 595#undef HELPER_REGISTER_REDUCTION_SEQ_VP 596 597///// } Reduction 598 599///// Shuffles { 600 601// The mask 'cond' operand of llvm.vp.select and llvm.vp.merge are not reported 602// as masks with the BEGIN_REGISTER_VP_* macros. This is because, unlike other 603// VP intrinsics, these two have a defined result on lanes where the mask is 604// false. 605// 606// llvm.vp.select(cond,on_true,on_false,vlen) 607BEGIN_REGISTER_VP(vp_select, std::nullopt, 3, VP_SELECT, -1) 608VP_PROPERTY_FUNCTIONAL_OPC(Select) 609END_REGISTER_VP(vp_select, VP_SELECT) 610 611// llvm.vp.merge(cond,on_true,on_false,pivot) 612BEGIN_REGISTER_VP(vp_merge, std::nullopt, 3, VP_MERGE, -1) 613END_REGISTER_VP(vp_merge, VP_MERGE) 614 615BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1) 616END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE) 617 618///// } Shuffles 619 620#undef BEGIN_REGISTER_VP 621#undef BEGIN_REGISTER_VP_INTRINSIC 622#undef BEGIN_REGISTER_VP_SDNODE 623#undef END_REGISTER_VP 624#undef END_REGISTER_VP_INTRINSIC 625#undef END_REGISTER_VP_SDNODE 626#undef HELPER_MAP_VPID_TO_VPSD 627#undef VP_PROPERTY_BINARYOP 628#undef VP_PROPERTY_CASTOP 629#undef VP_PROPERTY_CMP 630#undef VP_PROPERTY_CONSTRAINEDFP 631#undef VP_PROPERTY_FUNCTIONAL_INTRINSIC 632#undef VP_PROPERTY_FUNCTIONAL_OPC 633#undef VP_PROPERTY_MEMOP 634#undef VP_PROPERTY_REDUCTION 635