1//===-- IR/VPIntrinsics.def - Describes llvm.vp.* Intrinsics -*- C++ -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file contains descriptions of the various Vector Predication intrinsics. 10// This is used as a central place for enumerating the different instructions 11// and should eventually be the place to put comments about the instructions. 12// 13//===----------------------------------------------------------------------===// 14 15// NOTE: NO INCLUDE GUARD DESIRED! 16 17// Provide definitions of macros so that users of this file do not have to 18// define everything to use it... 19// 20// Register a VP intrinsic and begin its property scope. 21// All VP intrinsic scopes are top level, ie it is illegal to place a 22// BEGIN_REGISTER_VP_INTRINSIC within a VP intrinsic scope. 23// \p VPID The VP intrinsic id. 24// \p MASKPOS The mask operand position. 25// \p EVLPOS The explicit vector length operand position. 26#ifndef BEGIN_REGISTER_VP_INTRINSIC 27#define BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) 28#endif 29 30// End the property scope of a VP intrinsic. 31#ifndef END_REGISTER_VP_INTRINSIC 32#define END_REGISTER_VP_INTRINSIC(VPID) 33#endif 34 35// Register a new VP SDNode and begin its property scope. 36// When the SDNode scope is nested within a VP intrinsic scope, it is 37// implicitly registered as the canonical SDNode for this VP intrinsic. There 38// is one VP intrinsic that maps directly to one SDNode that goes by the 39// same name. Since the operands are also the same, we open the property 40// scopes for both the VPIntrinsic and the SDNode at once. 41// \p VPSD The SelectionDAG Node id (eg VP_ADD). 42// \p LEGALPOS The operand position of the SDNode that is used for legalizing. 43// If LEGALPOS < 0, then the return type given by 44// TheNode->getValueType(-1-LEGALPOS) is used. 45// \p TDNAME The name of the TableGen definition of this SDNode. 46// \p MASKPOS The mask operand position. 47// \p EVLPOS The explicit vector length operand position. 48#ifndef BEGIN_REGISTER_VP_SDNODE 49#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) 50#endif 51 52// End the property scope of a new VP SDNode. 53#ifndef END_REGISTER_VP_SDNODE 54#define END_REGISTER_VP_SDNODE(VPSD) 55#endif 56 57// Helper macro to set up the mapping from VP intrinsic to ISD opcode. 58// Note: More than one VP intrinsic may map to one ISD opcode. 59#ifndef HELPER_MAP_VPID_TO_VPSD 60#define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) 61#endif 62 63// Helper macros for the common "1:1 - Intrinsic : SDNode" case. 64// 65// There is one VP intrinsic that maps directly to one SDNode that goes by the 66// same name. Since the operands are also the same, we open the property 67// scopes for both the VPIntrinsic and the SDNode at once. 68// 69// \p VPID The canonical name (eg `vp_add`, which at the same time is the 70// name of the intrinsic and the TableGen def of the SDNode). 71// \p MASKPOS The mask operand position. 72// \p EVLPOS The explicit vector length operand position. 73// \p VPSD The SelectionDAG Node id (eg VP_ADD). 74// \p LEGALPOS The operand position of the SDNode that is used for legalizing 75// this SDNode. This can be `-1`, in which case the return type of 76// the SDNode is used. 77#define BEGIN_REGISTER_VP(VPID, MASKPOS, EVLPOS, VPSD, LEGALPOS) \ 78 BEGIN_REGISTER_VP_INTRINSIC(VPID, MASKPOS, EVLPOS) \ 79 BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, VPID, MASKPOS, EVLPOS) \ 80 HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) 81 82#define END_REGISTER_VP(VPID, VPSD) \ 83 END_REGISTER_VP_INTRINSIC(VPID) \ 84 END_REGISTER_VP_SDNODE(VPSD) 85 86// The following macros attach properties to the scope they are placed in. This 87// assigns the property to the VP Intrinsic and/or SDNode that belongs to the 88// scope. 89// 90// Property Macros { 91 92// The intrinsic and/or SDNode has the same function as this LLVM IR Opcode. 93// \p OPC The opcode of the instruction with the same function. 94#ifndef VP_PROPERTY_FUNCTIONAL_OPC 95#define VP_PROPERTY_FUNCTIONAL_OPC(OPC) 96#endif 97 98// If operation can have rounding or fp exceptions, maps to corresponding 99// constrained fp intrinsic. 100#ifndef VP_PROPERTY_CONSTRAINEDFP 101#define VP_PROPERTY_CONSTRAINEDFP(INTRINID) 102#endif 103 104// The intrinsic and/or SDNode has the same function as this ISD Opcode. 105// \p SDOPC The opcode of the instruction with the same function. 106#ifndef VP_PROPERTY_FUNCTIONAL_SDOPC 107#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) 108#endif 109 110// Map this VP intrinsic to its canonical functional intrinsic. 111// \p INTRIN The non-VP intrinsics with the same function. 112#ifndef VP_PROPERTY_FUNCTIONAL_INTRINSIC 113#define VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) 114#endif 115 116// This VP Intrinsic has no functionally-equivalent non-VP opcode or intrinsic. 117#ifndef VP_PROPERTY_NO_FUNCTIONAL 118#define VP_PROPERTY_NO_FUNCTIONAL 119#endif 120 121// A property to infer VP binary-op SDNode opcodes automatically. 122#ifndef VP_PROPERTY_BINARYOP 123#define VP_PROPERTY_BINARYOP 124#endif 125 126/// } Property Macros 127 128///// Integer Arithmetic { 129 130// Specialized helper macro for integer binary operators (%x, %y, %mask, %evl). 131#ifdef HELPER_REGISTER_BINARY_INT_VP 132#error \ 133 "The internal helper macro HELPER_REGISTER_BINARY_INT_VP is already defined!" 134#endif 135#define HELPER_REGISTER_BINARY_INT_VP(VPID, VPSD, IROPC, SDOPC) \ 136 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, -1) \ 137 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 138 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 139 VP_PROPERTY_BINARYOP \ 140 END_REGISTER_VP(VPID, VPSD) 141 142// llvm.vp.add(x,y,mask,vlen) 143HELPER_REGISTER_BINARY_INT_VP(vp_add, VP_ADD, Add, ADD) 144 145// llvm.vp.and(x,y,mask,vlen) 146HELPER_REGISTER_BINARY_INT_VP(vp_and, VP_AND, And, AND) 147 148// llvm.vp.ashr(x,y,mask,vlen) 149HELPER_REGISTER_BINARY_INT_VP(vp_ashr, VP_SRA, AShr, SRA) 150 151// llvm.vp.lshr(x,y,mask,vlen) 152HELPER_REGISTER_BINARY_INT_VP(vp_lshr, VP_SRL, LShr, SRL) 153 154// llvm.vp.mul(x,y,mask,vlen) 155HELPER_REGISTER_BINARY_INT_VP(vp_mul, VP_MUL, Mul, MUL) 156 157// llvm.vp.or(x,y,mask,vlen) 158HELPER_REGISTER_BINARY_INT_VP(vp_or, VP_OR, Or, OR) 159 160// llvm.vp.sdiv(x,y,mask,vlen) 161HELPER_REGISTER_BINARY_INT_VP(vp_sdiv, VP_SDIV, SDiv, SDIV) 162 163// llvm.vp.shl(x,y,mask,vlen) 164HELPER_REGISTER_BINARY_INT_VP(vp_shl, VP_SHL, Shl, SHL) 165 166// llvm.vp.srem(x,y,mask,vlen) 167HELPER_REGISTER_BINARY_INT_VP(vp_srem, VP_SREM, SRem, SREM) 168 169// llvm.vp.sub(x,y,mask,vlen) 170HELPER_REGISTER_BINARY_INT_VP(vp_sub, VP_SUB, Sub, SUB) 171 172// llvm.vp.udiv(x,y,mask,vlen) 173HELPER_REGISTER_BINARY_INT_VP(vp_udiv, VP_UDIV, UDiv, UDIV) 174 175// llvm.vp.urem(x,y,mask,vlen) 176HELPER_REGISTER_BINARY_INT_VP(vp_urem, VP_UREM, URem, UREM) 177 178// llvm.vp.xor(x,y,mask,vlen) 179HELPER_REGISTER_BINARY_INT_VP(vp_xor, VP_XOR, Xor, XOR) 180 181#undef HELPER_REGISTER_BINARY_INT_VP 182 183// llvm.vp.smin(x,y,mask,vlen) 184BEGIN_REGISTER_VP(vp_smin, 2, 3, VP_SMIN, -1) 185VP_PROPERTY_BINARYOP 186VP_PROPERTY_FUNCTIONAL_SDOPC(SMIN) 187VP_PROPERTY_FUNCTIONAL_INTRINSIC(smin) 188END_REGISTER_VP(vp_smin, VP_SMIN) 189 190// llvm.vp.smax(x,y,mask,vlen) 191BEGIN_REGISTER_VP(vp_smax, 2, 3, VP_SMAX, -1) 192VP_PROPERTY_BINARYOP 193VP_PROPERTY_FUNCTIONAL_SDOPC(SMAX) 194VP_PROPERTY_FUNCTIONAL_INTRINSIC(smax) 195END_REGISTER_VP(vp_smax, VP_SMAX) 196 197// llvm.vp.umin(x,y,mask,vlen) 198BEGIN_REGISTER_VP(vp_umin, 2, 3, VP_UMIN, -1) 199VP_PROPERTY_BINARYOP 200VP_PROPERTY_FUNCTIONAL_SDOPC(UMIN) 201VP_PROPERTY_FUNCTIONAL_INTRINSIC(umin) 202END_REGISTER_VP(vp_umin, VP_UMIN) 203 204// llvm.vp.umax(x,y,mask,vlen) 205BEGIN_REGISTER_VP(vp_umax, 2, 3, VP_UMAX, -1) 206VP_PROPERTY_BINARYOP 207VP_PROPERTY_FUNCTIONAL_SDOPC(UMAX) 208VP_PROPERTY_FUNCTIONAL_INTRINSIC(umax) 209END_REGISTER_VP(vp_umax, VP_UMAX) 210 211// llvm.vp.abs(x,is_int_min_poison,mask,vlen) 212BEGIN_REGISTER_VP_INTRINSIC(vp_abs, 2, 3) 213BEGIN_REGISTER_VP_SDNODE(VP_ABS, -1, vp_abs, 1, 2) 214HELPER_MAP_VPID_TO_VPSD(vp_abs, VP_ABS) 215VP_PROPERTY_FUNCTIONAL_INTRINSIC(abs) 216VP_PROPERTY_FUNCTIONAL_SDOPC(ABS) 217END_REGISTER_VP(vp_abs, VP_ABS) 218 219// llvm.vp.bswap(x,mask,vlen) 220BEGIN_REGISTER_VP(vp_bswap, 1, 2, VP_BSWAP, -1) 221VP_PROPERTY_FUNCTIONAL_INTRINSIC(bswap) 222VP_PROPERTY_FUNCTIONAL_SDOPC(BSWAP) 223END_REGISTER_VP(vp_bswap, VP_BSWAP) 224 225// llvm.vp.bitreverse(x,mask,vlen) 226BEGIN_REGISTER_VP(vp_bitreverse, 1, 2, VP_BITREVERSE, -1) 227VP_PROPERTY_FUNCTIONAL_INTRINSIC(bitreverse) 228VP_PROPERTY_FUNCTIONAL_SDOPC(BITREVERSE) 229END_REGISTER_VP(vp_bitreverse, VP_BITREVERSE) 230 231// llvm.vp.ctpop(x,mask,vlen) 232BEGIN_REGISTER_VP(vp_ctpop, 1, 2, VP_CTPOP, -1) 233VP_PROPERTY_FUNCTIONAL_INTRINSIC(ctpop) 234VP_PROPERTY_FUNCTIONAL_SDOPC(CTPOP) 235END_REGISTER_VP(vp_ctpop, VP_CTPOP) 236 237// llvm.vp.ctlz(x,is_zero_poison,mask,vlen) 238BEGIN_REGISTER_VP_INTRINSIC(vp_ctlz, 2, 3) 239BEGIN_REGISTER_VP_SDNODE(VP_CTLZ, -1, vp_ctlz, 1, 2) 240VP_PROPERTY_FUNCTIONAL_INTRINSIC(ctlz) 241VP_PROPERTY_FUNCTIONAL_SDOPC(CTLZ) 242END_REGISTER_VP_SDNODE(VP_CTLZ) 243BEGIN_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF, -1, vp_ctlz_zero_undef, 1, 2) 244VP_PROPERTY_FUNCTIONAL_SDOPC(CTLZ_ZERO_UNDEF) 245END_REGISTER_VP_SDNODE(VP_CTLZ_ZERO_UNDEF) 246END_REGISTER_VP_INTRINSIC(vp_ctlz) 247 248// llvm.vp.cttz(x,is_zero_poison,mask,vlen) 249BEGIN_REGISTER_VP_INTRINSIC(vp_cttz, 2, 3) 250BEGIN_REGISTER_VP_SDNODE(VP_CTTZ, -1, vp_cttz, 1, 2) 251VP_PROPERTY_FUNCTIONAL_INTRINSIC(cttz) 252VP_PROPERTY_FUNCTIONAL_SDOPC(CTTZ) 253END_REGISTER_VP_SDNODE(VP_CTTZ) 254BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF, -1, vp_cttz_zero_undef, 1, 2) 255END_REGISTER_VP_SDNODE(VP_CTTZ_ZERO_UNDEF) 256END_REGISTER_VP_INTRINSIC(vp_cttz) 257 258// llvm.vp.cttz.elts(x,is_zero_poison,mask,vl) 259BEGIN_REGISTER_VP_INTRINSIC(vp_cttz_elts, 2, 3) 260VP_PROPERTY_NO_FUNCTIONAL 261BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ELTS, 0, vp_cttz_elts, 1, 2) 262END_REGISTER_VP_SDNODE(VP_CTTZ_ELTS) 263BEGIN_REGISTER_VP_SDNODE(VP_CTTZ_ELTS_ZERO_UNDEF, 0, vp_cttz_elts_zero_undef, 1, 2) 264END_REGISTER_VP_SDNODE(VP_CTTZ_ELTS_ZERO_UNDEF) 265END_REGISTER_VP_INTRINSIC(vp_cttz_elts) 266 267// llvm.vp.fshl(x,y,z,mask,vlen) 268BEGIN_REGISTER_VP(vp_fshl, 3, 4, VP_FSHL, -1) 269VP_PROPERTY_FUNCTIONAL_INTRINSIC(fshl) 270VP_PROPERTY_FUNCTIONAL_SDOPC(FSHL) 271END_REGISTER_VP(vp_fshl, VP_FSHL) 272 273// llvm.vp.fshr(x,y,z,mask,vlen) 274BEGIN_REGISTER_VP(vp_fshr, 3, 4, VP_FSHR, -1) 275VP_PROPERTY_FUNCTIONAL_INTRINSIC(fshr) 276VP_PROPERTY_FUNCTIONAL_SDOPC(FSHR) 277END_REGISTER_VP(vp_fshr, VP_FSHR) 278 279// llvm.vp.sadd.sat(x,y,mask,vlen) 280BEGIN_REGISTER_VP(vp_sadd_sat, 2, 3, VP_SADDSAT, -1) 281VP_PROPERTY_FUNCTIONAL_INTRINSIC(sadd_sat) 282VP_PROPERTY_FUNCTIONAL_SDOPC(SADDSAT) 283END_REGISTER_VP(vp_sadd_sat, VP_SADDSAT) 284 285// llvm.vp.uadd.sat(x,y,mask,vlen) 286BEGIN_REGISTER_VP(vp_uadd_sat, 2, 3, VP_UADDSAT, -1) 287VP_PROPERTY_FUNCTIONAL_INTRINSIC(uadd_sat) 288VP_PROPERTY_FUNCTIONAL_SDOPC(UADDSAT) 289END_REGISTER_VP(vp_uadd_sat, VP_UADDSAT) 290 291// llvm.vp.ssub.sat(x,y,mask,vlen) 292BEGIN_REGISTER_VP(vp_ssub_sat, 2, 3, VP_SSUBSAT, -1) 293VP_PROPERTY_FUNCTIONAL_INTRINSIC(ssub_sat) 294VP_PROPERTY_FUNCTIONAL_SDOPC(SSUBSAT) 295END_REGISTER_VP(vp_ssub_sat, VP_SSUBSAT) 296 297// llvm.vp.usub.sat(x,y,mask,vlen) 298BEGIN_REGISTER_VP(vp_usub_sat, 2, 3, VP_USUBSAT, -1) 299VP_PROPERTY_FUNCTIONAL_INTRINSIC(usub_sat) 300VP_PROPERTY_FUNCTIONAL_SDOPC(USUBSAT) 301END_REGISTER_VP(vp_usub_sat, VP_USUBSAT) 302///// } Integer Arithmetic 303 304///// Floating-Point Arithmetic { 305 306// Specialized helper macro for floating-point binary operators 307// <operation>(%x, %y, %mask, %evl). 308#ifdef HELPER_REGISTER_BINARY_FP_VP 309#error \ 310 "The internal helper macro HELPER_REGISTER_BINARY_FP_VP is already defined!" 311#endif 312#define HELPER_REGISTER_BINARY_FP_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \ 313 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 2, 3, VPSD, -1) \ 314 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 315 VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_##OPSUFFIX) \ 316 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 317 VP_PROPERTY_BINARYOP \ 318 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 319 320// llvm.vp.fadd(x,y,mask,vlen) 321HELPER_REGISTER_BINARY_FP_VP(fadd, VP_FADD, FAdd, FADD) 322 323// llvm.vp.fsub(x,y,mask,vlen) 324HELPER_REGISTER_BINARY_FP_VP(fsub, VP_FSUB, FSub, FSUB) 325 326// llvm.vp.fmul(x,y,mask,vlen) 327HELPER_REGISTER_BINARY_FP_VP(fmul, VP_FMUL, FMul, FMUL) 328 329// llvm.vp.fdiv(x,y,mask,vlen) 330HELPER_REGISTER_BINARY_FP_VP(fdiv, VP_FDIV, FDiv, FDIV) 331 332// llvm.vp.frem(x,y,mask,vlen) 333HELPER_REGISTER_BINARY_FP_VP(frem, VP_FREM, FRem, FREM) 334 335#undef HELPER_REGISTER_BINARY_FP_VP 336 337// llvm.vp.fneg(x,mask,vlen) 338BEGIN_REGISTER_VP(vp_fneg, 1, 2, VP_FNEG, -1) 339VP_PROPERTY_FUNCTIONAL_OPC(FNeg) 340VP_PROPERTY_FUNCTIONAL_SDOPC(FNEG) 341END_REGISTER_VP(vp_fneg, VP_FNEG) 342 343// llvm.vp.fabs(x,mask,vlen) 344BEGIN_REGISTER_VP(vp_fabs, 1, 2, VP_FABS, -1) 345VP_PROPERTY_FUNCTIONAL_INTRINSIC(fabs) 346VP_PROPERTY_FUNCTIONAL_SDOPC(FABS) 347END_REGISTER_VP(vp_fabs, VP_FABS) 348 349// llvm.vp.sqrt(x,mask,vlen) 350BEGIN_REGISTER_VP(vp_sqrt, 1, 2, VP_SQRT, -1) 351VP_PROPERTY_FUNCTIONAL_INTRINSIC(sqrt) 352VP_PROPERTY_FUNCTIONAL_SDOPC(FSQRT) 353END_REGISTER_VP(vp_sqrt, VP_SQRT) 354 355// llvm.vp.fma(x,y,z,mask,vlen) 356BEGIN_REGISTER_VP(vp_fma, 3, 4, VP_FMA, -1) 357VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_fma) 358VP_PROPERTY_FUNCTIONAL_INTRINSIC(fma) 359VP_PROPERTY_FUNCTIONAL_SDOPC(FMA) 360END_REGISTER_VP(vp_fma, VP_FMA) 361 362// llvm.vp.fmuladd(x,y,z,mask,vlen) 363BEGIN_REGISTER_VP(vp_fmuladd, 3, 4, VP_FMULADD, -1) 364VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_fmuladd) 365VP_PROPERTY_FUNCTIONAL_INTRINSIC(fmuladd) 366VP_PROPERTY_FUNCTIONAL_SDOPC(FMAD) 367END_REGISTER_VP(vp_fmuladd, VP_FMULADD) 368 369// llvm.vp.copysign(x,y,mask,vlen) 370BEGIN_REGISTER_VP(vp_copysign, 2, 3, VP_FCOPYSIGN, -1) 371VP_PROPERTY_BINARYOP 372VP_PROPERTY_FUNCTIONAL_SDOPC(FCOPYSIGN) 373VP_PROPERTY_FUNCTIONAL_INTRINSIC(copysign) 374END_REGISTER_VP(vp_copysign, VP_FCOPYSIGN) 375 376// llvm.vp.minnum(x,y,mask,vlen) 377BEGIN_REGISTER_VP(vp_minnum, 2, 3, VP_FMINNUM, -1) 378VP_PROPERTY_BINARYOP 379VP_PROPERTY_FUNCTIONAL_SDOPC(FMINNUM) 380VP_PROPERTY_FUNCTIONAL_INTRINSIC(minnum) 381END_REGISTER_VP(vp_minnum, VP_FMINNUM) 382 383// llvm.vp.maxnum(x,y,mask,vlen) 384BEGIN_REGISTER_VP(vp_maxnum, 2, 3, VP_FMAXNUM, -1) 385VP_PROPERTY_BINARYOP 386VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXNUM) 387VP_PROPERTY_FUNCTIONAL_INTRINSIC(maxnum) 388END_REGISTER_VP(vp_maxnum, VP_FMAXNUM) 389 390// llvm.vp.minimum(x,y,mask,vlen) 391BEGIN_REGISTER_VP(vp_minimum, 2, 3, VP_FMINIMUM, -1) 392VP_PROPERTY_BINARYOP 393VP_PROPERTY_FUNCTIONAL_SDOPC(FMINIMUM) 394VP_PROPERTY_FUNCTIONAL_INTRINSIC(minimum) 395END_REGISTER_VP(vp_minimum, VP_FMINIMUM) 396 397// llvm.vp.maximum(x,y,mask,vlen) 398BEGIN_REGISTER_VP(vp_maximum, 2, 3, VP_FMAXIMUM, -1) 399VP_PROPERTY_BINARYOP 400VP_PROPERTY_FUNCTIONAL_SDOPC(FMAXIMUM) 401VP_PROPERTY_FUNCTIONAL_INTRINSIC(maximum) 402END_REGISTER_VP(vp_maximum, VP_FMAXIMUM) 403 404// llvm.vp.ceil(x,mask,vlen) 405BEGIN_REGISTER_VP(vp_ceil, 1, 2, VP_FCEIL, -1) 406VP_PROPERTY_FUNCTIONAL_INTRINSIC(ceil) 407VP_PROPERTY_FUNCTIONAL_SDOPC(FCEIL) 408END_REGISTER_VP(vp_ceil, VP_FCEIL) 409 410// llvm.vp.floor(x,mask,vlen) 411BEGIN_REGISTER_VP(vp_floor, 1, 2, VP_FFLOOR, -1) 412VP_PROPERTY_FUNCTIONAL_INTRINSIC(floor) 413VP_PROPERTY_FUNCTIONAL_SDOPC(FFLOOR) 414END_REGISTER_VP(vp_floor, VP_FFLOOR) 415 416// llvm.vp.round(x,mask,vlen) 417BEGIN_REGISTER_VP(vp_round, 1, 2, VP_FROUND, -1) 418VP_PROPERTY_FUNCTIONAL_INTRINSIC(round) 419VP_PROPERTY_FUNCTIONAL_SDOPC(FROUND) 420END_REGISTER_VP(vp_round, VP_FROUND) 421 422// llvm.vp.roundeven(x,mask,vlen) 423BEGIN_REGISTER_VP(vp_roundeven, 1, 2, VP_FROUNDEVEN, -1) 424VP_PROPERTY_FUNCTIONAL_INTRINSIC(roundeven) 425VP_PROPERTY_FUNCTIONAL_SDOPC(FROUNDEVEN) 426END_REGISTER_VP(vp_roundeven, VP_FROUNDEVEN) 427 428// llvm.vp.roundtozero(x,mask,vlen) 429BEGIN_REGISTER_VP(vp_roundtozero, 1, 2, VP_FROUNDTOZERO, -1) 430VP_PROPERTY_FUNCTIONAL_INTRINSIC(trunc) 431VP_PROPERTY_FUNCTIONAL_SDOPC(FTRUNC) 432END_REGISTER_VP(vp_roundtozero, VP_FROUNDTOZERO) 433 434// llvm.vp.rint(x,mask,vlen) 435BEGIN_REGISTER_VP(vp_rint, 1, 2, VP_FRINT, -1) 436VP_PROPERTY_FUNCTIONAL_INTRINSIC(rint) 437VP_PROPERTY_FUNCTIONAL_SDOPC(FRINT) 438END_REGISTER_VP(vp_rint, VP_FRINT) 439 440// llvm.vp.nearbyint(x,mask,vlen) 441BEGIN_REGISTER_VP(vp_nearbyint, 1, 2, VP_FNEARBYINT, -1) 442VP_PROPERTY_FUNCTIONAL_INTRINSIC(nearbyint) 443VP_PROPERTY_FUNCTIONAL_SDOPC(FNEARBYINT) 444END_REGISTER_VP(vp_nearbyint, VP_FNEARBYINT) 445 446// llvm.vp.lrint(x,mask,vlen) 447BEGIN_REGISTER_VP(vp_lrint, 1, 2, VP_LRINT, 0) 448VP_PROPERTY_FUNCTIONAL_INTRINSIC(lrint) 449VP_PROPERTY_FUNCTIONAL_SDOPC(LRINT) 450END_REGISTER_VP(vp_lrint, VP_LRINT) 451 452// llvm.vp.llrint(x,mask,vlen) 453BEGIN_REGISTER_VP(vp_llrint, 1, 2, VP_LLRINT, 0) 454VP_PROPERTY_FUNCTIONAL_INTRINSIC(llrint) 455VP_PROPERTY_FUNCTIONAL_SDOPC(LLRINT) 456END_REGISTER_VP(vp_llrint, VP_LLRINT) 457 458///// } Floating-Point Arithmetic 459 460///// Type Casts { 461// Specialized helper macro for type conversions. 462// <operation>(%x, %mask, %evl). 463#ifdef HELPER_REGISTER_FP_CAST_VP 464#error \ 465 "The internal helper macro HELPER_REGISTER_FP_CAST_VP is already defined!" 466#endif 467#define HELPER_REGISTER_FP_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \ 468 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \ 469 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 470 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 471 VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_##OPSUFFIX) \ 472 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 473 474// llvm.vp.fptoui(x,mask,vlen) 475HELPER_REGISTER_FP_CAST_VP(fptoui, VP_FP_TO_UINT, FPToUI, FP_TO_UINT) 476 477// llvm.vp.fptosi(x,mask,vlen) 478HELPER_REGISTER_FP_CAST_VP(fptosi, VP_FP_TO_SINT, FPToSI, FP_TO_SINT) 479 480// llvm.vp.uitofp(x,mask,vlen) 481HELPER_REGISTER_FP_CAST_VP(uitofp, VP_UINT_TO_FP, UIToFP, UINT_TO_FP) 482 483// llvm.vp.sitofp(x,mask,vlen) 484HELPER_REGISTER_FP_CAST_VP(sitofp, VP_SINT_TO_FP, SIToFP, SINT_TO_FP) 485 486// llvm.vp.fptrunc(x,mask,vlen) 487HELPER_REGISTER_FP_CAST_VP(fptrunc, VP_FP_ROUND, FPTrunc, FP_ROUND) 488 489// llvm.vp.fpext(x,mask,vlen) 490HELPER_REGISTER_FP_CAST_VP(fpext, VP_FP_EXTEND, FPExt, FP_EXTEND) 491 492#undef HELPER_REGISTER_FP_CAST_VP 493 494// Specialized helper macro for integer type conversions. 495// <operation>(%x, %mask, %evl). 496#ifdef HELPER_REGISTER_INT_CAST_VP 497#error \ 498 "The internal helper macro HELPER_REGISTER_INT_CAST_VP is already defined!" 499#endif 500#define HELPER_REGISTER_INT_CAST_VP(OPSUFFIX, VPSD, IROPC, SDOPC) \ 501 BEGIN_REGISTER_VP(vp_##OPSUFFIX, 1, 2, VPSD, -1) \ 502 VP_PROPERTY_FUNCTIONAL_OPC(IROPC) \ 503 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 504 END_REGISTER_VP(vp_##OPSUFFIX, VPSD) 505 506// llvm.vp.trunc(x,mask,vlen) 507HELPER_REGISTER_INT_CAST_VP(trunc, VP_TRUNCATE, Trunc, TRUNCATE) 508 509// llvm.vp.zext(x,mask,vlen) 510HELPER_REGISTER_INT_CAST_VP(zext, VP_ZERO_EXTEND, ZExt, ZERO_EXTEND) 511 512// llvm.vp.sext(x,mask,vlen) 513HELPER_REGISTER_INT_CAST_VP(sext, VP_SIGN_EXTEND, SExt, SIGN_EXTEND) 514 515// llvm.vp.ptrtoint(x,mask,vlen) 516BEGIN_REGISTER_VP(vp_ptrtoint, 1, 2, VP_PTRTOINT, -1) 517VP_PROPERTY_FUNCTIONAL_OPC(PtrToInt) 518END_REGISTER_VP(vp_ptrtoint, VP_PTRTOINT) 519 520// llvm.vp.inttoptr(x,mask,vlen) 521BEGIN_REGISTER_VP(vp_inttoptr, 1, 2, VP_INTTOPTR, -1) 522VP_PROPERTY_FUNCTIONAL_OPC(IntToPtr) 523END_REGISTER_VP(vp_inttoptr, VP_INTTOPTR) 524 525#undef HELPER_REGISTER_INT_CAST_VP 526 527///// } Type Casts 528 529///// Comparisons { 530 531// VP_SETCC (ISel only) 532BEGIN_REGISTER_VP_SDNODE(VP_SETCC, 0, vp_setcc, 3, 4) 533END_REGISTER_VP_SDNODE(VP_SETCC) 534 535// llvm.vp.fcmp(x,y,cc,mask,vlen) 536BEGIN_REGISTER_VP_INTRINSIC(vp_fcmp, 3, 4) 537HELPER_MAP_VPID_TO_VPSD(vp_fcmp, VP_SETCC) 538VP_PROPERTY_FUNCTIONAL_OPC(FCmp) 539VP_PROPERTY_CONSTRAINEDFP(experimental_constrained_fcmp) 540END_REGISTER_VP_INTRINSIC(vp_fcmp) 541 542// llvm.vp.icmp(x,y,cc,mask,vlen) 543BEGIN_REGISTER_VP_INTRINSIC(vp_icmp, 3, 4) 544HELPER_MAP_VPID_TO_VPSD(vp_icmp, VP_SETCC) 545VP_PROPERTY_FUNCTIONAL_OPC(ICmp) 546END_REGISTER_VP_INTRINSIC(vp_icmp) 547 548///// } Comparisons 549 550// llvm.vp.is.fpclass(on_true,on_false,mask,vlen) 551BEGIN_REGISTER_VP(vp_is_fpclass, 2, 3, VP_IS_FPCLASS, 0) 552VP_PROPERTY_FUNCTIONAL_INTRINSIC(is_fpclass) 553END_REGISTER_VP(vp_is_fpclass, VP_IS_FPCLASS) 554 555///// Memory Operations { 556// llvm.vp.store(val,ptr,mask,vlen) 557BEGIN_REGISTER_VP_INTRINSIC(vp_store, 2, 3) 558// chain = VP_STORE chain,val,base,offset,mask,evl 559BEGIN_REGISTER_VP_SDNODE(VP_STORE, 1, vp_store, 4, 5) 560HELPER_MAP_VPID_TO_VPSD(vp_store, VP_STORE) 561VP_PROPERTY_FUNCTIONAL_OPC(Store) 562VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_store) 563END_REGISTER_VP(vp_store, VP_STORE) 564 565// llvm.experimental.vp.strided.store(val,ptr,stride,mask,vlen) 566BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_store, 3, 4) 567// chain = EXPERIMENTAL_VP_STRIDED_STORE chain,val,base,offset,stride,mask,evl 568VP_PROPERTY_NO_FUNCTIONAL 569BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_STORE, 1, experimental_vp_strided_store, 5, 6) 570HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE) 571END_REGISTER_VP(experimental_vp_strided_store, EXPERIMENTAL_VP_STRIDED_STORE) 572 573// llvm.vp.scatter(ptr,val,mask,vlen) 574BEGIN_REGISTER_VP_INTRINSIC(vp_scatter, 2, 3) 575// chain = VP_SCATTER chain,val,base,indices,scale,mask,evl 576BEGIN_REGISTER_VP_SDNODE(VP_SCATTER, 1, vp_scatter, 5, 6) 577HELPER_MAP_VPID_TO_VPSD(vp_scatter, VP_SCATTER) 578VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_scatter) 579END_REGISTER_VP(vp_scatter, VP_SCATTER) 580 581// llvm.vp.load(ptr,mask,vlen) 582BEGIN_REGISTER_VP_INTRINSIC(vp_load, 1, 2) 583// val,chain = VP_LOAD chain,base,offset,mask,evl 584BEGIN_REGISTER_VP_SDNODE(VP_LOAD, -1, vp_load, 3, 4) 585HELPER_MAP_VPID_TO_VPSD(vp_load, VP_LOAD) 586VP_PROPERTY_FUNCTIONAL_OPC(Load) 587VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_load) 588END_REGISTER_VP(vp_load, VP_LOAD) 589 590// llvm.experimental.vp.strided.load(ptr,stride,mask,vlen) 591BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_strided_load, 2, 3) 592// chain = EXPERIMENTAL_VP_STRIDED_LOAD chain,base,offset,stride,mask,evl 593VP_PROPERTY_NO_FUNCTIONAL 594BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_STRIDED_LOAD, -1, experimental_vp_strided_load, 4, 5) 595HELPER_MAP_VPID_TO_VPSD(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD) 596END_REGISTER_VP(experimental_vp_strided_load, EXPERIMENTAL_VP_STRIDED_LOAD) 597 598// llvm.vp.gather(ptr,mask,vlen) 599BEGIN_REGISTER_VP_INTRINSIC(vp_gather, 1, 2) 600// val,chain = VP_GATHER chain,base,indices,scale,mask,evl 601BEGIN_REGISTER_VP_SDNODE(VP_GATHER, -1, vp_gather, 4, 5) 602HELPER_MAP_VPID_TO_VPSD(vp_gather, VP_GATHER) 603VP_PROPERTY_FUNCTIONAL_INTRINSIC(masked_gather) 604END_REGISTER_VP(vp_gather, VP_GATHER) 605 606///// } Memory Operations 607 608///// Reductions { 609 610// Specialized helper macro for VP reductions (%start, %x, %mask, %evl). 611#ifdef HELPER_REGISTER_REDUCTION_VP 612#error \ 613 "The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!" 614#endif 615#define HELPER_REGISTER_REDUCTION_VP(VPID, VPSD, INTRIN, SDOPC) \ 616 BEGIN_REGISTER_VP(VPID, 2, 3, VPSD, 1) \ 617 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \ 618 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 619 END_REGISTER_VP(VPID, VPSD) 620 621// llvm.vp.reduce.add(start,x,mask,vlen) 622HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD, 623 vector_reduce_add, VECREDUCE_ADD) 624 625// llvm.vp.reduce.mul(start,x,mask,vlen) 626HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL, 627 vector_reduce_mul, VECREDUCE_MUL) 628 629// llvm.vp.reduce.and(start,x,mask,vlen) 630HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND, 631 vector_reduce_and, VECREDUCE_AND) 632 633// llvm.vp.reduce.or(start,x,mask,vlen) 634HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR, 635 vector_reduce_or, VECREDUCE_OR) 636 637// llvm.vp.reduce.xor(start,x,mask,vlen) 638HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR, 639 vector_reduce_xor, VECREDUCE_XOR) 640 641// llvm.vp.reduce.smax(start,x,mask,vlen) 642HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX, 643 vector_reduce_smax, VECREDUCE_SMAX) 644 645// llvm.vp.reduce.smin(start,x,mask,vlen) 646HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN, 647 vector_reduce_smin, VECREDUCE_SMIN) 648 649// llvm.vp.reduce.umax(start,x,mask,vlen) 650HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX, 651 vector_reduce_umax, VECREDUCE_UMAX) 652 653// llvm.vp.reduce.umin(start,x,mask,vlen) 654HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN, 655 vector_reduce_umin, VECREDUCE_UMIN) 656 657// llvm.vp.reduce.fmax(start,x,mask,vlen) 658HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX, 659 vector_reduce_fmax, VECREDUCE_FMAX) 660 661// llvm.vp.reduce.fmin(start,x,mask,vlen) 662HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN, 663 vector_reduce_fmin, VECREDUCE_FMIN) 664 665// llvm.vp.reduce.fmaximum(start,x,mask,vlen) 666HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmaximum, VP_REDUCE_FMAXIMUM, 667 vector_reduce_fmaximum, VECREDUCE_FMAXIMUM) 668 669// llvm.vp.reduce.fminimum(start,x,mask,vlen) 670HELPER_REGISTER_REDUCTION_VP(vp_reduce_fminimum, VP_REDUCE_FMINIMUM, 671 vector_reduce_fminimum, VECREDUCE_FMINIMUM) 672 673#undef HELPER_REGISTER_REDUCTION_VP 674 675// Specialized helper macro for VP reductions as above but with two forms: 676// sequential and reassociative. These manifest as the presence of 'reassoc' 677// fast-math flags in the IR and as two distinct ISD opcodes in the 678// SelectionDAG. 679// Note we by default map from the VP intrinsic to the SEQ ISD opcode, which 680// can then be relaxed to the non-SEQ ISD opcode if the 'reassoc' flag is set. 681#ifdef HELPER_REGISTER_REDUCTION_SEQ_VP 682#error \ 683 "The internal helper macro HELPER_REGISTER_REDUCTION_SEQ_VP is already defined!" 684#endif 685#define HELPER_REGISTER_REDUCTION_SEQ_VP(VPID, VPSD, SEQ_VPSD, SDOPC, SEQ_SDOPC, INTRIN) \ 686 BEGIN_REGISTER_VP_INTRINSIC(VPID, 2, 3) \ 687 BEGIN_REGISTER_VP_SDNODE(VPSD, 1, VPID, 2, 3) \ 688 VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) \ 689 END_REGISTER_VP_SDNODE(VPSD) \ 690 BEGIN_REGISTER_VP_SDNODE(SEQ_VPSD, 1, VPID, 2, 3) \ 691 HELPER_MAP_VPID_TO_VPSD(VPID, SEQ_VPSD) \ 692 VP_PROPERTY_FUNCTIONAL_SDOPC(SEQ_SDOPC) \ 693 END_REGISTER_VP_SDNODE(SEQ_VPSD) \ 694 VP_PROPERTY_FUNCTIONAL_INTRINSIC(INTRIN) \ 695 END_REGISTER_VP_INTRINSIC(VPID) 696 697// llvm.vp.reduce.fadd(start,x,mask,vlen) 698HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fadd, VP_REDUCE_FADD, 699 VP_REDUCE_SEQ_FADD, VECREDUCE_FADD, 700 VECREDUCE_SEQ_FADD, vector_reduce_fadd) 701 702// llvm.vp.reduce.fmul(start,x,mask,vlen) 703HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL, 704 VP_REDUCE_SEQ_FMUL, VECREDUCE_FMUL, 705 VECREDUCE_SEQ_FMUL, vector_reduce_fmul) 706 707#undef HELPER_REGISTER_REDUCTION_SEQ_VP 708 709///// } Reduction 710 711///// Shuffles { 712 713// The mask 'cond' operand of llvm.vp.select and llvm.vp.merge are not reported 714// as masks with the BEGIN_REGISTER_VP_* macros. This is because, unlike other 715// VP intrinsics, these two have a defined result on lanes where the mask is 716// false. 717// 718// llvm.vp.select(cond,on_true,on_false,vlen) 719BEGIN_REGISTER_VP(vp_select, std::nullopt, 3, VP_SELECT, -1) 720VP_PROPERTY_FUNCTIONAL_OPC(Select) 721VP_PROPERTY_FUNCTIONAL_SDOPC(VSELECT) 722END_REGISTER_VP(vp_select, VP_SELECT) 723 724// llvm.vp.merge(cond,on_true,on_false,pivot) 725BEGIN_REGISTER_VP(vp_merge, std::nullopt, 3, VP_MERGE, -1) 726VP_PROPERTY_NO_FUNCTIONAL 727END_REGISTER_VP(vp_merge, VP_MERGE) 728 729BEGIN_REGISTER_VP(experimental_vp_splice, 3, 5, EXPERIMENTAL_VP_SPLICE, -1) 730VP_PROPERTY_NO_FUNCTIONAL 731END_REGISTER_VP(experimental_vp_splice, EXPERIMENTAL_VP_SPLICE) 732 733// llvm.experimental.vp.reverse(x,mask,vlen) 734BEGIN_REGISTER_VP(experimental_vp_reverse, 1, 2, 735 EXPERIMENTAL_VP_REVERSE, -1) 736VP_PROPERTY_NO_FUNCTIONAL 737END_REGISTER_VP(experimental_vp_reverse, EXPERIMENTAL_VP_REVERSE) 738 739///// } Shuffles 740 741// llvm.vp.splat(val,mask,vlen) 742BEGIN_REGISTER_VP_INTRINSIC(experimental_vp_splat, 1, 2) 743BEGIN_REGISTER_VP_SDNODE(EXPERIMENTAL_VP_SPLAT, -1, experimental_vp_splat, 1, 2) 744VP_PROPERTY_NO_FUNCTIONAL 745HELPER_MAP_VPID_TO_VPSD(experimental_vp_splat, EXPERIMENTAL_VP_SPLAT) 746END_REGISTER_VP(experimental_vp_splat, EXPERIMENTAL_VP_SPLAT) 747 748#undef BEGIN_REGISTER_VP 749#undef BEGIN_REGISTER_VP_INTRINSIC 750#undef BEGIN_REGISTER_VP_SDNODE 751#undef END_REGISTER_VP 752#undef END_REGISTER_VP_INTRINSIC 753#undef END_REGISTER_VP_SDNODE 754#undef HELPER_MAP_VPID_TO_VPSD 755#undef VP_PROPERTY_BINARYOP 756#undef VP_PROPERTY_CONSTRAINEDFP 757#undef VP_PROPERTY_FUNCTIONAL_INTRINSIC 758#undef VP_PROPERTY_FUNCTIONAL_OPC 759#undef VP_PROPERTY_FUNCTIONAL_SDOPC 760#undef VP_PROPERTY_NO_FUNCTIONAL 761