1//===-- BPFInstrInfo.td - Target Description for BPF Target ---------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file describes the BPF instructions in TableGen format. 10// 11//===----------------------------------------------------------------------===// 12 13include "BPFInstrFormats.td" 14 15// Instruction Operands and Patterns 16 17// These are target-independent nodes, but have target-specific formats. 18def SDT_BPFCallSeqStart : SDCallSeqStart<[SDTCisVT<0, iPTR>, 19 SDTCisVT<1, iPTR>]>; 20def SDT_BPFCallSeqEnd : SDCallSeqEnd<[SDTCisVT<0, iPTR>, SDTCisVT<1, iPTR>]>; 21def SDT_BPFCall : SDTypeProfile<0, -1, [SDTCisVT<0, iPTR>]>; 22def SDT_BPFSetFlag : SDTypeProfile<0, 3, [SDTCisSameAs<0, 1>]>; 23def SDT_BPFSelectCC : SDTypeProfile<1, 5, [SDTCisSameAs<1, 2>, 24 SDTCisSameAs<0, 4>, 25 SDTCisSameAs<4, 5>]>; 26def SDT_BPFBrCC : SDTypeProfile<0, 4, [SDTCisSameAs<0, 1>, 27 SDTCisVT<3, OtherVT>]>; 28def SDT_BPFWrapper : SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>, 29 SDTCisPtrTy<0>]>; 30def SDT_BPFMEMCPY : SDTypeProfile<0, 4, [SDTCisVT<0, i64>, 31 SDTCisVT<1, i64>, 32 SDTCisVT<2, i64>, 33 SDTCisVT<3, i64>]>; 34 35def BPFcall : SDNode<"BPFISD::CALL", SDT_BPFCall, 36 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue, 37 SDNPVariadic]>; 38def BPFretglue : SDNode<"BPFISD::RET_GLUE", SDTNone, 39 [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>; 40def BPFcallseq_start: SDNode<"ISD::CALLSEQ_START", SDT_BPFCallSeqStart, 41 [SDNPHasChain, SDNPOutGlue]>; 42def BPFcallseq_end : SDNode<"ISD::CALLSEQ_END", SDT_BPFCallSeqEnd, 43 [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>; 44def BPFbrcc : SDNode<"BPFISD::BR_CC", SDT_BPFBrCC, 45 [SDNPHasChain, SDNPOutGlue, SDNPInGlue]>; 46 47def BPFselectcc : SDNode<"BPFISD::SELECT_CC", SDT_BPFSelectCC>; 48def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>; 49def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY, 50 [SDNPHasChain, SDNPInGlue, SDNPOutGlue, 51 SDNPMayStore, SDNPMayLoad]>; 52def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">; 53def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">; 54def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">; 55def BPFNoALU32 : Predicate<"!Subtarget->getHasAlu32()">; 56def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">; 57def BPFHasMovsx : Predicate<"Subtarget->hasMovsx()">; 58def BPFHasBswap : Predicate<"Subtarget->hasBswap()">; 59def BPFHasSdivSmod : Predicate<"Subtarget->hasSdivSmod()">; 60def BPFNoMovsx : Predicate<"!Subtarget->hasMovsx()">; 61def BPFNoBswap : Predicate<"!Subtarget->hasBswap()">; 62def BPFHasStoreImm : Predicate<"Subtarget->hasStoreImm()">; 63 64class ImmediateAsmOperand<string name> : AsmOperandClass { 65 let Name = name; 66 let RenderMethod = "addImmOperands"; 67 let DiagnosticType = !strconcat("Invalid", name); 68} 69 70def SImm16AsmOperand : ImmediateAsmOperand<"SImm16">; 71 72def brtarget : Operand<OtherVT> { 73 let PrintMethod = "printBrTargetOperand"; 74 let ParserMatchClass = ImmediateAsmOperand<"BrTarget">; 75} 76def calltarget : Operand<i64>; 77 78def u64imm : Operand<i64> { 79 let PrintMethod = "printImm64Operand"; 80} 81 82def s16imm : Operand<i16> { 83 let ParserMatchClass = SImm16AsmOperand; 84} 85 86def gpr_or_imm : Operand<i64>; 87 88def i64immSExt32 : PatLeaf<(i64 imm), 89 [{return isInt<32>(N->getSExtValue()); }]>; 90def i32immSExt32 : PatLeaf<(i32 imm), 91 [{return isInt<32>(N->getSExtValue()); }]>; 92def i64immZExt32 : PatLeaf<(i64 imm), 93 [{return isUInt<32>(N->getZExtValue()); }]>; 94 95def imm_to_i64 : SDNodeXForm<timm, [{ 96 return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i64); 97}]>; 98 99// Addressing modes. 100def ADDRri : ComplexPattern<i64, 2, "SelectAddr", [], []>; 101def FIri : ComplexPattern<i64, 2, "SelectFIAddr", [add, or], []>; 102 103// Address operands 104def MEMri : Operand<i64> { 105 let PrintMethod = "printMemOperand"; 106 let EncoderMethod = "getMemoryOpValue"; 107 let DecoderMethod = "decodeMemoryOpValue"; 108 let MIOperandInfo = (ops GPR, s16imm); 109} 110 111// Conditional code predicates - used for pattern matching for jump instructions 112def BPF_CC_EQ : PatLeaf<(i64 imm), 113 [{return (N->getZExtValue() == ISD::SETEQ);}]>; 114def BPF_CC_NE : PatLeaf<(i64 imm), 115 [{return (N->getZExtValue() == ISD::SETNE);}]>; 116def BPF_CC_GE : PatLeaf<(i64 imm), 117 [{return (N->getZExtValue() == ISD::SETGE);}]>; 118def BPF_CC_GT : PatLeaf<(i64 imm), 119 [{return (N->getZExtValue() == ISD::SETGT);}]>; 120def BPF_CC_GTU : PatLeaf<(i64 imm), 121 [{return (N->getZExtValue() == ISD::SETUGT);}]>; 122def BPF_CC_GEU : PatLeaf<(i64 imm), 123 [{return (N->getZExtValue() == ISD::SETUGE);}]>; 124def BPF_CC_LE : PatLeaf<(i64 imm), 125 [{return (N->getZExtValue() == ISD::SETLE);}]>; 126def BPF_CC_LT : PatLeaf<(i64 imm), 127 [{return (N->getZExtValue() == ISD::SETLT);}]>; 128def BPF_CC_LTU : PatLeaf<(i64 imm), 129 [{return (N->getZExtValue() == ISD::SETULT);}]>; 130def BPF_CC_LEU : PatLeaf<(i64 imm), 131 [{return (N->getZExtValue() == ISD::SETULE);}]>; 132def BPF_CC_EQ_32 : PatLeaf<(i32 imm), 133 [{return (N->getZExtValue() == ISD::SETEQ);}]>; 134def BPF_CC_NE_32 : PatLeaf<(i32 imm), 135 [{return (N->getZExtValue() == ISD::SETNE);}]>; 136def BPF_CC_GE_32 : PatLeaf<(i32 imm), 137 [{return (N->getZExtValue() == ISD::SETGE);}]>; 138def BPF_CC_GT_32 : PatLeaf<(i32 imm), 139 [{return (N->getZExtValue() == ISD::SETGT);}]>; 140def BPF_CC_GTU_32 : PatLeaf<(i32 imm), 141 [{return (N->getZExtValue() == ISD::SETUGT);}]>; 142def BPF_CC_GEU_32 : PatLeaf<(i32 imm), 143 [{return (N->getZExtValue() == ISD::SETUGE);}]>; 144def BPF_CC_LE_32 : PatLeaf<(i32 imm), 145 [{return (N->getZExtValue() == ISD::SETLE);}]>; 146def BPF_CC_LT_32 : PatLeaf<(i32 imm), 147 [{return (N->getZExtValue() == ISD::SETLT);}]>; 148def BPF_CC_LTU_32 : PatLeaf<(i32 imm), 149 [{return (N->getZExtValue() == ISD::SETULT);}]>; 150def BPF_CC_LEU_32 : PatLeaf<(i32 imm), 151 [{return (N->getZExtValue() == ISD::SETULE);}]>; 152def NoCond : PatLeaf<(vt)> {} 153 154// For arithmetic and jump instructions the 8-bit 'code' 155// field is divided into three parts: 156// 157// +----------------+--------+--------------------+ 158// | 4 bits | 1 bit | 3 bits | 159// | operation code | source | instruction class | 160// +----------------+--------+--------------------+ 161// (MSB) (LSB) 162class TYPE_ALU_JMP<bits<4> op, bits<1> srctype, 163 dag outs, dag ins, string asmstr, list<dag> pattern> 164 : InstBPF<outs, ins, asmstr, pattern> { 165 166 let Inst{63-60} = op; 167 let Inst{59} = srctype; 168} 169 170//For load and store instructions the 8-bit 'code' field is divided as: 171// 172// +--------+--------+-------------------+ 173// | 3 bits | 2 bits | 3 bits | 174// | mode | size | instruction class | 175// +--------+--------+-------------------+ 176// (MSB) (LSB) 177class TYPE_LD_ST<bits<3> mode, bits<2> size, 178 dag outs, dag ins, string asmstr, list<dag> pattern> 179 : InstBPF<outs, ins, asmstr, pattern> { 180 181 let Inst{63-61} = mode; 182 let Inst{60-59} = size; 183} 184 185// jump instructions 186class JMP_RR<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 187 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, 188 (outs), 189 (ins GPR:$dst, GPR:$src, brtarget:$BrDst), 190 "if $dst "#OpcodeStr#" $src goto $BrDst", 191 [(BPFbrcc i64:$dst, i64:$src, Cond, bb:$BrDst)]> { 192 bits<4> dst; 193 bits<4> src; 194 bits<16> BrDst; 195 196 let Inst{55-52} = src; 197 let Inst{51-48} = dst; 198 let Inst{47-32} = BrDst; 199 let BPFClass = BPF_JMP; 200} 201 202class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 203 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 204 (outs), 205 (ins GPR:$dst, i64imm:$imm, brtarget:$BrDst), 206 "if $dst "#OpcodeStr#" $imm goto $BrDst", 207 [(BPFbrcc i64:$dst, i64immSExt32:$imm, Cond, bb:$BrDst)]> { 208 bits<4> dst; 209 bits<16> BrDst; 210 bits<32> imm; 211 212 let Inst{51-48} = dst; 213 let Inst{47-32} = BrDst; 214 let Inst{31-0} = imm; 215 let BPFClass = BPF_JMP; 216} 217 218class JMP_JCOND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 219 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 220 (outs), 221 (ins brtarget:$BrDst), 222 !strconcat(OpcodeStr, " $BrDst"), 223 Pattern> { 224 bits<16> BrDst; 225 226 let Inst{47-32} = BrDst; 227 let BPFClass = BPF_JMP; 228} 229 230class JMP_RR_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 231 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, 232 (outs), 233 (ins GPR32:$dst, GPR32:$src, brtarget:$BrDst), 234 "if $dst "#OpcodeStr#" $src goto $BrDst", 235 [(BPFbrcc i32:$dst, i32:$src, Cond, bb:$BrDst)]> { 236 bits<4> dst; 237 bits<4> src; 238 bits<16> BrDst; 239 240 let Inst{55-52} = src; 241 let Inst{51-48} = dst; 242 let Inst{47-32} = BrDst; 243 let BPFClass = BPF_JMP32; 244} 245 246class JMP_RI_32<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond> 247 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 248 (outs), 249 (ins GPR32:$dst, i32imm:$imm, brtarget:$BrDst), 250 "if $dst "#OpcodeStr#" $imm goto $BrDst", 251 [(BPFbrcc i32:$dst, i32immSExt32:$imm, Cond, bb:$BrDst)]> { 252 bits<4> dst; 253 bits<16> BrDst; 254 bits<32> imm; 255 256 let Inst{51-48} = dst; 257 let Inst{47-32} = BrDst; 258 let Inst{31-0} = imm; 259 let BPFClass = BPF_JMP32; 260} 261 262multiclass J<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond, PatLeaf Cond32> { 263 def _rr : JMP_RR<Opc, OpcodeStr, Cond>; 264 def _ri : JMP_RI<Opc, OpcodeStr, Cond>; 265 def _rr_32 : JMP_RR_32<Opc, OpcodeStr, Cond32>; 266 def _ri_32 : JMP_RI_32<Opc, OpcodeStr, Cond32>; 267} 268 269let isBranch = 1, isTerminator = 1, hasDelaySlot=0 in { 270// cmp+goto instructions 271defm JEQ : J<BPF_JEQ, "==", BPF_CC_EQ, BPF_CC_EQ_32>; 272defm JUGT : J<BPF_JGT, ">", BPF_CC_GTU, BPF_CC_GTU_32>; 273defm JUGE : J<BPF_JGE, ">=", BPF_CC_GEU, BPF_CC_GEU_32>; 274defm JNE : J<BPF_JNE, "!=", BPF_CC_NE, BPF_CC_NE_32>; 275defm JSGT : J<BPF_JSGT, "s>", BPF_CC_GT, BPF_CC_GT_32>; 276defm JSGE : J<BPF_JSGE, "s>=", BPF_CC_GE, BPF_CC_GE_32>; 277defm JULT : J<BPF_JLT, "<", BPF_CC_LTU, BPF_CC_LTU_32>; 278defm JULE : J<BPF_JLE, "<=", BPF_CC_LEU, BPF_CC_LEU_32>; 279defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>; 280defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>; 281defm JSET : J<BPF_JSET, "&", NoCond, NoCond>; 282def JCOND : JMP_JCOND<BPF_JCOND, "may_goto", []>; 283} 284 285// ALU instructions 286class ALU_RI<BPFOpClass Class, BPFArithOp Opc, int off, 287 dag outs, dag ins, string asmstr, list<dag> pattern> 288 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, outs, ins, asmstr, pattern> { 289 bits<4> dst; 290 bits<32> imm; 291 292 let Inst{51-48} = dst; 293 let Inst{47-32} = off; 294 let Inst{31-0} = imm; 295 let BPFClass = Class; 296} 297 298class ALU_RR<BPFOpClass Class, BPFArithOp Opc, int off, 299 dag outs, dag ins, string asmstr, list<dag> pattern> 300 : TYPE_ALU_JMP<Opc.Value, BPF_X.Value, outs, ins, asmstr, pattern> { 301 bits<4> dst; 302 bits<4> src; 303 304 let Inst{55-52} = src; 305 let Inst{51-48} = dst; 306 let Inst{47-32} = off; 307 let BPFClass = Class; 308} 309 310multiclass ALU<BPFArithOp Opc, int off, string OpcodeStr, SDNode OpNode> { 311 def _rr : ALU_RR<BPF_ALU64, Opc, off, 312 (outs GPR:$dst), 313 (ins GPR:$src2, GPR:$src), 314 "$dst "#OpcodeStr#" $src", 315 [(set GPR:$dst, (OpNode i64:$src2, i64:$src))]>; 316 def _ri : ALU_RI<BPF_ALU64, Opc, off, 317 (outs GPR:$dst), 318 (ins GPR:$src2, i64imm:$imm), 319 "$dst "#OpcodeStr#" $imm", 320 [(set GPR:$dst, (OpNode GPR:$src2, i64immSExt32:$imm))]>; 321 def _rr_32 : ALU_RR<BPF_ALU, Opc, off, 322 (outs GPR32:$dst), 323 (ins GPR32:$src2, GPR32:$src), 324 "$dst "#OpcodeStr#" $src", 325 [(set GPR32:$dst, (OpNode i32:$src2, i32:$src))]>; 326 def _ri_32 : ALU_RI<BPF_ALU, Opc, off, 327 (outs GPR32:$dst), 328 (ins GPR32:$src2, i32imm:$imm), 329 "$dst "#OpcodeStr#" $imm", 330 [(set GPR32:$dst, (OpNode GPR32:$src2, i32immSExt32:$imm))]>; 331} 332 333let Constraints = "$dst = $src2" in { 334let isAsCheapAsAMove = 1 in { 335 defm ADD : ALU<BPF_ADD, 0, "+=", add>; 336 defm SUB : ALU<BPF_SUB, 0, "-=", sub>; 337 defm OR : ALU<BPF_OR, 0, "|=", or>; 338 defm AND : ALU<BPF_AND, 0, "&=", and>; 339 defm SLL : ALU<BPF_LSH, 0, "<<=", shl>; 340 defm SRL : ALU<BPF_RSH, 0, ">>=", srl>; 341 defm XOR : ALU<BPF_XOR, 0, "^=", xor>; 342 defm SRA : ALU<BPF_ARSH, 0, "s>>=", sra>; 343} 344 defm MUL : ALU<BPF_MUL, 0, "*=", mul>; 345 defm DIV : ALU<BPF_DIV, 0, "/=", udiv>; 346 defm MOD : ALU<BPF_MOD, 0, "%=", urem>; 347 348 let Predicates = [BPFHasSdivSmod] in { 349 defm SDIV : ALU<BPF_DIV, 1, "s/=", sdiv>; 350 defm SMOD : ALU<BPF_MOD, 1, "s%=", srem>; 351 } 352} 353 354class NEG_RR<BPFOpClass Class, BPFArithOp Opc, 355 dag outs, dag ins, string asmstr, list<dag> pattern> 356 : TYPE_ALU_JMP<Opc.Value, 0, outs, ins, asmstr, pattern> { 357 bits<4> dst; 358 359 let Inst{51-48} = dst; 360 let BPFClass = Class; 361} 362 363let Constraints = "$dst = $src", isAsCheapAsAMove = 1 in { 364 def NEG_64: NEG_RR<BPF_ALU64, BPF_NEG, (outs GPR:$dst), (ins GPR:$src), 365 "$dst = -$src", 366 [(set GPR:$dst, (ineg i64:$src))]>; 367 def NEG_32: NEG_RR<BPF_ALU, BPF_NEG, (outs GPR32:$dst), (ins GPR32:$src), 368 "$dst = -$src", 369 [(set GPR32:$dst, (ineg i32:$src))]>; 370} 371 372class LD_IMM64<bits<4> Pseudo, string OpcodeStr> 373 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 374 (outs GPR:$dst), 375 (ins u64imm:$imm), 376 "$dst "#OpcodeStr#" ${imm} ll", 377 [(set GPR:$dst, (i64 imm:$imm))]> { 378 379 bits<4> dst; 380 bits<64> imm; 381 382 let Inst{51-48} = dst; 383 let Inst{55-52} = Pseudo; 384 let Inst{47-32} = 0; 385 let Inst{31-0} = imm{31-0}; 386 let BPFClass = BPF_LD; 387} 388 389let isReMaterializable = 1, isAsCheapAsAMove = 1 in { 390def LD_imm64 : LD_IMM64<0, "=">; 391def MOV_rr : ALU_RR<BPF_ALU64, BPF_MOV, 0, 392 (outs GPR:$dst), 393 (ins GPR:$src), 394 "$dst = $src", 395 []>; 396def MOV_ri : ALU_RI<BPF_ALU64, BPF_MOV, 0, 397 (outs GPR:$dst), 398 (ins i64imm:$imm), 399 "$dst = $imm", 400 [(set GPR:$dst, (i64 i64immSExt32:$imm))]>; 401def MOV_rr_32 : ALU_RR<BPF_ALU, BPF_MOV, 0, 402 (outs GPR32:$dst), 403 (ins GPR32:$src), 404 "$dst = $src", 405 []>; 406def MOV_ri_32 : ALU_RI<BPF_ALU, BPF_MOV, 0, 407 (outs GPR32:$dst), 408 (ins i32imm:$imm), 409 "$dst = $imm", 410 [(set GPR32:$dst, (i32 i32immSExt32:$imm))]>; 411 412let Predicates = [BPFHasMovsx] in { 413 def MOVSX_rr_8 : ALU_RR<BPF_ALU64, BPF_MOV, 8, 414 (outs GPR:$dst), (ins GPR:$src), 415 "$dst = (s8)$src", 416 [(set GPR:$dst, (sext_inreg GPR:$src, i8))]>; 417 def MOVSX_rr_16 : ALU_RR<BPF_ALU64, BPF_MOV, 16, 418 (outs GPR:$dst), (ins GPR:$src), 419 "$dst = (s16)$src", 420 [(set GPR:$dst, (sext_inreg GPR:$src, i16))]>; 421 def MOVSX_rr_32 : ALU_RR<BPF_ALU64, BPF_MOV, 32, 422 (outs GPR:$dst), (ins GPR:$src), 423 "$dst = (s32)$src", 424 [(set GPR:$dst, (sext_inreg GPR:$src, i32))]>; 425 def MOVSX_rr_32_8 : ALU_RR<BPF_ALU, BPF_MOV, 8, 426 (outs GPR32:$dst), (ins GPR32:$src), 427 "$dst = (s8)$src", 428 [(set GPR32:$dst, (sext_inreg GPR32:$src, i8))]>; 429 def MOVSX_rr_32_16 : ALU_RR<BPF_ALU, BPF_MOV, 16, 430 (outs GPR32:$dst), (ins GPR32:$src), 431 "$dst = (s16)$src", 432 [(set GPR32:$dst, (sext_inreg GPR32:$src, i16))]>; 433} 434} 435 436def ADDR_SPACE_CAST 437 : ALU_RR<BPF_ALU64, BPF_MOV, 64, 438 (outs GPR:$dst), 439 (ins GPR:$src, i64imm:$dst_as, i64imm:$src_as), 440 "$dst = addr_space_cast($src, $dst_as, $src_as)", 441 []> { 442 bits<64> dst_as; 443 bits<64> src_as; 444 445 let Inst{47-32} = 1; 446 let Inst{31-16} = dst_as{15-0}; 447 let Inst{15-0} = src_as{15-0}; 448} 449 450def SrcAddrSpace : SDNodeXForm<addrspacecast, [{ 451 return CurDAG->getTargetConstant( 452 cast<AddrSpaceCastSDNode>(N)->getSrcAddressSpace(), 453 SDLoc(N), MVT::i64); 454}]>; 455 456def DstAddrSpace : SDNodeXForm<addrspacecast, [{ 457 return CurDAG->getTargetConstant( 458 cast<AddrSpaceCastSDNode>(N)->getDestAddressSpace(), 459 SDLoc(N), MVT::i64); 460}]>; 461 462def : Pat<(addrspacecast:$this GPR:$src), 463 (ADDR_SPACE_CAST $src, (DstAddrSpace $this), (SrcAddrSpace $this))>; 464 465def FI_ri 466 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 467 (outs GPR:$dst), 468 (ins MEMri:$addr), 469 "lea\t$dst, $addr", 470 [(set i64:$dst, FIri:$addr)]> { 471 // This is a tentative instruction, and will be replaced 472 // with MOV_rr and ADD_ri in PEI phase 473 let Inst{51-48} = 0; 474 let Inst{55-52} = 2; 475 let Inst{47-32} = 0; 476 let Inst{31-0} = 0; 477 let BPFClass = BPF_LD; 478 bit isPseudo = true; 479} 480 481def LD_pseudo 482 : TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value, 483 (outs GPR:$dst), 484 (ins i64imm:$pseudo, u64imm:$imm), 485 "ld_pseudo\t$dst, $pseudo, $imm", 486 [(set GPR:$dst, (int_bpf_pseudo imm:$pseudo, imm:$imm))]> { 487 488 bits<4> dst; 489 bits<64> imm; 490 bits<4> pseudo; 491 492 let Inst{51-48} = dst; 493 let Inst{55-52} = pseudo; 494 let Inst{47-32} = 0; 495 let Inst{31-0} = imm{31-0}; 496 let BPFClass = BPF_LD; 497} 498 499// STORE instructions 500class STORE<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern> 501 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 502 (outs), 503 (ins GPR:$src, MEMri:$addr), 504 "*("#OpcodeStr#" *)($addr) = $src", 505 Pattern> { 506 bits<4> src; 507 bits<20> addr; 508 509 let Inst{51-48} = addr{19-16}; // base reg 510 let Inst{55-52} = src; 511 let Inst{47-32} = addr{15-0}; // offset 512 let BPFClass = BPF_STX; 513} 514 515class STOREi64<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode> 516 : STORE<Opc, OpcodeStr, [(OpNode GPR:$src, ADDRri:$addr)]>; 517 518let Predicates = [BPFNoALU32] in { 519 def STW : STOREi64<BPF_W, "u32", truncstorei32>; 520 def STH : STOREi64<BPF_H, "u16", truncstorei16>; 521 def STB : STOREi64<BPF_B, "u8", truncstorei8>; 522} 523def STD : STOREi64<BPF_DW, "u64", store>; 524 525class STORE_imm<BPFWidthModifer SizeOp, 526 string OpcodeStr, dag Pattern> 527 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 528 (outs), 529 (ins i64imm:$imm, MEMri:$addr), 530 "*("#OpcodeStr#" *)($addr) = $imm", 531 [Pattern]> { 532 bits<20> addr; 533 bits<32> imm; 534 535 let Inst{51-48} = addr{19-16}; // base reg 536 let Inst{47-32} = addr{15-0}; // offset 537 let Inst{31-0} = imm; 538 let BPFClass = BPF_ST; 539} 540 541let Predicates = [BPFHasStoreImm] in { 542 // Opcode (BPF_ST | BPF_MEM | BPF_DW) implies sign extension for 543 // value stored to memory: 544 // - it is fine to generate such write when immediate is -1 545 // - it is incorrect to generate such write when immediate is 546 // +0xffff_ffff. 547 // 548 // In the latter case two instructions would be generated instead of 549 // one BPF_ST: 550 // rA = 0xffffffff ll ; LD_imm64 551 // *(u64 *)(rB + 0) = rA ; STX 552 // 553 // For BPF_{B,H,W} the size of value stored matches size of the immediate. 554 def STD_imm : STORE_imm<BPF_DW, "u64", (store (i64 i64immSExt32:$imm), ADDRri:$addr)>; 555 def STW_imm : STORE_imm<BPF_W, "u32", (truncstorei32 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 556 def STH_imm : STORE_imm<BPF_H, "u16", (truncstorei16 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 557 def STB_imm : STORE_imm<BPF_B, "u8", (truncstorei8 (i64 i64immZExt32:$imm), ADDRri:$addr)>; 558} 559 560let Predicates = [BPFHasALU32, BPFHasStoreImm] in { 561 def : Pat<(store (i32 imm:$src), ADDRri:$dst), 562 (STW_imm (imm_to_i64 $src), ADDRri:$dst)>; 563 def : Pat<(truncstorei16 (i32 imm:$src), ADDRri:$dst), 564 (STH_imm (imm_to_i64 imm:$src), ADDRri:$dst)>; 565 def : Pat<(truncstorei8 (i32 imm:$src), ADDRri:$dst), 566 (STB_imm (imm_to_i64 imm:$src), ADDRri:$dst)>; 567} 568 569// LOAD instructions 570class LOAD<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern> 571 : TYPE_LD_ST<ModOp.Value, SizeOp.Value, 572 (outs GPR:$dst), 573 (ins MEMri:$addr), 574 "$dst = *("#OpcodeStr#" *)($addr)", 575 Pattern> { 576 bits<4> dst; 577 bits<20> addr; 578 579 let Inst{51-48} = dst; 580 let Inst{55-52} = addr{19-16}; 581 let Inst{47-32} = addr{15-0}; 582 let BPFClass = BPF_LDX; 583} 584 585class LOADi64<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode> 586 : LOAD<SizeOp, ModOp, OpcodeStr, [(set i64:$dst, (OpNode ADDRri:$addr))]>; 587 588let isCodeGenOnly = 1 in { 589 class CORE_LD<RegisterClass RegClass, string Sz> 590 : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value, 591 (outs RegClass:$dst), 592 (ins u64imm:$opcode, GPR:$src, u64imm:$offset), 593 "$dst = core_ld"#Sz#"($opcode, $src, $offset)", 594 []>; 595 def CORE_LD64 : CORE_LD<GPR, "64">; 596 def CORE_LD32 : CORE_LD<GPR32, "32">; 597 def CORE_ST : TYPE_LD_ST<BPF_MEM.Value, BPF_W.Value, 598 (outs), 599 (ins gpr_or_imm:$src, u64imm:$opcode, GPR:$ptr, u64imm:$offset), 600 "core_st($src, $opcode, $ptr, $offset)", 601 []>; 602 let Constraints = "$dst = $src" in { 603 def CORE_SHIFT : ALU_RR<BPF_ALU64, BPF_LSH, 0, 604 (outs GPR:$dst), 605 (ins u64imm:$opcode, GPR:$src, u64imm:$offset), 606 "$dst = core_shift($opcode, $src, $offset)", 607 []>; 608 } 609} 610 611let Predicates = [BPFNoALU32] in { 612 def LDW : LOADi64<BPF_W, BPF_MEM, "u32", zextloadi32>; 613 def LDH : LOADi64<BPF_H, BPF_MEM, "u16", zextloadi16>; 614 def LDB : LOADi64<BPF_B, BPF_MEM, "u8", zextloadi8>; 615} 616 617let Predicates = [BPFHasLdsx] in { 618 def LDWSX : LOADi64<BPF_W, BPF_MEMSX, "s32", sextloadi32>; 619 def LDHSX : LOADi64<BPF_H, BPF_MEMSX, "s16", sextloadi16>; 620 def LDBSX : LOADi64<BPF_B, BPF_MEMSX, "s8", sextloadi8>; 621} 622 623def LDD : LOADi64<BPF_DW, BPF_MEM, "u64", load>; 624 625class BRANCH<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 626 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 627 (outs), 628 (ins brtarget:$BrDst), 629 !strconcat(OpcodeStr, " $BrDst"), 630 Pattern> { 631 bits<16> BrDst; 632 633 let Inst{47-32} = BrDst; 634 let BPFClass = BPF_JMP; 635} 636 637class BRANCH_LONG<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern> 638 : TYPE_ALU_JMP<Opc.Value, BPF_K.Value, 639 (outs), 640 (ins brtarget:$BrDst), 641 !strconcat(OpcodeStr, " $BrDst"), 642 Pattern> { 643 bits<32> BrDst; 644 645 let Inst{31-0} = BrDst; 646 let BPFClass = BPF_JMP32; 647} 648 649class CALL<string OpcodeStr> 650 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_K.Value, 651 (outs), 652 (ins calltarget:$BrDst), 653 !strconcat(OpcodeStr, " $BrDst"), 654 []> { 655 bits<32> BrDst; 656 657 let Inst{31-0} = BrDst; 658 let BPFClass = BPF_JMP; 659} 660 661class CALLX<string OpcodeStr> 662 : TYPE_ALU_JMP<BPF_CALL.Value, BPF_X.Value, 663 (outs), 664 (ins GPR:$BrDst), 665 !strconcat(OpcodeStr, " $BrDst"), 666 []> { 667 bits<4> BrDst; 668 669 let Inst{51-48} = BrDst; 670 let BPFClass = BPF_JMP; 671} 672 673// Jump always 674let isBranch = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1 in { 675 def JMP : BRANCH<BPF_JA, "goto", [(br bb:$BrDst)]>; 676 def JMPL : BRANCH_LONG<BPF_JA, "gotol", []>; 677} 678 679// Jump and link 680let isCall=1, hasDelaySlot=0, Uses = [R11] in { 681 def JAL : CALL<"call">; 682 def JALX : CALLX<"callx">; 683} 684 685class NOP_I<string OpcodeStr> 686 : TYPE_ALU_JMP<BPF_MOV.Value, BPF_X.Value, 687 (outs), 688 (ins i32imm:$imm), 689 !strconcat(OpcodeStr, "\t$imm"), 690 []> { 691 // mov r0, r0 == nop 692 let Inst{55-52} = 0; 693 let Inst{51-48} = 0; 694 let BPFClass = BPF_ALU64; 695} 696 697let hasSideEffects = 0, isCodeGenOnly = 1 in 698 def NOP : NOP_I<"nop">; 699 700class RET<string OpcodeStr> 701 : TYPE_ALU_JMP<BPF_EXIT.Value, BPF_K.Value, 702 (outs), 703 (ins), 704 !strconcat(OpcodeStr, ""), 705 [(BPFretglue)]> { 706 let Inst{31-0} = 0; 707 let BPFClass = BPF_JMP; 708} 709 710let isReturn = 1, isTerminator = 1, hasDelaySlot=0, isBarrier = 1, 711 isNotDuplicable = 1 in { 712 def RET : RET<"exit">; 713} 714 715// ADJCALLSTACKDOWN/UP pseudo insns 716let Defs = [R11], Uses = [R11], isCodeGenOnly = 1 in { 717def ADJCALLSTACKDOWN : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2), 718 "#ADJCALLSTACKDOWN $amt1 $amt2", 719 [(BPFcallseq_start timm:$amt1, timm:$amt2)]>; 720def ADJCALLSTACKUP : Pseudo<(outs), (ins i64imm:$amt1, i64imm:$amt2), 721 "#ADJCALLSTACKUP $amt1 $amt2", 722 [(BPFcallseq_end timm:$amt1, timm:$amt2)]>; 723} 724 725let usesCustomInserter = 1, isCodeGenOnly = 1 in { 726 def Select : Pseudo<(outs GPR:$dst), 727 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR:$src, GPR:$src2), 728 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 729 [(set i64:$dst, 730 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i64:$src, i64:$src2))]>; 731 def Select_Ri : Pseudo<(outs GPR:$dst), 732 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR:$src, GPR:$src2), 733 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 734 [(set i64:$dst, 735 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i64:$src, i64:$src2))]>; 736 def Select_64_32 : Pseudo<(outs GPR32:$dst), 737 (ins GPR:$lhs, GPR:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2), 738 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 739 [(set i32:$dst, 740 (BPFselectcc i64:$lhs, i64:$rhs, (i64 imm:$imm), i32:$src, i32:$src2))]>; 741 def Select_Ri_64_32 : Pseudo<(outs GPR32:$dst), 742 (ins GPR:$lhs, i64imm:$rhs, i64imm:$imm, GPR32:$src, GPR32:$src2), 743 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 744 [(set i32:$dst, 745 (BPFselectcc i64:$lhs, (i64immSExt32:$rhs), (i64 imm:$imm), i32:$src, i32:$src2))]>; 746 def Select_32 : Pseudo<(outs GPR32:$dst), 747 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2), 748 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 749 [(set i32:$dst, 750 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i32:$src, i32:$src2))]>; 751 def Select_Ri_32 : Pseudo<(outs GPR32:$dst), 752 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR32:$src, GPR32:$src2), 753 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 754 [(set i32:$dst, 755 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i32:$src, i32:$src2))]>; 756 def Select_32_64 : Pseudo<(outs GPR:$dst), 757 (ins GPR32:$lhs, GPR32:$rhs, i32imm:$imm, GPR:$src, GPR:$src2), 758 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 759 [(set i64:$dst, 760 (BPFselectcc i32:$lhs, i32:$rhs, (i32 imm:$imm), i64:$src, i64:$src2))]>; 761 def Select_Ri_32_64 : Pseudo<(outs GPR:$dst), 762 (ins GPR32:$lhs, i32imm:$rhs, i32imm:$imm, GPR:$src, GPR:$src2), 763 "# Select PSEUDO $dst = $lhs $imm $rhs ? $src : $src2", 764 [(set i64:$dst, 765 (BPFselectcc i32:$lhs, (i32immSExt32:$rhs), (i32 imm:$imm), i64:$src, i64:$src2))]>; 766} 767 768// load 64-bit global addr into register 769def : Pat<(BPFWrapper tglobaladdr:$in), (LD_imm64 tglobaladdr:$in)>; 770def : Pat<(BPFWrapper tconstpool:$in), (LD_imm64 tconstpool:$in)>; 771 772// 0xffffFFFF doesn't fit into simm32, optimize common case 773def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)), 774 (SRL_ri (SLL_ri (i64 GPR:$src), 32), 32)>; 775 776// Calls 777def : Pat<(BPFcall tglobaladdr:$dst), (JAL tglobaladdr:$dst)>; 778def : Pat<(BPFcall texternalsym:$dst), (JAL texternalsym:$dst)>; 779def : Pat<(BPFcall imm:$dst), (JAL imm:$dst)>; 780def : Pat<(BPFcall GPR:$dst), (JALX GPR:$dst)>; 781 782// Loads 783let Predicates = [BPFNoALU32] in { 784 def : Pat<(i64 (extloadi8 ADDRri:$src)), (i64 (LDB ADDRri:$src))>; 785 def : Pat<(i64 (extloadi16 ADDRri:$src)), (i64 (LDH ADDRri:$src))>; 786 def : Pat<(i64 (extloadi32 ADDRri:$src)), (i64 (LDW ADDRri:$src))>; 787} 788 789// Atomic add, and, or, xor 790class ATOMIC_NOFETCH<BPFWidthModifer SizeOp, string OpType, RegisterClass RegTp, 791 BPFArithOp Opc, string Opstr> 792 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 793 (outs RegTp:$dst), 794 (ins MEMri:$addr, RegTp:$val), 795 "lock *(" #OpType# " *)($addr) " #Opstr# "= $val", 796 []> { 797 bits<4> dst; 798 bits<20> addr; 799 800 let Inst{51-48} = addr{19-16}; // base reg 801 let Inst{55-52} = dst; 802 let Inst{47-32} = addr{15-0}; // offset 803 let Inst{7-4} = Opc.Value; 804 let BPFClass = BPF_STX; 805} 806 807let Constraints = "$dst = $val" in { 808 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 809 def XADDW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_ADD, "+">; 810 def XANDW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_AND, "&">; 811 def XORW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_OR, "|">; 812 def XXORW32 : ATOMIC_NOFETCH<BPF_W, "u32", GPR32, BPF_XOR, "^">; 813 } 814 def XADDW : ATOMIC_NOFETCH<BPF_W, "u32", GPR, BPF_ADD, "+">; 815 def XADDD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_ADD, "+">; 816 def XANDD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_AND, "&">; 817 def XORD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_OR, "|">; 818 def XXORD : ATOMIC_NOFETCH<BPF_DW, "u64", GPR, BPF_XOR, "^">; 819} 820 821let Predicates = [BPFNoALU32] in { 822 def : Pat<(atomic_load_add_i32 ADDRri:$addr, GPR:$val), 823 (XADDW ADDRri:$addr, GPR:$val)>; 824 def : Pat<(atomic_load_add_i64 ADDRri:$addr, GPR:$val), 825 (XADDD ADDRri:$addr, GPR:$val)>; 826} 827 828// Atomic Fetch-and-<add, and, or, xor> operations 829class XFALU64<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, string OpcStr> 830 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 831 (outs GPR:$dst), 832 (ins MEMri:$addr, GPR:$val), 833 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)", 834 []> { 835 bits<4> dst; 836 bits<20> addr; 837 838 let Inst{51-48} = addr{19-16}; // base reg 839 let Inst{55-52} = dst; 840 let Inst{47-32} = addr{15-0}; // offset 841 let Inst{7-4} = Opc.Value; 842 let Inst{3-0} = BPF_FETCH.Value; 843 let BPFClass = BPF_STX; 844} 845 846class XFALU32<BPFWidthModifer SizeOp, BPFArithOp Opc, string OpcodeStr, string OpcStr> 847 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 848 (outs GPR32:$dst), 849 (ins MEMri:$addr, GPR32:$val), 850 "$dst = atomic_fetch_"#OpcStr#"(("#OpcodeStr#" *)($addr), $val)", 851 []> { 852 bits<4> dst; 853 bits<20> addr; 854 855 let Inst{51-48} = addr{19-16}; // base reg 856 let Inst{55-52} = dst; 857 let Inst{47-32} = addr{15-0}; // offset 858 let Inst{7-4} = Opc.Value; 859 let Inst{3-0} = BPF_FETCH.Value; 860 let BPFClass = BPF_STX; 861} 862 863let Constraints = "$dst = $val" in { 864 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 865 def XFADDW32 : XFALU32<BPF_W, BPF_ADD, "u32", "add">; 866 def XFANDW32 : XFALU32<BPF_W, BPF_AND, "u32", "and">; 867 def XFORW32 : XFALU32<BPF_W, BPF_OR, "u32", "or">; 868 def XFXORW32 : XFALU32<BPF_W, BPF_XOR, "u32", "xor">; 869 } 870 871 let Predicates = [BPFHasALU32] in { 872 def XFADDD : XFALU64<BPF_DW, BPF_ADD, "u64", "add">; 873 } 874 def XFANDD : XFALU64<BPF_DW, BPF_AND, "u64", "and">; 875 def XFORD : XFALU64<BPF_DW, BPF_OR, "u64", "or">; 876 def XFXORD : XFALU64<BPF_DW, BPF_XOR, "u64", "xor">; 877} 878 879let Predicates = [BPFHasALU32] in { 880 foreach P = [// add 881 [atomic_load_add_i32_monotonic, XADDW32], 882 [atomic_load_add_i32_acquire, XFADDW32], 883 [atomic_load_add_i32_release, XFADDW32], 884 [atomic_load_add_i32_acq_rel, XFADDW32], 885 [atomic_load_add_i32_seq_cst, XFADDW32], 886 // and 887 [atomic_load_and_i32_monotonic, XANDW32], 888 [atomic_load_and_i32_acquire, XFANDW32], 889 [atomic_load_and_i32_release, XFANDW32], 890 [atomic_load_and_i32_acq_rel, XFANDW32], 891 [atomic_load_and_i32_seq_cst, XFANDW32], 892 // or 893 [atomic_load_or_i32_monotonic, XORW32], 894 [atomic_load_or_i32_acquire, XFORW32], 895 [atomic_load_or_i32_release, XFORW32], 896 [atomic_load_or_i32_acq_rel, XFORW32], 897 [atomic_load_or_i32_seq_cst, XFORW32], 898 // xor 899 [atomic_load_xor_i32_monotonic, XXORW32], 900 [atomic_load_xor_i32_acquire, XFXORW32], 901 [atomic_load_xor_i32_release, XFXORW32], 902 [atomic_load_xor_i32_acq_rel, XFXORW32], 903 [atomic_load_xor_i32_seq_cst, XFXORW32], 904 ] in { 905 def : Pat<(P[0] ADDRri:$addr, GPR32:$val), (P[1] ADDRri:$addr, GPR32:$val)>; 906 } 907 908 // atomic_load_sub can be represented as a neg followed 909 // by an atomic_load_add. 910 foreach P = [[atomic_load_sub_i32_monotonic, XADDW32], 911 [atomic_load_sub_i32_acquire, XFADDW32], 912 [atomic_load_sub_i32_release, XFADDW32], 913 [atomic_load_sub_i32_acq_rel, XFADDW32], 914 [atomic_load_sub_i32_seq_cst, XFADDW32], 915 ] in { 916 def : Pat<(P[0] ADDRri:$addr, GPR32:$val), (P[1] ADDRri:$addr, (NEG_32 GPR32:$val))>; 917 } 918 919 foreach P = [// add 920 [atomic_load_add_i64_monotonic, XADDD], 921 [atomic_load_add_i64_acquire, XFADDD], 922 [atomic_load_add_i64_release, XFADDD], 923 [atomic_load_add_i64_acq_rel, XFADDD], 924 [atomic_load_add_i64_seq_cst, XFADDD], 925 ] in { 926 def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1] ADDRri:$addr, GPR:$val)>; 927 } 928} 929 930foreach P = [[atomic_load_sub_i64_monotonic, XADDD], 931 [atomic_load_sub_i64_acquire, XFADDD], 932 [atomic_load_sub_i64_release, XFADDD], 933 [atomic_load_sub_i64_acq_rel, XFADDD], 934 [atomic_load_sub_i64_seq_cst, XFADDD], 935 ] in { 936 def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1] ADDRri:$addr, (NEG_64 GPR:$val))>; 937} 938 939// Borrow the idea from X86InstrFragments.td 940class binop_no_use<SDPatternOperator operator> 941 : PatFrag<(ops node:$A, node:$B), 942 (operator node:$A, node:$B), 943 [{ return SDValue(N, 0).use_empty(); }]>; 944 945class binop_has_use<SDPatternOperator operator> 946 : PatFrag<(ops node:$A, node:$B), 947 (operator node:$A, node:$B), 948 [{ return !SDValue(N, 0).use_empty(); }]>; 949 950foreach op = [add, and, or, xor] in { 951def atomic_load_ # op # _i64_monotonic_nu: 952 binop_no_use <!cast<SDPatternOperator>("atomic_load_"#op# _i64_monotonic)>; 953def atomic_load_ # op # _i64_monotonic_hu: 954 binop_has_use<!cast<SDPatternOperator>("atomic_load_"#op# _i64_monotonic)>; 955} 956 957foreach P = [// and 958 [atomic_load_and_i64_monotonic_nu, XANDD], 959 [atomic_load_and_i64_monotonic_hu, XFANDD], 960 [atomic_load_and_i64_acquire, XFANDD], 961 [atomic_load_and_i64_release, XFANDD], 962 [atomic_load_and_i64_acq_rel, XFANDD], 963 [atomic_load_and_i64_seq_cst, XFANDD], 964 // or 965 [atomic_load_or_i64_monotonic_nu, XORD], 966 [atomic_load_or_i64_monotonic_hu, XFORD], 967 [atomic_load_or_i64_acquire, XFORD], 968 [atomic_load_or_i64_release, XFORD], 969 [atomic_load_or_i64_acq_rel, XFORD], 970 [atomic_load_or_i64_seq_cst, XFORD], 971 // xor 972 [atomic_load_xor_i64_monotonic_nu, XXORD], 973 [atomic_load_xor_i64_monotonic_hu, XFXORD], 974 [atomic_load_xor_i64_acquire, XFXORD], 975 [atomic_load_xor_i64_release, XFXORD], 976 [atomic_load_xor_i64_acq_rel, XFXORD], 977 [atomic_load_xor_i64_seq_cst, XFXORD], 978 ] in { 979 def : Pat<(P[0] ADDRri:$addr, GPR:$val), (P[1] ADDRri:$addr, GPR:$val)>; 980} 981 982// Atomic Exchange 983class XCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 984 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 985 (outs GPR:$dst), 986 (ins MEMri:$addr, GPR:$val), 987 "$dst = xchg_"#OpcodeStr#"($addr, $val)", 988 [(set GPR:$dst, (OpNode ADDRri:$addr,GPR:$val))]> { 989 bits<4> dst; 990 bits<20> addr; 991 992 let Inst{51-48} = addr{19-16}; // base reg 993 let Inst{55-52} = dst; 994 let Inst{47-32} = addr{15-0}; // offset 995 let Inst{7-4} = BPF_XCHG.Value; 996 let Inst{3-0} = BPF_FETCH.Value; 997 let BPFClass = BPF_STX; 998} 999 1000class XCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 1001 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 1002 (outs GPR32:$dst), 1003 (ins MEMri:$addr, GPR32:$val), 1004 "$dst = xchg32_"#OpcodeStr#"($addr, $val)", 1005 [(set GPR32:$dst, (OpNode ADDRri:$addr,GPR32:$val))]> { 1006 bits<4> dst; 1007 bits<20> addr; 1008 1009 let Inst{51-48} = addr{19-16}; // base reg 1010 let Inst{55-52} = dst; 1011 let Inst{47-32} = addr{15-0}; // offset 1012 let Inst{7-4} = BPF_XCHG.Value; 1013 let Inst{3-0} = BPF_FETCH.Value; 1014 let BPFClass = BPF_STX; 1015} 1016 1017let Constraints = "$dst = $val" in { 1018 let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1019 def XCHGW32 : XCHG32<BPF_W, "32", atomic_swap_i32>; 1020 } 1021 1022 def XCHGD : XCHG<BPF_DW, "64", atomic_swap_i64>; 1023} 1024 1025// Compare-And-Exchange 1026class CMPXCHG<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 1027 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 1028 (outs), 1029 (ins MEMri:$addr, GPR:$new), 1030 "r0 = cmpxchg_"#OpcodeStr#"($addr, r0, $new)", 1031 [(set R0, (OpNode ADDRri:$addr, R0, GPR:$new))]> { 1032 bits<4> new; 1033 bits<20> addr; 1034 1035 let Inst{51-48} = addr{19-16}; // base reg 1036 let Inst{55-52} = new; 1037 let Inst{47-32} = addr{15-0}; // offset 1038 let Inst{7-4} = BPF_CMPXCHG.Value; 1039 let Inst{3-0} = BPF_FETCH.Value; 1040 let BPFClass = BPF_STX; 1041} 1042 1043class CMPXCHG32<BPFWidthModifer SizeOp, string OpcodeStr, PatFrag OpNode> 1044 : TYPE_LD_ST<BPF_ATOMIC.Value, SizeOp.Value, 1045 (outs), 1046 (ins MEMri:$addr, GPR32:$new), 1047 "w0 = cmpxchg32_"#OpcodeStr#"($addr, w0, $new)", 1048 [(set W0, (OpNode ADDRri:$addr, W0, GPR32:$new))]> { 1049 bits<4> new; 1050 bits<20> addr; 1051 1052 let Inst{51-48} = addr{19-16}; // base reg 1053 let Inst{55-52} = new; 1054 let Inst{47-32} = addr{15-0}; // offset 1055 let Inst{7-4} = BPF_CMPXCHG.Value; 1056 let Inst{3-0} = BPF_FETCH.Value; 1057 let BPFClass = BPF_STX; 1058} 1059 1060let Predicates = [BPFHasALU32], Defs = [W0], Uses = [W0], 1061 DecoderNamespace = "BPFALU32" in { 1062 def CMPXCHGW32 : CMPXCHG32<BPF_W, "32", atomic_cmp_swap_i32>; 1063} 1064 1065let Defs = [R0], Uses = [R0] in { 1066 def CMPXCHGD : CMPXCHG<BPF_DW, "64", atomic_cmp_swap_i64>; 1067} 1068 1069// bswap16, bswap32, bswap64 1070class BSWAP<BPFOpClass Class, bits<32> SizeOp, string OpcodeStr, BPFSrcType SrcType, list<dag> Pattern> 1071 : TYPE_ALU_JMP<BPF_END.Value, SrcType.Value, 1072 (outs GPR:$dst), 1073 (ins GPR:$src), 1074 "$dst = "#OpcodeStr#" $src", 1075 Pattern> { 1076 bits<4> dst; 1077 1078 let Inst{51-48} = dst; 1079 let Inst{31-0} = SizeOp; 1080 let BPFClass = Class; 1081} 1082 1083 1084let Constraints = "$dst = $src" in { 1085 let Predicates = [BPFHasBswap] in { 1086 def BSWAP16 : BSWAP<BPF_ALU64, 16, "bswap16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 1087 def BSWAP32 : BSWAP<BPF_ALU64, 32, "bswap32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 1088 def BSWAP64 : BSWAP<BPF_ALU64, 64, "bswap64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>; 1089 } 1090 1091 let Predicates = [BPFNoBswap] in { 1092 let Predicates = [BPFIsLittleEndian] in { 1093 def BE16 : BSWAP<BPF_ALU, 16, "be16", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 1094 def BE32 : BSWAP<BPF_ALU, 32, "be32", BPF_TO_BE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 1095 def BE64 : BSWAP<BPF_ALU, 64, "be64", BPF_TO_BE, [(set GPR:$dst, (bswap GPR:$src))]>; 1096 } 1097 let Predicates = [BPFIsBigEndian] in { 1098 def LE16 : BSWAP<BPF_ALU, 16, "le16", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 48)))]>; 1099 def LE32 : BSWAP<BPF_ALU, 32, "le32", BPF_TO_LE, [(set GPR:$dst, (srl (bswap GPR:$src), (i64 32)))]>; 1100 def LE64 : BSWAP<BPF_ALU, 64, "le64", BPF_TO_LE, [(set GPR:$dst, (bswap GPR:$src))]>; 1101 } 1102 } 1103} 1104 1105let Defs = [R0, R1, R2, R3, R4, R5], Uses = [R6], hasSideEffects = 1, 1106 hasExtraDefRegAllocReq = 1, hasExtraSrcRegAllocReq = 1, mayLoad = 1 in { 1107class LOAD_ABS<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode> 1108 : TYPE_LD_ST<BPF_ABS.Value, SizeOp.Value, 1109 (outs), 1110 (ins GPR:$skb, i64imm:$imm), 1111 "r0 = *("#OpcodeStr#" *)skb[$imm]", 1112 [(set R0, (OpNode GPR:$skb, i64immSExt32:$imm))]> { 1113 bits<32> imm; 1114 1115 let Inst{31-0} = imm; 1116 let BPFClass = BPF_LD; 1117} 1118 1119class LOAD_IND<BPFWidthModifer SizeOp, string OpcodeStr, Intrinsic OpNode> 1120 : TYPE_LD_ST<BPF_IND.Value, SizeOp.Value, 1121 (outs), 1122 (ins GPR:$skb, GPR:$val), 1123 "r0 = *("#OpcodeStr#" *)skb[$val]", 1124 [(set R0, (OpNode GPR:$skb, GPR:$val))]> { 1125 bits<4> val; 1126 1127 let Inst{55-52} = val; 1128 let BPFClass = BPF_LD; 1129} 1130} 1131 1132def LD_ABS_B : LOAD_ABS<BPF_B, "u8", int_bpf_load_byte>; 1133def LD_ABS_H : LOAD_ABS<BPF_H, "u16", int_bpf_load_half>; 1134def LD_ABS_W : LOAD_ABS<BPF_W, "u32", int_bpf_load_word>; 1135 1136def LD_IND_B : LOAD_IND<BPF_B, "u8", int_bpf_load_byte>; 1137def LD_IND_H : LOAD_IND<BPF_H, "u16", int_bpf_load_half>; 1138def LD_IND_W : LOAD_IND<BPF_W, "u32", int_bpf_load_word>; 1139 1140let isCodeGenOnly = 1 in { 1141 def MOV_32_64 : ALU_RR<BPF_ALU, BPF_MOV, 0, 1142 (outs GPR:$dst), (ins GPR32:$src), 1143 "$dst = $src", []>; 1144} 1145 1146let Predicates = [BPFNoMovsx] in { 1147 def : Pat<(i64 (sext GPR32:$src)), 1148 (SRA_ri (SLL_ri (MOV_32_64 GPR32:$src), 32), 32)>; 1149} 1150 1151let Predicates = [BPFHasMovsx] in { 1152 def : Pat<(i64 (sext GPR32:$src)), 1153 (MOVSX_rr_32 (MOV_32_64 GPR32:$src))>; 1154} 1155 1156def : Pat<(i64 (zext GPR32:$src)), (MOV_32_64 GPR32:$src)>; 1157 1158// For i64 -> i32 truncation, use the 32-bit subregister directly. 1159def : Pat<(i32 (trunc GPR:$src)), 1160 (i32 (EXTRACT_SUBREG GPR:$src, sub_32))>; 1161 1162// For i32 -> i64 anyext, we don't care about the high bits. 1163def : Pat<(i64 (anyext GPR32:$src)), 1164 (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32)>; 1165 1166class STORE32<BPFWidthModifer SizeOp, string OpcodeStr, list<dag> Pattern> 1167 : TYPE_LD_ST<BPF_MEM.Value, SizeOp.Value, 1168 (outs), 1169 (ins GPR32:$src, MEMri:$addr), 1170 "*("#OpcodeStr#" *)($addr) = $src", 1171 Pattern> { 1172 bits<4> src; 1173 bits<20> addr; 1174 1175 let Inst{51-48} = addr{19-16}; // base reg 1176 let Inst{55-52} = src; 1177 let Inst{47-32} = addr{15-0}; // offset 1178 let BPFClass = BPF_STX; 1179} 1180 1181class STOREi32<BPFWidthModifer Opc, string OpcodeStr, PatFrag OpNode> 1182 : STORE32<Opc, OpcodeStr, [(OpNode GPR32:$src, ADDRri:$addr)]>; 1183 1184let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1185 def STW32 : STOREi32<BPF_W, "u32", store>; 1186 def STH32 : STOREi32<BPF_H, "u16", truncstorei16>; 1187 def STB32 : STOREi32<BPF_B, "u8", truncstorei8>; 1188} 1189 1190class LOAD32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, list<dag> Pattern> 1191 : TYPE_LD_ST<ModOp.Value, SizeOp.Value, 1192 (outs GPR32:$dst), 1193 (ins MEMri:$addr), 1194 "$dst = *("#OpcodeStr#" *)($addr)", 1195 Pattern> { 1196 bits<4> dst; 1197 bits<20> addr; 1198 1199 let Inst{51-48} = dst; 1200 let Inst{55-52} = addr{19-16}; 1201 let Inst{47-32} = addr{15-0}; 1202 let BPFClass = BPF_LDX; 1203} 1204 1205class LOADi32<BPFWidthModifer SizeOp, BPFModeModifer ModOp, string OpcodeStr, PatFrag OpNode> 1206 : LOAD32<SizeOp, ModOp, OpcodeStr, [(set i32:$dst, (OpNode ADDRri:$addr))]>; 1207 1208let Predicates = [BPFHasALU32], DecoderNamespace = "BPFALU32" in { 1209 def LDW32 : LOADi32<BPF_W, BPF_MEM, "u32", load>; 1210 def LDH32 : LOADi32<BPF_H, BPF_MEM, "u16", zextloadi16>; 1211 def LDB32 : LOADi32<BPF_B, BPF_MEM, "u8", zextloadi8>; 1212} 1213 1214let Predicates = [BPFHasALU32] in { 1215 def : Pat<(truncstorei8 GPR:$src, ADDRri:$dst), 1216 (STB32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1217 def : Pat<(truncstorei16 GPR:$src, ADDRri:$dst), 1218 (STH32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1219 def : Pat<(truncstorei32 GPR:$src, ADDRri:$dst), 1220 (STW32 (EXTRACT_SUBREG GPR:$src, sub_32), ADDRri:$dst)>; 1221 def : Pat<(i32 (extloadi8 ADDRri:$src)), (i32 (LDB32 ADDRri:$src))>; 1222 def : Pat<(i32 (extloadi16 ADDRri:$src)), (i32 (LDH32 ADDRri:$src))>; 1223 1224 let Predicates = [BPFHasLdsx] in { 1225 def : Pat<(i32 (sextloadi8 ADDRri:$src)), (EXTRACT_SUBREG (LDBSX ADDRri:$src), sub_32)>; 1226 def : Pat<(i32 (sextloadi16 ADDRri:$src)), (EXTRACT_SUBREG (LDHSX ADDRri:$src), sub_32)>; 1227 } 1228 1229 def : Pat<(i64 (zextloadi8 ADDRri:$src)), 1230 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>; 1231 def : Pat<(i64 (zextloadi16 ADDRri:$src)), 1232 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>; 1233 def : Pat<(i64 (zextloadi32 ADDRri:$src)), 1234 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>; 1235 def : Pat<(i64 (extloadi8 ADDRri:$src)), 1236 (SUBREG_TO_REG (i64 0), (LDB32 ADDRri:$src), sub_32)>; 1237 def : Pat<(i64 (extloadi16 ADDRri:$src)), 1238 (SUBREG_TO_REG (i64 0), (LDH32 ADDRri:$src), sub_32)>; 1239 def : Pat<(i64 (extloadi32 ADDRri:$src)), 1240 (SUBREG_TO_REG (i64 0), (LDW32 ADDRri:$src), sub_32)>; 1241} 1242 1243let usesCustomInserter = 1, isCodeGenOnly = 1 in { 1244 def MEMCPY : Pseudo< 1245 (outs), 1246 (ins GPR:$dst, GPR:$src, i64imm:$len, i64imm:$align, variable_ops), 1247 "#memcpy dst: $dst, src: $src, len: $len, align: $align", 1248 [(BPFmemcpy GPR:$dst, GPR:$src, imm:$len, imm:$align)]>; 1249} 1250