1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the AArch64SelectionDAGInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "AArch64TargetMachine.h" 14 #include "Utils/AArch64SMEAttributes.h" 15 16 using namespace llvm; 17 18 #define DEBUG_TYPE "aarch64-selectiondag-info" 19 20 static cl::opt<bool> 21 LowerToSMERoutines("aarch64-lower-to-sme-routines", cl::Hidden, 22 cl::desc("Enable AArch64 SME memory operations " 23 "to lower to librt functions"), 24 cl::init(true)); 25 26 bool AArch64SelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { 27 return Opcode >= AArch64ISD::FIRST_MEMORY_OPCODE && 28 Opcode <= AArch64ISD::LAST_MEMORY_OPCODE; 29 } 30 31 bool AArch64SelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const { 32 return Opcode >= AArch64ISD::FIRST_STRICTFP_OPCODE && 33 Opcode <= AArch64ISD::LAST_STRICTFP_OPCODE; 34 } 35 36 SDValue AArch64SelectionDAGInfo::EmitMOPS(unsigned Opcode, SelectionDAG &DAG, 37 const SDLoc &DL, SDValue Chain, 38 SDValue Dst, SDValue SrcOrValue, 39 SDValue Size, Align Alignment, 40 bool isVolatile, 41 MachinePointerInfo DstPtrInfo, 42 MachinePointerInfo SrcPtrInfo) const { 43 44 // Get the constant size of the copy/set. 45 uint64_t ConstSize = 0; 46 if (auto *C = dyn_cast<ConstantSDNode>(Size)) 47 ConstSize = C->getZExtValue(); 48 49 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo || 50 Opcode == AArch64::MOPSMemorySetTaggingPseudo; 51 52 MachineFunction &MF = DAG.getMachineFunction(); 53 54 auto Vol = 55 isVolatile ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; 56 auto DstFlags = MachineMemOperand::MOStore | Vol; 57 auto *DstOp = 58 MF.getMachineMemOperand(DstPtrInfo, DstFlags, ConstSize, Alignment); 59 60 if (IsSet) { 61 // Extend value to i64, if required. 62 if (SrcOrValue.getValueType() != MVT::i64) 63 SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue); 64 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain}; 65 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other}; 66 MachineSDNode *Node = DAG.getMachineNode(Opcode, DL, ResultTys, Ops); 67 DAG.setNodeMemRefs(Node, {DstOp}); 68 return SDValue(Node, 2); 69 } else { 70 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain}; 71 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other}; 72 MachineSDNode *Node = DAG.getMachineNode(Opcode, DL, ResultTys, Ops); 73 74 auto SrcFlags = MachineMemOperand::MOLoad | Vol; 75 auto *SrcOp = 76 MF.getMachineMemOperand(SrcPtrInfo, SrcFlags, ConstSize, Alignment); 77 DAG.setNodeMemRefs(Node, {DstOp, SrcOp}); 78 return SDValue(Node, 3); 79 } 80 } 81 82 SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall( 83 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, 84 SDValue Size, RTLIB::Libcall LC) const { 85 const AArch64Subtarget &STI = 86 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 87 const AArch64TargetLowering *TLI = STI.getTargetLowering(); 88 SDValue Symbol; 89 TargetLowering::ArgListEntry DstEntry; 90 DstEntry.Ty = PointerType::getUnqual(*DAG.getContext()); 91 DstEntry.Node = Dst; 92 TargetLowering::ArgListTy Args; 93 Args.push_back(DstEntry); 94 EVT PointerVT = TLI->getPointerTy(DAG.getDataLayout()); 95 96 switch (LC) { 97 case RTLIB::MEMCPY: { 98 TargetLowering::ArgListEntry Entry; 99 Entry.Ty = PointerType::getUnqual(*DAG.getContext()); 100 Symbol = DAG.getExternalSymbol("__arm_sc_memcpy", PointerVT); 101 Entry.Node = Src; 102 Args.push_back(Entry); 103 break; 104 } 105 case RTLIB::MEMMOVE: { 106 TargetLowering::ArgListEntry Entry; 107 Entry.Ty = PointerType::getUnqual(*DAG.getContext()); 108 Symbol = DAG.getExternalSymbol("__arm_sc_memmove", PointerVT); 109 Entry.Node = Src; 110 Args.push_back(Entry); 111 break; 112 } 113 case RTLIB::MEMSET: { 114 TargetLowering::ArgListEntry Entry; 115 Entry.Ty = Type::getInt32Ty(*DAG.getContext()); 116 Symbol = DAG.getExternalSymbol("__arm_sc_memset", PointerVT); 117 Src = DAG.getZExtOrTrunc(Src, DL, MVT::i32); 118 Entry.Node = Src; 119 Args.push_back(Entry); 120 break; 121 } 122 default: 123 return SDValue(); 124 } 125 126 TargetLowering::ArgListEntry SizeEntry; 127 SizeEntry.Node = Size; 128 SizeEntry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); 129 Args.push_back(SizeEntry); 130 assert(Symbol->getOpcode() == ISD::ExternalSymbol && 131 "Function name is not set"); 132 133 TargetLowering::CallLoweringInfo CLI(DAG); 134 PointerType *RetTy = PointerType::getUnqual(*DAG.getContext()); 135 CLI.setDebugLoc(DL).setChain(Chain).setLibCallee( 136 TLI->getLibcallCallingConv(LC), RetTy, Symbol, std::move(Args)); 137 return TLI->LowerCallTo(CLI).second; 138 } 139 140 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy( 141 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, 142 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, 143 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { 144 const AArch64Subtarget &STI = 145 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 146 147 if (STI.hasMOPS()) 148 return EmitMOPS(AArch64::MOPSMemoryCopyPseudo, DAG, DL, Chain, Dst, Src, 149 Size, Alignment, isVolatile, DstPtrInfo, SrcPtrInfo); 150 151 SMEAttrs Attrs(DAG.getMachineFunction().getFunction()); 152 if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody()) 153 return EmitStreamingCompatibleMemLibCall(DAG, DL, Chain, Dst, Src, Size, 154 RTLIB::MEMCPY); 155 return SDValue(); 156 } 157 158 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset( 159 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, 160 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, 161 MachinePointerInfo DstPtrInfo) const { 162 const AArch64Subtarget &STI = 163 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 164 165 if (STI.hasMOPS()) 166 return EmitMOPS(AArch64::MOPSMemorySetPseudo, DAG, dl, Chain, Dst, Src, 167 Size, Alignment, isVolatile, DstPtrInfo, 168 MachinePointerInfo{}); 169 170 SMEAttrs Attrs(DAG.getMachineFunction().getFunction()); 171 if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody()) 172 return EmitStreamingCompatibleMemLibCall(DAG, dl, Chain, Dst, Src, Size, 173 RTLIB::MEMSET); 174 return SDValue(); 175 } 176 177 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove( 178 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, 179 SDValue Size, Align Alignment, bool isVolatile, 180 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { 181 const AArch64Subtarget &STI = 182 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>(); 183 184 if (STI.hasMOPS()) 185 return EmitMOPS(AArch64::MOPSMemoryMovePseudo, DAG, dl, Chain, Dst, Src, 186 Size, Alignment, isVolatile, DstPtrInfo, SrcPtrInfo); 187 188 SMEAttrs Attrs(DAG.getMachineFunction().getFunction()); 189 if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody()) 190 return EmitStreamingCompatibleMemLibCall(DAG, dl, Chain, Dst, Src, Size, 191 RTLIB::MEMMOVE); 192 return SDValue(); 193 } 194 195 static const int kSetTagLoopThreshold = 176; 196 197 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, 198 SDValue Chain, SDValue Ptr, uint64_t ObjSize, 199 const MachineMemOperand *BaseMemOperand, 200 bool ZeroData) { 201 MachineFunction &MF = DAG.getMachineFunction(); 202 unsigned ObjSizeScaled = ObjSize / 16; 203 204 SDValue TagSrc = Ptr; 205 if (Ptr.getOpcode() == ISD::FrameIndex) { 206 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex(); 207 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64); 208 // A frame index operand may end up as [SP + offset] => it is fine to use SP 209 // register as the tag source. 210 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64); 211 } 212 213 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG; 214 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G; 215 216 SmallVector<SDValue, 8> OutChains; 217 unsigned OffsetScaled = 0; 218 while (OffsetScaled < ObjSizeScaled) { 219 if (ObjSizeScaled - OffsetScaled >= 2) { 220 SDValue AddrNode = DAG.getMemBasePlusOffset( 221 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl); 222 SDValue St = DAG.getMemIntrinsicNode( 223 OpCode2, dl, DAG.getVTList(MVT::Other), 224 {Chain, TagSrc, AddrNode}, 225 MVT::v4i64, 226 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2)); 227 OffsetScaled += 2; 228 OutChains.push_back(St); 229 continue; 230 } 231 232 if (ObjSizeScaled - OffsetScaled > 0) { 233 SDValue AddrNode = DAG.getMemBasePlusOffset( 234 Ptr, TypeSize::getFixed(OffsetScaled * 16), dl); 235 SDValue St = DAG.getMemIntrinsicNode( 236 OpCode1, dl, DAG.getVTList(MVT::Other), 237 {Chain, TagSrc, AddrNode}, 238 MVT::v2i64, 239 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16)); 240 OffsetScaled += 1; 241 OutChains.push_back(St); 242 } 243 } 244 245 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains); 246 return Res; 247 } 248 249 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag( 250 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr, 251 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const { 252 uint64_t ObjSize = Size->getAsZExtVal(); 253 assert(ObjSize % 16 == 0); 254 255 MachineFunction &MF = DAG.getMachineFunction(); 256 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand( 257 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16)); 258 259 bool UseSetTagRangeLoop = 260 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold; 261 if (!UseSetTagRangeLoop) 262 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand, 263 ZeroData); 264 265 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other}; 266 267 unsigned Opcode; 268 if (Addr.getOpcode() == ISD::FrameIndex) { 269 int FI = cast<FrameIndexSDNode>(Addr)->getIndex(); 270 Addr = DAG.getTargetFrameIndex(FI, MVT::i64); 271 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop; 272 } else { 273 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback; 274 } 275 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain}; 276 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops); 277 278 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand}); 279 return SDValue(St, 2); 280 } 281