1 //===-- AArch64SelectionDAGInfo.cpp - AArch64 SelectionDAG Info -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64SelectionDAGInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "AArch64TargetMachine.h"
14 using namespace llvm;
15
16 #define DEBUG_TYPE "aarch64-selectiondag-info"
17
EmitMOPS(AArch64ISD::NodeType SDOpcode,SelectionDAG & DAG,const SDLoc & DL,SDValue Chain,SDValue Dst,SDValue SrcOrValue,SDValue Size,Align Alignment,bool isVolatile,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo) const18 SDValue AArch64SelectionDAGInfo::EmitMOPS(AArch64ISD::NodeType SDOpcode,
19 SelectionDAG &DAG, const SDLoc &DL,
20 SDValue Chain, SDValue Dst,
21 SDValue SrcOrValue, SDValue Size,
22 Align Alignment, bool isVolatile,
23 MachinePointerInfo DstPtrInfo,
24 MachinePointerInfo SrcPtrInfo) const {
25
26 // Get the constant size of the copy/set.
27 uint64_t ConstSize = 0;
28 if (auto *C = dyn_cast<ConstantSDNode>(Size))
29 ConstSize = C->getZExtValue();
30
31 const bool IsSet = SDOpcode == AArch64ISD::MOPS_MEMSET ||
32 SDOpcode == AArch64ISD::MOPS_MEMSET_TAGGING;
33
34 const auto MachineOpcode = [&]() {
35 switch (SDOpcode) {
36 case AArch64ISD::MOPS_MEMSET:
37 return AArch64::MOPSMemorySetPseudo;
38 case AArch64ISD::MOPS_MEMSET_TAGGING:
39 return AArch64::MOPSMemorySetTaggingPseudo;
40 case AArch64ISD::MOPS_MEMCOPY:
41 return AArch64::MOPSMemoryCopyPseudo;
42 case AArch64ISD::MOPS_MEMMOVE:
43 return AArch64::MOPSMemoryMovePseudo;
44 default:
45 llvm_unreachable("Unhandled MOPS ISD Opcode");
46 }
47 }();
48
49 MachineMemOperand::Flags Flags = MachineMemOperand::MOStore;
50 if (isVolatile)
51 Flags |= MachineMemOperand::MOVolatile;
52 if (!IsSet)
53 Flags |= MachineMemOperand::MOLoad;
54
55 MachineFunction &MF = DAG.getMachineFunction();
56
57 auto *DstOp =
58 MF.getMachineMemOperand(DstPtrInfo, Flags, ConstSize, Alignment);
59 auto *SrcOp =
60 MF.getMachineMemOperand(SrcPtrInfo, Flags, ConstSize, Alignment);
61
62 if (IsSet) {
63 // Extend value to i64 if required
64 if (SrcOrValue.getValueType() != MVT::i64)
65 SrcOrValue = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, SrcOrValue);
66 SDValue Ops[] = {Dst, Size, SrcOrValue, Chain};
67 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};
68 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
69 DAG.setNodeMemRefs(Node, {DstOp});
70 return SDValue(Node, 2);
71 } else {
72 SDValue Ops[] = {Dst, SrcOrValue, Size, Chain};
73 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};
74 MachineSDNode *Node = DAG.getMachineNode(MachineOpcode, DL, ResultTys, Ops);
75 DAG.setNodeMemRefs(Node, {DstOp, SrcOp});
76 return SDValue(Node, 3);
77 }
78 }
79
EmitTargetCodeForMemcpy(SelectionDAG & DAG,const SDLoc & DL,SDValue Chain,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVolatile,bool AlwaysInline,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo) const80 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy(
81 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src,
82 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
83 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
84 const AArch64Subtarget &STI =
85 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
86 if (STI.hasMOPS())
87 return EmitMOPS(AArch64ISD::MOPS_MEMCOPY, DAG, DL, Chain, Dst, Src, Size,
88 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
89 return SDValue();
90 }
91
EmitTargetCodeForMemset(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVolatile,bool AlwaysInline,MachinePointerInfo DstPtrInfo) const92 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemset(
93 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
94 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,
95 MachinePointerInfo DstPtrInfo) const {
96 const AArch64Subtarget &STI =
97 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
98
99 if (STI.hasMOPS()) {
100 return EmitMOPS(AArch64ISD::MOPS_MEMSET, DAG, dl, Chain, Dst, Src, Size,
101 Alignment, isVolatile, DstPtrInfo, MachinePointerInfo{});
102 }
103 return SDValue();
104 }
105
EmitTargetCodeForMemmove(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Dst,SDValue Src,SDValue Size,Align Alignment,bool isVolatile,MachinePointerInfo DstPtrInfo,MachinePointerInfo SrcPtrInfo) const106 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForMemmove(
107 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src,
108 SDValue Size, Align Alignment, bool isVolatile,
109 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const {
110 const AArch64Subtarget &STI =
111 DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
112 if (STI.hasMOPS()) {
113 return EmitMOPS(AArch64ISD::MOPS_MEMMOVE, DAG, dl, Chain, Dst, Src, Size,
114 Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);
115 }
116 return SDValue();
117 }
118
119 static const int kSetTagLoopThreshold = 176;
120
EmitUnrolledSetTag(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Ptr,uint64_t ObjSize,const MachineMemOperand * BaseMemOperand,bool ZeroData)121 static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl,
122 SDValue Chain, SDValue Ptr, uint64_t ObjSize,
123 const MachineMemOperand *BaseMemOperand,
124 bool ZeroData) {
125 MachineFunction &MF = DAG.getMachineFunction();
126 unsigned ObjSizeScaled = ObjSize / 16;
127
128 SDValue TagSrc = Ptr;
129 if (Ptr.getOpcode() == ISD::FrameIndex) {
130 int FI = cast<FrameIndexSDNode>(Ptr)->getIndex();
131 Ptr = DAG.getTargetFrameIndex(FI, MVT::i64);
132 // A frame index operand may end up as [SP + offset] => it is fine to use SP
133 // register as the tag source.
134 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);
135 }
136
137 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;
138 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;
139
140 SmallVector<SDValue, 8> OutChains;
141 unsigned OffsetScaled = 0;
142 while (OffsetScaled < ObjSizeScaled) {
143 if (ObjSizeScaled - OffsetScaled >= 2) {
144 SDValue AddrNode =
145 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
146 SDValue St = DAG.getMemIntrinsicNode(
147 OpCode2, dl, DAG.getVTList(MVT::Other),
148 {Chain, TagSrc, AddrNode},
149 MVT::v4i64,
150 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16 * 2));
151 OffsetScaled += 2;
152 OutChains.push_back(St);
153 continue;
154 }
155
156 if (ObjSizeScaled - OffsetScaled > 0) {
157 SDValue AddrNode =
158 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(OffsetScaled * 16), dl);
159 SDValue St = DAG.getMemIntrinsicNode(
160 OpCode1, dl, DAG.getVTList(MVT::Other),
161 {Chain, TagSrc, AddrNode},
162 MVT::v2i64,
163 MF.getMachineMemOperand(BaseMemOperand, OffsetScaled * 16, 16));
164 OffsetScaled += 1;
165 OutChains.push_back(St);
166 }
167 }
168
169 SDValue Res = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
170 return Res;
171 }
172
EmitTargetCodeForSetTag(SelectionDAG & DAG,const SDLoc & dl,SDValue Chain,SDValue Addr,SDValue Size,MachinePointerInfo DstPtrInfo,bool ZeroData) const173 SDValue AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(
174 SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Addr,
175 SDValue Size, MachinePointerInfo DstPtrInfo, bool ZeroData) const {
176 uint64_t ObjSize = cast<ConstantSDNode>(Size)->getZExtValue();
177 assert(ObjSize % 16 == 0);
178
179 MachineFunction &MF = DAG.getMachineFunction();
180 MachineMemOperand *BaseMemOperand = MF.getMachineMemOperand(
181 DstPtrInfo, MachineMemOperand::MOStore, ObjSize, Align(16));
182
183 bool UseSetTagRangeLoop =
184 kSetTagLoopThreshold >= 0 && (int)ObjSize >= kSetTagLoopThreshold;
185 if (!UseSetTagRangeLoop)
186 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,
187 ZeroData);
188
189 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};
190
191 unsigned Opcode;
192 if (Addr.getOpcode() == ISD::FrameIndex) {
193 int FI = cast<FrameIndexSDNode>(Addr)->getIndex();
194 Addr = DAG.getTargetFrameIndex(FI, MVT::i64);
195 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;
196 } else {
197 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;
198 }
199 SDValue Ops[] = {DAG.getTargetConstant(ObjSize, dl, MVT::i64), Addr, Chain};
200 SDNode *St = DAG.getMachineNode(Opcode, dl, ResTys, Ops);
201
202 DAG.setNodeMemRefs(cast<MachineSDNode>(St), {BaseMemOperand});
203 return SDValue(St, 2);
204 }
205