1 //===-- SystemZSelectionDAGInfo.cpp - SystemZ SelectionDAG Info -----------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements the SystemZSelectionDAGInfo class. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "SystemZTargetMachine.h" 14 #include "llvm/CodeGen/SelectionDAG.h" 15 16 using namespace llvm; 17 18 #define DEBUG_TYPE "systemz-selectiondag-info" 19 20 bool SystemZSelectionDAGInfo::isTargetMemoryOpcode(unsigned Opcode) const { 21 return Opcode >= SystemZISD::FIRST_MEMORY_OPCODE && 22 Opcode <= SystemZISD::LAST_MEMORY_OPCODE; 23 } 24 25 bool SystemZSelectionDAGInfo::isTargetStrictFPOpcode(unsigned Opcode) const { 26 return Opcode >= SystemZISD::FIRST_STRICTFP_OPCODE && 27 Opcode <= SystemZISD::LAST_STRICTFP_OPCODE; 28 } 29 30 static unsigned getMemMemLenAdj(unsigned Op) { 31 return Op == SystemZISD::MEMSET_MVC ? 2 : 1; 32 } 33 34 static SDValue createMemMemNode(SelectionDAG &DAG, const SDLoc &DL, unsigned Op, 35 SDValue Chain, SDValue Dst, SDValue Src, 36 SDValue LenAdj, SDValue Byte) { 37 SDVTList VTs = Op == SystemZISD::CLC ? DAG.getVTList(MVT::i32, MVT::Other) 38 : DAG.getVTList(MVT::Other); 39 SmallVector<SDValue, 6> Ops; 40 if (Op == SystemZISD::MEMSET_MVC) 41 Ops = { Chain, Dst, LenAdj, Byte }; 42 else 43 Ops = { Chain, Dst, Src, LenAdj }; 44 return DAG.getNode(Op, DL, VTs, Ops); 45 } 46 47 // Emit a mem-mem operation after subtracting one (or two for memset) from 48 // size, which will be added back during pseudo expansion. As the Reg case 49 // emitted here may be converted by DAGCombiner into having an Imm length, 50 // they are both emitted the same way. 51 static SDValue emitMemMemImm(SelectionDAG &DAG, const SDLoc &DL, unsigned Op, 52 SDValue Chain, SDValue Dst, SDValue Src, 53 uint64_t Size, SDValue Byte = SDValue()) { 54 unsigned Adj = getMemMemLenAdj(Op); 55 assert(Size >= Adj && "Adjusted length overflow."); 56 SDValue LenAdj = DAG.getConstant(Size - Adj, DL, Dst.getValueType()); 57 return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte); 58 } 59 60 static SDValue emitMemMemReg(SelectionDAG &DAG, const SDLoc &DL, unsigned Op, 61 SDValue Chain, SDValue Dst, SDValue Src, 62 SDValue Size, SDValue Byte = SDValue()) { 63 int64_t Adj = getMemMemLenAdj(Op); 64 SDValue LenAdj = DAG.getNode(ISD::ADD, DL, MVT::i64, 65 DAG.getZExtOrTrunc(Size, DL, MVT::i64), 66 DAG.getSignedConstant(0 - Adj, DL, MVT::i64)); 67 return createMemMemNode(DAG, DL, Op, Chain, Dst, Src, LenAdj, Byte); 68 } 69 70 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy( 71 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue Src, 72 SDValue Size, Align Alignment, bool IsVolatile, bool AlwaysInline, 73 MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const { 74 if (IsVolatile) 75 return SDValue(); 76 77 if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) 78 return emitMemMemImm(DAG, DL, SystemZISD::MVC, Chain, Dst, Src, 79 CSize->getZExtValue()); 80 81 return emitMemMemReg(DAG, DL, SystemZISD::MVC, Chain, Dst, Src, Size); 82 } 83 84 // Handle a memset of 1, 2, 4 or 8 bytes with the operands given by 85 // Chain, Dst, ByteVal and Size. These cases are expected to use 86 // MVI, MVHHI, MVHI and MVGHI respectively. 87 static SDValue memsetStore(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, 88 SDValue Dst, uint64_t ByteVal, uint64_t Size, 89 Align Alignment, MachinePointerInfo DstPtrInfo) { 90 uint64_t StoreVal = ByteVal; 91 for (unsigned I = 1; I < Size; ++I) 92 StoreVal |= ByteVal << (I * 8); 93 return DAG.getStore( 94 Chain, DL, DAG.getConstant(StoreVal, DL, MVT::getIntegerVT(Size * 8)), 95 Dst, DstPtrInfo, Alignment); 96 } 97 98 SDValue SystemZSelectionDAGInfo::EmitTargetCodeForMemset( 99 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, 100 SDValue Byte, SDValue Size, Align Alignment, bool IsVolatile, 101 bool AlwaysInline, MachinePointerInfo DstPtrInfo) const { 102 EVT PtrVT = Dst.getValueType(); 103 104 if (IsVolatile) 105 return SDValue(); 106 107 auto *CByte = dyn_cast<ConstantSDNode>(Byte); 108 if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) { 109 uint64_t Bytes = CSize->getZExtValue(); 110 if (Bytes == 0) 111 return SDValue(); 112 if (CByte) { 113 // Handle cases that can be done using at most two of 114 // MVI, MVHI, MVHHI and MVGHI. The latter two can only be 115 // used if ByteVal is all zeros or all ones; in other cases, 116 // we can move at most 2 halfwords. 117 uint64_t ByteVal = CByte->getZExtValue(); 118 if (ByteVal == 0 || ByteVal == 255 119 ? Bytes <= 16 && llvm::popcount(Bytes) <= 2 120 : Bytes <= 4) { 121 unsigned Size1 = Bytes == 16 ? 8 : llvm::bit_floor(Bytes); 122 unsigned Size2 = Bytes - Size1; 123 SDValue Chain1 = memsetStore(DAG, DL, Chain, Dst, ByteVal, Size1, 124 Alignment, DstPtrInfo); 125 if (Size2 == 0) 126 return Chain1; 127 Dst = DAG.getNode(ISD::ADD, DL, PtrVT, Dst, 128 DAG.getConstant(Size1, DL, PtrVT)); 129 DstPtrInfo = DstPtrInfo.getWithOffset(Size1); 130 SDValue Chain2 = 131 memsetStore(DAG, DL, Chain, Dst, ByteVal, Size2, 132 std::min(Alignment, Align(Size1)), DstPtrInfo); 133 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2); 134 } 135 } else { 136 // Handle one and two bytes using STC. 137 if (Bytes <= 2) { 138 SDValue Chain1 = 139 DAG.getStore(Chain, DL, Byte, Dst, DstPtrInfo, Alignment); 140 if (Bytes == 1) 141 return Chain1; 142 SDValue Dst2 = DAG.getNode(ISD::ADD, DL, PtrVT, Dst, 143 DAG.getConstant(1, DL, PtrVT)); 144 SDValue Chain2 = DAG.getStore(Chain, DL, Byte, Dst2, 145 DstPtrInfo.getWithOffset(1), Align(1)); 146 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chain1, Chain2); 147 } 148 } 149 assert(Bytes >= 2 && "Should have dealt with 0- and 1-byte cases already"); 150 151 // Handle the special case of a memset of 0, which can use XC. 152 if (CByte && CByte->getZExtValue() == 0) 153 return emitMemMemImm(DAG, DL, SystemZISD::XC, Chain, Dst, Dst, Bytes); 154 155 return emitMemMemImm(DAG, DL, SystemZISD::MEMSET_MVC, Chain, Dst, SDValue(), 156 Bytes, DAG.getAnyExtOrTrunc(Byte, DL, MVT::i32)); 157 } 158 159 // Variable length 160 if (CByte && CByte->getZExtValue() == 0) 161 // Handle the special case of a variable length memset of 0 with XC. 162 return emitMemMemReg(DAG, DL, SystemZISD::XC, Chain, Dst, Dst, Size); 163 164 return emitMemMemReg(DAG, DL, SystemZISD::MEMSET_MVC, Chain, Dst, SDValue(), 165 Size, DAG.getAnyExtOrTrunc(Byte, DL, MVT::i32)); 166 } 167 168 // Convert the current CC value into an integer that is 0 if CC == 0, 169 // greater than zero if CC == 1 and less than zero if CC >= 2. 170 // The sequence starts with IPM, which puts CC into bits 29 and 28 171 // of an integer and clears bits 30 and 31. 172 static SDValue addIPMSequence(const SDLoc &DL, SDValue CCReg, 173 SelectionDAG &DAG) { 174 SDValue IPM = DAG.getNode(SystemZISD::IPM, DL, MVT::i32, CCReg); 175 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, IPM, 176 DAG.getConstant(30 - SystemZ::IPM_CC, DL, MVT::i32)); 177 SDValue SRA = DAG.getNode(ISD::SRA, DL, MVT::i32, SHL, 178 DAG.getConstant(30, DL, MVT::i32)); 179 return SRA; 180 } 181 182 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemcmp( 183 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1, 184 SDValue Src2, SDValue Size, MachinePointerInfo Op1PtrInfo, 185 MachinePointerInfo Op2PtrInfo) const { 186 SDValue CCReg; 187 // Swap operands to invert CC == 1 vs. CC == 2 cases. 188 if (auto *CSize = dyn_cast<ConstantSDNode>(Size)) { 189 uint64_t Bytes = CSize->getZExtValue(); 190 assert(Bytes > 0 && "Caller should have handled 0-size case"); 191 CCReg = emitMemMemImm(DAG, DL, SystemZISD::CLC, Chain, Src2, Src1, Bytes); 192 } else 193 CCReg = emitMemMemReg(DAG, DL, SystemZISD::CLC, Chain, Src2, Src1, Size); 194 Chain = CCReg.getValue(1); 195 return std::make_pair(addIPMSequence(DL, CCReg, DAG), Chain); 196 } 197 198 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForMemchr( 199 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, 200 SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const { 201 // Use SRST to find the character. End is its address on success. 202 EVT PtrVT = Src.getValueType(); 203 SDVTList VTs = DAG.getVTList(PtrVT, MVT::i32, MVT::Other); 204 Length = DAG.getZExtOrTrunc(Length, DL, PtrVT); 205 Char = DAG.getZExtOrTrunc(Char, DL, MVT::i32); 206 Char = DAG.getNode(ISD::AND, DL, MVT::i32, Char, 207 DAG.getConstant(255, DL, MVT::i32)); 208 SDValue Limit = DAG.getNode(ISD::ADD, DL, PtrVT, Src, Length); 209 SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain, 210 Limit, Src, Char); 211 SDValue CCReg = End.getValue(1); 212 Chain = End.getValue(2); 213 214 // Now select between End and null, depending on whether the character 215 // was found. 216 SDValue Ops[] = { 217 End, DAG.getConstant(0, DL, PtrVT), 218 DAG.getTargetConstant(SystemZ::CCMASK_SRST, DL, MVT::i32), 219 DAG.getTargetConstant(SystemZ::CCMASK_SRST_FOUND, DL, MVT::i32), CCReg}; 220 End = DAG.getNode(SystemZISD::SELECT_CCMASK, DL, PtrVT, Ops); 221 return std::make_pair(End, Chain); 222 } 223 224 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcpy( 225 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dest, 226 SDValue Src, MachinePointerInfo DestPtrInfo, MachinePointerInfo SrcPtrInfo, 227 bool isStpcpy) const { 228 SDVTList VTs = DAG.getVTList(Dest.getValueType(), MVT::Other); 229 SDValue EndDest = DAG.getNode(SystemZISD::STPCPY, DL, VTs, Chain, Dest, Src, 230 DAG.getConstant(0, DL, MVT::i32)); 231 return std::make_pair(isStpcpy ? EndDest : Dest, EndDest.getValue(1)); 232 } 233 234 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrcmp( 235 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src1, 236 SDValue Src2, MachinePointerInfo Op1PtrInfo, 237 MachinePointerInfo Op2PtrInfo) const { 238 SDVTList VTs = DAG.getVTList(Src1.getValueType(), MVT::i32, MVT::Other); 239 // Swap operands to invert CC == 1 vs. CC == 2 cases. 240 SDValue Unused = DAG.getNode(SystemZISD::STRCMP, DL, VTs, Chain, Src2, Src1, 241 DAG.getConstant(0, DL, MVT::i32)); 242 SDValue CCReg = Unused.getValue(1); 243 Chain = Unused.getValue(2); 244 return std::make_pair(addIPMSequence(DL, CCReg, DAG), Chain); 245 } 246 247 // Search from Src for a null character, stopping once Src reaches Limit. 248 // Return a pair of values, the first being the number of nonnull characters 249 // and the second being the out chain. 250 // 251 // This can be used for strlen by setting Limit to 0. 252 static std::pair<SDValue, SDValue> getBoundedStrlen(SelectionDAG &DAG, 253 const SDLoc &DL, 254 SDValue Chain, SDValue Src, 255 SDValue Limit) { 256 EVT PtrVT = Src.getValueType(); 257 SDVTList VTs = DAG.getVTList(PtrVT, MVT::i32, MVT::Other); 258 SDValue End = DAG.getNode(SystemZISD::SEARCH_STRING, DL, VTs, Chain, 259 Limit, Src, DAG.getConstant(0, DL, MVT::i32)); 260 Chain = End.getValue(2); 261 SDValue Len = DAG.getNode(ISD::SUB, DL, PtrVT, End, Src); 262 return std::make_pair(Len, Chain); 263 } 264 265 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrlen( 266 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, 267 MachinePointerInfo SrcPtrInfo) const { 268 EVT PtrVT = Src.getValueType(); 269 return getBoundedStrlen(DAG, DL, Chain, Src, DAG.getConstant(0, DL, PtrVT)); 270 } 271 272 std::pair<SDValue, SDValue> SystemZSelectionDAGInfo::EmitTargetCodeForStrnlen( 273 SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Src, 274 SDValue MaxLength, MachinePointerInfo SrcPtrInfo) const { 275 EVT PtrVT = Src.getValueType(); 276 MaxLength = DAG.getZExtOrTrunc(MaxLength, DL, PtrVT); 277 SDValue Limit = DAG.getNode(ISD::ADD, DL, PtrVT, Src, MaxLength); 278 return getBoundedStrlen(DAG, DL, Chain, Src, Limit); 279 } 280