10b57cec5SDimitry Andric //===-- ARMTargetMachine.cpp - Define TargetMachine for ARM ---------------===// 20b57cec5SDimitry Andric // 30b57cec5SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 40b57cec5SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 50b57cec5SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 60b57cec5SDimitry Andric // 70b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 80b57cec5SDimitry Andric // 90b57cec5SDimitry Andric // 100b57cec5SDimitry Andric //===----------------------------------------------------------------------===// 110b57cec5SDimitry Andric 120b57cec5SDimitry Andric #include "ARMTargetMachine.h" 130b57cec5SDimitry Andric #include "ARM.h" 14bdd1243dSDimitry Andric #include "ARMMachineFunctionInfo.h" 150b57cec5SDimitry Andric #include "ARMMacroFusion.h" 160b57cec5SDimitry Andric #include "ARMSubtarget.h" 170b57cec5SDimitry Andric #include "ARMTargetObjectFile.h" 180b57cec5SDimitry Andric #include "ARMTargetTransformInfo.h" 190b57cec5SDimitry Andric #include "MCTargetDesc/ARMMCTargetDesc.h" 200b57cec5SDimitry Andric #include "TargetInfo/ARMTargetInfo.h" 210b57cec5SDimitry Andric #include "llvm/ADT/STLExtras.h" 220b57cec5SDimitry Andric #include "llvm/ADT/StringRef.h" 230b57cec5SDimitry Andric #include "llvm/Analysis/TargetTransformInfo.h" 240b57cec5SDimitry Andric #include "llvm/CodeGen/ExecutionDomainFix.h" 2581ad6265SDimitry Andric #include "llvm/CodeGen/GlobalISel/CSEInfo.h" 260b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/CallLowering.h" 270b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/IRTranslator.h" 280b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelect.h" 290b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/InstructionSelector.h" 300b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/Legalizer.h" 310b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" 320b57cec5SDimitry Andric #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" 335f757f3fSDimitry Andric #include "llvm/CodeGen/MIRParser/MIParser.h" 340b57cec5SDimitry Andric #include "llvm/CodeGen/MachineFunction.h" 350b57cec5SDimitry Andric #include "llvm/CodeGen/MachineScheduler.h" 360b57cec5SDimitry Andric #include "llvm/CodeGen/Passes.h" 3781ad6265SDimitry Andric #include "llvm/CodeGen/RegisterBankInfo.h" 380b57cec5SDimitry Andric #include "llvm/CodeGen/TargetPassConfig.h" 390b57cec5SDimitry Andric #include "llvm/IR/Attributes.h" 400b57cec5SDimitry Andric #include "llvm/IR/DataLayout.h" 410b57cec5SDimitry Andric #include "llvm/IR/Function.h" 42349cc55cSDimitry Andric #include "llvm/MC/TargetRegistry.h" 430b57cec5SDimitry Andric #include "llvm/Pass.h" 440b57cec5SDimitry Andric #include "llvm/Support/CodeGen.h" 450b57cec5SDimitry Andric #include "llvm/Support/CommandLine.h" 460b57cec5SDimitry Andric #include "llvm/Support/ErrorHandling.h" 470b57cec5SDimitry Andric #include "llvm/Target/TargetLoweringObjectFile.h" 480b57cec5SDimitry Andric #include "llvm/Target/TargetOptions.h" 4906c3fb27SDimitry Andric #include "llvm/TargetParser/ARMTargetParser.h" 5006c3fb27SDimitry Andric #include "llvm/TargetParser/TargetParser.h" 5106c3fb27SDimitry Andric #include "llvm/TargetParser/Triple.h" 52480093f4SDimitry Andric #include "llvm/Transforms/CFGuard.h" 53fe6060f1SDimitry Andric #include "llvm/Transforms/IPO.h" 540b57cec5SDimitry Andric #include "llvm/Transforms/Scalar.h" 550b57cec5SDimitry Andric #include <cassert> 560b57cec5SDimitry Andric #include <memory> 57bdd1243dSDimitry Andric #include <optional> 580b57cec5SDimitry Andric #include <string> 590b57cec5SDimitry Andric 600b57cec5SDimitry Andric using namespace llvm; 610b57cec5SDimitry Andric 620b57cec5SDimitry Andric static cl::opt<bool> 630b57cec5SDimitry Andric DisableA15SDOptimization("disable-a15-sd-optimization", cl::Hidden, 640b57cec5SDimitry Andric cl::desc("Inhibit optimization of S->D register accesses on A15"), 650b57cec5SDimitry Andric cl::init(false)); 660b57cec5SDimitry Andric 670b57cec5SDimitry Andric static cl::opt<bool> 680b57cec5SDimitry Andric EnableAtomicTidy("arm-atomic-cfg-tidy", cl::Hidden, 690b57cec5SDimitry Andric cl::desc("Run SimplifyCFG after expanding atomic operations" 700b57cec5SDimitry Andric " to make use of cmpxchg flow-based information"), 710b57cec5SDimitry Andric cl::init(true)); 720b57cec5SDimitry Andric 730b57cec5SDimitry Andric static cl::opt<bool> 740b57cec5SDimitry Andric EnableARMLoadStoreOpt("arm-load-store-opt", cl::Hidden, 750b57cec5SDimitry Andric cl::desc("Enable ARM load/store optimization pass"), 760b57cec5SDimitry Andric cl::init(true)); 770b57cec5SDimitry Andric 780b57cec5SDimitry Andric // FIXME: Unify control over GlobalMerge. 790b57cec5SDimitry Andric static cl::opt<cl::boolOrDefault> 800b57cec5SDimitry Andric EnableGlobalMerge("arm-global-merge", cl::Hidden, 810b57cec5SDimitry Andric cl::desc("Enable the global merge pass")); 820b57cec5SDimitry Andric 830b57cec5SDimitry Andric namespace llvm { 840b57cec5SDimitry Andric void initializeARMExecutionDomainFixPass(PassRegistry&); 850b57cec5SDimitry Andric } 860b57cec5SDimitry Andric 87480093f4SDimitry Andric extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeARMTarget() { 880b57cec5SDimitry Andric // Register the target. 890b57cec5SDimitry Andric RegisterTargetMachine<ARMLETargetMachine> X(getTheARMLETarget()); 900b57cec5SDimitry Andric RegisterTargetMachine<ARMLETargetMachine> A(getTheThumbLETarget()); 910b57cec5SDimitry Andric RegisterTargetMachine<ARMBETargetMachine> Y(getTheARMBETarget()); 920b57cec5SDimitry Andric RegisterTargetMachine<ARMBETargetMachine> B(getTheThumbBETarget()); 930b57cec5SDimitry Andric 940b57cec5SDimitry Andric PassRegistry &Registry = *PassRegistry::getPassRegistry(); 950b57cec5SDimitry Andric initializeGlobalISel(Registry); 960b57cec5SDimitry Andric initializeARMLoadStoreOptPass(Registry); 970b57cec5SDimitry Andric initializeARMPreAllocLoadStoreOptPass(Registry); 980b57cec5SDimitry Andric initializeARMParallelDSPPass(Registry); 994824e7fdSDimitry Andric initializeARMBranchTargetsPass(Registry); 1000b57cec5SDimitry Andric initializeARMConstantIslandsPass(Registry); 1010b57cec5SDimitry Andric initializeARMExecutionDomainFixPass(Registry); 1020b57cec5SDimitry Andric initializeARMExpandPseudoPass(Registry); 1030b57cec5SDimitry Andric initializeThumb2SizeReducePass(Registry); 1040b57cec5SDimitry Andric initializeMVEVPTBlockPass(Registry); 105fe6060f1SDimitry Andric initializeMVETPAndVPTOptimisationsPass(Registry); 1068bcb0991SDimitry Andric initializeMVETailPredicationPass(Registry); 1070b57cec5SDimitry Andric initializeARMLowOverheadLoopsPass(Registry); 108e8d8bef9SDimitry Andric initializeARMBlockPlacementPass(Registry); 109480093f4SDimitry Andric initializeMVEGatherScatterLoweringPass(Registry); 110e8d8bef9SDimitry Andric initializeARMSLSHardeningPass(Registry); 111fe6060f1SDimitry Andric initializeMVELaneInterleavingPass(Registry); 11281ad6265SDimitry Andric initializeARMFixCortexA57AES1742098Pass(Registry); 1130fca6ea1SDimitry Andric initializeARMDAGToDAGISelLegacyPass(Registry); 1140b57cec5SDimitry Andric } 1150b57cec5SDimitry Andric 1160b57cec5SDimitry Andric static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { 1170b57cec5SDimitry Andric if (TT.isOSBinFormatMachO()) 1188bcb0991SDimitry Andric return std::make_unique<TargetLoweringObjectFileMachO>(); 1190b57cec5SDimitry Andric if (TT.isOSWindows()) 1208bcb0991SDimitry Andric return std::make_unique<TargetLoweringObjectFileCOFF>(); 1218bcb0991SDimitry Andric return std::make_unique<ARMElfTargetObjectFile>(); 1220b57cec5SDimitry Andric } 1230b57cec5SDimitry Andric 1240b57cec5SDimitry Andric static ARMBaseTargetMachine::ARMABI 1250b57cec5SDimitry Andric computeTargetABI(const Triple &TT, StringRef CPU, 1260b57cec5SDimitry Andric const TargetOptions &Options) { 1270b57cec5SDimitry Andric StringRef ABIName = Options.MCOptions.getABIName(); 1280b57cec5SDimitry Andric 1290b57cec5SDimitry Andric if (ABIName.empty()) 1300b57cec5SDimitry Andric ABIName = ARM::computeDefaultTargetABI(TT, CPU); 1310b57cec5SDimitry Andric 1320b57cec5SDimitry Andric if (ABIName == "aapcs16") 1330b57cec5SDimitry Andric return ARMBaseTargetMachine::ARM_ABI_AAPCS16; 1345f757f3fSDimitry Andric else if (ABIName.starts_with("aapcs")) 1350b57cec5SDimitry Andric return ARMBaseTargetMachine::ARM_ABI_AAPCS; 1365f757f3fSDimitry Andric else if (ABIName.starts_with("apcs")) 1370b57cec5SDimitry Andric return ARMBaseTargetMachine::ARM_ABI_APCS; 1380b57cec5SDimitry Andric 1390b57cec5SDimitry Andric llvm_unreachable("Unhandled/unknown ABI Name!"); 1400b57cec5SDimitry Andric return ARMBaseTargetMachine::ARM_ABI_UNKNOWN; 1410b57cec5SDimitry Andric } 1420b57cec5SDimitry Andric 1430b57cec5SDimitry Andric static std::string computeDataLayout(const Triple &TT, StringRef CPU, 1440b57cec5SDimitry Andric const TargetOptions &Options, 1450b57cec5SDimitry Andric bool isLittle) { 1460b57cec5SDimitry Andric auto ABI = computeTargetABI(TT, CPU, Options); 1470b57cec5SDimitry Andric std::string Ret; 1480b57cec5SDimitry Andric 1490b57cec5SDimitry Andric if (isLittle) 1500b57cec5SDimitry Andric // Little endian. 1510b57cec5SDimitry Andric Ret += "e"; 1520b57cec5SDimitry Andric else 1530b57cec5SDimitry Andric // Big endian. 1540b57cec5SDimitry Andric Ret += "E"; 1550b57cec5SDimitry Andric 1560b57cec5SDimitry Andric Ret += DataLayout::getManglingComponent(TT); 1570b57cec5SDimitry Andric 1580b57cec5SDimitry Andric // Pointers are 32 bits and aligned to 32 bits. 1590b57cec5SDimitry Andric Ret += "-p:32:32"; 1600b57cec5SDimitry Andric 1610b57cec5SDimitry Andric // Function pointers are aligned to 8 bits (because the LSB stores the 1620b57cec5SDimitry Andric // ARM/Thumb state). 1630b57cec5SDimitry Andric Ret += "-Fi8"; 1640b57cec5SDimitry Andric 1650b57cec5SDimitry Andric // ABIs other than APCS have 64 bit integers with natural alignment. 1660b57cec5SDimitry Andric if (ABI != ARMBaseTargetMachine::ARM_ABI_APCS) 1670b57cec5SDimitry Andric Ret += "-i64:64"; 1680b57cec5SDimitry Andric 1690b57cec5SDimitry Andric // We have 64 bits floats. The APCS ABI requires them to be aligned to 32 1700b57cec5SDimitry Andric // bits, others to 64 bits. We always try to align to 64 bits. 1710b57cec5SDimitry Andric if (ABI == ARMBaseTargetMachine::ARM_ABI_APCS) 1720b57cec5SDimitry Andric Ret += "-f64:32:64"; 1730b57cec5SDimitry Andric 1740b57cec5SDimitry Andric // We have 128 and 64 bit vectors. The APCS ABI aligns them to 32 bits, others 1750b57cec5SDimitry Andric // to 64. We always ty to give them natural alignment. 1760b57cec5SDimitry Andric if (ABI == ARMBaseTargetMachine::ARM_ABI_APCS) 1770b57cec5SDimitry Andric Ret += "-v64:32:64-v128:32:128"; 1780b57cec5SDimitry Andric else if (ABI != ARMBaseTargetMachine::ARM_ABI_AAPCS16) 1790b57cec5SDimitry Andric Ret += "-v128:64:128"; 1800b57cec5SDimitry Andric 1810b57cec5SDimitry Andric // Try to align aggregates to 32 bits (the default is 64 bits, which has no 1820b57cec5SDimitry Andric // particular hardware support on 32-bit ARM). 1830b57cec5SDimitry Andric Ret += "-a:0:32"; 1840b57cec5SDimitry Andric 1850b57cec5SDimitry Andric // Integer registers are 32 bits. 1860b57cec5SDimitry Andric Ret += "-n32"; 1870b57cec5SDimitry Andric 1880b57cec5SDimitry Andric // The stack is 128 bit aligned on NaCl, 64 bit aligned on AAPCS and 32 bit 1890b57cec5SDimitry Andric // aligned everywhere else. 1900b57cec5SDimitry Andric if (TT.isOSNaCl() || ABI == ARMBaseTargetMachine::ARM_ABI_AAPCS16) 1910b57cec5SDimitry Andric Ret += "-S128"; 1920b57cec5SDimitry Andric else if (ABI == ARMBaseTargetMachine::ARM_ABI_AAPCS) 1930b57cec5SDimitry Andric Ret += "-S64"; 1940b57cec5SDimitry Andric else 1950b57cec5SDimitry Andric Ret += "-S32"; 1960b57cec5SDimitry Andric 1970b57cec5SDimitry Andric return Ret; 1980b57cec5SDimitry Andric } 1990b57cec5SDimitry Andric 2000b57cec5SDimitry Andric static Reloc::Model getEffectiveRelocModel(const Triple &TT, 201bdd1243dSDimitry Andric std::optional<Reloc::Model> RM) { 20281ad6265SDimitry Andric if (!RM) 2030b57cec5SDimitry Andric // Default relocation model on Darwin is PIC. 2040b57cec5SDimitry Andric return TT.isOSBinFormatMachO() ? Reloc::PIC_ : Reloc::Static; 2050b57cec5SDimitry Andric 2060b57cec5SDimitry Andric if (*RM == Reloc::ROPI || *RM == Reloc::RWPI || *RM == Reloc::ROPI_RWPI) 2070b57cec5SDimitry Andric assert(TT.isOSBinFormatELF() && 2080b57cec5SDimitry Andric "ROPI/RWPI currently only supported for ELF"); 2090b57cec5SDimitry Andric 2100b57cec5SDimitry Andric // DynamicNoPIC is only used on darwin. 2110b57cec5SDimitry Andric if (*RM == Reloc::DynamicNoPIC && !TT.isOSDarwin()) 2120b57cec5SDimitry Andric return Reloc::Static; 2130b57cec5SDimitry Andric 2140b57cec5SDimitry Andric return *RM; 2150b57cec5SDimitry Andric } 2160b57cec5SDimitry Andric 2170b57cec5SDimitry Andric /// Create an ARM architecture model. 2180b57cec5SDimitry Andric /// 2190b57cec5SDimitry Andric ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT, 2200b57cec5SDimitry Andric StringRef CPU, StringRef FS, 2210b57cec5SDimitry Andric const TargetOptions &Options, 222bdd1243dSDimitry Andric std::optional<Reloc::Model> RM, 223bdd1243dSDimitry Andric std::optional<CodeModel::Model> CM, 2245f757f3fSDimitry Andric CodeGenOptLevel OL, bool isLittle) 2250b57cec5SDimitry Andric : LLVMTargetMachine(T, computeDataLayout(TT, CPU, Options, isLittle), TT, 2260b57cec5SDimitry Andric CPU, FS, Options, getEffectiveRelocModel(TT, RM), 2270b57cec5SDimitry Andric getEffectiveCodeModel(CM, CodeModel::Small), OL), 2280b57cec5SDimitry Andric TargetABI(computeTargetABI(TT, CPU, Options)), 2290b57cec5SDimitry Andric TLOF(createTLOF(getTargetTriple())), isLittle(isLittle) { 2300b57cec5SDimitry Andric 2310b57cec5SDimitry Andric // Default to triple-appropriate float ABI 2320b57cec5SDimitry Andric if (Options.FloatABIType == FloatABI::Default) { 2330b57cec5SDimitry Andric if (isTargetHardFloat()) 2340b57cec5SDimitry Andric this->Options.FloatABIType = FloatABI::Hard; 2350b57cec5SDimitry Andric else 2360b57cec5SDimitry Andric this->Options.FloatABIType = FloatABI::Soft; 2370b57cec5SDimitry Andric } 2380b57cec5SDimitry Andric 2390b57cec5SDimitry Andric // Default to triple-appropriate EABI 2400b57cec5SDimitry Andric if (Options.EABIVersion == EABI::Default || 2410b57cec5SDimitry Andric Options.EABIVersion == EABI::Unknown) { 2420b57cec5SDimitry Andric // musl is compatible with glibc with regard to EABI version 2430b57cec5SDimitry Andric if ((TargetTriple.getEnvironment() == Triple::GNUEABI || 244*d686ce93SDimitry Andric TargetTriple.getEnvironment() == Triple::GNUEABIT64 || 2450b57cec5SDimitry Andric TargetTriple.getEnvironment() == Triple::GNUEABIHF || 246*d686ce93SDimitry Andric TargetTriple.getEnvironment() == Triple::GNUEABIHFT64 || 2470b57cec5SDimitry Andric TargetTriple.getEnvironment() == Triple::MuslEABI || 24806c3fb27SDimitry Andric TargetTriple.getEnvironment() == Triple::MuslEABIHF || 24906c3fb27SDimitry Andric TargetTriple.getEnvironment() == Triple::OpenHOS) && 2500b57cec5SDimitry Andric !(TargetTriple.isOSWindows() || TargetTriple.isOSDarwin())) 2510b57cec5SDimitry Andric this->Options.EABIVersion = EABI::GNU; 2520b57cec5SDimitry Andric else 2530b57cec5SDimitry Andric this->Options.EABIVersion = EABI::EABI5; 2540b57cec5SDimitry Andric } 2550b57cec5SDimitry Andric 2560b57cec5SDimitry Andric if (TT.isOSBinFormatMachO()) { 2570b57cec5SDimitry Andric this->Options.TrapUnreachable = true; 2580b57cec5SDimitry Andric this->Options.NoTrapAfterNoreturn = true; 2590b57cec5SDimitry Andric } 2600b57cec5SDimitry Andric 2615ffd83dbSDimitry Andric // ARM supports the debug entry values. 2625ffd83dbSDimitry Andric setSupportsDebugEntryValues(true); 2635ffd83dbSDimitry Andric 2640b57cec5SDimitry Andric initAsmInfo(); 2655ffd83dbSDimitry Andric 2665ffd83dbSDimitry Andric // ARM supports the MachineOutliner. 2675ffd83dbSDimitry Andric setMachineOutliner(true); 268e8d8bef9SDimitry Andric setSupportsDefaultOutlining(true); 2690b57cec5SDimitry Andric } 2700b57cec5SDimitry Andric 2710b57cec5SDimitry Andric ARMBaseTargetMachine::~ARMBaseTargetMachine() = default; 2720b57cec5SDimitry Andric 273bdd1243dSDimitry Andric MachineFunctionInfo *ARMBaseTargetMachine::createMachineFunctionInfo( 274bdd1243dSDimitry Andric BumpPtrAllocator &Allocator, const Function &F, 275bdd1243dSDimitry Andric const TargetSubtargetInfo *STI) const { 276bdd1243dSDimitry Andric return ARMFunctionInfo::create<ARMFunctionInfo>( 277bdd1243dSDimitry Andric Allocator, F, static_cast<const ARMSubtarget *>(STI)); 278bdd1243dSDimitry Andric } 279bdd1243dSDimitry Andric 2800b57cec5SDimitry Andric const ARMSubtarget * 2810b57cec5SDimitry Andric ARMBaseTargetMachine::getSubtargetImpl(const Function &F) const { 2820b57cec5SDimitry Andric Attribute CPUAttr = F.getFnAttribute("target-cpu"); 2830b57cec5SDimitry Andric Attribute FSAttr = F.getFnAttribute("target-features"); 2840b57cec5SDimitry Andric 285e8d8bef9SDimitry Andric std::string CPU = 286e8d8bef9SDimitry Andric CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU; 287e8d8bef9SDimitry Andric std::string FS = 288e8d8bef9SDimitry Andric FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS; 2890b57cec5SDimitry Andric 2900b57cec5SDimitry Andric // FIXME: This is related to the code below to reset the target options, 2910b57cec5SDimitry Andric // we need to know whether or not the soft float flag is set on the 2920b57cec5SDimitry Andric // function before we can generate a subtarget. We also need to use 2930b57cec5SDimitry Andric // it as a key for the subtarget since that can be the only difference 2940b57cec5SDimitry Andric // between two functions. 295fe6060f1SDimitry Andric bool SoftFloat = F.getFnAttribute("use-soft-float").getValueAsBool(); 2960b57cec5SDimitry Andric // If the soft float attribute is set on the function turn on the soft float 2970b57cec5SDimitry Andric // subtarget feature. 2980b57cec5SDimitry Andric if (SoftFloat) 2990b57cec5SDimitry Andric FS += FS.empty() ? "+soft-float" : ",+soft-float"; 3000b57cec5SDimitry Andric 3010b57cec5SDimitry Andric // Use the optminsize to identify the subtarget, but don't use it in the 3020b57cec5SDimitry Andric // feature string. 3030b57cec5SDimitry Andric std::string Key = CPU + FS; 3040b57cec5SDimitry Andric if (F.hasMinSize()) 3050b57cec5SDimitry Andric Key += "+minsize"; 3060b57cec5SDimitry Andric 3070b57cec5SDimitry Andric auto &I = SubtargetMap[Key]; 3080b57cec5SDimitry Andric if (!I) { 3090b57cec5SDimitry Andric // This needs to be done before we create a new subtarget since any 3100b57cec5SDimitry Andric // creation will depend on the TM and the code generation flags on the 3110b57cec5SDimitry Andric // function that reside in TargetOptions. 3120b57cec5SDimitry Andric resetTargetOptions(F); 3138bcb0991SDimitry Andric I = std::make_unique<ARMSubtarget>(TargetTriple, CPU, FS, *this, isLittle, 3140b57cec5SDimitry Andric F.hasMinSize()); 3150b57cec5SDimitry Andric 3160b57cec5SDimitry Andric if (!I->isThumb() && !I->hasARMOps()) 3170b57cec5SDimitry Andric F.getContext().emitError("Function '" + F.getName() + "' uses ARM " 3180b57cec5SDimitry Andric "instructions, but the target does not support ARM mode execution."); 3190b57cec5SDimitry Andric } 3200b57cec5SDimitry Andric 3210b57cec5SDimitry Andric return I.get(); 3220b57cec5SDimitry Andric } 3230b57cec5SDimitry Andric 3240b57cec5SDimitry Andric TargetTransformInfo 32581ad6265SDimitry Andric ARMBaseTargetMachine::getTargetTransformInfo(const Function &F) const { 3260b57cec5SDimitry Andric return TargetTransformInfo(ARMTTIImpl(this, F)); 3270b57cec5SDimitry Andric } 3280b57cec5SDimitry Andric 3290b57cec5SDimitry Andric ARMLETargetMachine::ARMLETargetMachine(const Target &T, const Triple &TT, 3300b57cec5SDimitry Andric StringRef CPU, StringRef FS, 3310b57cec5SDimitry Andric const TargetOptions &Options, 332bdd1243dSDimitry Andric std::optional<Reloc::Model> RM, 333bdd1243dSDimitry Andric std::optional<CodeModel::Model> CM, 3345f757f3fSDimitry Andric CodeGenOptLevel OL, bool JIT) 3350b57cec5SDimitry Andric : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true) {} 3360b57cec5SDimitry Andric 3370b57cec5SDimitry Andric ARMBETargetMachine::ARMBETargetMachine(const Target &T, const Triple &TT, 3380b57cec5SDimitry Andric StringRef CPU, StringRef FS, 3390b57cec5SDimitry Andric const TargetOptions &Options, 340bdd1243dSDimitry Andric std::optional<Reloc::Model> RM, 341bdd1243dSDimitry Andric std::optional<CodeModel::Model> CM, 3425f757f3fSDimitry Andric CodeGenOptLevel OL, bool JIT) 3430b57cec5SDimitry Andric : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false) {} 3440b57cec5SDimitry Andric 3450b57cec5SDimitry Andric namespace { 3460b57cec5SDimitry Andric 3470b57cec5SDimitry Andric /// ARM Code Generator Pass Configuration Options. 3480b57cec5SDimitry Andric class ARMPassConfig : public TargetPassConfig { 3490b57cec5SDimitry Andric public: 3500b57cec5SDimitry Andric ARMPassConfig(ARMBaseTargetMachine &TM, PassManagerBase &PM) 351480093f4SDimitry Andric : TargetPassConfig(TM, PM) {} 3520b57cec5SDimitry Andric 3530b57cec5SDimitry Andric ARMBaseTargetMachine &getARMTargetMachine() const { 3540b57cec5SDimitry Andric return getTM<ARMBaseTargetMachine>(); 3550b57cec5SDimitry Andric } 3560b57cec5SDimitry Andric 3570b57cec5SDimitry Andric ScheduleDAGInstrs * 3580b57cec5SDimitry Andric createMachineScheduler(MachineSchedContext *C) const override { 3590b57cec5SDimitry Andric ScheduleDAGMILive *DAG = createGenericSchedLive(C); 3600b57cec5SDimitry Andric // add DAG Mutations here. 3610b57cec5SDimitry Andric const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>(); 3620b57cec5SDimitry Andric if (ST.hasFusion()) 3630b57cec5SDimitry Andric DAG->addMutation(createARMMacroFusionDAGMutation()); 3640b57cec5SDimitry Andric return DAG; 3650b57cec5SDimitry Andric } 3660b57cec5SDimitry Andric 3670b57cec5SDimitry Andric ScheduleDAGInstrs * 3680b57cec5SDimitry Andric createPostMachineScheduler(MachineSchedContext *C) const override { 3690b57cec5SDimitry Andric ScheduleDAGMI *DAG = createGenericSchedPostRA(C); 3700b57cec5SDimitry Andric // add DAG Mutations here. 3710b57cec5SDimitry Andric const ARMSubtarget &ST = C->MF->getSubtarget<ARMSubtarget>(); 3720b57cec5SDimitry Andric if (ST.hasFusion()) 3730b57cec5SDimitry Andric DAG->addMutation(createARMMacroFusionDAGMutation()); 3740b57cec5SDimitry Andric return DAG; 3750b57cec5SDimitry Andric } 3760b57cec5SDimitry Andric 3770b57cec5SDimitry Andric void addIRPasses() override; 3780b57cec5SDimitry Andric void addCodeGenPrepare() override; 3790b57cec5SDimitry Andric bool addPreISel() override; 3800b57cec5SDimitry Andric bool addInstSelector() override; 3810b57cec5SDimitry Andric bool addIRTranslator() override; 3820b57cec5SDimitry Andric bool addLegalizeMachineIR() override; 3830b57cec5SDimitry Andric bool addRegBankSelect() override; 3840b57cec5SDimitry Andric bool addGlobalInstructionSelect() override; 3850b57cec5SDimitry Andric void addPreRegAlloc() override; 3860b57cec5SDimitry Andric void addPreSched2() override; 3870b57cec5SDimitry Andric void addPreEmitPass() override; 3885ffd83dbSDimitry Andric void addPreEmitPass2() override; 3890b57cec5SDimitry Andric 3900b57cec5SDimitry Andric std::unique_ptr<CSEConfigBase> getCSEConfig() const override; 3910b57cec5SDimitry Andric }; 3920b57cec5SDimitry Andric 3930b57cec5SDimitry Andric class ARMExecutionDomainFix : public ExecutionDomainFix { 3940b57cec5SDimitry Andric public: 3950b57cec5SDimitry Andric static char ID; 3960b57cec5SDimitry Andric ARMExecutionDomainFix() : ExecutionDomainFix(ID, ARM::DPRRegClass) {} 3970b57cec5SDimitry Andric StringRef getPassName() const override { 3980b57cec5SDimitry Andric return "ARM Execution Domain Fix"; 3990b57cec5SDimitry Andric } 4000b57cec5SDimitry Andric }; 4010b57cec5SDimitry Andric char ARMExecutionDomainFix::ID; 4020b57cec5SDimitry Andric 4030b57cec5SDimitry Andric } // end anonymous namespace 4040b57cec5SDimitry Andric 4050b57cec5SDimitry Andric INITIALIZE_PASS_BEGIN(ARMExecutionDomainFix, "arm-execution-domain-fix", 4060b57cec5SDimitry Andric "ARM Execution Domain Fix", false, false) 4070b57cec5SDimitry Andric INITIALIZE_PASS_DEPENDENCY(ReachingDefAnalysis) 4080b57cec5SDimitry Andric INITIALIZE_PASS_END(ARMExecutionDomainFix, "arm-execution-domain-fix", 4090b57cec5SDimitry Andric "ARM Execution Domain Fix", false, false) 4100b57cec5SDimitry Andric 4110b57cec5SDimitry Andric TargetPassConfig *ARMBaseTargetMachine::createPassConfig(PassManagerBase &PM) { 4120b57cec5SDimitry Andric return new ARMPassConfig(*this, PM); 4130b57cec5SDimitry Andric } 4140b57cec5SDimitry Andric 4150b57cec5SDimitry Andric std::unique_ptr<CSEConfigBase> ARMPassConfig::getCSEConfig() const { 4160b57cec5SDimitry Andric return getStandardCSEConfigForOpt(TM->getOptLevel()); 4170b57cec5SDimitry Andric } 4180b57cec5SDimitry Andric 4190b57cec5SDimitry Andric void ARMPassConfig::addIRPasses() { 4200b57cec5SDimitry Andric if (TM->Options.ThreadModel == ThreadModel::Single) 4210b57cec5SDimitry Andric addPass(createLowerAtomicPass()); 4220b57cec5SDimitry Andric else 4230fca6ea1SDimitry Andric addPass(createAtomicExpandLegacyPass()); 4240b57cec5SDimitry Andric 4250b57cec5SDimitry Andric // Cmpxchg instructions are often used with a subsequent comparison to 4260b57cec5SDimitry Andric // determine whether it succeeded. We can exploit existing control-flow in 4270b57cec5SDimitry Andric // ldrex/strex loops to simplify this, but it needs tidying up. 4285f757f3fSDimitry Andric if (TM->getOptLevel() != CodeGenOptLevel::None && EnableAtomicTidy) 4290b57cec5SDimitry Andric addPass(createCFGSimplificationPass( 430e8d8bef9SDimitry Andric SimplifyCFGOptions().hoistCommonInsts(true).sinkCommonInsts(true), 431e8d8bef9SDimitry Andric [this](const Function &F) { 4320b57cec5SDimitry Andric const auto &ST = this->TM->getSubtarget<ARMSubtarget>(F); 4330b57cec5SDimitry Andric return ST.hasAnyDataBarrier() && !ST.isThumb1Only(); 4340b57cec5SDimitry Andric })); 4350b57cec5SDimitry Andric 436480093f4SDimitry Andric addPass(createMVEGatherScatterLoweringPass()); 437fe6060f1SDimitry Andric addPass(createMVELaneInterleavingPass()); 438480093f4SDimitry Andric 4390b57cec5SDimitry Andric TargetPassConfig::addIRPasses(); 4400b57cec5SDimitry Andric 4410b57cec5SDimitry Andric // Run the parallel DSP pass. 4425f757f3fSDimitry Andric if (getOptLevel() == CodeGenOptLevel::Aggressive) 4430b57cec5SDimitry Andric addPass(createARMParallelDSPPass()); 4440b57cec5SDimitry Andric 445bdd1243dSDimitry Andric // Match complex arithmetic patterns 4465f757f3fSDimitry Andric if (TM->getOptLevel() >= CodeGenOptLevel::Default) 447bdd1243dSDimitry Andric addPass(createComplexDeinterleavingPass(TM)); 448bdd1243dSDimitry Andric 4490b57cec5SDimitry Andric // Match interleaved memory accesses to ldN/stN intrinsics. 4505f757f3fSDimitry Andric if (TM->getOptLevel() != CodeGenOptLevel::None) 4510b57cec5SDimitry Andric addPass(createInterleavedAccessPass()); 452480093f4SDimitry Andric 453480093f4SDimitry Andric // Add Control Flow Guard checks. 454480093f4SDimitry Andric if (TM->getTargetTriple().isOSWindows()) 455480093f4SDimitry Andric addPass(createCFGuardCheckPass()); 45681ad6265SDimitry Andric 45781ad6265SDimitry Andric if (TM->Options.JMCInstrument) 45881ad6265SDimitry Andric addPass(createJMCInstrumenterPass()); 4590b57cec5SDimitry Andric } 4600b57cec5SDimitry Andric 4610b57cec5SDimitry Andric void ARMPassConfig::addCodeGenPrepare() { 4625f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None) 463bdd1243dSDimitry Andric addPass(createTypePromotionLegacyPass()); 4640b57cec5SDimitry Andric TargetPassConfig::addCodeGenPrepare(); 4650b57cec5SDimitry Andric } 4660b57cec5SDimitry Andric 4670b57cec5SDimitry Andric bool ARMPassConfig::addPreISel() { 4685f757f3fSDimitry Andric if ((TM->getOptLevel() != CodeGenOptLevel::None && 4690b57cec5SDimitry Andric EnableGlobalMerge == cl::BOU_UNSET) || 4700b57cec5SDimitry Andric EnableGlobalMerge == cl::BOU_TRUE) { 4710b57cec5SDimitry Andric // FIXME: This is using the thumb1 only constant value for 4720b57cec5SDimitry Andric // maximal global offset for merging globals. We may want 4730b57cec5SDimitry Andric // to look into using the old value for non-thumb1 code of 4740b57cec5SDimitry Andric // 4095 based on the TargetMachine, but this starts to become 4750b57cec5SDimitry Andric // tricky when doing code gen per function. 4765f757f3fSDimitry Andric bool OnlyOptimizeForSize = 4775f757f3fSDimitry Andric (TM->getOptLevel() < CodeGenOptLevel::Aggressive) && 4780b57cec5SDimitry Andric (EnableGlobalMerge == cl::BOU_UNSET); 4790b57cec5SDimitry Andric // Merging of extern globals is enabled by default on non-Mach-O as we 4800b57cec5SDimitry Andric // expect it to be generally either beneficial or harmless. On Mach-O it 4810b57cec5SDimitry Andric // is disabled as we emit the .subsections_via_symbols directive which 4820b57cec5SDimitry Andric // means that merging extern globals is not safe. 4830b57cec5SDimitry Andric bool MergeExternalByDefault = !TM->getTargetTriple().isOSBinFormatMachO(); 4840b57cec5SDimitry Andric addPass(createGlobalMergePass(TM, 127, OnlyOptimizeForSize, 4850b57cec5SDimitry Andric MergeExternalByDefault)); 4860b57cec5SDimitry Andric } 4870b57cec5SDimitry Andric 4885f757f3fSDimitry Andric if (TM->getOptLevel() != CodeGenOptLevel::None) { 48906c3fb27SDimitry Andric addPass(createHardwareLoopsLegacyPass()); 4908bcb0991SDimitry Andric addPass(createMVETailPredicationPass()); 491fe6060f1SDimitry Andric // FIXME: IR passes can delete address-taken basic blocks, deleting 492fe6060f1SDimitry Andric // corresponding blockaddresses. ARMConstantPoolConstant holds references to 493fe6060f1SDimitry Andric // address-taken basic blocks which can be invalidated if the function 494fe6060f1SDimitry Andric // containing the blockaddress has already been codegen'd and the basic 495fe6060f1SDimitry Andric // block is removed. Work around this by forcing all IR passes to run before 496fe6060f1SDimitry Andric // any ISel takes place. We should have a more principled way of handling 497fe6060f1SDimitry Andric // this. See D99707 for more details. 498fe6060f1SDimitry Andric addPass(createBarrierNoopPass()); 4998bcb0991SDimitry Andric } 5000b57cec5SDimitry Andric 5010b57cec5SDimitry Andric return false; 5020b57cec5SDimitry Andric } 5030b57cec5SDimitry Andric 5040b57cec5SDimitry Andric bool ARMPassConfig::addInstSelector() { 5050b57cec5SDimitry Andric addPass(createARMISelDag(getARMTargetMachine(), getOptLevel())); 5060b57cec5SDimitry Andric return false; 5070b57cec5SDimitry Andric } 5080b57cec5SDimitry Andric 5090b57cec5SDimitry Andric bool ARMPassConfig::addIRTranslator() { 510e8d8bef9SDimitry Andric addPass(new IRTranslator(getOptLevel())); 5110b57cec5SDimitry Andric return false; 5120b57cec5SDimitry Andric } 5130b57cec5SDimitry Andric 5140b57cec5SDimitry Andric bool ARMPassConfig::addLegalizeMachineIR() { 5150b57cec5SDimitry Andric addPass(new Legalizer()); 5160b57cec5SDimitry Andric return false; 5170b57cec5SDimitry Andric } 5180b57cec5SDimitry Andric 5190b57cec5SDimitry Andric bool ARMPassConfig::addRegBankSelect() { 5200b57cec5SDimitry Andric addPass(new RegBankSelect()); 5210b57cec5SDimitry Andric return false; 5220b57cec5SDimitry Andric } 5230b57cec5SDimitry Andric 5240b57cec5SDimitry Andric bool ARMPassConfig::addGlobalInstructionSelect() { 525fe6060f1SDimitry Andric addPass(new InstructionSelect(getOptLevel())); 5260b57cec5SDimitry Andric return false; 5270b57cec5SDimitry Andric } 5280b57cec5SDimitry Andric 5290b57cec5SDimitry Andric void ARMPassConfig::addPreRegAlloc() { 5305f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None) { 5315f757f3fSDimitry Andric if (getOptLevel() == CodeGenOptLevel::Aggressive) 53281ad6265SDimitry Andric addPass(&MachinePipelinerID); 53381ad6265SDimitry Andric 534fe6060f1SDimitry Andric addPass(createMVETPAndVPTOptimisationsPass()); 5355ffd83dbSDimitry Andric 5360b57cec5SDimitry Andric addPass(createMLxExpansionPass()); 5370b57cec5SDimitry Andric 5380b57cec5SDimitry Andric if (EnableARMLoadStoreOpt) 5390b57cec5SDimitry Andric addPass(createARMLoadStoreOptimizationPass(/* pre-register alloc */ true)); 5400b57cec5SDimitry Andric 5410b57cec5SDimitry Andric if (!DisableA15SDOptimization) 5420b57cec5SDimitry Andric addPass(createA15SDOptimizerPass()); 5430b57cec5SDimitry Andric } 5440b57cec5SDimitry Andric } 5450b57cec5SDimitry Andric 5460b57cec5SDimitry Andric void ARMPassConfig::addPreSched2() { 5475f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None) { 5480b57cec5SDimitry Andric if (EnableARMLoadStoreOpt) 5490b57cec5SDimitry Andric addPass(createARMLoadStoreOptimizationPass()); 5500b57cec5SDimitry Andric 5510b57cec5SDimitry Andric addPass(new ARMExecutionDomainFix()); 5520b57cec5SDimitry Andric addPass(createBreakFalseDeps()); 5530b57cec5SDimitry Andric } 5540b57cec5SDimitry Andric 5550b57cec5SDimitry Andric // Expand some pseudo instructions into multiple instructions to allow 5560b57cec5SDimitry Andric // proper scheduling. 5570b57cec5SDimitry Andric addPass(createARMExpandPseudoPass()); 5580b57cec5SDimitry Andric 5595f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None) { 5605ffd83dbSDimitry Andric // When optimising for size, always run the Thumb2SizeReduction pass before 5615ffd83dbSDimitry Andric // IfConversion. Otherwise, check whether IT blocks are restricted 5625ffd83dbSDimitry Andric // (e.g. in v8, IfConversion depends on Thumb instruction widths) 5630b57cec5SDimitry Andric addPass(createThumb2SizeReductionPass([this](const Function &F) { 5645ffd83dbSDimitry Andric return this->TM->getSubtarget<ARMSubtarget>(F).hasMinSize() || 5655ffd83dbSDimitry Andric this->TM->getSubtarget<ARMSubtarget>(F).restrictIT(); 5660b57cec5SDimitry Andric })); 5670b57cec5SDimitry Andric 5680b57cec5SDimitry Andric addPass(createIfConverter([](const MachineFunction &MF) { 5690b57cec5SDimitry Andric return !MF.getSubtarget<ARMSubtarget>().isThumb1Only(); 5700b57cec5SDimitry Andric })); 5710b57cec5SDimitry Andric } 5720b57cec5SDimitry Andric addPass(createThumb2ITBlockPass()); 573480093f4SDimitry Andric 574480093f4SDimitry Andric // Add both scheduling passes to give the subtarget an opportunity to pick 575480093f4SDimitry Andric // between them. 5765f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None) { 577480093f4SDimitry Andric addPass(&PostMachineSchedulerID); 578480093f4SDimitry Andric addPass(&PostRASchedulerID); 579480093f4SDimitry Andric } 580e8d8bef9SDimitry Andric 581349cc55cSDimitry Andric addPass(createMVEVPTBlockPass()); 582e8d8bef9SDimitry Andric addPass(createARMIndirectThunks()); 583e8d8bef9SDimitry Andric addPass(createARMSLSHardeningPass()); 5840b57cec5SDimitry Andric } 5850b57cec5SDimitry Andric 5860b57cec5SDimitry Andric void ARMPassConfig::addPreEmitPass() { 5870b57cec5SDimitry Andric addPass(createThumb2SizeReductionPass()); 5880b57cec5SDimitry Andric 5890b57cec5SDimitry Andric // Constant island pass work on unbundled instructions. 5900b57cec5SDimitry Andric addPass(createUnpackMachineBundles([](const MachineFunction &MF) { 5910b57cec5SDimitry Andric return MF.getSubtarget<ARMSubtarget>().isThumb2(); 5920b57cec5SDimitry Andric })); 5930b57cec5SDimitry Andric 594e8d8bef9SDimitry Andric // Don't optimize barriers or block placement at -O0. 5955f757f3fSDimitry Andric if (getOptLevel() != CodeGenOptLevel::None) { 596e8d8bef9SDimitry Andric addPass(createARMBlockPlacementPass()); 5970b57cec5SDimitry Andric addPass(createARMOptimizeBarriersPass()); 5985ffd83dbSDimitry Andric } 599e8d8bef9SDimitry Andric } 6000b57cec5SDimitry Andric 6015ffd83dbSDimitry Andric void ARMPassConfig::addPreEmitPass2() { 60281ad6265SDimitry Andric // Inserts fixup instructions before unsafe AES operations. Instructions may 60381ad6265SDimitry Andric // be inserted at the start of blocks and at within blocks so this pass has to 60481ad6265SDimitry Andric // come before those below. 60581ad6265SDimitry Andric addPass(createARMFixCortexA57AES1742098Pass()); 60681ad6265SDimitry Andric // Inserts BTIs at the start of functions and indirectly-called basic blocks, 60781ad6265SDimitry Andric // so passes cannot add to the start of basic blocks once this has run. 6084824e7fdSDimitry Andric addPass(createARMBranchTargetsPass()); 60981ad6265SDimitry Andric // Inserts Constant Islands. Block sizes cannot be increased after this point, 61081ad6265SDimitry Andric // as this may push the branch ranges and load offsets of accessing constant 61181ad6265SDimitry Andric // pools out of range.. 6120b57cec5SDimitry Andric addPass(createARMConstantIslandPass()); 61381ad6265SDimitry Andric // Finalises Low-Overhead Loops. This replaces pseudo instructions with real 61481ad6265SDimitry Andric // instructions, but the pseudos all have conservative sizes so that block 61581ad6265SDimitry Andric // sizes will only be decreased by this pass. 6160b57cec5SDimitry Andric addPass(createARMLowOverheadLoopsPass()); 617480093f4SDimitry Andric 618fe6060f1SDimitry Andric if (TM->getTargetTriple().isOSWindows()) { 619480093f4SDimitry Andric // Identify valid longjmp targets for Windows Control Flow Guard. 620480093f4SDimitry Andric addPass(createCFGuardLongjmpPass()); 621fe6060f1SDimitry Andric // Identify valid eh continuation targets for Windows EHCont Guard. 622fe6060f1SDimitry Andric addPass(createEHContGuardCatchretPass()); 623fe6060f1SDimitry Andric } 6240b57cec5SDimitry Andric } 6255f757f3fSDimitry Andric 6265f757f3fSDimitry Andric yaml::MachineFunctionInfo * 6275f757f3fSDimitry Andric ARMBaseTargetMachine::createDefaultFuncInfoYAML() const { 6285f757f3fSDimitry Andric return new yaml::ARMFunctionInfo(); 6295f757f3fSDimitry Andric } 6305f757f3fSDimitry Andric 6315f757f3fSDimitry Andric yaml::MachineFunctionInfo * 6325f757f3fSDimitry Andric ARMBaseTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const { 6335f757f3fSDimitry Andric const auto *MFI = MF.getInfo<ARMFunctionInfo>(); 6345f757f3fSDimitry Andric return new yaml::ARMFunctionInfo(*MFI); 6355f757f3fSDimitry Andric } 6365f757f3fSDimitry Andric 6375f757f3fSDimitry Andric bool ARMBaseTargetMachine::parseMachineFunctionInfo( 6385f757f3fSDimitry Andric const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS, 6395f757f3fSDimitry Andric SMDiagnostic &Error, SMRange &SourceRange) const { 6405f757f3fSDimitry Andric const auto &YamlMFI = static_cast<const yaml::ARMFunctionInfo &>(MFI); 6415f757f3fSDimitry Andric MachineFunction &MF = PFS.MF; 6425f757f3fSDimitry Andric MF.getInfo<ARMFunctionInfo>()->initializeBaseYamlFields(YamlMFI); 6435f757f3fSDimitry Andric return false; 6445f757f3fSDimitry Andric } 645