1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains a printer that converts from our internal representation
10 // of machine-dependent LLVM code to the AArch64 assembly language.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "AArch64.h"
15 #include "AArch64MCInstLower.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64RegisterInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "AArch64TargetObjectFile.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "MCTargetDesc/AArch64InstPrinter.h"
22 #include "MCTargetDesc/AArch64MCExpr.h"
23 #include "MCTargetDesc/AArch64MCTargetDesc.h"
24 #include "MCTargetDesc/AArch64TargetStreamer.h"
25 #include "TargetInfo/AArch64TargetInfo.h"
26 #include "Utils/AArch64BaseInfo.h"
27 #include "llvm/ADT/SmallString.h"
28 #include "llvm/ADT/SmallVector.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/BinaryFormat/COFF.h"
33 #include "llvm/BinaryFormat/ELF.h"
34 #include "llvm/CodeGen/AsmPrinter.h"
35 #include "llvm/CodeGen/FaultMaps.h"
36 #include "llvm/CodeGen/MachineBasicBlock.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineJumpTableInfo.h"
40 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
41 #include "llvm/CodeGen/MachineOperand.h"
42 #include "llvm/CodeGen/StackMaps.h"
43 #include "llvm/CodeGen/TargetRegisterInfo.h"
44 #include "llvm/IR/DataLayout.h"
45 #include "llvm/IR/DebugInfoMetadata.h"
46 #include "llvm/MC/MCAsmInfo.h"
47 #include "llvm/MC/MCContext.h"
48 #include "llvm/MC/MCInst.h"
49 #include "llvm/MC/MCInstBuilder.h"
50 #include "llvm/MC/MCSectionELF.h"
51 #include "llvm/MC/MCStreamer.h"
52 #include "llvm/MC/MCSymbol.h"
53 #include "llvm/Support/Casting.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/TargetRegistry.h"
56 #include "llvm/Support/raw_ostream.h"
57 #include "llvm/Target/TargetMachine.h"
58 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
59 #include <algorithm>
60 #include <cassert>
61 #include <cstdint>
62 #include <map>
63 #include <memory>
64
65 using namespace llvm;
66
67 #define DEBUG_TYPE "asm-printer"
68
69 namespace {
70
71 class AArch64AsmPrinter : public AsmPrinter {
72 AArch64MCInstLower MCInstLowering;
73 StackMaps SM;
74 FaultMaps FM;
75 const AArch64Subtarget *STI;
76
77 public:
AArch64AsmPrinter(TargetMachine & TM,std::unique_ptr<MCStreamer> Streamer)78 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
79 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
80 SM(*this), FM(*this) {}
81
getPassName() const82 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
83
84 /// Wrapper for MCInstLowering.lowerOperand() for the
85 /// tblgen'erated pseudo lowering.
lowerOperand(const MachineOperand & MO,MCOperand & MCOp) const86 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
87 return MCInstLowering.lowerOperand(MO, MCOp);
88 }
89
90 void emitStartOfAsmFile(Module &M) override;
91 void emitJumpTableInfo() override;
92
93 void emitFunctionEntryLabel() override;
94
95 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
96
97 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
98 const MachineInstr &MI);
99 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
100 const MachineInstr &MI);
101 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
102 const MachineInstr &MI);
103 void LowerFAULTING_OP(const MachineInstr &MI);
104
105 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
106 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
107 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
108
109 typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
110 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
111 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
112 void EmitHwasanMemaccessSymbols(Module &M);
113
114 void EmitSled(const MachineInstr &MI, SledKind Kind);
115
116 /// tblgen'erated driver function for lowering simple MI->MC
117 /// pseudo instructions.
118 bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
119 const MachineInstr *MI);
120
121 void emitInstruction(const MachineInstr *MI) override;
122
123 void emitFunctionHeaderComment() override;
124
getAnalysisUsage(AnalysisUsage & AU) const125 void getAnalysisUsage(AnalysisUsage &AU) const override {
126 AsmPrinter::getAnalysisUsage(AU);
127 AU.setPreservesAll();
128 }
129
runOnMachineFunction(MachineFunction & MF)130 bool runOnMachineFunction(MachineFunction &MF) override {
131 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
132 STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
133
134 SetupMachineFunction(MF);
135
136 if (STI->isTargetCOFF()) {
137 bool Internal = MF.getFunction().hasInternalLinkage();
138 COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
139 : COFF::IMAGE_SYM_CLASS_EXTERNAL;
140 int Type =
141 COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
142
143 OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
144 OutStreamer->EmitCOFFSymbolStorageClass(Scl);
145 OutStreamer->EmitCOFFSymbolType(Type);
146 OutStreamer->EndCOFFSymbolDef();
147 }
148
149 // Emit the rest of the function body.
150 emitFunctionBody();
151
152 // Emit the XRay table for this function.
153 emitXRayTable();
154
155 // We didn't modify anything.
156 return false;
157 }
158
159 private:
160 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
161 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
162 bool printAsmRegInClass(const MachineOperand &MO,
163 const TargetRegisterClass *RC, unsigned AltName,
164 raw_ostream &O);
165
166 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
167 const char *ExtraCode, raw_ostream &O) override;
168 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
169 const char *ExtraCode, raw_ostream &O) override;
170
171 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
172
173 void emitFunctionBodyEnd() override;
174
175 MCSymbol *GetCPISymbol(unsigned CPID) const override;
176 void emitEndOfAsmFile(Module &M) override;
177
178 AArch64FunctionInfo *AArch64FI = nullptr;
179
180 /// Emit the LOHs contained in AArch64FI.
181 void EmitLOHs();
182
183 /// Emit instruction to set float register to zero.
184 void EmitFMov0(const MachineInstr &MI);
185
186 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
187
188 MInstToMCSymbol LOHInstToLabel;
189 };
190
191 } // end anonymous namespace
192
emitStartOfAsmFile(Module & M)193 void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
194 const Triple &TT = TM.getTargetTriple();
195
196 if (TT.isOSBinFormatCOFF()) {
197 // Emit an absolute @feat.00 symbol. This appears to be some kind of
198 // compiler features bitfield read by link.exe.
199 MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00"));
200 OutStreamer->BeginCOFFSymbolDef(S);
201 OutStreamer->EmitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
202 OutStreamer->EmitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
203 OutStreamer->EndCOFFSymbolDef();
204 int64_t Feat00Flags = 0;
205
206 if (M.getModuleFlag("cfguard")) {
207 Feat00Flags |= 0x800; // Object is CFG-aware.
208 }
209
210 if (M.getModuleFlag("ehcontguard")) {
211 Feat00Flags |= 0x4000; // Object also has EHCont.
212 }
213
214 OutStreamer->emitSymbolAttribute(S, MCSA_Global);
215 OutStreamer->emitAssignment(
216 S, MCConstantExpr::create(Feat00Flags, MMI->getContext()));
217 }
218
219 if (!TT.isOSBinFormatELF())
220 return;
221
222 // Assemble feature flags that may require creation of a note section.
223 unsigned Flags = 0;
224 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
225 M.getModuleFlag("branch-target-enforcement")))
226 if (BTE->getZExtValue())
227 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_BTI;
228
229 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
230 M.getModuleFlag("sign-return-address")))
231 if (Sign->getZExtValue())
232 Flags |= ELF::GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
233
234 if (Flags == 0)
235 return;
236
237 // Emit a .note.gnu.property section with the flags.
238 if (auto *TS = static_cast<AArch64TargetStreamer *>(
239 OutStreamer->getTargetStreamer()))
240 TS->emitNoteSection(Flags);
241 }
242
emitFunctionHeaderComment()243 void AArch64AsmPrinter::emitFunctionHeaderComment() {
244 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
245 Optional<std::string> OutlinerString = FI->getOutliningStyle();
246 if (OutlinerString != None)
247 OutStreamer->GetCommentOS() << ' ' << OutlinerString;
248 }
249
LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr & MI)250 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
251 {
252 const Function &F = MF->getFunction();
253 if (F.hasFnAttribute("patchable-function-entry")) {
254 unsigned Num;
255 if (F.getFnAttribute("patchable-function-entry")
256 .getValueAsString()
257 .getAsInteger(10, Num))
258 return;
259 emitNops(Num);
260 return;
261 }
262
263 EmitSled(MI, SledKind::FUNCTION_ENTER);
264 }
265
LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr & MI)266 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
267 {
268 EmitSled(MI, SledKind::FUNCTION_EXIT);
269 }
270
LowerPATCHABLE_TAIL_CALL(const MachineInstr & MI)271 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
272 {
273 EmitSled(MI, SledKind::TAIL_CALL);
274 }
275
EmitSled(const MachineInstr & MI,SledKind Kind)276 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
277 {
278 static const int8_t NoopsInSledCount = 7;
279 // We want to emit the following pattern:
280 //
281 // .Lxray_sled_N:
282 // ALIGN
283 // B #32
284 // ; 7 NOP instructions (28 bytes)
285 // .tmpN
286 //
287 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
288 // over the full 32 bytes (8 instructions) with the following pattern:
289 //
290 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
291 // LDR W0, #12 ; W0 := function ID
292 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
293 // BLR X16 ; call the tracing trampoline
294 // ;DATA: 32 bits of function ID
295 // ;DATA: lower 32 bits of the address of the trampoline
296 // ;DATA: higher 32 bits of the address of the trampoline
297 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
298 //
299 OutStreamer->emitCodeAlignment(4);
300 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
301 OutStreamer->emitLabel(CurSled);
302 auto Target = OutContext.createTempSymbol();
303
304 // Emit "B #32" instruction, which jumps over the next 28 bytes.
305 // The operand has to be the number of 4-byte instructions to jump over,
306 // including the current instruction.
307 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
308
309 for (int8_t I = 0; I < NoopsInSledCount; I++)
310 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
311
312 OutStreamer->emitLabel(Target);
313 recordSled(CurSled, MI, Kind, 2);
314 }
315
LowerHWASAN_CHECK_MEMACCESS(const MachineInstr & MI)316 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
317 Register Reg = MI.getOperand(0).getReg();
318 bool IsShort =
319 MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES;
320 uint32_t AccessInfo = MI.getOperand(1).getImm();
321 MCSymbol *&Sym =
322 HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, IsShort, AccessInfo)];
323 if (!Sym) {
324 // FIXME: Make this work on non-ELF.
325 if (!TM.getTargetTriple().isOSBinFormatELF())
326 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
327
328 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
329 utostr(AccessInfo);
330 if (IsShort)
331 SymName += "_short_v2";
332 Sym = OutContext.getOrCreateSymbol(SymName);
333 }
334
335 EmitToStreamer(*OutStreamer,
336 MCInstBuilder(AArch64::BL)
337 .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
338 }
339
EmitHwasanMemaccessSymbols(Module & M)340 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
341 if (HwasanMemaccessSymbols.empty())
342 return;
343
344 const Triple &TT = TM.getTargetTriple();
345 assert(TT.isOSBinFormatELF());
346 std::unique_ptr<MCSubtargetInfo> STI(
347 TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
348 assert(STI && "Unable to create subtarget info");
349
350 MCSymbol *HwasanTagMismatchV1Sym =
351 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
352 MCSymbol *HwasanTagMismatchV2Sym =
353 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
354
355 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
356 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
357 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
358 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
359
360 for (auto &P : HwasanMemaccessSymbols) {
361 unsigned Reg = std::get<0>(P.first);
362 bool IsShort = std::get<1>(P.first);
363 uint32_t AccessInfo = std::get<2>(P.first);
364 const MCSymbolRefExpr *HwasanTagMismatchRef =
365 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
366 MCSymbol *Sym = P.second;
367
368 bool HasMatchAllTag =
369 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
370 uint8_t MatchAllTag =
371 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
372 unsigned Size =
373 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
374 bool CompileKernel =
375 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
376
377 OutStreamer->SwitchSection(OutContext.getELFSection(
378 ".text.hot", ELF::SHT_PROGBITS,
379 ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
380 Sym->getName(), /*IsComdat=*/true));
381
382 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
383 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
384 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
385 OutStreamer->emitLabel(Sym);
386
387 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
388 .addReg(AArch64::X16)
389 .addReg(Reg)
390 .addImm(4)
391 .addImm(55),
392 *STI);
393 OutStreamer->emitInstruction(
394 MCInstBuilder(AArch64::LDRBBroX)
395 .addReg(AArch64::W16)
396 .addReg(IsShort ? AArch64::X20 : AArch64::X9)
397 .addReg(AArch64::X16)
398 .addImm(0)
399 .addImm(0),
400 *STI);
401 OutStreamer->emitInstruction(
402 MCInstBuilder(AArch64::SUBSXrs)
403 .addReg(AArch64::XZR)
404 .addReg(AArch64::X16)
405 .addReg(Reg)
406 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
407 *STI);
408 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
409 OutStreamer->emitInstruction(
410 MCInstBuilder(AArch64::Bcc)
411 .addImm(AArch64CC::NE)
412 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
413 OutContext)),
414 *STI);
415 MCSymbol *ReturnSym = OutContext.createTempSymbol();
416 OutStreamer->emitLabel(ReturnSym);
417 OutStreamer->emitInstruction(
418 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
419 OutStreamer->emitLabel(HandleMismatchOrPartialSym);
420
421 if (HasMatchAllTag) {
422 OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
423 .addReg(AArch64::X16)
424 .addReg(Reg)
425 .addImm(56)
426 .addImm(63),
427 *STI);
428 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
429 .addReg(AArch64::XZR)
430 .addReg(AArch64::X16)
431 .addImm(MatchAllTag)
432 .addImm(0),
433 *STI);
434 OutStreamer->emitInstruction(
435 MCInstBuilder(AArch64::Bcc)
436 .addImm(AArch64CC::EQ)
437 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
438 *STI);
439 }
440
441 if (IsShort) {
442 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
443 .addReg(AArch64::WZR)
444 .addReg(AArch64::W16)
445 .addImm(15)
446 .addImm(0),
447 *STI);
448 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
449 OutStreamer->emitInstruction(
450 MCInstBuilder(AArch64::Bcc)
451 .addImm(AArch64CC::HI)
452 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
453 *STI);
454
455 OutStreamer->emitInstruction(
456 MCInstBuilder(AArch64::ANDXri)
457 .addReg(AArch64::X17)
458 .addReg(Reg)
459 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
460 *STI);
461 if (Size != 1)
462 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
463 .addReg(AArch64::X17)
464 .addReg(AArch64::X17)
465 .addImm(Size - 1)
466 .addImm(0),
467 *STI);
468 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
469 .addReg(AArch64::WZR)
470 .addReg(AArch64::W16)
471 .addReg(AArch64::W17)
472 .addImm(0),
473 *STI);
474 OutStreamer->emitInstruction(
475 MCInstBuilder(AArch64::Bcc)
476 .addImm(AArch64CC::LS)
477 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
478 *STI);
479
480 OutStreamer->emitInstruction(
481 MCInstBuilder(AArch64::ORRXri)
482 .addReg(AArch64::X16)
483 .addReg(Reg)
484 .addImm(AArch64_AM::encodeLogicalImmediate(0xf, 64)),
485 *STI);
486 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
487 .addReg(AArch64::W16)
488 .addReg(AArch64::X16)
489 .addImm(0),
490 *STI);
491 OutStreamer->emitInstruction(
492 MCInstBuilder(AArch64::SUBSXrs)
493 .addReg(AArch64::XZR)
494 .addReg(AArch64::X16)
495 .addReg(Reg)
496 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSR, 56)),
497 *STI);
498 OutStreamer->emitInstruction(
499 MCInstBuilder(AArch64::Bcc)
500 .addImm(AArch64CC::EQ)
501 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
502 *STI);
503
504 OutStreamer->emitLabel(HandleMismatchSym);
505 }
506
507 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
508 .addReg(AArch64::SP)
509 .addReg(AArch64::X0)
510 .addReg(AArch64::X1)
511 .addReg(AArch64::SP)
512 .addImm(-32),
513 *STI);
514 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
515 .addReg(AArch64::FP)
516 .addReg(AArch64::LR)
517 .addReg(AArch64::SP)
518 .addImm(29),
519 *STI);
520
521 if (Reg != AArch64::X0)
522 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
523 .addReg(AArch64::X0)
524 .addReg(AArch64::XZR)
525 .addReg(Reg)
526 .addImm(0),
527 *STI);
528 OutStreamer->emitInstruction(
529 MCInstBuilder(AArch64::MOVZXi)
530 .addReg(AArch64::X1)
531 .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask)
532 .addImm(0),
533 *STI);
534
535 if (CompileKernel) {
536 // The Linux kernel's dynamic loader doesn't support GOT relative
537 // relocations, but it doesn't support late binding either, so just call
538 // the function directly.
539 OutStreamer->emitInstruction(
540 MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
541 } else {
542 // Intentionally load the GOT entry and branch to it, rather than possibly
543 // late binding the function, which may clobber the registers before we
544 // have a chance to save them.
545 OutStreamer->emitInstruction(
546 MCInstBuilder(AArch64::ADRP)
547 .addReg(AArch64::X16)
548 .addExpr(AArch64MCExpr::create(
549 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
550 OutContext)),
551 *STI);
552 OutStreamer->emitInstruction(
553 MCInstBuilder(AArch64::LDRXui)
554 .addReg(AArch64::X16)
555 .addReg(AArch64::X16)
556 .addExpr(AArch64MCExpr::create(
557 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
558 OutContext)),
559 *STI);
560 OutStreamer->emitInstruction(
561 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
562 }
563 }
564 }
565
emitEndOfAsmFile(Module & M)566 void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
567 EmitHwasanMemaccessSymbols(M);
568
569 const Triple &TT = TM.getTargetTriple();
570 if (TT.isOSBinFormatMachO()) {
571 // Funny Darwin hack: This flag tells the linker that no global symbols
572 // contain code that falls through to other global symbols (e.g. the obvious
573 // implementation of multiple entry points). If this doesn't occur, the
574 // linker can safely perform dead code stripping. Since LLVM never
575 // generates code that does this, it is always safe to set.
576 OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
577 }
578
579 // Emit stack and fault map information.
580 emitStackMaps(SM);
581 FM.serializeToFaultMapSection();
582
583 }
584
EmitLOHs()585 void AArch64AsmPrinter::EmitLOHs() {
586 SmallVector<MCSymbol *, 3> MCArgs;
587
588 for (const auto &D : AArch64FI->getLOHContainer()) {
589 for (const MachineInstr *MI : D.getArgs()) {
590 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
591 assert(LabelIt != LOHInstToLabel.end() &&
592 "Label hasn't been inserted for LOH related instruction");
593 MCArgs.push_back(LabelIt->second);
594 }
595 OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
596 MCArgs.clear();
597 }
598 }
599
emitFunctionBodyEnd()600 void AArch64AsmPrinter::emitFunctionBodyEnd() {
601 if (!AArch64FI->getLOHRelated().empty())
602 EmitLOHs();
603 }
604
605 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
GetCPISymbol(unsigned CPID) const606 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
607 // Darwin uses a linker-private symbol name for constant-pools (to
608 // avoid addends on the relocation?), ELF has no such concept and
609 // uses a normal private symbol.
610 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
611 return OutContext.getOrCreateSymbol(
612 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
613 Twine(getFunctionNumber()) + "_" + Twine(CPID));
614
615 return AsmPrinter::GetCPISymbol(CPID);
616 }
617
printOperand(const MachineInstr * MI,unsigned OpNum,raw_ostream & O)618 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
619 raw_ostream &O) {
620 const MachineOperand &MO = MI->getOperand(OpNum);
621 switch (MO.getType()) {
622 default:
623 llvm_unreachable("<unknown operand type>");
624 case MachineOperand::MO_Register: {
625 Register Reg = MO.getReg();
626 assert(Register::isPhysicalRegister(Reg));
627 assert(!MO.getSubReg() && "Subregs should be eliminated!");
628 O << AArch64InstPrinter::getRegisterName(Reg);
629 break;
630 }
631 case MachineOperand::MO_Immediate: {
632 O << MO.getImm();
633 break;
634 }
635 case MachineOperand::MO_GlobalAddress: {
636 PrintSymbolOperand(MO, O);
637 break;
638 }
639 case MachineOperand::MO_BlockAddress: {
640 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
641 Sym->print(O, MAI);
642 break;
643 }
644 }
645 }
646
printAsmMRegister(const MachineOperand & MO,char Mode,raw_ostream & O)647 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
648 raw_ostream &O) {
649 Register Reg = MO.getReg();
650 switch (Mode) {
651 default:
652 return true; // Unknown mode.
653 case 'w':
654 Reg = getWRegFromXReg(Reg);
655 break;
656 case 'x':
657 Reg = getXRegFromWReg(Reg);
658 break;
659 }
660
661 O << AArch64InstPrinter::getRegisterName(Reg);
662 return false;
663 }
664
665 // Prints the register in MO using class RC using the offset in the
666 // new register class. This should not be used for cross class
667 // printing.
printAsmRegInClass(const MachineOperand & MO,const TargetRegisterClass * RC,unsigned AltName,raw_ostream & O)668 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
669 const TargetRegisterClass *RC,
670 unsigned AltName, raw_ostream &O) {
671 assert(MO.isReg() && "Should only get here with a register!");
672 const TargetRegisterInfo *RI = STI->getRegisterInfo();
673 Register Reg = MO.getReg();
674 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
675 if (!RI->regsOverlap(RegToPrint, Reg))
676 return true;
677 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
678 return false;
679 }
680
PrintAsmOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)681 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
682 const char *ExtraCode, raw_ostream &O) {
683 const MachineOperand &MO = MI->getOperand(OpNum);
684
685 // First try the generic code, which knows about modifiers like 'c' and 'n'.
686 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
687 return false;
688
689 // Does this asm operand have a single letter operand modifier?
690 if (ExtraCode && ExtraCode[0]) {
691 if (ExtraCode[1] != 0)
692 return true; // Unknown modifier.
693
694 switch (ExtraCode[0]) {
695 default:
696 return true; // Unknown modifier.
697 case 'w': // Print W register
698 case 'x': // Print X register
699 if (MO.isReg())
700 return printAsmMRegister(MO, ExtraCode[0], O);
701 if (MO.isImm() && MO.getImm() == 0) {
702 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
703 O << AArch64InstPrinter::getRegisterName(Reg);
704 return false;
705 }
706 printOperand(MI, OpNum, O);
707 return false;
708 case 'b': // Print B register.
709 case 'h': // Print H register.
710 case 's': // Print S register.
711 case 'd': // Print D register.
712 case 'q': // Print Q register.
713 case 'z': // Print Z register.
714 if (MO.isReg()) {
715 const TargetRegisterClass *RC;
716 switch (ExtraCode[0]) {
717 case 'b':
718 RC = &AArch64::FPR8RegClass;
719 break;
720 case 'h':
721 RC = &AArch64::FPR16RegClass;
722 break;
723 case 's':
724 RC = &AArch64::FPR32RegClass;
725 break;
726 case 'd':
727 RC = &AArch64::FPR64RegClass;
728 break;
729 case 'q':
730 RC = &AArch64::FPR128RegClass;
731 break;
732 case 'z':
733 RC = &AArch64::ZPRRegClass;
734 break;
735 default:
736 return true;
737 }
738 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
739 }
740 printOperand(MI, OpNum, O);
741 return false;
742 }
743 }
744
745 // According to ARM, we should emit x and v registers unless we have a
746 // modifier.
747 if (MO.isReg()) {
748 Register Reg = MO.getReg();
749
750 // If this is a w or x register, print an x register.
751 if (AArch64::GPR32allRegClass.contains(Reg) ||
752 AArch64::GPR64allRegClass.contains(Reg))
753 return printAsmMRegister(MO, 'x', O);
754
755 unsigned AltName = AArch64::NoRegAltName;
756 const TargetRegisterClass *RegClass;
757 if (AArch64::ZPRRegClass.contains(Reg)) {
758 RegClass = &AArch64::ZPRRegClass;
759 } else if (AArch64::PPRRegClass.contains(Reg)) {
760 RegClass = &AArch64::PPRRegClass;
761 } else {
762 RegClass = &AArch64::FPR128RegClass;
763 AltName = AArch64::vreg;
764 }
765
766 // If this is a b, h, s, d, or q register, print it as a v register.
767 return printAsmRegInClass(MO, RegClass, AltName, O);
768 }
769
770 printOperand(MI, OpNum, O);
771 return false;
772 }
773
PrintAsmMemoryOperand(const MachineInstr * MI,unsigned OpNum,const char * ExtraCode,raw_ostream & O)774 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
775 unsigned OpNum,
776 const char *ExtraCode,
777 raw_ostream &O) {
778 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
779 return true; // Unknown modifier.
780
781 const MachineOperand &MO = MI->getOperand(OpNum);
782 assert(MO.isReg() && "unexpected inline asm memory operand");
783 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
784 return false;
785 }
786
PrintDebugValueComment(const MachineInstr * MI,raw_ostream & OS)787 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
788 raw_ostream &OS) {
789 unsigned NOps = MI->getNumOperands();
790 assert(NOps == 4);
791 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
792 // cast away const; DIetc do not take const operands for some reason.
793 OS << MI->getDebugVariable()->getName();
794 OS << " <- ";
795 // Frame address. Currently handles register +- offset only.
796 assert(MI->isIndirectDebugValue());
797 OS << '[';
798 for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(),
799 MI->debug_operands().end());
800 I < E; ++I) {
801 if (I != 0)
802 OS << ", ";
803 printOperand(MI, I, OS);
804 }
805 OS << ']';
806 OS << "+";
807 printOperand(MI, NOps - 2, OS);
808 }
809
emitJumpTableInfo()810 void AArch64AsmPrinter::emitJumpTableInfo() {
811 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
812 if (!MJTI) return;
813
814 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
815 if (JT.empty()) return;
816
817 const Function &F = MF->getFunction();
818 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
819 bool JTInDiffSection =
820 !STI->isTargetCOFF() ||
821 !TLOF.shouldPutJumpTableInFunctionSection(
822 MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
823 F);
824 if (JTInDiffSection) {
825 // Drop it in the readonly section.
826 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
827 OutStreamer->SwitchSection(ReadOnlySec);
828 }
829
830 auto AFI = MF->getInfo<AArch64FunctionInfo>();
831 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
832 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
833
834 // If this jump table was deleted, ignore it.
835 if (JTBBs.empty()) continue;
836
837 unsigned Size = AFI->getJumpTableEntrySize(JTI);
838 emitAlignment(Align(Size));
839 OutStreamer->emitLabel(GetJTISymbol(JTI));
840
841 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
842 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
843
844 for (auto *JTBB : JTBBs) {
845 const MCExpr *Value =
846 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
847
848 // Each entry is:
849 // .byte/.hword (LBB - Lbase)>>2
850 // or plain:
851 // .word LBB - Lbase
852 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
853 if (Size != 4)
854 Value = MCBinaryExpr::createLShr(
855 Value, MCConstantExpr::create(2, OutContext), OutContext);
856
857 OutStreamer->emitValue(Value, Size);
858 }
859 }
860 }
861
emitFunctionEntryLabel()862 void AArch64AsmPrinter::emitFunctionEntryLabel() {
863 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
864 MF->getFunction().getCallingConv() ==
865 CallingConv::AArch64_SVE_VectorCall ||
866 STI->getRegisterInfo()->hasSVEArgsOrReturn(MF)) {
867 auto *TS =
868 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
869 TS->emitDirectiveVariantPCS(CurrentFnSym);
870 }
871
872 return AsmPrinter::emitFunctionEntryLabel();
873 }
874
875 /// Small jump tables contain an unsigned byte or half, representing the offset
876 /// from the lowest-addressed possible destination to the desired basic
877 /// block. Since all instructions are 4-byte aligned, this is further compressed
878 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
879 /// materialize the correct destination we need:
880 ///
881 /// adr xDest, .LBB0_0
882 /// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
883 /// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
LowerJumpTableDest(llvm::MCStreamer & OutStreamer,const llvm::MachineInstr & MI)884 void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
885 const llvm::MachineInstr &MI) {
886 Register DestReg = MI.getOperand(0).getReg();
887 Register ScratchReg = MI.getOperand(1).getReg();
888 Register ScratchRegW =
889 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
890 Register TableReg = MI.getOperand(2).getReg();
891 Register EntryReg = MI.getOperand(3).getReg();
892 int JTIdx = MI.getOperand(4).getIndex();
893 int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
894
895 // This has to be first because the compression pass based its reachability
896 // calculations on the start of the JumpTableDest instruction.
897 auto Label =
898 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
899
900 // If we don't already have a symbol to use as the base, use the ADR
901 // instruction itself.
902 if (!Label) {
903 Label = MF->getContext().createTempSymbol();
904 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
905 OutStreamer.emitLabel(Label);
906 }
907
908 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
909 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
910 .addReg(DestReg)
911 .addExpr(LabelExpr));
912
913 // Load the number of instruction-steps to offset from the label.
914 unsigned LdrOpcode;
915 switch (Size) {
916 case 1: LdrOpcode = AArch64::LDRBBroX; break;
917 case 2: LdrOpcode = AArch64::LDRHHroX; break;
918 case 4: LdrOpcode = AArch64::LDRSWroX; break;
919 default:
920 llvm_unreachable("Unknown jump table size");
921 }
922
923 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
924 .addReg(Size == 4 ? ScratchReg : ScratchRegW)
925 .addReg(TableReg)
926 .addReg(EntryReg)
927 .addImm(0)
928 .addImm(Size == 1 ? 0 : 1));
929
930 // Add to the already materialized base label address, multiplying by 4 if
931 // compressed.
932 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
933 .addReg(DestReg)
934 .addReg(DestReg)
935 .addReg(ScratchReg)
936 .addImm(Size == 4 ? 0 : 2));
937 }
938
LowerSTACKMAP(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)939 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
940 const MachineInstr &MI) {
941 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
942
943 auto &Ctx = OutStreamer.getContext();
944 MCSymbol *MILabel = Ctx.createTempSymbol();
945 OutStreamer.emitLabel(MILabel);
946
947 SM.recordStackMap(*MILabel, MI);
948 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
949
950 // Scan ahead to trim the shadow.
951 const MachineBasicBlock &MBB = *MI.getParent();
952 MachineBasicBlock::const_iterator MII(MI);
953 ++MII;
954 while (NumNOPBytes > 0) {
955 if (MII == MBB.end() || MII->isCall() ||
956 MII->getOpcode() == AArch64::DBG_VALUE ||
957 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
958 MII->getOpcode() == TargetOpcode::STACKMAP)
959 break;
960 ++MII;
961 NumNOPBytes -= 4;
962 }
963
964 // Emit nops.
965 for (unsigned i = 0; i < NumNOPBytes; i += 4)
966 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
967 }
968
969 // Lower a patchpoint of the form:
970 // [<def>], <id>, <numBytes>, <target>, <numArgs>
LowerPATCHPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)971 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
972 const MachineInstr &MI) {
973 auto &Ctx = OutStreamer.getContext();
974 MCSymbol *MILabel = Ctx.createTempSymbol();
975 OutStreamer.emitLabel(MILabel);
976 SM.recordPatchPoint(*MILabel, MI);
977
978 PatchPointOpers Opers(&MI);
979
980 int64_t CallTarget = Opers.getCallTarget().getImm();
981 unsigned EncodedBytes = 0;
982 if (CallTarget) {
983 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
984 "High 16 bits of call target should be zero.");
985 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
986 EncodedBytes = 16;
987 // Materialize the jump address:
988 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
989 .addReg(ScratchReg)
990 .addImm((CallTarget >> 32) & 0xFFFF)
991 .addImm(32));
992 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
993 .addReg(ScratchReg)
994 .addReg(ScratchReg)
995 .addImm((CallTarget >> 16) & 0xFFFF)
996 .addImm(16));
997 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
998 .addReg(ScratchReg)
999 .addReg(ScratchReg)
1000 .addImm(CallTarget & 0xFFFF)
1001 .addImm(0));
1002 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1003 }
1004 // Emit padding.
1005 unsigned NumBytes = Opers.getNumPatchBytes();
1006 assert(NumBytes >= EncodedBytes &&
1007 "Patchpoint can't request size less than the length of a call.");
1008 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1009 "Invalid number of NOP bytes requested!");
1010 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1011 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1012 }
1013
LowerSTATEPOINT(MCStreamer & OutStreamer,StackMaps & SM,const MachineInstr & MI)1014 void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1015 const MachineInstr &MI) {
1016 StatepointOpers SOpers(&MI);
1017 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1018 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1019 for (unsigned i = 0; i < PatchBytes; i += 4)
1020 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1021 } else {
1022 // Lower call target and choose correct opcode
1023 const MachineOperand &CallTarget = SOpers.getCallTarget();
1024 MCOperand CallTargetMCOp;
1025 unsigned CallOpcode;
1026 switch (CallTarget.getType()) {
1027 case MachineOperand::MO_GlobalAddress:
1028 case MachineOperand::MO_ExternalSymbol:
1029 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1030 CallOpcode = AArch64::BL;
1031 break;
1032 case MachineOperand::MO_Immediate:
1033 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1034 CallOpcode = AArch64::BL;
1035 break;
1036 case MachineOperand::MO_Register:
1037 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1038 CallOpcode = AArch64::BLR;
1039 break;
1040 default:
1041 llvm_unreachable("Unsupported operand type in statepoint call target");
1042 break;
1043 }
1044
1045 EmitToStreamer(OutStreamer,
1046 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1047 }
1048
1049 auto &Ctx = OutStreamer.getContext();
1050 MCSymbol *MILabel = Ctx.createTempSymbol();
1051 OutStreamer.emitLabel(MILabel);
1052 SM.recordStatepoint(*MILabel, MI);
1053 }
1054
LowerFAULTING_OP(const MachineInstr & FaultingMI)1055 void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1056 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1057 // <opcode>, <operands>
1058
1059 Register DefRegister = FaultingMI.getOperand(0).getReg();
1060 FaultMaps::FaultKind FK =
1061 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1062 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1063 unsigned Opcode = FaultingMI.getOperand(3).getImm();
1064 unsigned OperandsBeginIdx = 4;
1065
1066 auto &Ctx = OutStreamer->getContext();
1067 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1068 OutStreamer->emitLabel(FaultingLabel);
1069
1070 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1071 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1072
1073 MCInst MI;
1074 MI.setOpcode(Opcode);
1075
1076 if (DefRegister != (Register)0)
1077 MI.addOperand(MCOperand::createReg(DefRegister));
1078
1079 for (auto I = FaultingMI.operands_begin() + OperandsBeginIdx,
1080 E = FaultingMI.operands_end();
1081 I != E; ++I) {
1082 MCOperand Dest;
1083 lowerOperand(*I, Dest);
1084 MI.addOperand(Dest);
1085 }
1086
1087 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1088 OutStreamer->emitInstruction(MI, getSubtargetInfo());
1089 }
1090
EmitFMov0(const MachineInstr & MI)1091 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
1092 Register DestReg = MI.getOperand(0).getReg();
1093 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
1094 // Convert H/S register to corresponding D register
1095 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1096 DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1097 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1098 DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1099 else
1100 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1101
1102 MCInst MOVI;
1103 MOVI.setOpcode(AArch64::MOVID);
1104 MOVI.addOperand(MCOperand::createReg(DestReg));
1105 MOVI.addOperand(MCOperand::createImm(0));
1106 EmitToStreamer(*OutStreamer, MOVI);
1107 } else {
1108 MCInst FMov;
1109 switch (MI.getOpcode()) {
1110 default: llvm_unreachable("Unexpected opcode");
1111 case AArch64::FMOVH0:
1112 FMov.setOpcode(AArch64::FMOVWHr);
1113 FMov.addOperand(MCOperand::createReg(DestReg));
1114 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1115 break;
1116 case AArch64::FMOVS0:
1117 FMov.setOpcode(AArch64::FMOVWSr);
1118 FMov.addOperand(MCOperand::createReg(DestReg));
1119 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1120 break;
1121 case AArch64::FMOVD0:
1122 FMov.setOpcode(AArch64::FMOVXDr);
1123 FMov.addOperand(MCOperand::createReg(DestReg));
1124 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1125 break;
1126 }
1127 EmitToStreamer(*OutStreamer, FMov);
1128 }
1129 }
1130
1131 // Simple pseudo-instructions have their lowering (with expansion to real
1132 // instructions) auto-generated.
1133 #include "AArch64GenMCPseudoLowering.inc"
1134
emitInstruction(const MachineInstr * MI)1135 void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
1136 // Do any auto-generated pseudo lowerings.
1137 if (emitPseudoExpansionLowering(*OutStreamer, MI))
1138 return;
1139
1140 if (AArch64FI->getLOHRelated().count(MI)) {
1141 // Generate a label for LOH related instruction
1142 MCSymbol *LOHLabel = createTempSymbol("loh");
1143 // Associate the instruction with the label
1144 LOHInstToLabel[MI] = LOHLabel;
1145 OutStreamer->emitLabel(LOHLabel);
1146 }
1147
1148 AArch64TargetStreamer *TS =
1149 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1150 // Do any manual lowerings.
1151 switch (MI->getOpcode()) {
1152 default:
1153 break;
1154 case AArch64::HINT: {
1155 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
1156 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
1157 // non-empty. If MI is the initial BTI, place the
1158 // __patchable_function_entries label after BTI.
1159 if (CurrentPatchableFunctionEntrySym &&
1160 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
1161 MI == &MF->front().front()) {
1162 int64_t Imm = MI->getOperand(0).getImm();
1163 if ((Imm & 32) && (Imm & 6)) {
1164 MCInst Inst;
1165 MCInstLowering.Lower(MI, Inst);
1166 EmitToStreamer(*OutStreamer, Inst);
1167 CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
1168 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
1169 return;
1170 }
1171 }
1172 break;
1173 }
1174 case AArch64::MOVMCSym: {
1175 Register DestReg = MI->getOperand(0).getReg();
1176 const MachineOperand &MO_Sym = MI->getOperand(1);
1177 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
1178 MCOperand Hi_MCSym, Lo_MCSym;
1179
1180 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
1181 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
1182
1183 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
1184 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
1185
1186 MCInst MovZ;
1187 MovZ.setOpcode(AArch64::MOVZXi);
1188 MovZ.addOperand(MCOperand::createReg(DestReg));
1189 MovZ.addOperand(Hi_MCSym);
1190 MovZ.addOperand(MCOperand::createImm(16));
1191 EmitToStreamer(*OutStreamer, MovZ);
1192
1193 MCInst MovK;
1194 MovK.setOpcode(AArch64::MOVKXi);
1195 MovK.addOperand(MCOperand::createReg(DestReg));
1196 MovK.addOperand(MCOperand::createReg(DestReg));
1197 MovK.addOperand(Lo_MCSym);
1198 MovK.addOperand(MCOperand::createImm(0));
1199 EmitToStreamer(*OutStreamer, MovK);
1200 return;
1201 }
1202 case AArch64::MOVIv2d_ns:
1203 // If the target has <rdar://problem/16473581>, lower this
1204 // instruction to movi.16b instead.
1205 if (STI->hasZeroCycleZeroingFPWorkaround() &&
1206 MI->getOperand(1).getImm() == 0) {
1207 MCInst TmpInst;
1208 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
1209 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1210 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
1211 EmitToStreamer(*OutStreamer, TmpInst);
1212 return;
1213 }
1214 break;
1215
1216 case AArch64::DBG_VALUE:
1217 case AArch64::DBG_VALUE_LIST: {
1218 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
1219 SmallString<128> TmpStr;
1220 raw_svector_ostream OS(TmpStr);
1221 PrintDebugValueComment(MI, OS);
1222 OutStreamer->emitRawText(StringRef(OS.str()));
1223 }
1224 return;
1225
1226 case AArch64::EMITBKEY: {
1227 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
1228 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
1229 ExceptionHandlingType != ExceptionHandling::ARM)
1230 return;
1231
1232 if (getFunctionCFISectionType(*MF) == CFISection::None)
1233 return;
1234
1235 OutStreamer->emitCFIBKeyFrame();
1236 return;
1237 }
1238 }
1239
1240 // Tail calls use pseudo instructions so they have the proper code-gen
1241 // attributes (isCall, isReturn, etc.). We lower them to the real
1242 // instruction here.
1243 case AArch64::TCRETURNri:
1244 case AArch64::TCRETURNriBTI:
1245 case AArch64::TCRETURNriALL: {
1246 MCInst TmpInst;
1247 TmpInst.setOpcode(AArch64::BR);
1248 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
1249 EmitToStreamer(*OutStreamer, TmpInst);
1250 return;
1251 }
1252 case AArch64::TCRETURNdi: {
1253 MCOperand Dest;
1254 MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
1255 MCInst TmpInst;
1256 TmpInst.setOpcode(AArch64::B);
1257 TmpInst.addOperand(Dest);
1258 EmitToStreamer(*OutStreamer, TmpInst);
1259 return;
1260 }
1261 case AArch64::SpeculationBarrierISBDSBEndBB: {
1262 // Print DSB SYS + ISB
1263 MCInst TmpInstDSB;
1264 TmpInstDSB.setOpcode(AArch64::DSB);
1265 TmpInstDSB.addOperand(MCOperand::createImm(0xf));
1266 EmitToStreamer(*OutStreamer, TmpInstDSB);
1267 MCInst TmpInstISB;
1268 TmpInstISB.setOpcode(AArch64::ISB);
1269 TmpInstISB.addOperand(MCOperand::createImm(0xf));
1270 EmitToStreamer(*OutStreamer, TmpInstISB);
1271 return;
1272 }
1273 case AArch64::SpeculationBarrierSBEndBB: {
1274 // Print SB
1275 MCInst TmpInstSB;
1276 TmpInstSB.setOpcode(AArch64::SB);
1277 EmitToStreamer(*OutStreamer, TmpInstSB);
1278 return;
1279 }
1280 case AArch64::TLSDESC_CALLSEQ: {
1281 /// lower this to:
1282 /// adrp x0, :tlsdesc:var
1283 /// ldr x1, [x0, #:tlsdesc_lo12:var]
1284 /// add x0, x0, #:tlsdesc_lo12:var
1285 /// .tlsdesccall var
1286 /// blr x1
1287 /// (TPIDR_EL0 offset now in x0)
1288 const MachineOperand &MO_Sym = MI->getOperand(0);
1289 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
1290 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
1291 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
1292 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
1293 MCInstLowering.lowerOperand(MO_Sym, Sym);
1294 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
1295 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
1296
1297 MCInst Adrp;
1298 Adrp.setOpcode(AArch64::ADRP);
1299 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
1300 Adrp.addOperand(SymTLSDesc);
1301 EmitToStreamer(*OutStreamer, Adrp);
1302
1303 MCInst Ldr;
1304 if (STI->isTargetILP32()) {
1305 Ldr.setOpcode(AArch64::LDRWui);
1306 Ldr.addOperand(MCOperand::createReg(AArch64::W1));
1307 } else {
1308 Ldr.setOpcode(AArch64::LDRXui);
1309 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
1310 }
1311 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
1312 Ldr.addOperand(SymTLSDescLo12);
1313 Ldr.addOperand(MCOperand::createImm(0));
1314 EmitToStreamer(*OutStreamer, Ldr);
1315
1316 MCInst Add;
1317 if (STI->isTargetILP32()) {
1318 Add.setOpcode(AArch64::ADDWri);
1319 Add.addOperand(MCOperand::createReg(AArch64::W0));
1320 Add.addOperand(MCOperand::createReg(AArch64::W0));
1321 } else {
1322 Add.setOpcode(AArch64::ADDXri);
1323 Add.addOperand(MCOperand::createReg(AArch64::X0));
1324 Add.addOperand(MCOperand::createReg(AArch64::X0));
1325 }
1326 Add.addOperand(SymTLSDescLo12);
1327 Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
1328 EmitToStreamer(*OutStreamer, Add);
1329
1330 // Emit a relocation-annotation. This expands to no code, but requests
1331 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
1332 MCInst TLSDescCall;
1333 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
1334 TLSDescCall.addOperand(Sym);
1335 EmitToStreamer(*OutStreamer, TLSDescCall);
1336
1337 MCInst Blr;
1338 Blr.setOpcode(AArch64::BLR);
1339 Blr.addOperand(MCOperand::createReg(AArch64::X1));
1340 EmitToStreamer(*OutStreamer, Blr);
1341
1342 return;
1343 }
1344
1345 case AArch64::JumpTableDest32:
1346 case AArch64::JumpTableDest16:
1347 case AArch64::JumpTableDest8:
1348 LowerJumpTableDest(*OutStreamer, *MI);
1349 return;
1350
1351 case AArch64::FMOVH0:
1352 case AArch64::FMOVS0:
1353 case AArch64::FMOVD0:
1354 EmitFMov0(*MI);
1355 return;
1356
1357 case TargetOpcode::STACKMAP:
1358 return LowerSTACKMAP(*OutStreamer, SM, *MI);
1359
1360 case TargetOpcode::PATCHPOINT:
1361 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
1362
1363 case TargetOpcode::STATEPOINT:
1364 return LowerSTATEPOINT(*OutStreamer, SM, *MI);
1365
1366 case TargetOpcode::FAULTING_OP:
1367 return LowerFAULTING_OP(*MI);
1368
1369 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
1370 LowerPATCHABLE_FUNCTION_ENTER(*MI);
1371 return;
1372
1373 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
1374 LowerPATCHABLE_FUNCTION_EXIT(*MI);
1375 return;
1376
1377 case TargetOpcode::PATCHABLE_TAIL_CALL:
1378 LowerPATCHABLE_TAIL_CALL(*MI);
1379 return;
1380
1381 case AArch64::HWASAN_CHECK_MEMACCESS:
1382 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
1383 LowerHWASAN_CHECK_MEMACCESS(*MI);
1384 return;
1385
1386 case AArch64::SEH_StackAlloc:
1387 TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1388 return;
1389
1390 case AArch64::SEH_SaveFPLR:
1391 TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1392 return;
1393
1394 case AArch64::SEH_SaveFPLR_X:
1395 assert(MI->getOperand(0).getImm() < 0 &&
1396 "Pre increment SEH opcode must have a negative offset");
1397 TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1398 return;
1399
1400 case AArch64::SEH_SaveReg:
1401 TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1402 MI->getOperand(1).getImm());
1403 return;
1404
1405 case AArch64::SEH_SaveReg_X:
1406 assert(MI->getOperand(1).getImm() < 0 &&
1407 "Pre increment SEH opcode must have a negative offset");
1408 TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1409 -MI->getOperand(1).getImm());
1410 return;
1411
1412 case AArch64::SEH_SaveRegP:
1413 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
1414 MI->getOperand(0).getImm() <= 28) {
1415 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
1416 "Register paired with LR must be odd");
1417 TS->EmitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
1418 MI->getOperand(2).getImm());
1419 return;
1420 }
1421 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1422 "Non-consecutive registers not allowed for save_regp");
1423 TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1424 MI->getOperand(2).getImm());
1425 return;
1426
1427 case AArch64::SEH_SaveRegP_X:
1428 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1429 "Non-consecutive registers not allowed for save_regp_x");
1430 assert(MI->getOperand(2).getImm() < 0 &&
1431 "Pre increment SEH opcode must have a negative offset");
1432 TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1433 -MI->getOperand(2).getImm());
1434 return;
1435
1436 case AArch64::SEH_SaveFReg:
1437 TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1438 MI->getOperand(1).getImm());
1439 return;
1440
1441 case AArch64::SEH_SaveFReg_X:
1442 assert(MI->getOperand(1).getImm() < 0 &&
1443 "Pre increment SEH opcode must have a negative offset");
1444 TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1445 -MI->getOperand(1).getImm());
1446 return;
1447
1448 case AArch64::SEH_SaveFRegP:
1449 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1450 "Non-consecutive registers not allowed for save_regp");
1451 TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1452 MI->getOperand(2).getImm());
1453 return;
1454
1455 case AArch64::SEH_SaveFRegP_X:
1456 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1457 "Non-consecutive registers not allowed for save_regp_x");
1458 assert(MI->getOperand(2).getImm() < 0 &&
1459 "Pre increment SEH opcode must have a negative offset");
1460 TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1461 -MI->getOperand(2).getImm());
1462 return;
1463
1464 case AArch64::SEH_SetFP:
1465 TS->EmitARM64WinCFISetFP();
1466 return;
1467
1468 case AArch64::SEH_AddFP:
1469 TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1470 return;
1471
1472 case AArch64::SEH_Nop:
1473 TS->EmitARM64WinCFINop();
1474 return;
1475
1476 case AArch64::SEH_PrologEnd:
1477 TS->EmitARM64WinCFIPrologEnd();
1478 return;
1479
1480 case AArch64::SEH_EpilogStart:
1481 TS->EmitARM64WinCFIEpilogStart();
1482 return;
1483
1484 case AArch64::SEH_EpilogEnd:
1485 TS->EmitARM64WinCFIEpilogEnd();
1486 return;
1487 }
1488
1489 // Finally, do the automated lowerings for everything else.
1490 MCInst TmpInst;
1491 MCInstLowering.Lower(MI, TmpInst);
1492 EmitToStreamer(*OutStreamer, TmpInst);
1493 }
1494
1495 // Force static initialization.
LLVMInitializeAArch64AsmPrinter()1496 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter() {
1497 RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1498 RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1499 RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1500 RegisterAsmPrinter<AArch64AsmPrinter> W(getTheARM64_32Target());
1501 RegisterAsmPrinter<AArch64AsmPrinter> V(getTheAArch64_32Target());
1502 }
1503