LLVM 20.0.0git
AArch64AsmPrinter.cpp
Go to the documentation of this file.
1//===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a printer that converts from our internal representation
10// of machine-dependent LLVM code to the AArch64 assembly language.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64.h"
15#include "AArch64MCInstLower.h"
17#include "AArch64RegisterInfo.h"
18#include "AArch64Subtarget.h"
29#include "llvm/ADT/StringRef.h"
30#include "llvm/ADT/Twine.h"
44#include "llvm/IR/DataLayout.h"
46#include "llvm/IR/Module.h"
47#include "llvm/MC/MCAsmInfo.h"
48#include "llvm/MC/MCContext.h"
49#include "llvm/MC/MCInst.h"
53#include "llvm/MC/MCStreamer.h"
54#include "llvm/MC/MCSymbol.h"
63#include <algorithm>
64#include <cassert>
65#include <cstdint>
66#include <map>
67#include <memory>
68
69using namespace llvm;
70
73 "aarch64-ptrauth-auth-checks", cl::Hidden,
74 cl::values(clEnumValN(Unchecked, "none", "don't test for failure"),
75 clEnumValN(Poison, "poison", "poison on failure"),
76 clEnumValN(Trap, "trap", "trap on failure")),
77 cl::desc("Check pointer authentication auth/resign failures"),
79
80#define DEBUG_TYPE "asm-printer"
81
82namespace {
83
84class AArch64AsmPrinter : public AsmPrinter {
85 AArch64MCInstLower MCInstLowering;
86 FaultMaps FM;
87 const AArch64Subtarget *STI;
88 bool ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = false;
89
90public:
91 AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
92 : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
93 FM(*this) {}
94
95 StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
96
97 /// Wrapper for MCInstLowering.lowerOperand() for the
98 /// tblgen'erated pseudo lowering.
99 bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
100 return MCInstLowering.lowerOperand(MO, MCOp);
101 }
102
103 const MCExpr *lowerConstantPtrAuth(const ConstantPtrAuth &CPA) override;
104
105 const MCExpr *lowerBlockAddressConstant(const BlockAddress &BA) override;
106
107 void emitStartOfAsmFile(Module &M) override;
108 void emitJumpTableInfo() override;
109 std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
111 getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr,
112 const MCSymbol *BranchLabel) const override;
113
114 void emitFunctionEntryLabel() override;
115
116 void emitXXStructor(const DataLayout &DL, const Constant *CV) override;
117
118 void LowerJumpTableDest(MCStreamer &OutStreamer, const MachineInstr &MI);
119
120 void LowerHardenedBRJumpTable(const MachineInstr &MI);
121
122 void LowerMOPS(MCStreamer &OutStreamer, const MachineInstr &MI);
123
124 void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
125 const MachineInstr &MI);
126 void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
127 const MachineInstr &MI);
128 void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
129 const MachineInstr &MI);
130 void LowerFAULTING_OP(const MachineInstr &MI);
131
132 void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
133 void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
134 void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
135 void LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI, bool Typed);
136
137 typedef std::tuple<unsigned, bool, uint32_t, bool, uint64_t>
138 HwasanMemaccessTuple;
139 std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
140 void LowerKCFI_CHECK(const MachineInstr &MI);
141 void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
142 void emitHwasanMemaccessSymbols(Module &M);
143
144 void emitSled(const MachineInstr &MI, SledKind Kind);
145
146 // Emit the sequence for BRA/BLRA (authenticate + branch/call).
147 void emitPtrauthBranch(const MachineInstr *MI);
148
149 // Emit the sequence for AUT or AUTPAC.
150 void emitPtrauthAuthResign(const MachineInstr *MI);
151
152 // Emit the sequence to compute a discriminator into x17, or reuse AddrDisc.
153 unsigned emitPtrauthDiscriminator(uint16_t Disc, unsigned AddrDisc,
154 unsigned &InstsEmitted);
155
156 // Emit the sequence for LOADauthptrstatic
157 void LowerLOADauthptrstatic(const MachineInstr &MI);
158
159 // Emit the sequence for LOADgotPAC/MOVaddrPAC (either GOT adrp-ldr or
160 // adrp-add followed by PAC sign)
161 void LowerMOVaddrPAC(const MachineInstr &MI);
162
163 /// tblgen'erated driver function for lowering simple MI->MC
164 /// pseudo instructions.
165 bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
166
167 void emitInstruction(const MachineInstr *MI) override;
168
169 void emitFunctionHeaderComment() override;
170
171 void getAnalysisUsage(AnalysisUsage &AU) const override {
173 AU.setPreservesAll();
174 }
175
176 bool runOnMachineFunction(MachineFunction &MF) override {
177 AArch64FI = MF.getInfo<AArch64FunctionInfo>();
178 STI = &MF.getSubtarget<AArch64Subtarget>();
179
181
182 if (STI->isTargetCOFF()) {
183 bool Local = MF.getFunction().hasLocalLinkage();
186 int Type =
188
189 OutStreamer->beginCOFFSymbolDef(CurrentFnSym);
190 OutStreamer->emitCOFFSymbolStorageClass(Scl);
191 OutStreamer->emitCOFFSymbolType(Type);
192 OutStreamer->endCOFFSymbolDef();
193 }
194
195 // Emit the rest of the function body.
197
198 // Emit the XRay table for this function.
200
201 // We didn't modify anything.
202 return false;
203 }
204
205 const MCExpr *lowerConstant(const Constant *CV) override;
206
207private:
208 void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
209 bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
210 bool printAsmRegInClass(const MachineOperand &MO,
211 const TargetRegisterClass *RC, unsigned AltName,
212 raw_ostream &O);
213
214 bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
215 const char *ExtraCode, raw_ostream &O) override;
216 bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
217 const char *ExtraCode, raw_ostream &O) override;
218
219 void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
220
221 void emitFunctionBodyEnd() override;
222 void emitGlobalAlias(const Module &M, const GlobalAlias &GA) override;
223
224 MCSymbol *GetCPISymbol(unsigned CPID) const override;
225 void emitEndOfAsmFile(Module &M) override;
226
227 AArch64FunctionInfo *AArch64FI = nullptr;
228
229 /// Emit the LOHs contained in AArch64FI.
230 void emitLOHs();
231
232 /// Emit instruction to set float register to zero.
233 void emitFMov0(const MachineInstr &MI);
234
235 using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
236
237 MInstToMCSymbol LOHInstToLabel;
238
240 return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
241 }
242
243 const MCSubtargetInfo *getIFuncMCSubtargetInfo() const override {
244 assert(STI);
245 return STI;
246 }
247 void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
248 MCSymbol *LazyPointer) override;
250 MCSymbol *LazyPointer) override;
251};
252
253} // end anonymous namespace
254
255void AArch64AsmPrinter::emitStartOfAsmFile(Module &M) {
256 const Triple &TT = TM.getTargetTriple();
257
258 if (TT.isOSBinFormatCOFF()) {
259 // Emit an absolute @feat.00 symbol
260 MCSymbol *S = MMI->getContext().getOrCreateSymbol(StringRef("@feat.00"));
261 OutStreamer->beginCOFFSymbolDef(S);
262 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_STATIC);
263 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_NULL);
264 OutStreamer->endCOFFSymbolDef();
265 int64_t Feat00Value = 0;
266
267 if (M.getModuleFlag("cfguard")) {
268 // Object is CFG-aware.
269 Feat00Value |= COFF::Feat00Flags::GuardCF;
270 }
271
272 if (M.getModuleFlag("ehcontguard")) {
273 // Object also has EHCont.
274 Feat00Value |= COFF::Feat00Flags::GuardEHCont;
275 }
276
277 if (M.getModuleFlag("ms-kernel")) {
278 // Object is compiled with /kernel.
279 Feat00Value |= COFF::Feat00Flags::Kernel;
280 }
281
282 OutStreamer->emitSymbolAttribute(S, MCSA_Global);
283 OutStreamer->emitAssignment(
284 S, MCConstantExpr::create(Feat00Value, MMI->getContext()));
285 }
286
287 if (!TT.isOSBinFormatELF())
288 return;
289
290 // Assemble feature flags that may require creation of a note section.
291 unsigned Flags = 0;
292 if (const auto *BTE = mdconst::extract_or_null<ConstantInt>(
293 M.getModuleFlag("branch-target-enforcement")))
294 if (!BTE->isZero())
296
297 if (const auto *GCS = mdconst::extract_or_null<ConstantInt>(
298 M.getModuleFlag("guarded-control-stack")))
299 if (!GCS->isZero())
301
302 if (const auto *Sign = mdconst::extract_or_null<ConstantInt>(
303 M.getModuleFlag("sign-return-address")))
304 if (!Sign->isZero())
306
307 uint64_t PAuthABIPlatform = -1;
308 if (const auto *PAP = mdconst::extract_or_null<ConstantInt>(
309 M.getModuleFlag("aarch64-elf-pauthabi-platform")))
310 PAuthABIPlatform = PAP->getZExtValue();
311 uint64_t PAuthABIVersion = -1;
312 if (const auto *PAV = mdconst::extract_or_null<ConstantInt>(
313 M.getModuleFlag("aarch64-elf-pauthabi-version")))
314 PAuthABIVersion = PAV->getZExtValue();
315
316 // Emit a .note.gnu.property section with the flags.
317 auto *TS =
318 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
319 TS->emitNoteSection(Flags, PAuthABIPlatform, PAuthABIVersion);
320}
321
322void AArch64AsmPrinter::emitFunctionHeaderComment() {
323 const AArch64FunctionInfo *FI = MF->getInfo<AArch64FunctionInfo>();
324 std::optional<std::string> OutlinerString = FI->getOutliningStyle();
325 if (OutlinerString != std::nullopt)
326 OutStreamer->getCommentOS() << ' ' << OutlinerString;
327}
328
329void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
330{
331 const Function &F = MF->getFunction();
332 if (F.hasFnAttribute("patchable-function-entry")) {
333 unsigned Num;
334 if (F.getFnAttribute("patchable-function-entry")
335 .getValueAsString()
336 .getAsInteger(10, Num))
337 return;
338 emitNops(Num);
339 return;
340 }
341
342 emitSled(MI, SledKind::FUNCTION_ENTER);
343}
344
345void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) {
346 emitSled(MI, SledKind::FUNCTION_EXIT);
347}
348
349void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) {
350 emitSled(MI, SledKind::TAIL_CALL);
351}
352
353void AArch64AsmPrinter::emitSled(const MachineInstr &MI, SledKind Kind) {
354 static const int8_t NoopsInSledCount = 7;
355 // We want to emit the following pattern:
356 //
357 // .Lxray_sled_N:
358 // ALIGN
359 // B #32
360 // ; 7 NOP instructions (28 bytes)
361 // .tmpN
362 //
363 // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
364 // over the full 32 bytes (8 instructions) with the following pattern:
365 //
366 // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
367 // LDR W17, #12 ; W17 := function ID
368 // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
369 // BLR X16 ; call the tracing trampoline
370 // ;DATA: 32 bits of function ID
371 // ;DATA: lower 32 bits of the address of the trampoline
372 // ;DATA: higher 32 bits of the address of the trampoline
373 // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
374 //
375 OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
376 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
377 OutStreamer->emitLabel(CurSled);
378 auto Target = OutContext.createTempSymbol();
379
380 // Emit "B #32" instruction, which jumps over the next 28 bytes.
381 // The operand has to be the number of 4-byte instructions to jump over,
382 // including the current instruction.
383 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
384
385 for (int8_t I = 0; I < NoopsInSledCount; I++)
386 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
387
388 OutStreamer->emitLabel(Target);
389 recordSled(CurSled, MI, Kind, 2);
390}
391
392// Emit the following code for Intrinsic::{xray_customevent,xray_typedevent}
393// (built-in functions __xray_customevent/__xray_typedevent).
394//
395// .Lxray_event_sled_N:
396// b 1f
397// save x0 and x1 (and also x2 for TYPED_EVENT_CALL)
398// set up x0 and x1 (and also x2 for TYPED_EVENT_CALL)
399// bl __xray_CustomEvent or __xray_TypedEvent
400// restore x0 and x1 (and also x2 for TYPED_EVENT_CALL)
401// 1:
402//
403// There are 6 instructions for EVENT_CALL and 9 for TYPED_EVENT_CALL.
404//
405// Then record a sled of kind CUSTOM_EVENT or TYPED_EVENT.
406// After patching, b .+N will become a nop.
407void AArch64AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
408 bool Typed) {
409 auto &O = *OutStreamer;
410 MCSymbol *CurSled = OutContext.createTempSymbol("xray_sled_", true);
411 O.emitLabel(CurSled);
412 MCInst MovX0Op0 = MCInstBuilder(AArch64::ORRXrs)
413 .addReg(AArch64::X0)
414 .addReg(AArch64::XZR)
415 .addReg(MI.getOperand(0).getReg())
416 .addImm(0);
417 MCInst MovX1Op1 = MCInstBuilder(AArch64::ORRXrs)
418 .addReg(AArch64::X1)
419 .addReg(AArch64::XZR)
420 .addReg(MI.getOperand(1).getReg())
421 .addImm(0);
422 bool MachO = TM.getTargetTriple().isOSBinFormatMachO();
424 OutContext.getOrCreateSymbol(
425 Twine(MachO ? "_" : "") +
426 (Typed ? "__xray_TypedEvent" : "__xray_CustomEvent")),
427 OutContext);
428 if (Typed) {
429 O.AddComment("Begin XRay typed event");
430 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(9));
431 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
432 .addReg(AArch64::SP)
433 .addReg(AArch64::X0)
434 .addReg(AArch64::X1)
435 .addReg(AArch64::SP)
436 .addImm(-4));
437 EmitToStreamer(O, MCInstBuilder(AArch64::STRXui)
438 .addReg(AArch64::X2)
439 .addReg(AArch64::SP)
440 .addImm(2));
441 EmitToStreamer(O, MovX0Op0);
442 EmitToStreamer(O, MovX1Op1);
443 EmitToStreamer(O, MCInstBuilder(AArch64::ORRXrs)
444 .addReg(AArch64::X2)
445 .addReg(AArch64::XZR)
446 .addReg(MI.getOperand(2).getReg())
447 .addImm(0));
448 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
449 EmitToStreamer(O, MCInstBuilder(AArch64::LDRXui)
450 .addReg(AArch64::X2)
451 .addReg(AArch64::SP)
452 .addImm(2));
453 O.AddComment("End XRay typed event");
454 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
455 .addReg(AArch64::SP)
456 .addReg(AArch64::X0)
457 .addReg(AArch64::X1)
458 .addReg(AArch64::SP)
459 .addImm(4));
460
461 recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
462 } else {
463 O.AddComment("Begin XRay custom event");
464 EmitToStreamer(O, MCInstBuilder(AArch64::B).addImm(6));
465 EmitToStreamer(O, MCInstBuilder(AArch64::STPXpre)
466 .addReg(AArch64::SP)
467 .addReg(AArch64::X0)
468 .addReg(AArch64::X1)
469 .addReg(AArch64::SP)
470 .addImm(-2));
471 EmitToStreamer(O, MovX0Op0);
472 EmitToStreamer(O, MovX1Op1);
473 EmitToStreamer(O, MCInstBuilder(AArch64::BL).addExpr(Sym));
474 O.AddComment("End XRay custom event");
475 EmitToStreamer(O, MCInstBuilder(AArch64::LDPXpost)
476 .addReg(AArch64::SP)
477 .addReg(AArch64::X0)
478 .addReg(AArch64::X1)
479 .addReg(AArch64::SP)
480 .addImm(2));
481
482 recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
483 }
484}
485
486void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
487 Register AddrReg = MI.getOperand(0).getReg();
488 assert(std::next(MI.getIterator())->isCall() &&
489 "KCFI_CHECK not followed by a call instruction");
490 assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
491 "KCFI_CHECK call target doesn't match call operand");
492
493 // Default to using the intra-procedure-call temporary registers for
494 // comparing the hashes.
495 unsigned ScratchRegs[] = {AArch64::W16, AArch64::W17};
496 if (AddrReg == AArch64::XZR) {
497 // Checking XZR makes no sense. Instead of emitting a load, zero
498 // ScratchRegs[0] and use it for the ESR AddrIndex below.
499 AddrReg = getXRegFromWReg(ScratchRegs[0]);
500 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
501 .addReg(AddrReg)
502 .addReg(AArch64::XZR)
503 .addReg(AArch64::XZR)
504 .addImm(0));
505 } else {
506 // If one of the scratch registers is used for the call target (e.g.
507 // with AArch64::TCRETURNriBTI), we can clobber another caller-saved
508 // temporary register instead (in this case, AArch64::W9) as the check
509 // is immediately followed by the call instruction.
510 for (auto &Reg : ScratchRegs) {
511 if (Reg == getWRegFromXReg(AddrReg)) {
512 Reg = AArch64::W9;
513 break;
514 }
515 }
516 assert(ScratchRegs[0] != AddrReg && ScratchRegs[1] != AddrReg &&
517 "Invalid scratch registers for KCFI_CHECK");
518
519 // Adjust the offset for patchable-function-prefix. This assumes that
520 // patchable-function-prefix is the same for all functions.
521 int64_t PrefixNops = 0;
522 (void)MI.getMF()
523 ->getFunction()
524 .getFnAttribute("patchable-function-prefix")
525 .getValueAsString()
526 .getAsInteger(10, PrefixNops);
527
528 // Load the target function type hash.
529 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi)
530 .addReg(ScratchRegs[0])
531 .addReg(AddrReg)
532 .addImm(-(PrefixNops * 4 + 4)));
533 }
534
535 // Load the expected type hash.
536 const int64_t Type = MI.getOperand(1).getImm();
537 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
538 .addReg(ScratchRegs[1])
539 .addReg(ScratchRegs[1])
540 .addImm(Type & 0xFFFF)
541 .addImm(0));
542 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
543 .addReg(ScratchRegs[1])
544 .addReg(ScratchRegs[1])
545 .addImm((Type >> 16) & 0xFFFF)
546 .addImm(16));
547
548 // Compare the hashes and trap if there's a mismatch.
549 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs)
550 .addReg(AArch64::WZR)
551 .addReg(ScratchRegs[0])
552 .addReg(ScratchRegs[1])
553 .addImm(0));
554
555 MCSymbol *Pass = OutContext.createTempSymbol();
556 EmitToStreamer(*OutStreamer,
557 MCInstBuilder(AArch64::Bcc)
558 .addImm(AArch64CC::EQ)
559 .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
560
561 // The base ESR is 0x8000 and the register information is encoded in bits
562 // 0-9 as follows:
563 // - 0-4: n, where the register Xn contains the target address
564 // - 5-9: m, where the register Wm contains the expected type hash
565 // Where n, m are in [0, 30].
566 unsigned TypeIndex = ScratchRegs[1] - AArch64::W0;
567 unsigned AddrIndex;
568 switch (AddrReg) {
569 default:
570 AddrIndex = AddrReg - AArch64::X0;
571 break;
572 case AArch64::FP:
573 AddrIndex = 29;
574 break;
575 case AArch64::LR:
576 AddrIndex = 30;
577 break;
578 }
579
580 assert(AddrIndex < 31 && TypeIndex < 31);
581
582 unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
583 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR));
584 OutStreamer->emitLabel(Pass);
585}
586
587void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
588 Register Reg = MI.getOperand(0).getReg();
589 bool IsShort =
590 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES) ||
591 (MI.getOpcode() ==
592 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
593 uint32_t AccessInfo = MI.getOperand(1).getImm();
594 bool IsFixedShadow =
595 ((MI.getOpcode() == AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW) ||
596 (MI.getOpcode() ==
597 AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW));
598 uint64_t FixedShadowOffset = IsFixedShadow ? MI.getOperand(2).getImm() : 0;
599
600 MCSymbol *&Sym = HwasanMemaccessSymbols[HwasanMemaccessTuple(
601 Reg, IsShort, AccessInfo, IsFixedShadow, FixedShadowOffset)];
602 if (!Sym) {
603 // FIXME: Make this work on non-ELF.
604 if (!TM.getTargetTriple().isOSBinFormatELF())
605 report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
606
607 std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
608 utostr(AccessInfo);
609 if (IsFixedShadow)
610 SymName += "_fixed_" + utostr(FixedShadowOffset);
611 if (IsShort)
612 SymName += "_short_v2";
613 Sym = OutContext.getOrCreateSymbol(SymName);
614 }
615
616 EmitToStreamer(*OutStreamer,
617 MCInstBuilder(AArch64::BL)
618 .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
619}
620
621void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
622 if (HwasanMemaccessSymbols.empty())
623 return;
624
625 const Triple &TT = TM.getTargetTriple();
626 assert(TT.isOSBinFormatELF());
627 std::unique_ptr<MCSubtargetInfo> STI(
628 TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
629 assert(STI && "Unable to create subtarget info");
630
631 MCSymbol *HwasanTagMismatchV1Sym =
632 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
633 MCSymbol *HwasanTagMismatchV2Sym =
634 OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
635
636 const MCSymbolRefExpr *HwasanTagMismatchV1Ref =
637 MCSymbolRefExpr::create(HwasanTagMismatchV1Sym, OutContext);
638 const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
639 MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
640
641 for (auto &P : HwasanMemaccessSymbols) {
642 unsigned Reg = std::get<0>(P.first);
643 bool IsShort = std::get<1>(P.first);
644 uint32_t AccessInfo = std::get<2>(P.first);
645 bool IsFixedShadow = std::get<3>(P.first);
646 uint64_t FixedShadowOffset = std::get<4>(P.first);
647 const MCSymbolRefExpr *HwasanTagMismatchRef =
648 IsShort ? HwasanTagMismatchV2Ref : HwasanTagMismatchV1Ref;
649 MCSymbol *Sym = P.second;
650
651 bool HasMatchAllTag =
652 (AccessInfo >> HWASanAccessInfo::HasMatchAllShift) & 1;
653 uint8_t MatchAllTag =
654 (AccessInfo >> HWASanAccessInfo::MatchAllShift) & 0xff;
655 unsigned Size =
656 1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
657 bool CompileKernel =
658 (AccessInfo >> HWASanAccessInfo::CompileKernelShift) & 1;
659
660 OutStreamer->switchSection(OutContext.getELFSection(
661 ".text.hot", ELF::SHT_PROGBITS,
663 /*IsComdat=*/true));
664
665 OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
666 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
667 OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
668 OutStreamer->emitLabel(Sym);
669
670 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SBFMXri)
671 .addReg(AArch64::X16)
672 .addReg(Reg)
673 .addImm(4)
674 .addImm(55),
675 *STI);
676
677 if (IsFixedShadow) {
678 // Aarch64 makes it difficult to embed large constants in the code.
679 // Fortuitously, kShadowBaseAlignment == 32, so we use the 32-bit
680 // left-shift option in the MOV instruction. Combined with the 16-bit
681 // immediate, this is enough to represent any offset up to 2**48.
682 OutStreamer->emitInstruction(MCInstBuilder(AArch64::MOVZXi)
683 .addReg(AArch64::X17)
684 .addImm(FixedShadowOffset >> 32)
685 .addImm(32),
686 *STI);
687 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBroX)
688 .addReg(AArch64::W16)
689 .addReg(AArch64::X17)
690 .addReg(AArch64::X16)
691 .addImm(0)
692 .addImm(0),
693 *STI);
694 } else {
695 OutStreamer->emitInstruction(
696 MCInstBuilder(AArch64::LDRBBroX)
697 .addReg(AArch64::W16)
698 .addReg(IsShort ? AArch64::X20 : AArch64::X9)
699 .addReg(AArch64::X16)
700 .addImm(0)
701 .addImm(0),
702 *STI);
703 }
704
705 OutStreamer->emitInstruction(
706 MCInstBuilder(AArch64::SUBSXrs)
707 .addReg(AArch64::XZR)
708 .addReg(AArch64::X16)
709 .addReg(Reg)
711 *STI);
712 MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
713 OutStreamer->emitInstruction(
714 MCInstBuilder(AArch64::Bcc)
716 .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
717 OutContext)),
718 *STI);
719 MCSymbol *ReturnSym = OutContext.createTempSymbol();
720 OutStreamer->emitLabel(ReturnSym);
721 OutStreamer->emitInstruction(
722 MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
723 OutStreamer->emitLabel(HandleMismatchOrPartialSym);
724
725 if (HasMatchAllTag) {
726 OutStreamer->emitInstruction(MCInstBuilder(AArch64::UBFMXri)
727 .addReg(AArch64::X17)
728 .addReg(Reg)
729 .addImm(56)
730 .addImm(63),
731 *STI);
732 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSXri)
733 .addReg(AArch64::XZR)
734 .addReg(AArch64::X17)
735 .addImm(MatchAllTag)
736 .addImm(0),
737 *STI);
738 OutStreamer->emitInstruction(
739 MCInstBuilder(AArch64::Bcc)
741 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
742 *STI);
743 }
744
745 if (IsShort) {
746 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWri)
747 .addReg(AArch64::WZR)
748 .addReg(AArch64::W16)
749 .addImm(15)
750 .addImm(0),
751 *STI);
752 MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
753 OutStreamer->emitInstruction(
754 MCInstBuilder(AArch64::Bcc)
756 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
757 *STI);
758
759 OutStreamer->emitInstruction(
760 MCInstBuilder(AArch64::ANDXri)
761 .addReg(AArch64::X17)
762 .addReg(Reg)
764 *STI);
765 if (Size != 1)
766 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
767 .addReg(AArch64::X17)
768 .addReg(AArch64::X17)
769 .addImm(Size - 1)
770 .addImm(0),
771 *STI);
772 OutStreamer->emitInstruction(MCInstBuilder(AArch64::SUBSWrs)
773 .addReg(AArch64::WZR)
774 .addReg(AArch64::W16)
775 .addReg(AArch64::W17)
776 .addImm(0),
777 *STI);
778 OutStreamer->emitInstruction(
779 MCInstBuilder(AArch64::Bcc)
781 .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
782 *STI);
783
784 OutStreamer->emitInstruction(
785 MCInstBuilder(AArch64::ORRXri)
786 .addReg(AArch64::X16)
787 .addReg(Reg)
789 *STI);
790 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRBBui)
791 .addReg(AArch64::W16)
792 .addReg(AArch64::X16)
793 .addImm(0),
794 *STI);
795 OutStreamer->emitInstruction(
796 MCInstBuilder(AArch64::SUBSXrs)
797 .addReg(AArch64::XZR)
798 .addReg(AArch64::X16)
799 .addReg(Reg)
801 *STI);
802 OutStreamer->emitInstruction(
803 MCInstBuilder(AArch64::Bcc)
805 .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
806 *STI);
807
808 OutStreamer->emitLabel(HandleMismatchSym);
809 }
810
811 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
812 .addReg(AArch64::SP)
813 .addReg(AArch64::X0)
814 .addReg(AArch64::X1)
815 .addReg(AArch64::SP)
816 .addImm(-32),
817 *STI);
818 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXi)
819 .addReg(AArch64::FP)
820 .addReg(AArch64::LR)
821 .addReg(AArch64::SP)
822 .addImm(29),
823 *STI);
824
825 if (Reg != AArch64::X0)
826 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ORRXrs)
827 .addReg(AArch64::X0)
828 .addReg(AArch64::XZR)
829 .addReg(Reg)
830 .addImm(0),
831 *STI);
832 OutStreamer->emitInstruction(
833 MCInstBuilder(AArch64::MOVZXi)
834 .addReg(AArch64::X1)
836 .addImm(0),
837 *STI);
838
839 if (CompileKernel) {
840 // The Linux kernel's dynamic loader doesn't support GOT relative
841 // relocations, but it doesn't support late binding either, so just call
842 // the function directly.
843 OutStreamer->emitInstruction(
844 MCInstBuilder(AArch64::B).addExpr(HwasanTagMismatchRef), *STI);
845 } else {
846 // Intentionally load the GOT entry and branch to it, rather than possibly
847 // late binding the function, which may clobber the registers before we
848 // have a chance to save them.
849 OutStreamer->emitInstruction(
850 MCInstBuilder(AArch64::ADRP)
851 .addReg(AArch64::X16)
853 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_PAGE,
854 OutContext)),
855 *STI);
856 OutStreamer->emitInstruction(
857 MCInstBuilder(AArch64::LDRXui)
858 .addReg(AArch64::X16)
859 .addReg(AArch64::X16)
861 HwasanTagMismatchRef, AArch64MCExpr::VariantKind::VK_GOT_LO12,
862 OutContext)),
863 *STI);
864 OutStreamer->emitInstruction(
865 MCInstBuilder(AArch64::BR).addReg(AArch64::X16), *STI);
866 }
867 }
868}
869
870static void emitAuthenticatedPointer(MCStreamer &OutStreamer,
871 MCSymbol *StubLabel,
872 const MCExpr *StubAuthPtrRef) {
873 // sym$auth_ptr$key$disc:
874 OutStreamer.emitLabel(StubLabel);
875 OutStreamer.emitValue(StubAuthPtrRef, /*size=*/8);
876}
877
878void AArch64AsmPrinter::emitEndOfAsmFile(Module &M) {
879 emitHwasanMemaccessSymbols(M);
880
881 const Triple &TT = TM.getTargetTriple();
882 if (TT.isOSBinFormatMachO()) {
883 // Output authenticated pointers as indirect symbols, if we have any.
884 MachineModuleInfoMachO &MMIMacho =
885 MMI->getObjFileInfo<MachineModuleInfoMachO>();
886
887 auto Stubs = MMIMacho.getAuthGVStubList();
888
889 if (!Stubs.empty()) {
890 // Switch to the "__auth_ptr" section.
891 OutStreamer->switchSection(
892 OutContext.getMachOSection("__DATA", "__auth_ptr", MachO::S_REGULAR,
894 emitAlignment(Align(8));
895
896 for (const auto &Stub : Stubs)
897 emitAuthenticatedPointer(*OutStreamer, Stub.first, Stub.second);
898
899 OutStreamer->addBlankLine();
900 }
901
902 // Funny Darwin hack: This flag tells the linker that no global symbols
903 // contain code that falls through to other global symbols (e.g. the obvious
904 // implementation of multiple entry points). If this doesn't occur, the
905 // linker can safely perform dead code stripping. Since LLVM never
906 // generates code that does this, it is always safe to set.
907 OutStreamer->emitAssemblerFlag(MCAF_SubsectionsViaSymbols);
908 }
909
910 if (TT.isOSBinFormatELF()) {
911 // Output authenticated pointers as indirect symbols, if we have any.
912 MachineModuleInfoELF &MMIELF = MMI->getObjFileInfo<MachineModuleInfoELF>();
913
914 auto Stubs = MMIELF.getAuthGVStubList();
915
916 if (!Stubs.empty()) {
917 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
918 OutStreamer->switchSection(TLOF.getDataSection());
919 emitAlignment(Align(8));
920
921 for (const auto &Stub : Stubs)
922 emitAuthenticatedPointer(*OutStreamer, Stub.first, Stub.second);
923
924 OutStreamer->addBlankLine();
925 }
926 }
927
928 // Emit stack and fault map information.
929 FM.serializeToFaultMapSection();
930
931}
932
933void AArch64AsmPrinter::emitLOHs() {
935
936 for (const auto &D : AArch64FI->getLOHContainer()) {
937 for (const MachineInstr *MI : D.getArgs()) {
938 MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
939 assert(LabelIt != LOHInstToLabel.end() &&
940 "Label hasn't been inserted for LOH related instruction");
941 MCArgs.push_back(LabelIt->second);
942 }
943 OutStreamer->emitLOHDirective(D.getKind(), MCArgs);
944 MCArgs.clear();
945 }
946}
947
948void AArch64AsmPrinter::emitFunctionBodyEnd() {
949 if (!AArch64FI->getLOHRelated().empty())
950 emitLOHs();
951}
952
953/// GetCPISymbol - Return the symbol for the specified constant pool entry.
954MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
955 // Darwin uses a linker-private symbol name for constant-pools (to
956 // avoid addends on the relocation?), ELF has no such concept and
957 // uses a normal private symbol.
958 if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
959 return OutContext.getOrCreateSymbol(
960 Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
961 Twine(getFunctionNumber()) + "_" + Twine(CPID));
962
963 return AsmPrinter::GetCPISymbol(CPID);
964}
965
966void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
967 raw_ostream &O) {
968 const MachineOperand &MO = MI->getOperand(OpNum);
969 switch (MO.getType()) {
970 default:
971 llvm_unreachable("<unknown operand type>");
973 Register Reg = MO.getReg();
974 assert(Reg.isPhysical());
975 assert(!MO.getSubReg() && "Subregs should be eliminated!");
977 break;
978 }
980 O << MO.getImm();
981 break;
982 }
984 PrintSymbolOperand(MO, O);
985 break;
986 }
988 MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
989 Sym->print(O, MAI);
990 break;
991 }
992 }
993}
994
995bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
996 raw_ostream &O) {
997 Register Reg = MO.getReg();
998 switch (Mode) {
999 default:
1000 return true; // Unknown mode.
1001 case 'w':
1002 Reg = getWRegFromXReg(Reg);
1003 break;
1004 case 'x':
1005 Reg = getXRegFromWReg(Reg);
1006 break;
1007 case 't':
1009 break;
1010 }
1011
1013 return false;
1014}
1015
1016// Prints the register in MO using class RC using the offset in the
1017// new register class. This should not be used for cross class
1018// printing.
1019bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
1020 const TargetRegisterClass *RC,
1021 unsigned AltName, raw_ostream &O) {
1022 assert(MO.isReg() && "Should only get here with a register!");
1023 const TargetRegisterInfo *RI = STI->getRegisterInfo();
1024 Register Reg = MO.getReg();
1025 unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
1026 if (!RI->regsOverlap(RegToPrint, Reg))
1027 return true;
1028 O << AArch64InstPrinter::getRegisterName(RegToPrint, AltName);
1029 return false;
1030}
1031
1032bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
1033 const char *ExtraCode, raw_ostream &O) {
1034 const MachineOperand &MO = MI->getOperand(OpNum);
1035
1036 // First try the generic code, which knows about modifiers like 'c' and 'n'.
1037 if (!AsmPrinter::PrintAsmOperand(MI, OpNum, ExtraCode, O))
1038 return false;
1039
1040 // Does this asm operand have a single letter operand modifier?
1041 if (ExtraCode && ExtraCode[0]) {
1042 if (ExtraCode[1] != 0)
1043 return true; // Unknown modifier.
1044
1045 switch (ExtraCode[0]) {
1046 default:
1047 return true; // Unknown modifier.
1048 case 'w': // Print W register
1049 case 'x': // Print X register
1050 if (MO.isReg())
1051 return printAsmMRegister(MO, ExtraCode[0], O);
1052 if (MO.isImm() && MO.getImm() == 0) {
1053 unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
1055 return false;
1056 }
1057 printOperand(MI, OpNum, O);
1058 return false;
1059 case 'b': // Print B register.
1060 case 'h': // Print H register.
1061 case 's': // Print S register.
1062 case 'd': // Print D register.
1063 case 'q': // Print Q register.
1064 case 'z': // Print Z register.
1065 if (MO.isReg()) {
1066 const TargetRegisterClass *RC;
1067 switch (ExtraCode[0]) {
1068 case 'b':
1069 RC = &AArch64::FPR8RegClass;
1070 break;
1071 case 'h':
1072 RC = &AArch64::FPR16RegClass;
1073 break;
1074 case 's':
1075 RC = &AArch64::FPR32RegClass;
1076 break;
1077 case 'd':
1078 RC = &AArch64::FPR64RegClass;
1079 break;
1080 case 'q':
1081 RC = &AArch64::FPR128RegClass;
1082 break;
1083 case 'z':
1084 RC = &AArch64::ZPRRegClass;
1085 break;
1086 default:
1087 return true;
1088 }
1089 return printAsmRegInClass(MO, RC, AArch64::NoRegAltName, O);
1090 }
1091 printOperand(MI, OpNum, O);
1092 return false;
1093 }
1094 }
1095
1096 // According to ARM, we should emit x and v registers unless we have a
1097 // modifier.
1098 if (MO.isReg()) {
1099 Register Reg = MO.getReg();
1100
1101 // If this is a w or x register, print an x register.
1102 if (AArch64::GPR32allRegClass.contains(Reg) ||
1103 AArch64::GPR64allRegClass.contains(Reg))
1104 return printAsmMRegister(MO, 'x', O);
1105
1106 // If this is an x register tuple, print an x register.
1107 if (AArch64::GPR64x8ClassRegClass.contains(Reg))
1108 return printAsmMRegister(MO, 't', O);
1109
1110 unsigned AltName = AArch64::NoRegAltName;
1111 const TargetRegisterClass *RegClass;
1112 if (AArch64::ZPRRegClass.contains(Reg)) {
1113 RegClass = &AArch64::ZPRRegClass;
1114 } else if (AArch64::PPRRegClass.contains(Reg)) {
1115 RegClass = &AArch64::PPRRegClass;
1116 } else if (AArch64::PNRRegClass.contains(Reg)) {
1117 RegClass = &AArch64::PNRRegClass;
1118 } else {
1119 RegClass = &AArch64::FPR128RegClass;
1120 AltName = AArch64::vreg;
1121 }
1122
1123 // If this is a b, h, s, d, or q register, print it as a v register.
1124 return printAsmRegInClass(MO, RegClass, AltName, O);
1125 }
1126
1127 printOperand(MI, OpNum, O);
1128 return false;
1129}
1130
1131bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
1132 unsigned OpNum,
1133 const char *ExtraCode,
1134 raw_ostream &O) {
1135 if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
1136 return true; // Unknown modifier.
1137
1138 const MachineOperand &MO = MI->getOperand(OpNum);
1139 assert(MO.isReg() && "unexpected inline asm memory operand");
1140 O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
1141 return false;
1142}
1143
1144void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
1145 raw_ostream &OS) {
1146 unsigned NOps = MI->getNumOperands();
1147 assert(NOps == 4);
1148 OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
1149 // cast away const; DIetc do not take const operands for some reason.
1150 OS << MI->getDebugVariable()->getName();
1151 OS << " <- ";
1152 // Frame address. Currently handles register +- offset only.
1153 assert(MI->isIndirectDebugValue());
1154 OS << '[';
1155 for (unsigned I = 0, E = std::distance(MI->debug_operands().begin(),
1156 MI->debug_operands().end());
1157 I < E; ++I) {
1158 if (I != 0)
1159 OS << ", ";
1160 printOperand(MI, I, OS);
1161 }
1162 OS << ']';
1163 OS << "+";
1164 printOperand(MI, NOps - 2, OS);
1165}
1166
1167void AArch64AsmPrinter::emitJumpTableInfo() {
1168 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1169 if (!MJTI) return;
1170
1171 const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
1172 if (JT.empty()) return;
1173
1174 const TargetLoweringObjectFile &TLOF = getObjFileLowering();
1175 MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(MF->getFunction(), TM);
1176 OutStreamer->switchSection(ReadOnlySec);
1177
1178 auto AFI = MF->getInfo<AArch64FunctionInfo>();
1179 for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
1180 const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
1181
1182 // If this jump table was deleted, ignore it.
1183 if (JTBBs.empty()) continue;
1184
1185 unsigned Size = AFI->getJumpTableEntrySize(JTI);
1186 emitAlignment(Align(Size));
1187 OutStreamer->emitLabel(GetJTISymbol(JTI));
1188
1189 const MCSymbol *BaseSym = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1190 const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
1191
1192 for (auto *JTBB : JTBBs) {
1193 const MCExpr *Value =
1194 MCSymbolRefExpr::create(JTBB->getSymbol(), OutContext);
1195
1196 // Each entry is:
1197 // .byte/.hword (LBB - Lbase)>>2
1198 // or plain:
1199 // .word LBB - Lbase
1200 Value = MCBinaryExpr::createSub(Value, Base, OutContext);
1201 if (Size != 4)
1203 Value, MCConstantExpr::create(2, OutContext), OutContext);
1204
1205 OutStreamer->emitValue(Value, Size);
1206 }
1207 }
1208}
1209
1210std::tuple<const MCSymbol *, uint64_t, const MCSymbol *,
1212AArch64AsmPrinter::getCodeViewJumpTableInfo(int JTI,
1213 const MachineInstr *BranchInstr,
1214 const MCSymbol *BranchLabel) const {
1215 const auto AFI = MF->getInfo<AArch64FunctionInfo>();
1216 const auto Base = AArch64FI->getJumpTableEntryPCRelSymbol(JTI);
1218 switch (AFI->getJumpTableEntrySize(JTI)) {
1219 case 1:
1220 EntrySize = codeview::JumpTableEntrySize::UInt8ShiftLeft;
1221 break;
1222 case 2:
1223 EntrySize = codeview::JumpTableEntrySize::UInt16ShiftLeft;
1224 break;
1225 case 4:
1226 EntrySize = codeview::JumpTableEntrySize::Int32;
1227 break;
1228 default:
1229 llvm_unreachable("Unexpected jump table entry size");
1230 }
1231 return std::make_tuple(Base, 0, BranchLabel, EntrySize);
1232}
1233
1234void AArch64AsmPrinter::emitFunctionEntryLabel() {
1235 if (MF->getFunction().getCallingConv() == CallingConv::AArch64_VectorCall ||
1236 MF->getFunction().getCallingConv() ==
1238 MF->getInfo<AArch64FunctionInfo>()->isSVECC()) {
1239 auto *TS =
1240 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
1241 TS->emitDirectiveVariantPCS(CurrentFnSym);
1242 }
1243
1245
1246 if (TM.getTargetTriple().isWindowsArm64EC() &&
1247 !MF->getFunction().hasLocalLinkage()) {
1248 // For ARM64EC targets, a function definition's name is mangled differently
1249 // from the normal symbol, emit required aliases here.
1250 auto emitFunctionAlias = [&](MCSymbol *Src, MCSymbol *Dst) {
1251 OutStreamer->emitSymbolAttribute(Src, MCSA_WeakAntiDep);
1252 OutStreamer->emitAssignment(
1254 MMI->getContext()));
1255 };
1256
1257 auto getSymbolFromMetadata = [&](StringRef Name) {
1258 MCSymbol *Sym = nullptr;
1259 if (MDNode *Node = MF->getFunction().getMetadata(Name)) {
1260 StringRef NameStr = cast<MDString>(Node->getOperand(0))->getString();
1261 Sym = MMI->getContext().getOrCreateSymbol(NameStr);
1262 }
1263 return Sym;
1264 };
1265
1266 if (MCSymbol *UnmangledSym =
1267 getSymbolFromMetadata("arm64ec_unmangled_name")) {
1268 MCSymbol *ECMangledSym = getSymbolFromMetadata("arm64ec_ecmangled_name");
1269
1270 if (ECMangledSym) {
1271 // An external function, emit the alias from the unmangled symbol to
1272 // mangled symbol name and the alias from the mangled symbol to guest
1273 // exit thunk.
1274 emitFunctionAlias(UnmangledSym, ECMangledSym);
1275 emitFunctionAlias(ECMangledSym, CurrentFnSym);
1276 } else {
1277 // A function implementation, emit the alias from the unmangled symbol
1278 // to mangled symbol name.
1279 emitFunctionAlias(UnmangledSym, CurrentFnSym);
1280 }
1281 }
1282 }
1283}
1284
1285void AArch64AsmPrinter::emitXXStructor(const DataLayout &DL,
1286 const Constant *CV) {
1287 if (const auto *CPA = dyn_cast<ConstantPtrAuth>(CV))
1288 if (CPA->hasAddressDiscriminator() &&
1289 !CPA->hasSpecialAddressDiscriminator(
1292 "unexpected address discrimination value for ctors/dtors entry, only "
1293 "'ptr inttoptr (i64 1 to ptr)' is allowed");
1294 // If we have signed pointers in xxstructors list, they'll be lowered to @AUTH
1295 // MCExpr's via AArch64AsmPrinter::lowerConstantPtrAuth. It does not look at
1296 // actual address discrimination value and only checks
1297 // hasAddressDiscriminator(), so it's OK to leave special address
1298 // discrimination value here.
1300}
1301
1302void AArch64AsmPrinter::emitGlobalAlias(const Module &M,
1303 const GlobalAlias &GA) {
1304 if (auto F = dyn_cast_or_null<Function>(GA.getAliasee())) {
1305 // Global aliases must point to a definition, but unmangled patchable
1306 // symbols are special and need to point to an undefined symbol with "EXP+"
1307 // prefix. Such undefined symbol is resolved by the linker by creating
1308 // x86 thunk that jumps back to the actual EC target.
1309 if (MDNode *Node = F->getMetadata("arm64ec_exp_name")) {
1310 StringRef ExpStr = cast<MDString>(Node->getOperand(0))->getString();
1311 MCSymbol *ExpSym = MMI->getContext().getOrCreateSymbol(ExpStr);
1312 MCSymbol *Sym = MMI->getContext().getOrCreateSymbol(GA.getName());
1313
1314 OutStreamer->beginCOFFSymbolDef(ExpSym);
1315 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_EXTERNAL);
1316 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION
1318 OutStreamer->endCOFFSymbolDef();
1319
1320 OutStreamer->beginCOFFSymbolDef(Sym);
1321 OutStreamer->emitCOFFSymbolStorageClass(COFF::IMAGE_SYM_CLASS_EXTERNAL);
1322 OutStreamer->emitCOFFSymbolType(COFF::IMAGE_SYM_DTYPE_FUNCTION
1324 OutStreamer->endCOFFSymbolDef();
1325 OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
1326 OutStreamer->emitAssignment(
1328 MMI->getContext()));
1329 return;
1330 }
1331 }
1333}
1334
1335/// Small jump tables contain an unsigned byte or half, representing the offset
1336/// from the lowest-addressed possible destination to the desired basic
1337/// block. Since all instructions are 4-byte aligned, this is further compressed
1338/// by counting in instructions rather than bytes (i.e. divided by 4). So, to
1339/// materialize the correct destination we need:
1340///
1341/// adr xDest, .LBB0_0
1342/// ldrb wScratch, [xTable, xEntry] (with "lsl #1" for ldrh).
1343/// add xDest, xDest, xScratch (with "lsl #2" for smaller entries)
1344void AArch64AsmPrinter::LowerJumpTableDest(llvm::MCStreamer &OutStreamer,
1345 const llvm::MachineInstr &MI) {
1346 Register DestReg = MI.getOperand(0).getReg();
1347 Register ScratchReg = MI.getOperand(1).getReg();
1348 Register ScratchRegW =
1349 STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
1350 Register TableReg = MI.getOperand(2).getReg();
1351 Register EntryReg = MI.getOperand(3).getReg();
1352 int JTIdx = MI.getOperand(4).getIndex();
1353 int Size = AArch64FI->getJumpTableEntrySize(JTIdx);
1354
1355 // This has to be first because the compression pass based its reachability
1356 // calculations on the start of the JumpTableDest instruction.
1357 auto Label =
1358 MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
1359
1360 // If we don't already have a symbol to use as the base, use the ADR
1361 // instruction itself.
1362 if (!Label) {
1363 Label = MF->getContext().createTempSymbol();
1364 AArch64FI->setJumpTableEntryInfo(JTIdx, Size, Label);
1365 OutStreamer.emitLabel(Label);
1366 }
1367
1368 auto LabelExpr = MCSymbolRefExpr::create(Label, MF->getContext());
1369 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
1370 .addReg(DestReg)
1371 .addExpr(LabelExpr));
1372
1373 // Load the number of instruction-steps to offset from the label.
1374 unsigned LdrOpcode;
1375 switch (Size) {
1376 case 1: LdrOpcode = AArch64::LDRBBroX; break;
1377 case 2: LdrOpcode = AArch64::LDRHHroX; break;
1378 case 4: LdrOpcode = AArch64::LDRSWroX; break;
1379 default:
1380 llvm_unreachable("Unknown jump table size");
1381 }
1382
1383 EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
1384 .addReg(Size == 4 ? ScratchReg : ScratchRegW)
1385 .addReg(TableReg)
1386 .addReg(EntryReg)
1387 .addImm(0)
1388 .addImm(Size == 1 ? 0 : 1));
1389
1390 // Add to the already materialized base label address, multiplying by 4 if
1391 // compressed.
1392 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1393 .addReg(DestReg)
1394 .addReg(DestReg)
1395 .addReg(ScratchReg)
1396 .addImm(Size == 4 ? 0 : 2));
1397}
1398
1399void AArch64AsmPrinter::LowerHardenedBRJumpTable(const MachineInstr &MI) {
1400 unsigned InstsEmitted = 0;
1401
1402 const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
1403 assert(MJTI && "Can't lower jump-table dispatch without JTI");
1404
1405 const std::vector<MachineJumpTableEntry> &JTs = MJTI->getJumpTables();
1406 assert(!JTs.empty() && "Invalid JT index for jump-table dispatch");
1407
1408 // Emit:
1409 // mov x17, #<size of table> ; depending on table size, with MOVKs
1410 // cmp x16, x17 ; or #imm if table size fits in 12-bit
1411 // csel x16, x16, xzr, ls ; check for index overflow
1412 //
1413 // adrp x17, Ltable@PAGE ; materialize table address
1414 // add x17, Ltable@PAGEOFF
1415 // ldrsw x16, [x17, x16, lsl #2] ; load table entry
1416 //
1417 // Lanchor:
1418 // adr x17, Lanchor ; compute target address
1419 // add x16, x17, x16
1420 // br x16 ; branch to target
1421
1422 MachineOperand JTOp = MI.getOperand(0);
1423
1424 unsigned JTI = JTOp.getIndex();
1425 assert(!AArch64FI->getJumpTableEntryPCRelSymbol(JTI) &&
1426 "unsupported compressed jump table");
1427
1428 const uint64_t NumTableEntries = JTs[JTI].MBBs.size();
1429
1430 // cmp only supports a 12-bit immediate. If we need more, materialize the
1431 // immediate, using x17 as a scratch register.
1432 uint64_t MaxTableEntry = NumTableEntries - 1;
1433 if (isUInt<12>(MaxTableEntry)) {
1434 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXri)
1435 .addReg(AArch64::XZR)
1436 .addReg(AArch64::X16)
1437 .addImm(MaxTableEntry)
1438 .addImm(0));
1439 ++InstsEmitted;
1440 } else {
1441 EmitToStreamer(*OutStreamer,
1442 MCInstBuilder(AArch64::MOVZXi)
1443 .addReg(AArch64::X17)
1444 .addImm(static_cast<uint16_t>(MaxTableEntry))
1445 .addImm(0));
1446 ++InstsEmitted;
1447 // It's sad that we have to manually materialize instructions, but we can't
1448 // trivially reuse the main pseudo expansion logic.
1449 // A MOVK sequence is easy enough to generate and handles the general case.
1450 for (int Offset = 16; Offset < 64; Offset += 16) {
1451 if ((MaxTableEntry >> Offset) == 0)
1452 break;
1453 EmitToStreamer(*OutStreamer,
1454 MCInstBuilder(AArch64::MOVKXi)
1455 .addReg(AArch64::X17)
1456 .addReg(AArch64::X17)
1457 .addImm(static_cast<uint16_t>(MaxTableEntry >> Offset))
1458 .addImm(Offset));
1459 ++InstsEmitted;
1460 }
1461 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXrs)
1462 .addReg(AArch64::XZR)
1463 .addReg(AArch64::X16)
1464 .addReg(AArch64::X17)
1465 .addImm(0));
1466 ++InstsEmitted;
1467 }
1468
1469 // This picks entry #0 on failure.
1470 // We might want to trap instead.
1471 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::CSELXr)
1472 .addReg(AArch64::X16)
1473 .addReg(AArch64::X16)
1474 .addReg(AArch64::XZR)
1475 .addImm(AArch64CC::LS));
1476 ++InstsEmitted;
1477
1478 // Prepare the @PAGE/@PAGEOFF low/high operands.
1479 MachineOperand JTMOHi(JTOp), JTMOLo(JTOp);
1480 MCOperand JTMCHi, JTMCLo;
1481
1482 JTMOHi.setTargetFlags(AArch64II::MO_PAGE);
1483 JTMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1484
1485 MCInstLowering.lowerOperand(JTMOHi, JTMCHi);
1486 MCInstLowering.lowerOperand(JTMOLo, JTMCLo);
1487
1488 EmitToStreamer(
1489 *OutStreamer,
1490 MCInstBuilder(AArch64::ADRP).addReg(AArch64::X17).addOperand(JTMCHi));
1491 ++InstsEmitted;
1492
1493 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXri)
1494 .addReg(AArch64::X17)
1495 .addReg(AArch64::X17)
1496 .addOperand(JTMCLo)
1497 .addImm(0));
1498 ++InstsEmitted;
1499
1500 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
1501 .addReg(AArch64::X16)
1502 .addReg(AArch64::X17)
1503 .addReg(AArch64::X16)
1504 .addImm(0)
1505 .addImm(1));
1506 ++InstsEmitted;
1507
1508 MCSymbol *AdrLabel = MF->getContext().createTempSymbol();
1509 const auto *AdrLabelE = MCSymbolRefExpr::create(AdrLabel, MF->getContext());
1510 AArch64FI->setJumpTableEntryInfo(JTI, 4, AdrLabel);
1511
1512 OutStreamer->emitLabel(AdrLabel);
1513 EmitToStreamer(
1514 *OutStreamer,
1515 MCInstBuilder(AArch64::ADR).addReg(AArch64::X17).addExpr(AdrLabelE));
1516 ++InstsEmitted;
1517
1518 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
1519 .addReg(AArch64::X16)
1520 .addReg(AArch64::X17)
1521 .addReg(AArch64::X16)
1522 .addImm(0));
1523 ++InstsEmitted;
1524
1525 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BR).addReg(AArch64::X16));
1526 ++InstsEmitted;
1527
1528 (void)InstsEmitted;
1529 assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4);
1530}
1531
1532void AArch64AsmPrinter::LowerMOPS(llvm::MCStreamer &OutStreamer,
1533 const llvm::MachineInstr &MI) {
1534 unsigned Opcode = MI.getOpcode();
1535 assert(STI->hasMOPS());
1536 assert(STI->hasMTE() || Opcode != AArch64::MOPSMemorySetTaggingPseudo);
1537
1538 const auto Ops = [Opcode]() -> std::array<unsigned, 3> {
1539 if (Opcode == AArch64::MOPSMemoryCopyPseudo)
1540 return {AArch64::CPYFP, AArch64::CPYFM, AArch64::CPYFE};
1541 if (Opcode == AArch64::MOPSMemoryMovePseudo)
1542 return {AArch64::CPYP, AArch64::CPYM, AArch64::CPYE};
1543 if (Opcode == AArch64::MOPSMemorySetPseudo)
1544 return {AArch64::SETP, AArch64::SETM, AArch64::SETE};
1545 if (Opcode == AArch64::MOPSMemorySetTaggingPseudo)
1546 return {AArch64::SETGP, AArch64::SETGM, AArch64::MOPSSETGE};
1547 llvm_unreachable("Unhandled memory operation pseudo");
1548 }();
1549 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||
1550 Opcode == AArch64::MOPSMemorySetTaggingPseudo;
1551
1552 for (auto Op : Ops) {
1553 int i = 0;
1554 auto MCIB = MCInstBuilder(Op);
1555 // Destination registers
1556 MCIB.addReg(MI.getOperand(i++).getReg());
1557 MCIB.addReg(MI.getOperand(i++).getReg());
1558 if (!IsSet)
1559 MCIB.addReg(MI.getOperand(i++).getReg());
1560 // Input registers
1561 MCIB.addReg(MI.getOperand(i++).getReg());
1562 MCIB.addReg(MI.getOperand(i++).getReg());
1563 MCIB.addReg(MI.getOperand(i++).getReg());
1564
1565 EmitToStreamer(OutStreamer, MCIB);
1566 }
1567}
1568
1569void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
1570 const MachineInstr &MI) {
1571 unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
1572
1573 auto &Ctx = OutStreamer.getContext();
1574 MCSymbol *MILabel = Ctx.createTempSymbol();
1575 OutStreamer.emitLabel(MILabel);
1576
1577 SM.recordStackMap(*MILabel, MI);
1578 assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1579
1580 // Scan ahead to trim the shadow.
1581 const MachineBasicBlock &MBB = *MI.getParent();
1583 ++MII;
1584 while (NumNOPBytes > 0) {
1585 if (MII == MBB.end() || MII->isCall() ||
1586 MII->getOpcode() == AArch64::DBG_VALUE ||
1587 MII->getOpcode() == TargetOpcode::PATCHPOINT ||
1588 MII->getOpcode() == TargetOpcode::STACKMAP)
1589 break;
1590 ++MII;
1591 NumNOPBytes -= 4;
1592 }
1593
1594 // Emit nops.
1595 for (unsigned i = 0; i < NumNOPBytes; i += 4)
1596 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1597}
1598
1599// Lower a patchpoint of the form:
1600// [<def>], <id>, <numBytes>, <target>, <numArgs>
1601void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1602 const MachineInstr &MI) {
1603 auto &Ctx = OutStreamer.getContext();
1604 MCSymbol *MILabel = Ctx.createTempSymbol();
1605 OutStreamer.emitLabel(MILabel);
1606 SM.recordPatchPoint(*MILabel, MI);
1607
1608 PatchPointOpers Opers(&MI);
1609
1610 int64_t CallTarget = Opers.getCallTarget().getImm();
1611 unsigned EncodedBytes = 0;
1612 if (CallTarget) {
1613 assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
1614 "High 16 bits of call target should be zero.");
1615 Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
1616 EncodedBytes = 16;
1617 // Materialize the jump address:
1618 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
1619 .addReg(ScratchReg)
1620 .addImm((CallTarget >> 32) & 0xFFFF)
1621 .addImm(32));
1622 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1623 .addReg(ScratchReg)
1624 .addReg(ScratchReg)
1625 .addImm((CallTarget >> 16) & 0xFFFF)
1626 .addImm(16));
1627 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1628 .addReg(ScratchReg)
1629 .addReg(ScratchReg)
1630 .addImm(CallTarget & 0xFFFF)
1631 .addImm(0));
1632 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
1633 }
1634 // Emit padding.
1635 unsigned NumBytes = Opers.getNumPatchBytes();
1636 assert(NumBytes >= EncodedBytes &&
1637 "Patchpoint can't request size less than the length of a call.");
1638 assert((NumBytes - EncodedBytes) % 4 == 0 &&
1639 "Invalid number of NOP bytes requested!");
1640 for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
1641 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1642}
1643
1644void AArch64AsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
1645 const MachineInstr &MI) {
1646 StatepointOpers SOpers(&MI);
1647 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1648 assert(PatchBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
1649 for (unsigned i = 0; i < PatchBytes; i += 4)
1650 EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
1651 } else {
1652 // Lower call target and choose correct opcode
1653 const MachineOperand &CallTarget = SOpers.getCallTarget();
1654 MCOperand CallTargetMCOp;
1655 unsigned CallOpcode;
1656 switch (CallTarget.getType()) {
1659 MCInstLowering.lowerOperand(CallTarget, CallTargetMCOp);
1660 CallOpcode = AArch64::BL;
1661 break;
1663 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1664 CallOpcode = AArch64::BL;
1665 break;
1667 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1668 CallOpcode = AArch64::BLR;
1669 break;
1670 default:
1671 llvm_unreachable("Unsupported operand type in statepoint call target");
1672 break;
1673 }
1674
1675 EmitToStreamer(OutStreamer,
1676 MCInstBuilder(CallOpcode).addOperand(CallTargetMCOp));
1677 }
1678
1679 auto &Ctx = OutStreamer.getContext();
1680 MCSymbol *MILabel = Ctx.createTempSymbol();
1681 OutStreamer.emitLabel(MILabel);
1682 SM.recordStatepoint(*MILabel, MI);
1683}
1684
1685void AArch64AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI) {
1686 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1687 // <opcode>, <operands>
1688
1689 Register DefRegister = FaultingMI.getOperand(0).getReg();
1691 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1692 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1693 unsigned Opcode = FaultingMI.getOperand(3).getImm();
1694 unsigned OperandsBeginIdx = 4;
1695
1696 auto &Ctx = OutStreamer->getContext();
1697 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1698 OutStreamer->emitLabel(FaultingLabel);
1699
1700 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1701 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1702
1703 MCInst MI;
1704 MI.setOpcode(Opcode);
1705
1706 if (DefRegister != (Register)0)
1707 MI.addOperand(MCOperand::createReg(DefRegister));
1708
1709 for (const MachineOperand &MO :
1710 llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx)) {
1711 MCOperand Dest;
1712 lowerOperand(MO, Dest);
1713 MI.addOperand(Dest);
1714 }
1715
1716 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1717 OutStreamer->emitInstruction(MI, getSubtargetInfo());
1718}
1719
1720void AArch64AsmPrinter::emitFMov0(const MachineInstr &MI) {
1721 Register DestReg = MI.getOperand(0).getReg();
1722 if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround() &&
1723 STI->isNeonAvailable()) {
1724 // Convert H/S register to corresponding D register
1725 if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
1726 DestReg = AArch64::D0 + (DestReg - AArch64::H0);
1727 else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
1728 DestReg = AArch64::D0 + (DestReg - AArch64::S0);
1729 else
1730 assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
1731
1732 MCInst MOVI;
1733 MOVI.setOpcode(AArch64::MOVID);
1734 MOVI.addOperand(MCOperand::createReg(DestReg));
1735 MOVI.addOperand(MCOperand::createImm(0));
1736 EmitToStreamer(*OutStreamer, MOVI);
1737 } else {
1738 MCInst FMov;
1739 switch (MI.getOpcode()) {
1740 default: llvm_unreachable("Unexpected opcode");
1741 case AArch64::FMOVH0:
1742 FMov.setOpcode(STI->hasFullFP16() ? AArch64::FMOVWHr : AArch64::FMOVWSr);
1743 if (!STI->hasFullFP16())
1744 DestReg = (AArch64::S0 + (DestReg - AArch64::H0));
1745 FMov.addOperand(MCOperand::createReg(DestReg));
1746 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1747 break;
1748 case AArch64::FMOVS0:
1749 FMov.setOpcode(AArch64::FMOVWSr);
1750 FMov.addOperand(MCOperand::createReg(DestReg));
1751 FMov.addOperand(MCOperand::createReg(AArch64::WZR));
1752 break;
1753 case AArch64::FMOVD0:
1754 FMov.setOpcode(AArch64::FMOVXDr);
1755 FMov.addOperand(MCOperand::createReg(DestReg));
1756 FMov.addOperand(MCOperand::createReg(AArch64::XZR));
1757 break;
1758 }
1759 EmitToStreamer(*OutStreamer, FMov);
1760 }
1761}
1762
1763unsigned AArch64AsmPrinter::emitPtrauthDiscriminator(uint16_t Disc,
1764 unsigned AddrDisc,
1765 unsigned &InstsEmitted) {
1766 // So far we've used NoRegister in pseudos. Now we need real encodings.
1767 if (AddrDisc == AArch64::NoRegister)
1768 AddrDisc = AArch64::XZR;
1769
1770 // If there is no constant discriminator, there's no blend involved:
1771 // just use the address discriminator register as-is (XZR or not).
1772 if (!Disc)
1773 return AddrDisc;
1774
1775 // If there's only a constant discriminator, MOV it into x17.
1776 if (AddrDisc == AArch64::XZR) {
1777 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
1778 .addReg(AArch64::X17)
1779 .addImm(Disc)
1780 .addImm(/*shift=*/0));
1781 ++InstsEmitted;
1782 return AArch64::X17;
1783 }
1784
1785 // If there are both, emit a blend into x17.
1786 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
1787 .addReg(AArch64::X17)
1788 .addReg(AArch64::XZR)
1789 .addReg(AddrDisc)
1790 .addImm(0));
1791 ++InstsEmitted;
1792 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
1793 .addReg(AArch64::X17)
1794 .addReg(AArch64::X17)
1795 .addImm(Disc)
1796 .addImm(/*shift=*/48));
1797 ++InstsEmitted;
1798 return AArch64::X17;
1799}
1800
1801void AArch64AsmPrinter::emitPtrauthAuthResign(const MachineInstr *MI) {
1802 unsigned InstsEmitted = 0;
1803 const bool IsAUTPAC = MI->getOpcode() == AArch64::AUTPAC;
1804
1805 // We can expand AUT/AUTPAC into 3 possible sequences:
1806 // - unchecked:
1807 // autia x16, x0
1808 // pacib x16, x1 ; if AUTPAC
1809 //
1810 // - checked and clearing:
1811 // mov x17, x0
1812 // movk x17, #disc, lsl #48
1813 // autia x16, x17
1814 // mov x17, x16
1815 // xpaci x17
1816 // cmp x16, x17
1817 // b.eq Lsuccess
1818 // mov x16, x17
1819 // b Lend
1820 // Lsuccess:
1821 // mov x17, x1
1822 // movk x17, #disc, lsl #48
1823 // pacib x16, x17
1824 // Lend:
1825 // Where we only emit the AUT if we started with an AUT.
1826 //
1827 // - checked and trapping:
1828 // mov x17, x0
1829 // movk x17, #disc, lsl #48
1830 // autia x16, x0
1831 // mov x17, x16
1832 // xpaci x17
1833 // cmp x16, x17
1834 // b.eq Lsuccess
1835 // brk #<0xc470 + aut key>
1836 // Lsuccess:
1837 // mov x17, x1
1838 // movk x17, #disc, lsl #48
1839 // pacib x16, x17 ; if AUTPAC
1840 // Where the b.eq skips over the trap if the PAC is valid.
1841 //
1842 // This sequence is expensive, but we need more information to be able to
1843 // do better.
1844 //
1845 // We can't TBZ the poison bit because EnhancedPAC2 XORs the PAC bits
1846 // on failure.
1847 // We can't TST the PAC bits because we don't always know how the address
1848 // space is setup for the target environment (and the bottom PAC bit is
1849 // based on that).
1850 // Either way, we also don't always know whether TBI is enabled or not for
1851 // the specific target environment.
1852
1853 // By default, auth/resign sequences check for auth failures.
1854 bool ShouldCheck = true;
1855 // In the checked sequence, we only trap if explicitly requested.
1856 bool ShouldTrap = MF->getFunction().hasFnAttribute("ptrauth-auth-traps");
1857
1858 // On an FPAC CPU, you get traps whether you want them or not: there's
1859 // no point in emitting checks or traps.
1860 if (STI->hasFPAC())
1861 ShouldCheck = ShouldTrap = false;
1862
1863 // However, command-line flags can override this, for experimentation.
1864 switch (PtrauthAuthChecks) {
1866 break;
1868 ShouldCheck = ShouldTrap = false;
1869 break;
1871 ShouldCheck = true;
1872 ShouldTrap = false;
1873 break;
1875 ShouldCheck = ShouldTrap = true;
1876 break;
1877 }
1878
1879 auto AUTKey = (AArch64PACKey::ID)MI->getOperand(0).getImm();
1880 uint64_t AUTDisc = MI->getOperand(1).getImm();
1881 unsigned AUTAddrDisc = MI->getOperand(2).getReg();
1882
1883 unsigned XPACOpc = getXPACOpcodeForKey(AUTKey);
1884
1885 // Compute aut discriminator into x17
1886 assert(isUInt<16>(AUTDisc));
1887 unsigned AUTDiscReg =
1888 emitPtrauthDiscriminator(AUTDisc, AUTAddrDisc, InstsEmitted);
1889 bool AUTZero = AUTDiscReg == AArch64::XZR;
1890 unsigned AUTOpc = getAUTOpcodeForKey(AUTKey, AUTZero);
1891
1892 // autiza x16 ; if AUTZero
1893 // autia x16, x17 ; if !AUTZero
1894 MCInst AUTInst;
1895 AUTInst.setOpcode(AUTOpc);
1896 AUTInst.addOperand(MCOperand::createReg(AArch64::X16));
1897 AUTInst.addOperand(MCOperand::createReg(AArch64::X16));
1898 if (!AUTZero)
1899 AUTInst.addOperand(MCOperand::createReg(AUTDiscReg));
1900 EmitToStreamer(*OutStreamer, AUTInst);
1901 ++InstsEmitted;
1902
1903 // Unchecked or checked-but-non-trapping AUT is just an "AUT": we're done.
1904 if (!IsAUTPAC && (!ShouldCheck || !ShouldTrap)) {
1905 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
1906 return;
1907 }
1908
1909 MCSymbol *EndSym = nullptr;
1910
1911 // Checked sequences do an additional strip-and-compare.
1912 if (ShouldCheck) {
1913 MCSymbol *SuccessSym = createTempSymbol("auth_success_");
1914
1915 // XPAC has tied src/dst: use x17 as a temporary copy.
1916 // mov x17, x16
1917 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
1918 .addReg(AArch64::X17)
1919 .addReg(AArch64::XZR)
1920 .addReg(AArch64::X16)
1921 .addImm(0));
1922 ++InstsEmitted;
1923
1924 // xpaci x17
1925 EmitToStreamer(
1926 *OutStreamer,
1927 MCInstBuilder(XPACOpc).addReg(AArch64::X17).addReg(AArch64::X17));
1928 ++InstsEmitted;
1929
1930 // cmp x16, x17
1931 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSXrs)
1932 .addReg(AArch64::XZR)
1933 .addReg(AArch64::X16)
1934 .addReg(AArch64::X17)
1935 .addImm(0));
1936 ++InstsEmitted;
1937
1938 // b.eq Lsuccess
1939 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::Bcc)
1940 .addImm(AArch64CC::EQ)
1941 .addExpr(MCSymbolRefExpr::create(
1942 SuccessSym, OutContext)));
1943 ++InstsEmitted;
1944
1945 if (ShouldTrap) {
1946 // Trapping sequences do a 'brk'.
1947 // brk #<0xc470 + aut key>
1948 EmitToStreamer(*OutStreamer,
1949 MCInstBuilder(AArch64::BRK).addImm(0xc470 | AUTKey));
1950 ++InstsEmitted;
1951 } else {
1952 // Non-trapping checked sequences return the stripped result in x16,
1953 // skipping over the PAC if there is one.
1954
1955 // FIXME: can we simply return the AUT result, already in x16? without..
1956 // ..traps this is usable as an oracle anyway, based on high bits
1957 // mov x17, x16
1958 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
1959 .addReg(AArch64::X16)
1960 .addReg(AArch64::XZR)
1961 .addReg(AArch64::X17)
1962 .addImm(0));
1963 ++InstsEmitted;
1964
1965 if (IsAUTPAC) {
1966 EndSym = createTempSymbol("resign_end_");
1967
1968 // b Lend
1969 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B)
1970 .addExpr(MCSymbolRefExpr::create(
1971 EndSym, OutContext)));
1972 ++InstsEmitted;
1973 }
1974 }
1975
1976 // If the auth check succeeds, we can continue.
1977 // Lsuccess:
1978 OutStreamer->emitLabel(SuccessSym);
1979 }
1980
1981 // We already emitted unchecked and checked-but-non-trapping AUTs.
1982 // That left us with trapping AUTs, and AUTPACs.
1983 // Trapping AUTs don't need PAC: we're done.
1984 if (!IsAUTPAC) {
1985 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
1986 return;
1987 }
1988
1989 auto PACKey = (AArch64PACKey::ID)MI->getOperand(3).getImm();
1990 uint64_t PACDisc = MI->getOperand(4).getImm();
1991 unsigned PACAddrDisc = MI->getOperand(5).getReg();
1992
1993 // Compute pac discriminator into x17
1994 assert(isUInt<16>(PACDisc));
1995 unsigned PACDiscReg =
1996 emitPtrauthDiscriminator(PACDisc, PACAddrDisc, InstsEmitted);
1997 bool PACZero = PACDiscReg == AArch64::XZR;
1998 unsigned PACOpc = getPACOpcodeForKey(PACKey, PACZero);
1999
2000 // pacizb x16 ; if PACZero
2001 // pacib x16, x17 ; if !PACZero
2002 MCInst PACInst;
2003 PACInst.setOpcode(PACOpc);
2004 PACInst.addOperand(MCOperand::createReg(AArch64::X16));
2005 PACInst.addOperand(MCOperand::createReg(AArch64::X16));
2006 if (!PACZero)
2007 PACInst.addOperand(MCOperand::createReg(PACDiscReg));
2008 EmitToStreamer(*OutStreamer, PACInst);
2009 ++InstsEmitted;
2010
2011 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
2012 // Lend:
2013 if (EndSym)
2014 OutStreamer->emitLabel(EndSym);
2015}
2016
2017void AArch64AsmPrinter::emitPtrauthBranch(const MachineInstr *MI) {
2018 unsigned InstsEmitted = 0;
2019 bool IsCall = MI->getOpcode() == AArch64::BLRA;
2020 unsigned BrTarget = MI->getOperand(0).getReg();
2021
2022 auto Key = (AArch64PACKey::ID)MI->getOperand(1).getImm();
2023 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2024 "Invalid auth call key");
2025
2026 uint64_t Disc = MI->getOperand(2).getImm();
2027 assert(isUInt<16>(Disc));
2028
2029 unsigned AddrDisc = MI->getOperand(3).getReg();
2030
2031 // Compute discriminator into x17
2032 unsigned DiscReg = emitPtrauthDiscriminator(Disc, AddrDisc, InstsEmitted);
2033 bool IsZeroDisc = DiscReg == AArch64::XZR;
2034
2035 unsigned Opc;
2036 if (IsCall) {
2037 if (Key == AArch64PACKey::IA)
2038 Opc = IsZeroDisc ? AArch64::BLRAAZ : AArch64::BLRAA;
2039 else
2040 Opc = IsZeroDisc ? AArch64::BLRABZ : AArch64::BLRAB;
2041 } else {
2042 if (Key == AArch64PACKey::IA)
2043 Opc = IsZeroDisc ? AArch64::BRAAZ : AArch64::BRAA;
2044 else
2045 Opc = IsZeroDisc ? AArch64::BRABZ : AArch64::BRAB;
2046 }
2047
2048 MCInst BRInst;
2049 BRInst.setOpcode(Opc);
2050 BRInst.addOperand(MCOperand::createReg(BrTarget));
2051 if (!IsZeroDisc)
2052 BRInst.addOperand(MCOperand::createReg(DiscReg));
2053 EmitToStreamer(*OutStreamer, BRInst);
2054 ++InstsEmitted;
2055
2056 assert(STI->getInstrInfo()->getInstSizeInBytes(*MI) >= InstsEmitted * 4);
2057}
2058
2059const MCExpr *
2060AArch64AsmPrinter::lowerConstantPtrAuth(const ConstantPtrAuth &CPA) {
2061 MCContext &Ctx = OutContext;
2062
2063 // Figure out the base symbol and the addend, if any.
2064 APInt Offset(64, 0);
2065 const Value *BaseGV = CPA.getPointer()->stripAndAccumulateConstantOffsets(
2066 getDataLayout(), Offset, /*AllowNonInbounds=*/true);
2067
2068 auto *BaseGVB = dyn_cast<GlobalValue>(BaseGV);
2069
2070 // If we can't understand the referenced ConstantExpr, there's nothing
2071 // else we can do: emit an error.
2072 if (!BaseGVB) {
2073 BaseGV->getContext().emitError(
2074 "cannot resolve target base/addend of ptrauth constant");
2075 return nullptr;
2076 }
2077
2078 // If there is an addend, turn that into the appropriate MCExpr.
2079 const MCExpr *Sym = MCSymbolRefExpr::create(getSymbol(BaseGVB), Ctx);
2080 if (Offset.sgt(0))
2082 Sym, MCConstantExpr::create(Offset.getSExtValue(), Ctx), Ctx);
2083 else if (Offset.slt(0))
2085 Sym, MCConstantExpr::create((-Offset).getSExtValue(), Ctx), Ctx);
2086
2087 uint64_t KeyID = CPA.getKey()->getZExtValue();
2088 // We later rely on valid KeyID value in AArch64PACKeyIDToString call from
2089 // AArch64AuthMCExpr::printImpl, so fail fast.
2090 if (KeyID > AArch64PACKey::LAST)
2091 report_fatal_error("AArch64 PAC Key ID '" + Twine(KeyID) +
2092 "' out of range [0, " +
2093 Twine((unsigned)AArch64PACKey::LAST) + "]");
2094
2095 uint64_t Disc = CPA.getDiscriminator()->getZExtValue();
2096 if (!isUInt<16>(Disc))
2097 report_fatal_error("AArch64 PAC Discriminator '" + Twine(Disc) +
2098 "' out of range [0, 0xFFFF]");
2099
2100 // Finally build the complete @AUTH expr.
2101 return AArch64AuthMCExpr::create(Sym, Disc, AArch64PACKey::ID(KeyID),
2102 CPA.hasAddressDiscriminator(), Ctx);
2103}
2104
2105void AArch64AsmPrinter::LowerLOADauthptrstatic(const MachineInstr &MI) {
2106 unsigned DstReg = MI.getOperand(0).getReg();
2107 const MachineOperand &GAOp = MI.getOperand(1);
2108 const uint64_t KeyC = MI.getOperand(2).getImm();
2109 assert(KeyC <= AArch64PACKey::LAST &&
2110 "key is out of range [0, AArch64PACKey::LAST]");
2111 const auto Key = (AArch64PACKey::ID)KeyC;
2112 const uint64_t Disc = MI.getOperand(3).getImm();
2113 assert(isUInt<16>(Disc) &&
2114 "constant discriminator is out of range [0, 0xffff]");
2115
2116 // Emit instruction sequence like the following:
2117 // ADRP x16, symbol$auth_ptr$key$disc
2118 // LDR x16, [x16, :lo12:symbol$auth_ptr$key$disc]
2119 //
2120 // Where the $auth_ptr$ symbol is the stub slot containing the signed pointer
2121 // to symbol.
2122 MCSymbol *AuthPtrStubSym;
2123 if (TM.getTargetTriple().isOSBinFormatELF()) {
2124 const auto &TLOF =
2125 static_cast<const AArch64_ELFTargetObjectFile &>(getObjFileLowering());
2126
2127 assert(GAOp.getOffset() == 0 &&
2128 "non-zero offset for $auth_ptr$ stub slots is not supported");
2129 const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
2130 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, GASym, Key, Disc);
2131 } else {
2132 assert(TM.getTargetTriple().isOSBinFormatMachO() &&
2133 "LOADauthptrstatic is implemented only for MachO/ELF");
2134
2135 const auto &TLOF = static_cast<const AArch64_MachoTargetObjectFile &>(
2136 getObjFileLowering());
2137
2138 assert(GAOp.getOffset() == 0 &&
2139 "non-zero offset for $auth_ptr$ stub slots is not supported");
2140 const MCSymbol *GASym = TM.getSymbol(GAOp.getGlobal());
2141 AuthPtrStubSym = TLOF.getAuthPtrSlotSymbol(TM, MMI, GASym, Key, Disc);
2142 }
2143
2144 MachineOperand StubMOHi =
2147 AuthPtrStubSym, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2148 MCOperand StubMCHi, StubMCLo;
2149
2150 MCInstLowering.lowerOperand(StubMOHi, StubMCHi);
2151 MCInstLowering.lowerOperand(StubMOLo, StubMCLo);
2152
2153 EmitToStreamer(
2154 *OutStreamer,
2155 MCInstBuilder(AArch64::ADRP).addReg(DstReg).addOperand(StubMCHi));
2156
2157 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRXui)
2158 .addReg(DstReg)
2159 .addReg(DstReg)
2160 .addOperand(StubMCLo));
2161}
2162
2163void AArch64AsmPrinter::LowerMOVaddrPAC(const MachineInstr &MI) {
2164 unsigned InstsEmitted = 0;
2165 auto EmitAndIncrement = [this, &InstsEmitted](const MCInst &Inst) {
2166 EmitToStreamer(*OutStreamer, Inst);
2167 ++InstsEmitted;
2168 };
2169
2170 const bool IsGOTLoad = MI.getOpcode() == AArch64::LOADgotPAC;
2171 MachineOperand GAOp = MI.getOperand(0);
2172 const uint64_t KeyC = MI.getOperand(1).getImm();
2173 assert(KeyC <= AArch64PACKey::LAST &&
2174 "key is out of range [0, AArch64PACKey::LAST]");
2175 const auto Key = (AArch64PACKey::ID)KeyC;
2176 const unsigned AddrDisc = MI.getOperand(2).getReg();
2177 const uint64_t Disc = MI.getOperand(3).getImm();
2178 assert(isUInt<16>(Disc) &&
2179 "constant discriminator is out of range [0, 0xffff]");
2180
2181 const int64_t Offset = GAOp.getOffset();
2182 GAOp.setOffset(0);
2183
2184 // Emit:
2185 // target materialization:
2186 // - via GOT:
2187 // adrp x16, :got:target
2188 // ldr x16, [x16, :got_lo12:target]
2189 // add offset to x16 if offset != 0
2190 //
2191 // - direct:
2192 // adrp x16, target
2193 // add x16, x16, :lo12:target
2194 // add offset to x16 if offset != 0
2195 //
2196 // add offset to x16:
2197 // - abs(offset) fits 24 bits:
2198 // add/sub x16, x16, #<offset>[, #lsl 12] (up to 2 instructions)
2199 // - abs(offset) does not fit 24 bits:
2200 // - offset < 0:
2201 // movn+movk sequence filling x17 register with the offset (up to 4
2202 // instructions)
2203 // add x16, x16, x17
2204 // - offset > 0:
2205 // movz+movk sequence filling x17 register with the offset (up to 4
2206 // instructions)
2207 // add x16, x16, x17
2208 //
2209 // signing:
2210 // - 0 discriminator:
2211 // paciza x16
2212 // - Non-0 discriminator, no address discriminator:
2213 // mov x17, #Disc
2214 // pacia x16, x17
2215 // - address discriminator (with potentially folded immediate discriminator):
2216 // pacia x16, xAddrDisc
2217
2218 MachineOperand GAMOHi(GAOp), GAMOLo(GAOp);
2219 MCOperand GAMCHi, GAMCLo;
2220
2221 GAMOHi.setTargetFlags(AArch64II::MO_PAGE);
2222 GAMOLo.setTargetFlags(AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
2223 if (IsGOTLoad) {
2224 GAMOHi.addTargetFlag(AArch64II::MO_GOT);
2225 GAMOLo.addTargetFlag(AArch64II::MO_GOT);
2226 }
2227
2228 MCInstLowering.lowerOperand(GAMOHi, GAMCHi);
2229 MCInstLowering.lowerOperand(GAMOLo, GAMCLo);
2230
2231 EmitAndIncrement(
2232 MCInstBuilder(AArch64::ADRP).addReg(AArch64::X16).addOperand(GAMCHi));
2233
2234 if (IsGOTLoad) {
2235 EmitAndIncrement(MCInstBuilder(AArch64::LDRXui)
2236 .addReg(AArch64::X16)
2237 .addReg(AArch64::X16)
2238 .addOperand(GAMCLo));
2239 } else {
2240 EmitAndIncrement(MCInstBuilder(AArch64::ADDXri)
2241 .addReg(AArch64::X16)
2242 .addReg(AArch64::X16)
2243 .addOperand(GAMCLo)
2244 .addImm(0));
2245 }
2246
2247 if (Offset != 0) {
2248 const uint64_t AbsOffset = (Offset > 0 ? Offset : -((uint64_t)Offset));
2249 const bool IsNeg = Offset < 0;
2250 if (isUInt<24>(AbsOffset)) {
2251 for (int BitPos = 0; BitPos != 24 && (AbsOffset >> BitPos);
2252 BitPos += 12) {
2253 EmitAndIncrement(
2254 MCInstBuilder(IsNeg ? AArch64::SUBXri : AArch64::ADDXri)
2255 .addReg(AArch64::X16)
2256 .addReg(AArch64::X16)
2257 .addImm((AbsOffset >> BitPos) & 0xfff)
2258 .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, BitPos)));
2259 }
2260 } else {
2261 const uint64_t UOffset = Offset;
2262 EmitAndIncrement(MCInstBuilder(IsNeg ? AArch64::MOVNXi : AArch64::MOVZXi)
2263 .addReg(AArch64::X17)
2264 .addImm((IsNeg ? ~UOffset : UOffset) & 0xffff)
2265 .addImm(/*shift=*/0));
2266 auto NeedMovk = [IsNeg, UOffset](int BitPos) -> bool {
2267 assert(BitPos == 16 || BitPos == 32 || BitPos == 48);
2268 uint64_t Shifted = UOffset >> BitPos;
2269 if (!IsNeg)
2270 return Shifted != 0;
2271 for (int I = 0; I != 64 - BitPos; I += 16)
2272 if (((Shifted >> I) & 0xffff) != 0xffff)
2273 return true;
2274 return false;
2275 };
2276 for (int BitPos = 16; BitPos != 64 && NeedMovk(BitPos); BitPos += 16) {
2277 EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
2278 .addReg(AArch64::X17)
2279 .addReg(AArch64::X17)
2280 .addImm((UOffset >> BitPos) & 0xffff)
2281 .addImm(/*shift=*/BitPos));
2282 }
2283 EmitAndIncrement(MCInstBuilder(AArch64::ADDXrs)
2284 .addReg(AArch64::X16)
2285 .addReg(AArch64::X16)
2286 .addReg(AArch64::X17)
2287 .addImm(/*shift=*/0));
2288 }
2289 }
2290
2291 unsigned DiscReg = AddrDisc;
2292 if (Disc != 0) {
2293 if (AddrDisc != AArch64::XZR) {
2294 EmitAndIncrement(MCInstBuilder(AArch64::ORRXrs)
2295 .addReg(AArch64::X17)
2296 .addReg(AArch64::XZR)
2297 .addReg(AddrDisc)
2298 .addImm(0));
2299 EmitAndIncrement(MCInstBuilder(AArch64::MOVKXi)
2300 .addReg(AArch64::X17)
2301 .addReg(AArch64::X17)
2302 .addImm(Disc)
2303 .addImm(/*shift=*/48));
2304 } else {
2305 EmitAndIncrement(MCInstBuilder(AArch64::MOVZXi)
2306 .addReg(AArch64::X17)
2307 .addImm(Disc)
2308 .addImm(/*shift=*/0));
2309 }
2310 DiscReg = AArch64::X17;
2311 }
2312
2313 auto MIB = MCInstBuilder(getPACOpcodeForKey(Key, DiscReg == AArch64::XZR))
2314 .addReg(AArch64::X16)
2315 .addReg(AArch64::X16);
2316 if (DiscReg != AArch64::XZR)
2317 MIB.addReg(DiscReg);
2318 EmitAndIncrement(MIB);
2319
2320 assert(STI->getInstrInfo()->getInstSizeInBytes(MI) >= InstsEmitted * 4);
2321}
2322
2323const MCExpr *
2324AArch64AsmPrinter::lowerBlockAddressConstant(const BlockAddress &BA) {
2326 const Function &Fn = *BA.getFunction();
2327
2328 if (std::optional<uint16_t> BADisc =
2329 STI->getPtrAuthBlockAddressDiscriminatorIfEnabled(Fn))
2330 return AArch64AuthMCExpr::create(BAE, *BADisc, AArch64PACKey::IA,
2331 /*HasAddressDiversity=*/false, OutContext);
2332
2333 return BAE;
2334}
2335
2336// Simple pseudo-instructions have their lowering (with expansion to real
2337// instructions) auto-generated.
2338#include "AArch64GenMCPseudoLowering.inc"
2339
2340void AArch64AsmPrinter::emitInstruction(const MachineInstr *MI) {
2341 AArch64_MC::verifyInstructionPredicates(MI->getOpcode(), STI->getFeatureBits());
2342
2343 // Do any auto-generated pseudo lowerings.
2344 if (MCInst OutInst; lowerPseudoInstExpansion(MI, OutInst)) {
2345 EmitToStreamer(*OutStreamer, OutInst);
2346 return;
2347 }
2348
2349 if (MI->getOpcode() == AArch64::ADRP) {
2350 for (auto &Opd : MI->operands()) {
2351 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
2352 "swift_async_extendedFramePointerFlags") {
2353 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
2354 }
2355 }
2356 }
2357
2358 if (AArch64FI->getLOHRelated().count(MI)) {
2359 // Generate a label for LOH related instruction
2360 MCSymbol *LOHLabel = createTempSymbol("loh");
2361 // Associate the instruction with the label
2362 LOHInstToLabel[MI] = LOHLabel;
2363 OutStreamer->emitLabel(LOHLabel);
2364 }
2365
2367 static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
2368 // Do any manual lowerings.
2369 switch (MI->getOpcode()) {
2370 default:
2371 break;
2372 case AArch64::HINT: {
2373 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2374 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2375 // non-empty. If MI is the initial BTI, place the
2376 // __patchable_function_entries label after BTI.
2377 if (CurrentPatchableFunctionEntrySym &&
2378 CurrentPatchableFunctionEntrySym == CurrentFnBegin &&
2379 MI == &MF->front().front()) {
2380 int64_t Imm = MI->getOperand(0).getImm();
2381 if ((Imm & 32) && (Imm & 6)) {
2382 MCInst Inst;
2383 MCInstLowering.Lower(MI, Inst);
2384 EmitToStreamer(*OutStreamer, Inst);
2385 CurrentPatchableFunctionEntrySym = createTempSymbol("patch");
2386 OutStreamer->emitLabel(CurrentPatchableFunctionEntrySym);
2387 return;
2388 }
2389 }
2390 break;
2391 }
2392 case AArch64::MOVMCSym: {
2393 Register DestReg = MI->getOperand(0).getReg();
2394 const MachineOperand &MO_Sym = MI->getOperand(1);
2395 MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
2396 MCOperand Hi_MCSym, Lo_MCSym;
2397
2398 Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
2399 Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
2400
2401 MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
2402 MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
2403
2404 MCInst MovZ;
2405 MovZ.setOpcode(AArch64::MOVZXi);
2406 MovZ.addOperand(MCOperand::createReg(DestReg));
2407 MovZ.addOperand(Hi_MCSym);
2409 EmitToStreamer(*OutStreamer, MovZ);
2410
2411 MCInst MovK;
2412 MovK.setOpcode(AArch64::MOVKXi);
2413 MovK.addOperand(MCOperand::createReg(DestReg));
2414 MovK.addOperand(MCOperand::createReg(DestReg));
2415 MovK.addOperand(Lo_MCSym);
2417 EmitToStreamer(*OutStreamer, MovK);
2418 return;
2419 }
2420 case AArch64::MOVIv2d_ns:
2421 // It is generally beneficial to rewrite "fmov s0, wzr" to "movi d0, #0".
2422 // as movi is more efficient across all cores. Newer cores can eliminate
2423 // fmovs early and there is no difference with movi, but this not true for
2424 // all implementations.
2425 //
2426 // The floating-point version doesn't quite work in rare cases on older
2427 // CPUs, so on those targets we lower this instruction to movi.16b instead.
2428 if (STI->hasZeroCycleZeroingFPWorkaround() &&
2429 MI->getOperand(1).getImm() == 0) {
2430 MCInst TmpInst;
2431 TmpInst.setOpcode(AArch64::MOVIv16b_ns);
2432 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
2433 TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
2434 EmitToStreamer(*OutStreamer, TmpInst);
2435 return;
2436 }
2437 break;
2438
2439 case AArch64::DBG_VALUE:
2440 case AArch64::DBG_VALUE_LIST:
2441 if (isVerbose() && OutStreamer->hasRawTextSupport()) {
2442 SmallString<128> TmpStr;
2443 raw_svector_ostream OS(TmpStr);
2444 PrintDebugValueComment(MI, OS);
2445 OutStreamer->emitRawText(StringRef(OS.str()));
2446 }
2447 return;
2448
2449 case AArch64::EMITBKEY: {
2450 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
2451 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
2452 ExceptionHandlingType != ExceptionHandling::ARM)
2453 return;
2454
2455 if (getFunctionCFISectionType(*MF) == CFISection::None)
2456 return;
2457
2458 OutStreamer->emitCFIBKeyFrame();
2459 return;
2460 }
2461
2462 case AArch64::EMITMTETAGGED: {
2463 ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
2464 if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
2465 ExceptionHandlingType != ExceptionHandling::ARM)
2466 return;
2467
2468 if (getFunctionCFISectionType(*MF) != CFISection::None)
2469 OutStreamer->emitCFIMTETaggedFrame();
2470 return;
2471 }
2472
2473 case AArch64::AUT:
2474 case AArch64::AUTPAC:
2475 emitPtrauthAuthResign(MI);
2476 return;
2477
2478 case AArch64::LOADauthptrstatic:
2479 LowerLOADauthptrstatic(*MI);
2480 return;
2481
2482 case AArch64::LOADgotPAC:
2483 case AArch64::MOVaddrPAC:
2484 LowerMOVaddrPAC(*MI);
2485 return;
2486
2487 case AArch64::BRA:
2488 case AArch64::BLRA:
2489 emitPtrauthBranch(MI);
2490 return;
2491
2492 // Tail calls use pseudo instructions so they have the proper code-gen
2493 // attributes (isCall, isReturn, etc.). We lower them to the real
2494 // instruction here.
2495 case AArch64::AUTH_TCRETURN:
2496 case AArch64::AUTH_TCRETURN_BTI: {
2497 const uint64_t Key = MI->getOperand(2).getImm();
2498 assert((Key == AArch64PACKey::IA || Key == AArch64PACKey::IB) &&
2499 "Invalid auth key for tail-call return");
2500
2501 const uint64_t Disc = MI->getOperand(3).getImm();
2502 assert(isUInt<16>(Disc) && "Integer discriminator is too wide");
2503
2504 Register AddrDisc = MI->getOperand(4).getReg();
2505
2506 Register ScratchReg = MI->getOperand(0).getReg() == AArch64::X16
2507 ? AArch64::X17
2508 : AArch64::X16;
2509
2510 unsigned DiscReg = AddrDisc;
2511 if (Disc) {
2512 if (AddrDisc != AArch64::NoRegister) {
2513 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
2514 .addReg(ScratchReg)
2515 .addReg(AArch64::XZR)
2516 .addReg(AddrDisc)
2517 .addImm(0));
2518 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKXi)
2519 .addReg(ScratchReg)
2520 .addReg(ScratchReg)
2521 .addImm(Disc)
2522 .addImm(/*shift=*/48));
2523 } else {
2524 EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVZXi)
2525 .addReg(ScratchReg)
2526 .addImm(Disc)
2527 .addImm(/*shift=*/0));
2528 }
2529 DiscReg = ScratchReg;
2530 }
2531
2532 const bool IsZero = DiscReg == AArch64::NoRegister;
2533 const unsigned Opcodes[2][2] = {{AArch64::BRAA, AArch64::BRAAZ},
2534 {AArch64::BRAB, AArch64::BRABZ}};
2535
2536 MCInst TmpInst;
2537 TmpInst.setOpcode(Opcodes[Key][IsZero]);
2538 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
2539 if (!IsZero)
2540 TmpInst.addOperand(MCOperand::createReg(DiscReg));
2541 EmitToStreamer(*OutStreamer, TmpInst);
2542 return;
2543 }
2544
2545 case AArch64::TCRETURNri:
2546 case AArch64::TCRETURNrix16x17:
2547 case AArch64::TCRETURNrix17:
2548 case AArch64::TCRETURNrinotx16:
2549 case AArch64::TCRETURNriALL: {
2550 MCInst TmpInst;
2551 TmpInst.setOpcode(AArch64::BR);
2552 TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
2553 EmitToStreamer(*OutStreamer, TmpInst);
2554 return;
2555 }
2556 case AArch64::TCRETURNdi: {
2557 MCOperand Dest;
2558 MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
2559 MCInst TmpInst;
2560 TmpInst.setOpcode(AArch64::B);
2561 TmpInst.addOperand(Dest);
2562 EmitToStreamer(*OutStreamer, TmpInst);
2563 return;
2564 }
2565 case AArch64::SpeculationBarrierISBDSBEndBB: {
2566 // Print DSB SYS + ISB
2567 MCInst TmpInstDSB;
2568 TmpInstDSB.setOpcode(AArch64::DSB);
2569 TmpInstDSB.addOperand(MCOperand::createImm(0xf));
2570 EmitToStreamer(*OutStreamer, TmpInstDSB);
2571 MCInst TmpInstISB;
2572 TmpInstISB.setOpcode(AArch64::ISB);
2573 TmpInstISB.addOperand(MCOperand::createImm(0xf));
2574 EmitToStreamer(*OutStreamer, TmpInstISB);
2575 return;
2576 }
2577 case AArch64::SpeculationBarrierSBEndBB: {
2578 // Print SB
2579 MCInst TmpInstSB;
2580 TmpInstSB.setOpcode(AArch64::SB);
2581 EmitToStreamer(*OutStreamer, TmpInstSB);
2582 return;
2583 }
2584 case AArch64::TLSDESC_CALLSEQ: {
2585 /// lower this to:
2586 /// adrp x0, :tlsdesc:var
2587 /// ldr x1, [x0, #:tlsdesc_lo12:var]
2588 /// add x0, x0, #:tlsdesc_lo12:var
2589 /// .tlsdesccall var
2590 /// blr x1
2591 /// (TPIDR_EL0 offset now in x0)
2592 const MachineOperand &MO_Sym = MI->getOperand(0);
2593 MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
2594 MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
2595 MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
2596 MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
2597 MCInstLowering.lowerOperand(MO_Sym, Sym);
2598 MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
2599 MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
2600
2601 MCInst Adrp;
2602 Adrp.setOpcode(AArch64::ADRP);
2603 Adrp.addOperand(MCOperand::createReg(AArch64::X0));
2604 Adrp.addOperand(SymTLSDesc);
2605 EmitToStreamer(*OutStreamer, Adrp);
2606
2607 MCInst Ldr;
2608 if (STI->isTargetILP32()) {
2609 Ldr.setOpcode(AArch64::LDRWui);
2610 Ldr.addOperand(MCOperand::createReg(AArch64::W1));
2611 } else {
2612 Ldr.setOpcode(AArch64::LDRXui);
2613 Ldr.addOperand(MCOperand::createReg(AArch64::X1));
2614 }
2615 Ldr.addOperand(MCOperand::createReg(AArch64::X0));
2616 Ldr.addOperand(SymTLSDescLo12);
2618 EmitToStreamer(*OutStreamer, Ldr);
2619
2620 MCInst Add;
2621 if (STI->isTargetILP32()) {
2622 Add.setOpcode(AArch64::ADDWri);
2623 Add.addOperand(MCOperand::createReg(AArch64::W0));
2624 Add.addOperand(MCOperand::createReg(AArch64::W0));
2625 } else {
2626 Add.setOpcode(AArch64::ADDXri);
2627 Add.addOperand(MCOperand::createReg(AArch64::X0));
2628 Add.addOperand(MCOperand::createReg(AArch64::X0));
2629 }
2630 Add.addOperand(SymTLSDescLo12);
2632 EmitToStreamer(*OutStreamer, Add);
2633
2634 // Emit a relocation-annotation. This expands to no code, but requests
2635 // the following instruction gets an R_AARCH64_TLSDESC_CALL.
2636 MCInst TLSDescCall;
2637 TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
2638 TLSDescCall.addOperand(Sym);
2639 EmitToStreamer(*OutStreamer, TLSDescCall);
2640
2641 MCInst Blr;
2642 Blr.setOpcode(AArch64::BLR);
2643 Blr.addOperand(MCOperand::createReg(AArch64::X1));
2644 EmitToStreamer(*OutStreamer, Blr);
2645
2646 return;
2647 }
2648
2649 case AArch64::JumpTableDest32:
2650 case AArch64::JumpTableDest16:
2651 case AArch64::JumpTableDest8:
2652 LowerJumpTableDest(*OutStreamer, *MI);
2653 return;
2654
2655 case AArch64::BR_JumpTable:
2656 LowerHardenedBRJumpTable(*MI);
2657 return;
2658
2659 case AArch64::FMOVH0:
2660 case AArch64::FMOVS0:
2661 case AArch64::FMOVD0:
2662 emitFMov0(*MI);
2663 return;
2664
2665 case AArch64::MOPSMemoryCopyPseudo:
2666 case AArch64::MOPSMemoryMovePseudo:
2667 case AArch64::MOPSMemorySetPseudo:
2668 case AArch64::MOPSMemorySetTaggingPseudo:
2669 LowerMOPS(*OutStreamer, *MI);
2670 return;
2671
2672 case TargetOpcode::STACKMAP:
2673 return LowerSTACKMAP(*OutStreamer, SM, *MI);
2674
2675 case TargetOpcode::PATCHPOINT:
2676 return LowerPATCHPOINT(*OutStreamer, SM, *MI);
2677
2678 case TargetOpcode::STATEPOINT:
2679 return LowerSTATEPOINT(*OutStreamer, SM, *MI);
2680
2681 case TargetOpcode::FAULTING_OP:
2682 return LowerFAULTING_OP(*MI);
2683
2684 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2685 LowerPATCHABLE_FUNCTION_ENTER(*MI);
2686 return;
2687
2688 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
2689 LowerPATCHABLE_FUNCTION_EXIT(*MI);
2690 return;
2691
2692 case TargetOpcode::PATCHABLE_TAIL_CALL:
2693 LowerPATCHABLE_TAIL_CALL(*MI);
2694 return;
2695 case TargetOpcode::PATCHABLE_EVENT_CALL:
2696 return LowerPATCHABLE_EVENT_CALL(*MI, false);
2697 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2698 return LowerPATCHABLE_EVENT_CALL(*MI, true);
2699
2700 case AArch64::KCFI_CHECK:
2701 LowerKCFI_CHECK(*MI);
2702 return;
2703
2704 case AArch64::HWASAN_CHECK_MEMACCESS:
2705 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
2706 case AArch64::HWASAN_CHECK_MEMACCESS_FIXEDSHADOW:
2707 case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES_FIXEDSHADOW:
2708 LowerHWASAN_CHECK_MEMACCESS(*MI);
2709 return;
2710
2711 case AArch64::SEH_StackAlloc:
2712 TS->emitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
2713 return;
2714
2715 case AArch64::SEH_SaveFPLR:
2716 TS->emitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
2717 return;
2718
2719 case AArch64::SEH_SaveFPLR_X:
2720 assert(MI->getOperand(0).getImm() < 0 &&
2721 "Pre increment SEH opcode must have a negative offset");
2722 TS->emitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
2723 return;
2724
2725 case AArch64::SEH_SaveReg:
2726 TS->emitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
2727 MI->getOperand(1).getImm());
2728 return;
2729
2730 case AArch64::SEH_SaveReg_X:
2731 assert(MI->getOperand(1).getImm() < 0 &&
2732 "Pre increment SEH opcode must have a negative offset");
2733 TS->emitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
2734 -MI->getOperand(1).getImm());
2735 return;
2736
2737 case AArch64::SEH_SaveRegP:
2738 if (MI->getOperand(1).getImm() == 30 && MI->getOperand(0).getImm() >= 19 &&
2739 MI->getOperand(0).getImm() <= 28) {
2740 assert((MI->getOperand(0).getImm() - 19) % 2 == 0 &&
2741 "Register paired with LR must be odd");
2742 TS->emitARM64WinCFISaveLRPair(MI->getOperand(0).getImm(),
2743 MI->getOperand(2).getImm());
2744 return;
2745 }
2746 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2747 "Non-consecutive registers not allowed for save_regp");
2748 TS->emitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
2749 MI->getOperand(2).getImm());
2750 return;
2751
2752 case AArch64::SEH_SaveRegP_X:
2753 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2754 "Non-consecutive registers not allowed for save_regp_x");
2755 assert(MI->getOperand(2).getImm() < 0 &&
2756 "Pre increment SEH opcode must have a negative offset");
2757 TS->emitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
2758 -MI->getOperand(2).getImm());
2759 return;
2760
2761 case AArch64::SEH_SaveFReg:
2762 TS->emitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
2763 MI->getOperand(1).getImm());
2764 return;
2765
2766 case AArch64::SEH_SaveFReg_X:
2767 assert(MI->getOperand(1).getImm() < 0 &&
2768 "Pre increment SEH opcode must have a negative offset");
2769 TS->emitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
2770 -MI->getOperand(1).getImm());
2771 return;
2772
2773 case AArch64::SEH_SaveFRegP:
2774 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2775 "Non-consecutive registers not allowed for save_regp");
2776 TS->emitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
2777 MI->getOperand(2).getImm());
2778 return;
2779
2780 case AArch64::SEH_SaveFRegP_X:
2781 assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
2782 "Non-consecutive registers not allowed for save_regp_x");
2783 assert(MI->getOperand(2).getImm() < 0 &&
2784 "Pre increment SEH opcode must have a negative offset");
2785 TS->emitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
2786 -MI->getOperand(2).getImm());
2787 return;
2788
2789 case AArch64::SEH_SetFP:
2791 return;
2792
2793 case AArch64::SEH_AddFP:
2794 TS->emitARM64WinCFIAddFP(MI->getOperand(0).getImm());
2795 return;
2796
2797 case AArch64::SEH_Nop:
2798 TS->emitARM64WinCFINop();
2799 return;
2800
2801 case AArch64::SEH_PrologEnd:
2803 return;
2804
2805 case AArch64::SEH_EpilogStart:
2807 return;
2808
2809 case AArch64::SEH_EpilogEnd:
2811 return;
2812
2813 case AArch64::SEH_PACSignLR:
2815 return;
2816
2817 case AArch64::SEH_SaveAnyRegQP:
2818 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
2819 "Non-consecutive registers not allowed for save_any_reg");
2820 assert(MI->getOperand(2).getImm() >= 0 &&
2821 "SaveAnyRegQP SEH opcode offset must be non-negative");
2822 assert(MI->getOperand(2).getImm() <= 1008 &&
2823 "SaveAnyRegQP SEH opcode offset must fit into 6 bits");
2824 TS->emitARM64WinCFISaveAnyRegQP(MI->getOperand(0).getImm(),
2825 MI->getOperand(2).getImm());
2826 return;
2827
2828 case AArch64::SEH_SaveAnyRegQPX:
2829 assert(MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1 &&
2830 "Non-consecutive registers not allowed for save_any_reg");
2831 assert(MI->getOperand(2).getImm() < 0 &&
2832 "SaveAnyRegQPX SEH opcode offset must be negative");
2833 assert(MI->getOperand(2).getImm() >= -1008 &&
2834 "SaveAnyRegQPX SEH opcode offset must fit into 6 bits");
2835 TS->emitARM64WinCFISaveAnyRegQPX(MI->getOperand(0).getImm(),
2836 -MI->getOperand(2).getImm());
2837 return;
2838 }
2839
2840 // Finally, do the automated lowerings for everything else.
2841 MCInst TmpInst;
2842 MCInstLowering.Lower(MI, TmpInst);
2843 EmitToStreamer(*OutStreamer, TmpInst);
2844}
2845
2846void AArch64AsmPrinter::emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI,
2847 MCSymbol *LazyPointer) {
2848 // _ifunc:
2849 // adrp x16, lazy_pointer@GOTPAGE
2850 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
2851 // ldr x16, [x16]
2852 // br x16
2853
2854 {
2855 MCInst Adrp;
2856 Adrp.setOpcode(AArch64::ADRP);
2857 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
2858 MCOperand SymPage;
2859 MCInstLowering.lowerOperand(
2862 SymPage);
2863 Adrp.addOperand(SymPage);
2864 OutStreamer->emitInstruction(Adrp, *STI);
2865 }
2866
2867 {
2868 MCInst Ldr;
2869 Ldr.setOpcode(AArch64::LDRXui);
2870 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
2871 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
2872 MCOperand SymPageOff;
2873 MCInstLowering.lowerOperand(
2876 SymPageOff);
2877 Ldr.addOperand(SymPageOff);
2879 OutStreamer->emitInstruction(Ldr, *STI);
2880 }
2881
2882 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDRXui)
2883 .addReg(AArch64::X16)
2884 .addReg(AArch64::X16)
2885 .addImm(0),
2886 *STI);
2887
2888 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
2889 ? AArch64::BRAAZ
2890 : AArch64::BR)
2891 .addReg(AArch64::X16),
2892 *STI);
2893}
2894
2895void AArch64AsmPrinter::emitMachOIFuncStubHelperBody(Module &M,
2896 const GlobalIFunc &GI,
2897 MCSymbol *LazyPointer) {
2898 // These stub helpers are only ever called once, so here we're optimizing for
2899 // minimum size by using the pre-indexed store variants, which saves a few
2900 // bytes of instructions to bump & restore sp.
2901
2902 // _ifunc.stub_helper:
2903 // stp fp, lr, [sp, #-16]!
2904 // mov fp, sp
2905 // stp x1, x0, [sp, #-16]!
2906 // stp x3, x2, [sp, #-16]!
2907 // stp x5, x4, [sp, #-16]!
2908 // stp x7, x6, [sp, #-16]!
2909 // stp d1, d0, [sp, #-16]!
2910 // stp d3, d2, [sp, #-16]!
2911 // stp d5, d4, [sp, #-16]!
2912 // stp d7, d6, [sp, #-16]!
2913 // bl _resolver
2914 // adrp x16, lazy_pointer@GOTPAGE
2915 // ldr x16, [x16, lazy_pointer@GOTPAGEOFF]
2916 // str x0, [x16]
2917 // mov x16, x0
2918 // ldp d7, d6, [sp], #16
2919 // ldp d5, d4, [sp], #16
2920 // ldp d3, d2, [sp], #16
2921 // ldp d1, d0, [sp], #16
2922 // ldp x7, x6, [sp], #16
2923 // ldp x5, x4, [sp], #16
2924 // ldp x3, x2, [sp], #16
2925 // ldp x1, x0, [sp], #16
2926 // ldp fp, lr, [sp], #16
2927 // br x16
2928
2929 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
2930 .addReg(AArch64::SP)
2931 .addReg(AArch64::FP)
2932 .addReg(AArch64::LR)
2933 .addReg(AArch64::SP)
2934 .addImm(-2),
2935 *STI);
2936
2937 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
2938 .addReg(AArch64::FP)
2939 .addReg(AArch64::SP)
2940 .addImm(0)
2941 .addImm(0),
2942 *STI);
2943
2944 for (int I = 0; I != 4; ++I)
2945 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPXpre)
2946 .addReg(AArch64::SP)
2947 .addReg(AArch64::X1 + 2 * I)
2948 .addReg(AArch64::X0 + 2 * I)
2949 .addReg(AArch64::SP)
2950 .addImm(-2),
2951 *STI);
2952
2953 for (int I = 0; I != 4; ++I)
2954 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STPDpre)
2955 .addReg(AArch64::SP)
2956 .addReg(AArch64::D1 + 2 * I)
2957 .addReg(AArch64::D0 + 2 * I)
2958 .addReg(AArch64::SP)
2959 .addImm(-2),
2960 *STI);
2961
2962 OutStreamer->emitInstruction(
2963 MCInstBuilder(AArch64::BL)
2965 *STI);
2966
2967 {
2968 MCInst Adrp;
2969 Adrp.setOpcode(AArch64::ADRP);
2970 Adrp.addOperand(MCOperand::createReg(AArch64::X16));
2971 MCOperand SymPage;
2972 MCInstLowering.lowerOperand(
2973 MachineOperand::CreateES(LazyPointer->getName().data() + 1,
2975 SymPage);
2976 Adrp.addOperand(SymPage);
2977 OutStreamer->emitInstruction(Adrp, *STI);
2978 }
2979
2980 {
2981 MCInst Ldr;
2982 Ldr.setOpcode(AArch64::LDRXui);
2983 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
2984 Ldr.addOperand(MCOperand::createReg(AArch64::X16));
2985 MCOperand SymPageOff;
2986 MCInstLowering.lowerOperand(
2987 MachineOperand::CreateES(LazyPointer->getName().data() + 1,
2989 SymPageOff);
2990 Ldr.addOperand(SymPageOff);
2992 OutStreamer->emitInstruction(Ldr, *STI);
2993 }
2994
2995 OutStreamer->emitInstruction(MCInstBuilder(AArch64::STRXui)
2996 .addReg(AArch64::X0)
2997 .addReg(AArch64::X16)
2998 .addImm(0),
2999 *STI);
3000
3001 OutStreamer->emitInstruction(MCInstBuilder(AArch64::ADDXri)
3002 .addReg(AArch64::X16)
3003 .addReg(AArch64::X0)
3004 .addImm(0)
3005 .addImm(0),
3006 *STI);
3007
3008 for (int I = 3; I != -1; --I)
3009 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPDpost)
3010 .addReg(AArch64::SP)
3011 .addReg(AArch64::D1 + 2 * I)
3012 .addReg(AArch64::D0 + 2 * I)
3013 .addReg(AArch64::SP)
3014 .addImm(2),
3015 *STI);
3016
3017 for (int I = 3; I != -1; --I)
3018 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
3019 .addReg(AArch64::SP)
3020 .addReg(AArch64::X1 + 2 * I)
3021 .addReg(AArch64::X0 + 2 * I)
3022 .addReg(AArch64::SP)
3023 .addImm(2),
3024 *STI);
3025
3026 OutStreamer->emitInstruction(MCInstBuilder(AArch64::LDPXpost)
3027 .addReg(AArch64::SP)
3028 .addReg(AArch64::FP)
3029 .addReg(AArch64::LR)
3030 .addReg(AArch64::SP)
3031 .addImm(2),
3032 *STI);
3033
3034 OutStreamer->emitInstruction(MCInstBuilder(TM.getTargetTriple().isArm64e()
3035 ? AArch64::BRAAZ
3036 : AArch64::BR)
3037 .addReg(AArch64::X16),
3038 *STI);
3039}
3040
3041const MCExpr *AArch64AsmPrinter::lowerConstant(const Constant *CV) {
3042 if (const GlobalValue *GV = dyn_cast<GlobalValue>(CV)) {
3043 return MCSymbolRefExpr::create(MCInstLowering.GetGlobalValueSymbol(GV, 0),
3044 OutContext);
3045 }
3046
3047 return AsmPrinter::lowerConstant(CV);
3048}
3049
3050// Force static initialization.
3057}
static cl::opt< PtrauthCheckMode > PtrauthAuthChecks("aarch64-ptrauth-auth-checks", cl::Hidden, cl::values(clEnumValN(Unchecked, "none", "don't test for failure"), clEnumValN(Poison, "poison", "poison on failure"), clEnumValN(Trap, "trap", "trap on failure")), cl::desc("Check pointer authentication auth/resign failures"), cl::init(Default))
PtrauthCheckMode
@ Poison
@ Default
@ Unchecked
static void emitAuthenticatedPointer(MCStreamer &OutStreamer, MCSymbol *StubLabel, const MCExpr *StubAuthPtrRef)
LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAArch64AsmPrinter()
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:686
#define LLVM_EXTERNAL_VISIBILITY
Definition: Compiler.h:131
std::string Name
uint64_t Size
Symbol * Sym
Definition: ELF_riscv.cpp:479
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
static bool printOperand(raw_ostream &OS, const SelectionDAG *G, const SDValue Value)
This file defines the SmallString class.
This file defines the SmallVector class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition: Value.cpp:469
static bool printAsmMRegister(const X86AsmPrinter &P, const MachineOperand &MO, char Mode, raw_ostream &O)
static const AArch64AuthMCExpr * create(const MCExpr *Expr, uint16_t Discriminator, AArch64PACKey::ID Key, bool HasAddressDiversity, MCContext &Ctx)
AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...
std::optional< std::string > getOutliningStyle() const
static const char * getRegisterName(MCRegister Reg, unsigned AltIdx=AArch64::NoRegAltName)
static const AArch64MCExpr * create(const MCExpr *Expr, VariantKind Kind, MCContext &Ctx)
AArch64MCInstLower - This class is used to lower an MachineInstr into an MCInst.
bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const
virtual void emitARM64WinCFISaveRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveRegPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveRegX(unsigned Reg, int Offset)
virtual void emitARM64WinCFIAllocStack(unsigned Size)
virtual void emitARM64WinCFISaveFPLRX(int Offset)
virtual void emitDirectiveVariantPCS(MCSymbol *Symbol)
Callback used to implement the .variant_pcs directive.
virtual void emitARM64WinCFIAddFP(unsigned Size)
virtual void emitARM64WinCFISaveFPLR(int Offset)
virtual void emitARM64WinCFISaveFRegP(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveAnyRegQPX(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveFRegX(unsigned Reg, int Offset)
void emitNoteSection(unsigned Flags, uint64_t PAuthABIPlatform=-1, uint64_t PAuthABIVersion=-1)
Callback used to implement the .note.gnu.property section.
virtual void emitARM64WinCFISaveReg(unsigned Reg, int Offset)
virtual void emitARM64WinCFISaveLRPair(unsigned Reg, int Offset)
This implementation is used for AArch64 ELF targets (Linux in particular).
AArch64_MachoTargetObjectFile - This TLOF implementation is used for Darwin.
Class for arbitrary precision integers.
Definition: APInt.h:77
Represent the analysis usage information of a pass.
void setPreservesAll()
Set by analyses that do not transform their input at all.
This class is intended to be used as a driving class for all asm writers.
Definition: AsmPrinter.h:86
virtual void emitInstruction(const MachineInstr *)
Targets should implement this to emit instructions.
Definition: AsmPrinter.h:561
virtual const MCExpr * lowerConstantPtrAuth(const ConstantPtrAuth &CPA)
Definition: AsmPrinter.h:582
void emitXRayTable()
Emit a table with all XRay instrumentation points.
virtual void emitGlobalAlias(const Module &M, const GlobalAlias &GA)
virtual MCSymbol * GetCPISymbol(unsigned CPID) const
Return the symbol for the specified constant pool entry.
virtual void emitMachOIFuncStubHelperBody(Module &M, const GlobalIFunc &GI, MCSymbol *LazyPointer)
Definition: AsmPrinter.h:613
virtual void emitJumpTableInfo()
Print assembly representations of the jump tables used by the current function to the current output ...
virtual void SetupMachineFunction(MachineFunction &MF)
This should be called when a new MachineFunction is being processed from runOnMachineFunction.
void emitFunctionBody()
This method emits the body and trailer for a function.
virtual void emitStartOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the start of their fi...
Definition: AsmPrinter.h:537
virtual void emitEndOfAsmFile(Module &)
This virtual method can be overridden by targets that want to emit something at the end of their file...
Definition: AsmPrinter.h:541
virtual void emitMachOIFuncStubBody(Module &M, const GlobalIFunc &GI, MCSymbol *LazyPointer)
Definition: AsmPrinter.h:607
void getAnalysisUsage(AnalysisUsage &AU) const override
Record analysis usage.
Definition: AsmPrinter.cpp:425
virtual bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const
Definition: AsmPrinter.h:912
virtual const MCSubtargetInfo * getIFuncMCSubtargetInfo() const
getSubtargetInfo() cannot be used where this is needed because we don't have a MachineFunction when w...
Definition: AsmPrinter.h:603
bool runOnMachineFunction(MachineFunction &MF) override
Emit the specified function out to the OutStreamer.
Definition: AsmPrinter.h:387
virtual const MCExpr * lowerConstant(const Constant *CV)
Lower the specified LLVM Constant to an MCExpr.
virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant as...
virtual void emitXXStructor(const DataLayout &DL, const Constant *CV)
Targets can override this to change how global constants that are part of a C++ static/global constru...
Definition: AsmPrinter.h:578
virtual void emitFunctionBodyEnd()
Targets can override this to emit stuff after the last basic block in the function.
Definition: AsmPrinter.h:549
virtual void emitFunctionEntryLabel()
EmitFunctionEntryLabel - Emit the label that is the entrypoint for the function.
virtual std::tuple< const MCSymbol *, uint64_t, const MCSymbol *, codeview::JumpTableEntrySize > getCodeViewJumpTableInfo(int JTI, const MachineInstr *BranchInstr, const MCSymbol *BranchLabel) const
Gets information required to create a CodeView debug symbol for a jump table.
virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &OS)
Print the specified operand of MI, an INLINEASM instruction, using the specified assembler variant.
virtual const MCExpr * lowerBlockAddressConstant(const BlockAddress &BA)
Lower the specified BlockAddress to an MCExpr.
The address of a basic block.
Definition: Constants.h:890
Function * getFunction() const
Definition: Constants.h:918
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:155
A signed pointer, in the ptrauth sense.
Definition: Constants.h:1012
Constant * getPointer() const
The pointer that is signed in this ptrauth signed pointer.
Definition: Constants.h:1037
ConstantInt * getKey() const
The Key ID, an i32 constant.
Definition: Constants.h:1040
bool hasAddressDiscriminator() const
Whether there is any non-null address discriminator.
Definition: Constants.h:1055
ConstantInt * getDiscriminator() const
The integer discriminator, an i64 constant, or 0.
Definition: Constants.h:1043
This is an important base class in LLVM.
Definition: Constant.h:42
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:63
const Constant * getAliasee() const
Definition: GlobalAlias.h:84
const Constant * getResolver() const
Definition: GlobalIFunc.h:70
bool hasLocalLinkage() const
Definition: GlobalValue.h:528
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
static const MCBinaryExpr * createLShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:612
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:532
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:617
static const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Definition: MCExpr.cpp:193
Context object for machine code objects.
Definition: MCContext.h:83
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:34
MCInstBuilder & addReg(unsigned Reg)
Add a new register operand.
Definition: MCInstBuilder.h:37
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Definition: MCInstBuilder.h:43
MCInstBuilder & addExpr(const MCExpr *Val)
Add a new MCExpr operand.
Definition: MCInstBuilder.h:61
Instances of this class represent a single low-level machine instruction.
Definition: MCInst.h:184
void addOperand(const MCOperand Op)
Definition: MCInst.h:210
void setOpcode(unsigned Op)
Definition: MCInst.h:197
MCSection * getDataSection() const
Instances of this class represent operands of the MCInst class.
Definition: MCInst.h:36
static MCOperand createReg(unsigned Reg)
Definition: MCInst.h:134
static MCOperand createExpr(const MCExpr *Val)
Definition: MCInst.h:162
static MCOperand createImm(int64_t Val)
Definition: MCInst.h:141
uint16_t getEncodingValue(MCRegister RegNo) const
Returns the encoding for RegNo.
Instances of this class represent a uniqued identifier for a section in the current translation unit.
Definition: MCSection.h:36
Streaming machine code generation interface.
Definition: MCStreamer.h:213
virtual void emitCFIBKeyFrame()
Definition: MCStreamer.cpp:248
virtual void beginCOFFSymbolDef(const MCSymbol *Symbol)
Start emitting COFF symbol definition.
virtual void emitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI)
Emit the given Instruction into the current section.
virtual void emitCOFFSymbolType(int Type)
Emit the type of the symbol.
virtual bool hasRawTextSupport() const
Return true if this asm streamer supports emitting unformatted text to the .s file with EmitRawText.
Definition: MCStreamer.h:345
virtual void endCOFFSymbolDef()
Marks the end of the symbol definition.
MCContext & getContext() const
Definition: MCStreamer.h:300
virtual void AddComment(const Twine &T, bool EOL=true)
Add a textual comment.
Definition: MCStreamer.h:364
virtual void emitCFIMTETaggedFrame()
Definition: MCStreamer.cpp:255
void emitValue(const MCExpr *Value, unsigned Size, SMLoc Loc=SMLoc())
Definition: MCStreamer.cpp:179
virtual void emitLabel(MCSymbol *Symbol, SMLoc Loc=SMLoc())
Emit a label for Symbol into the current section.
Definition: MCStreamer.cpp:414
MCTargetStreamer * getTargetStreamer()
Definition: MCStreamer.h:309
void emitRawText(const Twine &String)
If this file is backed by a assembly streamer, this dumps the specified string in the output ....
Definition: MCStreamer.cpp:998
virtual void emitCOFFSymbolStorageClass(int StorageClass)
Emit the storage class of the symbol.
Generic base class for all target subtargets.
Represent a reference to a symbol from inside an expression.
Definition: MCExpr.h:188
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:393
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
StringRef getName() const
getName - Get the symbol name.
Definition: MCSymbol.h:205
Metadata node.
Definition: Metadata.h:1069
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Representation of each machine instruction.
Definition: MachineInstr.h:69
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:685
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:579
const std::vector< MachineJumpTableEntry > & getJumpTables() const
MachineModuleInfoELF - This is a MachineModuleInfoImpl implementation for ELF targets.
MachineModuleInfoMachO - This is a MachineModuleInfoImpl implementation for MachO targets.
MachineOperand class - Representation of each machine instruction operand.
unsigned getSubReg() const
static MachineOperand CreateMCSymbol(MCSymbol *Sym, unsigned TargetFlags=0)
const GlobalValue * getGlobal() const
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
const BlockAddress * getBlockAddress() const
void setOffset(int64_t Offset)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
@ MO_Immediate
Immediate operand.
@ MO_GlobalAddress
Address of a global value.
@ MO_BlockAddress
Address of a basic block.
@ MO_Register
Register operand.
@ MO_ExternalSymbol
Name of external global symbol.
int64_t getOffset() const
Return the offset from the symbol in this operand.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
MI-level patchpoint operands.
Definition: StackMaps.h:76
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
static SectionKind getMetadata()
Definition: SectionKind.h:188
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
MI-level stackmap operands.
Definition: StackMaps.h:35
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
Definition: StackMaps.h:50
void recordStatepoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a statepoint instruction.
Definition: StackMaps.cpp:569
void recordPatchPoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a patchpoint instruction.
Definition: StackMaps.cpp:548
void recordStackMap(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a stackmap instruction.
Definition: StackMaps.cpp:538
MI-level Statepoint operands.
Definition: StackMaps.h:158
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
Definition: StringRef.h:131
virtual MCSection * getSectionForJumpTable(const Function &F, const TargetMachine &TM) const
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
MCRegister getRegister(unsigned i) const
Return the specified register in the class.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr) const
Accumulate the constant offset this value has compared to a base pointer.
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an SmallVector or SmallString.
Definition: raw_ostream.h:691
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_G1
MO_G1 - A symbol operand with this flag (granule 1) represents the bits 16-31 of a 64-bit address,...
@ MO_S
MO_S - Indicates that the bits of the symbol operand represented by MO_G0 etc are signed.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_G0
MO_G0 - A symbol operand with this flag (granule 0) represents the bits 0-15 of a 64-bit address,...
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TLS
MO_TLS - Indicates that the operand being accessed is some kind of thread-local symbol.
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static uint64_t encodeLogicalImmediate(uint64_t imm, unsigned regSize)
encodeLogicalImmediate - Return the encoded immediate value for a logical immediate instruction of th...
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
Key
PAL metadata keys.
SymbolStorageClass
Storage class tells where and what the symbol represents.
Definition: COFF.h:217
@ IMAGE_SYM_CLASS_EXTERNAL
External symbol.
Definition: COFF.h:223
@ IMAGE_SYM_CLASS_STATIC
Static.
Definition: COFF.h:224
@ IMAGE_SYM_DTYPE_NULL
No complex type; simple scalar variable.
Definition: COFF.h:273
@ IMAGE_SYM_DTYPE_FUNCTION
A function that returns a base type.
Definition: COFF.h:275
@ SCT_COMPLEX_TYPE_SHIFT
Type is formed as (base + (derived << SCT_COMPLEX_TYPE_SHIFT))
Definition: COFF.h:279
@ AArch64_VectorCall
Used between AArch64 Advanced SIMD functions.
Definition: CallingConv.h:221
@ AArch64_SVE_VectorCall
Used between AArch64 SVE functions.
Definition: CallingConv.h:224
@ GNU_PROPERTY_AARCH64_FEATURE_1_BTI
Definition: ELF.h:1790
@ GNU_PROPERTY_AARCH64_FEATURE_1_PAC
Definition: ELF.h:1791
@ GNU_PROPERTY_AARCH64_FEATURE_1_GCS
Definition: ELF.h:1792
@ SHT_PROGBITS
Definition: ELF.h:1089
@ SHF_ALLOC
Definition: ELF.h:1186
@ SHF_GROUP
Definition: ELF.h:1208
@ SHF_EXECINSTR
Definition: ELF.h:1189
@ S_REGULAR
S_REGULAR - Regular section.
Definition: MachO.h:127
Reg
All possible values of the reg field in the ModR/M byte.
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:711
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
constexpr double e
Definition: MathExtras.h:47
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
@ Offset
Definition: DWP.cpp:480
static unsigned getXPACOpcodeForKey(AArch64PACKey::ID K)
Return XPAC opcode to be used for a ptrauth strip using the given key.
ExceptionHandling
Target & getTheAArch64beTarget()
Target & getTheAArch64leTarget()
static unsigned getXRegFromWReg(unsigned Reg)
static unsigned getXRegFromXRegTuple(unsigned RegTuple)
Target & getTheAArch64_32Target()
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
Target & getTheARM64_32Target()
@ MCAF_SubsectionsViaSymbols
.subsections_via_symbols (MachO)
Definition: MCDirectives.h:55
@ Add
Sum of integers.
static unsigned getWRegFromXReg(unsigned Reg)
Target & getTheARM64Target()
static unsigned getPACOpcodeForKey(AArch64PACKey::ID K, bool Zero)
Return PAC opcode to be used for a ptrauth sign using the given key, or its PAC*Z variant that doesn'...
static unsigned getAUTOpcodeForKey(AArch64PACKey::ID K, bool Zero)
Return AUT opcode to be used for a ptrauth auth using the given key, or its AUT*Z variant that doesn'...
@ MCSA_Weak
.weak
Definition: MCDirectives.h:45
@ MCSA_Global
.type _foo, @gnu_unique_object
Definition: MCDirectives.h:30
@ MCSA_WeakAntiDep
.weak_anti_dep (COFF)
Definition: MCDirectives.h:49
@ MCSA_ELF_TypeFunction
.type _foo, STT_FUNC # aka @function
Definition: MCDirectives.h:23
@ MCSA_Hidden
.hidden (ELF)
Definition: MCDirectives.h:33
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
RegisterAsmPrinter - Helper template for registering a target specific assembly printer,...