13#ifndef LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H
14#define LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H
363 constexpr uint32_t LoadStoreImm12Mask = 0x3b000000;
364 return (Instr & LoadStoreImm12Mask) == 0x39000000;
368 constexpr uint32_t TestAndBranchImm14Mask = 0x7e000000;
369 return (Instr & TestAndBranchImm14Mask) == 0x36000000;
373 constexpr uint32_t CondBranchImm19Mask = 0xfe000000;
374 return (Instr & CondBranchImm19Mask) == 0x54000000;
378 constexpr uint32_t CompAndBranchImm19Mask = 0x7e000000;
379 return (Instr & CompAndBranchImm19Mask) == 0x34000000;
383 constexpr uint32_t ADRMask = 0x9f000000;
384 return (Instr & ADRMask) == 0x10000000;
388 constexpr uint32_t LDRLitMask = 0x3b000000;
389 return (Instr & LDRLitMask) == 0x18000000;
399 constexpr uint32_t Vec128Mask = 0x04800000;
402 uint32_t ImplicitShift = Instr >> 30;
403 if (ImplicitShift == 0)
404 if ((Instr & Vec128Mask) == Vec128Mask)
407 return ImplicitShift;
415 constexpr uint32_t MoveWideImm16Mask = 0x5f9fffe0;
416 return (Instr & MoveWideImm16Mask) == 0x52800000;
425 uint32_t ImplicitShift = (Instr >> 21) & 0b11;
426 return ImplicitShift << 4;
434 using namespace support;
436 char *BlockWorkingMem =
B.getAlreadyMutableContent().data();
437 char *FixupPtr = BlockWorkingMem +
E.getOffset();
440 switch (
E.getKind()) {
442 uint64_t Value =
E.getTarget().getAddress().getValue() +
E.getAddend();
443 *(ulittle64_t *)FixupPtr =
Value;
447 uint64_t Value =
E.getTarget().getAddress().getValue() +
E.getAddend();
448 if (
Value > std::numeric_limits<uint32_t>::max())
450 *(ulittle32_t *)FixupPtr =
Value;
459 Value =
E.getTarget().getAddress() - FixupAddress +
E.getAddend();
461 Value = FixupAddress -
E.getTarget().getAddress() +
E.getAddend();
464 if (
Value < std::numeric_limits<int32_t>::min() ||
465 Value > std::numeric_limits<int32_t>::max())
467 *(little32_t *)FixupPtr =
Value;
469 *(little64_t *)FixupPtr =
Value;
474 "Branch-inst is not 32-bit aligned");
476 int64_t
Value =
E.getTarget().getAddress() - FixupAddress +
E.getAddend();
479 return make_error<JITLinkError>(
"BranchPCRel26 target is not 32-bit "
482 if (
Value < -(1 << 27) ||
Value > ((1 << 27) - 1))
485 uint32_t RawInstr = *(little32_t *)FixupPtr;
486 assert((RawInstr & 0x7fffffff) == 0x14000000 &&
487 "RawInstr isn't a B or BR immediate instruction");
489 uint32_t FixedInstr = RawInstr | Imm;
490 *(little32_t *)FixupPtr = FixedInstr;
495 (
E.getTarget().getAddress() +
E.getAddend()).getValue();
497 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
499 "RawInstr isn't a MOVK/MOVZ instruction");
502 uint32_t Imm = (TargetOffset >> ImmShift) & 0xffff;
503 uint32_t FixedInstr = RawInstr | (Imm << 5);
504 *(ulittle32_t *)FixupPtr = FixedInstr;
508 assert((FixupAddress.
getValue() & 0x3) == 0 &&
"LDR is not 32-bit aligned");
509 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
511 int64_t Delta =
E.getTarget().getAddress() +
E.getAddend() - FixupAddress;
513 return make_error<JITLinkError>(
"LDR literal target is not 32-bit "
515 if (!isInt<21>(Delta))
518 uint32_t FixedInstr = RawInstr | EncodedImm;
519 *(ulittle32_t *)FixupPtr = FixedInstr;
523 assert((FixupAddress.
getValue() & 0x3) == 0 &&
"ADR is not 32-bit aligned");
524 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
525 assert(
isADR(RawInstr) &&
"RawInstr is not an ADR");
526 int64_t Delta =
E.getTarget().getAddress() +
E.getAddend() - FixupAddress;
527 if (!isInt<21>(Delta))
529 auto UDelta =
static_cast<uint32_t>(Delta);
530 uint32_t EncodedImmHi = ((UDelta >> 2) & 0x7ffff) << 5;
531 uint32_t EncodedImmLo = (UDelta & 0x3) << 29;
532 uint32_t FixedInstr = RawInstr | EncodedImmHi | EncodedImmLo;
533 *(ulittle32_t *)FixupPtr = FixedInstr;
538 "Test and branch is not 32-bit aligned");
539 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
541 "RawInstr is not a test and branch");
542 int64_t Delta =
E.getTarget().getAddress() +
E.getAddend() - FixupAddress;
544 return make_error<JITLinkError>(
545 "Test and branch literal target is not 32-bit aligned");
546 if (!isInt<16>(Delta))
549 uint32_t FixedInstr = RawInstr | EncodedImm;
550 *(ulittle32_t *)FixupPtr = FixedInstr;
555 "Conditional branch is not 32-bit aligned");
556 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
558 "RawInstr is not a conditional branch");
559 int64_t Delta =
E.getTarget().getAddress() +
E.getAddend() - FixupAddress;
561 return make_error<JITLinkError>(
562 "Conditional branch literal target is not 32-bit "
564 if (!isInt<21>(Delta))
567 uint32_t FixedInstr = RawInstr | EncodedImm;
568 *(ulittle32_t *)FixupPtr = FixedInstr;
573 (
E.getTarget().getAddress().getValue() +
E.getAddend()) &
576 FixupAddress.
getValue() & ~static_cast<uint64_t>(4096 - 1);
578 int64_t PageDelta = TargetPage - PCPage;
579 if (!isInt<33>(PageDelta))
582 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
583 assert((RawInstr & 0xffffffe0) == 0x90000000 &&
584 "RawInstr isn't an ADRP instruction");
587 uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5);
588 *(ulittle32_t *)FixupPtr = FixedInstr;
593 (
E.getTarget().getAddress() +
E.getAddend()).getValue() & 0xfff;
595 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
598 if (TargetOffset & ((1 << ImmShift) - 1))
599 return make_error<JITLinkError>(
"PAGEOFF12 target is not aligned");
601 uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;
602 uint32_t FixedInstr = RawInstr | EncodedImm;
603 *(ulittle32_t *)FixupPtr = FixedInstr;
607 return make_error<JITLinkError>(
608 "In graph " +
G.getName() +
", section " +
B.getSection().getName() +
641 Symbol *InitialTarget =
nullptr,
646 B.addEdge(
Pointer64, 0, *InitialTarget, InitialAddend);
647 return G.addAnonymousSymbol(
B, 0, 8,
false,
false);
660 B.addEdge(
Page21, 0, PointerSymbol, 0);
672 return G.addAnonymousSymbol(
684 const char *BlockWorkingMem =
B->getContent().data();
685 const char *FixupPtr = BlockWorkingMem +
E.getOffset();
687 switch (
E.getKind()) {
699 "GOTPageOffset12/TLVPageOffset12 with non-zero addend");
700 assert((RawInstr & 0xfffffc00) == 0xf9400000 &&
701 "RawInstr isn't a 64-bit LDR immediate");
712 "Fell through switch, but no new kind to set");
714 dbgs() <<
" Fixing " <<
G.getEdgeKindName(
E.getKind()) <<
" edge at "
715 <<
B->getFixupAddress(
E) <<
" (" <<
B->getAddress() <<
" + "
716 <<
formatv(
"{0:x}",
E.getOffset()) <<
")\n";
718 E.setKind(KindToSet);
748 dbgs() <<
" Fixing " <<
G.getEdgeKindName(
E.getKind()) <<
" edge at "
749 <<
B->getFixupAddress(
E) <<
" (" <<
B->getAddress() <<
" + "
750 <<
formatv(
"{0:x}",
E.getOffset()) <<
")\n";
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define DEBUG_WITH_TYPE(TYPE, X)
DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
StringRef - Represent a constant reference to a string, i.e.
Target - Wrapper for Target specific information.
LLVM Value Representation.
An Addressable with content and edges.
Represents fixups and constraints in the LinkGraph.
Represents an object file section.
A CRTP base for tables that are built on demand, e.g.
Symbol & getEntryForTarget(LinkGraph &G, Symbol &Target)
Return the constructed entry.
Global Offset Table Builder.
Symbol & createEntry(LinkGraph &G, Symbol &Target)
bool visitEdge(LinkGraph &G, Block *B, Edge &E)
static StringRef getSectionName()
Procedure Linkage Table Builder.
static StringRef getSectionName()
bool visitEdge(LinkGraph &G, Block *B, Edge &E)
PLTTableManager(GOTTableManager &GOT)
Symbol & createEntry(LinkGraph &G, Symbol &Target)
Section & getStubsSection(LinkGraph &G)
Represents an address in the executor process.
uint64_t getValue() const
const char NullPointerContent[PointerSize]
AArch64 null pointer content.
bool isLDRLiteral(uint32_t Instr)
bool isADR(uint32_t Instr)
bool isCondBranchImm19(uint32_t Instr)
constexpr uint64_t PointerSize
aarch64 pointer size.
bool isMoveWideImm16(uint32_t Instr)
bool isCompAndBranchImm19(uint32_t Instr)
Error applyFixup(LinkGraph &G, Block &B, const Edge &E)
Apply fixup expression for edge to block content.
const char PointerJumpStubContent[12]
bool isLoadStoreImm12(uint32_t Instr)
const char * getEdgeKindName(Edge::Kind K)
Returns a string name for the given aarch64 edge.
Block & createPointerJumpStubBlock(LinkGraph &G, Section &StubSection, Symbol &PointerSymbol)
Create a jump stub block that jumps via the pointer at the given symbol.
Symbol & createAnonymousPointer(LinkGraph &G, Section &PointerSection, Symbol *InitialTarget=nullptr, uint64_t InitialAddend=0)
Creates a new pointer block in the given section and returns an Anonymous symbol pointing to it.
EdgeKind_aarch64
Represents aarch64 fixups and other aarch64-specific edge kinds.
@ LDRLiteral19
The signed 21-bit delta from the fixup to the target.
@ CondBranch19PCRel
A 19-bit PC-relative conditional branch.
@ RequestTLVPAndTransformToPageOffset12
A TLVP entry getter/constructor, transformed to PageOffset12.
@ RequestTLSDescEntryAndTransformToPageOffset12
A TLSDesc entry getter/constructor, transformed to PageOffset12.
@ Page21
The signed 21-bit delta from the fixup page to the page containing the target.
@ Branch26PCRel
A 26-bit PC-relative branch.
@ Pointer64
A plain 64-bit pointer value relocation.
@ Pointer32
A plain 32-bit pointer value relocation.
@ RequestTLVPAndTransformToPage21
A TLVP entry getter/constructor, transformed to Page21.
@ MoveWide16
A 16-bit slice of the target address (which slice depends on the instruction at the fixup location).
@ TestAndBranch14PCRel
A 14-bit PC-relative test and branch.
@ RequestGOTAndTransformToPage21
A GOT entry getter/constructor, transformed to Page21 pointing at the GOT entry for the original targ...
@ ADRLiteral21
The signed 21-bit delta from the fixup to the target.
@ RequestGOTAndTransformToPageOffset12
A GOT entry getter/constructor, transformed to Pageoffset12 pointing at the GOT entry for the origina...
@ NegDelta32
A 32-bit negative delta.
@ NegDelta64
A 64-bit negative delta.
@ RequestGOTAndTransformToDelta32
A GOT entry getter/constructor, transformed to Delta32 pointing at the GOT entry for the original tar...
@ PageOffset12
The 12-bit (potentially shifted) offset of the target within its page.
@ RequestTLSDescEntryAndTransformToPage21
A TLSDesc entry getter/constructor, transformed to Page21.
bool isTestAndBranchImm14(uint32_t Instr)
unsigned getPageOffset12Shift(uint32_t Instr)
unsigned getMoveWide16Shift(uint32_t Instr)
Symbol & createAnonymousPointerJumpStub(LinkGraph &G, Section &StubSection, Symbol &PointerSymbol)
Create a jump stub that jumps via the pointer at the given symbol and an anonymous symbol pointing to...
Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B, const Edge &E)
Create an out of range error for the given edge in the given block.
This is an optimization pass for GlobalISel generic memory operations.
auto formatv(const char *Fmt, Ts &&...Vals) -> formatv_object< decltype(std::make_tuple(support::detail::build_format_adapter(std::forward< Ts >(Vals))...))>
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.