LLVM 20.0.0git
aarch64.cpp
Go to the documentation of this file.
1//===---- aarch64.cpp - Generic JITLink aarch64 edge kinds, utilities -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Generic utilities for graphs representing aarch64 objects.
10//
11//===----------------------------------------------------------------------===//
12
14
16
17#define DEBUG_TYPE "jitlink"
18
19namespace llvm {
20namespace jitlink {
21namespace aarch64 {
22
23const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00,
24 0x00, 0x00, 0x00, 0x00};
25
26const char PointerJumpStubContent[12] = {
27 0x10, 0x00, 0x00, (char)0x90u, // ADRP x16, <imm>@page21
28 0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12]
29 0x00, 0x02, 0x1f, (char)0xd6u // BR x16
30};
31
32const char ReentryTrampolineContent[8] = {
33 (char)0xfd, 0x7b, (char)0xbf, (char)0xa9, // STP x30, [sp, #-8]
34 0x00, 0x00, 0x00, (char)0x94 // BL
35};
36
38 switch (R) {
39 case Pointer64:
40 return "Pointer64";
42 return "Pointer64Authenticated";
43 case Pointer32:
44 return "Pointer32";
45 case Delta64:
46 return "Delta64";
47 case Delta32:
48 return "Delta32";
49 case NegDelta64:
50 return "NegDelta64";
51 case NegDelta32:
52 return "NegDelta32";
53 case Branch26PCRel:
54 return "Branch26PCRel";
55 case MoveWide16:
56 return "MoveWide16";
57 case LDRLiteral19:
58 return "LDRLiteral19";
60 return "TestAndBranch14PCRel";
62 return "CondBranch19PCRel";
63 case ADRLiteral21:
64 return "ADRLiteral21";
65 case Page21:
66 return "Page21";
67 case PageOffset12:
68 return "PageOffset12";
69 case GotPageOffset15:
70 return "GotPageOffset15";
72 return "RequestGOTAndTransformToPage21";
74 return "RequestGOTAndTransformToPageOffset12";
76 return "RequestGOTAndTransformToPageOffset15";
78 return "RequestGOTAndTransformToDelta32";
80 return "RequestTLVPAndTransformToPage21";
82 return "RequestTLVPAndTransformToPageOffset12";
84 return "RequestTLSDescEntryAndTransformToPage21";
86 return "RequestTLSDescEntryAndTransformToPageOffset12";
87 default:
88 return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
89 }
90}
91
92// Write a 64-bit GPR -> GPR move.
93template <typename AppendFtor>
94static Error writeMovRegRegSeq(AppendFtor &Append, uint64_t DstReg,
95 uint64_t SrcReg) {
96 assert(DstReg < 32 && "Dst reg out of range");
97 assert(SrcReg < 32 && "Src reg out of range");
98
99 if (DstReg == SrcReg)
100 return Error::success();
101
102 constexpr uint32_t MOVGPR64Template = 0xaa0003e0;
103 constexpr uint32_t DstRegIndex = 0;
104 constexpr uint32_t SrcRegIndex = 16;
105 uint32_t Instr = MOVGPR64Template;
106 Instr |= DstReg << DstRegIndex;
107 Instr |= SrcReg << SrcRegIndex;
108 return Append(Instr);
109}
110
111// Generate a sequence of imm writes to assign the given value.
112template <typename AppendFtor>
113static Error writeMovRegImm64Seq(AppendFtor &Append, uint64_t Reg,
114 uint64_t Imm) {
115 assert(Reg < 32 && "Invalid register number");
116
117 constexpr uint32_t MovRegImm64Template = 0xd2800000;
118 constexpr unsigned PreserveBitIndex = 29;
119 constexpr unsigned ShiftBitsIndex = 21;
120 constexpr unsigned ImmBitsIndex = 5;
121
122 bool PreserveRegValue = false;
123 for (unsigned I = 0; I != 4; ++I) {
124 uint32_t ImmBits = Imm & 0xffff;
125 Imm >>= 16;
126
127 // Skip any all-zero immediates after the first one.
128 if (PreserveRegValue && !ImmBits)
129 continue;
130
131 uint32_t Instr = MovRegImm64Template;
132 Instr |= PreserveRegValue << PreserveBitIndex;
133 Instr |= (I << ShiftBitsIndex);
134 Instr |= ImmBits << ImmBitsIndex;
135 Instr |= Reg;
136 if (auto Err = Append(Instr))
137 return Err;
138 PreserveRegValue = true;
139 }
140
141 return Error::success();
142}
143
144template <typename AppendFtor>
145static Error
146writePACSignSeq(AppendFtor &Append, unsigned DstReg, orc::ExecutorAddr RawAddr,
147 unsigned RawAddrReg, unsigned DiscriminatorReg, unsigned Key,
148 uint64_t EncodedDiscriminator, bool AddressDiversify) {
149 assert(DstReg < 32 && "DstReg out of range");
150 assert(RawAddrReg < 32 && "AddrReg out of range");
151 assert(DiscriminatorReg < 32 && "DiscriminatorReg out of range");
152 assert(EncodedDiscriminator < 0x10000 && "EncodedDiscriminator out of range");
153
154 if (AddressDiversify) {
155 // Move the address into the discriminator register.
156 if (auto Err = writeMovRegRegSeq(Append, DiscriminatorReg, RawAddrReg))
157 return Err;
158 // Blend encoded discriminator if there is one.
159 if (EncodedDiscriminator) {
160 constexpr uint32_t MOVKTemplate = 0xf2e00000;
161 constexpr unsigned ImmIndex = 5;
162 uint32_t BlendInstr = MOVKTemplate;
163 BlendInstr |= EncodedDiscriminator << ImmIndex;
164 BlendInstr |= DiscriminatorReg;
165 if (auto Err = Append(BlendInstr))
166 return Err;
167 }
168 } else if (EncodedDiscriminator) {
169 // Move the encoded discriminator into the discriminator register.
170 if (auto Err =
171 writeMovRegImm64Seq(Append, DiscriminatorReg, EncodedDiscriminator))
172 return Err;
173 } else
174 DiscriminatorReg = 31; // WZR
175
176 constexpr uint32_t PACTemplate = 0xdac10000;
177 constexpr unsigned ZBitIndex = 13;
178 constexpr unsigned KeyIndex = 10;
179 constexpr unsigned DiscriminatorRegIndex = 5;
180
181 uint32_t Instr = PACTemplate;
182 Instr |= (DiscriminatorReg == 31) << ZBitIndex;
183 Instr |= Key << KeyIndex;
184 Instr |= DiscriminatorReg << DiscriminatorRegIndex;
185 Instr |= DstReg;
186
187 return Append(Instr);
188}
189
190template <typename AppendFtor>
191static Error writeStoreRegSeq(AppendFtor &Append, unsigned DstLocReg,
192 unsigned SrcReg) {
193 assert(DstLocReg < 32 && "DstLocReg out of range");
194 assert(SrcReg < 32 && "SrcReg out of range");
195
196 constexpr uint32_t STRTemplate = 0xf9000000;
197 constexpr unsigned DstLocRegIndex = 5;
198 constexpr unsigned SrcRegIndex = 0;
199
200 uint32_t Instr = STRTemplate;
201 Instr |= DstLocReg << DstLocRegIndex;
202 Instr |= SrcReg << SrcRegIndex;
203
204 return Append(Instr);
205}
206
207void GOTTableManager::registerExistingEntries() {
208 for (auto *EntrySym : GOTSection->symbols()) {
209 assert(EntrySym->getBlock().edges_size() == 1 &&
210 "GOT block edge count != 1");
211 registerPreExistingEntry(EntrySym->getBlock().edges().begin()->getTarget(),
212 *EntrySym);
213 }
214}
215
217 for (auto *EntrySym : StubsSection->symbols()) {
218 assert(EntrySym->getBlock().edges_size() == 2 &&
219 "PLT block edge count != 2");
220 auto &GOTSym = EntrySym->getBlock().edges().begin()->getTarget();
221 assert(GOTSym.getBlock().edges_size() == 1 && "GOT block edge count != 1");
222 registerPreExistingEntry(GOTSym.getBlock().edges().begin()->getTarget(),
223 *EntrySym);
224 }
225}
226
227const char *getPointerSigningFunctionSectionName() { return "$__ptrauth_sign"; }
228
229/// Creates a pointer signing function section, block, and symbol to reserve
230/// space for a signing function for this LinkGraph. Clients should insert this
231/// pass in the post-prune phase, and add the paired
232/// lowerPointer64AuthEdgesToSigningFunction pass to the pre-fixup phase.
234 LLVM_DEBUG({
235 dbgs() << "Creating empty pointer signing function for " << G.getName()
236 << "\n";
237 });
238
239 // FIXME: We could put a tighter bound on this if we inspected the ptrauth
240 // info encoded in the addend -- the only actually unknown quantity is the
241 // fixup location, and we can probably put constraints even on that.
242 size_t NumPtrAuthFixupLocations = 0;
243 for (auto *B : G.blocks())
244 for (auto &E : B->edges())
245 NumPtrAuthFixupLocations +=
247
248 constexpr size_t MaxPtrSignSeqLength =
249 4 + // To materialize the value to sign.
250 4 + // To materialize the fixup location.
251 3 + // To copy, blend discriminator, and sign
252 1; // To store the result.
253
254 // The maximum number of signing instructions required is the maximum per
255 // location, times the number of locations, plus three instructions to
256 // materialize the return value and return.
257 size_t NumSigningInstrs = NumPtrAuthFixupLocations * MaxPtrSignSeqLength + 3;
258
259 // Create signing function section.
260 auto &SigningSection =
263 SigningSection.setMemLifetime(orc::MemLifetime::Finalize);
264
265 size_t SigningFunctionSize = NumSigningInstrs * 4;
266 auto &SigningFunctionBlock = G.createMutableContentBlock(
267 SigningSection, G.allocateBuffer(SigningFunctionSize),
268 orc::ExecutorAddr(), 4, 0);
269 G.addAnonymousSymbol(SigningFunctionBlock, 0, SigningFunctionBlock.getSize(),
270 true, true);
271
272 LLVM_DEBUG({
273 dbgs() << " " << NumPtrAuthFixupLocations << " location(s) to sign, up to "
274 << NumSigningInstrs << " instructions required ("
275 << formatv("{0:x}", SigningFunctionBlock.getSize()) << " bytes)\n";
276 });
277
278 return Error::success();
279}
280
281/// Given a LinkGraph containing Pointer64Auth edges, transform those edges to
282/// Pointer64 and add code to sign the pointers in the executor.
283///
284/// This function will add a $__ptrauth_sign section with finalization-lifetime
285/// containing an anonymous function that will sign all pointers in the graph.
286/// An allocation action will be added to run this function during finalization.
288 LLVM_DEBUG({
289 dbgs() << "Writing pointer signing function for " << G.getName() << "\n";
290 });
291
292 constexpr unsigned Reg1 = 8; // Holds pointer value to sign.
293 constexpr unsigned Reg2 = 9; // Holds fixup address.
294 constexpr unsigned Reg3 = 10; // Temporary for discriminator value if needed.
295
296 // Find the signing function.
297 auto *SigningSection =
298 G.findSectionByName(getPointerSigningFunctionSectionName());
299 assert(SigningSection && "Siging section missing");
300 assert(SigningSection->blocks_size() == 1 &&
301 "Unexpected number of blocks in signing section");
302 assert(SigningSection->symbols_size() == 1 &&
303 "Unexpected number of symbols in signing section");
304
305 auto &SigningFunctionSym = **SigningSection->symbols().begin();
306 auto &SigningFunctionBlock = SigningFunctionSym.getBlock();
307 auto SigningFunctionBuf = SigningFunctionBlock.getAlreadyMutableContent();
308
309 // Write the instructions to the block content.
310 BinaryStreamWriter InstrWriter(
311 {reinterpret_cast<uint8_t *>(SigningFunctionBuf.data()),
312 SigningFunctionBuf.size()},
313 G.getEndianness());
314
315 auto AppendInstr = [&](uint32_t Instr) {
316 return InstrWriter.writeInteger(Instr);
317 };
318
319 for (auto *B : G.blocks()) {
320 for (auto &E : B->edges()) {
321 // We're only concerned with Pointer64Authenticated edges here.
322 if (E.getKind() != aarch64::Pointer64Authenticated)
323 continue;
324
325 uint64_t EncodedInfo = E.getAddend();
326 int32_t RealAddend = (uint32_t)(EncodedInfo & 0xffffffff);
327 uint32_t InitialDiscriminator = (EncodedInfo >> 32) & 0xffff;
328 bool AddressDiversify = (EncodedInfo >> 48) & 0x1;
329 uint32_t Key = (EncodedInfo >> 49) & 0x3;
330 uint32_t HighBits = EncodedInfo >> 51;
331 auto ValueToSign = E.getTarget().getAddress() + RealAddend;
332
333 if (HighBits != 0x1000)
334 return make_error<JITLinkError>(
335 "Pointer64Auth edge at " +
336 formatv("{0:x}", B->getFixupAddress(E).getValue()) +
337 " has invalid encoded addend " + formatv("{0:x}", EncodedInfo));
338
339 LLVM_DEBUG({
340 const char *const KeyNames[] = {"IA", "IB", "DA", "DB"};
341 dbgs() << " " << B->getFixupAddress(E) << " <- " << ValueToSign
342 << " : key = " << KeyNames[Key] << ", discriminator = "
343 << formatv("{0:x4}", InitialDiscriminator)
344 << ", address diversified = "
345 << (AddressDiversify ? "yes" : "no") << "\n";
346 });
347
348 // Materialize pointer value.
349 cantFail(writeMovRegImm64Seq(AppendInstr, Reg1, ValueToSign.getValue()));
350
351 // Materialize fixup pointer.
352 cantFail(writeMovRegImm64Seq(AppendInstr, Reg2,
353 B->getFixupAddress(E).getValue()));
354
355 // Write signing instruction(s).
356 cantFail(writePACSignSeq(AppendInstr, Reg1, ValueToSign, Reg2, Reg3, Key,
357 InitialDiscriminator, AddressDiversify));
358
359 // Store signed pointer.
360 cantFail(writeStoreRegSeq(AppendInstr, Reg2, Reg1));
361
362 // Replace edge with a keep-alive to preserve dependence info.
363 E.setKind(Edge::KeepAlive);
364 }
365 }
366
367 // Write epilogue. x0 = 0, x1 = 1 is an SPS serialized Error::success value.
368 constexpr uint32_t RETInstr = 0xd65f03c0;
369 cantFail(writeMovRegImm64Seq(AppendInstr, 0, 0)); // mov x0, #0
370 cantFail(writeMovRegImm64Seq(AppendInstr, 1, 1)); // mov x1, #1
371 cantFail(AppendInstr(RETInstr)); // ret
372
373 // Add an allocation action to call the signing function.
374 using namespace orc::shared;
375 G.allocActions().push_back(
376 {cantFail(WrapperFunctionCall::Create<SPSArgList<>>(
377 SigningFunctionSym.getAddress())),
378 {}});
379
380 return Error::success();
381}
382
383} // namespace aarch64
384} // namespace jitlink
385} // namespace llvm
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_DEBUG(...)
Definition: Debug.h:106
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
unsigned Reg
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Provides write only access to a subclass of WritableBinaryStream.
Error writeInteger(T Value)
Write the integer Value to the underlying stream in the specified endianness.
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
static ErrorSuccess success()
Create a success value.
Definition: Error.h:337
Represents an address in the executor process.
@ Finalize
Finalize memory should be allocated by the allocator, and then be overwritten and deallocated after a...
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void cantFail(Error Err, const char *Msg=nullptr)
Report a fatal error if Err is a failure value.
Definition: Error.h:756