LLVM 19.0.0git
BPFAbstractMemberAccess.cpp
Go to the documentation of this file.
1//===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass abstracted struct/union member accesses in order to support
10// compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program
11// which can run on different kernels. In particular, if bpf program tries to
12// access a particular kernel data structure member, the details of the
13// intermediate member access will be remembered so bpf loader can do
14// necessary adjustment right before program loading.
15//
16// For example,
17//
18// struct s {
19// int a;
20// int b;
21// };
22// struct t {
23// struct s c;
24// int d;
25// };
26// struct t e;
27//
28// For the member access e.c.b, the compiler will generate code
29// &e + 4
30//
31// The compile-once run-everywhere instead generates the following code
32// r = 4
33// &e + r
34// The "4" in "r = 4" can be changed based on a particular kernel version.
35// For example, on a particular kernel version, if struct s is changed to
36//
37// struct s {
38// int new_field;
39// int a;
40// int b;
41// }
42//
43// By repeating the member access on the host, the bpf loader can
44// adjust "r = 4" as "r = 8".
45//
46// This feature relies on the following three intrinsic calls:
47// addr = preserve_array_access_index(base, dimension, index)
48// addr = preserve_union_access_index(base, di_index)
49// !llvm.preserve.access.index <union_ditype>
50// addr = preserve_struct_access_index(base, gep_index, di_index)
51// !llvm.preserve.access.index <struct_ditype>
52//
53// Bitfield member access needs special attention. User cannot take the
54// address of a bitfield acceess. To facilitate kernel verifier
55// for easy bitfield code optimization, a new clang intrinsic is introduced:
56// uint32_t __builtin_preserve_field_info(member_access, info_kind)
57// In IR, a chain with two (or more) intrinsic calls will be generated:
58// ...
59// addr = preserve_struct_access_index(base, 1, 1) !struct s
60// uint32_t result = bpf_preserve_field_info(addr, info_kind)
61//
62// Suppose the info_kind is FIELD_SIGNEDNESS,
63// The above two IR intrinsics will be replaced with
64// a relocatable insn:
65// signness = /* signness of member_access */
66// and signness can be changed by bpf loader based on the
67// types on the host.
68//
69// User can also test whether a field exists or not with
70// uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE)
71// The field will be always available (result = 1) during initial
72// compilation, but bpf loader can patch with the correct value
73// on the target host where the member_access may or may not be available
74//
75//===----------------------------------------------------------------------===//
76
77#include "BPF.h"
78#include "BPFCORE.h"
79#include "BPFTargetMachine.h"
84#include "llvm/IR/Instruction.h"
86#include "llvm/IR/IntrinsicsBPF.h"
87#include "llvm/IR/Module.h"
88#include "llvm/IR/PassManager.h"
89#include "llvm/IR/Type.h"
90#include "llvm/IR/User.h"
91#include "llvm/IR/Value.h"
92#include "llvm/IR/ValueHandle.h"
93#include "llvm/Pass.h"
95#include <stack>
96
97#define DEBUG_TYPE "bpf-abstract-member-access"
98
99namespace llvm {
102
104 Instruction *Input,
107 M, Intrinsic::bpf_passthrough, {Input->getType(), Input->getType()});
108 Constant *SeqNumVal = ConstantInt::get(Type::getInt32Ty(BB->getContext()),
110
111 auto *NewInst = CallInst::Create(Fn, {SeqNumVal, Input});
112 NewInst->insertBefore(Before);
113 return NewInst;
114}
115} // namespace llvm
116
117using namespace llvm;
118
119namespace {
120class BPFAbstractMemberAccess final {
121public:
122 BPFAbstractMemberAccess(BPFTargetMachine *TM) : TM(TM) {}
123
124 bool run(Function &F);
125
126 struct CallInfo {
128 uint32_t AccessIndex;
129 MaybeAlign RecordAlignment;
132 };
133 typedef std::stack<std::pair<CallInst *, CallInfo>> CallInfoStack;
134
135private:
136 enum : uint32_t {
137 BPFPreserveArrayAI = 1,
138 BPFPreserveUnionAI = 2,
139 BPFPreserveStructAI = 3,
140 BPFPreserveFieldInfoAI = 4,
141 };
142
144 const DataLayout *DL = nullptr;
145 Module *M = nullptr;
146
147 static std::map<std::string, GlobalVariable *> GEPGlobals;
148 // A map to link preserve_*_access_index intrinsic calls.
149 std::map<CallInst *, std::pair<CallInst *, CallInfo>> AIChain;
150 // A map to hold all the base preserve_*_access_index intrinsic calls.
151 // The base call is not an input of any other preserve_*
152 // intrinsics.
153 std::map<CallInst *, CallInfo> BaseAICalls;
154 // A map to hold <AnonRecord, TypeDef> relationships
155 std::map<DICompositeType *, DIDerivedType *> AnonRecords;
156
157 void CheckAnonRecordType(DIDerivedType *ParentTy, DIType *Ty);
158 void CheckCompositeType(DIDerivedType *ParentTy, DICompositeType *CTy);
159 void CheckDerivedType(DIDerivedType *ParentTy, DIDerivedType *DTy);
160 void ResetMetadata(struct CallInfo &CInfo);
161
162 bool doTransformation(Function &F);
163
164 void traceAICall(CallInst *Call, CallInfo &ParentInfo);
165 void traceBitCast(BitCastInst *BitCast, CallInst *Parent,
166 CallInfo &ParentInfo);
167 void traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
168 CallInfo &ParentInfo);
169 void collectAICallChains(Function &F);
170
171 bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo);
172 bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI,
173 const MDNode *ChildMeta);
174 bool removePreserveAccessIndexIntrinsic(Function &F);
175 bool HasPreserveFieldInfoCall(CallInfoStack &CallStack);
176 void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment,
177 uint32_t &StartBitOffset, uint32_t &EndBitOffset);
178 uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy,
179 uint32_t AccessIndex, uint32_t PatchImm,
180 MaybeAlign RecordAlignment);
181
182 Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo,
183 std::string &AccessKey, MDNode *&BaseMeta);
184 MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo,
185 std::string &AccessKey, bool &IsInt32Ret);
186 bool transformGEPChain(CallInst *Call, CallInfo &CInfo);
187};
188
189std::map<std::string, GlobalVariable *> BPFAbstractMemberAccess::GEPGlobals;
190} // End anonymous namespace
191
192bool BPFAbstractMemberAccess::run(Function &F) {
193 LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n");
194
195 M = F.getParent();
196 if (!M)
197 return false;
198
199 // Bail out if no debug info.
200 if (M->debug_compile_units().empty())
201 return false;
202
203 // For each argument/return/local_variable type, trace the type
204 // pattern like '[derived_type]* [composite_type]' to check
205 // and remember (anon record -> typedef) relations where the
206 // anon record is defined as
207 // typedef [const/volatile/restrict]* [anon record]
208 DISubprogram *SP = F.getSubprogram();
209 if (SP && SP->isDefinition()) {
210 for (DIType *Ty: SP->getType()->getTypeArray())
211 CheckAnonRecordType(nullptr, Ty);
212 for (const DINode *DN : SP->getRetainedNodes()) {
213 if (const auto *DV = dyn_cast<DILocalVariable>(DN))
214 CheckAnonRecordType(nullptr, DV->getType());
215 }
216 }
217
218 DL = &M->getDataLayout();
219 return doTransformation(F);
220}
221
222void BPFAbstractMemberAccess::ResetMetadata(struct CallInfo &CInfo) {
223 if (auto Ty = dyn_cast<DICompositeType>(CInfo.Metadata)) {
224 if (AnonRecords.find(Ty) != AnonRecords.end()) {
225 if (AnonRecords[Ty] != nullptr)
226 CInfo.Metadata = AnonRecords[Ty];
227 }
228 }
229}
230
231void BPFAbstractMemberAccess::CheckCompositeType(DIDerivedType *ParentTy,
232 DICompositeType *CTy) {
233 if (!CTy->getName().empty() || !ParentTy ||
234 ParentTy->getTag() != dwarf::DW_TAG_typedef)
235 return;
236
237 if (AnonRecords.find(CTy) == AnonRecords.end()) {
238 AnonRecords[CTy] = ParentTy;
239 return;
240 }
241
242 // Two or more typedef's may point to the same anon record.
243 // If this is the case, set the typedef DIType to be nullptr
244 // to indicate the duplication case.
245 DIDerivedType *CurrTy = AnonRecords[CTy];
246 if (CurrTy == ParentTy)
247 return;
248 AnonRecords[CTy] = nullptr;
249}
250
251void BPFAbstractMemberAccess::CheckDerivedType(DIDerivedType *ParentTy,
252 DIDerivedType *DTy) {
253 DIType *BaseType = DTy->getBaseType();
254 if (!BaseType)
255 return;
256
257 unsigned Tag = DTy->getTag();
258 if (Tag == dwarf::DW_TAG_pointer_type)
259 CheckAnonRecordType(nullptr, BaseType);
260 else if (Tag == dwarf::DW_TAG_typedef)
261 CheckAnonRecordType(DTy, BaseType);
262 else
263 CheckAnonRecordType(ParentTy, BaseType);
264}
265
266void BPFAbstractMemberAccess::CheckAnonRecordType(DIDerivedType *ParentTy,
267 DIType *Ty) {
268 if (!Ty)
269 return;
270
271 if (auto *CTy = dyn_cast<DICompositeType>(Ty))
272 return CheckCompositeType(ParentTy, CTy);
273 else if (auto *DTy = dyn_cast<DIDerivedType>(Ty))
274 return CheckDerivedType(ParentTy, DTy);
275}
276
277static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) {
278 if (Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type &&
279 Tag != dwarf::DW_TAG_volatile_type &&
280 Tag != dwarf::DW_TAG_restrict_type &&
281 Tag != dwarf::DW_TAG_member)
282 return false;
283 if (Tag == dwarf::DW_TAG_typedef && !skipTypedef)
284 return false;
285 return true;
286}
287
288static DIType * stripQualifiers(DIType *Ty, bool skipTypedef = true) {
289 while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) {
290 if (!SkipDIDerivedTag(DTy->getTag(), skipTypedef))
291 break;
292 Ty = DTy->getBaseType();
293 }
294 return Ty;
295}
296
297static const DIType * stripQualifiers(const DIType *Ty) {
298 while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) {
299 if (!SkipDIDerivedTag(DTy->getTag(), true))
300 break;
301 Ty = DTy->getBaseType();
302 }
303 return Ty;
304}
305
306static uint32_t calcArraySize(const DICompositeType *CTy, uint32_t StartDim) {
307 DINodeArray Elements = CTy->getElements();
308 uint32_t DimSize = 1;
309 for (uint32_t I = StartDim; I < Elements.size(); ++I) {
310 if (auto *Element = dyn_cast_or_null<DINode>(Elements[I]))
311 if (Element->getTag() == dwarf::DW_TAG_subrange_type) {
312 const DISubrange *SR = cast<DISubrange>(Element);
313 auto *CI = SR->getCount().dyn_cast<ConstantInt *>();
314 DimSize *= CI->getSExtValue();
315 }
316 }
317
318 return DimSize;
319}
320
321static Type *getBaseElementType(const CallInst *Call) {
322 // Element type is stored in an elementtype() attribute on the first param.
323 return Call->getParamElementType(0);
324}
325
326static uint64_t getConstant(const Value *IndexValue) {
327 const ConstantInt *CV = dyn_cast<ConstantInt>(IndexValue);
328 assert(CV);
329 return CV->getValue().getZExtValue();
330}
331
332/// Check whether a call is a preserve_*_access_index intrinsic call or not.
333bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call,
334 CallInfo &CInfo) {
335 if (!Call)
336 return false;
337
338 const auto *GV = dyn_cast<GlobalValue>(Call->getCalledOperand());
339 if (!GV)
340 return false;
341 if (GV->getName().starts_with("llvm.preserve.array.access.index")) {
342 CInfo.Kind = BPFPreserveArrayAI;
343 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);
344 if (!CInfo.Metadata)
345 report_fatal_error("Missing metadata for llvm.preserve.array.access.index intrinsic");
346 CInfo.AccessIndex = getConstant(Call->getArgOperand(2));
347 CInfo.Base = Call->getArgOperand(0);
348 CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call));
349 return true;
350 }
351 if (GV->getName().starts_with("llvm.preserve.union.access.index")) {
352 CInfo.Kind = BPFPreserveUnionAI;
353 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);
354 if (!CInfo.Metadata)
355 report_fatal_error("Missing metadata for llvm.preserve.union.access.index intrinsic");
356 ResetMetadata(CInfo);
357 CInfo.AccessIndex = getConstant(Call->getArgOperand(1));
358 CInfo.Base = Call->getArgOperand(0);
359 return true;
360 }
361 if (GV->getName().starts_with("llvm.preserve.struct.access.index")) {
362 CInfo.Kind = BPFPreserveStructAI;
363 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);
364 if (!CInfo.Metadata)
365 report_fatal_error("Missing metadata for llvm.preserve.struct.access.index intrinsic");
366 ResetMetadata(CInfo);
367 CInfo.AccessIndex = getConstant(Call->getArgOperand(2));
368 CInfo.Base = Call->getArgOperand(0);
369 CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call));
370 return true;
371 }
372 if (GV->getName().starts_with("llvm.bpf.preserve.field.info")) {
373 CInfo.Kind = BPFPreserveFieldInfoAI;
374 CInfo.Metadata = nullptr;
375 // Check validity of info_kind as clang did not check this.
376 uint64_t InfoKind = getConstant(Call->getArgOperand(1));
377 if (InfoKind >= BTF::MAX_FIELD_RELOC_KIND)
378 report_fatal_error("Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic");
379 CInfo.AccessIndex = InfoKind;
380 return true;
381 }
382 if (GV->getName().starts_with("llvm.bpf.preserve.type.info")) {
383 CInfo.Kind = BPFPreserveFieldInfoAI;
384 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);
385 if (!CInfo.Metadata)
386 report_fatal_error("Missing metadata for llvm.preserve.type.info intrinsic");
387 uint64_t Flag = getConstant(Call->getArgOperand(1));
389 report_fatal_error("Incorrect flag for llvm.bpf.preserve.type.info intrinsic");
391 CInfo.AccessIndex = BTF::TYPE_EXISTENCE;
393 CInfo.AccessIndex = BTF::TYPE_MATCH;
394 else
395 CInfo.AccessIndex = BTF::TYPE_SIZE;
396 return true;
397 }
398 if (GV->getName().starts_with("llvm.bpf.preserve.enum.value")) {
399 CInfo.Kind = BPFPreserveFieldInfoAI;
400 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index);
401 if (!CInfo.Metadata)
402 report_fatal_error("Missing metadata for llvm.preserve.enum.value intrinsic");
403 uint64_t Flag = getConstant(Call->getArgOperand(2));
405 report_fatal_error("Incorrect flag for llvm.bpf.preserve.enum.value intrinsic");
407 CInfo.AccessIndex = BTF::ENUM_VALUE_EXISTENCE;
408 else
409 CInfo.AccessIndex = BTF::ENUM_VALUE;
410 return true;
411 }
412
413 return false;
414}
415
416static void replaceWithGEP(CallInst *Call, uint32_t DimensionIndex,
417 uint32_t GEPIndex) {
418 uint32_t Dimension = 1;
419 if (DimensionIndex > 0)
420 Dimension = getConstant(Call->getArgOperand(DimensionIndex));
421
422 Constant *Zero =
423 ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0);
425 for (unsigned I = 0; I < Dimension; ++I)
426 IdxList.push_back(Zero);
427 IdxList.push_back(Call->getArgOperand(GEPIndex));
428
430 getBaseElementType(Call), Call->getArgOperand(0), IdxList, "", Call);
431 Call->replaceAllUsesWith(GEP);
432 Call->eraseFromParent();
433}
434
436 replaceWithGEP(Call, 1, 2);
437}
438
440 replaceWithGEP(Call, 0, 1);
441}
442
444 Call->replaceAllUsesWith(Call->getArgOperand(0));
445 Call->eraseFromParent();
446}
447
448bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) {
449 std::vector<CallInst *> PreserveArrayIndexCalls;
450 std::vector<CallInst *> PreserveUnionIndexCalls;
451 std::vector<CallInst *> PreserveStructIndexCalls;
452 bool Found = false;
453
454 for (auto &BB : F)
455 for (auto &I : BB) {
456 auto *Call = dyn_cast<CallInst>(&I);
457 CallInfo CInfo;
458 if (!IsPreserveDIAccessIndexCall(Call, CInfo))
459 continue;
460
461 Found = true;
462 if (CInfo.Kind == BPFPreserveArrayAI)
463 PreserveArrayIndexCalls.push_back(Call);
464 else if (CInfo.Kind == BPFPreserveUnionAI)
465 PreserveUnionIndexCalls.push_back(Call);
466 else
467 PreserveStructIndexCalls.push_back(Call);
468 }
469
470 // do the following transformation:
471 // . addr = preserve_array_access_index(base, dimension, index)
472 // is transformed to
473 // addr = GEP(base, dimenion's zero's, index)
474 // . addr = preserve_union_access_index(base, di_index)
475 // is transformed to
476 // addr = base, i.e., all usages of "addr" are replaced by "base".
477 // . addr = preserve_struct_access_index(base, gep_index, di_index)
478 // is transformed to
479 // addr = GEP(base, 0, gep_index)
480 for (CallInst *Call : PreserveArrayIndexCalls)
482 for (CallInst *Call : PreserveStructIndexCalls)
484 for (CallInst *Call : PreserveUnionIndexCalls)
486
487 return Found;
488}
489
490/// Check whether the access index chain is valid. We check
491/// here because there may be type casts between two
492/// access indexes. We want to ensure memory access still valid.
493bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType,
494 uint32_t ParentAI,
495 const MDNode *ChildType) {
496 if (!ChildType)
497 return true; // preserve_field_info, no type comparison needed.
498
499 const DIType *PType = stripQualifiers(cast<DIType>(ParentType));
500 const DIType *CType = stripQualifiers(cast<DIType>(ChildType));
501
502 // Child is a derived/pointer type, which is due to type casting.
503 // Pointer type cannot be in the middle of chain.
504 if (isa<DIDerivedType>(CType))
505 return false;
506
507 // Parent is a pointer type.
508 if (const auto *PtrTy = dyn_cast<DIDerivedType>(PType)) {
509 if (PtrTy->getTag() != dwarf::DW_TAG_pointer_type)
510 return false;
511 return stripQualifiers(PtrTy->getBaseType()) == CType;
512 }
513
514 // Otherwise, struct/union/array types
515 const auto *PTy = dyn_cast<DICompositeType>(PType);
516 const auto *CTy = dyn_cast<DICompositeType>(CType);
517 assert(PTy && CTy && "ParentType or ChildType is null or not composite");
518
519 uint32_t PTyTag = PTy->getTag();
520 assert(PTyTag == dwarf::DW_TAG_array_type ||
521 PTyTag == dwarf::DW_TAG_structure_type ||
522 PTyTag == dwarf::DW_TAG_union_type);
523
524 uint32_t CTyTag = CTy->getTag();
525 assert(CTyTag == dwarf::DW_TAG_array_type ||
526 CTyTag == dwarf::DW_TAG_structure_type ||
527 CTyTag == dwarf::DW_TAG_union_type);
528
529 // Multi dimensional arrays, base element should be the same
530 if (PTyTag == dwarf::DW_TAG_array_type && PTyTag == CTyTag)
531 return PTy->getBaseType() == CTy->getBaseType();
532
533 DIType *Ty;
534 if (PTyTag == dwarf::DW_TAG_array_type)
535 Ty = PTy->getBaseType();
536 else
537 Ty = dyn_cast<DIType>(PTy->getElements()[ParentAI]);
538
539 return dyn_cast<DICompositeType>(stripQualifiers(Ty)) == CTy;
540}
541
542void BPFAbstractMemberAccess::traceAICall(CallInst *Call,
543 CallInfo &ParentInfo) {
544 for (User *U : Call->users()) {
545 Instruction *Inst = dyn_cast<Instruction>(U);
546 if (!Inst)
547 continue;
548
549 if (auto *BI = dyn_cast<BitCastInst>(Inst)) {
550 traceBitCast(BI, Call, ParentInfo);
551 } else if (auto *CI = dyn_cast<CallInst>(Inst)) {
552 CallInfo ChildInfo;
553
554 if (IsPreserveDIAccessIndexCall(CI, ChildInfo) &&
555 IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex,
556 ChildInfo.Metadata)) {
557 AIChain[CI] = std::make_pair(Call, ParentInfo);
558 traceAICall(CI, ChildInfo);
559 } else {
560 BaseAICalls[Call] = ParentInfo;
561 }
562 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
563 if (GI->hasAllZeroIndices())
564 traceGEP(GI, Call, ParentInfo);
565 else
566 BaseAICalls[Call] = ParentInfo;
567 } else {
568 BaseAICalls[Call] = ParentInfo;
569 }
570 }
571}
572
573void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast,
574 CallInst *Parent,
575 CallInfo &ParentInfo) {
576 for (User *U : BitCast->users()) {
577 Instruction *Inst = dyn_cast<Instruction>(U);
578 if (!Inst)
579 continue;
580
581 if (auto *BI = dyn_cast<BitCastInst>(Inst)) {
582 traceBitCast(BI, Parent, ParentInfo);
583 } else if (auto *CI = dyn_cast<CallInst>(Inst)) {
584 CallInfo ChildInfo;
585 if (IsPreserveDIAccessIndexCall(CI, ChildInfo) &&
586 IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex,
587 ChildInfo.Metadata)) {
588 AIChain[CI] = std::make_pair(Parent, ParentInfo);
589 traceAICall(CI, ChildInfo);
590 } else {
591 BaseAICalls[Parent] = ParentInfo;
592 }
593 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
594 if (GI->hasAllZeroIndices())
595 traceGEP(GI, Parent, ParentInfo);
596 else
597 BaseAICalls[Parent] = ParentInfo;
598 } else {
599 BaseAICalls[Parent] = ParentInfo;
600 }
601 }
602}
603
604void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent,
605 CallInfo &ParentInfo) {
606 for (User *U : GEP->users()) {
607 Instruction *Inst = dyn_cast<Instruction>(U);
608 if (!Inst)
609 continue;
610
611 if (auto *BI = dyn_cast<BitCastInst>(Inst)) {
612 traceBitCast(BI, Parent, ParentInfo);
613 } else if (auto *CI = dyn_cast<CallInst>(Inst)) {
614 CallInfo ChildInfo;
615 if (IsPreserveDIAccessIndexCall(CI, ChildInfo) &&
616 IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex,
617 ChildInfo.Metadata)) {
618 AIChain[CI] = std::make_pair(Parent, ParentInfo);
619 traceAICall(CI, ChildInfo);
620 } else {
621 BaseAICalls[Parent] = ParentInfo;
622 }
623 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) {
624 if (GI->hasAllZeroIndices())
625 traceGEP(GI, Parent, ParentInfo);
626 else
627 BaseAICalls[Parent] = ParentInfo;
628 } else {
629 BaseAICalls[Parent] = ParentInfo;
630 }
631 }
632}
633
634void BPFAbstractMemberAccess::collectAICallChains(Function &F) {
635 AIChain.clear();
636 BaseAICalls.clear();
637
638 for (auto &BB : F)
639 for (auto &I : BB) {
640 CallInfo CInfo;
641 auto *Call = dyn_cast<CallInst>(&I);
642 if (!IsPreserveDIAccessIndexCall(Call, CInfo) ||
643 AIChain.find(Call) != AIChain.end())
644 continue;
645
646 traceAICall(Call, CInfo);
647 }
648}
649
650/// Get the start and the end of storage offset for \p MemberTy.
651void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy,
652 Align RecordAlignment,
653 uint32_t &StartBitOffset,
654 uint32_t &EndBitOffset) {
655 uint32_t MemberBitSize = MemberTy->getSizeInBits();
656 uint32_t MemberBitOffset = MemberTy->getOffsetInBits();
657
658 if (RecordAlignment > 8) {
659 // If the Bits are within an aligned 8-byte, set the RecordAlignment
660 // to 8, other report the fatal error.
661 if (MemberBitOffset / 64 != (MemberBitOffset + MemberBitSize) / 64)
662 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, "
663 "requiring too big alignment");
664 RecordAlignment = Align(8);
665 }
666
667 uint32_t AlignBits = RecordAlignment.value() * 8;
668 if (MemberBitSize > AlignBits)
669 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, "
670 "bitfield size greater than record alignment");
671
672 StartBitOffset = MemberBitOffset & ~(AlignBits - 1);
673 if ((StartBitOffset + AlignBits) < (MemberBitOffset + MemberBitSize))
674 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, "
675 "cross alignment boundary");
676 EndBitOffset = StartBitOffset + AlignBits;
677}
678
679uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind,
680 DICompositeType *CTy,
681 uint32_t AccessIndex,
682 uint32_t PatchImm,
683 MaybeAlign RecordAlignment) {
684 if (InfoKind == BTF::FIELD_EXISTENCE)
685 return 1;
686
687 uint32_t Tag = CTy->getTag();
688 if (InfoKind == BTF::FIELD_BYTE_OFFSET) {
689 if (Tag == dwarf::DW_TAG_array_type) {
690 auto *EltTy = stripQualifiers(CTy->getBaseType());
691 PatchImm += AccessIndex * calcArraySize(CTy, 1) *
692 (EltTy->getSizeInBits() >> 3);
693 } else if (Tag == dwarf::DW_TAG_structure_type) {
694 auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);
695 if (!MemberTy->isBitField()) {
696 PatchImm += MemberTy->getOffsetInBits() >> 3;
697 } else {
698 unsigned SBitOffset, NextSBitOffset;
699 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset,
700 NextSBitOffset);
701 PatchImm += SBitOffset >> 3;
702 }
703 }
704 return PatchImm;
705 }
706
707 if (InfoKind == BTF::FIELD_BYTE_SIZE) {
708 if (Tag == dwarf::DW_TAG_array_type) {
709 auto *EltTy = stripQualifiers(CTy->getBaseType());
710 return calcArraySize(CTy, 1) * (EltTy->getSizeInBits() >> 3);
711 } else {
712 auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);
713 uint32_t SizeInBits = MemberTy->getSizeInBits();
714 if (!MemberTy->isBitField())
715 return SizeInBits >> 3;
716
717 unsigned SBitOffset, NextSBitOffset;
718 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset,
719 NextSBitOffset);
720 SizeInBits = NextSBitOffset - SBitOffset;
721 if (SizeInBits & (SizeInBits - 1))
722 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info");
723 return SizeInBits >> 3;
724 }
725 }
726
727 if (InfoKind == BTF::FIELD_SIGNEDNESS) {
728 const DIType *BaseTy;
729 if (Tag == dwarf::DW_TAG_array_type) {
730 // Signedness only checked when final array elements are accessed.
731 if (CTy->getElements().size() != 1)
732 report_fatal_error("Invalid array expression for llvm.bpf.preserve.field.info");
734 } else {
735 auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);
736 BaseTy = stripQualifiers(MemberTy->getBaseType());
737 }
738
739 // Only basic types and enum types have signedness.
740 const auto *BTy = dyn_cast<DIBasicType>(BaseTy);
741 while (!BTy) {
742 const auto *CompTy = dyn_cast<DICompositeType>(BaseTy);
743 // Report an error if the field expression does not have signedness.
744 if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type)
745 report_fatal_error("Invalid field expression for llvm.bpf.preserve.field.info");
746 BaseTy = stripQualifiers(CompTy->getBaseType());
747 BTy = dyn_cast<DIBasicType>(BaseTy);
748 }
749 uint32_t Encoding = BTy->getEncoding();
750 return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char);
751 }
752
753 if (InfoKind == BTF::FIELD_LSHIFT_U64) {
754 // The value is loaded into a value with FIELD_BYTE_SIZE size,
755 // and then zero or sign extended to U64.
756 // FIELD_LSHIFT_U64 and FIELD_RSHIFT_U64 are operations
757 // to extract the original value.
758 const Triple &Triple = TM->getTargetTriple();
759 DIDerivedType *MemberTy = nullptr;
760 bool IsBitField = false;
761 uint32_t SizeInBits;
762
763 if (Tag == dwarf::DW_TAG_array_type) {
764 auto *EltTy = stripQualifiers(CTy->getBaseType());
765 SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits();
766 } else {
767 MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);
768 SizeInBits = MemberTy->getSizeInBits();
769 IsBitField = MemberTy->isBitField();
770 }
771
772 if (!IsBitField) {
773 if (SizeInBits > 64)
774 report_fatal_error("too big field size for llvm.bpf.preserve.field.info");
775 return 64 - SizeInBits;
776 }
777
778 unsigned SBitOffset, NextSBitOffset;
779 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset);
780 if (NextSBitOffset - SBitOffset > 64)
781 report_fatal_error("too big field size for llvm.bpf.preserve.field.info");
782
783 unsigned OffsetInBits = MemberTy->getOffsetInBits();
785 return SBitOffset + 64 - OffsetInBits - SizeInBits;
786 else
787 return OffsetInBits + 64 - NextSBitOffset;
788 }
789
790 if (InfoKind == BTF::FIELD_RSHIFT_U64) {
791 DIDerivedType *MemberTy = nullptr;
792 bool IsBitField = false;
793 uint32_t SizeInBits;
794 if (Tag == dwarf::DW_TAG_array_type) {
795 auto *EltTy = stripQualifiers(CTy->getBaseType());
796 SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits();
797 } else {
798 MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]);
799 SizeInBits = MemberTy->getSizeInBits();
800 IsBitField = MemberTy->isBitField();
801 }
802
803 if (!IsBitField) {
804 if (SizeInBits > 64)
805 report_fatal_error("too big field size for llvm.bpf.preserve.field.info");
806 return 64 - SizeInBits;
807 }
808
809 unsigned SBitOffset, NextSBitOffset;
810 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset);
811 if (NextSBitOffset - SBitOffset > 64)
812 report_fatal_error("too big field size for llvm.bpf.preserve.field.info");
813
814 return 64 - SizeInBits;
815 }
816
817 llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind");
818}
819
820bool BPFAbstractMemberAccess::HasPreserveFieldInfoCall(CallInfoStack &CallStack) {
821 // This is called in error return path, no need to maintain CallStack.
822 while (CallStack.size()) {
823 auto StackElem = CallStack.top();
824 if (StackElem.second.Kind == BPFPreserveFieldInfoAI)
825 return true;
826 CallStack.pop();
827 }
828 return false;
829}
830
831/// Compute the base of the whole preserve_* intrinsics chains, i.e., the base
832/// pointer of the first preserve_*_access_index call, and construct the access
833/// string, which will be the name of a global variable.
834Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call,
835 CallInfo &CInfo,
836 std::string &AccessKey,
837 MDNode *&TypeMeta) {
838 Value *Base = nullptr;
839 std::string TypeName;
840 CallInfoStack CallStack;
841
842 // Put the access chain into a stack with the top as the head of the chain.
843 while (Call) {
844 CallStack.push(std::make_pair(Call, CInfo));
845 CInfo = AIChain[Call].second;
846 Call = AIChain[Call].first;
847 }
848
849 // The access offset from the base of the head of chain is also
850 // calculated here as all debuginfo types are available.
851
852 // Get type name and calculate the first index.
853 // We only want to get type name from typedef, structure or union.
854 // If user wants a relocation like
855 // int *p; ... __builtin_preserve_access_index(&p[4]) ...
856 // or
857 // int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ...
858 // we will skip them.
859 uint32_t FirstIndex = 0;
860 uint32_t PatchImm = 0; // AccessOffset or the requested field info
862 while (CallStack.size()) {
863 auto StackElem = CallStack.top();
864 Call = StackElem.first;
865 CInfo = StackElem.second;
866
867 if (!Base)
868 Base = CInfo.Base;
869
870 DIType *PossibleTypeDef = stripQualifiers(cast<DIType>(CInfo.Metadata),
871 false);
872 DIType *Ty = stripQualifiers(PossibleTypeDef);
873 if (CInfo.Kind == BPFPreserveUnionAI ||
874 CInfo.Kind == BPFPreserveStructAI) {
875 // struct or union type. If the typedef is in the metadata, always
876 // use the typedef.
877 TypeName = std::string(PossibleTypeDef->getName());
878 TypeMeta = PossibleTypeDef;
879 PatchImm += FirstIndex * (Ty->getSizeInBits() >> 3);
880 break;
881 }
882
883 assert(CInfo.Kind == BPFPreserveArrayAI);
884
885 // Array entries will always be consumed for accumulative initial index.
886 CallStack.pop();
887
888 // BPFPreserveArrayAI
889 uint64_t AccessIndex = CInfo.AccessIndex;
890
891 DIType *BaseTy = nullptr;
892 bool CheckElemType = false;
893 if (const auto *CTy = dyn_cast<DICompositeType>(Ty)) {
894 // array type
895 assert(CTy->getTag() == dwarf::DW_TAG_array_type);
896
897
898 FirstIndex += AccessIndex * calcArraySize(CTy, 1);
900 CheckElemType = CTy->getElements().size() == 1;
901 } else {
902 // pointer type
903 auto *DTy = cast<DIDerivedType>(Ty);
904 assert(DTy->getTag() == dwarf::DW_TAG_pointer_type);
905
906 BaseTy = stripQualifiers(DTy->getBaseType());
907 CTy = dyn_cast<DICompositeType>(BaseTy);
908 if (!CTy) {
909 CheckElemType = true;
910 } else if (CTy->getTag() != dwarf::DW_TAG_array_type) {
911 FirstIndex += AccessIndex;
912 CheckElemType = true;
913 } else {
914 FirstIndex += AccessIndex * calcArraySize(CTy, 0);
915 }
916 }
917
918 if (CheckElemType) {
919 auto *CTy = dyn_cast<DICompositeType>(BaseTy);
920 if (!CTy) {
921 if (HasPreserveFieldInfoCall(CallStack))
922 report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic");
923 return nullptr;
924 }
925
926 unsigned CTag = CTy->getTag();
927 if (CTag == dwarf::DW_TAG_structure_type || CTag == dwarf::DW_TAG_union_type) {
928 TypeName = std::string(CTy->getName());
929 } else {
930 if (HasPreserveFieldInfoCall(CallStack))
931 report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic");
932 return nullptr;
933 }
934 TypeMeta = CTy;
935 PatchImm += FirstIndex * (CTy->getSizeInBits() >> 3);
936 break;
937 }
938 }
939 assert(TypeName.size());
940 AccessKey += std::to_string(FirstIndex);
941
942 // Traverse the rest of access chain to complete offset calculation
943 // and access key construction.
944 while (CallStack.size()) {
945 auto StackElem = CallStack.top();
946 CInfo = StackElem.second;
947 CallStack.pop();
948
949 if (CInfo.Kind == BPFPreserveFieldInfoAI) {
950 InfoKind = CInfo.AccessIndex;
951 if (InfoKind == BTF::FIELD_EXISTENCE)
952 PatchImm = 1;
953 break;
954 }
955
956 // If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI,
957 // the action will be extracting field info.
958 if (CallStack.size()) {
959 auto StackElem2 = CallStack.top();
960 CallInfo CInfo2 = StackElem2.second;
961 if (CInfo2.Kind == BPFPreserveFieldInfoAI) {
962 InfoKind = CInfo2.AccessIndex;
963 assert(CallStack.size() == 1);
964 }
965 }
966
967 // Access Index
968 uint64_t AccessIndex = CInfo.AccessIndex;
969 AccessKey += ":" + std::to_string(AccessIndex);
970
971 MDNode *MDN = CInfo.Metadata;
972 // At this stage, it cannot be pointer type.
973 auto *CTy = cast<DICompositeType>(stripQualifiers(cast<DIType>(MDN)));
974 PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex, PatchImm,
975 CInfo.RecordAlignment);
976 }
977
978 // Access key is the
979 // "llvm." + type name + ":" + reloc type + ":" + patched imm + "$" +
980 // access string,
981 // uniquely identifying one relocation.
982 // The prefix "llvm." indicates this is a temporary global, which should
983 // not be emitted to ELF file.
984 AccessKey = "llvm." + TypeName + ":" + std::to_string(InfoKind) + ":" +
985 std::to_string(PatchImm) + "$" + AccessKey;
986
987 return Base;
988}
989
990MDNode *BPFAbstractMemberAccess::computeAccessKey(CallInst *Call,
991 CallInfo &CInfo,
992 std::string &AccessKey,
993 bool &IsInt32Ret) {
994 DIType *Ty = stripQualifiers(cast<DIType>(CInfo.Metadata), false);
995 assert(!Ty->getName().empty());
996
997 int64_t PatchImm;
998 std::string AccessStr("0");
999 if (CInfo.AccessIndex == BTF::TYPE_EXISTENCE ||
1000 CInfo.AccessIndex == BTF::TYPE_MATCH) {
1001 PatchImm = 1;
1002 } else if (CInfo.AccessIndex == BTF::TYPE_SIZE) {
1003 // typedef debuginfo type has size 0, get the eventual base type.
1004 DIType *BaseTy = stripQualifiers(Ty, true);
1005 PatchImm = BaseTy->getSizeInBits() / 8;
1006 } else {
1007 // ENUM_VALUE_EXISTENCE and ENUM_VALUE
1008 IsInt32Ret = false;
1009
1010 // The argument could be a global variable or a getelementptr with base to
1011 // a global variable depending on whether the clang option `opaque-options`
1012 // is set or not.
1013 const GlobalVariable *GV =
1014 cast<GlobalVariable>(Call->getArgOperand(1)->stripPointerCasts());
1015 assert(GV->hasInitializer());
1016 const ConstantDataArray *DA = cast<ConstantDataArray>(GV->getInitializer());
1017 assert(DA->isString());
1018 StringRef ValueStr = DA->getAsString();
1019
1020 // ValueStr format: <EnumeratorStr>:<Value>
1021 size_t Separator = ValueStr.find_first_of(':');
1022 StringRef EnumeratorStr = ValueStr.substr(0, Separator);
1023
1024 // Find enumerator index in the debuginfo
1025 DIType *BaseTy = stripQualifiers(Ty, true);
1026 const auto *CTy = cast<DICompositeType>(BaseTy);
1027 assert(CTy->getTag() == dwarf::DW_TAG_enumeration_type);
1028 int EnumIndex = 0;
1029 for (const auto Element : CTy->getElements()) {
1030 const auto *Enum = cast<DIEnumerator>(Element);
1031 if (Enum->getName() == EnumeratorStr) {
1032 AccessStr = std::to_string(EnumIndex);
1033 break;
1034 }
1035 EnumIndex++;
1036 }
1037
1038 if (CInfo.AccessIndex == BTF::ENUM_VALUE) {
1039 StringRef EValueStr = ValueStr.substr(Separator + 1);
1040 PatchImm = std::stoll(std::string(EValueStr));
1041 } else {
1042 PatchImm = 1;
1043 }
1044 }
1045
1046 AccessKey = "llvm." + Ty->getName().str() + ":" +
1047 std::to_string(CInfo.AccessIndex) + std::string(":") +
1048 std::to_string(PatchImm) + std::string("$") + AccessStr;
1049
1050 return Ty;
1051}
1052
1053/// Call/Kind is the base preserve_*_access_index() call. Attempts to do
1054/// transformation to a chain of relocable GEPs.
1055bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call,
1056 CallInfo &CInfo) {
1057 std::string AccessKey;
1058 MDNode *TypeMeta;
1059 Value *Base = nullptr;
1060 bool IsInt32Ret;
1061
1062 IsInt32Ret = CInfo.Kind == BPFPreserveFieldInfoAI;
1063 if (CInfo.Kind == BPFPreserveFieldInfoAI && CInfo.Metadata) {
1064 TypeMeta = computeAccessKey(Call, CInfo, AccessKey, IsInt32Ret);
1065 } else {
1066 Base = computeBaseAndAccessKey(Call, CInfo, AccessKey, TypeMeta);
1067 if (!Base)
1068 return false;
1069 }
1070
1071 BasicBlock *BB = Call->getParent();
1072 GlobalVariable *GV;
1073
1074 if (GEPGlobals.find(AccessKey) == GEPGlobals.end()) {
1075 IntegerType *VarType;
1076 if (IsInt32Ret)
1077 VarType = Type::getInt32Ty(BB->getContext()); // 32bit return value
1078 else
1079 VarType = Type::getInt64Ty(BB->getContext()); // 64bit ptr or enum value
1080
1081 GV = new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage,
1082 nullptr, AccessKey);
1084 GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta);
1085 GEPGlobals[AccessKey] = GV;
1086 } else {
1087 GV = GEPGlobals[AccessKey];
1088 }
1089
1090 if (CInfo.Kind == BPFPreserveFieldInfoAI) {
1091 // Load the global variable which represents the returned field info.
1092 LoadInst *LDInst;
1093 if (IsInt32Ret)
1094 LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV, "", Call);
1095 else
1096 LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", Call);
1097
1098 Instruction *PassThroughInst =
1099 BPFCoreSharedInfo::insertPassThrough(M, BB, LDInst, Call);
1100 Call->replaceAllUsesWith(PassThroughInst);
1101 Call->eraseFromParent();
1102 return true;
1103 }
1104
1105 // For any original GEP Call and Base %2 like
1106 // %4 = bitcast %struct.net_device** %dev1 to i64*
1107 // it is transformed to:
1108 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0
1109 // %7 = bitcast %struct.sk_buff* %2 to i8*
1110 // %8 = getelementptr i8, i8* %7, %6
1111 // %9 = bitcast i8* %8 to i64*
1112 // using %9 instead of %4
1113 // The original Call inst is removed.
1114
1115 // Load the global variable.
1116 auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", Call);
1117
1118 // Generate a BitCast
1119 auto *BCInst =
1121 BCInst->insertBefore(Call);
1122
1123 // Generate a GetElementPtr
1125 BCInst, LDInst);
1126 GEP->insertBefore(Call);
1127
1128 // Generate a BitCast
1129 auto *BCInst2 = new BitCastInst(GEP, Call->getType());
1130 BCInst2->insertBefore(Call);
1131
1132 // For the following code,
1133 // Block0:
1134 // ...
1135 // if (...) goto Block1 else ...
1136 // Block1:
1137 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0
1138 // %7 = bitcast %struct.sk_buff* %2 to i8*
1139 // %8 = getelementptr i8, i8* %7, %6
1140 // ...
1141 // goto CommonExit
1142 // Block2:
1143 // ...
1144 // if (...) goto Block3 else ...
1145 // Block3:
1146 // %6 = load llvm.bpf_map:0:40$0:0:0:2:0
1147 // %7 = bitcast %struct.sk_buff* %2 to i8*
1148 // %8 = getelementptr i8, i8* %7, %6
1149 // ...
1150 // goto CommonExit
1151 // CommonExit
1152 // SimplifyCFG may generate:
1153 // Block0:
1154 // ...
1155 // if (...) goto Block_Common else ...
1156 // Block2:
1157 // ...
1158 // if (...) goto Block_Common else ...
1159 // Block_Common:
1160 // PHI = [llvm.sk_buff:0:50$0:0:0:2:0, llvm.bpf_map:0:40$0:0:0:2:0]
1161 // %6 = load PHI
1162 // %7 = bitcast %struct.sk_buff* %2 to i8*
1163 // %8 = getelementptr i8, i8* %7, %6
1164 // ...
1165 // goto CommonExit
1166 // For the above code, we cannot perform proper relocation since
1167 // "load PHI" has two possible relocations.
1168 //
1169 // To prevent above tail merging, we use __builtin_bpf_passthrough()
1170 // where one of its parameters is a seq_num. Since two
1171 // __builtin_bpf_passthrough() funcs will always have different seq_num,
1172 // tail merging cannot happen. The __builtin_bpf_passthrough() will be
1173 // removed in the beginning of Target IR passes.
1174 //
1175 // This approach is also used in other places when global var
1176 // representing a relocation is used.
1177 Instruction *PassThroughInst =
1178 BPFCoreSharedInfo::insertPassThrough(M, BB, BCInst2, Call);
1179 Call->replaceAllUsesWith(PassThroughInst);
1180 Call->eraseFromParent();
1181
1182 return true;
1183}
1184
1185bool BPFAbstractMemberAccess::doTransformation(Function &F) {
1186 bool Transformed = false;
1187
1188 // Collect PreserveDIAccessIndex Intrinsic call chains.
1189 // The call chains will be used to generate the access
1190 // patterns similar to GEP.
1191 collectAICallChains(F);
1192
1193 for (auto &C : BaseAICalls)
1194 Transformed = transformGEPChain(C.first, C.second) || Transformed;
1195
1196 return removePreserveAccessIndexIntrinsic(F) || Transformed;
1197}
1198
1201 return BPFAbstractMemberAccess(TM).run(F) ? PreservedAnalyses::none()
1203}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void replaceWithGEP(CallInst *Call, uint32_t DimensionIndex, uint32_t GEPIndex)
static uint64_t getConstant(const Value *IndexValue)
static Type * getBaseElementType(const CallInst *Call)
static uint32_t calcArraySize(const DICompositeType *CTy, uint32_t StartDim)
static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef)
static DIType * stripQualifiers(DIType *Ty, bool skipTypedef=true)
This file contains the layout of .BTF and .BTF.ext ELF sections.
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file contains constants used for implementing Dwarf debug support.
Hexagon Common GEP
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Module.h This file contains the declarations for the Module class.
const char LLVMTargetMachineRef TM
This header defines various interfaces for pass management in LLVM.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1485
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:348
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
static void removeArrayAccessCall(CallInst *Call)
static uint32_t SeqNum
llvm.bpf.passthrough builtin seq number
Definition: BPFCORE.h:51
static void removeStructAccessCall(CallInst *Call)
static void removeUnionAccessCall(CallInst *Call)
static Instruction * insertPassThrough(Module *M, BasicBlock *BB, Instruction *Input, Instruction *Before)
Insert a bpf passthrough builtin function.
static constexpr StringRef AmaAttr
The attribute attached to globals representing a field access.
Definition: BPFCORE.h:46
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:213
This class represents a no-op cast from one type to another.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
An array constant whose element type is a simple 1/2/4/8-byte integer or float/double,...
Definition: Constants.h:691
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:144
This is an important base class in LLVM.
Definition: Constant.h:41
DINodeArray getElements() const
DIType * getBaseType() const
Tagged DWARF-like metadata node.
dwarf::Tag getTag() const
Subprogram description.
Array subrange.
BoundType getCount() const
Base class for types.
StringRef getName() const
uint64_t getSizeInBits() const
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:949
static GetElementPtrInst * CreateInBounds(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Create an "inbounds" getelementptr.
Definition: Instructions.h:998
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Definition: Instructions.h:975
void setMetadata(unsigned KindID, MDNode *Node)
Set a particular kind of metadata attachment.
Definition: Metadata.cpp:1485
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:52
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
bool hasInitializer() const
Definitions have initializers, declarations don't.
void addAttribute(Attribute::AttrKind Kind)
Add attribute to this global.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
Definition: Instruction.cpp:98
Class to represent integer types.
Definition: DerivedTypes.h:40
An instruction for reading from memory.
Definition: Instructions.h:178
Metadata node.
Definition: Metadata.h:1067
Root of the metadata hierarchy.
Definition: Metadata.h:62
Metadata(unsigned ID, StorageType Storage)
Definition: Metadata.h:86
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:222
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:567
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
size_t find_first_of(char C, size_t From=0) const
Find the first character in the string that is C, or npos if not found.
Definition: StringRef.h:373
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ArchType getArch() const
Get the parsed architecture type of this triple.
Definition: Triple.h:361
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
iterator_range< user_iterator > users()
Definition: Value.h:421
Value handle that is nullable, but tries to track the Value.
Definition: ValueHandle.h:204
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char TypeName[]
Key for Kernel::Arg::Metadata::mTypeName.
@ FIELD_RSHIFT_U64
Definition: BTF.h:287
@ ENUM_VALUE
Definition: BTF.h:293
@ FIELD_SIGNEDNESS
Definition: BTF.h:285
@ FIELD_BYTE_OFFSET
Definition: BTF.h:282
@ FIELD_BYTE_SIZE
Definition: BTF.h:283
@ ENUM_VALUE_EXISTENCE
Definition: BTF.h:292
@ MAX_FIELD_RELOC_KIND
Definition: BTF.h:295
@ TYPE_EXISTENCE
Definition: BTF.h:290
@ FIELD_LSHIFT_U64
Definition: BTF.h:286
@ TYPE_MATCH
Definition: BTF.h:294
@ TYPE_SIZE
Definition: BTF.h:291
@ FIELD_EXISTENCE
Definition: BTF.h:284
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1447
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117