diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td --- a/llvm/include/llvm/IR/IntrinsicsBPF.td +++ b/llvm/include/llvm/IR/IntrinsicsBPF.td @@ -32,4 +32,6 @@ def int_bpf_preserve_enum_value : GCCBuiltin<"__builtin_bpf_preserve_enum_value">, Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_ptr_ty, llvm_i64_ty], [IntrNoMem]>; + def int_bpf_passthrough : GCCBuiltin<"__builtin_bpf_passthrough">, + Intrinsic<[llvm_any_ty], [llvm_any_ty], [IntrNoMem]>; } diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -15,9 +15,10 @@ namespace llvm { class BPFTargetMachine; -ModulePass *createBPFAbstractMemberAccess(BPFTargetMachine *TM); -ModulePass *createBPFPreserveDIType(); +ModulePass *createBPFCheckAndAdjustIR(); +FunctionPass *createBPFAbstractMemberAccess(BPFTargetMachine *TM); +FunctionPass *createBPFPreserveDIType(); FunctionPass *createBPFISelDag(BPFTargetMachine &TM); FunctionPass *createBPFMISimplifyPatchablePass(); FunctionPass *createBPFMIPeepholePass(); @@ -25,6 +26,8 @@ FunctionPass *createBPFMIPreEmitPeepholePass(); FunctionPass *createBPFMIPreEmitCheckingPass(); +void initializeBPFCheckAndAdjustIRPass(PassRegistry&); + void initializeBPFAbstractMemberAccessPass(PassRegistry&); void initializeBPFPreserveDITypePass(PassRegistry&); void initializeBPFMISimplifyPatchablePass(PassRegistry&); diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp --- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp +++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp @@ -81,6 +81,7 @@ #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicsBPF.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/User.h" @@ -93,18 +94,26 @@ namespace llvm { constexpr StringRef BPFCoreSharedInfo::AmaAttr; + +Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB, + Instruction *Input, + Instruction *Before) { + Function *Fn = Intrinsic::getDeclaration(M, Intrinsic::bpf_passthrough, + {Input->getType(), Input->getType()}); + auto *NewInst = CallInst::Create(Fn, {Input}); + // Prevent sink common which may create a PHI node for reloc globals. + NewInst->addAttribute(AttributeList::FunctionIndex, Attribute::NoMerge); + BB->getInstList().insert(Before->getIterator(), NewInst); + return NewInst; +} } // namespace llvm using namespace llvm; namespace { -class BPFAbstractMemberAccess final : public ModulePass { - StringRef getPassName() const override { - return "BPF Abstract Member Access"; - } - - bool runOnModule(Module &M) override; +class BPFAbstractMemberAccess final : public FunctionPass { + bool runOnFunction(Function &F) override; public: static char ID; @@ -112,7 +121,8 @@ // Add optional BPFTargetMachine parameter so that BPF backend can add the phase // with target machine to find out the endianness. The default constructor (without // parameters) is used by the pass manager for managing purposes. - BPFAbstractMemberAccess(BPFTargetMachine *TM = nullptr) : ModulePass(ID), TM(TM) {} + BPFAbstractMemberAccess(BPFTargetMachine *TM = nullptr) + : FunctionPass(ID), TM(TM) {} struct CallInfo { uint32_t Kind; @@ -132,6 +142,7 @@ }; const DataLayout *DL = nullptr; + Module *M = nullptr; std::map GEPGlobals; // A map to link preserve_*_access_index instrinsic calls. @@ -141,19 +152,19 @@ // intrinsics. std::map BaseAICalls; - bool doTransformation(Module &M); + bool doTransformation(Function &F); void traceAICall(CallInst *Call, CallInfo &ParentInfo); void traceBitCast(BitCastInst *BitCast, CallInst *Parent, CallInfo &ParentInfo); void traceGEP(GetElementPtrInst *GEP, CallInst *Parent, CallInfo &ParentInfo); - void collectAICallChains(Module &M, Function &F); + void collectAICallChains(Function &F); bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo); bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI, const MDNode *ChildMeta); - bool removePreserveAccessIndexIntrinsic(Module &M); + bool removePreserveAccessIndexIntrinsic(Function &F); void replaceWithGEP(std::vector &CallList, uint32_t NumOfZerosIndex, uint32_t DIIndex); bool HasPreserveFieldInfoCall(CallInfoStack &CallStack); @@ -168,27 +179,31 @@ MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo, std::string &AccessKey, bool &IsInt32Ret); uint64_t getConstant(const Value *IndexValue); - bool transformGEPChain(Module &M, CallInst *Call, CallInfo &CInfo); + bool transformGEPChain(CallInst *Call, CallInfo &CInfo); }; } // End anonymous namespace char BPFAbstractMemberAccess::ID = 0; INITIALIZE_PASS(BPFAbstractMemberAccess, DEBUG_TYPE, - "abstracting struct/union member accessees", false, false) + "BPF Abstract Member Access", false, false) -ModulePass *llvm::createBPFAbstractMemberAccess(BPFTargetMachine *TM) { +FunctionPass *llvm::createBPFAbstractMemberAccess(BPFTargetMachine *TM) { return new BPFAbstractMemberAccess(TM); } -bool BPFAbstractMemberAccess::runOnModule(Module &M) { +bool BPFAbstractMemberAccess::runOnFunction(Function &F) { LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n"); + M = F.getParent(); + if (!M) + return false; + // Bail out if no debug info. - if (M.debug_compile_units().empty()) + if (M->debug_compile_units().empty()) return false; - DL = &M.getDataLayout(); - return doTransformation(M); + DL = &M->getDataLayout(); + return doTransformation(F); } static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) { @@ -341,28 +356,27 @@ } } -bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Module &M) { +bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) { std::vector PreserveArrayIndexCalls; std::vector PreserveUnionIndexCalls; std::vector PreserveStructIndexCalls; bool Found = false; - for (Function &F : M) - for (auto &BB : F) - for (auto &I : BB) { - auto *Call = dyn_cast(&I); - CallInfo CInfo; - if (!IsPreserveDIAccessIndexCall(Call, CInfo)) - continue; - - Found = true; - if (CInfo.Kind == BPFPreserveArrayAI) - PreserveArrayIndexCalls.push_back(Call); - else if (CInfo.Kind == BPFPreserveUnionAI) - PreserveUnionIndexCalls.push_back(Call); - else - PreserveStructIndexCalls.push_back(Call); - } + for (auto &BB : F) + for (auto &I : BB) { + auto *Call = dyn_cast(&I); + CallInfo CInfo; + if (!IsPreserveDIAccessIndexCall(Call, CInfo)) + continue; + + Found = true; + if (CInfo.Kind == BPFPreserveArrayAI) + PreserveArrayIndexCalls.push_back(Call); + else if (CInfo.Kind == BPFPreserveUnionAI) + PreserveUnionIndexCalls.push_back(Call); + else + PreserveStructIndexCalls.push_back(Call); + } // do the following transformation: // . addr = preserve_array_access_index(base, dimension, index) @@ -528,7 +542,7 @@ } } -void BPFAbstractMemberAccess::collectAICallChains(Module &M, Function &F) { +void BPFAbstractMemberAccess::collectAICallChains(Function &F) { AIChain.clear(); BaseAICalls.clear(); @@ -938,7 +952,7 @@ /// Call/Kind is the base preserve_*_access_index() call. Attempts to do /// transformation to a chain of relocable GEPs. -bool BPFAbstractMemberAccess::transformGEPChain(Module &M, CallInst *Call, +bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call, CallInfo &CInfo) { std::string AccessKey; MDNode *TypeMeta; @@ -964,7 +978,7 @@ else VarType = Type::getInt64Ty(BB->getContext()); // 64bit ptr or enum value - GV = new GlobalVariable(M, VarType, false, GlobalVariable::ExternalLinkage, + GV = new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage, NULL, AccessKey); GV->addAttribute(BPFCoreSharedInfo::AmaAttr); GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta); @@ -980,7 +994,10 @@ LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV, "", Call); else LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", Call); - Call->replaceAllUsesWith(LDInst); + + Instruction *PassThroughInst = + BPFCoreSharedInfo::insertPassThrough(M, BB, LDInst, Call); + Call->replaceAllUsesWith(PassThroughInst); Call->eraseFromParent(); return true; } @@ -988,7 +1005,7 @@ // For any original GEP Call and Base %2 like // %4 = bitcast %struct.net_device** %dev1 to i64* // it is transformed to: - // %6 = load sk_buff:50:$0:0:0:2:0 + // %6 = load llvm.sk_buff:50:$0:0:0:2:0 // %7 = bitcast %struct.sk_buff* %2 to i8* // %8 = getelementptr i8, i8* %7, %6 // %9 = bitcast i8* %8 to i64* @@ -1011,24 +1028,72 @@ auto *BCInst2 = new BitCastInst(GEP, Call->getType()); BB->getInstList().insert(Call->getIterator(), BCInst2); - Call->replaceAllUsesWith(BCInst2); + // For the following code, + // Block0: + // ... + // if (...) goto Block1 else ... + // Block1: + // %6 = load llvm.sk_buff:50:$0:0:0:2:0 + // %7 = bitcast %struct.sk_buff* %2 to i8* + // %8 = getelementptr i8, i8* %7, %6 + // ... + // goto CommonExit + // Block2: + // ... + // if (...) goto Block3 else ... + // Block3: + // %6 = load llvm.bpf_map:40:$0:0:0:2:0 + // %7 = bitcast %struct.sk_buff* %2 to i8* + // %8 = getelementptr i8, i8* %7, %6 + // ... + // goto CommonExit + // CommonExit + // SimplifyCFG may generate: + // Block0: + // ... + // if (...) goto Block_Common else ... + // Block2: + // ... + // if (...) goto Block_Common else ... + // Block_Common: + // PHI = [llvm.sk_buff:50:$0:0:0:2:0, llvm.bpf_map:40:$0:0:0:2:0] + // %6 = load PHI + // %7 = bitcast %struct.sk_buff* %2 to i8* + // %8 = getelementptr i8, i8* %7, %6 + // ... + // goto CommonExit + // For the above code, we cannot perform proper relocation since + // "load PHI" has two possible relocations. + // + // Current SimplifyCFG has a couple of ways to prevent the above + // transformation: + // - insert an inline asm in the basic block + // - insert a call and mark it with NoMerge function attribute. + // We use __builtin_bpf_passthrough() call marked with NoMerge approach. + // Compared to inline assembly approach, this has less impact on inlining + // and other optimizations. The __builtin_bpf_passthrough() will be removed + // in the beginning of Target IR passes. + // + // This approach is also used in other places when global var + // representing a relocation is used. + Instruction *PassThroughInst = + BPFCoreSharedInfo::insertPassThrough(M, BB, BCInst2, Call); + Call->replaceAllUsesWith(PassThroughInst); Call->eraseFromParent(); return true; } -bool BPFAbstractMemberAccess::doTransformation(Module &M) { +bool BPFAbstractMemberAccess::doTransformation(Function &F) { bool Transformed = false; - for (Function &F : M) { - // Collect PreserveDIAccessIndex Intrinsic call chains. - // The call chains will be used to generate the access - // patterns similar to GEP. - collectAICallChains(M, F); + // Collect PreserveDIAccessIndex Intrinsic call chains. + // The call chains will be used to generate the access + // patterns similar to GEP. + collectAICallChains(F); - for (auto &C : BaseAICalls) - Transformed = transformGEPChain(M, C.first, C.second) || Transformed; - } + for (auto &C : BaseAICalls) + Transformed = transformGEPChain(C.first, C.second) || Transformed; - return removePreserveAccessIndexIntrinsic(M) || Transformed; + return removePreserveAccessIndexIntrinsic(F) || Transformed; } diff --git a/llvm/lib/Target/BPF/BPFCORE.h b/llvm/lib/Target/BPF/BPFCORE.h --- a/llvm/lib/Target/BPF/BPFCORE.h +++ b/llvm/lib/Target/BPF/BPFCORE.h @@ -13,6 +13,11 @@ namespace llvm { +class BasicBlock; +class CallInst; +class Instruction; +class Module; + class BPFCoreSharedInfo { public: enum PatchableRelocKind : uint32_t { @@ -57,6 +62,11 @@ static constexpr StringRef AmaAttr = "btf_ama"; /// The attribute attached to globals representing a type id static constexpr StringRef TypeIdAttr = "btf_type_id"; + + /// Insert a bpf passthrough builtin function. + static Instruction *insertPassThrough(Module *M, BasicBlock *BB, + Instruction *Input, + Instruction *Before); }; } // namespace llvm diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp @@ -0,0 +1,310 @@ +//===------------ BPFCheckAndAdjustIR.cpp - Check and Adjust IR -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Check IR and adjust IR for verifier friendly codes. +// The following are done for IR checking: +// - no relocation globals in PHI node. +// The following are done for IR adjustment: +// - remove __builtin_bpf_passthrough builtins. Target independent IR +// optimizations are done and those builtins can be removed. +// - coalesce possible relocations. The builtin->relocation_global +// transformation is done before inlining. Inlining may expose +// additional opportunities. +// +//===----------------------------------------------------------------------===// + +#include "BPF.h" +#include "BPFCORE.h" +#include "BPFTargetMachine.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" +#include "llvm/Pass.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" + +#define DEBUG_TYPE "bpf-check-and-opt-ir" + +using namespace llvm; + +namespace { + +class BPFCheckAndAdjustIR final : public ModulePass { + struct RelocOptInfo { + GetElementPtrInst *GEP1; + GetElementPtrInst *GEP2; + std::string RelocStr; + MDNode *TypeMeta; + RelocOptInfo(GetElementPtrInst *I1, GetElementPtrInst *I2, std::string Str, MDNode *Meta) + : GEP1(I1), GEP2(I2), RelocStr(Str), TypeMeta(Meta) {} + }; + + bool runOnModule(Module &F) override; + +public: + static char ID; + BPFCheckAndAdjustIR() : ModulePass(ID) {} + +private: + std::map GEPGlobals; + SmallVector RelocOptCands; + + void checkIR(Module &M); + bool adjustIR(Module &M); + bool removePassThroughBuiltin(Module &M); + bool optimizeReloc(Module &M); + void findOptRelocCand(Instruction &I); + void checkBitCast(BitCastInst *BI, GetElementPtrInst *GEP1, GlobalVariable *GV1); + void checkGEP(GetElementPtrInst *GEP2, GetElementPtrInst *GEP1, GlobalVariable *GV1); +}; +} // End anonymous namespace + +char BPFCheckAndAdjustIR::ID = 0; +INITIALIZE_PASS(BPFCheckAndAdjustIR, DEBUG_TYPE, "BPF Check And Adjust IR", + false, false) + +ModulePass *llvm::createBPFCheckAndAdjustIR() { + return new BPFCheckAndAdjustIR(); +} + +bool BPFCheckAndAdjustIR::runOnModule(Module &M) { + checkIR(M); + return adjustIR(M); +} + +void BPFCheckAndAdjustIR::checkIR(Module &M) { + // Ensure relocation global won't appear in PHI node + // This may happen if the compiler generated the following code: + // B1: + // g1 = @llvm.skb_buff:0:1... + // ... + // goto B_COMMON + // B2: + // g2 = @llvm.skb_buff:0:2... + // ... + // goto B_COMMON + // B_COMMON: + // g = PHI(g1, g2) + // x = load g + // ... + // If anything likes the above "g = PHI(g1, g2)", issue a fatal error. + for (Function &F : M) + for (auto &BB : F) + for (auto &I : BB) { + PHINode *PN = dyn_cast(&I); + if (!PN || PN->use_empty()) + continue; + for (int i = 0, e = PN->getNumIncomingValues(); i < e; ++i) { + auto *GV = dyn_cast(PN->getIncomingValue(i)); + if (!GV) + continue; + if (GV->hasAttribute(BPFCoreSharedInfo::AmaAttr) || + GV->hasAttribute(BPFCoreSharedInfo::TypeIdAttr)) + report_fatal_error("relocation global in PHI node"); + } + } +} + +bool BPFCheckAndAdjustIR::removePassThroughBuiltin(Module &M) { + // Remove __builtin_bpf_passthrough()'s which are used to prevent + // certain IR optimizations. Now major IR optimizations are done, + // remove them. + bool Changed = false; + CallInst *ToBeDeleted = nullptr; + for (Function &F : M) + for (auto &BB : F) + for (auto &I : BB) { + if (ToBeDeleted) { + ToBeDeleted->eraseFromParent(); + ToBeDeleted = nullptr; + } + + auto *Call = dyn_cast(&I); + if (!Call) + continue; + auto *GV = dyn_cast(Call->getCalledOperand()); + if (!GV) + continue; + if (!GV->getName().startswith("llvm.bpf.passthrough")) + continue; + Changed = true; + Value *Arg = Call->getArgOperand(0); + Call->replaceAllUsesWith(Arg); + ToBeDeleted = Call; + } + return Changed; +} + +// Adjust/Optimize the following pattern +// %0 = load i64, i64* @"llvm.bpf_htab:0:0$0:0", align 8 +// %1 = getelementptr i8, i8* bitcast (%struct.anon.2* @m_hash to i8*), +// i64 %0 +// %2 = bitcast i8* %1 to %struct.bpf_map* +// %3 = load i64, i64* @"llvm.bpf_map:0:0$0:0", align 8 +// %4 = bitcast %struct.bpf_map* %2 to i8* +// %5 = getelementptr i8, i8* %4, i64 %3 +// In the above sequence, we have address computation like +// @m_hash + @"llvm.bpf_htab:0:0$0:0" + @"llvm.bpf_map:0:0$0:0" +// The above two relocations can be merged to +// @"llvm.bpf_htab:0:0$0:0:0" +// and new address computation looks like +// @m_hash + @"llvm.bpf_htab:0:0$0:0:0" +bool BPFCheckAndAdjustIR::optimizeReloc(Module &M) { + // Find candidates. + for (Function &F : M) + for (auto &BB : F) { + BasicBlock::iterator I = BB.begin(), E = BB.end(); + for (; I != E; ++I) + findOptRelocCand(*I); + } + + if (RelocOptCands.empty()) + return false; + + // Do transformations one candidate at a time. + for (auto &Cand : RelocOptCands) { + GetElementPtrInst *GEP1 = Cand.GEP1; + GetElementPtrInst *GEP2 = Cand.GEP2; + std::string RelocStr = Cand.RelocStr; + IntegerType *OffsetType = Type::getInt64Ty(M.getContext()); + GlobalVariable *GV; + + if (GEPGlobals.find(RelocStr) == GEPGlobals.end()) { + MDNode *TypeMeta = Cand.TypeMeta; + GV = new GlobalVariable(M, OffsetType, false, + GlobalVariable::ExternalLinkage, NULL, RelocStr); + GV->addAttribute(BPFCoreSharedInfo::AmaAttr); + GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta); + GEPGlobals[RelocStr] = GV; + } else { + GV = GEPGlobals[RelocStr]; + } + + auto *LDInst = new LoadInst(OffsetType, GV, "", GEP2); + auto *NewGEP = GetElementPtrInst::Create(Type::getInt8Ty(M.getContext()), + GEP1->getOperand(0), LDInst); + GEP2->getParent()->getInstList().insert(GEP2->getIterator(), NewGEP); + GEP2->replaceAllUsesWith(NewGEP); + } + return true; +} + +void BPFCheckAndAdjustIR::checkBitCast(BitCastInst *BI, GetElementPtrInst *GEP1, + GlobalVariable *GV1) { + for (User *U : BI->users()) { + if (auto *BI2 = dyn_cast(U)) + checkBitCast(BI2, GEP1, GV1); + else if (auto *GEP2 = dyn_cast(U)) + checkGEP(GEP2, GEP1, GV1); + } +} + +void BPFCheckAndAdjustIR::checkGEP(GetElementPtrInst *GEP2, GetElementPtrInst *GEP1, + GlobalVariable *GV1) { + // Find the second reloc global. + if (GEP2->getNumOperands() != 2) + return; + + // GEP2 cannot be the first GEP of another candidate. + // We do not optimize chain of three relocation globals, which should be rare. + for (auto &Cand : RelocOptCands) { + if (GEP2 == Cand.GEP1) + return; + } + + auto *LdInst2 = dyn_cast(GEP2->getOperand(1)); + if (!LdInst2) + return; + + auto *GV2 = dyn_cast(LdInst2->getOperand(0)); + if (!GV2 || !GV2->hasAttribute(BPFCoreSharedInfo::AmaAttr)) + return; + + // Calculate the combined relocation access string and patch immediate. + StringRef GV1Name = GV1->getName(); + size_t GV1_FirstColon = GV1Name.find_first_of(':'); + size_t GV1_SecondColon = GV1Name.find_first_of(':', GV1_FirstColon + 1); + size_t GV1_FirstDollar = GV1Name.find_first_of('$'); + StringRef IndexPattern1 = GV1Name.substr(GV1_FirstDollar + 1); + StringRef RelocKindStr1 = + GV1Name.substr(GV1_FirstColon + 1, GV1_SecondColon - GV1_FirstColon); + StringRef PatchImmStr1 = + GV1Name.substr(GV1_SecondColon + 1, GV1_FirstDollar - GV1_SecondColon); + + StringRef GV2Name = GV2->getName(); + size_t GV2_FirstColon = GV2Name.find_first_of(':'); + size_t GV2_SecondColon = GV2Name.find_first_of(':', GV2_FirstColon + 1); + size_t GV2_FirstDollar = GV2Name.find_first_of('$'); + StringRef IndexPattern2 = GV2Name.substr(GV2_FirstDollar + 1); + StringRef RelocKindStr2 = + GV2Name.substr(GV2_FirstColon + 1, GV2_SecondColon - GV2_FirstColon); + StringRef PatchImmStr2 = + GV2Name.substr(GV2_SecondColon + 1, GV2_FirstDollar - GV2_SecondColon); + + // RelocKind should be the same and the second access string should have + // "0" as the first index. + if (RelocKindStr1 != RelocKindStr2 || IndexPattern2[0] != '0') + return; + + uint32_t NewPatchImm = std::stoll(std::string(PatchImmStr1)) + + std::stoll(std::string(PatchImmStr2)); + std::string NewIndexPattern = + std::string(IndexPattern1) + std::string(IndexPattern2.substr(1)); + + std::string NewTypeStr = std::string(GV1Name.substr(0, GV1_SecondColon)) + + ":" + std::to_string(NewPatchImm) + "$" + + NewIndexPattern; + + // A candidate to merge two relocations into one + MDNode *MDN = GV1->getMetadata(LLVMContext::MD_preserve_access_index); + RelocOptInfo Candidate(GEP1, GEP2, NewTypeStr, MDN); + RelocOptCands.push_back(Candidate); +} + +void BPFCheckAndAdjustIR::findOptRelocCand(Instruction &I) { + // Find the first reloc global. + auto *LdInst = dyn_cast(&I); + if (!LdInst) + return; + + auto *GV1 = dyn_cast(LdInst->getOperand(0)); + if (!GV1 || !GV1->hasAttribute(BPFCoreSharedInfo::AmaAttr)) + return; + + // Remember the RelocStr=>GlobalVariable matching so we do not + // create a global with identical name. + if (GEPGlobals.find(std::string(GV1->getName())) == GEPGlobals.end()) + GEPGlobals[std::string(GV1->getName())] = GV1; + + for (User *U : LdInst->users()) { + GetElementPtrInst *GEP1 = dyn_cast(U); + if (!GEP1 || GEP1->getNumOperands() != 2) + continue; + + // GEP1: op0: Base, op1: reloc_global + for (User *U2 : GEP1->users()) { + if (auto *BI = dyn_cast(U2)) + checkBitCast(BI, GEP1, GV1); + else if (auto *GEP2 = dyn_cast(U2)) + checkGEP(GEP2, GEP1, GV1); + } + } +} + +bool BPFCheckAndAdjustIR::adjustIR(Module &M) { + bool Changed = false; + + Changed = removePassThroughBuiltin(M); + Changed = optimizeReloc(M) || Changed; + + return false; +} diff --git a/llvm/lib/Target/BPF/BPFPreserveDIType.cpp b/llvm/lib/Target/BPF/BPFPreserveDIType.cpp --- a/llvm/lib/Target/BPF/BPFPreserveDIType.cpp +++ b/llvm/lib/Target/BPF/BPFPreserveDIType.cpp @@ -33,58 +33,58 @@ namespace { -class BPFPreserveDIType final : public ModulePass { - StringRef getPassName() const override { - return "BPF Preserve DebugInfo Type"; - } - - bool runOnModule(Module &M) override; +class BPFPreserveDIType final : public FunctionPass { + bool runOnFunction(Function &F) override; public: static char ID; - BPFPreserveDIType() : ModulePass(ID) {} + BPFPreserveDIType() : FunctionPass(ID) {} private: - bool doTransformation(Module &M); + Module *M = nullptr; + + bool doTransformation(Function &F); }; } // End anonymous namespace char BPFPreserveDIType::ID = 0; -INITIALIZE_PASS(BPFPreserveDIType, DEBUG_TYPE, "preserve debuginfo type", false, - false) +INITIALIZE_PASS(BPFPreserveDIType, DEBUG_TYPE, "BPF Preserve Debuginfo Type", + false, false) -ModulePass *llvm::createBPFPreserveDIType() { return new BPFPreserveDIType(); } +FunctionPass *llvm::createBPFPreserveDIType() { return new BPFPreserveDIType(); } -bool BPFPreserveDIType::runOnModule(Module &M) { +bool BPFPreserveDIType::runOnFunction(Function &F) { LLVM_DEBUG(dbgs() << "********** preserve debuginfo type **********\n"); + M = F.getParent(); + if (!M) + return false; + // Bail out if no debug info. - if (M.debug_compile_units().empty()) + if (M->debug_compile_units().empty()) return false; - return doTransformation(M); + return doTransformation(F); } -bool BPFPreserveDIType::doTransformation(Module &M) { +bool BPFPreserveDIType::doTransformation(Function &F) { std::vector PreserveDITypeCalls; - for (auto &F : M) { - for (auto &BB : F) { - for (auto &I : BB) { - auto *Call = dyn_cast(&I); - if (!Call) - continue; - - const auto *GV = dyn_cast(Call->getCalledOperand()); - if (!GV) - continue; - - if (GV->getName().startswith("llvm.bpf.btf.type.id")) { - if (!Call->getMetadata(LLVMContext::MD_preserve_access_index)) - report_fatal_error( - "Missing metadata for llvm.bpf.btf.type.id intrinsic"); - PreserveDITypeCalls.push_back(Call); - } + for (auto &BB : F) { + for (auto &I : BB) { + auto *Call = dyn_cast(&I); + if (!Call) + continue; + + const auto *GV = dyn_cast(Call->getCalledOperand()); + if (!GV) + continue; + + if (GV->getName().startswith("llvm.bpf.btf.type.id")) { + if (!Call->getMetadata(LLVMContext::MD_preserve_access_index)) + report_fatal_error( + "Missing metadata for llvm.bpf.btf.type.id intrinsic"); + PreserveDITypeCalls.push_back(Call); } } } @@ -119,7 +119,7 @@ std::string GVName = BaseName + std::to_string(Count) + "$" + std::to_string(Reloc); GlobalVariable *GV = - new GlobalVariable(M, VarType, false, GlobalVariable::ExternalLinkage, + new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage, NULL, GVName); GV->addAttribute(BPFCoreSharedInfo::TypeIdAttr); GV->setMetadata(LLVMContext::MD_preserve_access_index, MD); @@ -127,7 +127,9 @@ // Load the global variable which represents the type info. auto *LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV, "", Call); - Call->replaceAllUsesWith(LDInst); + Instruction *PassThroughInst = + BPFCoreSharedInfo::insertPassThrough(M, BB, LDInst, Call); + Call->replaceAllUsesWith(PassThroughInst); Call->eraseFromParent(); Count++; } diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -39,6 +39,7 @@ PassRegistry &PR = *PassRegistry::getPassRegistry(); initializeBPFAbstractMemberAccessPass(PR); initializeBPFPreserveDITypePass(PR); + initializeBPFCheckAndAdjustIRPass(PR); initializeBPFMIPeepholePass(PR); initializeBPFMIPeepholeTruncElimPass(PR); } @@ -98,6 +99,13 @@ } void BPFTargetMachine::adjustPassManager(PassManagerBuilder &Builder) { + Builder.addExtension( + PassManagerBuilder::EP_EarlyAsPossible, + [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) { + PM.add(createBPFAbstractMemberAccess(this)); + PM.add(createBPFPreserveDIType()); + }); + Builder.addExtension( PassManagerBuilder::EP_Peephole, [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -107,9 +115,7 @@ } void BPFPassConfig::addIRPasses() { - addPass(createBPFAbstractMemberAccess(&getBPFTargetMachine())); - addPass(createBPFPreserveDIType()); - + addPass(createBPFCheckAndAdjustIR()); TargetPassConfig::addIRPasses(); } diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -15,6 +15,7 @@ add_llvm_target(BPFCodeGen BPFAbstractMemberAccess.cpp BPFAsmPrinter.cpp + BPFCheckAndAdjustIR.cpp BPFFrameLowering.cpp BPFInstrInfo.cpp BPFISelDAGToDAG.cpp