diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -15,9 +15,8 @@ namespace llvm { class BPFTargetMachine; -ModulePass *createBPFAbstractMemberAccess(BPFTargetMachine *TM); -ModulePass *createBPFPreserveDIType(); - +FunctionPass *createBPFAbstractMemberAccess(BPFTargetMachine *TM); +FunctionPass *createBPFPreserveDIType(); FunctionPass *createBPFISelDag(BPFTargetMachine &TM); FunctionPass *createBPFMISimplifyPatchablePass(); FunctionPass *createBPFMIPeepholePass(); diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp --- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp +++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp @@ -99,12 +99,8 @@ namespace { -class BPFAbstractMemberAccess final : public ModulePass { - StringRef getPassName() const override { - return "BPF Abstract Member Access"; - } - - bool runOnModule(Module &M) override; +class BPFAbstractMemberAccess final : public FunctionPass { + bool runOnFunction(Function &F) override; public: static char ID; @@ -112,7 +108,7 @@ // Add optional BPFTargetMachine parameter so that BPF backend can add the phase // with target machine to find out the endianness. The default constructor (without // parameters) is used by the pass manager for managing purposes. - BPFAbstractMemberAccess(BPFTargetMachine *TM = nullptr) : ModulePass(ID), TM(TM) {} + BPFAbstractMemberAccess(BPFTargetMachine *TM = nullptr) : FunctionPass(ID), TM(TM) {} struct CallInfo { uint32_t Kind; @@ -132,6 +128,7 @@ }; const DataLayout *DL = nullptr; + Module *M = nullptr; std::map GEPGlobals; // A map to link preserve_*_access_index instrinsic calls. @@ -141,19 +138,19 @@ // intrinsics. std::map BaseAICalls; - bool doTransformation(Module &M); + bool doTransformation(Function &F); void traceAICall(CallInst *Call, CallInfo &ParentInfo); void traceBitCast(BitCastInst *BitCast, CallInst *Parent, CallInfo &ParentInfo); void traceGEP(GetElementPtrInst *GEP, CallInst *Parent, CallInfo &ParentInfo); - void collectAICallChains(Module &M, Function &F); + void collectAICallChains(Function &F); bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo); bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI, const MDNode *ChildMeta); - bool removePreserveAccessIndexIntrinsic(Module &M); + bool removePreserveAccessIndexIntrinsic(Function &F); void replaceWithGEP(std::vector &CallList, uint32_t NumOfZerosIndex, uint32_t DIIndex); bool HasPreserveFieldInfoCall(CallInfoStack &CallStack); @@ -168,27 +165,32 @@ MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo, std::string &AccessKey, bool &IsInt32Ret); uint64_t getConstant(const Value *IndexValue); - bool transformGEPChain(Module &M, CallInst *Call, CallInfo &CInfo); + bool transformGEPChain(CallInst *Call, CallInfo &CInfo); + void insertMemoryBarrier(BasicBlock *BB); }; } // End anonymous namespace char BPFAbstractMemberAccess::ID = 0; INITIALIZE_PASS(BPFAbstractMemberAccess, DEBUG_TYPE, - "abstracting struct/union member accessees", false, false) + "BPF Abstract Member Access", false, false) -ModulePass *llvm::createBPFAbstractMemberAccess(BPFTargetMachine *TM) { +FunctionPass *llvm::createBPFAbstractMemberAccess(BPFTargetMachine *TM) { return new BPFAbstractMemberAccess(TM); } -bool BPFAbstractMemberAccess::runOnModule(Module &M) { +bool BPFAbstractMemberAccess::runOnFunction(Function &F) { LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n"); + M = F.getParent(); + if (!M) + return false; + // Bail out if no debug info. - if (M.debug_compile_units().empty()) + if (M->debug_compile_units().empty()) return false; - DL = &M.getDataLayout(); - return doTransformation(M); + DL = &M->getDataLayout(); + return doTransformation(F); } static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) { @@ -341,28 +343,27 @@ } } -bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Module &M) { +bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) { std::vector PreserveArrayIndexCalls; std::vector PreserveUnionIndexCalls; std::vector PreserveStructIndexCalls; bool Found = false; - for (Function &F : M) - for (auto &BB : F) - for (auto &I : BB) { - auto *Call = dyn_cast(&I); - CallInfo CInfo; - if (!IsPreserveDIAccessIndexCall(Call, CInfo)) - continue; - - Found = true; - if (CInfo.Kind == BPFPreserveArrayAI) - PreserveArrayIndexCalls.push_back(Call); - else if (CInfo.Kind == BPFPreserveUnionAI) - PreserveUnionIndexCalls.push_back(Call); - else - PreserveStructIndexCalls.push_back(Call); - } + for (auto &BB : F) + for (auto &I : BB) { + auto *Call = dyn_cast(&I); + CallInfo CInfo; + if (!IsPreserveDIAccessIndexCall(Call, CInfo)) + continue; + + Found = true; + if (CInfo.Kind == BPFPreserveArrayAI) + PreserveArrayIndexCalls.push_back(Call); + else if (CInfo.Kind == BPFPreserveUnionAI) + PreserveUnionIndexCalls.push_back(Call); + else + PreserveStructIndexCalls.push_back(Call); + } // do the following transformation: // . addr = preserve_array_access_index(base, dimension, index) @@ -528,7 +529,7 @@ } } -void BPFAbstractMemberAccess::collectAICallChains(Module &M, Function &F) { +void BPFAbstractMemberAccess::collectAICallChains(Function &F) { AIChain.clear(); BaseAICalls.clear(); @@ -936,9 +937,66 @@ return Ty; } +/// For the following code, +/// Block0: +/// ... +/// if (...) goto Block1 else ... +/// Block1: +/// %6 = load llvm.sk_buff:50:$0:0:0:2:0 +/// %7 = bitcast %struct.sk_buff* %2 to i8* +/// %8 = getelementptr i8, i8* %7, %6 +/// ... +/// goto CommonExit +/// Block2: +/// ... +/// if (...) goto Block3 else ... +/// Block3: +/// %6 = load llvm.bpf_map:40:$0:0:0:2:0 +/// %7 = bitcast %struct.sk_buff* %2 to i8* +/// %8 = getelementptr i8, i8* %7, %6 +/// ... +/// goto CommonExit +/// CommonExit +/// SimplifyCFG may generate: +/// Block0: +/// ... +/// if (...) goto Block_Common else ... +// Block2: +// ... +/// if (...) goto Block_Common else ... +/// Block_Common: +/// PHI = [llvm.sk_buff:50:$0:0:0:2:0, llvm.bpf_map:40:$0:0:0:2:0] +/// %6 = load PHI +/// %7 = bitcast %struct.sk_buff* %2 to i8* +/// %8 = getelementptr i8, i8* %7, %6 +/// ... +/// goto CommonExit +/// For the above code, we cannot perform proper relocation since +/// "load PHI" has two possible relocations. +/// +/// Current SimplifyCFG has a couple of ways to prevent the above +/// transformation: +/// - insert an inline asm in the basic block +/// - insert a call and mark it with NoMerge function attribute. +/// We use the inline asm approach and inserted one memory barrier +/// at the end of the basic block to prevent the optimization. +void BPFAbstractMemberAccess::insertMemoryBarrier(BasicBlock *BB) { + Instruction *TermInst = BB->getTerminator(); + auto *Call = dyn_cast(TermInst->getPrevNonDebugInstruction()); + if (Call && Call->isInlineAsm()) + return; + + std::vector ArgTypes; + std::vector Args; + FunctionType *AsmFTy = FunctionType::get(Type::getVoidTy(BB->getContext()), ArgTypes, false); + InlineAsm *Asm = InlineAsm::get(AsmFTy, StringRef(""), StringRef("~{memory}"), true); + auto *Barrier = CallInst::Create(Asm, Args); + BB->getInstList().insert(TermInst->getIterator(), Barrier); +} + /// Call/Kind is the base preserve_*_access_index() call. Attempts to do /// transformation to a chain of relocable GEPs. -bool BPFAbstractMemberAccess::transformGEPChain(Module &M, CallInst *Call, +bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call, CallInfo &CInfo) { std::string AccessKey; MDNode *TypeMeta; @@ -964,7 +1022,7 @@ else VarType = Type::getInt64Ty(BB->getContext()); // 64bit ptr or enum value - GV = new GlobalVariable(M, VarType, false, GlobalVariable::ExternalLinkage, + GV = new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage, NULL, AccessKey); GV->addAttribute(BPFCoreSharedInfo::AmaAttr); GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta); @@ -982,13 +1040,14 @@ LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", Call); Call->replaceAllUsesWith(LDInst); Call->eraseFromParent(); + insertMemoryBarrier(BB); return true; } // For any original GEP Call and Base %2 like // %4 = bitcast %struct.net_device** %dev1 to i64* // it is transformed to: - // %6 = load sk_buff:50:$0:0:0:2:0 + // %6 = load llvm.sk_buff:50:$0:0:0:2:0 // %7 = bitcast %struct.sk_buff* %2 to i8* // %8 = getelementptr i8, i8* %7, %6 // %9 = bitcast i8* %8 to i64* @@ -1013,22 +1072,21 @@ Call->replaceAllUsesWith(BCInst2); Call->eraseFromParent(); + insertMemoryBarrier(BB); return true; } -bool BPFAbstractMemberAccess::doTransformation(Module &M) { +bool BPFAbstractMemberAccess::doTransformation(Function &F) { bool Transformed = false; - for (Function &F : M) { - // Collect PreserveDIAccessIndex Intrinsic call chains. - // The call chains will be used to generate the access - // patterns similar to GEP. - collectAICallChains(M, F); + // Collect PreserveDIAccessIndex Intrinsic call chains. + // The call chains will be used to generate the access + // patterns similar to GEP. + collectAICallChains(F); - for (auto &C : BaseAICalls) - Transformed = transformGEPChain(M, C.first, C.second) || Transformed; - } + for (auto &C : BaseAICalls) + Transformed = transformGEPChain(C.first, C.second) || Transformed; - return removePreserveAccessIndexIntrinsic(M) || Transformed; + return removePreserveAccessIndexIntrinsic(F) || Transformed; } diff --git a/llvm/lib/Target/BPF/BPFPreserveDIType.cpp b/llvm/lib/Target/BPF/BPFPreserveDIType.cpp --- a/llvm/lib/Target/BPF/BPFPreserveDIType.cpp +++ b/llvm/lib/Target/BPF/BPFPreserveDIType.cpp @@ -12,6 +12,7 @@ #include "BPF.h" #include "BPFCORE.h" +#include "BPFTargetMachine.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instruction.h" @@ -33,58 +34,73 @@ namespace { -class BPFPreserveDIType final : public ModulePass { - StringRef getPassName() const override { - return "BPF Preserve DebugInfo Type"; - } - - bool runOnModule(Module &M) override; +class BPFPreserveDIType final : public FunctionPass { + bool runOnFunction(Function &F) override; public: static char ID; - BPFPreserveDIType() : ModulePass(ID) {} + BPFPreserveDIType() : FunctionPass(ID) {} private: - bool doTransformation(Module &M); + Module *M = nullptr; + + bool doTransformation(Function &F); + void insertMemoryBarrier(BasicBlock *BB); }; } // End anonymous namespace char BPFPreserveDIType::ID = 0; -INITIALIZE_PASS(BPFPreserveDIType, DEBUG_TYPE, "preserve debuginfo type", false, - false) +INITIALIZE_PASS(BPFPreserveDIType, DEBUG_TYPE, "BPF Preserve Debuginfo Type", + false, false) -ModulePass *llvm::createBPFPreserveDIType() { return new BPFPreserveDIType(); } +FunctionPass *llvm::createBPFPreserveDIType() { return new BPFPreserveDIType(); } -bool BPFPreserveDIType::runOnModule(Module &M) { +bool BPFPreserveDIType::runOnFunction(Function &F) { LLVM_DEBUG(dbgs() << "********** preserve debuginfo type **********\n"); + M = F.getParent(); + if (!M) + return false; + // Bail out if no debug info. - if (M.debug_compile_units().empty()) + if (M->debug_compile_units().empty()) return false; - return doTransformation(M); + return doTransformation(F); +} + +void BPFPreserveDIType::insertMemoryBarrier(BasicBlock *BB) { + Instruction *TermInst = BB->getTerminator(); + auto *Call = dyn_cast(TermInst->getPrevNonDebugInstruction()); + if (Call && Call->isInlineAsm()) + return; + + std::vector ArgTypes; + std::vector Args; + FunctionType *AsmFTy = FunctionType::get(Type::getVoidTy(BB->getContext()), ArgTypes, false); + InlineAsm *Asm = InlineAsm::get(AsmFTy, StringRef(""), StringRef("~{memory}"), true); + auto *Barrier = CallInst::Create(Asm, Args); + BB->getInstList().insert(TermInst->getIterator(), Barrier); } -bool BPFPreserveDIType::doTransformation(Module &M) { +bool BPFPreserveDIType::doTransformation(Function &F) { std::vector PreserveDITypeCalls; - for (auto &F : M) { - for (auto &BB : F) { - for (auto &I : BB) { - auto *Call = dyn_cast(&I); - if (!Call) - continue; - - const auto *GV = dyn_cast(Call->getCalledOperand()); - if (!GV) - continue; - - if (GV->getName().startswith("llvm.bpf.btf.type.id")) { - if (!Call->getMetadata(LLVMContext::MD_preserve_access_index)) - report_fatal_error( - "Missing metadata for llvm.bpf.btf.type.id intrinsic"); - PreserveDITypeCalls.push_back(Call); - } + for (auto &BB : F) { + for (auto &I : BB) { + auto *Call = dyn_cast(&I); + if (!Call) + continue; + + const auto *GV = dyn_cast(Call->getCalledOperand()); + if (!GV) + continue; + + if (GV->getName().startswith("llvm.bpf.btf.type.id")) { + if (!Call->getMetadata(LLVMContext::MD_preserve_access_index)) + report_fatal_error( + "Missing metadata for llvm.bpf.btf.type.id intrinsic"); + PreserveDITypeCalls.push_back(Call); } } } @@ -119,7 +135,7 @@ std::string GVName = BaseName + std::to_string(Count) + "$" + std::to_string(Reloc); GlobalVariable *GV = - new GlobalVariable(M, VarType, false, GlobalVariable::ExternalLinkage, + new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage, NULL, GVName); GV->addAttribute(BPFCoreSharedInfo::TypeIdAttr); GV->setMetadata(LLVMContext::MD_preserve_access_index, MD); @@ -129,6 +145,7 @@ Call); Call->replaceAllUsesWith(LDInst); Call->eraseFromParent(); + insertMemoryBarrier(BB); Count++; } diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -98,6 +98,13 @@ } void BPFTargetMachine::adjustPassManager(PassManagerBuilder &Builder) { + Builder.addExtension( + PassManagerBuilder::EP_EarlyAsPossible, + [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) { + PM.add(createBPFAbstractMemberAccess(this)); + PM.add(createBPFPreserveDIType()); + }); + Builder.addExtension( PassManagerBuilder::EP_Peephole, [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -107,9 +114,6 @@ } void BPFPassConfig::addIRPasses() { - addPass(createBPFAbstractMemberAccess(&getBPFTargetMachine())); - addPass(createBPFPreserveDIType()); - TargetPassConfig::addIRPasses(); }