diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td --- a/llvm/include/llvm/IR/IntrinsicsBPF.td +++ b/llvm/include/llvm/IR/IntrinsicsBPF.td @@ -33,8 +33,14 @@ Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_ptr_ty, llvm_i64_ty], [IntrNoMem]>; def int_bpf_passthrough : ClangBuiltin<"__builtin_bpf_passthrough">, - Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_any_ty], [IntrNoMem]>; + Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_any_ty], [IntrNoMem]> { + bit DisableDefaultAttributes = false; + } def int_bpf_compare : ClangBuiltin<"__builtin_bpf_compare">, Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_anyint_ty, llvm_anyint_ty], [IntrNoMem]>; + def int_bpf_dummy_use : ClangBuiltin<"__builtin_bpf_dummy_use">, + Intrinsic<[], [llvm_ptr_ty], [IntrArgMemOnly, NoCapture>]> { + bit DisableDefaultAttributes = false; + } } diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -72,6 +72,14 @@ public: PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); }; + +class BPFPreventLoadSinking : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + + static bool isRequired() { return true; } +}; + } // namespace llvm #endif diff --git a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp --- a/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp +++ b/llvm/lib/Target/BPF/BPFAbstractMemberAccess.cpp @@ -99,7 +99,7 @@ uint32_t BPFCoreSharedInfo::SeqNum; Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB, - Instruction *Input, + Value *Input, Instruction *Before) { Function *Fn = Intrinsic::getDeclaration( M, Intrinsic::bpf_passthrough, {Input->getType(), Input->getType()}); diff --git a/llvm/lib/Target/BPF/BPFCORE.h b/llvm/lib/Target/BPF/BPFCORE.h --- a/llvm/lib/Target/BPF/BPFCORE.h +++ b/llvm/lib/Target/BPF/BPFCORE.h @@ -10,6 +10,7 @@ #define LLVM_LIB_TARGET_BPF_BPFCORE_H #include "llvm/ADT/StringRef.h" +#include "llvm/IR/Value.h" namespace llvm { @@ -68,8 +69,7 @@ static uint32_t SeqNum; /// Insert a bpf passthrough builtin function. - static Instruction *insertPassThrough(Module *M, BasicBlock *BB, - Instruction *Input, + static Instruction *insertPassThrough(Module *M, BasicBlock *BB, Value *Input, Instruction *Before); }; diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp --- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp +++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp @@ -22,6 +22,7 @@ #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicsBPF.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/User.h" @@ -111,12 +112,18 @@ auto *GV = dyn_cast(Call->getCalledOperand()); if (!GV) continue; - if (!GV->getName().startswith("llvm.bpf.passthrough")) + if (GV->getName().startswith("llvm.bpf.passthrough")) { + Changed = true; + Value *Arg = Call->getArgOperand(1); + Call->replaceAllUsesWith(Arg); + ToBeDeleted = Call; continue; - Changed = true; - Value *Arg = Call->getArgOperand(1); - Call->replaceAllUsesWith(Arg); - ToBeDeleted = Call; + } + if (Call->getIntrinsicID() == Intrinsic::bpf_dummy_use) { + Changed = true; + ToBeDeleted = Call; + continue; + } } return Changed; } diff --git a/llvm/lib/Target/BPF/BPFPreventLoadSinking.cpp b/llvm/lib/Target/BPF/BPFPreventLoadSinking.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Target/BPF/BPFPreventLoadSinking.cpp @@ -0,0 +1,141 @@ +//===------------ BPFPreventLoadSinking.cpp -------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// +//===----------------------------------------------------------------------===// + +#include "BPF.h" +#include "BPFCORE.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/IntrinsicsBPF.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Support/Casting.h" + +#define DEBUG_TYPE "bpf-prevent-load-sinking" + +using namespace llvm; + +namespace { +// This is a simplified version of LockstepReverseIterator from SimplifyCFG +class LockstepReverseIterator { + SmallVector Insts; + bool Fail = false; + +public: + LockstepReverseIterator(ArrayRef Blocks) { + for (auto *BB : Blocks) { + Instruction *Inst = BB->getTerminator(); + for (Inst = Inst->getPrevNode(); isa_and_nonnull(Inst);) + Inst = Inst->getPrevNode(); + if (!Inst) { + // Block wasn't big enough. + Fail = true; + return; + } + Insts.push_back(Inst); + } + } + + void operator--() { + if (Fail) + return; + for (auto *&Inst : Insts) { + for (Inst = Inst->getPrevNode(); isa_and_nonnull(Inst);) + Inst = Inst->getPrevNode(); + // Already at beginning of block. + if (!Inst) { + Fail = true; + return; + } + } + } + + ArrayRef operator*() const { return Insts; } + + bool isValid() { return !Fail; } +}; +} // anonymous namespace + +static bool maybeSunk(ArrayRef Insts) { + Instruction *I0 = Insts[0]; + for (auto *I : Insts) + if (!I->isSameOperationAs(I0)) + return false; + return true; +} + +// Scan unconditional basic block predecessors looking for +// instructions that might get sunk by SimplifyCFG. +// +// If load or store could get sunk add a call to llvm.bpf.passthrough +// or llvm.bpf.dummy.use in order to prevent sinking. +// +// It is hard to determine for sure if particular instruction would be +// sunk w/o repeating 90% of logic from SimplifyCFG, so this +// transformation has to be conservative. +static bool maybePatchBB(BasicBlock *BB) { + SmallVector UnconditionalPreds; + for (auto *PredBB : predecessors(BB)) { + auto *PredBr = dyn_cast(PredBB->getTerminator()); + if (PredBr && PredBr->isUnconditional()) + UnconditionalPreds.push_back(PredBB); + } + if (UnconditionalPreds.size() < 2) + return false; + + auto *M = BB->getModule(); + + LockstepReverseIterator LRI(UnconditionalPreds); + while (LRI.isValid() && maybeSunk(*LRI)) { + Instruction *I0 = (*LRI)[0]; + auto SameAsI0 = [=](const Instruction *I) { + for (unsigned OpNum = 0; OpNum < I0->getNumOperands(); ++OpNum) + if (I0->getOperand(OpNum) != I->getOperand(OpNum)) + return false; + return true; + }; + if (!all_of(*LRI, SameAsI0)) { + if (isa(I0)) { + auto *PassThrough = BPFCoreSharedInfo::insertPassThrough( + M, I0->getParent(), I0, I0->getNextNode()); + I0->replaceUsesWithIf(PassThrough, [=](auto &Use) { + return Use.getUser() != PassThrough; + }); + return true; + } + if (auto *Store = dyn_cast(I0)) { + Function *Fn = + Intrinsic::getDeclaration(M, Intrinsic::bpf_dummy_use, {}); + auto *DummyUse = CallInst::Create(Fn, {Store->getPointerOperand()}, "", + Store->getNextNode()); + DummyUse->setAAMetadata(Store->getAAMetadata()); + return true; + } + } + --LRI; + } + return false; +} + +static bool preventLoadSinking(Function &F) { + bool Changed = false; + for (auto &BB : F) + Changed |= maybePatchBB(&BB); + return Changed; +} + +PreservedAnalyses BPFPreventLoadSinking::run(Function &F, + FunctionAnalysisManager &AM) { + return preventLoadSinking(F) ? PreservedAnalyses::none() + : PreservedAnalyses::all(); +} diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -34,6 +34,9 @@ opt DisableMIPeephole("disable-bpf-peephole", cl::Hidden, cl::desc("Disable machine peepholes for BPF")); +static cl::opt DisablePreventLoadSinking("disable-prevent-load-sinking", + cl::Hidden, cl::desc("")); + extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() { // Register the target. RegisterTargetMachine X(getTheBPFleTarget()); @@ -137,6 +140,11 @@ OptimizationLevel Level) { FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true))); }); + PB.registerScalarOptimizerLateEPCallback( + [=](FunctionPassManager &FPM, OptimizationLevel Level) { + if (!DisablePreventLoadSinking) + FPM.addPass(BPFPreventLoadSinking()); + }); PB.registerPipelineEarlySimplificationEPCallback( [=](ModulePassManager &MPM, OptimizationLevel) { MPM.addPass(BPFAdjustOptPass()); diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -26,6 +26,7 @@ BPFISelLowering.cpp BPFMCInstLower.cpp BPFPreserveDIType.cpp + BPFPreventLoadSinking.cpp BPFRegisterInfo.cpp BPFSelectionDAGInfo.cpp BPFSubtarget.cpp