diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1649,6 +1649,16 @@ /// false, but it shouldn't matter what it returns anyway. bool hasArmWideBranch(bool Thumb) const; + /// \returns True if the target wants to fold AndOr ICmp comparisons using + /// ranges. + bool shouldFoldAndOrOfICmpsUsingRanges() const; + + /// \returns True if the target wants to fold two entry PHI nodes. + bool shouldFoldTwoEntryPHINode() const; + + /// \returns True if the target wants to hoist min/max operations in LICM.. + bool shouldHoistMinMax() const; + /// @} private: @@ -2010,6 +2020,9 @@ virtual VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const = 0; virtual bool hasArmWideBranch(bool Thumb) const = 0; + virtual bool shouldFoldAndOrOfICmpsUsingRanges() const = 0; + virtual bool shouldFoldTwoEntryPHINode() const = 0; + virtual bool shouldHoistMinMax() const = 0; }; template @@ -2704,6 +2717,18 @@ bool hasArmWideBranch(bool Thumb) const override { return Impl.hasArmWideBranch(Thumb); } + + bool shouldFoldAndOrOfICmpsUsingRanges() const override { + return Impl.shouldFoldAndOrOfICmpsUsingRanges(); + } + + bool shouldFoldTwoEntryPHINode() const override { + return Impl.shouldFoldTwoEntryPHINode(); + } + + bool shouldHoistMinMax() const override { + return Impl.shouldHoistMinMax(); + } }; template diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -880,6 +880,12 @@ bool hasArmWideBranch(bool) const { return false; } + bool shouldFoldAndOrOfICmpsUsingRanges() const { return true; } + + bool shouldFoldTwoEntryPHINode() const { return true; } + + bool shouldHoistMinMax() const { return true; } + protected: // Obtain the minimum required size to hold the value (without the sign) // In case of a vector it returns the min required size for one element. diff --git a/llvm/include/llvm/IR/IntrinsicsBPF.td b/llvm/include/llvm/IR/IntrinsicsBPF.td --- a/llvm/include/llvm/IR/IntrinsicsBPF.td +++ b/llvm/include/llvm/IR/IntrinsicsBPF.td @@ -34,7 +34,4 @@ [IntrNoMem]>; def int_bpf_passthrough : ClangBuiltin<"__builtin_bpf_passthrough">, Intrinsic<[llvm_any_ty], [llvm_i32_ty, llvm_any_ty], [IntrNoMem]>; - def int_bpf_compare : ClangBuiltin<"__builtin_bpf_compare">, - Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_anyint_ty, llvm_anyint_ty], - [IntrNoMem]>; } diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h --- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h @@ -43,6 +43,7 @@ /// This class provides both the logic to recursively visit instructions and /// combine them. class LLVM_LIBRARY_VISIBILITY InstCombiner { +protected: /// Only used to call target specific intrinsic combining. /// It must **NOT** be used for any other purpose, as InstCombine is a /// target-independent canonicalization transform. diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -169,10 +169,10 @@ /// \p AllowSpeculation is whether values should be hoisted even if they are not /// guaranteed to execute in the loop, but are safe to speculatively execute. bool hoistRegion(DomTreeNode *, AAResults *, LoopInfo *, DominatorTree *, - AssumptionCache *, TargetLibraryInfo *, Loop *, - MemorySSAUpdater &, ScalarEvolution *, ICFLoopSafetyInfo *, - SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *, bool, - bool AllowSpeculation); + AssumptionCache *, TargetLibraryInfo *, TargetTransformInfo *, + Loop *, MemorySSAUpdater &, ScalarEvolution *, + ICFLoopSafetyInfo *, SinkAndHoistLICMFlags &, + OptimizationRemarkEmitter *, bool, bool AllowSpeculation); /// Return true if the induction variable \p IV in a Loop whose latch is /// \p LatchBlock would become dead if the exit test \p Cond were removed. diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -1198,6 +1198,18 @@ return TTIImpl->hasArmWideBranch(Thumb); } +bool TargetTransformInfo::shouldFoldAndOrOfICmpsUsingRanges() const { + return TTIImpl->shouldFoldAndOrOfICmpsUsingRanges(); +} + +bool TargetTransformInfo::shouldFoldTwoEntryPHINode() const { + return TTIImpl->shouldFoldTwoEntryPHINode(); +} + +bool TargetTransformInfo::shouldHoistMinMax() const { + return TTIImpl->shouldHoistMinMax(); +} + bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const { return TTIImpl->shouldExpandReduction(II); } diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -18,7 +18,6 @@ class BPFTargetMachine; class PassRegistry; -ModulePass *createBPFAdjustOpt(); ModulePass *createBPFCheckAndAdjustIR(); FunctionPass *createBPFAbstractMemberAccess(BPFTargetMachine *TM); @@ -32,7 +31,6 @@ FunctionPass *createBPFMIPreEmitCheckingPass(); void initializeBPFAbstractMemberAccessLegacyPassPass(PassRegistry &); -void initializeBPFAdjustOptPass(PassRegistry&); void initializeBPFCheckAndAdjustIRPass(PassRegistry&); void initializeBPFDAGToDAGISelPass(PassRegistry &); void initializeBPFIRPeepholePass(PassRegistry &); @@ -67,11 +65,6 @@ static bool isRequired() { return true; } }; - -class BPFAdjustOptPass : public PassInfoMixin { -public: - PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); -}; } // namespace llvm #endif diff --git a/llvm/lib/Target/BPF/BPFAdjustOpt.cpp b/llvm/lib/Target/BPF/BPFAdjustOpt.cpp deleted file mode 100644 --- a/llvm/lib/Target/BPF/BPFAdjustOpt.cpp +++ /dev/null @@ -1,393 +0,0 @@ -//===---------------- BPFAdjustOpt.cpp - Adjust Optimization --------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Adjust optimization to make the code more kernel verifier friendly. -// -//===----------------------------------------------------------------------===// - -#include "BPF.h" -#include "BPFCORE.h" -#include "BPFTargetMachine.h" -#include "llvm/IR/Instruction.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/IntrinsicsBPF.h" -#include "llvm/IR/Module.h" -#include "llvm/IR/PatternMatch.h" -#include "llvm/IR/Type.h" -#include "llvm/IR/User.h" -#include "llvm/IR/Value.h" -#include "llvm/Pass.h" -#include "llvm/Transforms/Utils/BasicBlockUtils.h" - -#define DEBUG_TYPE "bpf-adjust-opt" - -using namespace llvm; -using namespace llvm::PatternMatch; - -static cl::opt - DisableBPFserializeICMP("bpf-disable-serialize-icmp", cl::Hidden, - cl::desc("BPF: Disable Serializing ICMP insns."), - cl::init(false)); - -static cl::opt DisableBPFavoidSpeculation( - "bpf-disable-avoid-speculation", cl::Hidden, - cl::desc("BPF: Disable Avoiding Speculative Code Motion."), - cl::init(false)); - -namespace { - -class BPFAdjustOpt final : public ModulePass { -public: - static char ID; - - BPFAdjustOpt() : ModulePass(ID) {} - bool runOnModule(Module &M) override; -}; - -class BPFAdjustOptImpl { - struct PassThroughInfo { - Instruction *Input; - Instruction *UsedInst; - uint32_t OpIdx; - PassThroughInfo(Instruction *I, Instruction *U, uint32_t Idx) - : Input(I), UsedInst(U), OpIdx(Idx) {} - }; - -public: - BPFAdjustOptImpl(Module *M) : M(M) {} - - bool run(); - -private: - Module *M; - SmallVector PassThroughs; - - bool adjustICmpToBuiltin(); - void adjustBasicBlock(BasicBlock &BB); - bool serializeICMPCrossBB(BasicBlock &BB); - void adjustInst(Instruction &I); - bool serializeICMPInBB(Instruction &I); - bool avoidSpeculation(Instruction &I); - bool insertPassThrough(); -}; - -} // End anonymous namespace - -char BPFAdjustOpt::ID = 0; -INITIALIZE_PASS(BPFAdjustOpt, "bpf-adjust-opt", "BPF Adjust Optimization", - false, false) - -ModulePass *llvm::createBPFAdjustOpt() { return new BPFAdjustOpt(); } - -bool BPFAdjustOpt::runOnModule(Module &M) { return BPFAdjustOptImpl(&M).run(); } - -bool BPFAdjustOptImpl::run() { - bool Changed = adjustICmpToBuiltin(); - - for (Function &F : *M) - for (auto &BB : F) { - adjustBasicBlock(BB); - for (auto &I : BB) - adjustInst(I); - } - return insertPassThrough() || Changed; -} - -// Commit acabad9ff6bf ("[InstCombine] try to canonicalize icmp with -// trunc op into mask and cmp") added a transformation to -// convert "(conv)a < power_2_const" to "a & " in certain -// cases and bpf kernel verifier has to handle the resulted code -// conservatively and this may reject otherwise legitimate program. -// Here, we change related icmp code to a builtin which will -// be restored to original icmp code later to prevent that -// InstCombine transformatin. -bool BPFAdjustOptImpl::adjustICmpToBuiltin() { - bool Changed = false; - ICmpInst *ToBeDeleted = nullptr; - for (Function &F : *M) - for (auto &BB : F) - for (auto &I : BB) { - if (ToBeDeleted) { - ToBeDeleted->eraseFromParent(); - ToBeDeleted = nullptr; - } - - auto *Icmp = dyn_cast(&I); - if (!Icmp) - continue; - - Value *Op0 = Icmp->getOperand(0); - if (!isa(Op0)) - continue; - - auto ConstOp1 = dyn_cast(Icmp->getOperand(1)); - if (!ConstOp1) - continue; - - auto ConstOp1Val = ConstOp1->getValue().getZExtValue(); - auto Op = Icmp->getPredicate(); - if (Op == ICmpInst::ICMP_ULT || Op == ICmpInst::ICMP_UGE) { - if ((ConstOp1Val - 1) & ConstOp1Val) - continue; - } else if (Op == ICmpInst::ICMP_ULE || Op == ICmpInst::ICMP_UGT) { - if (ConstOp1Val & (ConstOp1Val + 1)) - continue; - } else { - continue; - } - - Constant *Opcode = - ConstantInt::get(Type::getInt32Ty(BB.getContext()), Op); - Function *Fn = Intrinsic::getDeclaration( - M, Intrinsic::bpf_compare, {Op0->getType(), ConstOp1->getType()}); - auto *NewInst = CallInst::Create(Fn, {Opcode, Op0, ConstOp1}); - NewInst->insertBefore(&I); - Icmp->replaceAllUsesWith(NewInst); - Changed = true; - ToBeDeleted = Icmp; - } - - return Changed; -} - -bool BPFAdjustOptImpl::insertPassThrough() { - for (auto &Info : PassThroughs) { - auto *CI = BPFCoreSharedInfo::insertPassThrough( - M, Info.UsedInst->getParent(), Info.Input, Info.UsedInst); - Info.UsedInst->setOperand(Info.OpIdx, CI); - } - - return !PassThroughs.empty(); -} - -// To avoid combining conditionals in the same basic block by -// instrcombine optimization. -bool BPFAdjustOptImpl::serializeICMPInBB(Instruction &I) { - // For: - // comp1 = icmp ...; - // comp2 = icmp ...; - // ... or comp1 comp2 ... - // changed to: - // comp1 = icmp ...; - // comp2 = icmp ...; - // new_comp1 = __builtin_bpf_passthrough(seq_num, comp1) - // ... or new_comp1 comp2 ... - Value *Op0, *Op1; - // Use LogicalOr (accept `or i1` as well as `select i1 Op0, true, Op1`) - if (!match(&I, m_LogicalOr(m_Value(Op0), m_Value(Op1)))) - return false; - auto *Icmp1 = dyn_cast(Op0); - if (!Icmp1) - return false; - auto *Icmp2 = dyn_cast(Op1); - if (!Icmp2) - return false; - - Value *Icmp1Op0 = Icmp1->getOperand(0); - Value *Icmp2Op0 = Icmp2->getOperand(0); - if (Icmp1Op0 != Icmp2Op0) - return false; - - // Now we got two icmp instructions which feed into - // an "or" instruction. - PassThroughInfo Info(Icmp1, &I, 0); - PassThroughs.push_back(Info); - return true; -} - -// To avoid combining conditionals in the same basic block by -// instrcombine optimization. -bool BPFAdjustOptImpl::serializeICMPCrossBB(BasicBlock &BB) { - // For: - // B1: - // comp1 = icmp ...; - // if (comp1) goto B2 else B3; - // B2: - // comp2 = icmp ...; - // if (comp2) goto B4 else B5; - // B4: - // ... - // changed to: - // B1: - // comp1 = icmp ...; - // comp1 = __builtin_bpf_passthrough(seq_num, comp1); - // if (comp1) goto B2 else B3; - // B2: - // comp2 = icmp ...; - // if (comp2) goto B4 else B5; - // B4: - // ... - - // Check basic predecessors, if two of them (say B1, B2) are using - // icmp instructions to generate conditions and one is the predesessor - // of another (e.g., B1 is the predecessor of B2). Add a passthrough - // barrier after icmp inst of block B1. - BasicBlock *B2 = BB.getSinglePredecessor(); - if (!B2) - return false; - - BasicBlock *B1 = B2->getSinglePredecessor(); - if (!B1) - return false; - - Instruction *TI = B2->getTerminator(); - auto *BI = dyn_cast(TI); - if (!BI || !BI->isConditional()) - return false; - auto *Cond = dyn_cast(BI->getCondition()); - if (!Cond || B2->getFirstNonPHI() != Cond) - return false; - Value *B2Op0 = Cond->getOperand(0); - auto Cond2Op = Cond->getPredicate(); - - TI = B1->getTerminator(); - BI = dyn_cast(TI); - if (!BI || !BI->isConditional()) - return false; - Cond = dyn_cast(BI->getCondition()); - if (!Cond) - return false; - Value *B1Op0 = Cond->getOperand(0); - auto Cond1Op = Cond->getPredicate(); - - if (B1Op0 != B2Op0) - return false; - - if (Cond1Op == ICmpInst::ICMP_SGT || Cond1Op == ICmpInst::ICMP_SGE) { - if (Cond2Op != ICmpInst::ICMP_SLT && Cond2Op != ICmpInst::ICMP_SLE) - return false; - } else if (Cond1Op == ICmpInst::ICMP_SLT || Cond1Op == ICmpInst::ICMP_SLE) { - if (Cond2Op != ICmpInst::ICMP_SGT && Cond2Op != ICmpInst::ICMP_SGE) - return false; - } else if (Cond1Op == ICmpInst::ICMP_ULT || Cond1Op == ICmpInst::ICMP_ULE) { - if (Cond2Op != ICmpInst::ICMP_UGT && Cond2Op != ICmpInst::ICMP_UGE) - return false; - } else if (Cond1Op == ICmpInst::ICMP_UGT || Cond1Op == ICmpInst::ICMP_UGE) { - if (Cond2Op != ICmpInst::ICMP_ULT && Cond2Op != ICmpInst::ICMP_ULE) - return false; - } else { - return false; - } - - PassThroughInfo Info(Cond, BI, 0); - PassThroughs.push_back(Info); - - return true; -} - -// To avoid speculative hoisting certain computations out of -// a basic block. -bool BPFAdjustOptImpl::avoidSpeculation(Instruction &I) { - if (auto *LdInst = dyn_cast(&I)) { - if (auto *GV = dyn_cast(LdInst->getOperand(0))) { - if (GV->hasAttribute(BPFCoreSharedInfo::AmaAttr) || - GV->hasAttribute(BPFCoreSharedInfo::TypeIdAttr)) - return false; - } - } - - if (!isa(&I) && !isa(&I)) - return false; - - // For: - // B1: - // var = ... - // ... - // /* icmp may not be in the same block as var = ... */ - // comp1 = icmp var, ; - // if (comp1) goto B2 else B3; - // B2: - // ... var ... - // change to: - // B1: - // var = ... - // ... - // /* icmp may not be in the same block as var = ... */ - // comp1 = icmp var, ; - // if (comp1) goto B2 else B3; - // B2: - // var = __builtin_bpf_passthrough(seq_num, var); - // ... var ... - bool isCandidate = false; - SmallVector Candidates; - for (User *U : I.users()) { - Instruction *Inst = dyn_cast(U); - if (!Inst) - continue; - - // May cover a little bit more than the - // above pattern. - if (auto *Icmp1 = dyn_cast(Inst)) { - Value *Icmp1Op1 = Icmp1->getOperand(1); - if (!isa(Icmp1Op1)) - return false; - isCandidate = true; - continue; - } - - // Ignore the use in the same basic block as the definition. - if (Inst->getParent() == I.getParent()) - continue; - - // use in a different basic block, If there is a call or - // load/store insn before this instruction in this basic - // block. Most likely it cannot be hoisted out. Skip it. - for (auto &I2 : *Inst->getParent()) { - if (isa(&I2)) - return false; - if (isa(&I2) || isa(&I2)) - return false; - if (&I2 == Inst) - break; - } - - // It should be used in a GEP or a simple arithmetic like - // ZEXT/SEXT which is used for GEP. - if (Inst->getOpcode() == Instruction::ZExt || - Inst->getOpcode() == Instruction::SExt) { - PassThroughInfo Info(&I, Inst, 0); - Candidates.push_back(Info); - } else if (auto *GI = dyn_cast(Inst)) { - // traverse GEP inst to find Use operand index - unsigned i, e; - for (i = 1, e = GI->getNumOperands(); i != e; ++i) { - Value *V = GI->getOperand(i); - if (V == &I) - break; - } - if (i == e) - continue; - - PassThroughInfo Info(&I, GI, i); - Candidates.push_back(Info); - } - } - - if (!isCandidate || Candidates.empty()) - return false; - - llvm::append_range(PassThroughs, Candidates); - return true; -} - -void BPFAdjustOptImpl::adjustBasicBlock(BasicBlock &BB) { - if (!DisableBPFserializeICMP && serializeICMPCrossBB(BB)) - return; -} - -void BPFAdjustOptImpl::adjustInst(Instruction &I) { - if (!DisableBPFserializeICMP && serializeICMPInBB(I)) - return; - if (!DisableBPFavoidSpeculation && avoidSpeculation(I)) - return; -} - -PreservedAnalyses BPFAdjustOptPass::run(Module &M, ModuleAnalysisManager &AM) { - return BPFAdjustOptImpl(&M).run() ? PreservedAnalyses::none() - : PreservedAnalyses::all(); -} diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp --- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp +++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp @@ -46,7 +46,6 @@ void checkIR(Module &M); bool adjustIR(Module &M); bool removePassThroughBuiltin(Module &M); - bool removeCompareBuiltin(Module &M); }; } // End anonymous namespace @@ -121,50 +120,8 @@ return Changed; } -bool BPFCheckAndAdjustIR::removeCompareBuiltin(Module &M) { - // Remove __builtin_bpf_compare()'s which are used to prevent - // certain IR optimizations. Now major IR optimizations are done, - // remove them. - bool Changed = false; - CallInst *ToBeDeleted = nullptr; - for (Function &F : M) - for (auto &BB : F) - for (auto &I : BB) { - if (ToBeDeleted) { - ToBeDeleted->eraseFromParent(); - ToBeDeleted = nullptr; - } - - auto *Call = dyn_cast(&I); - if (!Call) - continue; - auto *GV = dyn_cast(Call->getCalledOperand()); - if (!GV) - continue; - if (!GV->getName().startswith("llvm.bpf.compare")) - continue; - - Changed = true; - Value *Arg0 = Call->getArgOperand(0); - Value *Arg1 = Call->getArgOperand(1); - Value *Arg2 = Call->getArgOperand(2); - - auto OpVal = cast(Arg0)->getValue().getZExtValue(); - CmpInst::Predicate Opcode = (CmpInst::Predicate)OpVal; - - auto *ICmp = new ICmpInst(Opcode, Arg1, Arg2); - ICmp->insertBefore(Call); - - Call->replaceAllUsesWith(ICmp); - ToBeDeleted = Call; - } - return Changed; -} - bool BPFCheckAndAdjustIR::adjustIR(Module &M) { - bool Changed = removePassThroughBuiltin(M); - Changed = removeCompareBuiltin(M) || Changed; - return Changed; + return removePassThroughBuiltin(M); } bool BPFCheckAndAdjustIR::runOnModule(Module &M) { diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -43,7 +43,6 @@ initializeBPFAbstractMemberAccessLegacyPassPass(PR); initializeBPFPreserveDITypePass(PR); initializeBPFIRPeepholePass(PR); - initializeBPFAdjustOptPass(PR); initializeBPFCheckAndAdjustIRPass(PR); initializeBPFMIPeepholePass(PR); initializeBPFMIPeepholeTruncElimPass(PR); @@ -115,10 +114,6 @@ OptimizationLevel Level) { FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true))); }); - PB.registerPipelineEarlySimplificationEPCallback( - [=](ModulePassManager &MPM, OptimizationLevel) { - MPM.addPass(BPFAdjustOptPass()); - }); } void BPFPassConfig::addIRPasses() { diff --git a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h --- a/llvm/lib/Target/BPF/BPFTargetTransformInfo.h +++ b/llvm/lib/Target/BPF/BPFTargetTransformInfo.h @@ -77,6 +77,17 @@ return Options; } + bool shouldFoldAndOrOfICmpsUsingRanges() const { + return false; + } + + bool shouldFoldTwoEntryPHINode() const { + return false; + } + + bool shouldHoistMinMax() const { + return false; + } }; } // end namespace llvm diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -16,7 +16,6 @@ add_llvm_target(BPFCodeGen BPFAbstractMemberAccess.cpp - BPFAdjustOpt.cpp BPFAsmPrinter.cpp BPFCheckAndAdjustIR.cpp BPFFrameLowering.cpp diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp --- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp @@ -13,6 +13,7 @@ #include "InstCombineInternal.h" #include "llvm/Analysis/CmpInstAnalysis.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/PatternMatch.h" @@ -1168,6 +1169,9 @@ Value *InstCombinerImpl::foldAndOrOfICmpsUsingRanges(ICmpInst *ICmp1, ICmpInst *ICmp2, bool IsAnd) { + if (!TTI.shouldFoldAndOrOfICmpsUsingRanges()) + return nullptr; + ICmpInst::Predicate Pred1, Pred2; Value *V1, *V2; const APInt *C1, *C2; diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -173,7 +173,7 @@ static bool pointerInvalidatedByBlock(BasicBlock &BB, MemorySSA &MSSA, MemoryUse &MU); /// Aggregates various functions for hoisting computations out of loop. -static bool hoistArithmetics(Instruction &I, Loop &L, +static bool hoistArithmetics(TargetTransformInfo *TTI, Instruction &I, Loop &L, ICFLoopSafetyInfo &SafetyInfo, MemorySSAUpdater &MSSAU, AssumptionCache *AC, DominatorTree *DT); @@ -455,9 +455,9 @@ MSSAU, &SafetyInfo, Flags, ORE); Flags.setIsSink(false); if (Preheader) - Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, AC, TLI, L, - MSSAU, SE, &SafetyInfo, Flags, ORE, LoopNestMode, - LicmAllowSpeculation); + Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, AC, TLI, + TTI, L, MSSAU, SE, &SafetyInfo, Flags, ORE, + LoopNestMode, LicmAllowSpeculation); // Now that all loop invariants have been removed from the loop, promote any // memory references to scalars that we can. @@ -860,9 +860,9 @@ /// bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, - TargetLibraryInfo *TLI, Loop *CurLoop, - MemorySSAUpdater &MSSAU, ScalarEvolution *SE, - ICFLoopSafetyInfo *SafetyInfo, + TargetLibraryInfo *TLI, TargetTransformInfo *TTI, + Loop *CurLoop, MemorySSAUpdater &MSSAU, + ScalarEvolution *SE, ICFLoopSafetyInfo *SafetyInfo, SinkAndHoistLICMFlags &Flags, OptimizationRemarkEmitter *ORE, bool LoopNestMode, bool AllowSpeculation) { @@ -987,7 +987,7 @@ // Try to reassociate instructions so that part of computations can be // done out of loop. - if (hoistArithmetics(I, *CurLoop, *SafetyInfo, MSSAU, AC, DT)) { + if (hoistArithmetics(TTI, I, *CurLoop, *SafetyInfo, MSSAU, AC, DT)) { Changed = true; continue; } @@ -2418,8 +2418,11 @@ /// Try to simplify things like (A < INV_1 AND icmp A < INV_2) into (A < /// min(INV_1, INV_2)), if INV_1 and INV_2 are both loop invariants and their /// minimun can be computed outside of loop, and X is not a loop-invariant. -static bool hoistMinMax(Instruction &I, Loop &L, ICFLoopSafetyInfo &SafetyInfo, - MemorySSAUpdater &MSSAU) { +static bool hoistMinMax(TargetTransformInfo *TTI, Instruction &I, Loop &L, + ICFLoopSafetyInfo &SafetyInfo, MemorySSAUpdater &MSSAU) { + if (!TTI->shouldHoistMinMax()) + return false; + bool Inverse = false; bool IsLogical = false; using namespace PatternMatch; @@ -2547,14 +2550,14 @@ return true; } -static bool hoistArithmetics(Instruction &I, Loop &L, +static bool hoistArithmetics(TargetTransformInfo *TTI, Instruction &I, Loop &L, ICFLoopSafetyInfo &SafetyInfo, MemorySSAUpdater &MSSAU, AssumptionCache *AC, DominatorTree *DT) { // Optimize complex patterns, such as (x < INV1 && x < INV2), turning them // into (x < min(INV1, INV2)), and hoisting the invariant part of this // expression out of the loop. - if (hoistMinMax(I, L, SafetyInfo, MSSAU)) { + if (hoistMinMax(TTI, I, L, SafetyInfo, MSSAU)) { ++NumHoisted; ++NumMinMaxHoisted; return true; diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -3289,6 +3289,9 @@ /// see if we can eliminate it. static bool FoldTwoEntryPHINode(PHINode *PN, const TargetTransformInfo &TTI, DomTreeUpdater *DTU, const DataLayout &DL) { + if (!TTI.shouldFoldTwoEntryPHINode()) + return false; + // Ok, this is a two entry PHI node. Check to see if this is a simple "if // statement", which has a very simple dominance structure. Basically, we // are trying to find the condition that is being branched on, which diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll --- a/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll +++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll @@ -2,10 +2,6 @@ ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s ; RUN: opt -passes='default' -mtriple=bpf-pc-linux %s | llvm-dis > %t1 ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s -; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1 -; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s -; RUN: opt -passes='default' -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1 -; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s ; ; Source: ; int foo(); diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll --- a/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll +++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll @@ -1,7 +1,5 @@ ; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1 ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s -; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1 -; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s ; ; Source: ; int foo(); diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll --- a/llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll +++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp3.ll @@ -1,7 +1,7 @@ ; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1 -; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK,CHECK-V1 %s +; RUN: llc %t1 -o - | FileCheck %s ; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1 -; RUN: llc %t1 -mcpu=v3 -o - | FileCheck -check-prefixes=CHECK,CHECK-V3 %s +; RUN: llc %t1 -mcpu=v3 -o - | FileCheck %s ; ; Source: ; int test1(unsigned long a) { @@ -40,8 +40,8 @@ } ; CHECK-LABEL: test1 -; CHECK-V1: if r[[#]] > r[[#]] goto -; CHECK-V3: if w[[#]] < 4 goto +; CHECK: r[[#]] &= r[[#]] +; CHECK: if r[[#]] == 0 goto ; Function Attrs: nounwind define dso_local i32 @test2(i64 %a) #0 { @@ -68,8 +68,8 @@ } ; CHECK-LABEL: test2 -; CHECK-V1: if r[[#]] > r[[#]] goto -; CHECK-V3: if w[[#]] < 4 goto +; CHECK: r[[#]] &= r[[#]] +; CHECK: if r[[#]] == 0 goto attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll --- a/llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll +++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp4.ll @@ -1,7 +1,7 @@ ; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1 -; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK,CHECK-V1 %s +; RUN: llc %t1 -o - | FileCheck %s ; RUN: opt -O2 -S -mtriple=bpf-pc-linux %s -o %t1 -; RUN: llc %t1 -mcpu=v3 -o - | FileCheck -check-prefixes=CHECK,CHECK-V3 %s +; RUN: llc %t1 -mcpu=v3 -o - | FileCheck %s ; ; Source: ; int test1(unsigned long a) { @@ -40,8 +40,8 @@ } ; CHECK-LABEL: test1 -; CHECK-V1: if r[[#]] > 3 goto -; CHECK-V3: if w[[#]] > 3 goto +; CHECK: r[[#]] &= r[[#]] +; CHECK: if r[[#]] == 0 goto ; Function Attrs: nounwind define dso_local i32 @test2(i64 %a) #0 { @@ -68,8 +68,8 @@ } ; CHECK-LABEL: test2 -; CHECK-V1: if r[[#]] > 3 goto -; CHECK-V3: if w[[#]] > 3 goto +; CHECK: r[[#]] &= r[[#]] +; CHECK: if r[[#]] == 0 goto attributes #0 = { nounwind "frame-pointer"="all" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" } diff --git a/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll b/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll --- a/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll +++ b/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll @@ -1,7 +1,5 @@ ; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1 ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s -; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1 -; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s ; ; Source: ; unsigned long foo(); diff --git a/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll b/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll --- a/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll +++ b/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll @@ -1,7 +1,5 @@ ; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1 ; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s -; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1 -; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s ; ; Source: ; unsigned foo();