Index: include/llvm/Analysis/Loads.h =================================================================== --- include/llvm/Analysis/Loads.h +++ include/llvm/Analysis/Loads.h @@ -15,6 +15,7 @@ #define LLVM_ANALYSIS_LOADS_H #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/InvariantInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/Support/CommandLine.h" @@ -54,6 +55,7 @@ Value *FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan = DefMaxInstsToScan, + InvariantInfo *InvInfo = nullptr, AliasAnalysis *AA = nullptr, AAMDNodes *AATags = nullptr); Index: lib/Analysis/Lint.cpp =================================================================== --- lib/Analysis/Lint.cpp +++ lib/Analysis/Lint.cpp @@ -643,7 +643,7 @@ break; if (Value *U = FindAvailableLoadedValue(L->getPointerOperand(), - BB, BBI, DefMaxInstsToScan, AA)) + BB, BBI, DefMaxInstsToScan, nullptr, AA)) return findValueImpl(U, OffsetOk, Visited); if (BBI != BB->begin()) break; BB = BB->getUniquePredecessor(); Index: lib/Analysis/Loads.cpp =================================================================== --- lib/Analysis/Loads.cpp +++ lib/Analysis/Loads.cpp @@ -196,6 +196,7 @@ Value *llvm::FindAvailableLoadedValue(Value *Ptr, BasicBlock *ScanBB, BasicBlock::iterator &ScanFrom, unsigned MaxInstsToScan, + InvariantInfo *InvInfo, AliasAnalysis *AA, AAMDNodes *AATags) { if (MaxInstsToScan == 0) MaxInstsToScan = ~0U; @@ -209,6 +210,10 @@ Value *StrippedPtr = Ptr->stripPointerCasts(); + // We're about to scan backwards. Preserve the initial invariant_start + // intrinsic marking on this load, for subsequent instructions. + PreserveInvariantInfoRAII Preserved(StrippedPtr, DL, InvInfo); + while (ScanFrom != ScanBB->begin()) { // We must ignore debug info directives when counting (otherwise they // would affect codegen). @@ -216,6 +221,12 @@ if (isa(Inst)) continue; + // Same for invariant intrinsics, which may also unset preserved + // invariant info. + if (IntrinsicInst *II = dyn_cast(Inst)) { + if (AA && BackwardScanInvariantIntrinsic(II, Preserved, InvInfo)) continue; + } + // Restore ScanFrom to expected value in case next test succeeds ScanFrom++; Index: lib/Passes/PassBuilder.cpp =================================================================== --- lib/Passes/PassBuilder.cpp +++ lib/Passes/PassBuilder.cpp @@ -18,6 +18,7 @@ #include "llvm/Passes/PassBuilder.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Analysis/InvariantInfo.h" #include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ScalarEvolution.h" Index: lib/Passes/PassRegistry.def =================================================================== --- lib/Passes/PassRegistry.def +++ lib/Passes/PassRegistry.def @@ -53,6 +53,7 @@ #endif FUNCTION_ANALYSIS("assumptions", AssumptionAnalysis()) FUNCTION_ANALYSIS("domtree", DominatorTreeAnalysis()) +FUNCTION_ANALYSIS("invariantinfo", InvariantInfoAnalysis()) FUNCTION_ANALYSIS("loops", LoopAnalysis()) FUNCTION_ANALYSIS("no-op-function", NoOpFunctionAnalysis()) FUNCTION_ANALYSIS("scalar-evolution", ScalarEvolutionAnalysis()) Index: lib/Transforms/InstCombine/InstCombineCalls.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineCalls.cpp +++ lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -14,6 +14,7 @@ #include "InstCombineInternal.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/InvariantInfo.h" #include "llvm/Analysis/MemoryBuiltins.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Dominators.h" @@ -1743,6 +1744,10 @@ break; } + case Intrinsic::invariant_start: + case Intrinsic::invariant_end: + processInvariantIntrinsic(II, *InvInfo); + break; case Intrinsic::experimental_gc_relocate: { // Translate facts known about a pointer before relocating into // facts about the relocate value, while being careful to Index: lib/Transforms/InstCombine/InstCombineInternal.h =================================================================== --- lib/Transforms/InstCombine/InstCombineInternal.h +++ lib/Transforms/InstCombine/InstCombineInternal.h @@ -35,6 +35,7 @@ class CallSite; class DataLayout; class DominatorTree; +class InvariantInfo; class TargetLibraryInfo; class DbgDeclareInst; class MemIntrinsic; @@ -183,6 +184,7 @@ // Required analyses. // FIXME: These can never be null and should be references. AssumptionCache *AC; + InvariantInfo *InvInfo; TargetLibraryInfo *TLI; DominatorTree *DT; const DataLayout &DL; @@ -197,9 +199,11 @@ InstCombiner(InstCombineWorklist &Worklist, BuilderTy *Builder, bool MinimizeSize, AliasAnalysis *AA, AssumptionCache *AC, TargetLibraryInfo *TLI, - DominatorTree *DT, const DataLayout &DL, LoopInfo *LI) + DominatorTree *DT, const DataLayout &DL, + InvariantInfo *InvInfo, LoopInfo *LI) : Worklist(Worklist), Builder(Builder), MinimizeSize(MinimizeSize), - AA(AA), AC(AC), TLI(TLI), DT(DT), DL(DL), LI(LI), MadeIRChange(false) {} + AA(AA), AC(AC), InvInfo(InvInfo), TLI(TLI), DT(DT), DL(DL), LI(LI), + MadeIRChange(false) {} /// \brief Run the combiner over the entire worklist until it is empty. /// Index: lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -761,7 +761,7 @@ AAMDNodes AATags; if (Value *AvailableVal = FindAvailableLoadedValue(Op, LI.getParent(), BBI, - DefMaxInstsToScan, AA, &AATags)) { + DefMaxInstsToScan, InvInfo, AA, &AATags)) { if (LoadInst *NLI = dyn_cast(AvailableVal)) { unsigned KnownIDs[] = { LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope, Index: lib/Transforms/InstCombine/InstructionCombining.cpp =================================================================== --- lib/Transforms/InstCombine/InstructionCombining.cpp +++ lib/Transforms/InstCombine/InstructionCombining.cpp @@ -44,6 +44,7 @@ #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/InvariantInfo.h" #include "llvm/Analysis/LibCallSemantics.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MemoryBuiltins.h" @@ -3039,6 +3040,7 @@ static bool combineInstructionsOverFunction(Function &F, InstCombineWorklist &Worklist, AliasAnalysis *AA, AssumptionCache &AC, + InvariantInfo &InvInfo, TargetLibraryInfo &TLI, DominatorTree &DT, LoopInfo *LI = nullptr) { auto &DL = F.getParent()->getDataLayout(); @@ -3064,7 +3066,7 @@ Changed = true; InstCombiner IC(Worklist, &Builder, F.optForMinSize(), - AA, &AC, &TLI, &DT, DL, LI); + AA, &AC, &TLI, &DT, DL, &InvInfo, LI); if (IC.run()) Changed = true; @@ -3078,13 +3080,15 @@ PreservedAnalyses InstCombinePass::run(Function &F, AnalysisManager *AM) { auto &AC = AM->getResult(F); + auto &InvInfo = AM->getResult(F); auto &DT = AM->getResult(F); auto &TLI = AM->getResult(F); auto *LI = AM->getCachedResult(F); // FIXME: The AliasAnalysis is not yet supported in the new pass manager - if (!combineInstructionsOverFunction(F, Worklist, nullptr, AC, TLI, DT, LI)) + if (!combineInstructionsOverFunction(F, Worklist, nullptr, AC, InvInfo, + TLI, DT, LI)) // No changes, all analyses are preserved. return PreservedAnalyses::all(); @@ -3119,6 +3123,7 @@ AU.setPreservesCFG(); AU.addRequired(); AU.addRequired(); + AU.addRequired(); AU.addRequired(); AU.addRequired(); AU.addPreserved(); @@ -3132,6 +3137,7 @@ // Required analyses. auto AA = &getAnalysis().getAAResults(); auto &AC = getAnalysis().getAssumptionCache(F); + auto &InvInfo = getAnalysis().getInvariantInfo(); auto &TLI = getAnalysis().getTLI(); auto &DT = getAnalysis().getDomTree(); @@ -3139,13 +3145,15 @@ auto *LIWP = getAnalysisIfAvailable(); auto *LI = LIWP ? &LIWP->getLoopInfo() : nullptr; - return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, DT, LI); + return combineInstructionsOverFunction(F, Worklist, AA, AC, InvInfo, + TLI, DT, LI); } char InstructionCombiningPass::ID = 0; INITIALIZE_PASS_BEGIN(InstructionCombiningPass, "instcombine", "Combine redundant instructions", false, false) INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) +INITIALIZE_PASS_DEPENDENCY(InvariantInfoMarkerPass) INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) Index: lib/Transforms/Scalar/JumpThreading.cpp =================================================================== --- lib/Transforms/Scalar/JumpThreading.cpp +++ lib/Transforms/Scalar/JumpThreading.cpp @@ -994,7 +994,7 @@ BBIt = PredBB->end(); AAMDNodes ThisAATags; Value *PredAvailable = FindAvailableLoadedValue(LoadedPtr, PredBB, BBIt, - DefMaxInstsToScan, + DefMaxInstsToScan, nullptr, nullptr, &ThisAATags); if (!PredAvailable) { OneUnavailablePred = PredBB; Index: test/Transforms/LoadElim/invariant.ll =================================================================== --- test/Transforms/LoadElim/invariant.ll +++ test/Transforms/LoadElim/invariant.ll @@ -1,5 +1,7 @@ -; RUN: opt < %s -gvn -S | FileCheck %s --check-prefix=CHECKL --check-prefix=CHECK -; RUN: opt < %s -globalopt -gvn -S | FileCheck %s --check-prefix=CHECKG --check-prefix=CHECK +; RUN: opt < %s -gvn -S | FileCheck %s --check-prefix=CHECKL --check-prefix=CHECK +; RUN: opt < %s -instcombine -S | FileCheck %s --check-prefix=CHECKL --check-prefix=CHECK +; RUN: opt < %s -globalopt -gvn -S | FileCheck %s --check-prefix=CHECKG --check-prefix=CHECK +; RUN: opt < %s -globalopt -instcombine -S | FileCheck %s --check-prefix=CHECKG --check-prefix=CHECK ; On a given pointer to allocated memory, invariant_start/end intrinsic ; calls indicate that the memory can be considered constant for load