Index: lib/Analysis/Loads.cpp =================================================================== --- lib/Analysis/Loads.cpp +++ lib/Analysis/Loads.cpp @@ -167,11 +167,13 @@ /// threading in part by eliminating partially redundant loads. /// At that point, the value of MaxInstsToScan was already set to '6' /// without documented explanation. -cl::opt -llvm::DefMaxInstsToScan("available-load-scan-limit", cl::init(6), cl::Hidden, - cl::desc("Use this to specify the default maximum number of instructions " - "to scan backward from a given instruction, when searching for " - "available loaded value")); +/// Let's bump this number up to '8' to improve the chances of behaviorial +/// match between -instcombine and -inline -instcombine. +cl::opt llvm::DefMaxInstsToScan( + "available-load-scan-limit", cl::init(8), cl::Hidden, + cl::desc("Use this to specify the default maximum number of instructions " + "to scan backward from a given instruction, when searching for " + "available loaded value")); /// \brief Scan the ScanBB block backwards to see if we have the value at the /// memory address *Ptr locally available within a small number of instructions. @@ -208,6 +210,14 @@ Value *StrippedPtr = Ptr->stripPointerCasts(); + // We're about to scan backwards. Preserve the initial invariant_start + // intrinsic marking on this load, for subsequent instructions. + // First, compute the info to preserve. + // Then, actually preserve the info before backward scanning starts. + InvariantInfo &InvInfo = ScanBB->getModule()->getInvariantInfo(); + PreservedInvariantInfo Preserved(StrippedPtr, DL, InvInfo); + PreserveInvariantInfo PIO(Preserved); + while (ScanFrom != ScanBB->begin()) { // We must ignore debug info directives when counting (otherwise they // would affect codegen). @@ -215,6 +225,12 @@ if (isa(Inst)) continue; + // Same for invariant intrinsics, which may also unset preserved + // invariant info. + if (IntrinsicInst *II = dyn_cast(Inst)) { + if (AA && BackwardScanInvariantIntrinsic(II, Preserved, InvInfo)) continue; + } + // Restore ScanFrom to expected value in case next test succeeds ScanFrom++; Index: lib/Transforms/InstCombine/InstCombineCalls.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineCalls.cpp +++ lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -1505,6 +1505,10 @@ break; } + case Intrinsic::invariant_start: + case Intrinsic::invariant_end: + processInvariantIntrinsic(II, InvInfo); + break; case Intrinsic::experimental_gc_relocate: { // Translate facts known about a pointer before relocating into // facts about the relocate value, while being careful to Index: lib/Transforms/InstCombine/InstCombineInternal.h =================================================================== --- lib/Transforms/InstCombine/InstCombineInternal.h +++ lib/Transforms/InstCombine/InstCombineInternal.h @@ -35,6 +35,7 @@ class CallSite; class DataLayout; class DominatorTree; +class InvariantInfo; class TargetLibraryInfo; class DbgDeclareInst; class MemIntrinsic; @@ -186,6 +187,7 @@ TargetLibraryInfo *TLI; DominatorTree *DT; const DataLayout &DL; + InvariantInfo &InvInfo; // Optional analyses. When non-null, these can both be used to do better // combining and will be updated to reflect any changes. @@ -197,9 +199,11 @@ InstCombiner(InstCombineWorklist &Worklist, BuilderTy *Builder, bool MinimizeSize, AliasAnalysis *AA, AssumptionCache *AC, TargetLibraryInfo *TLI, - DominatorTree *DT, const DataLayout &DL, LoopInfo *LI) + DominatorTree *DT, const DataLayout &DL, + InvariantInfo &InvInfo, LoopInfo *LI) : Worklist(Worklist), Builder(Builder), MinimizeSize(MinimizeSize), - AA(AA), AC(AC), TLI(TLI), DT(DT), DL(DL), LI(LI), MadeIRChange(false) {} + AA(AA), AC(AC), TLI(TLI), DT(DT), DL(DL), InvInfo(InvInfo), LI(LI), + MadeIRChange(false) {} /// \brief Run the combiner over the entire worklist until it is empty. /// Index: lib/Transforms/InstCombine/InstructionCombining.cpp =================================================================== --- lib/Transforms/InstCombine/InstructionCombining.cpp +++ lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1957,6 +1957,20 @@ uint64_t DontKnow = CI->isZero() ? -1ULL : 0; ReplaceInstUsesWith(*I, ConstantInt::get(I->getType(), DontKnow)); } + + // If this is a paired invariant_start, then erase its invariant_end, + // and all of its other uses. + if (II->getIntrinsicID() == Intrinsic::invariant_start) { + SmallVector UsesToRemove; + for (Use &U : II->uses()) { + IntrinsicInst *User = cast(U.getUser()); + assert(User->getIntrinsicID() == Intrinsic::invariant_end && + "The paired instruction should be an invariant_end."); + UsesToRemove.push_back(User); + } + for (unsigned i = 0, e = UsesToRemove.size(); i != e; ++i) + EraseInstFromFunction(*UsesToRemove[i]); + } } EraseInstFromFunction(*I); } @@ -3002,6 +3016,7 @@ TargetLibraryInfo &TLI, DominatorTree &DT, LoopInfo *LI = nullptr) { auto &DL = F.getParent()->getDataLayout(); + auto &InvInfo = F.getParent()->getInvariantInfo(); /// Builder - This is an IRBuilder that automatically inserts new /// instructions into the worklist when they are created. @@ -3024,7 +3039,7 @@ Changed = true; InstCombiner IC(Worklist, &Builder, F.optForMinSize(), - AA, &AC, &TLI, &DT, DL, LI); + AA, &AC, &TLI, &DT, DL, InvInfo, LI); if (IC.run()) Changed = true;