Index: include/llvm/Analysis/MemoryDependenceAnalysis.h =================================================================== --- include/llvm/Analysis/MemoryDependenceAnalysis.h +++ include/llvm/Analysis/MemoryDependenceAnalysis.h @@ -397,7 +397,7 @@ /// critical edges. void invalidateCachedPredecessors(); - /// getPointerDependencyFrom - Return the instruction on which a memory + /// \brief - Return the instruction on which a memory /// location depends. If isLoad is true, this routine ignores may-aliases /// with read-only operations. If isLoad is false, this routine ignores /// may-aliases with reads from read-only locations. If possible, pass @@ -412,6 +412,17 @@ BasicBlock *BB, Instruction *QueryInst = nullptr); + MemDepResult getSimplePointerDependencyFrom( + const MemoryLocation &MemLoc, bool isLoad, BasicBlock::iterator ScanIt, + BasicBlock *BB, Instruction *QueryInst); + + + /// \brief - This analysis looks for other + /// loads and stores with invariant.group metadata and the same + /// pointer operand. Returns Unknown if does not find anything, and Def + /// if it can be assumed that 2 instructions load or store the same value. + MemDepResult getInvariantGroupPointerDependency(LoadInst *LI); + /// getLoadLoadClobberFullWidthSize - This is a little bit of analysis that /// looks at a memory location for a load (specified by MemLocBase, Offs, /// and Size) and compares it against a load. If the specified load could Index: lib/Analysis/MemoryDependenceAnalysis.cpp =================================================================== --- lib/Analysis/MemoryDependenceAnalysis.cpp +++ lib/Analysis/MemoryDependenceAnalysis.cpp @@ -380,6 +380,52 @@ const MemoryLocation &MemLoc, bool isLoad, BasicBlock::iterator ScanIt, BasicBlock *BB, Instruction *QueryInst) { + if (QueryInst != nullptr) + if (auto *LI = dyn_cast(QueryInst)) { + MemDepResult invariantGroupPointerDependency = + getInvariantGroupPointerDependency(LI); + if (!invariantGroupPointerDependency.isUnknown()) + return invariantGroupPointerDependency; + } + + return getSimplePointerDependencyFrom(MemLoc, isLoad, ScanIt, BB, QueryInst); +} + +MemDepResult MemoryDependenceAnalysis::getInvariantGroupPointerDependency( + LoadInst *LI) { + + Value *Ptr = LI->getPointerOperand(); + // It's is not safe to walk the uses of global value, because nobody guarantee + // that uses list won't change during iteration (uses list doesn't have + // any lock and there might be loads in different functions being optimized + // in the same time). + if (isa(Ptr)) + return MemDepResult::getUnknown(); + + auto *InvariantGroupMD = LI->getMetadata(LLVMContext::MD_invariant_group); + if (!InvariantGroupMD) + return MemDepResult::getUnknown(); + + for (Use &Us : Ptr->uses()) { + auto *U = dyn_cast(Us.getUser()); + if (!U || U == LI || DT->dominates(LI, Us)) + continue; + + // If we hit load/store with the same invariant.group metadata (and the + // same pointer operand) we can assume that value pointed by pointer + // operand didn't change. + if ((isa(U) || isa(U)) && + U->getMetadata(LLVMContext::MD_invariant_group) == InvariantGroupMD) + return MemDepResult::getDef(U); + } + + return MemDepResult::getUnknown(); +} + +MemDepResult MemoryDependenceAnalysis::getSimplePointerDependencyFrom( + const MemoryLocation &MemLoc, bool isLoad, BasicBlock::iterator ScanIt, + BasicBlock *BB, Instruction *QueryInst) { + const Value *MemLocBase = nullptr; int64_t MemLocOffset = 0; unsigned Limit = BlockScanLimit; Index: lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -761,6 +761,7 @@ LLVMContext::MD_range, LLVMContext::MD_invariant_load, LLVMContext::MD_nonnull, + LLVMContext::MD_invariant_group }; combineMetadata(NLI, &LI, KnownIDs); }; Index: lib/Transforms/Scalar/GVN.cpp =================================================================== --- lib/Transforms/Scalar/GVN.cpp +++ lib/Transforms/Scalar/GVN.cpp @@ -1669,6 +1669,9 @@ if (Tags) NewLoad->setAAMetadata(Tags); + if (auto *InvGroupMD = LI->getMetadata(LLVMContext::MD_invariant_group)) + NewLoad->setMetadata(LLVMContext::MD_invariant_group, InvGroupMD); + // Transfer DebugLoc. NewLoad->setDebugLoc(LI->getDebugLoc()); @@ -1852,6 +1855,7 @@ LLVMContext::MD_range, LLVMContext::MD_fpmath, LLVMContext::MD_invariant_load, + LLVMContext::MD_invariant_group }; combineMetadata(ReplInst, I, KnownIDs); } @@ -2106,6 +2110,8 @@ if (it != ReplaceWithConstMap.end()) { assert(!isa(Operand) && "Replacing constants with constants is invalid"); + DEBUG(dbgs() << "GVN replacing: " << *Operand << + " with " << *it->second << " in instruction " << *Instr << '\n'); Instr->setOperand(OpNum, it->second); Changed = true; } @@ -2461,7 +2467,6 @@ return Changed; } - bool GVN::processBlock(BasicBlock *BB) { // FIXME: Kill off InstrsToErase by doing erasing eagerly in a helper function // (and incrementing BI before processing an instruction). @@ -2476,6 +2481,7 @@ for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); BI != BE;) { + if (!ReplaceWithConstMap.empty()) ChangedFunction |= replaceOperandsWithConsts(BI); ChangedFunction |= processInstruction(BI); Index: lib/Transforms/Scalar/MemCpyOptimizer.cpp =================================================================== --- lib/Transforms/Scalar/MemCpyOptimizer.cpp +++ lib/Transforms/Scalar/MemCpyOptimizer.cpp @@ -748,6 +748,7 @@ LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope, LLVMContext::MD_noalias, + LLVMContext::MD_invariant_group }; combineMetadata(C, cpy, KnownIDs); Index: lib/Transforms/Utils/Local.cpp =================================================================== --- lib/Transforms/Utils/Local.cpp +++ lib/Transforms/Utils/Local.cpp @@ -1330,7 +1330,8 @@ return true; } -void llvm::combineMetadata(Instruction *K, const Instruction *J, ArrayRef KnownIDs) { +void llvm::combineMetadata(Instruction *K, const Instruction *J, + ArrayRef KnownIDs) { SmallVector, 4> Metadata; K->dropUnknownNonDebugMetadata(KnownIDs); K->getAllMetadataOtherThanDebugLoc(Metadata); @@ -1368,8 +1369,18 @@ // Only set the !nonnull if it is present in both instructions. K->setMetadata(Kind, JMD); break; + case LLVMContext::MD_invariant_group: + // Preserve !invariant.group in K. + break; } } + // Set !invariant.group from J if J has it. If both instructions have it + // then we will just pick it from J - even when they are different. + // FIXME: we should try to preserve both invariant.group md if they are + // different, but right now instruction can only have one invariant.group. + if (auto *JMD = J->getMetadata(LLVMContext::MD_invariant_group)) + K->setMetadata(LLVMContext::MD_invariant_group, JMD); + } unsigned llvm::replaceDominatedUsesWith(Value *From, Value *To, Index: lib/Transforms/Utils/SimplifyCFG.cpp =================================================================== --- lib/Transforms/Utils/SimplifyCFG.cpp +++ lib/Transforms/Utils/SimplifyCFG.cpp @@ -1099,7 +1099,8 @@ LLVMContext::MD_range, LLVMContext::MD_fpmath, LLVMContext::MD_invariant_load, - LLVMContext::MD_nonnull + LLVMContext::MD_nonnull, + LLVMContext::MD_invariant_group }; combineMetadata(I1, I2, KnownIDs); I2->eraseFromParent(); Index: lib/Transforms/Vectorize/BBVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/BBVectorize.cpp +++ lib/Transforms/Vectorize/BBVectorize.cpp @@ -3124,7 +3124,8 @@ LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope, LLVMContext::MD_noalias, - LLVMContext::MD_fpmath + LLVMContext::MD_fpmath, + LLVMContext::MD_invariant_group }; combineMetadata(K, H, KnownIDs); K->intersectOptionalDataWith(H);