Index: llvm/include/llvm/IR/Instruction.h =================================================================== --- llvm/include/llvm/IR/Instruction.h +++ llvm/include/llvm/IR/Instruction.h @@ -357,6 +357,10 @@ /// Return the debug location for this node as a DebugLoc. const DebugLoc &getDebugLoc() const { return DbgLoc; } + /// Fetch the debug location for this node, unless this is a debug intrinsic, + /// in which case fetch the debug location of the next non-debug node. + const DebugLoc &getStableDebugLoc() const; + /// Set or clear the nuw flag on this instruction, which must be an operator /// which supports this flag. See LangRef.html for the meaning of this flag. void setHasNoUnsignedWrap(bool b = true); Index: llvm/lib/IR/BasicBlock.cpp =================================================================== --- llvm/lib/IR/BasicBlock.cpp +++ llvm/lib/IR/BasicBlock.cpp @@ -420,7 +420,7 @@ this->getNextNode()); // Save DebugLoc of split point before invalidating iterator. - DebugLoc Loc = I->getDebugLoc(); + DebugLoc Loc = I->getStableDebugLoc(); // Move all of the specified instructions from the original basic block into // the new basic block. New->splice(New->end(), this, I, end()); Index: llvm/lib/IR/Instruction.cpp =================================================================== --- llvm/lib/IR/Instruction.cpp +++ llvm/lib/IR/Instruction.cpp @@ -848,6 +848,12 @@ return nullptr; } +const DebugLoc &Instruction::getStableDebugLoc() const { + if (isa(this)) + return getNextNonDebugInstruction()->getDebugLoc(); + return getDebugLoc(); +} + bool Instruction::isAssociative() const { unsigned Opcode = getOpcode(); if (isAssociative(Opcode)) Index: llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp +++ llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp @@ -1886,7 +1886,7 @@ return false; BasicBlock *LoopBody = *(CurLoop->block_begin()); - if (LoopBody->size() >= 20) { + if (LoopBody->sizeWithoutDebug() >= 20) { // The loop is too big, bail out. return false; } Index: llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -5348,7 +5348,7 @@ // IP consistent across expansions and allows the previously inserted // instructions to be reused by subsequent expansion. while (Rewriter.isInsertedInstruction(&*IP) && IP != LowestIP) - ++IP; + IP = IP->getNextNonDebugInstruction()->getIterator();; return IP; } Index: llvm/lib/Transforms/Utils/BasicBlockUtils.cpp =================================================================== --- llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -1304,7 +1304,7 @@ // The new block unconditionally branches to the old block. BranchInst *BI1 = BranchInst::Create(OrigBB, NewBB1); - BI1->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc()); + BI1->setDebugLoc(OrigBB->getFirstNonPHIOrDbg()->getDebugLoc()); // Move the edges from Preds to point to NewBB1 instead of OrigBB. for (BasicBlock *Pred : Preds) { @@ -1345,7 +1345,7 @@ // The new block unconditionally branches to the old block. BranchInst *BI2 = BranchInst::Create(OrigBB, NewBB2); - BI2->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc()); + BI2->setDebugLoc(OrigBB->getFirstNonPHIOrDbg()->getDebugLoc()); // Move the remaining edges from OrigBB to point to NewBB2. for (BasicBlock *NewBB2Pred : NewBB2Preds) @@ -1479,6 +1479,7 @@ SmallVector Updates; BasicBlock *Head = SplitBefore->getParent(); BasicBlock *Tail = Head->splitBasicBlock(SplitBefore->getIterator()); + DebugLoc DbgLoc = SplitBefore->getStableDebugLoc(); if (DTU) { SmallPtrSet UniqueSuccessorsOfHead; Updates.push_back({DominatorTree::Insert, Head, Tail}); @@ -1502,7 +1503,7 @@ if (DTU) Updates.push_back({DominatorTree::Insert, ThenBlock, Tail}); } - CheckTerm->setDebugLoc(SplitBefore->getDebugLoc()); + CheckTerm->setDebugLoc(DbgLoc); } else CheckTerm = ThenBlock->getTerminator(); BranchInst *HeadNewTerm = @@ -1567,6 +1568,7 @@ MDNode *BranchWeights, DomTreeUpdater *DTU) { BasicBlock *Head = SplitBefore->getParent(); + DebugLoc DbgLoc = SplitBefore->getStableDebugLoc(); SmallPtrSet UniqueOrigSuccessors; if (DTU) @@ -1578,9 +1580,9 @@ BasicBlock *ThenBlock = BasicBlock::Create(C, "", Head->getParent(), Tail); BasicBlock *ElseBlock = BasicBlock::Create(C, "", Head->getParent(), Tail); *ThenTerm = BranchInst::Create(Tail, ThenBlock); - (*ThenTerm)->setDebugLoc(SplitBefore->getDebugLoc()); + (*ThenTerm)->setDebugLoc(DbgLoc); *ElseTerm = BranchInst::Create(Tail, ElseBlock); - (*ElseTerm)->setDebugLoc(SplitBefore->getDebugLoc()); + (*ElseTerm)->setDebugLoc(DbgLoc); BranchInst *HeadNewTerm = BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/ElseBlock, Cond); HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights); Index: llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp =================================================================== --- llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -98,10 +98,12 @@ BasicBlock::iterator SCEVExpander::findInsertPointAfter(Instruction *I, Instruction *MustDominate) const { - BasicBlock::iterator IP = ++I->getIterator(); + BasicBlock::iterator IP = std::next(I->getIterator()); + IP = skipDebugIntrinsics(IP); if (auto *II = dyn_cast(I)) IP = II->getNormalDest()->begin(); + IP = skipDebugIntrinsics(IP); while (isa(IP)) ++IP; @@ -119,6 +121,7 @@ while (isInsertedInstruction(&*IP) && &*IP != MustDominate) ++IP; + IP = skipDebugIntrinsics(IP); return IP; } @@ -144,10 +147,10 @@ // so let's plop this cast into the function's entry block. assert(isa(V) && "Expected the cast argument to be a global/constant"); - return Builder.GetInsertBlock() + return skipDebugIntrinsics(Builder.GetInsertBlock() ->getParent() ->getEntryBlock() - .getFirstInsertionPt(); + .getFirstInsertionPt()); } /// InsertNoopCastOfTo - Insert a cast of V to the specified type, Index: llvm/lib/Transforms/Utils/SimplifyCFG.cpp =================================================================== --- llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -3985,7 +3985,9 @@ Value *QPHI = ensureValueAvailableInSuccessor(QStore->getValueOperand(), QStore->getParent(), PPHI); - IRBuilder<> QB(&*PostBB->getFirstInsertionPt()); + Instruction *PostBBFirst = &*PostBB->getFirstInsertionPt(); + IRBuilder<> QB(PostBBFirst); + QB.SetCurrentDebugLocation(PostBBFirst->getStableDebugLoc()); Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond); Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond); Index: llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3388,7 +3388,7 @@ ScheduleStart->comesBefore(ScheduleEnd) && "Not a valid scheduling region?"); - for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { + for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNonDebugInstruction()) { auto *SD = getScheduleData(I); if (!SD) continue; @@ -3421,7 +3421,7 @@ /// Put all instructions into the ReadyList which are ready for scheduling. template void initialFillReadyList(ReadyListType &ReadyList) { - for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { + for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNonDebugInstruction()) { doForAllOpcodes(I, [&](ScheduleData *SD) { if (SD->isSchedulingEntity() && SD->hasValidDependencies() && SD->isReady()) { @@ -11136,7 +11136,7 @@ // It is seldom that this needs to be done a second time after adding the // initial bundle to the region. if (ScheduleEnd != OldScheduleEnd) { - for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) + for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNonDebugInstruction()) doForAllOpcodes(I, [](ScheduleData *SD) { SD->clearDependencies(); }); ReSchedule = true; } @@ -11282,9 +11282,9 @@ return true; if (!ScheduleStart) { // It's the first instruction in the new region. - initScheduleData(I, I->getNextNode(), nullptr, nullptr); + initScheduleData(I, I->getNextNonDebugInstruction(), nullptr, nullptr); ScheduleStart = I; - ScheduleEnd = I->getNextNode(); + ScheduleEnd = I->getNextNonDebugInstruction(); if (isOneOf(S, I) != I) CheckScheduleForI(I); assert(ScheduleEnd && "tried to vectorize a terminator?"); @@ -11324,9 +11324,9 @@ "lower end."); assert(I->getParent() == ScheduleEnd->getParent() && "Instruction is in wrong basic block."); - initScheduleData(ScheduleEnd, I->getNextNode(), LastLoadStoreInRegion, + initScheduleData(ScheduleEnd, I->getNextNonDebugInstruction(), LastLoadStoreInRegion, nullptr); - ScheduleEnd = I->getNextNode(); + ScheduleEnd = I->getNextNonDebugInstruction(); if (isOneOf(S, I) != I) CheckScheduleForI(I); assert(ScheduleEnd && "tried to vectorize a terminator?"); @@ -11339,7 +11339,7 @@ ScheduleData *PrevLoadStore, ScheduleData *NextLoadStore) { ScheduleData *CurrentLoadStore = PrevLoadStore; - for (Instruction *I = FromI; I != ToI; I = I->getNextNode()) { + for (Instruction *I = FromI; I != ToI; I = I->getNextNonDebugInstruction()) { // No need to allocate data for non-schedulable instructions. if (doesNotNeedToBeScheduled(I)) continue; @@ -11439,8 +11439,8 @@ // block is control dependend on any early exit or non-willreturn call // which proceeds it. if (!isGuaranteedToTransferExecutionToSuccessor(BundleMember->Inst)) { - for (Instruction *I = BundleMember->Inst->getNextNode(); - I != ScheduleEnd; I = I->getNextNode()) { + for (Instruction *I = BundleMember->Inst->getNextNonDebugInstruction(); + I != ScheduleEnd; I = I->getNextNonDebugInstruction()) { if (isSafeToSpeculativelyExecute(I, &*BB->begin(), SLP->AC)) continue; @@ -11459,8 +11459,8 @@ // from reordering above a preceeding stackrestore. if (match(BundleMember->Inst, m_Intrinsic()) || match(BundleMember->Inst, m_Intrinsic())) { - for (Instruction *I = BundleMember->Inst->getNextNode(); - I != ScheduleEnd; I = I->getNextNode()) { + for (Instruction *I = BundleMember->Inst->getNextNonDebugInstruction(); + I != ScheduleEnd; I = I->getNextNonDebugInstruction()) { if (match(I, m_Intrinsic()) || match(I, m_Intrinsic())) // Any allocas past here must be control dependent on I, and I @@ -11482,8 +11482,8 @@ // can lead to incorrect code. if (isa(BundleMember->Inst) || BundleMember->Inst->mayReadOrWriteMemory()) { - for (Instruction *I = BundleMember->Inst->getNextNode(); - I != ScheduleEnd; I = I->getNextNode()) { + for (Instruction *I = BundleMember->Inst->getNextNonDebugInstruction(); + I != ScheduleEnd; I = I->getNextNonDebugInstruction()) { if (!match(I, m_Intrinsic()) && !match(I, m_Intrinsic())) continue; @@ -11567,7 +11567,7 @@ void BoUpSLP::BlockScheduling::resetSchedule() { assert(ScheduleStart && "tried to reset schedule on block which has not been scheduled"); - for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) { + for (Instruction *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNonDebugInstruction()) { doForAllOpcodes(I, [&](ScheduleData *SD) { assert(isInSchedulingRegion(SD) && "ScheduleData not in scheduling region"); @@ -11607,7 +11607,7 @@ // and fill the ready-list with initial instructions. int Idx = 0; for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; - I = I->getNextNode()) { + I = I->getNextNonDebugInstruction()) { BS->doForAllOpcodes(I, [this, &Idx, BS](ScheduleData *SD) { TreeEntry *SDTE = getTreeEntry(SD->Inst); (void)SDTE; @@ -11635,7 +11635,7 @@ for (ScheduleData *BundleMember = picked; BundleMember; BundleMember = BundleMember->NextInBundle) { Instruction *pickedInst = BundleMember->Inst; - if (pickedInst->getNextNode() != LastScheduledInst) + if (pickedInst->getNextNonDebugInstruction() != LastScheduledInst) pickedInst->moveBefore(LastScheduledInst); LastScheduledInst = pickedInst; } @@ -11650,7 +11650,7 @@ #if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS) // Check that all schedulable entities got scheduled - for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; I = I->getNextNode()) { + for (auto *I = BS->ScheduleStart; I != BS->ScheduleEnd; I = I->getNextNonDebugInstruction()) { BS->doForAllOpcodes(I, [&](ScheduleData *SD) { if (SD->isSchedulingEntity() && SD->hasValidDependencies()) { assert(SD->IsScheduled && "must be scheduled at this point");