Index: lib/CodeGen/CodeGenPrepare.cpp =================================================================== --- lib/CodeGen/CodeGenPrepare.cpp +++ lib/CodeGen/CodeGenPrepare.cpp @@ -606,7 +606,7 @@ isa(Pred->getTerminator()))) return true; - if (BB->getTerminator() != BB->getFirstNonPHI()) + if (BB->getTerminator() != BB->getFirstNonPHIOrDbg()) return true; // We use a simple cost heuristic which determine skipping merging is Index: lib/CodeGen/MachineCopyPropagation.cpp =================================================================== --- lib/CodeGen/MachineCopyPropagation.cpp +++ lib/CodeGen/MachineCopyPropagation.cpp @@ -392,12 +392,39 @@ } } +/// collectDebgValues - Scan instructions following MI and collect any +/// matching DBG_VALUEs. +static void collectDebugValues(MachineInstr &MI, + SmallVectorImpl &DbgValues) { + DbgValues.clear(); + if (!MI.getOperand(0).isReg()) + return; + + MachineBasicBlock::iterator DI = MI; ++DI; + for (MachineBasicBlock::iterator DE = MI.getParent()->end(); + DI != DE; ++DI) { + if (!DI->isDebugValue()) + return; + if (DI->getOperand(0).isReg() && + DI->getOperand(0).getReg() == MI.getOperand(0).getReg()) + DbgValues.push_back(&*DI); + } +} + +static void eraseDebugValues(SmallVectorImpl &DbgValues) { + for (SmallVectorImpl::iterator DBI = DbgValues.begin(), + DBE = DbgValues.end(); DBI != DBE; ++DBI) + (*DBI)->eraseFromParent(); +} + void MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) { DEBUG(dbgs() << "MCP: CopyPropagateBlock " << MBB.getName() << "\n"); for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) { MachineInstr *MI = &*I; ++I; + if (MI->isDebugValue()) + continue; // Analyze copies (which don't overlap themselves). if (MI->isCopy() && !TRI->regsOverlap(MI->getOperand(0).getReg(), @@ -542,7 +569,10 @@ // erase() will return the next valid iterator pointing to the next // element after the erased one. DI = MaybeDeadCopies.erase(DI); + SmallVector DbgValues; + collectDebugValues(*MaybeDead, DbgValues); MaybeDead->eraseFromParent(); + eraseDebugValues(DbgValues); Changed = true; ++NumDeletes; } Index: lib/CodeGen/MachineSink.cpp =================================================================== --- lib/CodeGen/MachineSink.cpp +++ lib/CodeGen/MachineSink.cpp @@ -1122,6 +1122,9 @@ MachineInstr *MI = &*I; ++I; + if (MI->isDebugValue()) + continue; + // Do not move any instruction across function call. if (MI->isCall()) return false; @@ -1159,7 +1162,29 @@ // block. clearKillFlags(MI, CurBB, UsedOpsInCopy, UsedRegs, TRI); MachineBasicBlock::iterator InsertPos = SuccBB->getFirstNonPHI(); + + // collect matching debug values. + SmallVector DbgValuesToSink; + collectDebugValues(*MI, DbgValuesToSink); + + // Merge or erase debug location to ensure consistent stepping in profilers + // and debuggers. + if (!SuccBB->empty() && InsertPos != SuccBB->end()) + MI->setDebugLoc(DILocation::getMergedLocation(MI->getDebugLoc(), + InsertPos->getDebugLoc())); + else + MI->setDebugLoc(DebugLoc()); + SuccBB->splice(InsertPos, &CurBB, MI); + + // Move previously adjacent debug value instructions to the insert position. + for (SmallVectorImpl::iterator DBI = DbgValuesToSink.begin(), + DBE = DbgValuesToSink.end(); DBI != DBE; ++DBI) { + MachineInstr *DbgMI = *DBI; + SuccBB->splice(InsertPos, &CurBB, DbgMI, + ++MachineBasicBlock::iterator(DbgMI)); + } + updateLiveIn(MI, SuccBB, UsedOpsInCopy, DefedRegsInCopy); Changed = true; Index: lib/CodeGen/TwoAddressInstructionPass.cpp =================================================================== --- lib/CodeGen/TwoAddressInstructionPass.cpp +++ lib/CodeGen/TwoAddressInstructionPass.cpp @@ -928,11 +928,14 @@ // Move the copies connected to MI down as well. MachineBasicBlock::iterator Begin = MI; MachineBasicBlock::iterator AfterMI = std::next(Begin); - MachineBasicBlock::iterator End = AfterMI; + MachineBasicBlock::iterator End = + skipDebugInstructionsForward(AfterMI, MBB->end()); + assert(End != MBB->end() && "Expecting following instruction"); while (End->isCopy() && regOverlapsSet(Defs, End->getOperand(1).getReg(), TRI)) { Defs.push_back(End->getOperand(0).getReg()); - ++End; + End = skipDebugInstructionsForward(++End, MBB->end()); + assert(End != MBB->end() && "Expecting following instruction"); } // Check if the reschedule will not break dependencies. Index: lib/Target/SystemZ/SystemZElimCompare.cpp =================================================================== --- lib/Target/SystemZ/SystemZElimCompare.cpp +++ lib/Target/SystemZ/SystemZElimCompare.cpp @@ -187,6 +187,37 @@ return reg; } +/// collectDebgValues - Scan instructions following MI and collect any +/// matching DBG_VALUEs. +static void collectDebugValues(MachineInstr &MI, + SmallVectorImpl &DbgValues) { + DbgValues.clear(); + if (!MI.getOperand(0).isReg()) + return; + + MachineBasicBlock::iterator DI = MI; ++DI; + for (MachineBasicBlock::iterator DE = MI.getParent()->end(); + DI != DE; ++DI) { + if (!DI->isDebugValue()) + return; + if (DI->getOperand(0).isReg() && + DI->getOperand(0).getReg() == MI.getOperand(0).getReg()) + DbgValues.push_back(&*DI); + } +} + +static void moveDebugValues(MachineBasicBlock *ToMBB, + MachineBasicBlock::iterator InsPos, + SmallVectorImpl &DbgValuesToSink) { + for (SmallVectorImpl::iterator DBI = DbgValuesToSink.begin(), + DBE = DbgValuesToSink.end(); DBI != DBE; ++DBI) { + MachineInstr *DbgMI = *DBI; + MachineBasicBlock *FromMBB = DbgMI->getParent(); + ToMBB->splice(InsPos, FromMBB, + DbgMI, ++MachineBasicBlock::iterator(DbgMI)); + } +} + // Compare compares the result of MI against zero. If MI is an addition // of -1 and if CCUsers is a single branch on nonzero, eliminate the addition // and convert the branch to a BRCT(G) or BRCTH. Return true on success. @@ -221,11 +252,17 @@ // Compare and Branch. unsigned SrcReg = getCompareSourceReg(Compare); MachineBasicBlock::iterator MBBI = Compare, MBBE = Branch; - for (++MBBI; MBBI != MBBE; ++MBBI) + for (++MBBI; MBBI != MBBE; ++MBBI) { + if (MBBI->isDebugValue()) + continue; if (getRegReferences(*MBBI, SrcReg)) return false; + } // The transformation is OK. Rebuild Branch as a BRCT(G) or BRCTH. + SmallVector DbgValuesToSink; + collectDebugValues(MI, DbgValuesToSink); + MachineOperand Target(Branch->getOperand(2)); while (Branch->getNumOperands()) Branch->RemoveOperand(0); @@ -238,6 +275,9 @@ if (BRCT != SystemZ::BRCTH) MIB.addReg(SystemZ::CC, RegState::ImplicitDefine | RegState::Dead); MI.eraseFromParent(); + + moveDebugValues(Branch->getParent(), ++MachineBasicBlock::iterator(Branch), + DbgValuesToSink); return true; } @@ -265,11 +305,17 @@ // Compare and Branch. unsigned SrcReg = getCompareSourceReg(Compare); MachineBasicBlock::iterator MBBI = Compare, MBBE = Branch; - for (++MBBI; MBBI != MBBE; ++MBBI) + for (++MBBI; MBBI != MBBE; ++MBBI) { + if (MBBI->isDebugValue()) + continue; if (getRegReferences(*MBBI, SrcReg)) return false; + } // The transformation is OK. Rebuild Branch as a load-and-trap. + SmallVector DbgValuesToSink; + collectDebugValues(MI, DbgValuesToSink); + while (Branch->getNumOperands()) Branch->RemoveOperand(0); Branch->setDesc(TII->get(LATOpcode)); @@ -279,6 +325,9 @@ .add(MI.getOperand(2)) .add(MI.getOperand(3)); MI.eraseFromParent(); + + moveDebugValues(Branch->getParent(), ++MachineBasicBlock::iterator(Branch), + DbgValuesToSink); return true; } @@ -431,6 +480,9 @@ while (MBBI != MBBE) { --MBBI; MachineInstr &MI = *MBBI; + if (MI.isDebugValue()) + continue; + if (resultTests(MI, SrcReg)) { // Try to remove both MI and Compare by converting a branch to BRCT(G). // or a load-and-trap instruction. We don't care in this case whether Index: lib/Target/SystemZ/SystemZMachineScheduler.cpp =================================================================== --- lib/Target/SystemZ/SystemZMachineScheduler.cpp +++ lib/Target/SystemZ/SystemZMachineScheduler.cpp @@ -60,6 +60,10 @@ void SystemZPostRASchedStrategy:: advanceTo(MachineBasicBlock::iterator NextBegin) { MachineBasicBlock::iterator LastEmittedMI = HazardRec->getLastEmittedMI(); + if (NextBegin == LastEmittedMI) + // In the precence of DBG_VALUEs after a BRCT, the BRCT may have been + // emitted already at this point. + return; MachineBasicBlock::iterator I = ((LastEmittedMI != nullptr && LastEmittedMI->getParent() == MBB) ? std::next(LastEmittedMI) : MBB->begin()); Index: lib/Transforms/IPO/FunctionAttrs.cpp =================================================================== --- lib/Transforms/IPO/FunctionAttrs.cpp +++ lib/Transforms/IPO/FunctionAttrs.cpp @@ -42,6 +42,7 @@ #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Type.h" @@ -1271,13 +1272,16 @@ // If all of the calls in F are identifiable and are to norecurse functions, F // is norecurse. This check also detects self-recursion as F is not currently // marked norecurse, so any called from F to F will not be marked norecurse. - for (Instruction &I : instructions(*F)) + for (Instruction &I : instructions(*F)) { + if (isa(I)) + continue; if (auto CS = CallSite(&I)) { Function *Callee = CS.getCalledFunction(); if (!Callee || Callee == F || !Callee->doesNotRecurse()) // Function calls a potentially recursive function. return false; } + } // Every call was to a non-recursive function other than this function, and // we have no indirect recursion as the SCC size is one. This function cannot Index: lib/Transforms/Scalar/MergedLoadStoreMotion.cpp =================================================================== --- lib/Transforms/Scalar/MergedLoadStoreMotion.cpp +++ lib/Transforms/Scalar/MergedLoadStoreMotion.cpp @@ -81,6 +81,7 @@ #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Metadata.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -285,7 +286,10 @@ return false; // No. More than 2 predecessors. // #Instructions in Succ1 for Compile Time Control - int Size1 = Pred1->size(); + int Size1 = 0; + for (auto &I : *Pred1) + if (!isa(I)) + Size1++; int NStores = 0; for (BasicBlock::reverse_iterator RBI = Pred0->rbegin(), RBE = Pred0->rend();