Index: include/llvm/CodeGen/MachineInstr.h =================================================================== --- include/llvm/CodeGen/MachineInstr.h +++ include/llvm/CodeGen/MachineInstr.h @@ -1169,6 +1169,12 @@ assert(NumMemRefs == NewMemRefsEnd - NewMemRefs && "Too many memrefs"); } + /// clearMemRefs - Clear this MachineInstr's memory reference descriptor list. + void clearMemRefs() { + MemRefs = nullptr; + NumMemRefs = 0; + } + private: /// getRegInfo - If this instruction is embedded into a MachineFunction, /// return the MachineRegisterInfo object for the current function, otherwise Index: lib/CodeGen/BranchFolding.cpp =================================================================== --- lib/CodeGen/BranchFolding.cpp +++ lib/CodeGen/BranchFolding.cpp @@ -24,6 +24,7 @@ #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" @@ -727,6 +728,99 @@ return true; } +static bool hasIdenticalMMOs(const MachineInstr *MI1, + const MachineInstr *MI2) { + auto MMOI1 = MI1->memoperands_begin(), MMOE1 = MI1->memoperands_end(); + auto MMOI2 = MI2->memoperands_begin(), MMOE2 = MI2->memoperands_end(); + + if ((MMOE1 - MMOI1) != (MMOE2 - MMOI2)) + return false; + + for (; MMOI1 != MMOE1; ++MMOI1, ++MMOI2) { + const Value *V1 = (*MMOI1)->getValue(); + const Value *V2 = (*MMOI2)->getValue(); + + if (V1 != V2) + return false; + + // And check all other MMO fields. + if ((*MMOI1)->getPseudoValue() != (*MMOI2)->getPseudoValue()) + return false; + + if ((*MMOI1)->getSize() != (*MMOI2)->getSize()) + return false; + + if ((*MMOI1)->getOffset() != (*MMOI2)->getOffset()) + return false; + + if ((*MMOI1)->getFlags() != (*MMOI2)->getFlags()) + return false; + + if ((*MMOI1)->getAAInfo() != (*MMOI2)->getAAInfo()) + return false; + + if ((*MMOI1)->getRanges() != (*MMOI2)->getRanges()) + return false; + + if ((*MMOI1)->getAlignment() != (*MMOI2)->getAlignment()) + return false; + + if ((*MMOI1)->getBaseAlignment() != (*MMOI2)->getBaseAlignment()) + return false; + + if ((*MMOI1)->getAddrSpace() != (*MMOI2)->getAddrSpace()) + return false; + } + + return true; +} + +static void removeMMOsFromMemoryOperations( + MachineBasicBlock::iterator MBBIStartPos, + MachineBasicBlock &MBBCommon) { + // Remove MMOs from memory operations in the common block + // when they do not match the ones from the block being tail-merged. + // This ensures later passes conservatively compute dependencies. + MachineBasicBlock *MBB = MBBIStartPos->getParent(); + // Note CommonTailLen does not necessarily matches the size of + // the common BB nor all its instructions because of debug + // instructions differences. + unsigned CommonTailLen = 0; + for (auto E = MBB->end(); MBBIStartPos != E; ++MBBIStartPos) + ++CommonTailLen; + + MachineBasicBlock::reverse_iterator MBBI = MBB->rbegin(); + MachineBasicBlock::reverse_iterator MBBICommon = MBBCommon.rbegin(); + MachineBasicBlock::reverse_iterator MBBIE = MBB->rend(); + MachineBasicBlock::reverse_iterator MBBIECommon = MBBCommon.rend(); + + while (CommonTailLen--) { + assert(MBBI != MBBIE && "Reached BB end within common tail length!"); + (void)MBBIE; + + if (MBBI->isDebugValue()) { + ++MBBI; + continue; + } + + while (MBBICommon->isDebugValue()) + ++MBBICommon; + + assert(MBBICommon != MBBIECommon && + "Reached BB end within common tail length!"); + (void)MBBIECommon; + + assert(MBBICommon->isIdenticalTo(&*MBBI) && "Expected matching MIIs!"); + + if (MBBICommon->mayLoad() || MBBICommon->mayStore()) + if (!hasIdenticalMMOs(&*MBBI, &*MBBICommon)) + MBBICommon->clearMemRefs(); + + ++MBBI; + ++MBBICommon; + } +} + // See if any of the blocks in MergePotentials (which all have a common single // successor, or all have no successor) can be tail-merged. If there is a // successor, any blocks in MergePotentials that are not tail-merged and @@ -840,6 +934,8 @@ continue; DEBUG(dbgs() << "BB#" << SameTails[i].getBlock()->getNumber() << (i == e-1 ? "" : ", ")); + // Remove MMOs from memory operations as needed. + removeMMOsFromMemoryOperations(SameTails[i].getTailStartPos(), *MBB); // Hack the end off BB i, making it jump to BB commonTailIndex instead. ReplaceTailWithBranchTo(SameTails[i].getTailStartPos(), MBB); // BB i is no longer a predecessor of SuccBB; remove it from the worklist.