Index: include/llvm/CodeGen/MachineInstr.h =================================================================== --- include/llvm/CodeGen/MachineInstr.h +++ include/llvm/CodeGen/MachineInstr.h @@ -1170,6 +1170,12 @@ assert(NumMemRefs == NewMemRefsEnd - NewMemRefs && "Too many memrefs"); } + /// clearMemRefs - Clear this MachineInstr's memory reference descriptor list. + void clearMemRefs() { + MemRefs = nullptr; + NumMemRefs = 0; + } + private: /// getRegInfo - If this instruction is embedded into a MachineFunction, /// return the MachineRegisterInfo object for the current function, otherwise Index: lib/CodeGen/BranchFolding.cpp =================================================================== --- lib/CodeGen/BranchFolding.cpp +++ lib/CodeGen/BranchFolding.cpp @@ -20,10 +20,12 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineBranchProbabilityInfo.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/Passes.h" @@ -728,6 +730,92 @@ return true; } +static bool hasIdentialMMOs(const MachineInstr *MI1, + const MachineInstr *MI2) { + auto MMOI1 = MI1->memoperands_begin(), MMOE1 = MI1->memoperands_end(); + auto MMOI2 = MI2->memoperands_begin(), MMOE2 = MI2->memoperands_end(); + + if ((MMOE1 - MMOI1) != (MMOE2 - MMOI2)) + return false; + + for (; MMOI1 != MMOE1; ++MMOI1, ++MMOI2) { + const Value *V1 = (*MMOI1)->getValue(); + const Value *V2 = (*MMOI2)->getValue(); + + // Check MMO PtrInfo V field and its underlying objects. + if (V1 != V2) + return false; + + SmallVector Objs1; + SmallVector Objs2; + + if (V1) + GetUnderlyingObjects(const_cast(V1), Objs1); + if (V2) + GetUnderlyingObjects(const_cast(V2), Objs2); + + if (Objs1.size() != Objs2.size()) + return false; + + auto OI1 = Objs1.begin(); + for (auto OI2 = Objs2.begin(), OIE2 = Objs2.end(); OI2 != OIE2; + ++OI2, ++OI1) + if (*OI1 != *OI2) + return false; + + // Check MMO PtrInfo getPseudoValue field. + if ((*MMOI1)->getPseudoValue() != (*MMOI2)->getPseudoValue()) + return false; + + // And check all other MMO fields. + if ((*MMOI1)->getSize() != (*MMOI2)->getSize()) + return false; + + if ((*MMOI1)->getOffset() != (*MMOI2)->getOffset()) + return false; + + // Check MMO Flags field, discounting Load/Store flags. + if (((*MMOI1)->getFlags() | + MachineMemOperand::MOLoad | MachineMemOperand::MOStore) != + ((*MMOI2)->getFlags() | + MachineMemOperand::MOLoad | MachineMemOperand::MOStore)) + return false; + + if ((*MMOI1)->getAAInfo() != (*MMOI2)->getAAInfo()) + return false; + + if ((*MMOI1)->getRanges() != (*MMOI2)->getRanges()) + return false; + + if ((*MMOI1)->getAlignment() != (*MMOI2)->getAlignment()) + return false; + + if ((*MMOI1)->getBaseAlignment() != (*MMOI2)->getBaseAlignment()) + return false; + + if ((*MMOI1)->getAddrSpace() != (*MMOI2)->getAddrSpace()) + return false; + } + + return true; +} + +static void removeMMOsFromMemoryOperations( + MachineBasicBlock &MBB, + MachineBasicBlock &MBBCommon) { + // Remove MMOs from memory operations in the common block + // when they do not match the ones from the block being tail-merged. + // This ensures later passes conservatively compute dependencies. + for (auto &MIICommon : MBBCommon) + if (MIICommon.mayLoad() || MIICommon.mayStore()) + for (auto &MII : MBB) + if (MII.mayLoad() || MII.mayStore()) + if (!hasIdentialMMOs(&MII, &MIICommon)) { + MIICommon.clearMemRefs(); + break; + } +} + // See if any of the blocks in MergePotentials (which all have a common single // successor, or all have no successor) can be tail-merged. If there is a // successor, any blocks in MergePotentials that are not tail-merged and @@ -841,6 +929,10 @@ continue; DEBUG(dbgs() << "BB#" << SameTails[i].getBlock()->getNumber() << (i == e-1 ? "" : ", ")); + + // Remove MMOs from memory operations as needed. + removeMMOsFromMemoryOperations(*SameTails[i].getBlock(), *MBB); + // Hack the end off BB i, making it jump to BB commonTailIndex instead. ReplaceTailWithBranchTo(SameTails[i].getTailStartPos(), MBB); // BB i is no longer a predecessor of SuccBB; remove it from the worklist.