Index: lib/Target/X86/CMakeLists.txt =================================================================== --- lib/Target/X86/CMakeLists.txt +++ lib/Target/X86/CMakeLists.txt @@ -31,7 +31,7 @@ X86FastISel.cpp X86FixupBWInsts.cpp X86FixupLEAs.cpp - X86FixupSFB.cpp + X86AvoidStoreForwardingBlocks.cpp X86FixupSetCC.cpp X86FloatingPoint.cpp X86FrameLowering.cpp Index: lib/Target/X86/X86.h =================================================================== --- lib/Target/X86/X86.h +++ lib/Target/X86/X86.h @@ -71,7 +71,7 @@ FunctionPass *createX86FixupSetCC(); /// Return a pass that avoids creating store forward block issues in the hardware. -FunctionPass *createX86FixupSFB(); +FunctionPass *createX86AvoidStoreForwardingBlocks(); /// Return a pass that expands WinAlloca pseudo-instructions. FunctionPass *createX86WinAllocaExpander(); Index: lib/Target/X86/X86FixupSFB.cpp =================================================================== --- lib/Target/X86/X86FixupSFB.cpp +++ lib/Target/X86/X86FixupSFB.cpp @@ -1,4 +1,4 @@ -//===- X86FixupSFB.cpp - Avoid HW Store Forward Block issues -----------===// +//===- X86AvoidStoreForwardingBlockis.cpp - Avoid HW Store Forward Block --===// // // The LLVM Compiler Infrastructure // @@ -50,19 +50,25 @@ using namespace llvm; -#define DEBUG_TYPE "x86-fixup-SFB" +#define DEBUG_TYPE "x86-avoid-SFB" -static cl::opt DisableX86FixupSFB("disable-fixup-SFB", cl::Hidden, - cl::desc("X86: Disable SFB fixup."), +static cl::opt DisableX86AvoidStoreForwardBlocks("disable-avoid-SFB", cl::Hidden, + cl::desc("X86: Disable Store Forwarding Blocks fixup."), cl::init(false)); + +static cl::opt X86AvoidSFBInspectionLimit( + "sfb-inspection-limit", + cl::desc("X86: Number of instructions backward to inspect for store forwarding blocks."), + cl::init(20), cl::Hidden); + namespace { -class FixupSFBPass : public MachineFunctionPass { +class AvoidSFBPass : public MachineFunctionPass { public: - FixupSFBPass() : MachineFunctionPass(ID) {} + AvoidSFBPass() : MachineFunctionPass(ID) {} StringRef getPassName() const override { - return "X86 Fixup Store Forward Block"; + return "X86 Avoid Store Forwarding Blocks"; } bool runOnMachineFunction(MachineFunction &MF) override; @@ -81,9 +87,9 @@ /// \brief Break the memcpy's load and store into smaller copies /// such that each memory load that was blocked by a smaller store /// would now be copied separately. - void - breakBlockedCopies(MachineInstr *LoadInst, MachineInstr *StoreInst, - const std::map &BlockingStoresDisp); + using DisplacementSizeMap = DenseMap; + void breakBlockedCopies(MachineInstr *LoadInst, MachineInstr *StoreInst, + const DisplacementSizeMap &BlockingStoresDispSizeMap); /// \brief Break a copy of size Size to smaller copies. void buildCopies(int Size, MachineInstr *LoadInst, int64_t LdDispImm, MachineInstr *StoreInst, int64_t StDispImm, @@ -100,9 +106,9 @@ } // end anonymous namespace -char FixupSFBPass::ID = 0; +char AvoidSFBPass::ID = 0; -FunctionPass *llvm::createX86FixupSFB() { return new FixupSFBPass(); } +FunctionPass *llvm::createX86AvoidStoreForwardingBlocks() { return new AvoidSFBPass(); } static bool isXMMLoadOpcode(unsigned Opcode) { return Opcode == X86::MOVUPSrm || Opcode == X86::MOVAPSrm || @@ -128,45 +134,58 @@ return isXMMLoadOpcode(Opcode) || isYMMLoadOpcode(Opcode); } -std::map> PotentialBlockedMemCpy{ - {X86::MOVUPSrm, {X86::MOVUPSmr, X86::MOVAPSmr}}, - {X86::MOVAPSrm, {X86::MOVUPSmr, X86::MOVAPSmr}}, - {X86::VMOVUPSrm, {X86::VMOVUPSmr, X86::VMOVAPSmr}}, - {X86::VMOVAPSrm, {X86::VMOVUPSmr, X86::VMOVAPSmr}}, - {X86::VMOVUPDrm, {X86::VMOVUPDmr, X86::VMOVAPDmr}}, - {X86::VMOVAPDrm, {X86::VMOVUPDmr, X86::VMOVAPDmr}}, - {X86::VMOVDQUrm, {X86::VMOVDQUmr, X86::VMOVDQAmr}}, - {X86::VMOVDQArm, {X86::VMOVDQUmr, X86::VMOVDQAmr}}, - {X86::VMOVUPSZ128rm, {X86::VMOVUPSZ128mr, X86::VMOVAPSZ128mr}}, - {X86::VMOVAPSZ128rm, {X86::VMOVUPSZ128mr, X86::VMOVAPSZ128mr}}, - {X86::VMOVUPDZ128rm, {X86::VMOVUPDZ128mr, X86::VMOVAPDZ128mr}}, - {X86::VMOVAPDZ128rm, {X86::VMOVUPDZ128mr, X86::VMOVAPDZ128mr}}, - {X86::VMOVUPSYrm, {X86::VMOVUPSYmr, X86::VMOVAPSYmr}}, - {X86::VMOVAPSYrm, {X86::VMOVUPSYmr, X86::VMOVAPSYmr}}, - {X86::VMOVUPDYrm, {X86::VMOVUPDYmr, X86::VMOVAPDYmr}}, - {X86::VMOVAPDYrm, {X86::VMOVUPDYmr, X86::VMOVAPDYmr}}, - {X86::VMOVDQUYrm, {X86::VMOVDQUYmr, X86::VMOVDQAYmr}}, - {X86::VMOVDQAYrm, {X86::VMOVDQUYmr, X86::VMOVDQAYmr}}, - {X86::VMOVUPSZ256rm, {X86::VMOVUPSZ256mr, X86::VMOVAPSZ256mr}}, - {X86::VMOVAPSZ256rm, {X86::VMOVUPSZ256mr, X86::VMOVAPSZ256mr}}, - {X86::VMOVUPDZ256rm, {X86::VMOVUPDZ256mr, X86::VMOVAPDZ256mr}}, - {X86::VMOVAPDZ256rm, {X86::VMOVUPDZ256mr, X86::VMOVAPDZ256mr}}, - {X86::VMOVDQU64Z128rm, {X86::VMOVDQU64Z128mr, X86::VMOVDQA64Z128mr}}, - {X86::VMOVDQA64Z128rm, {X86::VMOVDQU64Z128mr, X86::VMOVDQA64Z128mr}}, - {X86::VMOVDQU32Z128rm, {X86::VMOVDQU32Z128mr, X86::VMOVDQA32Z128mr}}, - {X86::VMOVDQA32Z128rm, {X86::VMOVDQU32Z128mr, X86::VMOVDQA32Z128mr}}, - {X86::VMOVDQU64Z256rm, {X86::VMOVDQU64Z256mr, X86::VMOVDQA64Z256mr}}, - {X86::VMOVDQA64Z256rm, {X86::VMOVDQU64Z256mr, X86::VMOVDQA64Z256mr}}, - {X86::VMOVDQU32Z256rm, {X86::VMOVDQU32Z256mr, X86::VMOVDQA32Z256mr}}, - {X86::VMOVDQA32Z256rm, {X86::VMOVDQU32Z256mr, X86::VMOVDQA32Z256mr}}, +static bool isPotentialBlockedMemCpyPair(int LdOpcode, int StOpcode) { + switch (LdOpcode) { + case X86::MOVUPSrm: + case X86::MOVAPSrm: + return StOpcode == X86::MOVUPSmr || StOpcode == X86::MOVAPSmr; + case X86::VMOVUPSrm: + case X86::VMOVAPSrm: + return StOpcode == X86::VMOVUPSmr || StOpcode == X86::VMOVAPSmr; + case X86::VMOVUPDrm: + case X86::VMOVAPDrm: + return StOpcode == X86::VMOVUPDmr || StOpcode == X86::VMOVAPDmr; + case X86::VMOVDQUrm: + case X86::VMOVDQArm: + return StOpcode == X86::VMOVDQUmr || StOpcode == X86::VMOVDQAmr; + case X86::VMOVUPSZ128rm: + case X86::VMOVAPSZ128rm: + return StOpcode == X86::VMOVUPSZ128mr || StOpcode == X86::VMOVAPSZ128mr; + case X86::VMOVUPDZ128rm: + case X86::VMOVAPDZ128rm: + return StOpcode == X86::VMOVUPDZ128mr || StOpcode == X86::VMOVAPDZ128mr; + case X86::VMOVUPSYrm: + case X86::VMOVAPSYrm: + return StOpcode == X86::VMOVUPSYmr || StOpcode == X86::VMOVAPSYmr; + case X86::VMOVUPDYrm: + case X86::VMOVAPDYrm: + return StOpcode == X86::VMOVUPDYmr || StOpcode == X86::VMOVAPDYmr; + case X86::VMOVDQUYrm: + case X86::VMOVDQAYrm: + return StOpcode == X86::VMOVDQUYmr || StOpcode == X86::VMOVDQAYmr; + case X86::VMOVUPSZ256rm: + case X86::VMOVAPSZ256rm: + return StOpcode == X86::VMOVUPSZ256mr || StOpcode == X86::VMOVAPSZ256mr; + case X86::VMOVUPDZ256rm: + case X86::VMOVAPDZ256rm: + return StOpcode == X86::VMOVUPDZ256mr || StOpcode == X86::VMOVAPDZ256mr; + case X86::VMOVDQU64Z128rm: + case X86::VMOVDQA64Z128rm: + return StOpcode == X86::VMOVDQU64Z128mr || StOpcode == X86::VMOVDQA64Z128mr; + case X86::VMOVDQU32Z128rm: + case X86::VMOVDQA32Z128rm: + return StOpcode == X86::VMOVDQU32Z128mr || StOpcode == X86::VMOVDQA32Z128mr; + case X86::VMOVDQU64Z256rm: + case X86::VMOVDQA64Z256rm: + return StOpcode == X86::VMOVDQU64Z256mr || StOpcode == X86::VMOVDQA64Z256mr; + case X86::VMOVDQU32Z256rm: + case X86::VMOVDQA32Z256rm: + return StOpcode == X86::VMOVDQU32Z256mr || StOpcode == X86::VMOVDQA32Z256mr; + default: + return false; + } }; -static bool isPotentialBlockedMemCpyPair(unsigned LdOpcode, unsigned StOpcode) { - auto PotentialStores = PotentialBlockedMemCpy.at(LdOpcode); - return PotentialStores.first == StOpcode || - PotentialStores.second == StOpcode; -} - static bool isPotentialBlockingStoreInst(int Opcode, int LoadOpcode) { bool PBlock = false; PBlock |= Opcode == X86::MOV64mr || Opcode == X86::MOV64mi32 || @@ -191,38 +210,62 @@ static const int MOV16SZ = 2; static const int MOV8SZ = 1; -std::map YMMtoXMMLoadMap = { - {X86::VMOVUPSYrm, X86::VMOVUPSrm}, - {X86::VMOVAPSYrm, X86::VMOVUPSrm}, - {X86::VMOVUPDYrm, X86::VMOVUPDrm}, - {X86::VMOVAPDYrm, X86::VMOVUPDrm}, - {X86::VMOVDQUYrm, X86::VMOVDQUrm}, - {X86::VMOVDQAYrm, X86::VMOVDQUrm}, - {X86::VMOVUPSZ256rm, X86::VMOVUPSZ128rm}, - {X86::VMOVAPSZ256rm, X86::VMOVUPSZ128rm}, - {X86::VMOVUPDZ256rm, X86::VMOVUPDZ128rm}, - {X86::VMOVAPDZ256rm, X86::VMOVUPDZ128rm}, - {X86::VMOVDQU64Z256rm, X86::VMOVDQU64Z128rm}, - {X86::VMOVDQA64Z256rm, X86::VMOVDQU64Z128rm}, - {X86::VMOVDQU32Z256rm, X86::VMOVDQU32Z128rm}, - {X86::VMOVDQA32Z256rm, X86::VMOVDQU32Z128rm}, +static unsigned getYMMtoXMMLoadOpcode(unsigned LoadOpcode) { + switch (LoadOpcode) { + case X86::VMOVUPSYrm: + case X86::VMOVAPSYrm: + return X86::VMOVUPSrm; + case X86::VMOVUPDYrm: + case X86::VMOVAPDYrm: + return X86::VMOVUPDrm; + case X86::VMOVDQUYrm: + case X86::VMOVDQAYrm: + return X86::VMOVDQUrm; + case X86::VMOVUPSZ256rm: + case X86::VMOVAPSZ256rm: + return X86::VMOVUPSZ128rm; + case X86::VMOVUPDZ256rm: + case X86::VMOVAPDZ256rm: + return X86::VMOVUPDZ128rm; + case X86::VMOVDQU64Z256rm: + case X86::VMOVDQA64Z256rm: + return X86::VMOVDQU64Z128rm; + case X86::VMOVDQU32Z256rm: + case X86::VMOVDQA32Z256rm: + return X86::VMOVDQU32Z128rm; + default: + llvm_unreachable("Unexpected Load Instruction Opcode"); + } + return 0; }; -std::map YMMtoXMMStoreMap = { - {X86::VMOVUPSYmr, X86::VMOVUPSmr}, - {X86::VMOVAPSYmr, X86::VMOVUPSmr}, - {X86::VMOVUPDYmr, X86::VMOVUPDmr}, - {X86::VMOVAPDYmr, X86::VMOVUPDmr}, - {X86::VMOVDQUYmr, X86::VMOVDQUmr}, - {X86::VMOVDQAYmr, X86::VMOVDQUmr}, - {X86::VMOVUPSZ256mr, X86::VMOVUPSZ128mr}, - {X86::VMOVAPSZ256mr, X86::VMOVUPSZ128mr}, - {X86::VMOVUPDZ256mr, X86::VMOVUPDZ128mr}, - {X86::VMOVAPDZ256mr, X86::VMOVUPDZ128mr}, - {X86::VMOVDQU64Z256mr, X86::VMOVDQU64Z128mr}, - {X86::VMOVDQA64Z256mr, X86::VMOVDQU64Z128mr}, - {X86::VMOVDQU32Z256mr, X86::VMOVDQU32Z128mr}, - {X86::VMOVDQA32Z256mr, X86::VMOVDQU32Z128mr}, +static unsigned getYMMtoXMMStoreOpcode(unsigned StoreOpcode) { + switch (StoreOpcode) { + case X86::VMOVUPSYmr: + case X86::VMOVAPSYmr: + return X86::VMOVUPSmr; + case X86::VMOVUPDYmr: + case X86::VMOVAPDYmr: + return X86::VMOVUPDmr; + case X86::VMOVDQUYmr: + case X86::VMOVDQAYmr: + return X86::VMOVDQUmr; + case X86::VMOVUPSZ256mr: + case X86::VMOVAPSZ256mr: + return X86::VMOVUPSZ128mr; + case X86::VMOVUPDZ256mr: + case X86::VMOVAPDZ256mr: + return X86::VMOVUPDZ128mr; + case X86::VMOVDQU64Z256mr: + case X86::VMOVDQA64Z256mr: + return X86::VMOVDQU64Z128mr; + case X86::VMOVDQU32Z256mr: + case X86::VMOVDQA32Z256mr: + return X86::VMOVDQU32Z128mr; + default: + llvm_unreachable("Unexpected Load Instruction Opcode"); + } + return 0; }; static int getAddrOffset(MachineInstr *MI) { @@ -248,8 +291,8 @@ // TODO: Consider expanding to other addressing modes in the future static bool isRelevantAddressingMode(MachineInstr *MI) { int AddrOffset = getAddrOffset(MI); - MachineOperand &Base = MI->getOperand(AddrOffset + X86::AddrBaseReg); - MachineOperand &Disp = MI->getOperand(AddrOffset + X86::AddrDisp); + MachineOperand &Base = getBaseOperand(MI); + MachineOperand &Disp = getDispOperand(MI); MachineOperand &Scale = MI->getOperand(AddrOffset + X86::AddrScaleAmt); MachineOperand &Index = MI->getOperand(AddrOffset + X86::AddrIndexReg); MachineOperand &Segment = MI->getOperand(AddrOffset + X86::AddrSegmentReg); @@ -272,16 +315,16 @@ // since the effect of store block won't be visible if the store // and load instructions have enough instructions in between to // keep the core busy. -static const unsigned LIMIT = 20; static SmallVector findPotentialBlockers(MachineInstr *LoadInst) { SmallVector PotentialBlockers; unsigned BlockLimit = 0; + unsigned InspectionLimit = X86AvoidSFBInspectionLimit; for (MachineBasicBlock::iterator LI = LoadInst, BB = LoadInst->getParent()->begin(); LI != BB; --LI) { BlockLimit++; - if (BlockLimit >= LIMIT) + if (BlockLimit >= InspectionLimit) break; MachineInstr &MI = *LI; if (MI.getDesc().isCall()) @@ -292,9 +335,9 @@ // Ideally we should traverse the predecessor blocks in depth with some // coloring algorithm, but for now let's just look at the first order // predecessors. - if (BlockLimit < LIMIT) { + if (BlockLimit < InspectionLimit) { MachineBasicBlock *MBB = LoadInst->getParent(); - int LimitLeft = LIMIT - BlockLimit; + int LimitLeft = InspectionLimit - BlockLimit; for (MachineBasicBlock::pred_iterator PB = MBB->pred_begin(), PE = MBB->pred_end(); PB != PE; ++PB) { @@ -315,7 +358,7 @@ return PotentialBlockers; } -void FixupSFBPass::buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode, +void AvoidSFBPass::buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode, int64_t LoadDisp, MachineInstr *StoreInst, unsigned NStoreOpcode, int64_t StoreDisp, unsigned Size, int64_t LMMOffset, @@ -354,7 +397,7 @@ DEBUG(StInst->getPrevNode()->dump()); } -void FixupSFBPass::buildCopies(int Size, MachineInstr *LoadInst, +void AvoidSFBPass::buildCopies(int Size, MachineInstr *LoadInst, int64_t LdDispImm, MachineInstr *StoreInst, int64_t StDispImm, int64_t LMMOffset, int64_t SMMOffset) { @@ -363,9 +406,9 @@ while (Size > 0) { if ((Size - MOV128SZ >= 0) && isYMMLoadOpcode(LoadInst->getOpcode())) { Size = Size - MOV128SZ; - buildCopy(LoadInst, YMMtoXMMLoadMap.at(LoadInst->getOpcode()), LdDisp, - StoreInst, YMMtoXMMStoreMap.at(StoreInst->getOpcode()), StDisp, - MOV128SZ, LMMOffset, SMMOffset); + buildCopy(LoadInst, getYMMtoXMMLoadOpcode(LoadInst->getOpcode()), LdDisp, + StoreInst, getYMMtoXMMStoreOpcode(StoreInst->getOpcode()), + StDisp, MOV128SZ, LMMOffset, SMMOffset); LdDisp += MOV128SZ; StDisp += MOV128SZ; LMMOffset += MOV128SZ; @@ -437,35 +480,36 @@ } } -void FixupSFBPass::findPotentiallylBlockedCopies(MachineFunction &MF) { +void AvoidSFBPass::findPotentiallylBlockedCopies(MachineFunction &MF) { for (auto &MBB : MF) - for (auto &MI : MBB) - if (isPotentialBlockedMemCpyLd(MI.getOpcode())) { - int DefVR = MI.getOperand(0).getReg(); - if (MRI->hasOneUse(DefVR)) - for (auto UI = MRI->use_nodbg_begin(DefVR), UE = MRI->use_nodbg_end(); - UI != UE;) { - MachineOperand &StoreMO = *UI++; - MachineInstr &StoreMI = *StoreMO.getParent(); - if (isPotentialBlockedMemCpyPair(MI.getOpcode(), - StoreMI.getOpcode()) && - (StoreMI.getParent() == MI.getParent())) - if (isRelevantAddressingMode(&MI) && - isRelevantAddressingMode(&StoreMI)) - BlockedLoadsStores.push_back( - std::pair(&MI, &StoreMI)); - } + for (auto &MI : MBB) { + if (!isPotentialBlockedMemCpyLd(MI.getOpcode())) + continue; + int DefVR = MI.getOperand(0).getReg(); + if (!MRI->hasOneUse(DefVR)) + continue; + for (auto UI = MRI->use_nodbg_begin(DefVR), UE = MRI->use_nodbg_end(); + UI != UE;) { + MachineOperand &StoreMO = *UI++; + MachineInstr &StoreMI = *StoreMO.getParent(); + if (isPotentialBlockedMemCpyPair(MI.getOpcode(), StoreMI.getOpcode()) && + StoreMI.getParent() == MI.getParent() && + isRelevantAddressingMode(&MI) && isRelevantAddressingMode(&StoreMI)) + BlockedLoadsStores.push_back( + std::pair(&MI, &StoreMI)); } + } } -unsigned FixupSFBPass::getRegSizeInBytes(MachineInstr *LoadInst) { + +unsigned AvoidSFBPass::getRegSizeInBytes(MachineInstr *LoadInst) { auto TRC = TII->getRegClass(TII->get(LoadInst->getOpcode()), 0, TRI, *LoadInst->getParent()->getParent()); return TRI->getRegSizeInBits(*TRC) / 8; } -void FixupSFBPass::breakBlockedCopies( +void AvoidSFBPass::breakBlockedCopies( MachineInstr *LoadInst, MachineInstr *StoreInst, - const std::map &BlockingStoresDisp) { + const DisplacementSizeMap &BlockingStoresDispSizeMap) { int64_t LdDispImm = getDispOperand(LoadInst).getImm(); int64_t StDispImm = getDispOperand(StoreInst).getImm(); int64_t LMMOffset = (*LoadInst->memoperands_begin())->getOffset(); @@ -478,13 +522,17 @@ unsigned Size1 = 0; unsigned Size2 = 0; int64_t LdStDelta = StDispImm - LdDispImm; - for (auto inst : BlockingStoresDisp) { - LdDisp2 = inst.first; - StDisp2 = inst.first + LdStDelta; + + for (auto DispSizePair : BlockingStoresDispSizeMap) { + LdDisp2 = DispSizePair.first; + StDisp2 = DispSizePair.first + LdStDelta; Size1 = std::abs(std::abs(LdDisp2) - std::abs(LdDisp1)); - Size2 = inst.second; + Size2 = DispSizePair.second; + // Build a copy for the point until the current blocking store's + // displacement. buildCopies(Size1, LoadInst, LdDisp1, StoreInst, StDisp1, LMMOffset, SMMOffset); + // Build a copy for the current blocking store. buildCopies(Size2, LoadInst, LdDisp2, StoreInst, StDisp2, LMMOffset + Size1, SMMOffset + Size1); LdDisp1 = LdDisp2 + Size2; @@ -497,10 +545,10 @@ LMMOffset); } -bool FixupSFBPass::runOnMachineFunction(MachineFunction &MF) { +bool AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) { bool Changed = false; - if (DisableX86FixupSFB || skipFunction(MF.getFunction())) + if (DisableX86AvoidStoreForwardBlocks || skipFunction(MF.getFunction())) return false; MRI = &MF.getRegInfo(); @@ -508,7 +556,7 @@ TII = MF.getSubtarget().getInstrInfo(); TRI = MF.getSubtarget().getRegisterInfo(); Is64Bit = MF.getSubtarget().is64Bit(); - DEBUG(dbgs() << "Start X86FixupSFB\n";); + DEBUG(dbgs() << "Start X86AvoidStoreForwardBlocks\n";); // Look for a load then a store to XMM/YMM which look like a memcpy findPotentiallylBlockedCopies(MF); @@ -519,41 +567,41 @@ MachineOperand &LoadBase = getBaseOperand(LoadInst); int64_t LdDispImm = getDispOperand(LoadInst).getImm(); - std::map BlockingStoresDisp; + DisplacementSizeMap BlockingStoresDispSizeMap; int LdBaseReg = LoadBase.isReg() ? LoadBase.getReg() : LoadBase.getIndex(); for (auto PBInst : PotentialBlockers) { - if (isPotentialBlockingStoreInst(PBInst->getOpcode(), - LoadInst->getOpcode())) { - if (!isRelevantAddressingMode(PBInst)) - continue; - MachineOperand &PBstoreBase = getBaseOperand(PBInst); - int64_t PBstDispImm = getDispOperand(PBInst).getImm(); - assert(PBInst->hasOneMemOperand() && "Expected One Memory Operand"); - unsigned PBstSize = (*PBInst->memoperands_begin())->getSize(); - int PBstBaseReg = - PBstoreBase.isReg() ? PBstoreBase.getReg() : PBstoreBase.getIndex(); - // This check doesn't cover all cases, but it will suffice for now. - // TODO: take branch probability into consideration, if the blocking - // store is in an unreached block, breaking the memcopy could lose - // performance. - if (((LoadBase.isReg() && PBstoreBase.isReg()) || - (LoadBase.isFI() && PBstoreBase.isFI())) && - LdBaseReg == PBstBaseReg && - ((PBstDispImm >= LdDispImm) && - (PBstDispImm <= - LdDispImm + (getRegSizeInBytes(LoadInst) - PBstSize)))) { - if (BlockingStoresDisp.count(PBstDispImm)) { - if (BlockingStoresDisp[PBstDispImm] > PBstSize) - BlockingStoresDisp[PBstDispImm] = PBstSize; - - } else - BlockingStoresDisp[PBstDispImm] = PBstSize; - } + if (!isPotentialBlockingStoreInst(PBInst->getOpcode(), + LoadInst->getOpcode()) || + !isRelevantAddressingMode(PBInst)) + continue; + MachineOperand &PBstoreBase = getBaseOperand(PBInst); + int64_t PBstDispImm = getDispOperand(PBInst).getImm(); + assert(PBInst->hasOneMemOperand() && "Expected One Memory Operand"); + unsigned PBstSize = (*PBInst->memoperands_begin())->getSize(); + int PBstBaseReg = + PBstoreBase.isReg() ? PBstoreBase.getReg() : PBstoreBase.getIndex(); + // This check doesn't cover all cases, but it will suffice for now. + // TODO: take branch probability into consideration, if the blocking + // store is in an unreached block, breaking the memcopy could lose + // performance. + if (((LoadBase.isReg() && PBstoreBase.isReg()) || + (LoadBase.isFI() && PBstoreBase.isFI())) && + LdBaseReg == PBstBaseReg && + ((PBstDispImm >= LdDispImm) && + (PBstDispImm <= + LdDispImm + (getRegSizeInBytes(LoadInst) - PBstSize)))) { + if (BlockingStoresDispSizeMap.count(PBstDispImm)) { + // Choose the smallest blocking store starting at this displacement. + if (BlockingStoresDispSizeMap[PBstDispImm] > PBstSize) + BlockingStoresDispSizeMap[PBstDispImm] = PBstSize; + + } else + BlockingStoresDispSizeMap[PBstDispImm] = PBstSize; } } - if (BlockingStoresDisp.size() == 0) + if (BlockingStoresDispSizeMap.size() == 0) continue; // We found a store forward block, break the memcpy's load and store @@ -564,7 +612,7 @@ DEBUG(LoadInst->dump()); DEBUG(StoreInst->dump()); DEBUG(dbgs() << "Replaced with:\n"); - breakBlockedCopies(LoadInst, StoreInst, BlockingStoresDisp); + breakBlockedCopies(LoadInst, StoreInst, BlockingStoresDispSizeMap); updateKillStatus(LoadInst, StoreInst); ForRemoval.push_back(LoadInst); ForRemoval.push_back(StoreInst); @@ -574,7 +622,7 @@ } ForRemoval.clear(); BlockedLoadsStores.clear(); - DEBUG(dbgs() << "End X86FixupSFB\n";); + DEBUG(dbgs() << "End X86AvoidStoreForwardBlocks\n";); return Changed; } Index: lib/Target/X86/X86TargetMachine.cpp =================================================================== --- lib/Target/X86/X86TargetMachine.cpp +++ lib/Target/X86/X86TargetMachine.cpp @@ -449,7 +449,7 @@ addPass(createX86FixupSetCC()); addPass(createX86OptimizeLEAs()); addPass(createX86CallFrameOptimization()); - addPass(createX86FixupSFB()); + addPass(createX86AvoidStoreForwardingBlocks()); } addPass(createX86WinAllocaExpander()); Index: test/CodeGen/X86/avoid-sfb-32.ll =================================================================== --- test/CodeGen/X86/avoid-sfb-32.ll +++ test/CodeGen/X86/avoid-sfb-32.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=i686-linux | FileCheck %s -check-prefix=CHECK -; RUN: llc < %s -mtriple=i686-linux --disable-fixup-SFB | FileCheck %s --check-prefix=DISABLED +; RUN: llc < %s -mtriple=i686-linux --disable-avoid-SFB | FileCheck %s --check-prefix=DISABLED ; RUN: llc < %s -mtriple=i686-linux -mattr +sse4.1 | FileCheck %s -check-prefix=CHECK-AVX2 ; RUN: llc < %s -mtriple=i686-linux -mattr=+avx512f,+avx512bw,+avx512vl,+avx512dq | FileCheck %s -check-prefix=CHECK-AVX512 Index: test/CodeGen/X86/avoid-sfb.ll =================================================================== --- test/CodeGen/X86/avoid-sfb.ll +++ test/CodeGen/X86/avoid-sfb.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -check-prefix=CHECK -; RUN: llc < %s -mtriple=x86_64-linux --disable-fixup-SFB | FileCheck %s --check-prefix=DISABLED +; RUN: llc < %s -mtriple=x86_64-linux --disable-avoid-SFB | FileCheck %s --check-prefix=DISABLED ; RUN: llc < %s -mtriple=x86_64-linux -mcpu=core-avx2 | FileCheck %s -check-prefix=CHECK-AVX2 ; RUN: llc < %s -mtriple=x86_64-linux -mcpu=skx | FileCheck %s -check-prefix=CHECK-AVX512 ; RUN: llc < %s -mtriple=i686-linux -; RUN: llc < %s -mtriple=i686-linux --disable-fixup-SFB +; RUN: llc < %s -mtriple=i686-linux --disable-avoid-SFB ; RUN: llc < %s -mtriple=i686-linux -mattr sse4 ; RUN: llc < %s -mtriple=i686-linux -mattr avx512