Index: lib/Target/X86/CMakeLists.txt =================================================================== --- lib/Target/X86/CMakeLists.txt +++ lib/Target/X86/CMakeLists.txt @@ -31,6 +31,7 @@ X86FastISel.cpp X86FixupBWInsts.cpp X86FixupLEAs.cpp + X86AvoidStoreForwardingBlocks.cpp X86FixupSetCC.cpp X86FloatingPoint.cpp X86FrameLowering.cpp Index: lib/Target/X86/X86.h =================================================================== --- lib/Target/X86/X86.h +++ lib/Target/X86/X86.h @@ -70,6 +70,9 @@ /// Return a pass that transforms setcc + movzx pairs into xor + setcc. FunctionPass *createX86FixupSetCC(); +/// Return a pass that avoids creating store forward block issues in the hardware. +FunctionPass *createX86AvoidStoreForwardingBlocks(); + /// Return a pass that expands WinAlloca pseudo-instructions. FunctionPass *createX86WinAllocaExpander(); Index: lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp =================================================================== --- /dev/null +++ lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp @@ -0,0 +1,721 @@ +//===- X86AvoidStoreForwardingBlockis.cpp - Avoid HW Store Forward Block --===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// If a load follows a store and reloads data that the store has written to +// memory, Intel microarchitectures can in many cases forward the data directly +// from the store to the load, This "store forwarding" saves cycles by enabling +// the load to directly obtain the data instead of accessing the data from +// cache or memory. +// A "store forward block" occurs in cases that a store cannot be forwarded to +// the load. The most typical case of store forward block on Intel Core +// microarchitecture that a small store cannot be forwarded to a large load. +// The estimated penalty for a store forward block is ~13 cycles. +// +// This pass tries to recognize and handle cases where "store forward block" +// is created by the compiler when lowering memcpy calls to a sequence +// of a load and a store. +// +// The pass currently only handles cases where memcpy is lowered to +// XMM/YMM registers, it tries to break the memcpy into smaller copies. +// breaking the memcpy should be possible since there is no atomicity +// guarantee for loads and stores to XMM/YMM. +// +// It could be better for performance to solve the problem by loading +// to XMM/YMM then inserting the partial store before storing back from XMM/YMM +// to memory, but this will result in a more conservative optimization since it +// requires we prove that all memory accesses between the blocking store and the +// load must alias/don't alias before we can move the store, whereas the +// transformation done here is correct regardless to other memory accesses. +//===----------------------------------------------------------------------===// + +#include "X86InstrInfo.h" +#include "X86Subtarget.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/IR/DebugInfoMetadata.h" +#include "llvm/IR/DebugLoc.h" +#include "llvm/IR/Function.h" +#include "llvm/MC/MCInstrDesc.h" + +using namespace llvm; + +#define DEBUG_TYPE "x86-avoid-SFB" + +namespace llvm { +void initializeX86AvoidSFBPassPass(PassRegistry &); +} // end namespace llvm + +static cl::opt DisableX86AvoidStoreForwardBlocks( + "disable-avoid-SFB", cl::Hidden, + cl::desc("X86: Disable Store Forwarding Blocks fixup."), cl::init(false)); + +static cl::opt X86AvoidSFBInspectionLimit( + "sfb-inspection-limit", cl::desc("X86: Number of instructions backward to " + "inspect for store forwarding blocks."), + cl::init(20), cl::Hidden); + +namespace { + +using DisplacementSizeMap = std::map; + +class X86AvoidSFBPass : public MachineFunctionPass { +public: + static char ID; + X86AvoidSFBPass() : MachineFunctionPass(ID) { + initializeX86AvoidSFBPassPass(*PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { + return "X86 Avoid Store Forwarding Blocks"; + } + + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + MachineFunctionPass::getAnalysisUsage(AU); + AU.addRequired(); + } + +private: + MachineRegisterInfo *MRI; + const X86InstrInfo *TII; + const X86RegisterInfo *TRI; + SmallVector, 2> + BlockedLoadsStoresPairs; + SmallVector ForRemoval; + AliasAnalysis *AA; + + /// \brief Returns couples of Load then Store to memory which look + /// like a memcpy. + void findPotentiallylBlockedCopies(MachineFunction &MF); + /// \brief Break the memcpy's load and store into smaller copies + /// such that each memory load that was blocked by a smaller store + /// would now be copied separately. + void breakBlockedCopies(MachineInstr *LoadInst, MachineInstr *StoreInst, + const DisplacementSizeMap &BlockingStoresDispSizeMap); + /// \brief Break a copy of size Size to smaller copies. + void buildCopies(int Size, MachineInstr *LoadInst, int64_t LdDispImm, + MachineInstr *StoreInst, int64_t StDispImm, + int64_t LMMOffset, int64_t SMMOffset); + + void buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode, int64_t LoadDisp, + MachineInstr *StoreInst, unsigned NStoreOpcode, + int64_t StoreDisp, unsigned Size, int64_t LMMOffset, + int64_t SMMOffset); + + bool alias(const MachineMemOperand &Op1, const MachineMemOperand &Op2) const; + + unsigned getRegSizeInBytes(MachineInstr *Inst); +}; + +} // end anonymous namespace + +char X86AvoidSFBPass::ID = 0; + +INITIALIZE_PASS_BEGIN(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking", + false, false) +INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) +INITIALIZE_PASS_END(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking", false, + false) + +FunctionPass *llvm::createX86AvoidStoreForwardingBlocks() { + return new X86AvoidSFBPass(); +} + +static bool isXMMLoadOpcode(unsigned Opcode) { + return Opcode == X86::MOVUPSrm || Opcode == X86::MOVAPSrm || + Opcode == X86::VMOVUPSrm || Opcode == X86::VMOVAPSrm || + Opcode == X86::VMOVUPDrm || Opcode == X86::VMOVAPDrm || + Opcode == X86::VMOVDQUrm || Opcode == X86::VMOVDQArm || + Opcode == X86::VMOVUPSZ128rm || Opcode == X86::VMOVAPSZ128rm || + Opcode == X86::VMOVUPDZ128rm || Opcode == X86::VMOVAPDZ128rm || + Opcode == X86::VMOVDQU64Z128rm || Opcode == X86::VMOVDQA64Z128rm || + Opcode == X86::VMOVDQU32Z128rm || Opcode == X86::VMOVDQA32Z128rm; +} +static bool isYMMLoadOpcode(unsigned Opcode) { + return Opcode == X86::VMOVUPSYrm || Opcode == X86::VMOVAPSYrm || + Opcode == X86::VMOVUPDYrm || Opcode == X86::VMOVAPDYrm || + Opcode == X86::VMOVDQUYrm || Opcode == X86::VMOVDQAYrm || + Opcode == X86::VMOVUPSZ256rm || Opcode == X86::VMOVAPSZ256rm || + Opcode == X86::VMOVUPDZ256rm || Opcode == X86::VMOVAPDZ256rm || + Opcode == X86::VMOVDQU64Z256rm || Opcode == X86::VMOVDQA64Z256rm || + Opcode == X86::VMOVDQU32Z256rm || Opcode == X86::VMOVDQA32Z256rm; +} + +static bool isPotentialBlockedMemCpyLd(unsigned Opcode) { + return isXMMLoadOpcode(Opcode) || isYMMLoadOpcode(Opcode); +} + +static bool isPotentialBlockedMemCpyPair(int LdOpcode, int StOpcode) { + switch (LdOpcode) { + case X86::MOVUPSrm: + case X86::MOVAPSrm: + return StOpcode == X86::MOVUPSmr || StOpcode == X86::MOVAPSmr; + case X86::VMOVUPSrm: + case X86::VMOVAPSrm: + return StOpcode == X86::VMOVUPSmr || StOpcode == X86::VMOVAPSmr; + case X86::VMOVUPDrm: + case X86::VMOVAPDrm: + return StOpcode == X86::VMOVUPDmr || StOpcode == X86::VMOVAPDmr; + case X86::VMOVDQUrm: + case X86::VMOVDQArm: + return StOpcode == X86::VMOVDQUmr || StOpcode == X86::VMOVDQAmr; + case X86::VMOVUPSZ128rm: + case X86::VMOVAPSZ128rm: + return StOpcode == X86::VMOVUPSZ128mr || StOpcode == X86::VMOVAPSZ128mr; + case X86::VMOVUPDZ128rm: + case X86::VMOVAPDZ128rm: + return StOpcode == X86::VMOVUPDZ128mr || StOpcode == X86::VMOVAPDZ128mr; + case X86::VMOVUPSYrm: + case X86::VMOVAPSYrm: + return StOpcode == X86::VMOVUPSYmr || StOpcode == X86::VMOVAPSYmr; + case X86::VMOVUPDYrm: + case X86::VMOVAPDYrm: + return StOpcode == X86::VMOVUPDYmr || StOpcode == X86::VMOVAPDYmr; + case X86::VMOVDQUYrm: + case X86::VMOVDQAYrm: + return StOpcode == X86::VMOVDQUYmr || StOpcode == X86::VMOVDQAYmr; + case X86::VMOVUPSZ256rm: + case X86::VMOVAPSZ256rm: + return StOpcode == X86::VMOVUPSZ256mr || StOpcode == X86::VMOVAPSZ256mr; + case X86::VMOVUPDZ256rm: + case X86::VMOVAPDZ256rm: + return StOpcode == X86::VMOVUPDZ256mr || StOpcode == X86::VMOVAPDZ256mr; + case X86::VMOVDQU64Z128rm: + case X86::VMOVDQA64Z128rm: + return StOpcode == X86::VMOVDQU64Z128mr || StOpcode == X86::VMOVDQA64Z128mr; + case X86::VMOVDQU32Z128rm: + case X86::VMOVDQA32Z128rm: + return StOpcode == X86::VMOVDQU32Z128mr || StOpcode == X86::VMOVDQA32Z128mr; + case X86::VMOVDQU64Z256rm: + case X86::VMOVDQA64Z256rm: + return StOpcode == X86::VMOVDQU64Z256mr || StOpcode == X86::VMOVDQA64Z256mr; + case X86::VMOVDQU32Z256rm: + case X86::VMOVDQA32Z256rm: + return StOpcode == X86::VMOVDQU32Z256mr || StOpcode == X86::VMOVDQA32Z256mr; + default: + return false; + } +}; + +static bool isPotentialBlockingStoreInst(int Opcode, int LoadOpcode) { + bool PBlock = false; + PBlock |= Opcode == X86::MOV64mr || Opcode == X86::MOV64mi32 || + Opcode == X86::MOV32mr || Opcode == X86::MOV32mi || + Opcode == X86::MOV16mr || Opcode == X86::MOV16mi || + Opcode == X86::MOV8mr || Opcode == X86::MOV8mi; + if (isYMMLoadOpcode(LoadOpcode)) + PBlock |= Opcode == X86::VMOVUPSmr || Opcode == X86::VMOVAPSmr || + Opcode == X86::VMOVUPDmr || Opcode == X86::VMOVAPDmr || + Opcode == X86::VMOVDQUmr || Opcode == X86::VMOVDQAmr || + Opcode == X86::VMOVUPSZ128mr || Opcode == X86::VMOVAPSZ128mr || + Opcode == X86::VMOVUPDZ128mr || Opcode == X86::VMOVAPDZ128mr || + Opcode == X86::VMOVDQU64Z128mr || + Opcode == X86::VMOVDQA64Z128mr || + Opcode == X86::VMOVDQU32Z128mr || Opcode == X86::VMOVDQA32Z128mr; + return PBlock; +} + +static const int MOV128SZ = 16; +static const int MOV64SZ = 8; +static const int MOV32SZ = 4; +static const int MOV16SZ = 2; +static const int MOV8SZ = 1; + +static unsigned getYMMtoXMMLoadOpcode(unsigned LoadOpcode) { + switch (LoadOpcode) { + case X86::VMOVUPSYrm: + case X86::VMOVAPSYrm: + return X86::VMOVUPSrm; + case X86::VMOVUPDYrm: + case X86::VMOVAPDYrm: + return X86::VMOVUPDrm; + case X86::VMOVDQUYrm: + case X86::VMOVDQAYrm: + return X86::VMOVDQUrm; + case X86::VMOVUPSZ256rm: + case X86::VMOVAPSZ256rm: + return X86::VMOVUPSZ128rm; + case X86::VMOVUPDZ256rm: + case X86::VMOVAPDZ256rm: + return X86::VMOVUPDZ128rm; + case X86::VMOVDQU64Z256rm: + case X86::VMOVDQA64Z256rm: + return X86::VMOVDQU64Z128rm; + case X86::VMOVDQU32Z256rm: + case X86::VMOVDQA32Z256rm: + return X86::VMOVDQU32Z128rm; + default: + llvm_unreachable("Unexpected Load Instruction Opcode"); + } + return 0; +}; + +static unsigned getYMMtoXMMStoreOpcode(unsigned StoreOpcode) { + switch (StoreOpcode) { + case X86::VMOVUPSYmr: + case X86::VMOVAPSYmr: + return X86::VMOVUPSmr; + case X86::VMOVUPDYmr: + case X86::VMOVAPDYmr: + return X86::VMOVUPDmr; + case X86::VMOVDQUYmr: + case X86::VMOVDQAYmr: + return X86::VMOVDQUmr; + case X86::VMOVUPSZ256mr: + case X86::VMOVAPSZ256mr: + return X86::VMOVUPSZ128mr; + case X86::VMOVUPDZ256mr: + case X86::VMOVAPDZ256mr: + return X86::VMOVUPDZ128mr; + case X86::VMOVDQU64Z256mr: + case X86::VMOVDQA64Z256mr: + return X86::VMOVDQU64Z128mr; + case X86::VMOVDQU32Z256mr: + case X86::VMOVDQA32Z256mr: + return X86::VMOVDQU32Z128mr; + default: + llvm_unreachable("Unexpected Load Instruction Opcode"); + } + return 0; +}; + +static int getAddrOffset(MachineInstr *MI) { + const MCInstrDesc &Descl = MI->getDesc(); + int AddrOffset = X86II::getMemoryOperandNo(Descl.TSFlags); + assert(AddrOffset != -1 && "Expected Memory Operand"); + AddrOffset += X86II::getOperandBias(Descl); + return AddrOffset; +} + +static MachineOperand &getBaseOperand(MachineInstr *MI) { + int AddrOffset = getAddrOffset(MI); + return MI->getOperand(AddrOffset + X86::AddrBaseReg); +} + +static MachineOperand &getDispOperand(MachineInstr *MI) { + int AddrOffset = getAddrOffset(MI); + return MI->getOperand(AddrOffset + X86::AddrDisp); +} + +// Relevant addressing modes contain only base register and immediate +// displacement or frameindex and immediate displacement. +// TODO: Consider expanding to other addressing modes in the future +static bool isRelevantAddressingMode(MachineInstr *MI) { + int AddrOffset = getAddrOffset(MI); + MachineOperand &Base = getBaseOperand(MI); + MachineOperand &Disp = getDispOperand(MI); + MachineOperand &Scale = MI->getOperand(AddrOffset + X86::AddrScaleAmt); + MachineOperand &Index = MI->getOperand(AddrOffset + X86::AddrIndexReg); + MachineOperand &Segment = MI->getOperand(AddrOffset + X86::AddrSegmentReg); + + if (!((Base.isReg() && Base.getReg() != X86::NoRegister) || Base.isFI())) + return false; + if (!Disp.isImm()) + return false; + if (Scale.getImm() != 1) + return false; + if (!(Index.isReg() && Index.getReg() == X86::NoRegister)) + return false; + if (!(Segment.isReg() && Segment.getReg() == X86::NoRegister)) + return false; + return true; +} + +// Collect potentially blocking stores. +// Limit the number of instructions backwards we want to inspect +// since the effect of store block won't be visible if the store +// and load instructions have enough instructions in between to +// keep the core busy. +static SmallVector +findPotentialBlockers(MachineInstr *LoadInst) { + SmallVector PotentialBlockers; + unsigned BlockLimit = 0; + unsigned InspectionLimit = X86AvoidSFBInspectionLimit; + for (MachineBasicBlock::iterator LI = LoadInst, + BB = LoadInst->getParent()->begin(); + LI != BB; --LI) { + BlockLimit++; + if (BlockLimit >= InspectionLimit) + break; + MachineInstr &MI = *LI; + if (MI.getDesc().isCall()) + break; + PotentialBlockers.push_back(&MI); + } + // If we didn't get to the instructions limit try predecessing blocks. + // Ideally we should traverse the predecessor blocks in depth with some + // coloring algorithm, but for now let's just look at the first order + // predecessors. + if (BlockLimit < InspectionLimit) { + MachineBasicBlock *MBB = LoadInst->getParent(); + int LimitLeft = InspectionLimit - BlockLimit; + for (MachineBasicBlock::pred_iterator PB = MBB->pred_begin(), + PE = MBB->pred_end(); + PB != PE; ++PB) { + MachineBasicBlock *PMBB = *PB; + int PredLimit = 0; + for (MachineBasicBlock::reverse_iterator PMI = PMBB->rbegin(), + PME = PMBB->rend(); + PMI != PME; ++PMI) { + PredLimit++; + if (PredLimit >= LimitLeft) + break; + if (PMI->getDesc().isCall()) + break; + PotentialBlockers.push_back(&*PMI); + } + } + } + return PotentialBlockers; +} + +void X86AvoidSFBPass::buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode, + int64_t LoadDisp, MachineInstr *StoreInst, + unsigned NStoreOpcode, int64_t StoreDisp, + unsigned Size, int64_t LMMOffset, + int64_t SMMOffset) { + MachineOperand &LoadBase = getBaseOperand(LoadInst); + MachineOperand &StoreBase = getBaseOperand(StoreInst); + MachineBasicBlock *MBB = LoadInst->getParent(); + MachineMemOperand *LMMO = *LoadInst->memoperands_begin(); + MachineMemOperand *SMMO = *StoreInst->memoperands_begin(); + + unsigned Reg1 = MRI->createVirtualRegister( + TII->getRegClass(TII->get(NLoadOpcode), 0, TRI, *(MBB->getParent()))); + BuildMI(*MBB, LoadInst, LoadInst->getDebugLoc(), TII->get(NLoadOpcode), Reg1) + .add(LoadBase) + .addImm(1) + .addReg(X86::NoRegister) + .addImm(LoadDisp) + .addReg(X86::NoRegister) + .addMemOperand( + MBB->getParent()->getMachineMemOperand(LMMO, LMMOffset, Size)); + DEBUG(LoadInst->getPrevNode()->dump()); + // If the load and store are consecutive, use the loadInst location to + // reduce register pressure. + MachineInstr *StInst = StoreInst; + if (StoreInst->getPrevNode() == LoadInst) + StInst = LoadInst; + BuildMI(*MBB, StInst, StInst->getDebugLoc(), TII->get(NStoreOpcode)) + .add(StoreBase) + .addImm(1) + .addReg(X86::NoRegister) + .addImm(StoreDisp) + .addReg(X86::NoRegister) + .addReg(Reg1) + .addMemOperand( + MBB->getParent()->getMachineMemOperand(SMMO, SMMOffset, Size)); + DEBUG(StInst->getPrevNode()->dump()); +} + +void X86AvoidSFBPass::buildCopies(int Size, MachineInstr *LoadInst, + int64_t LdDispImm, MachineInstr *StoreInst, + int64_t StDispImm, int64_t LMMOffset, + int64_t SMMOffset) { + int LdDisp = LdDispImm; + int StDisp = StDispImm; + while (Size > 0) { + if ((Size - MOV128SZ >= 0) && isYMMLoadOpcode(LoadInst->getOpcode())) { + Size = Size - MOV128SZ; + buildCopy(LoadInst, getYMMtoXMMLoadOpcode(LoadInst->getOpcode()), LdDisp, + StoreInst, getYMMtoXMMStoreOpcode(StoreInst->getOpcode()), + StDisp, MOV128SZ, LMMOffset, SMMOffset); + LdDisp += MOV128SZ; + StDisp += MOV128SZ; + LMMOffset += MOV128SZ; + SMMOffset += MOV128SZ; + continue; + } + if (Size - MOV64SZ >= 0) { + Size = Size - MOV64SZ; + buildCopy(LoadInst, X86::MOV64rm, LdDisp, StoreInst, X86::MOV64mr, StDisp, + MOV64SZ, LMMOffset, SMMOffset); + LdDisp += MOV64SZ; + StDisp += MOV64SZ; + LMMOffset += MOV64SZ; + SMMOffset += MOV64SZ; + continue; + } + if (Size - MOV32SZ >= 0) { + Size = Size - MOV32SZ; + buildCopy(LoadInst, X86::MOV32rm, LdDisp, StoreInst, X86::MOV32mr, StDisp, + MOV32SZ, LMMOffset, SMMOffset); + LdDisp += MOV32SZ; + StDisp += MOV32SZ; + LMMOffset += MOV32SZ; + SMMOffset += MOV32SZ; + continue; + } + if (Size - MOV16SZ >= 0) { + Size = Size - MOV16SZ; + buildCopy(LoadInst, X86::MOV16rm, LdDisp, StoreInst, X86::MOV16mr, StDisp, + MOV16SZ, LMMOffset, SMMOffset); + LdDisp += MOV16SZ; + StDisp += MOV16SZ; + LMMOffset += MOV16SZ; + SMMOffset += MOV16SZ; + continue; + } + if (Size - MOV8SZ >= 0) { + Size = Size - MOV8SZ; + buildCopy(LoadInst, X86::MOV8rm, LdDisp, StoreInst, X86::MOV8mr, StDisp, + MOV8SZ, LMMOffset, SMMOffset); + LdDisp += MOV8SZ; + StDisp += MOV8SZ; + LMMOffset += MOV8SZ; + SMMOffset += MOV8SZ; + continue; + } + } + assert(Size == 0 && "Wrong size division"); +} + +static void updateKillStatus(MachineInstr *LoadInst, MachineInstr *StoreInst) { + MachineOperand &LoadBase = getBaseOperand(LoadInst); + MachineOperand &StoreBase = getBaseOperand(StoreInst); + if (LoadBase.isReg()) { + MachineInstr *LastLoad = LoadInst->getPrevNode(); + // If the original load and store to xmm/ymm were consecutive + // then the partial copies were also created in + // a consecutive order to reduce register pressure, + // and the location of the last load is before the last store. + if (StoreInst->getPrevNode() == LoadInst) + LastLoad = LoadInst->getPrevNode()->getPrevNode(); + getBaseOperand(LastLoad).setIsKill(LoadBase.isKill()); + } + if (StoreBase.isReg()) { + MachineInstr *StInst = StoreInst; + if (StoreInst->getPrevNode() == LoadInst) + StInst = LoadInst; + getBaseOperand(StInst->getPrevNode()).setIsKill(StoreBase.isKill()); + } +} + +bool X86AvoidSFBPass::alias(const MachineMemOperand &Op1, + const MachineMemOperand &Op2) const { + if (!Op1.getValue() || !Op2.getValue()) + return true; + + int64_t MinOffset = std::min(Op1.getOffset(), Op2.getOffset()); + int64_t Overlapa = Op1.getSize() + Op1.getOffset() - MinOffset; + int64_t Overlapb = Op2.getSize() + Op2.getOffset() - MinOffset; + + AliasResult AAResult = + AA->alias(MemoryLocation(Op1.getValue(), Overlapa, Op1.getAAInfo()), + MemoryLocation(Op2.getValue(), Overlapb, Op2.getAAInfo())); + return AAResult != NoAlias; +} + +void X86AvoidSFBPass::findPotentiallylBlockedCopies(MachineFunction &MF) { + for (auto &MBB : MF) + for (auto &MI : MBB) { + if (!isPotentialBlockedMemCpyLd(MI.getOpcode())) + continue; + int DefVR = MI.getOperand(0).getReg(); + if (!MRI->hasOneUse(DefVR)) + continue; + for (auto UI = MRI->use_nodbg_begin(DefVR), UE = MRI->use_nodbg_end(); + UI != UE;) { + MachineOperand &StoreMO = *UI++; + MachineInstr &StoreMI = *StoreMO.getParent(); + // Skip cases where the memcpy may overlap. + if (isPotentialBlockedMemCpyPair(MI.getOpcode(), StoreMI.getOpcode()) && + StoreMI.getParent() == MI.getParent() && + isRelevantAddressingMode(&MI) && + isRelevantAddressingMode(&StoreMI) && + !alias(**MI.memoperands_begin(), **StoreMI.memoperands_begin())) + BlockedLoadsStoresPairs.push_back( + std::pair(&MI, &StoreMI)); + } + } +} + +unsigned X86AvoidSFBPass::getRegSizeInBytes(MachineInstr *LoadInst) { + auto TRC = TII->getRegClass(TII->get(LoadInst->getOpcode()), 0, TRI, + *LoadInst->getParent()->getParent()); + return TRI->getRegSizeInBits(*TRC) / 8; +} + +void X86AvoidSFBPass::breakBlockedCopies( + MachineInstr *LoadInst, MachineInstr *StoreInst, + const DisplacementSizeMap &BlockingStoresDispSizeMap) { + int64_t LdDispImm = getDispOperand(LoadInst).getImm(); + int64_t StDispImm = getDispOperand(StoreInst).getImm(); + int64_t LMMOffset = (*LoadInst->memoperands_begin())->getOffset(); + int64_t SMMOffset = (*StoreInst->memoperands_begin())->getOffset(); + + int64_t LdDisp1 = LdDispImm; + int64_t LdDisp2 = 0; + int64_t StDisp1 = StDispImm; + int64_t StDisp2 = 0; + unsigned Size1 = 0; + unsigned Size2 = 0; + int64_t LdStDelta = StDispImm - LdDispImm; + + for (auto DispSizePair : BlockingStoresDispSizeMap) { + LdDisp2 = DispSizePair.first; + StDisp2 = DispSizePair.first + LdStDelta; + Size2 = DispSizePair.second; + // Avoid copying overlapping areas. + if (LdDisp2 < LdDisp1) { + int overlapDelta = LdDisp1 - LdDisp2; + LdDisp2 += overlapDelta; + StDisp2 += overlapDelta; + Size2 -= overlapDelta; + } + Size1 = std::abs(std::abs(LdDisp2) - std::abs(LdDisp1)); + + // Build a copy for the point until the current blocking store's + // displacement. + buildCopies(Size1, LoadInst, LdDisp1, StoreInst, StDisp1, LMMOffset, + SMMOffset); + // Build a copy for the current blocking store. + buildCopies(Size2, LoadInst, LdDisp2, StoreInst, StDisp2, LMMOffset + Size1, + SMMOffset + Size1); + LdDisp1 = LdDisp2 + Size2; + StDisp1 = StDisp2 + Size2; + LMMOffset += Size1 + Size2; + SMMOffset += Size1 + Size2; + } + unsigned Size3 = (LdDispImm + getRegSizeInBytes(LoadInst)) - LdDisp1; + buildCopies(Size3, LoadInst, LdDisp1, StoreInst, StDisp1, LMMOffset, + LMMOffset); +} + +static int getBaseOperandValue(MachineOperand &BaseOperand) { + return BaseOperand.isReg() ? BaseOperand.getReg() : BaseOperand.getIndex(); +} + +static bool hasSameBaseOpValue(MachineInstr *LoadInst, + MachineInstr *StoreInst) { + MachineOperand &LoadBase = getBaseOperand(LoadInst); + MachineOperand &StoreBase = getBaseOperand(StoreInst); + int LoadBaseVal = getBaseOperandValue(LoadBase); + int StoreBaseVal = getBaseOperandValue(StoreBase); + return ((LoadBase.isReg() && StoreBase.isReg()) || + (LoadBase.isFI() && StoreBase.isFI())) && + LoadBaseVal == StoreBaseVal; +} + +static bool isBlockingStore(int64_t LoadDispImm, unsigned LoadSize, + int64_t StoreDispImm, unsigned StoreSize) { + return ((StoreDispImm >= LoadDispImm) && + (StoreDispImm <= LoadDispImm + (LoadSize - StoreSize))); +} + +// Keep track of all stores blocking a load +static void +updateBlockingStoresDispSizeMap(DisplacementSizeMap &BlockingStoresDispSizeMap, + int64_t DispImm, unsigned Size) { + if (BlockingStoresDispSizeMap.count(DispImm)) { + // Choose the smallest blocking store starting at this displacement. + if (BlockingStoresDispSizeMap[DispImm] > Size) + BlockingStoresDispSizeMap[DispImm] = Size; + + } else + BlockingStoresDispSizeMap[DispImm] = Size; +} + +// Remove blocking stores contained in each other. +static void +removeRedundantBlockingStores(DisplacementSizeMap &BlockingStoresDispSizeMap) { + if (BlockingStoresDispSizeMap.size() <= 1) + return; + + int64_t prevDisp = BlockingStoresDispSizeMap.begin()->first; + unsigned prevSize = BlockingStoresDispSizeMap.begin()->second; + SmallVector forRemoval; + for (auto inst = std::next(BlockingStoresDispSizeMap.begin()); + inst != BlockingStoresDispSizeMap.end(); ++inst) { + int64_t currDisp = inst->first; + unsigned currSize = inst->second; + if (currDisp + currSize <= prevDisp + prevSize) { + forRemoval.push_back(prevDisp); + } + prevDisp = currDisp; + prevSize = currSize; + } + for (auto disp : forRemoval) + BlockingStoresDispSizeMap.erase(disp); +} + +bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) { + bool Changed = false; + + if (DisableX86AvoidStoreForwardBlocks || skipFunction(MF.getFunction()) || + !MF.getSubtarget().is64Bit()) + return false; + + MRI = &MF.getRegInfo(); + assert(MRI->isSSA() && "Expected MIR to be in SSA form"); + TII = MF.getSubtarget().getInstrInfo(); + TRI = MF.getSubtarget().getRegisterInfo(); + AA = &getAnalysis().getAAResults(); + DEBUG(dbgs() << "Start X86AvoidStoreForwardBlocks\n";); + // Look for a load then a store to XMM/YMM which look like a memcpy + findPotentiallylBlockedCopies(MF); + + for (auto LoadStoreInstPair : BlockedLoadsStoresPairs) { + MachineInstr *LoadInst = LoadStoreInstPair.first; + int64_t LdDispImm = getDispOperand(LoadInst).getImm(); + DisplacementSizeMap BlockingStoresDispSizeMap; + + SmallVector PotentialBlockers = + findPotentialBlockers(LoadInst); + for (auto PBInst : PotentialBlockers) { + if (!isPotentialBlockingStoreInst(PBInst->getOpcode(), + LoadInst->getOpcode()) || + !isRelevantAddressingMode(PBInst)) + continue; + int64_t PBstDispImm = getDispOperand(PBInst).getImm(); + assert(PBInst->hasOneMemOperand() && "Expected One Memory Operand"); + unsigned PBstSize = (*PBInst->memoperands_begin())->getSize(); + // This check doesn't cover all cases, but it will suffice for now. + // TODO: take branch probability into consideration, if the blocking + // store is in an unreached block, breaking the memcopy could lose + // performance. + if (hasSameBaseOpValue(LoadInst, PBInst) && + isBlockingStore(LdDispImm, getRegSizeInBytes(LoadInst), + PBstDispImm, PBstSize)) + updateBlockingStoresDispSizeMap(BlockingStoresDispSizeMap, PBstDispImm, + PBstSize); + } + + if (BlockingStoresDispSizeMap.size() == 0) + continue; + + // We found a store forward block, break the memcpy's load and store + // into smaller copies such that each smaller store that was causing + // a store block would now be copied separately. + MachineInstr *StoreInst = LoadStoreInstPair.second; + DEBUG(dbgs() << "Blocked load and store instructions: \n"); + DEBUG(LoadInst->dump()); + DEBUG(StoreInst->dump()); + DEBUG(dbgs() << "Replaced with:\n"); + removeRedundantBlockingStores(BlockingStoresDispSizeMap); + breakBlockedCopies(LoadInst, StoreInst, BlockingStoresDispSizeMap); + updateKillStatus(LoadInst, StoreInst); + ForRemoval.push_back(LoadInst); + ForRemoval.push_back(StoreInst); + } + for (auto RemovedInst : ForRemoval) { + RemovedInst->eraseFromParent(); + } + ForRemoval.clear(); + BlockedLoadsStoresPairs.clear(); + DEBUG(dbgs() << "End X86AvoidStoreForwardBlocks\n";); + + return Changed; +} Index: lib/Target/X86/X86TargetMachine.cpp =================================================================== --- lib/Target/X86/X86TargetMachine.cpp +++ lib/Target/X86/X86TargetMachine.cpp @@ -62,6 +62,7 @@ void initializeX86CmovConverterPassPass(PassRegistry &); void initializeX86ExecutionDomainFixPass(PassRegistry &); void initializeX86DomainReassignmentPass(PassRegistry &); +void initializeX86AvoidSFBPassPass(PassRegistry &); } // end namespace llvm @@ -80,6 +81,7 @@ initializeX86CmovConverterPassPass(PR); initializeX86ExecutionDomainFixPass(PR); initializeX86DomainReassignmentPass(PR); + initializeX86AvoidSFBPassPass(PR); } static std::unique_ptr createTLOF(const Triple &TT) { @@ -449,6 +451,7 @@ addPass(createX86FixupSetCC()); addPass(createX86OptimizeLEAs()); addPass(createX86CallFrameOptimization()); + addPass(createX86AvoidStoreForwardingBlocks()); } addPass(createX86WinAllocaExpander()); Index: test/CodeGen/X86/avoid-sfb-overlaps.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/avoid-sfb-overlaps.ll @@ -0,0 +1,521 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -check-prefix=CHECK +; RUN: llc < %s -mtriple=x86_64-linux --disable-avoid-SFB | FileCheck %s --check-prefix=DISABLED +; RUN: llc < %s -mtriple=x86_64-linux -mcpu=core-avx2 | FileCheck %s -check-prefix=CHECK-AVX2 +; RUN: llc < %s -mtriple=x86_64-linux -mcpu=skx | FileCheck %s -check-prefix=CHECK-AVX512 + +; ModuleID = '../testSFB/testOverlapBlocks.c' +source_filename = "../testSFB/testOverlapBlocks.c" +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +; Function Attrs: nounwind uwtable +define dso_local void @test_overlap_1(i8* nocapture %A, i32 %x) local_unnamed_addr #0 { +; CHECK-LABEL: test_overlap_1: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl $7, -8(%rdi) +; CHECK-NEXT: movq -16(%rdi), %rax +; CHECK-NEXT: movq %rax, (%rdi) +; CHECK-NEXT: movl -8(%rdi), %eax +; CHECK-NEXT: movl %eax, 8(%rdi) +; CHECK-NEXT: movl -4(%rdi), %eax +; CHECK-NEXT: movl %eax, 12(%rdi) +; CHECK-NEXT: movslq %esi, %rax +; CHECK-NEXT: movq %rax, -9(%rdi) +; CHECK-NEXT: movq %rax, -16(%rdi) +; CHECK-NEXT: movb $0, -1(%rdi) +; CHECK-NEXT: movq -16(%rdi), %rax +; CHECK-NEXT: movq %rax, 16(%rdi) +; CHECK-NEXT: movl -8(%rdi), %eax +; CHECK-NEXT: movl %eax, 24(%rdi) +; CHECK-NEXT: movzwl -4(%rdi), %eax +; CHECK-NEXT: movw %ax, 28(%rdi) +; CHECK-NEXT: movb -2(%rdi), %al +; CHECK-NEXT: movb %al, 30(%rdi) +; CHECK-NEXT: movb -1(%rdi), %al +; CHECK-NEXT: movb %al, 31(%rdi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_overlap_1: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movl $7, -8(%rdi) +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rdi) +; DISABLED-NEXT: movslq %esi, %rax +; DISABLED-NEXT: movq %rax, -9(%rdi) +; DISABLED-NEXT: movq %rax, -16(%rdi) +; DISABLED-NEXT: movb $0, -1(%rdi) +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, 16(%rdi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_overlap_1: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: movl $7, -8(%rdi) +; CHECK-AVX2-NEXT: movq -16(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, (%rdi) +; CHECK-AVX2-NEXT: movl -8(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 8(%rdi) +; CHECK-AVX2-NEXT: movl -4(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 12(%rdi) +; CHECK-AVX2-NEXT: movslq %esi, %rax +; CHECK-AVX2-NEXT: movq %rax, -9(%rdi) +; CHECK-AVX2-NEXT: movq %rax, -16(%rdi) +; CHECK-AVX2-NEXT: movb $0, -1(%rdi) +; CHECK-AVX2-NEXT: movq -16(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 16(%rdi) +; CHECK-AVX2-NEXT: movl -8(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 24(%rdi) +; CHECK-AVX2-NEXT: movzwl -4(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 28(%rdi) +; CHECK-AVX2-NEXT: movb -2(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 30(%rdi) +; CHECK-AVX2-NEXT: movb -1(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 31(%rdi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_overlap_1: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: movl $7, -8(%rdi) +; CHECK-AVX512-NEXT: movq -16(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, (%rdi) +; CHECK-AVX512-NEXT: movl -8(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 8(%rdi) +; CHECK-AVX512-NEXT: movl -4(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 12(%rdi) +; CHECK-AVX512-NEXT: movslq %esi, %rax +; CHECK-AVX512-NEXT: movq %rax, -9(%rdi) +; CHECK-AVX512-NEXT: movq %rax, -16(%rdi) +; CHECK-AVX512-NEXT: movb $0, -1(%rdi) +; CHECK-AVX512-NEXT: movq -16(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 16(%rdi) +; CHECK-AVX512-NEXT: movl -8(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 24(%rdi) +; CHECK-AVX512-NEXT: movzwl -4(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 28(%rdi) +; CHECK-AVX512-NEXT: movb -2(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 30(%rdi) +; CHECK-AVX512-NEXT: movb -1(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 31(%rdi) +; CHECK-AVX512-NEXT: retq +entry: + %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16 + %add.ptr1 = getelementptr inbounds i8, i8* %A, i64 -8 + %0 = bitcast i8* %add.ptr1 to i32* + store i32 7, i32* %0, align 4 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + %conv = sext i32 %x to i64 + %add.ptr2 = getelementptr inbounds i8, i8* %A, i64 -9 + %1 = bitcast i8* %add.ptr2 to i64* + store i64 %conv, i64* %1, align 8 + %2 = bitcast i8* %add.ptr to i64* + store i64 %conv, i64* %2, align 8 + %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 -1 + store i8 0, i8* %add.ptr5, align 1 + %add.ptr6 = getelementptr inbounds i8, i8* %A, i64 16 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr6, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + ret void +} + +; Function Attrs: argmemonly nounwind +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1 + +; Function Attrs: nounwind uwtable +define dso_local void @test_overlap_2(i8* nocapture %A, i32 %x) local_unnamed_addr #0 { +; CHECK-LABEL: test_overlap_2: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movslq %esi, %rax +; CHECK-NEXT: movq %rax, -16(%rdi) +; CHECK-NEXT: movq -16(%rdi), %rcx +; CHECK-NEXT: movq %rcx, (%rdi) +; CHECK-NEXT: movq -8(%rdi), %rcx +; CHECK-NEXT: movq %rcx, 8(%rdi) +; CHECK-NEXT: movq %rax, -8(%rdi) +; CHECK-NEXT: movl $7, -12(%rdi) +; CHECK-NEXT: movl -16(%rdi), %eax +; CHECK-NEXT: movl %eax, 16(%rdi) +; CHECK-NEXT: movl -12(%rdi), %eax +; CHECK-NEXT: movl %eax, 20(%rdi) +; CHECK-NEXT: movq -8(%rdi), %rax +; CHECK-NEXT: movq %rax, 24(%rdi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_overlap_2: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movslq %esi, %rax +; DISABLED-NEXT: movq %rax, -16(%rdi) +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rdi) +; DISABLED-NEXT: movq %rax, -8(%rdi) +; DISABLED-NEXT: movl $7, -12(%rdi) +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, 16(%rdi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_overlap_2: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: movslq %esi, %rax +; CHECK-AVX2-NEXT: movq %rax, -16(%rdi) +; CHECK-AVX2-NEXT: movq -16(%rdi), %rcx +; CHECK-AVX2-NEXT: movq %rcx, (%rdi) +; CHECK-AVX2-NEXT: movq -8(%rdi), %rcx +; CHECK-AVX2-NEXT: movq %rcx, 8(%rdi) +; CHECK-AVX2-NEXT: movq %rax, -8(%rdi) +; CHECK-AVX2-NEXT: movl $7, -12(%rdi) +; CHECK-AVX2-NEXT: movl -16(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 16(%rdi) +; CHECK-AVX2-NEXT: movl -12(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 20(%rdi) +; CHECK-AVX2-NEXT: movq -8(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 24(%rdi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_overlap_2: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: movslq %esi, %rax +; CHECK-AVX512-NEXT: movq %rax, -16(%rdi) +; CHECK-AVX512-NEXT: movq -16(%rdi), %rcx +; CHECK-AVX512-NEXT: movq %rcx, (%rdi) +; CHECK-AVX512-NEXT: movq -8(%rdi), %rcx +; CHECK-AVX512-NEXT: movq %rcx, 8(%rdi) +; CHECK-AVX512-NEXT: movq %rax, -8(%rdi) +; CHECK-AVX512-NEXT: movl $7, -12(%rdi) +; CHECK-AVX512-NEXT: movl -16(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 16(%rdi) +; CHECK-AVX512-NEXT: movl -12(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 20(%rdi) +; CHECK-AVX512-NEXT: movq -8(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 24(%rdi) +; CHECK-AVX512-NEXT: retq +entry: + %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16 + %conv = sext i32 %x to i64 + %0 = bitcast i8* %add.ptr to i64* + store i64 %conv, i64* %0, align 8 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + %add.ptr3 = getelementptr inbounds i8, i8* %A, i64 -8 + %1 = bitcast i8* %add.ptr3 to i64* + store i64 %conv, i64* %1, align 8 + %add.ptr4 = getelementptr inbounds i8, i8* %A, i64 -12 + %2 = bitcast i8* %add.ptr4 to i32* + store i32 7, i32* %2, align 4 + %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 16 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr5, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + ret void +} + +; Function Attrs: nounwind uwtable +define dso_local void @test_overlap_3(i8* nocapture %A, i32 %x) local_unnamed_addr #0 { +; CHECK-LABEL: test_overlap_3: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl $7, -10(%rdi) +; CHECK-NEXT: movl -16(%rdi), %eax +; CHECK-NEXT: movl %eax, (%rdi) +; CHECK-NEXT: movzwl -12(%rdi), %eax +; CHECK-NEXT: movw %ax, 4(%rdi) +; CHECK-NEXT: movl -10(%rdi), %eax +; CHECK-NEXT: movl %eax, 6(%rdi) +; CHECK-NEXT: movl -6(%rdi), %eax +; CHECK-NEXT: movl %eax, 10(%rdi) +; CHECK-NEXT: movzwl -2(%rdi), %eax +; CHECK-NEXT: movw %ax, 14(%rdi) +; CHECK-NEXT: movslq %esi, %rax +; CHECK-NEXT: movq %rax, -9(%rdi) +; CHECK-NEXT: movq %rax, -16(%rdi) +; CHECK-NEXT: movb $0, -1(%rdi) +; CHECK-NEXT: movq -16(%rdi), %rax +; CHECK-NEXT: movq %rax, 16(%rdi) +; CHECK-NEXT: movzwl -8(%rdi), %eax +; CHECK-NEXT: movw %ax, 24(%rdi) +; CHECK-NEXT: movl -6(%rdi), %eax +; CHECK-NEXT: movl %eax, 26(%rdi) +; CHECK-NEXT: movb -2(%rdi), %al +; CHECK-NEXT: movb %al, 30(%rdi) +; CHECK-NEXT: movb -1(%rdi), %al +; CHECK-NEXT: movb %al, 31(%rdi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_overlap_3: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movl $7, -10(%rdi) +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rdi) +; DISABLED-NEXT: movslq %esi, %rax +; DISABLED-NEXT: movq %rax, -9(%rdi) +; DISABLED-NEXT: movq %rax, -16(%rdi) +; DISABLED-NEXT: movb $0, -1(%rdi) +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, 16(%rdi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_overlap_3: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: movl $7, -10(%rdi) +; CHECK-AVX2-NEXT: movl -16(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, (%rdi) +; CHECK-AVX2-NEXT: movzwl -12(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 4(%rdi) +; CHECK-AVX2-NEXT: movl -10(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 6(%rdi) +; CHECK-AVX2-NEXT: movl -6(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 10(%rdi) +; CHECK-AVX2-NEXT: movzwl -2(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 14(%rdi) +; CHECK-AVX2-NEXT: movslq %esi, %rax +; CHECK-AVX2-NEXT: movq %rax, -9(%rdi) +; CHECK-AVX2-NEXT: movq %rax, -16(%rdi) +; CHECK-AVX2-NEXT: movb $0, -1(%rdi) +; CHECK-AVX2-NEXT: movq -16(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 16(%rdi) +; CHECK-AVX2-NEXT: movzwl -8(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 24(%rdi) +; CHECK-AVX2-NEXT: movl -6(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 26(%rdi) +; CHECK-AVX2-NEXT: movb -2(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 30(%rdi) +; CHECK-AVX2-NEXT: movb -1(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 31(%rdi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_overlap_3: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: movl $7, -10(%rdi) +; CHECK-AVX512-NEXT: movl -16(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, (%rdi) +; CHECK-AVX512-NEXT: movzwl -12(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 4(%rdi) +; CHECK-AVX512-NEXT: movl -10(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 6(%rdi) +; CHECK-AVX512-NEXT: movl -6(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 10(%rdi) +; CHECK-AVX512-NEXT: movzwl -2(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 14(%rdi) +; CHECK-AVX512-NEXT: movslq %esi, %rax +; CHECK-AVX512-NEXT: movq %rax, -9(%rdi) +; CHECK-AVX512-NEXT: movq %rax, -16(%rdi) +; CHECK-AVX512-NEXT: movb $0, -1(%rdi) +; CHECK-AVX512-NEXT: movq -16(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 16(%rdi) +; CHECK-AVX512-NEXT: movzwl -8(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 24(%rdi) +; CHECK-AVX512-NEXT: movl -6(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 26(%rdi) +; CHECK-AVX512-NEXT: movb -2(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 30(%rdi) +; CHECK-AVX512-NEXT: movb -1(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 31(%rdi) +; CHECK-AVX512-NEXT: retq +entry: + %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16 + %add.ptr1 = getelementptr inbounds i8, i8* %A, i64 -10 + %0 = bitcast i8* %add.ptr1 to i32* + store i32 7, i32* %0, align 4 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + %conv = sext i32 %x to i64 + %add.ptr2 = getelementptr inbounds i8, i8* %A, i64 -9 + %1 = bitcast i8* %add.ptr2 to i64* + store i64 %conv, i64* %1, align 8 + %2 = bitcast i8* %add.ptr to i64* + store i64 %conv, i64* %2, align 8 + %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 -1 + store i8 0, i8* %add.ptr5, align 1 + %add.ptr6 = getelementptr inbounds i8, i8* %A, i64 16 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr6, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + ret void +} + +; Function Attrs: nounwind uwtable +define dso_local void @test_overlap_4(i8* nocapture %A, i32 %x) local_unnamed_addr #0 { +; CHECK-LABEL: test_overlap_4: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movups -16(%rdi), %xmm0 +; CHECK-NEXT: movups %xmm0, (%rdi) +; CHECK-NEXT: movslq %esi, %rax +; CHECK-NEXT: movq %rax, -8(%rdi) +; CHECK-NEXT: movl %eax, -16(%rdi) +; CHECK-NEXT: movl $0, -11(%rdi) +; CHECK-NEXT: movl -16(%rdi), %eax +; CHECK-NEXT: movl %eax, 16(%rdi) +; CHECK-NEXT: movb -12(%rdi), %al +; CHECK-NEXT: movb %al, 20(%rdi) +; CHECK-NEXT: movl -11(%rdi), %eax +; CHECK-NEXT: movl %eax, 21(%rdi) +; CHECK-NEXT: movl -7(%rdi), %eax +; CHECK-NEXT: movl %eax, 25(%rdi) +; CHECK-NEXT: movzwl -3(%rdi), %eax +; CHECK-NEXT: movw %ax, 29(%rdi) +; CHECK-NEXT: movb -1(%rdi), %al +; CHECK-NEXT: movb %al, 31(%rdi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_overlap_4: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rdi) +; DISABLED-NEXT: movslq %esi, %rax +; DISABLED-NEXT: movq %rax, -8(%rdi) +; DISABLED-NEXT: movl %eax, -16(%rdi) +; DISABLED-NEXT: movl $0, -11(%rdi) +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, 16(%rdi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_overlap_4: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: vmovups -16(%rdi), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%rdi) +; CHECK-AVX2-NEXT: movslq %esi, %rax +; CHECK-AVX2-NEXT: movq %rax, -8(%rdi) +; CHECK-AVX2-NEXT: movl %eax, -16(%rdi) +; CHECK-AVX2-NEXT: movl $0, -11(%rdi) +; CHECK-AVX2-NEXT: movl -16(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 16(%rdi) +; CHECK-AVX2-NEXT: movb -12(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 20(%rdi) +; CHECK-AVX2-NEXT: movl -11(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 21(%rdi) +; CHECK-AVX2-NEXT: movl -7(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 25(%rdi) +; CHECK-AVX2-NEXT: movzwl -3(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 29(%rdi) +; CHECK-AVX2-NEXT: movb -1(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 31(%rdi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_overlap_4: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: vmovups -16(%rdi), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%rdi) +; CHECK-AVX512-NEXT: movslq %esi, %rax +; CHECK-AVX512-NEXT: movq %rax, -8(%rdi) +; CHECK-AVX512-NEXT: movl %eax, -16(%rdi) +; CHECK-AVX512-NEXT: movl $0, -11(%rdi) +; CHECK-AVX512-NEXT: movl -16(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 16(%rdi) +; CHECK-AVX512-NEXT: movb -12(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 20(%rdi) +; CHECK-AVX512-NEXT: movl -11(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 21(%rdi) +; CHECK-AVX512-NEXT: movl -7(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 25(%rdi) +; CHECK-AVX512-NEXT: movzwl -3(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 29(%rdi) +; CHECK-AVX512-NEXT: movb -1(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 31(%rdi) +; CHECK-AVX512-NEXT: retq +entry: + %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + %conv = sext i32 %x to i64 + %add.ptr1 = getelementptr inbounds i8, i8* %A, i64 -8 + %0 = bitcast i8* %add.ptr1 to i64* + store i64 %conv, i64* %0, align 8 + %1 = bitcast i8* %add.ptr to i32* + store i32 %x, i32* %1, align 4 + %add.ptr3 = getelementptr inbounds i8, i8* %A, i64 -11 + %2 = bitcast i8* %add.ptr3 to i32* + store i32 0, i32* %2, align 4 + %add.ptr4 = getelementptr inbounds i8, i8* %A, i64 16 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr4, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + ret void +} + +; Function Attrs: nounwind uwtable +define dso_local void @test_overlap_5(i8* nocapture %A, i32 %x) local_unnamed_addr #0 { +; CHECK-LABEL: test_overlap_5: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movups -16(%rdi), %xmm0 +; CHECK-NEXT: movups %xmm0, (%rdi) +; CHECK-NEXT: movslq %esi, %rax +; CHECK-NEXT: movq %rax, -16(%rdi) +; CHECK-NEXT: movb %al, -14(%rdi) +; CHECK-NEXT: movb $0, -11(%rdi) +; CHECK-NEXT: movzwl -16(%rdi), %eax +; CHECK-NEXT: movw %ax, 16(%rdi) +; CHECK-NEXT: movb -14(%rdi), %al +; CHECK-NEXT: movb %al, 18(%rdi) +; CHECK-NEXT: movzwl -13(%rdi), %eax +; CHECK-NEXT: movw %ax, 19(%rdi) +; CHECK-NEXT: movb -11(%rdi), %al +; CHECK-NEXT: movb %al, 21(%rdi) +; CHECK-NEXT: movq -10(%rdi), %rax +; CHECK-NEXT: movq %rax, 22(%rdi) +; CHECK-NEXT: movzwl -2(%rdi), %eax +; CHECK-NEXT: movw %ax, 30(%rdi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_overlap_5: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rdi) +; DISABLED-NEXT: movslq %esi, %rax +; DISABLED-NEXT: movq %rax, -16(%rdi) +; DISABLED-NEXT: movb %al, -14(%rdi) +; DISABLED-NEXT: movb $0, -11(%rdi) +; DISABLED-NEXT: movups -16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, 16(%rdi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_overlap_5: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: vmovups -16(%rdi), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%rdi) +; CHECK-AVX2-NEXT: movslq %esi, %rax +; CHECK-AVX2-NEXT: movq %rax, -16(%rdi) +; CHECK-AVX2-NEXT: movb %al, -14(%rdi) +; CHECK-AVX2-NEXT: movb $0, -11(%rdi) +; CHECK-AVX2-NEXT: movzwl -16(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 16(%rdi) +; CHECK-AVX2-NEXT: movb -14(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 18(%rdi) +; CHECK-AVX2-NEXT: movzwl -13(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 19(%rdi) +; CHECK-AVX2-NEXT: movb -11(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 21(%rdi) +; CHECK-AVX2-NEXT: movq -10(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 22(%rdi) +; CHECK-AVX2-NEXT: movzwl -2(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 30(%rdi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_overlap_5: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: vmovups -16(%rdi), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%rdi) +; CHECK-AVX512-NEXT: movslq %esi, %rax +; CHECK-AVX512-NEXT: movq %rax, -16(%rdi) +; CHECK-AVX512-NEXT: movb %al, -14(%rdi) +; CHECK-AVX512-NEXT: movb $0, -11(%rdi) +; CHECK-AVX512-NEXT: movzwl -16(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 16(%rdi) +; CHECK-AVX512-NEXT: movb -14(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 18(%rdi) +; CHECK-AVX512-NEXT: movzwl -13(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 19(%rdi) +; CHECK-AVX512-NEXT: movb -11(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 21(%rdi) +; CHECK-AVX512-NEXT: movq -10(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 22(%rdi) +; CHECK-AVX512-NEXT: movzwl -2(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 30(%rdi) +; CHECK-AVX512-NEXT: retq +entry: + %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + %conv = sext i32 %x to i64 + %0 = bitcast i8* %add.ptr to i64* + store i64 %conv, i64* %0, align 8 + %conv2 = trunc i32 %x to i8 + %add.ptr3 = getelementptr inbounds i8, i8* %A, i64 -14 + store i8 %conv2, i8* %add.ptr3, align 1 + %add.ptr4 = getelementptr inbounds i8, i8* %A, i64 -11 + store i8 0, i8* %add.ptr4, align 1 + %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 16 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr5, i8* nonnull align 4 %add.ptr, i64 16, i1 false) + ret void +} + +attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { argmemonly nounwind } + + Index: test/CodeGen/X86/avoid-sfb.ll =================================================================== --- /dev/null +++ test/CodeGen/X86/avoid-sfb.ll @@ -0,0 +1,1491 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -check-prefix=CHECK +; RUN: llc < %s -mtriple=x86_64-linux --disable-avoid-SFB | FileCheck %s --check-prefix=DISABLED +; RUN: llc < %s -mtriple=x86_64-linux -mcpu=core-avx2 | FileCheck %s -check-prefix=CHECK-AVX2 +; RUN: llc < %s -mtriple=x86_64-linux -mcpu=skx | FileCheck %s -check-prefix=CHECK-AVX512 + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +%struct.S = type { i32, i32, i32, i32 } + +; Function Attrs: nounwind uwtable +define void @test_conditional_block(%struct.S* nocapture noalias %s1 , %struct.S* nocapture noalias %s2, i32 %x, %struct.S* nocapture noalias %s3, %struct.S* nocapture noalias readonly %s4) local_unnamed_addr #0 { +; CHECK-LABEL: test_conditional_block: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $18, %edx +; CHECK-NEXT: jl .LBB0_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movl %edx, 4(%rdi) +; CHECK-NEXT: .LBB0_2: # %if.end +; CHECK-NEXT: movups (%r8), %xmm0 +; CHECK-NEXT: movups %xmm0, (%rcx) +; CHECK-NEXT: movl (%rdi), %eax +; CHECK-NEXT: movl %eax, (%rsi) +; CHECK-NEXT: movl 4(%rdi), %eax +; CHECK-NEXT: movl %eax, 4(%rsi) +; CHECK-NEXT: movq 8(%rdi), %rax +; CHECK-NEXT: movq %rax, 8(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_conditional_block: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: cmpl $18, %edx +; DISABLED-NEXT: jl .LBB0_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movl %edx, 4(%rdi) +; DISABLED-NEXT: .LBB0_2: # %if.end +; DISABLED-NEXT: movups (%r8), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rcx) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_conditional_block: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: cmpl $18, %edx +; CHECK-AVX2-NEXT: jl .LBB0_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movl %edx, 4(%rdi) +; CHECK-AVX2-NEXT: .LBB0_2: # %if.end +; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX2-NEXT: movl (%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, (%rsi) +; CHECK-AVX2-NEXT: movl 4(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 4(%rsi) +; CHECK-AVX2-NEXT: movq 8(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 8(%rsi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_conditional_block: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: cmpl $18, %edx +; CHECK-AVX512-NEXT: jl .LBB0_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movl %edx, 4(%rdi) +; CHECK-AVX512-NEXT: .LBB0_2: # %if.end +; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX512-NEXT: movl (%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, (%rsi) +; CHECK-AVX512-NEXT: movl 4(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 4(%rsi) +; CHECK-AVX512-NEXT: movq 8(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 8(%rsi) +; CHECK-AVX512-NEXT: retq +entry: + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1 + store i32 %x, i32* %b, align 4 + br label %if.end + +if.end: ; preds = %if.then, %entry + %0 = bitcast %struct.S* %s3 to i8* + %1 = bitcast %struct.S* %s4 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false) + %2 = bitcast %struct.S* %s2 to i8* + %3 = bitcast %struct.S* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false) + ret void +} + +; Function Attrs: nounwind uwtable +define void @test_imm_store(%struct.S* nocapture noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3) local_unnamed_addr #0 { +; CHECK-LABEL: test_imm_store: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl $0, (%rdi) +; CHECK-NEXT: movl $1, (%rcx) +; CHECK-NEXT: movl (%rdi), %eax +; CHECK-NEXT: movl %eax, (%rsi) +; CHECK-NEXT: movq 4(%rdi), %rax +; CHECK-NEXT: movq %rax, 4(%rsi) +; CHECK-NEXT: movl 12(%rdi), %eax +; CHECK-NEXT: movl %eax, 12(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_imm_store: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movl $0, (%rdi) +; DISABLED-NEXT: movl $1, (%rcx) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_imm_store: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: movl $0, (%rdi) +; CHECK-AVX2-NEXT: movl $1, (%rcx) +; CHECK-AVX2-NEXT: movl (%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, (%rsi) +; CHECK-AVX2-NEXT: movq 4(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 4(%rsi) +; CHECK-AVX2-NEXT: movl 12(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 12(%rsi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_imm_store: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: movl $0, (%rdi) +; CHECK-AVX512-NEXT: movl $1, (%rcx) +; CHECK-AVX512-NEXT: movl (%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, (%rsi) +; CHECK-AVX512-NEXT: movq 4(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 4(%rsi) +; CHECK-AVX512-NEXT: movl 12(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 12(%rsi) +; CHECK-AVX512-NEXT: retq +entry: + %a = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 0 + store i32 0, i32* %a, align 4 + %a1 = getelementptr inbounds %struct.S, %struct.S* %s3, i64 0, i32 0 + store i32 1, i32* %a1, align 4 + %0 = bitcast %struct.S* %s2 to i8* + %1 = bitcast %struct.S* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false) + ret void +} + +; Function Attrs: nounwind uwtable +define void @test_nondirect_br(%struct.S* nocapture noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 { +; CHECK-LABEL: test_nondirect_br: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $18, %edx +; CHECK-NEXT: jl .LBB2_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movl %edx, 4(%rdi) +; CHECK-NEXT: .LBB2_2: # %if.end +; CHECK-NEXT: cmpl $14, %r9d +; CHECK-NEXT: jl .LBB2_4 +; CHECK-NEXT: # %bb.3: # %if.then2 +; CHECK-NEXT: movl %r9d, 12(%rdi) +; CHECK-NEXT: .LBB2_4: # %if.end3 +; CHECK-NEXT: movups (%r8), %xmm0 +; CHECK-NEXT: movups %xmm0, (%rcx) +; CHECK-NEXT: movq (%rdi), %rax +; CHECK-NEXT: movq %rax, (%rsi) +; CHECK-NEXT: movl 8(%rdi), %eax +; CHECK-NEXT: movl %eax, 8(%rsi) +; CHECK-NEXT: movl 12(%rdi), %eax +; CHECK-NEXT: movl %eax, 12(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_nondirect_br: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: cmpl $18, %edx +; DISABLED-NEXT: jl .LBB2_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movl %edx, 4(%rdi) +; DISABLED-NEXT: .LBB2_2: # %if.end +; DISABLED-NEXT: cmpl $14, %r9d +; DISABLED-NEXT: jl .LBB2_4 +; DISABLED-NEXT: # %bb.3: # %if.then2 +; DISABLED-NEXT: movl %r9d, 12(%rdi) +; DISABLED-NEXT: .LBB2_4: # %if.end3 +; DISABLED-NEXT: movups (%r8), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rcx) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_nondirect_br: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: cmpl $18, %edx +; CHECK-AVX2-NEXT: jl .LBB2_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movl %edx, 4(%rdi) +; CHECK-AVX2-NEXT: .LBB2_2: # %if.end +; CHECK-AVX2-NEXT: cmpl $14, %r9d +; CHECK-AVX2-NEXT: jl .LBB2_4 +; CHECK-AVX2-NEXT: # %bb.3: # %if.then2 +; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi) +; CHECK-AVX2-NEXT: .LBB2_4: # %if.end3 +; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX2-NEXT: movq (%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, (%rsi) +; CHECK-AVX2-NEXT: movl 8(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 8(%rsi) +; CHECK-AVX2-NEXT: movl 12(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 12(%rsi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_nondirect_br: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: cmpl $18, %edx +; CHECK-AVX512-NEXT: jl .LBB2_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movl %edx, 4(%rdi) +; CHECK-AVX512-NEXT: .LBB2_2: # %if.end +; CHECK-AVX512-NEXT: cmpl $14, %r9d +; CHECK-AVX512-NEXT: jl .LBB2_4 +; CHECK-AVX512-NEXT: # %bb.3: # %if.then2 +; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi) +; CHECK-AVX512-NEXT: .LBB2_4: # %if.end3 +; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX512-NEXT: movq (%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, (%rsi) +; CHECK-AVX512-NEXT: movl 8(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 8(%rsi) +; CHECK-AVX512-NEXT: movl 12(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 12(%rsi) +; CHECK-AVX512-NEXT: retq +entry: + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1 + store i32 %x, i32* %b, align 4 + br label %if.end + +if.end: ; preds = %if.then, %entry + %cmp1 = icmp sgt i32 %x2, 13 + br i1 %cmp1, label %if.then2, label %if.end3 + +if.then2: ; preds = %if.end + %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3 + store i32 %x2, i32* %d, align 4 + br label %if.end3 + +if.end3: ; preds = %if.then2, %if.end + %0 = bitcast %struct.S* %s3 to i8* + %1 = bitcast %struct.S* %s4 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false) + %2 = bitcast %struct.S* %s2 to i8* + %3 = bitcast %struct.S* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false) + ret void +} + +; Function Attrs: nounwind uwtable +define void @test_2preds_block(%struct.S* nocapture noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 { +; CHECK-LABEL: test_2preds_block: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %r9d, 12(%rdi) +; CHECK-NEXT: cmpl $18, %edx +; CHECK-NEXT: jl .LBB3_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movl %edx, 4(%rdi) +; CHECK-NEXT: .LBB3_2: # %if.end +; CHECK-NEXT: movups (%r8), %xmm0 +; CHECK-NEXT: movups %xmm0, (%rcx) +; CHECK-NEXT: movl (%rdi), %eax +; CHECK-NEXT: movl %eax, (%rsi) +; CHECK-NEXT: movl 4(%rdi), %eax +; CHECK-NEXT: movl %eax, 4(%rsi) +; CHECK-NEXT: movl 8(%rdi), %eax +; CHECK-NEXT: movl %eax, 8(%rsi) +; CHECK-NEXT: movl 12(%rdi), %eax +; CHECK-NEXT: movl %eax, 12(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_2preds_block: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movl %r9d, 12(%rdi) +; DISABLED-NEXT: cmpl $18, %edx +; DISABLED-NEXT: jl .LBB3_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movl %edx, 4(%rdi) +; DISABLED-NEXT: .LBB3_2: # %if.end +; DISABLED-NEXT: movups (%r8), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rcx) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_2preds_block: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi) +; CHECK-AVX2-NEXT: cmpl $18, %edx +; CHECK-AVX2-NEXT: jl .LBB3_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movl %edx, 4(%rdi) +; CHECK-AVX2-NEXT: .LBB3_2: # %if.end +; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX2-NEXT: movl (%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, (%rsi) +; CHECK-AVX2-NEXT: movl 4(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 4(%rsi) +; CHECK-AVX2-NEXT: movl 8(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 8(%rsi) +; CHECK-AVX2-NEXT: movl 12(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 12(%rsi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_2preds_block: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi) +; CHECK-AVX512-NEXT: cmpl $18, %edx +; CHECK-AVX512-NEXT: jl .LBB3_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movl %edx, 4(%rdi) +; CHECK-AVX512-NEXT: .LBB3_2: # %if.end +; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX512-NEXT: movl (%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, (%rsi) +; CHECK-AVX512-NEXT: movl 4(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 4(%rsi) +; CHECK-AVX512-NEXT: movl 8(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 8(%rsi) +; CHECK-AVX512-NEXT: movl 12(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 12(%rsi) +; CHECK-AVX512-NEXT: retq +entry: + %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3 + store i32 %x2, i32* %d, align 4 + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1 + store i32 %x, i32* %b, align 4 + br label %if.end + +if.end: ; preds = %if.then, %entry + %0 = bitcast %struct.S* %s3 to i8* + %1 = bitcast %struct.S* %s4 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false) + %2 = bitcast %struct.S* %s2 to i8* + %3 = bitcast %struct.S* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false) + ret void +} +%struct.S2 = type { i64, i64 } + +; Function Attrs: nounwind uwtable +define void @test_type64(%struct.S2* nocapture noalias %s1, %struct.S2* nocapture %s2, i32 %x, %struct.S2* nocapture %s3, %struct.S2* nocapture readonly %s4) local_unnamed_addr #0 { +; CHECK-LABEL: test_type64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $18, %edx +; CHECK-NEXT: jl .LBB4_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movslq %edx, %rax +; CHECK-NEXT: movq %rax, 8(%rdi) +; CHECK-NEXT: .LBB4_2: # %if.end +; CHECK-NEXT: movups (%r8), %xmm0 +; CHECK-NEXT: movups %xmm0, (%rcx) +; CHECK-NEXT: movq (%rdi), %rax +; CHECK-NEXT: movq %rax, (%rsi) +; CHECK-NEXT: movq 8(%rdi), %rax +; CHECK-NEXT: movq %rax, 8(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_type64: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: cmpl $18, %edx +; DISABLED-NEXT: jl .LBB4_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movslq %edx, %rax +; DISABLED-NEXT: movq %rax, 8(%rdi) +; DISABLED-NEXT: .LBB4_2: # %if.end +; DISABLED-NEXT: movups (%r8), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rcx) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_type64: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: cmpl $18, %edx +; CHECK-AVX2-NEXT: jl .LBB4_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movslq %edx, %rax +; CHECK-AVX2-NEXT: movq %rax, 8(%rdi) +; CHECK-AVX2-NEXT: .LBB4_2: # %if.end +; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX2-NEXT: movq (%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, (%rsi) +; CHECK-AVX2-NEXT: movq 8(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 8(%rsi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_type64: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: cmpl $18, %edx +; CHECK-AVX512-NEXT: jl .LBB4_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movslq %edx, %rax +; CHECK-AVX512-NEXT: movq %rax, 8(%rdi) +; CHECK-AVX512-NEXT: .LBB4_2: # %if.end +; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX512-NEXT: movq (%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, (%rsi) +; CHECK-AVX512-NEXT: movq 8(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 8(%rsi) +; CHECK-AVX512-NEXT: retq +entry: + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %conv = sext i32 %x to i64 + %b = getelementptr inbounds %struct.S2, %struct.S2* %s1, i64 0, i32 1 + store i64 %conv, i64* %b, align 8 + br label %if.end + +if.end: ; preds = %if.then, %entry + %0 = bitcast %struct.S2* %s3 to i8* + %1 = bitcast %struct.S2* %s4 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 8, i1 false) + %2 = bitcast %struct.S2* %s2 to i8* + %3 = bitcast %struct.S2* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 8, i1 false) + ret void +} +%struct.S3 = type { i64, i8, i8, i16, i32 } + +; Function Attrs: noinline nounwind uwtable +define void @test_mixed_type(%struct.S3* nocapture noalias %s1, %struct.S3* nocapture %s2, i32 %x, %struct.S3* nocapture readnone %s3, %struct.S3* nocapture readnone %s4) local_unnamed_addr #0 { +; CHECK-LABEL: test_mixed_type: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $18, %edx +; CHECK-NEXT: jl .LBB5_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movslq %edx, %rax +; CHECK-NEXT: movq %rax, (%rdi) +; CHECK-NEXT: movb %dl, 8(%rdi) +; CHECK-NEXT: .LBB5_2: # %if.end +; CHECK-NEXT: movq (%rdi), %rax +; CHECK-NEXT: movq %rax, (%rsi) +; CHECK-NEXT: movb 8(%rdi), %al +; CHECK-NEXT: movb %al, 8(%rsi) +; CHECK-NEXT: movl 9(%rdi), %eax +; CHECK-NEXT: movl %eax, 9(%rsi) +; CHECK-NEXT: movzwl 13(%rdi), %eax +; CHECK-NEXT: movw %ax, 13(%rsi) +; CHECK-NEXT: movb 15(%rdi), %al +; CHECK-NEXT: movb %al, 15(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_mixed_type: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: cmpl $18, %edx +; DISABLED-NEXT: jl .LBB5_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movslq %edx, %rax +; DISABLED-NEXT: movq %rax, (%rdi) +; DISABLED-NEXT: movb %dl, 8(%rdi) +; DISABLED-NEXT: .LBB5_2: # %if.end +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_mixed_type: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: cmpl $18, %edx +; CHECK-AVX2-NEXT: jl .LBB5_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movslq %edx, %rax +; CHECK-AVX2-NEXT: movq %rax, (%rdi) +; CHECK-AVX2-NEXT: movb %dl, 8(%rdi) +; CHECK-AVX2-NEXT: .LBB5_2: # %if.end +; CHECK-AVX2-NEXT: movq (%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, (%rsi) +; CHECK-AVX2-NEXT: movb 8(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 8(%rsi) +; CHECK-AVX2-NEXT: movl 9(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 9(%rsi) +; CHECK-AVX2-NEXT: movzwl 13(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 13(%rsi) +; CHECK-AVX2-NEXT: movb 15(%rdi), %al +; CHECK-AVX2-NEXT: movb %al, 15(%rsi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_mixed_type: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: cmpl $18, %edx +; CHECK-AVX512-NEXT: jl .LBB5_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movslq %edx, %rax +; CHECK-AVX512-NEXT: movq %rax, (%rdi) +; CHECK-AVX512-NEXT: movb %dl, 8(%rdi) +; CHECK-AVX512-NEXT: .LBB5_2: # %if.end +; CHECK-AVX512-NEXT: movq (%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, (%rsi) +; CHECK-AVX512-NEXT: movb 8(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 8(%rsi) +; CHECK-AVX512-NEXT: movl 9(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 9(%rsi) +; CHECK-AVX512-NEXT: movzwl 13(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 13(%rsi) +; CHECK-AVX512-NEXT: movb 15(%rdi), %al +; CHECK-AVX512-NEXT: movb %al, 15(%rsi) +; CHECK-AVX512-NEXT: retq +entry: + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %conv = sext i32 %x to i64 + %a = getelementptr inbounds %struct.S3, %struct.S3* %s1, i64 0, i32 0 + store i64 %conv, i64* %a, align 8 + %conv1 = trunc i32 %x to i8 + %b = getelementptr inbounds %struct.S3, %struct.S3* %s1, i64 0, i32 1 + store i8 %conv1, i8* %b, align 8 + br label %if.end + +if.end: ; preds = %if.then, %entry + %0 = bitcast %struct.S3* %s2 to i8* + %1 = bitcast %struct.S3* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 8, i1 false) + ret void +} +%struct.S4 = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 } + +; Function Attrs: nounwind uwtable +define void @test_multiple_blocks(%struct.S4* nocapture noalias %s1, %struct.S4* nocapture %s2) local_unnamed_addr #0 { +; CHECK-LABEL: test_multiple_blocks: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl $0, 4(%rdi) +; CHECK-NEXT: movl $0, 36(%rdi) +; CHECK-NEXT: movups 16(%rdi), %xmm0 +; CHECK-NEXT: movups %xmm0, 16(%rsi) +; CHECK-NEXT: movl 32(%rdi), %eax +; CHECK-NEXT: movl %eax, 32(%rsi) +; CHECK-NEXT: movl 36(%rdi), %eax +; CHECK-NEXT: movl %eax, 36(%rsi) +; CHECK-NEXT: movq 40(%rdi), %rax +; CHECK-NEXT: movq %rax, 40(%rsi) +; CHECK-NEXT: movl (%rdi), %eax +; CHECK-NEXT: movl %eax, (%rsi) +; CHECK-NEXT: movl 4(%rdi), %eax +; CHECK-NEXT: movl %eax, 4(%rsi) +; CHECK-NEXT: movq 8(%rdi), %rax +; CHECK-NEXT: movq %rax, 8(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_multiple_blocks: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movl $0, 4(%rdi) +; DISABLED-NEXT: movl $0, 36(%rdi) +; DISABLED-NEXT: movups 16(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, 16(%rsi) +; DISABLED-NEXT: movups 32(%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, 32(%rsi) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_multiple_blocks: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: movl $0, 4(%rdi) +; CHECK-AVX2-NEXT: movl $0, 36(%rdi) +; CHECK-AVX2-NEXT: vmovups 16(%rdi), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, 16(%rsi) +; CHECK-AVX2-NEXT: movl 32(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 32(%rsi) +; CHECK-AVX2-NEXT: movl 36(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 36(%rsi) +; CHECK-AVX2-NEXT: movq 40(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 40(%rsi) +; CHECK-AVX2-NEXT: movl (%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, (%rsi) +; CHECK-AVX2-NEXT: movl 4(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 4(%rsi) +; CHECK-AVX2-NEXT: vmovups 8(%rdi), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, 8(%rsi) +; CHECK-AVX2-NEXT: movq 24(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 24(%rsi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_multiple_blocks: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: movl $0, 4(%rdi) +; CHECK-AVX512-NEXT: movl $0, 36(%rdi) +; CHECK-AVX512-NEXT: vmovups 16(%rdi), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, 16(%rsi) +; CHECK-AVX512-NEXT: movl 32(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 32(%rsi) +; CHECK-AVX512-NEXT: movl 36(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 36(%rsi) +; CHECK-AVX512-NEXT: movq 40(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 40(%rsi) +; CHECK-AVX512-NEXT: movl (%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, (%rsi) +; CHECK-AVX512-NEXT: movl 4(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 4(%rsi) +; CHECK-AVX512-NEXT: vmovups 8(%rdi), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, 8(%rsi) +; CHECK-AVX512-NEXT: movq 24(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 24(%rsi) +; CHECK-AVX512-NEXT: retq +entry: + %b = getelementptr inbounds %struct.S4, %struct.S4* %s1, i64 0, i32 1 + store i32 0, i32* %b, align 4 + %b3 = getelementptr inbounds %struct.S4, %struct.S4* %s1, i64 0, i32 9 + store i32 0, i32* %b3, align 4 + %0 = bitcast %struct.S4* %s2 to i8* + %1 = bitcast %struct.S4* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 48, i32 4, i1 false) + ret void +} +%struct.S5 = type { i16, i16, i16, i16, i16, i16, i16, i16 } + +; Function Attrs: nounwind uwtable +define void @test_type16(%struct.S5* nocapture noalias %s1, %struct.S5* nocapture %s2, i32 %x, %struct.S5* nocapture %s3, %struct.S5* nocapture readonly %s4) local_unnamed_addr #0 { +; CHECK-LABEL: test_type16: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $18, %edx +; CHECK-NEXT: jl .LBB7_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movw %dx, 2(%rdi) +; CHECK-NEXT: .LBB7_2: # %if.end +; CHECK-NEXT: movups (%r8), %xmm0 +; CHECK-NEXT: movups %xmm0, (%rcx) +; CHECK-NEXT: movzwl (%rdi), %eax +; CHECK-NEXT: movw %ax, (%rsi) +; CHECK-NEXT: movzwl 2(%rdi), %eax +; CHECK-NEXT: movw %ax, 2(%rsi) +; CHECK-NEXT: movq 4(%rdi), %rax +; CHECK-NEXT: movq %rax, 4(%rsi) +; CHECK-NEXT: movl 12(%rdi), %eax +; CHECK-NEXT: movl %eax, 12(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_type16: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: cmpl $18, %edx +; DISABLED-NEXT: jl .LBB7_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movw %dx, 2(%rdi) +; DISABLED-NEXT: .LBB7_2: # %if.end +; DISABLED-NEXT: movups (%r8), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rcx) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_type16: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: cmpl $18, %edx +; CHECK-AVX2-NEXT: jl .LBB7_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movw %dx, 2(%rdi) +; CHECK-AVX2-NEXT: .LBB7_2: # %if.end +; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX2-NEXT: movzwl (%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, (%rsi) +; CHECK-AVX2-NEXT: movzwl 2(%rdi), %eax +; CHECK-AVX2-NEXT: movw %ax, 2(%rsi) +; CHECK-AVX2-NEXT: movq 4(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 4(%rsi) +; CHECK-AVX2-NEXT: movl 12(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 12(%rsi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_type16: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: cmpl $18, %edx +; CHECK-AVX512-NEXT: jl .LBB7_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movw %dx, 2(%rdi) +; CHECK-AVX512-NEXT: .LBB7_2: # %if.end +; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx) +; CHECK-AVX512-NEXT: movzwl (%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, (%rsi) +; CHECK-AVX512-NEXT: movzwl 2(%rdi), %eax +; CHECK-AVX512-NEXT: movw %ax, 2(%rsi) +; CHECK-AVX512-NEXT: movq 4(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 4(%rsi) +; CHECK-AVX512-NEXT: movl 12(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 12(%rsi) +; CHECK-AVX512-NEXT: retq +entry: + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %conv = trunc i32 %x to i16 + %b = getelementptr inbounds %struct.S5, %struct.S5* %s1, i64 0, i32 1 + store i16 %conv, i16* %b, align 2 + br label %if.end + +if.end: ; preds = %if.then, %entry + %0 = bitcast %struct.S5* %s3 to i8* + %1 = bitcast %struct.S5* %s4 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 2, i1 false) + %2 = bitcast %struct.S5* %s2 to i8* + %3 = bitcast %struct.S5* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 2, i1 false) + ret void +} + +%struct.S6 = type { [4 x i32], i32, i32, i32, i32 } + +; Function Attrs: nounwind uwtable +define void @test_stack(%struct.S6* noalias nocapture sret %agg.result, %struct.S6* byval nocapture readnone align 8 %s1, %struct.S6* byval nocapture align 8 %s2, i32 %x) local_unnamed_addr #0 { +; CHECK-LABEL: test_stack: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %esi, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0 +; CHECK-NEXT: movups %xmm0, (%rdi) +; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax +; CHECK-NEXT: movq %rax, 16(%rdi) +; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-NEXT: movl %eax, 24(%rdi) +; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-NEXT: movl %eax, 28(%rdi) +; CHECK-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0 +; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax +; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %ecx +; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %edx +; CHECK-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movl %ecx, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movl %edx, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movq %rdi, %rax +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_stack: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movl %esi, {{[0-9]+}}(%rsp) +; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%rdi) +; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0 +; DISABLED-NEXT: movups %xmm0, 16(%rdi) +; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0 +; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1 +; DISABLED-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp) +; DISABLED-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp) +; DISABLED-NEXT: movq %rdi, %rax +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_stack: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: movl %esi, {{[0-9]+}}(%rsp) +; CHECK-AVX2-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%rdi) +; CHECK-AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax +; CHECK-AVX2-NEXT: movq %rax, 16(%rdi) +; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-AVX2-NEXT: movl %eax, 24(%rdi) +; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-AVX2-NEXT: movl %eax, 28(%rdi) +; CHECK-AVX2-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, {{[0-9]+}}(%rsp) +; CHECK-AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax +; CHECK-AVX2-NEXT: movq %rax, {{[0-9]+}}(%rsp) +; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-AVX2-NEXT: movl %eax, {{[0-9]+}}(%rsp) +; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-AVX2-NEXT: movl %eax, {{[0-9]+}}(%rsp) +; CHECK-AVX2-NEXT: movq %rdi, %rax +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_stack: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: movl %esi, {{[0-9]+}}(%rsp) +; CHECK-AVX512-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%rdi) +; CHECK-AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax +; CHECK-AVX512-NEXT: movq %rax, 16(%rdi) +; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-AVX512-NEXT: movl %eax, 24(%rdi) +; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-AVX512-NEXT: movl %eax, 28(%rdi) +; CHECK-AVX512-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, {{[0-9]+}}(%rsp) +; CHECK-AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax +; CHECK-AVX512-NEXT: movq %rax, {{[0-9]+}}(%rsp) +; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-AVX512-NEXT: movl %eax, {{[0-9]+}}(%rsp) +; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax +; CHECK-AVX512-NEXT: movl %eax, {{[0-9]+}}(%rsp) +; CHECK-AVX512-NEXT: movq %rdi, %rax +; CHECK-AVX512-NEXT: retq +entry: + %s6.sroa.0.0..sroa_cast1 = bitcast %struct.S6* %s2 to i8* + %s6.sroa.3.0..sroa_idx4 = getelementptr inbounds %struct.S6, %struct.S6* %s2, i64 0, i32 3 + store i32 %x, i32* %s6.sroa.3.0..sroa_idx4, align 8 + %0 = bitcast %struct.S6* %agg.result to i8* + %s6.sroa.0.0..sroa_cast2 = bitcast %struct.S6* %s1 to i8* + call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* nonnull %s6.sroa.0.0..sroa_cast1, i64 32, i32 4, i1 false) + call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull %s6.sroa.0.0..sroa_cast2, i8* nonnull %s6.sroa.0.0..sroa_cast1, i64 32, i32 4, i1 false) + + ret void +} + +; Function Attrs: nounwind uwtable +define void @test_limit_all(%struct.S* noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 { +; CHECK-LABEL: test_limit_all: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: pushq %r15 +; CHECK-NEXT: .cfi_def_cfa_offset 24 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: pushq %r12 +; CHECK-NEXT: .cfi_def_cfa_offset 40 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: .cfi_def_cfa_offset 48 +; CHECK-NEXT: .cfi_offset %rbx, -48 +; CHECK-NEXT: .cfi_offset %r12, -40 +; CHECK-NEXT: .cfi_offset %r14, -32 +; CHECK-NEXT: .cfi_offset %r15, -24 +; CHECK-NEXT: .cfi_offset %rbp, -16 +; CHECK-NEXT: movq %r8, %r15 +; CHECK-NEXT: movq %rcx, %r14 +; CHECK-NEXT: movl %edx, %ebp +; CHECK-NEXT: movq %rsi, %r12 +; CHECK-NEXT: movq %rdi, %rbx +; CHECK-NEXT: movl %r9d, 12(%rdi) +; CHECK-NEXT: callq bar +; CHECK-NEXT: cmpl $18, %ebp +; CHECK-NEXT: jl .LBB9_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movl %ebp, 4(%rbx) +; CHECK-NEXT: movq %rbx, %rdi +; CHECK-NEXT: callq bar +; CHECK-NEXT: .LBB9_2: # %if.end +; CHECK-NEXT: movups (%r15), %xmm0 +; CHECK-NEXT: movups %xmm0, (%r14) +; CHECK-NEXT: movups (%rbx), %xmm0 +; CHECK-NEXT: movups %xmm0, (%r12) +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r12 +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_limit_all: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: pushq %rbp +; DISABLED-NEXT: .cfi_def_cfa_offset 16 +; DISABLED-NEXT: pushq %r15 +; DISABLED-NEXT: .cfi_def_cfa_offset 24 +; DISABLED-NEXT: pushq %r14 +; DISABLED-NEXT: .cfi_def_cfa_offset 32 +; DISABLED-NEXT: pushq %r12 +; DISABLED-NEXT: .cfi_def_cfa_offset 40 +; DISABLED-NEXT: pushq %rbx +; DISABLED-NEXT: .cfi_def_cfa_offset 48 +; DISABLED-NEXT: .cfi_offset %rbx, -48 +; DISABLED-NEXT: .cfi_offset %r12, -40 +; DISABLED-NEXT: .cfi_offset %r14, -32 +; DISABLED-NEXT: .cfi_offset %r15, -24 +; DISABLED-NEXT: .cfi_offset %rbp, -16 +; DISABLED-NEXT: movq %r8, %r15 +; DISABLED-NEXT: movq %rcx, %r14 +; DISABLED-NEXT: movl %edx, %ebp +; DISABLED-NEXT: movq %rsi, %r12 +; DISABLED-NEXT: movq %rdi, %rbx +; DISABLED-NEXT: movl %r9d, 12(%rdi) +; DISABLED-NEXT: callq bar +; DISABLED-NEXT: cmpl $18, %ebp +; DISABLED-NEXT: jl .LBB9_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movl %ebp, 4(%rbx) +; DISABLED-NEXT: movq %rbx, %rdi +; DISABLED-NEXT: callq bar +; DISABLED-NEXT: .LBB9_2: # %if.end +; DISABLED-NEXT: movups (%r15), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%r14) +; DISABLED-NEXT: movups (%rbx), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%r12) +; DISABLED-NEXT: popq %rbx +; DISABLED-NEXT: popq %r12 +; DISABLED-NEXT: popq %r14 +; DISABLED-NEXT: popq %r15 +; DISABLED-NEXT: popq %rbp +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_limit_all: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: pushq %rbp +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 16 +; CHECK-AVX2-NEXT: pushq %r15 +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 24 +; CHECK-AVX2-NEXT: pushq %r14 +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 32 +; CHECK-AVX2-NEXT: pushq %r12 +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 40 +; CHECK-AVX2-NEXT: pushq %rbx +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 48 +; CHECK-AVX2-NEXT: .cfi_offset %rbx, -48 +; CHECK-AVX2-NEXT: .cfi_offset %r12, -40 +; CHECK-AVX2-NEXT: .cfi_offset %r14, -32 +; CHECK-AVX2-NEXT: .cfi_offset %r15, -24 +; CHECK-AVX2-NEXT: .cfi_offset %rbp, -16 +; CHECK-AVX2-NEXT: movq %r8, %r15 +; CHECK-AVX2-NEXT: movq %rcx, %r14 +; CHECK-AVX2-NEXT: movl %edx, %ebp +; CHECK-AVX2-NEXT: movq %rsi, %r12 +; CHECK-AVX2-NEXT: movq %rdi, %rbx +; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi) +; CHECK-AVX2-NEXT: callq bar +; CHECK-AVX2-NEXT: cmpl $18, %ebp +; CHECK-AVX2-NEXT: jl .LBB9_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movl %ebp, 4(%rbx) +; CHECK-AVX2-NEXT: movq %rbx, %rdi +; CHECK-AVX2-NEXT: callq bar +; CHECK-AVX2-NEXT: .LBB9_2: # %if.end +; CHECK-AVX2-NEXT: vmovups (%r15), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%r14) +; CHECK-AVX2-NEXT: vmovups (%rbx), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%r12) +; CHECK-AVX2-NEXT: popq %rbx +; CHECK-AVX2-NEXT: popq %r12 +; CHECK-AVX2-NEXT: popq %r14 +; CHECK-AVX2-NEXT: popq %r15 +; CHECK-AVX2-NEXT: popq %rbp +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_limit_all: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: pushq %rbp +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 16 +; CHECK-AVX512-NEXT: pushq %r15 +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 24 +; CHECK-AVX512-NEXT: pushq %r14 +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 32 +; CHECK-AVX512-NEXT: pushq %r12 +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 40 +; CHECK-AVX512-NEXT: pushq %rbx +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 48 +; CHECK-AVX512-NEXT: .cfi_offset %rbx, -48 +; CHECK-AVX512-NEXT: .cfi_offset %r12, -40 +; CHECK-AVX512-NEXT: .cfi_offset %r14, -32 +; CHECK-AVX512-NEXT: .cfi_offset %r15, -24 +; CHECK-AVX512-NEXT: .cfi_offset %rbp, -16 +; CHECK-AVX512-NEXT: movq %r8, %r15 +; CHECK-AVX512-NEXT: movq %rcx, %r14 +; CHECK-AVX512-NEXT: movl %edx, %ebp +; CHECK-AVX512-NEXT: movq %rsi, %r12 +; CHECK-AVX512-NEXT: movq %rdi, %rbx +; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi) +; CHECK-AVX512-NEXT: callq bar +; CHECK-AVX512-NEXT: cmpl $18, %ebp +; CHECK-AVX512-NEXT: jl .LBB9_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movl %ebp, 4(%rbx) +; CHECK-AVX512-NEXT: movq %rbx, %rdi +; CHECK-AVX512-NEXT: callq bar +; CHECK-AVX512-NEXT: .LBB9_2: # %if.end +; CHECK-AVX512-NEXT: vmovups (%r15), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%r14) +; CHECK-AVX512-NEXT: vmovups (%rbx), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%r12) +; CHECK-AVX512-NEXT: popq %rbx +; CHECK-AVX512-NEXT: popq %r12 +; CHECK-AVX512-NEXT: popq %r14 +; CHECK-AVX512-NEXT: popq %r15 +; CHECK-AVX512-NEXT: popq %rbp +; CHECK-AVX512-NEXT: retq +entry: + %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3 + store i32 %x2, i32* %d, align 4 + tail call void @bar(%struct.S* %s1) #3 + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1 + store i32 %x, i32* %b, align 4 + tail call void @bar(%struct.S* nonnull %s1) #3 + br label %if.end + +if.end: ; preds = %if.then, %entry + %0 = bitcast %struct.S* %s3 to i8* + %1 = bitcast %struct.S* %s4 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false) + %2 = bitcast %struct.S* %s2 to i8* + %3 = bitcast %struct.S* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false) + ret void +} + +; Function Attrs: nounwind uwtable +define void @test_limit_one_pred(%struct.S* noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 { +; CHECK-LABEL: test_limit_one_pred: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %r15 +; CHECK-NEXT: .cfi_def_cfa_offset 16 +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: .cfi_def_cfa_offset 24 +; CHECK-NEXT: pushq %r12 +; CHECK-NEXT: .cfi_def_cfa_offset 32 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: .cfi_def_cfa_offset 40 +; CHECK-NEXT: pushq %rax +; CHECK-NEXT: .cfi_def_cfa_offset 48 +; CHECK-NEXT: .cfi_offset %rbx, -40 +; CHECK-NEXT: .cfi_offset %r12, -32 +; CHECK-NEXT: .cfi_offset %r14, -24 +; CHECK-NEXT: .cfi_offset %r15, -16 +; CHECK-NEXT: movq %r8, %r12 +; CHECK-NEXT: movq %rcx, %r15 +; CHECK-NEXT: movq %rsi, %r14 +; CHECK-NEXT: movq %rdi, %rbx +; CHECK-NEXT: movl %r9d, 12(%rdi) +; CHECK-NEXT: cmpl $18, %edx +; CHECK-NEXT: jl .LBB10_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movl %edx, 4(%rbx) +; CHECK-NEXT: movq %rbx, %rdi +; CHECK-NEXT: callq bar +; CHECK-NEXT: .LBB10_2: # %if.end +; CHECK-NEXT: movups (%r12), %xmm0 +; CHECK-NEXT: movups %xmm0, (%r15) +; CHECK-NEXT: movq (%rbx), %rax +; CHECK-NEXT: movq %rax, (%r14) +; CHECK-NEXT: movl 8(%rbx), %eax +; CHECK-NEXT: movl %eax, 8(%r14) +; CHECK-NEXT: movl 12(%rbx), %eax +; CHECK-NEXT: movl %eax, 12(%r14) +; CHECK-NEXT: addq $8, %rsp +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r12 +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %r15 +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_limit_one_pred: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: pushq %r15 +; DISABLED-NEXT: .cfi_def_cfa_offset 16 +; DISABLED-NEXT: pushq %r14 +; DISABLED-NEXT: .cfi_def_cfa_offset 24 +; DISABLED-NEXT: pushq %r12 +; DISABLED-NEXT: .cfi_def_cfa_offset 32 +; DISABLED-NEXT: pushq %rbx +; DISABLED-NEXT: .cfi_def_cfa_offset 40 +; DISABLED-NEXT: pushq %rax +; DISABLED-NEXT: .cfi_def_cfa_offset 48 +; DISABLED-NEXT: .cfi_offset %rbx, -40 +; DISABLED-NEXT: .cfi_offset %r12, -32 +; DISABLED-NEXT: .cfi_offset %r14, -24 +; DISABLED-NEXT: .cfi_offset %r15, -16 +; DISABLED-NEXT: movq %r8, %r15 +; DISABLED-NEXT: movq %rcx, %r14 +; DISABLED-NEXT: movq %rsi, %r12 +; DISABLED-NEXT: movq %rdi, %rbx +; DISABLED-NEXT: movl %r9d, 12(%rdi) +; DISABLED-NEXT: cmpl $18, %edx +; DISABLED-NEXT: jl .LBB10_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movl %edx, 4(%rbx) +; DISABLED-NEXT: movq %rbx, %rdi +; DISABLED-NEXT: callq bar +; DISABLED-NEXT: .LBB10_2: # %if.end +; DISABLED-NEXT: movups (%r15), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%r14) +; DISABLED-NEXT: movups (%rbx), %xmm0 +; DISABLED-NEXT: movups %xmm0, (%r12) +; DISABLED-NEXT: addq $8, %rsp +; DISABLED-NEXT: popq %rbx +; DISABLED-NEXT: popq %r12 +; DISABLED-NEXT: popq %r14 +; DISABLED-NEXT: popq %r15 +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_limit_one_pred: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: pushq %r15 +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 16 +; CHECK-AVX2-NEXT: pushq %r14 +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 24 +; CHECK-AVX2-NEXT: pushq %r12 +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 32 +; CHECK-AVX2-NEXT: pushq %rbx +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 40 +; CHECK-AVX2-NEXT: pushq %rax +; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 48 +; CHECK-AVX2-NEXT: .cfi_offset %rbx, -40 +; CHECK-AVX2-NEXT: .cfi_offset %r12, -32 +; CHECK-AVX2-NEXT: .cfi_offset %r14, -24 +; CHECK-AVX2-NEXT: .cfi_offset %r15, -16 +; CHECK-AVX2-NEXT: movq %r8, %r12 +; CHECK-AVX2-NEXT: movq %rcx, %r15 +; CHECK-AVX2-NEXT: movq %rsi, %r14 +; CHECK-AVX2-NEXT: movq %rdi, %rbx +; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi) +; CHECK-AVX2-NEXT: cmpl $18, %edx +; CHECK-AVX2-NEXT: jl .LBB10_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movl %edx, 4(%rbx) +; CHECK-AVX2-NEXT: movq %rbx, %rdi +; CHECK-AVX2-NEXT: callq bar +; CHECK-AVX2-NEXT: .LBB10_2: # %if.end +; CHECK-AVX2-NEXT: vmovups (%r12), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, (%r15) +; CHECK-AVX2-NEXT: movq (%rbx), %rax +; CHECK-AVX2-NEXT: movq %rax, (%r14) +; CHECK-AVX2-NEXT: movl 8(%rbx), %eax +; CHECK-AVX2-NEXT: movl %eax, 8(%r14) +; CHECK-AVX2-NEXT: movl 12(%rbx), %eax +; CHECK-AVX2-NEXT: movl %eax, 12(%r14) +; CHECK-AVX2-NEXT: addq $8, %rsp +; CHECK-AVX2-NEXT: popq %rbx +; CHECK-AVX2-NEXT: popq %r12 +; CHECK-AVX2-NEXT: popq %r14 +; CHECK-AVX2-NEXT: popq %r15 +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_limit_one_pred: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: pushq %r15 +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 16 +; CHECK-AVX512-NEXT: pushq %r14 +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 24 +; CHECK-AVX512-NEXT: pushq %r12 +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 32 +; CHECK-AVX512-NEXT: pushq %rbx +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 40 +; CHECK-AVX512-NEXT: pushq %rax +; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 48 +; CHECK-AVX512-NEXT: .cfi_offset %rbx, -40 +; CHECK-AVX512-NEXT: .cfi_offset %r12, -32 +; CHECK-AVX512-NEXT: .cfi_offset %r14, -24 +; CHECK-AVX512-NEXT: .cfi_offset %r15, -16 +; CHECK-AVX512-NEXT: movq %r8, %r12 +; CHECK-AVX512-NEXT: movq %rcx, %r15 +; CHECK-AVX512-NEXT: movq %rsi, %r14 +; CHECK-AVX512-NEXT: movq %rdi, %rbx +; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi) +; CHECK-AVX512-NEXT: cmpl $18, %edx +; CHECK-AVX512-NEXT: jl .LBB10_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movl %edx, 4(%rbx) +; CHECK-AVX512-NEXT: movq %rbx, %rdi +; CHECK-AVX512-NEXT: callq bar +; CHECK-AVX512-NEXT: .LBB10_2: # %if.end +; CHECK-AVX512-NEXT: vmovups (%r12), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, (%r15) +; CHECK-AVX512-NEXT: movq (%rbx), %rax +; CHECK-AVX512-NEXT: movq %rax, (%r14) +; CHECK-AVX512-NEXT: movl 8(%rbx), %eax +; CHECK-AVX512-NEXT: movl %eax, 8(%r14) +; CHECK-AVX512-NEXT: movl 12(%rbx), %eax +; CHECK-AVX512-NEXT: movl %eax, 12(%r14) +; CHECK-AVX512-NEXT: addq $8, %rsp +; CHECK-AVX512-NEXT: popq %rbx +; CHECK-AVX512-NEXT: popq %r12 +; CHECK-AVX512-NEXT: popq %r14 +; CHECK-AVX512-NEXT: popq %r15 +; CHECK-AVX512-NEXT: retq +entry: + %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3 + store i32 %x2, i32* %d, align 4 + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1 + store i32 %x, i32* %b, align 4 + tail call void @bar(%struct.S* nonnull %s1) #3 + br label %if.end + +if.end: ; preds = %if.then, %entry + %0 = bitcast %struct.S* %s3 to i8* + %1 = bitcast %struct.S* %s4 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false) + %2 = bitcast %struct.S* %s2 to i8* + %3 = bitcast %struct.S* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false) + ret void +} + + +declare void @bar(%struct.S*) local_unnamed_addr #1 + + +; Function Attrs: argmemonly nounwind +declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #1 + +attributes #0 = { nounwind uwtable "target-cpu"="x86-64" } + +%struct.S7 = type { float, float, float , float, float, float, float, float } + +; Function Attrs: nounwind uwtable +define void @test_conditional_block_float(%struct.S7* nocapture noalias %s1, %struct.S7* nocapture %s2, i32 %x, %struct.S7* nocapture %s3, %struct.S7* nocapture readonly %s4, float %y) local_unnamed_addr #0 { +; CHECK-LABEL: test_conditional_block_float: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $18, %edx +; CHECK-NEXT: jl .LBB11_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000 +; CHECK-NEXT: .LBB11_2: # %if.end +; CHECK-NEXT: movups (%r8), %xmm0 +; CHECK-NEXT: movups 16(%r8), %xmm1 +; CHECK-NEXT: movups %xmm1, 16(%rcx) +; CHECK-NEXT: movups %xmm0, (%rcx) +; CHECK-NEXT: movl (%rdi), %eax +; CHECK-NEXT: movl 4(%rdi), %ecx +; CHECK-NEXT: movq 8(%rdi), %rdx +; CHECK-NEXT: movups 16(%rdi), %xmm0 +; CHECK-NEXT: movups %xmm0, 16(%rsi) +; CHECK-NEXT: movl %eax, (%rsi) +; CHECK-NEXT: movl %ecx, 4(%rsi) +; CHECK-NEXT: movq %rdx, 8(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_conditional_block_float: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: cmpl $18, %edx +; DISABLED-NEXT: jl .LBB11_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000 +; DISABLED-NEXT: .LBB11_2: # %if.end +; DISABLED-NEXT: movups (%r8), %xmm0 +; DISABLED-NEXT: movups 16(%r8), %xmm1 +; DISABLED-NEXT: movups %xmm1, 16(%rcx) +; DISABLED-NEXT: movups %xmm0, (%rcx) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups 16(%rdi), %xmm1 +; DISABLED-NEXT: movups %xmm1, 16(%rsi) +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_conditional_block_float: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: cmpl $18, %edx +; CHECK-AVX2-NEXT: jl .LBB11_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000 +; CHECK-AVX2-NEXT: .LBB11_2: # %if.end +; CHECK-AVX2-NEXT: vmovups (%r8), %ymm0 +; CHECK-AVX2-NEXT: vmovups %ymm0, (%rcx) +; CHECK-AVX2-NEXT: movl (%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, (%rsi) +; CHECK-AVX2-NEXT: movl 4(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 4(%rsi) +; CHECK-AVX2-NEXT: vmovups 8(%rdi), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, 8(%rsi) +; CHECK-AVX2-NEXT: movq 24(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 24(%rsi) +; CHECK-AVX2-NEXT: vzeroupper +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_conditional_block_float: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: cmpl $18, %edx +; CHECK-AVX512-NEXT: jl .LBB11_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000 +; CHECK-AVX512-NEXT: .LBB11_2: # %if.end +; CHECK-AVX512-NEXT: vmovups (%r8), %ymm0 +; CHECK-AVX512-NEXT: vmovups %ymm0, (%rcx) +; CHECK-AVX512-NEXT: movl (%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, (%rsi) +; CHECK-AVX512-NEXT: movl 4(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 4(%rsi) +; CHECK-AVX512-NEXT: vmovups 8(%rdi), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, 8(%rsi) +; CHECK-AVX512-NEXT: movq 24(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 24(%rsi) +; CHECK-AVX512-NEXT: vzeroupper +; CHECK-AVX512-NEXT: retq +entry: + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %b = getelementptr inbounds %struct.S7, %struct.S7* %s1, i64 0, i32 1 + store float 1.0, float* %b, align 4 + br label %if.end + +if.end: ; preds = %if.then, %entry + %0 = bitcast %struct.S7* %s3 to i8* + %1 = bitcast %struct.S7* %s4 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 32, i32 4, i1 false) + %2 = bitcast %struct.S7* %s2 to i8* + %3 = bitcast %struct.S7* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 32, i32 4, i1 false) + ret void +} + +%struct.S8 = type { i64, i64, i64, i64, i64, i64 } + +; Function Attrs: nounwind uwtable +define void @test_conditional_block_ymm(%struct.S8* nocapture noalias %s1, %struct.S8* nocapture %s2, i32 %x, %struct.S8* nocapture %s3, %struct.S8* nocapture readonly %s4) local_unnamed_addr #0 { +; CHECK-LABEL: test_conditional_block_ymm: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: cmpl $18, %edx +; CHECK-NEXT: jl .LBB12_2 +; CHECK-NEXT: # %bb.1: # %if.then +; CHECK-NEXT: movq $1, 8(%rdi) +; CHECK-NEXT: .LBB12_2: # %if.end +; CHECK-NEXT: movups (%r8), %xmm0 +; CHECK-NEXT: movups 16(%r8), %xmm1 +; CHECK-NEXT: movups %xmm1, 16(%rcx) +; CHECK-NEXT: movups %xmm0, (%rcx) +; CHECK-NEXT: movq (%rdi), %rax +; CHECK-NEXT: movq 8(%rdi), %rcx +; CHECK-NEXT: movups 16(%rdi), %xmm0 +; CHECK-NEXT: movups %xmm0, 16(%rsi) +; CHECK-NEXT: movq %rax, (%rsi) +; CHECK-NEXT: movq %rcx, 8(%rsi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_conditional_block_ymm: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: cmpl $18, %edx +; DISABLED-NEXT: jl .LBB12_2 +; DISABLED-NEXT: # %bb.1: # %if.then +; DISABLED-NEXT: movq $1, 8(%rdi) +; DISABLED-NEXT: .LBB12_2: # %if.end +; DISABLED-NEXT: movups (%r8), %xmm0 +; DISABLED-NEXT: movups 16(%r8), %xmm1 +; DISABLED-NEXT: movups %xmm1, 16(%rcx) +; DISABLED-NEXT: movups %xmm0, (%rcx) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups 16(%rdi), %xmm1 +; DISABLED-NEXT: movups %xmm1, 16(%rsi) +; DISABLED-NEXT: movups %xmm0, (%rsi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_conditional_block_ymm: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: cmpl $18, %edx +; CHECK-AVX2-NEXT: jl .LBB12_2 +; CHECK-AVX2-NEXT: # %bb.1: # %if.then +; CHECK-AVX2-NEXT: movq $1, 8(%rdi) +; CHECK-AVX2-NEXT: .LBB12_2: # %if.end +; CHECK-AVX2-NEXT: vmovups (%r8), %ymm0 +; CHECK-AVX2-NEXT: vmovups %ymm0, (%rcx) +; CHECK-AVX2-NEXT: movq (%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, (%rsi) +; CHECK-AVX2-NEXT: movq 8(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 8(%rsi) +; CHECK-AVX2-NEXT: vmovups 16(%rdi), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, 16(%rsi) +; CHECK-AVX2-NEXT: vzeroupper +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_conditional_block_ymm: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: cmpl $18, %edx +; CHECK-AVX512-NEXT: jl .LBB12_2 +; CHECK-AVX512-NEXT: # %bb.1: # %if.then +; CHECK-AVX512-NEXT: movq $1, 8(%rdi) +; CHECK-AVX512-NEXT: .LBB12_2: # %if.end +; CHECK-AVX512-NEXT: vmovups (%r8), %ymm0 +; CHECK-AVX512-NEXT: vmovups %ymm0, (%rcx) +; CHECK-AVX512-NEXT: movq (%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, (%rsi) +; CHECK-AVX512-NEXT: movq 8(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 8(%rsi) +; CHECK-AVX512-NEXT: vmovups 16(%rdi), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, 16(%rsi) +; CHECK-AVX512-NEXT: vzeroupper +; CHECK-AVX512-NEXT: retq +entry: + %cmp = icmp sgt i32 %x, 17 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %b = getelementptr inbounds %struct.S8, %struct.S8* %s1, i64 0, i32 1 + store i64 1, i64* %b, align 4 + br label %if.end + +if.end: ; preds = %if.then, %entry + %0 = bitcast %struct.S8* %s3 to i8* + %1 = bitcast %struct.S8* %s4 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 32, i32 4, i1 false) + %2 = bitcast %struct.S8* %s2 to i8* + %3 = bitcast %struct.S8* %s1 to i8* + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 32, i32 4, i1 false) + ret void +} + +define dso_local void @test_alias(i8* nocapture %A, i32 %x) local_unnamed_addr #0 { +; CHECK-LABEL: test_alias: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %esi, (%rdi) +; CHECK-NEXT: movups (%rdi), %xmm0 +; CHECK-NEXT: movups %xmm0, 4(%rdi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_alias: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movl %esi, (%rdi) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, 4(%rdi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_alias: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: movl %esi, (%rdi) +; CHECK-AVX2-NEXT: vmovups (%rdi), %xmm0 +; CHECK-AVX2-NEXT: vmovups %xmm0, 4(%rdi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_alias: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: movl %esi, (%rdi) +; CHECK-AVX512-NEXT: vmovups (%rdi), %xmm0 +; CHECK-AVX512-NEXT: vmovups %xmm0, 4(%rdi) +; CHECK-AVX512-NEXT: retq +entry: + %a = bitcast i8* %A to i32* + store i32 %x, i32* %a, align 4 + %add.ptr = getelementptr inbounds i8, i8* %A, i64 4 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr, i8* align 4 %A, i64 16, i32 4, i1 false) + ret void +} + +; Function Attrs: nounwind uwtable +define dso_local void @test_noalias(i8* nocapture %A, i32 %x) local_unnamed_addr #0 { +; CHECK-LABEL: test_noalias: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movl %esi, (%rdi) +; CHECK-NEXT: movl (%rdi), %eax +; CHECK-NEXT: movl %eax, 20(%rdi) +; CHECK-NEXT: movq 4(%rdi), %rax +; CHECK-NEXT: movq %rax, 24(%rdi) +; CHECK-NEXT: movl 12(%rdi), %eax +; CHECK-NEXT: movl %eax, 32(%rdi) +; CHECK-NEXT: retq +; +; DISABLED-LABEL: test_noalias: +; DISABLED: # %bb.0: # %entry +; DISABLED-NEXT: movl %esi, (%rdi) +; DISABLED-NEXT: movups (%rdi), %xmm0 +; DISABLED-NEXT: movups %xmm0, 20(%rdi) +; DISABLED-NEXT: retq +; +; CHECK-AVX2-LABEL: test_noalias: +; CHECK-AVX2: # %bb.0: # %entry +; CHECK-AVX2-NEXT: movl %esi, (%rdi) +; CHECK-AVX2-NEXT: movl (%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 20(%rdi) +; CHECK-AVX2-NEXT: movq 4(%rdi), %rax +; CHECK-AVX2-NEXT: movq %rax, 24(%rdi) +; CHECK-AVX2-NEXT: movl 12(%rdi), %eax +; CHECK-AVX2-NEXT: movl %eax, 32(%rdi) +; CHECK-AVX2-NEXT: retq +; +; CHECK-AVX512-LABEL: test_noalias: +; CHECK-AVX512: # %bb.0: # %entry +; CHECK-AVX512-NEXT: movl %esi, (%rdi) +; CHECK-AVX512-NEXT: movl (%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 20(%rdi) +; CHECK-AVX512-NEXT: movq 4(%rdi), %rax +; CHECK-AVX512-NEXT: movq %rax, 24(%rdi) +; CHECK-AVX512-NEXT: movl 12(%rdi), %eax +; CHECK-AVX512-NEXT: movl %eax, 32(%rdi) +; CHECK-AVX512-NEXT: retq +entry: + %a = bitcast i8* %A to i32* + store i32 %x, i32* %a, align 4 + %add.ptr = getelementptr inbounds i8, i8* %A, i64 20 + tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr, i8* align 4 %A, i64 16, i32 4, i1 false) + ret void +} + + +