Index: include/llvm/CodeGen/GlobalISel/LegalizerHelper.h =================================================================== --- include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -123,6 +123,15 @@ LegalizeResult fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + /// Legalize a simple vector instruction where all operands are the same type + /// by splitting into multiple components. + LegalizeResult fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy); + LegalizeResult + fewerElementsVectorLoadStore(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI, + unsigned TypeIdx, LLT NarrowTy); + LegalizeResult narrowScalarMul(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty); Index: lib/CodeGen/GlobalISel/LegalizerHelper.cpp =================================================================== --- lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -1330,11 +1330,86 @@ = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); NewCmp->setFlags(MI.getFlags()); } + } + + if (NarrowTy1.isVector()) + MIRBuilder.buildConcatVectors(DstReg, DstRegs); + else + MIRBuilder.buildBuildVector(DstReg, DstRegs); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef( + MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { + SmallVector DstRegs; + + unsigned NarrowSize = NarrowTy.getSizeInBits(); + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned Size = MRI.getType(DstReg).getSizeInBits(); + int NumParts = Size / NarrowSize; + // FIXME: Don't know how to handle the situation where the small vectors + // aren't all the same size yet. + if (Size % NarrowSize != 0) + return UnableToLegalize; + + for (int i = 0; i < NumParts; ++i) { + unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy); + MIRBuilder.buildUndef(TmpReg); + DstRegs.push_back(TmpReg); + } + + if (NarrowTy.isVector()) + MIRBuilder.buildConcatVectors(DstReg, DstRegs); + else + MIRBuilder.buildBuildVector(DstReg, DstRegs); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + unsigned Opc = MI.getOpcode(); + unsigned NarrowSize = NarrowTy.getSizeInBits(); + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned Flags = MI.getFlags(); + unsigned Size = MRI.getType(DstReg).getSizeInBits(); + int NumParts = Size / NarrowSize; + // FIXME: Don't know how to handle the situation where the small vectors + // aren't all the same size yet. + if (Size % NarrowSize != 0) + return UnableToLegalize; + + unsigned NumOps = MI.getNumOperands() - 1; + SmallVector DstRegs, Src0Regs, Src1Regs, Src2Regs; + + extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs); + + if (NumOps >= 2) + extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs); + + if (NumOps >= 3) + extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs); + + for (int i = 0; i < NumParts; ++i) { + unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy); + + if (NumOps == 1) + MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags); + else if (NumOps == 2) { + MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags); + } else if (NumOps == 3) { + MIRBuilder.buildInstr(Opc, {DstReg}, + {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags); + } DstRegs.push_back(DstReg); } - if (NarrowTy1.isVector()) + if (NarrowTy.isVector()) MIRBuilder.buildConcatVectors(DstReg, DstRegs); else MIRBuilder.buildBuildVector(DstReg, DstRegs); @@ -1343,6 +1418,58 @@ return Legalized; } +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVectorLoadStore(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + // FIXME: Don't know how to handle secondary types yet. + if (TypeIdx != 0) + return UnableToLegalize; + + bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD; + unsigned ValReg = MI.getOperand(0).getReg(); + unsigned AddrReg = MI.getOperand(1).getReg(); + unsigned NarrowSize = NarrowTy.getSizeInBits(); + unsigned Size = MRI.getType(ValReg).getSizeInBits(); + unsigned NumParts = Size / NarrowSize; + if (NumParts * NarrowSize != Size) + return UnableToLegalize; + + SmallVector NarrowRegs; + if (!IsLoad) + extractParts(ValReg, NarrowTy, NumParts, NarrowRegs); + + const LLT OffsetTy = + LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits()); + MachineFunction &MF = *MI.getMF(); + MachineMemOperand *MMO = *MI.memoperands_begin(); + for (unsigned Idx = 0; Idx < NumParts; ++Idx) { + unsigned Adjustment = Idx * NarrowTy.getSizeInBits() / 8; + unsigned Alignment = MinAlign(MMO->getAlignment(), Adjustment); + unsigned NewAddrReg = 0; + MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, Adjustment); + MachineMemOperand &NewMMO = *MF.getMachineMemOperand( + MMO->getPointerInfo().getWithOffset(Adjustment), MMO->getFlags(), + NarrowTy.getSizeInBits() / 8, Alignment); + if (IsLoad) { + unsigned Dst = MRI.createGenericVirtualRegister(NarrowTy); + NarrowRegs.push_back(Dst); + MIRBuilder.buildLoad(Dst, NewAddrReg, NewMMO); + } else { + MIRBuilder.buildStore(NarrowRegs[Idx], NewAddrReg, NewMMO); + } + } + if (IsLoad) { + if (NarrowTy.isVector()) + MIRBuilder.buildConcatVectors(ValReg, NarrowRegs); + else + MIRBuilder.buildBuildVector(ValReg, NarrowRegs); + } + + MI.eraseFromParent(); + return Legalized; +} + + LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { @@ -1353,32 +1480,8 @@ switch (Opc) { default: return UnableToLegalize; - case TargetOpcode::G_IMPLICIT_DEF: { - SmallVector DstRegs; - - unsigned NarrowSize = NarrowTy.getSizeInBits(); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned Size = MRI.getType(DstReg).getSizeInBits(); - int NumParts = Size / NarrowSize; - // FIXME: Don't know how to handle the situation where the small vectors - // aren't all the same size yet. - if (Size % NarrowSize != 0) - return UnableToLegalize; - - for (int i = 0; i < NumParts; ++i) { - unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy); - MIRBuilder.buildUndef(TmpReg); - DstRegs.push_back(TmpReg); - } - - if (NarrowTy.isVector()) - MIRBuilder.buildConcatVectors(DstReg, DstRegs); - else - MIRBuilder.buildBuildVector(DstReg, DstRegs); - - MI.eraseFromParent(); - return Legalized; - } + case TargetOpcode::G_IMPLICIT_DEF: + return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy); case TargetOpcode::G_AND: case TargetOpcode::G_OR: case TargetOpcode::G_XOR: @@ -1393,102 +1496,14 @@ case TargetOpcode::G_FABS: case TargetOpcode::G_FDIV: case TargetOpcode::G_FREM: - case TargetOpcode::G_FMA: { - unsigned NarrowSize = NarrowTy.getSizeInBits(); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned Flags = MI.getFlags(); - unsigned Size = MRI.getType(DstReg).getSizeInBits(); - int NumParts = Size / NarrowSize; - // FIXME: Don't know how to handle the situation where the small vectors - // aren't all the same size yet. - if (Size % NarrowSize != 0) - return UnableToLegalize; - - unsigned NumOps = MI.getNumOperands() - 1; - SmallVector DstRegs, Src0Regs, Src1Regs, Src2Regs; - - extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs); - - if (NumOps >= 2) - extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs); - - if (NumOps >= 3) - extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs); - - for (int i = 0; i < NumParts; ++i) { - unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy); - - if (NumOps == 1) - MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags); - else if (NumOps == 2) { - MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags); - } else if (NumOps == 3) { - MIRBuilder.buildInstr(Opc, {DstReg}, - {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags); - } - - DstRegs.push_back(DstReg); - } - - if (NarrowTy.isVector()) - MIRBuilder.buildConcatVectors(DstReg, DstRegs); - else - MIRBuilder.buildBuildVector(DstReg, DstRegs); - - MI.eraseFromParent(); - return Legalized; - } + case TargetOpcode::G_FMA: + return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy); case TargetOpcode::G_ICMP: case TargetOpcode::G_FCMP: return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy); case TargetOpcode::G_LOAD: - case TargetOpcode::G_STORE: { - // FIXME: Don't know how to handle secondary types yet. - if (TypeIdx != 0) - return UnableToLegalize; - - bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD; - unsigned ValReg = MI.getOperand(0).getReg(); - unsigned AddrReg = MI.getOperand(1).getReg(); - unsigned NarrowSize = NarrowTy.getSizeInBits(); - unsigned Size = MRI.getType(ValReg).getSizeInBits(); - unsigned NumParts = Size / NarrowSize; - if (NumParts * NarrowSize != Size) - return UnableToLegalize; - - SmallVector NarrowRegs; - if (!IsLoad) - extractParts(ValReg, NarrowTy, NumParts, NarrowRegs); - - const LLT OffsetTy = - LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits()); - MachineFunction &MF = *MI.getMF(); - MachineMemOperand *MMO = *MI.memoperands_begin(); - for (unsigned Idx = 0; Idx < NumParts; ++Idx) { - unsigned Adjustment = Idx * NarrowTy.getSizeInBits() / 8; - unsigned Alignment = MinAlign(MMO->getAlignment(), Adjustment); - unsigned NewAddrReg = 0; - MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, Adjustment); - MachineMemOperand &NewMMO = *MF.getMachineMemOperand( - MMO->getPointerInfo().getWithOffset(Adjustment), MMO->getFlags(), - NarrowTy.getSizeInBits() / 8, Alignment); - if (IsLoad) { - unsigned Dst = MRI.createGenericVirtualRegister(NarrowTy); - NarrowRegs.push_back(Dst); - MIRBuilder.buildLoad(Dst, NewAddrReg, NewMMO); - } else { - MIRBuilder.buildStore(NarrowRegs[Idx], NewAddrReg, NewMMO); - } - } - if (IsLoad) { - if (NarrowTy.isVector()) - MIRBuilder.buildConcatVectors(ValReg, NarrowRegs); - else - MIRBuilder.buildBuildVector(ValReg, NarrowRegs); - } - MI.eraseFromParent(); - return Legalized; - } + case TargetOpcode::G_STORE: + return fewerElementsVectorLoadStore(MI, TypeIdx, NarrowTy); case TargetOpcode::G_ZEXT: case TargetOpcode::G_SEXT: case TargetOpcode::G_ANYEXT: