Index: include/llvm/Target/Target.td =================================================================== --- include/llvm/Target/Target.td +++ include/llvm/Target/Target.td @@ -21,6 +21,55 @@ class RegisterClass; // Forward def +class HwMode { + // A string representing subtarget features that turn on this hw mode. + // For example, "+feat1,-feat2" will indicate that the mode is active + // when "feat1" is enabled and "feat2" is disabled at the same time. + // Any other features are not checked. + // When multiple modes are used, they should be mutually exclusive, + // otherwise the results are unpredictable. + string Features = FS; +} + +// A special mode recognized by tablegen. This mode is considered active +// when no other mode is active. For targets that do not use specific hw +// modes, this is the only mode. +def DefaultMode : HwMode<"">; + +// A class used to associate objects with hw modes. It is only intended to +// be used as a base class, where the derived class should contain a member +// "Objects", which is a list of the same length as the list of modes. +// The n-th element on the Objects list will be associated with the n-th +// element on the Modes list. +class HwModeSelect Ms> { + list Modes = Ms; +} + +// A common class that implements a counterpart of ValueType, which is +// dependent on a hw mode. This class inherits from ValueType itself, +// which makes it possible to use objects of this class where ValueType +// objects could be used. This is specifically applicable to selection +// patterns. +class ValueTypeByHwMode Ms, list Ts> + : HwModeSelect, ValueType<0, 0> { + // The length of this list must be the same as the length of Ms. + list Objects = Ts; +} + +// A class representing size/alignment characteristics of a register. +class RegInfo { + int RegSize = RS; // Register size in bits. + int SpillSize = SS; // Spill slot size in bits. + int SpillAlignment = SA; // Spill slot alignment in bits. +} + +// The register size/alignment information, parametrized by a hw mode. +class RegInfoByHwMode Ms = [], list Ts = []> + : HwModeSelect { + // The length of this list must be the same as the length of Ms. + list Objects = Ts; +} + // SubRegIndex - Use instances of SubRegIndex to identify subregisters. class SubRegIndex { string Namespace = ""; @@ -156,6 +205,9 @@ : DAGOperand { string Namespace = namespace; + // The register size/alignment information, parametrized by a hw mode. + RegInfoByHwMode RegInfos; + // RegType - Specify the list ValueType of the registers in this register // class. Note that all registers in a register class must have the same // ValueTypes. This is a list because some targets permit storing different Index: include/llvm/Target/TargetRegisterInfo.h =================================================================== --- include/llvm/Target/TargetRegisterInfo.h +++ include/llvm/Target/TargetRegisterInfo.h @@ -49,8 +49,6 @@ // Instance variables filled by tablegen, do not use! const MCRegisterClass *MC; - const uint16_t SpillSize, SpillAlignment; - const MVT::SimpleValueType *VTs; const uint32_t *SubClassMask; const uint16_t *SuperRegIndices; const LaneBitmask LaneMask; @@ -222,7 +220,10 @@ public: using regclass_iterator = const TargetRegisterClass * const *; using vt_iterator = const MVT::SimpleValueType *; - + struct RegClassInfo { + unsigned RegSize, SpillSize, SpillAlignment; + vt_iterator VTList; + }; private: const TargetRegisterInfoDesc *InfoDesc; // Extra desc array for codegen const char *const *SubRegIndexNames; // Names of subreg indexes. @@ -231,6 +232,8 @@ regclass_iterator RegClassBegin, RegClassEnd; // List of regclasses LaneBitmask CoveringLanes; + const RegClassInfo *const RCInfos; + unsigned HwMode; protected: TargetRegisterInfo(const TargetRegisterInfoDesc *ID, @@ -238,7 +241,9 @@ regclass_iterator RegClassEnd, const char *const *SRINames, const LaneBitmask *SRILaneMasks, - LaneBitmask CoveringLanes); + LaneBitmask CoveringLanes, + const RegClassInfo *const RSI, + unsigned Mode = 0); virtual ~TargetRegisterInfo(); public: @@ -306,25 +311,25 @@ /// Return the size in bits of a register from class RC. unsigned getRegSizeInBits(const TargetRegisterClass &RC) const { - return RC.SpillSize * 8; + return getRegClassInfo(RC).RegSize; } /// Return the size in bytes of the stack slot allocated to hold a spilled /// copy of a register from class RC. unsigned getSpillSize(const TargetRegisterClass &RC) const { - return RC.SpillSize; + return getRegClassInfo(RC).SpillSize / 8; } /// Return the minimum required alignment for a spill slot for a register /// of this class. unsigned getSpillAlignment(const TargetRegisterClass &RC) const { - return RC.SpillAlignment; + return getRegClassInfo(RC).SpillAlignment / 8; } /// Return true if the given TargetRegisterClass has the ValueType T. bool isTypeLegalForClass(const TargetRegisterClass &RC, MVT T) const { - for (int i = 0; RC.VTs[i] != MVT::Other; ++i) - if (MVT(RC.VTs[i]) == T) + for (auto I = legalclasstypes_begin(RC); *I != MVT::Other; ++I) + if (MVT(*I) == T) return true; return false; } @@ -332,11 +337,11 @@ /// Loop over all of the value types that can be represented by values // in the given register class. vt_iterator legalclasstypes_begin(const TargetRegisterClass &RC) const { - return RC.VTs; + return getRegClassInfo(RC).VTList; } vt_iterator legalclasstypes_end(const TargetRegisterClass &RC) const { - vt_iterator I = RC.VTs; + vt_iterator I = legalclasstypes_begin(RC); while (*I != MVT::Other) ++I; return I; @@ -654,7 +659,12 @@ //===--------------------------------------------------------------------===// // Register Class Information // +protected: + const RegClassInfo &getRegClassInfo(const TargetRegisterClass &RC) const { + return RCInfos[getNumRegClasses() * HwMode + RC.getID()]; + } +public: /// Register class iterators regclass_iterator regclass_begin() const { return RegClassBegin; } regclass_iterator regclass_end() const { return RegClassEnd; } Index: lib/CodeGen/TargetRegisterInfo.cpp =================================================================== --- lib/CodeGen/TargetRegisterInfo.cpp +++ lib/CodeGen/TargetRegisterInfo.cpp @@ -41,11 +41,14 @@ regclass_iterator RCB, regclass_iterator RCE, const char *const *SRINames, const LaneBitmask *SRILaneMasks, - LaneBitmask SRICoveringLanes) + LaneBitmask SRICoveringLanes, + const RegClassInfo *const RCIs, + unsigned Mode) : InfoDesc(ID), SubRegIndexNames(SRINames), SubRegIndexLaneMasks(SRILaneMasks), RegClassBegin(RCB), RegClassEnd(RCE), - CoveringLanes(SRICoveringLanes) { + CoveringLanes(SRICoveringLanes), + RCInfos(RCIs), HwMode(Mode) { } TargetRegisterInfo::~TargetRegisterInfo() = default; Index: test/TableGen/GlobalISelEmitter.td =================================================================== --- test/TableGen/GlobalISelEmitter.td +++ test/TableGen/GlobalISelEmitter.td @@ -118,7 +118,7 @@ // CHECK-NEXT: // MIs[0] src3 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckComplexPattern, /*MI*/0, /*Op*/3, /*Renderer*/1, GICP_gi_complex, -// CHECK-NEXT: // (select:i32 GPR32:i32:$src1, complex:i32:$src2, complex:i32:$src3) => (INSN2:i32 GPR32:i32:$src1, complex:i32:$src3, complex:i32:$src2) +// CHECK-NEXT: // (select:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2, complex:{ *:[i32] }:$src3) => (INSN2:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN2, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 @@ -167,7 +167,7 @@ // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/3, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckComplexPattern, /*MI*/1, /*Op*/3, /*Renderer*/2, GICP_gi_complex, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, -// CHECK-NEXT: // (select:i32 GPR32:i32:$src1, complex:i32:$src2, (select:i32 GPR32:i32:$src3, complex:i32:$src4, complex:i32:$src5)) => (INSN3:i32 GPR32:i32:$src1, complex:i32:$src2, GPR32:i32:$src3, complex:i32:$src4, complex:i32:$src5) +// CHECK-NEXT: // (select:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2, (select:{ *:[i32] } GPR32:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src4, complex:{ *:[i32] }:$src5)) => (INSN3:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src4, complex:{ *:[i32] }:$src5) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN3, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 @@ -201,7 +201,7 @@ // CHECK-NEXT: // MIs[0] src2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // (add:i32 GPR32:i32:$src1, GPR32:i32:$src2) => (ADD:i32 GPR32:i32:$src1, GPR32:i32:$src2) +// CHECK-NEXT: // (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) => (ADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ADD, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, // CHECK-NEXT: GIR_Done, @@ -224,7 +224,7 @@ // CHECK-NEXT: // MIs[0] src1 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // (intrinsic_wo_chain:i32 [[ID:[0-9]+]]:iPTR, GPR32:i32:$src1) => (MOV:i32 GPR32:i32:$src1) +// CHECK-NEXT: // (intrinsic_wo_chain:{ *:[i32] } [[ID:[0-9]+]]:{ *:[iPTR] }, GPR32:{ *:[i32] }:$src1) => (MOV:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOV, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst @@ -263,7 +263,7 @@ // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, -// CHECK-NEXT: // (mul:i32 (add:i32 GPR32:i32:$src1, GPR32:i32:$src2), GPR32:i32:$src3) => (MULADD:i32 GPR32:i32:$src1, GPR32:i32:$src2, GPR32:i32:$src3) +// CHECK-NEXT: // (mul:{ *:[i32] } (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src3) => (MULADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MULADD, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 @@ -299,7 +299,7 @@ // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, -// CHECK-NEXT: // (mul:i32 GPR32:i32:$src3, (add:i32 GPR32:i32:$src1, GPR32:i32:$src2)) => (MULADD:i32 GPR32:i32:$src1, GPR32:i32:$src2, GPR32:i32:$src3) +// CHECK-NEXT: // (mul:{ *:[i32] } GPR32:{ *:[i32] }:$src3, (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)) => (MULADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MULADD, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 @@ -330,7 +330,7 @@ // CHECK-NEXT: // MIs[0] src2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // (mul:i32 GPR32:i32:$src1, GPR32:i32:$src2) => (MUL:i32 GPR32:i32:$src2, GPR32:i32:$src1) +// CHECK-NEXT: // (mul:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) => (MUL:{ *:[i32] } GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src1) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MUL, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src2 @@ -381,7 +381,7 @@ // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/2, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/2, -// CHECK-NEXT: // (sub:i32 (sub:i32 GPR32:i32:$src1, GPR32:i32:$src2), (sub:i32 GPR32:i32:$src3, GPR32:i32:$src4)) => (INSNBOB:i32 GPR32:i32:$src1, GPR32:i32:$src2, GPR32:i32:$src3, GPR32:i32:$src4) +// CHECK-NEXT: // (sub:{ *:[i32] } (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src3, GPR32:{ *:[i32] }:$src4)) => (INSNBOB:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3, GPR32:{ *:[i32] }:$src4) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSNBOB, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 @@ -413,7 +413,7 @@ // CHECK-NEXT: // MIs[0] src2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/0, GICP_gi_complex, -// CHECK-NEXT: // (sub:i32 GPR32:i32:$src1, complex:i32:$src2) => (INSN1:i32 GPR32:i32:$src1, complex:i32:$src2) +// CHECK-NEXT: // (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2) => (INSN1:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN1, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 @@ -441,7 +441,7 @@ // CHECK-NEXT: // MIs[0] Operand 2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -2 -// CHECK-NEXT: // (xor:i32 GPR32:i32:$src1, -2:i32) => (XORI:i32 GPR32:i32:$src1) +// CHECK-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -2:{ *:[i32] }) => (XORI:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORI, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/-1, @@ -470,7 +470,7 @@ // CHECK-NEXT: // MIs[0] Operand 2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -3 -// CHECK-NEXT: // (xor:i32 GPR32:i32:$src1, -3:i32) => (XOR:i32 GPR32:i32:$src1) +// CHECK-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -3:{ *:[i32] }) => (XOR:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XOR, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, @@ -499,7 +499,7 @@ // CHECK-NEXT: // MIs[0] Operand 2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -4 -// CHECK-NEXT: // (xor:i32 GPR32:i32:$src1, -4:i32) => (XORlike:i32 GPR32:i32:$src1) +// CHECK-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -4:{ *:[i32] }) => (XORlike:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORlike, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/-1, @@ -529,7 +529,7 @@ // CHECK-NEXT: // MIs[0] Operand 2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -5, -// CHECK-NEXT: // (xor:i32 GPR32:i32:$src1, -5:i32) => (XORManyDefaults:i32 GPR32:i32:$src1) +// CHECK-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -5:{ *:[i32] }) => (XORManyDefaults:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORManyDefaults, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/-1, @@ -562,7 +562,7 @@ // CHECK-NEXT: // MIs[0] Operand 2 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -1, -// CHECK-NEXT: // (xor:i32 GPR32:i32:$Wm, -1:i32) => (ORN:i32 R0:i32, GPR32:i32:$Wm) +// CHECK-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$Wm, -1:{ *:[i32] }) => (ORN:{ *:[i32] } R0:{ *:[i32] }, GPR32:{ *:[i32] }:$Wm) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ORN, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, @@ -587,7 +587,7 @@ // CHECK-NEXT: // MIs[0] src1 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::FPR32RegClassID, -// CHECK-NEXT: // (bitconvert:i32 FPR32:f32:$src1) => (COPY_TO_REGCLASS:i32 FPR32:f32:$src1, GPR32:i32) +// CHECK-NEXT: // (bitconvert:{ *:[i32] } FPR32:{ *:[f32] }:$src1) => (COPY_TO_REGCLASS:{ *:[i32] } FPR32:{ *:[f32] }:$src1, GPR32:{ *:[i32] }) // CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/TargetOpcode::COPY, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, /*RC GPR32*/1, // CHECK-NEXT: GIR_Done, @@ -606,7 +606,7 @@ // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] Operand 1 // CHECK-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, 1, -// CHECK-NEXT: // 1:i32 => (MOV1:i32) +// CHECK-NEXT: // 1:{ *:[i32] } => (MOV1:{ *:[i32] }) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOV1, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -626,7 +626,7 @@ // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] Operand 1 // CHECK-NEXT: // No predicates -// CHECK-NEXT: // (imm:i32):$imm => (MOVimm:i32 (imm:i32):$imm) +// CHECK-NEXT: // (imm:{ *:[i32] }):$imm => (MOVimm:{ *:[i32] } (imm:{ *:[i32] }):$imm) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst // CHECK-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm @@ -644,7 +644,7 @@ // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_BR, // CHECK-NEXT: // MIs[0] target // CHECK-NEXT: GIM_CheckIsMBB, /*MI*/0, /*Op*/0, -// CHECK-NEXT: // (br (bb:Other):$target) => (BR (bb:Other):$target) +// CHECK-NEXT: // (br (bb:{ *:[Other] }):$target) => (BR (bb:{ *:[Other] }):$target) // CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::BR, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, // CHECK-NEXT: GIR_Done, Index: test/TableGen/HwModeSelect.td =================================================================== --- /dev/null +++ test/TableGen/HwModeSelect.td @@ -0,0 +1,20 @@ +// RUN: not llvm-tblgen -gen-dag-isel -I %p/../../include %s 2>&1 | FileCheck %s + +include "llvm/Target/Target.td" + +def TestTargetInstrInfo : InstrInfo; + +def TestTarget : Target { + let InstructionSet = TestTargetInstrInfo; +} + +def TestReg : Register<"testreg">; +def TestClass : RegisterClass<"TestTarget", [i32], 32, (add TestReg)>; + +def TestMode1 : HwMode<"+feat1">; +def TestMode2 : HwMode<"+feat2">; + +def BadDef : ValueTypeByHwMode<[TestMode1, TestMode2, DefaultMode], + [i8, i16, i32, i64]>; + +// CHECK: error: in record BadDef derived from HwModeSelect: the lists Modes and Objects should have the same size Index: utils/TableGen/CMakeLists.txt =================================================================== --- utils/TableGen/CMakeLists.txt +++ utils/TableGen/CMakeLists.txt @@ -8,6 +8,7 @@ CallingConvEmitter.cpp CodeEmitterGen.cpp CodeGenDAGPatterns.cpp + CodeGenHwModes.cpp CodeGenInstruction.cpp CodeGenMapTable.cpp CodeGenRegisters.cpp @@ -23,6 +24,7 @@ FastISelEmitter.cpp FixedLenDecoderEmitter.cpp GlobalISelEmitter.cpp + InfoByHwMode.cpp InstrInfoEmitter.cpp IntrinsicEmitter.cpp OptParserEmitter.cpp Index: utils/TableGen/CodeGenDAGPatterns.h =================================================================== --- utils/TableGen/CodeGenDAGPatterns.h +++ utils/TableGen/CodeGenDAGPatterns.h @@ -15,6 +15,7 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H #define LLVM_UTILS_TABLEGEN_CODEGENDAGPATTERNS_H +#include "CodeGenHwModes.h" #include "CodeGenIntrinsics.h" #include "CodeGenTarget.h" #include "llvm/ADT/SmallVector.h" @@ -36,134 +37,111 @@ class CodeGenDAGPatterns; class ComplexPattern; -/// EEVT::DAGISelGenValueType - These are some extended forms of -/// MVT::SimpleValueType that we use as lattice values during type inference. -/// The existing MVT iAny, fAny and vAny types suffice to represent -/// arbitrary integer, floating-point, and vector types, so only an unknown -/// value is needed. -namespace EEVT { - /// TypeSet - This is either empty if it's completely unknown, or holds a set - /// of types. It is used during type inference because register classes can - /// have multiple possible types and we don't know which one they get until - /// type inference is complete. - /// - /// TypeSet can have three states: - /// Vector is empty: The type is completely unknown, it can be any valid - /// target type. - /// Vector has multiple constrained types: (e.g. v4i32 + v4f32) it is one - /// of those types only. - /// Vector has one concrete type: The type is completely known. - /// - class TypeSet { - SmallVector TypeVec; - public: - TypeSet() {} - TypeSet(MVT::SimpleValueType VT, TreePattern &TP); - TypeSet(ArrayRef VTList); - - bool isCompletelyUnknown() const { return TypeVec.empty(); } - - bool isConcrete() const { - if (TypeVec.size() != 1) return false; - unsigned char T = TypeVec[0]; (void)T; - assert(T < MVT::LAST_VALUETYPE || T == MVT::iPTR || T == MVT::iPTRAny); - return true; - } - - MVT::SimpleValueType getConcrete() const { - assert(isConcrete() && "Type isn't concrete yet"); - return (MVT::SimpleValueType)TypeVec[0]; - } - - bool isDynamicallyResolved() const { - return getConcrete() == MVT::iPTR || getConcrete() == MVT::iPTRAny; - } - - const SmallVectorImpl &getTypeList() const { - assert(!TypeVec.empty() && "Not a type list!"); - return TypeVec; - } - - bool isVoid() const { - return TypeVec.size() == 1 && TypeVec[0] == MVT::isVoid; - } - - /// hasIntegerTypes - Return true if this TypeSet contains any integer value - /// types. - bool hasIntegerTypes() const; - - /// hasFloatingPointTypes - Return true if this TypeSet contains an fAny or - /// a floating point value type. - bool hasFloatingPointTypes() const; - - /// hasScalarTypes - Return true if this TypeSet contains a scalar value - /// type. - bool hasScalarTypes() const; - - /// hasVectorTypes - Return true if this TypeSet contains a vector value - /// type. - bool hasVectorTypes() const; - - /// getName() - Return this TypeSet as a string. - std::string getName() const; - - /// MergeInTypeInfo - This merges in type information from the specified - /// argument. If 'this' changes, it returns true. If the two types are - /// contradictory (e.g. merge f32 into i32) then this flags an error. - bool MergeInTypeInfo(const EEVT::TypeSet &InVT, TreePattern &TP); - - bool MergeInTypeInfo(MVT::SimpleValueType InVT, TreePattern &TP) { - return MergeInTypeInfo(EEVT::TypeSet(InVT, TP), TP); - } - - /// Force this type list to only contain integer types. - bool EnforceInteger(TreePattern &TP); - - /// Force this type list to only contain floating point types. - bool EnforceFloatingPoint(TreePattern &TP); - - /// EnforceScalar - Remove all vector types from this type list. - bool EnforceScalar(TreePattern &TP); - - /// EnforceVector - Remove all non-vector types from this type list. - bool EnforceVector(TreePattern &TP); - - /// EnforceSmallerThan - 'this' must be a smaller VT than Other. Update - /// this an other based on this information. - bool EnforceSmallerThan(EEVT::TypeSet &Other, TreePattern &TP); - - /// EnforceVectorEltTypeIs - 'this' is now constrained to be a vector type - /// whose element is VT. - bool EnforceVectorEltTypeIs(EEVT::TypeSet &VT, TreePattern &TP); - - /// EnforceVectorEltTypeIs - 'this' is now constrained to be a vector type - /// whose element is VT. - bool EnforceVectorEltTypeIs(MVT::SimpleValueType VT, TreePattern &TP); - - /// EnforceVectorSubVectorTypeIs - 'this' is now constrained to - /// be a vector type VT. - bool EnforceVectorSubVectorTypeIs(EEVT::TypeSet &VT, TreePattern &TP); - - /// EnforceSameNumElts - If VTOperand is a scalar, then 'this' is a scalar. - /// If VTOperand is a vector, then 'this' must have the same number of - /// elements. - bool EnforceSameNumElts(EEVT::TypeSet &VT, TreePattern &TP); - - /// EnforceSameSize - 'this' is now constrained to be the same size as VT. - bool EnforceSameSize(EEVT::TypeSet &VT, TreePattern &TP); - - bool operator!=(const TypeSet &RHS) const { return TypeVec != RHS.TypeVec; } - bool operator==(const TypeSet &RHS) const { return TypeVec == RHS.TypeVec; } - - private: - /// FillWithPossibleTypes - Set to all legal types and return true, only - /// valid on completely unknown type sets. If Pred is non-null, only MVTs - /// that pass the predicate are added. - bool FillWithPossibleTypes(TreePattern &TP, - bool (*Pred)(MVT::SimpleValueType) = nullptr, - const char *PredicateName = nullptr); +struct TypeSetByHwMode : public InfoByHwMode> { + typedef std::set SetType; + + TypeSetByHwMode() = default; + TypeSetByHwMode(const TypeSetByHwMode &VTS) = default; + TypeSetByHwMode(MVT::SimpleValueType VT) + : TypeSetByHwMode(ValueTypeByHwMode(VT)) {} + TypeSetByHwMode(ValueTypeByHwMode VT) + : TypeSetByHwMode(ArrayRef(&VT, 1)) {} + TypeSetByHwMode(ArrayRef VTList); + + SetType &getOrCreate(unsigned Mode) { + if (hasMode(Mode)) + return get(Mode); + return Map.insert({Mode,SetType()}).first->second; + } + + bool isValueTypeByHwMode(bool AllowEmpty) const; + ValueTypeByHwMode getValueTypeByHwMode() const; + bool isMachineValueType() const { + return isDefaultOnly() && Map.begin()->second.size() == 1; + } + MVT getMachineValueType() const { + assert(isMachineValueType()); + return *Map.begin()->second.begin(); + } + + bool isPossible() const; + bool isDefaultOnly() const { + return Map.size() == 1 && + Map.begin()->first == DefaultMode; + } + + bool insert(const ValueTypeByHwMode &VVT); + bool constrain(const TypeSetByHwMode &VTS); + template bool constrain(Predicate P); + template bool assign_if(const TypeSetByHwMode &VTS, + Predicate P); + + std::string getAsString() const; + static std::string getAsString(const SetType &S); + + bool operator==(const TypeSetByHwMode &VTS) const; + bool operator!=(const TypeSetByHwMode &VTS) const { return !(*this == VTS); } + + void dump() const; + void validate() const; + +private: + /// Intersect two sets. Return true if anything has changed. + bool intersect(SetType &Out, const SetType &In); +}; + +struct TypeInfer { + TypeInfer(TreePattern &T) : TP(T), ForceMode(0) {} + + bool isConcrete(const TypeSetByHwMode &VTS, bool AllowEmpty) const { + return VTS.isValueTypeByHwMode(AllowEmpty); + } + ValueTypeByHwMode getConcrete(const TypeSetByHwMode &VTS, + bool AllowEmpty) const { + assert(VTS.isValueTypeByHwMode(AllowEmpty)); + return VTS.getValueTypeByHwMode(); + } + + bool MergeInTypeInfo(TypeSetByHwMode &Out, const TypeSetByHwMode &In); + bool MergeInTypeInfo(TypeSetByHwMode &Out, MVT::SimpleValueType InVT) { + return MergeInTypeInfo(Out, TypeSetByHwMode(InVT)); + } + bool MergeInTypeInfo(TypeSetByHwMode &Out, ValueTypeByHwMode InVT) { + return MergeInTypeInfo(Out, TypeSetByHwMode(InVT)); + } + bool forceArbitrary(TypeSetByHwMode &Out); + + bool EnforceInteger(TypeSetByHwMode &Out); + bool EnforceFloatingPoint(TypeSetByHwMode &Out); + bool EnforceScalar(TypeSetByHwMode &Out); + bool EnforceVector(TypeSetByHwMode &Out); + bool EnforceAny(TypeSetByHwMode &Out); + bool EnforceSmallerThan(TypeSetByHwMode &Small, TypeSetByHwMode &Big); + bool EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, TypeSetByHwMode &Elem); + bool EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, + const ValueTypeByHwMode &VVT); + bool EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec, + TypeSetByHwMode &Sub); + bool EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W); + bool EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B); + + void expandOverloads(TypeSetByHwMode &VTS); + void expandOverloads(TypeSetByHwMode::SetType &Out, + const TypeSetByHwMode::SetType &Legal); + + struct ValidateOnExit { + ValidateOnExit(TypeSetByHwMode &T) : VTS(T) {} + ~ValidateOnExit() { VTS.validate(); } + TypeSetByHwMode &VTS; }; -} + + TreePattern &TP; + unsigned ForceMode; // Mode to use when set. + bool CodeGen = false; // Set during generation of matcher code. + +private: + TypeSetByHwMode getLegalTypes(); +}; /// Set type used to track multiply used variables in patterns typedef std::set MultipleUseVarSet; @@ -171,7 +149,7 @@ /// SDTypeConstraint - This is a discriminated union of constraints, /// corresponding to the SDTypeConstraint tablegen class in Target.td. struct SDTypeConstraint { - SDTypeConstraint(Record *R); + SDTypeConstraint(Record *R, const CodeGenHwModes &CGH); unsigned OperandNo; // The operand # this constraint applies to. enum { @@ -182,9 +160,6 @@ union { // The discriminated union. struct { - MVT::SimpleValueType VT; - } SDTCisVT_Info; - struct { unsigned OtherOperandNum; } SDTCisSameAs_Info; struct { @@ -200,9 +175,6 @@ unsigned OtherOperandNum; } SDTCisSubVecOfVec_Info; struct { - MVT::SimpleValueType VT; - } SDTCVecEltisVT_Info; - struct { unsigned OtherOperandNum; } SDTCisSameNumEltsAs_Info; struct { @@ -210,6 +182,10 @@ } SDTCisSameSizeAs_Info; } x; + // The VT for SDTCisVT and SDTCVecEltisVT. + // Must not be in the union because it has a non-trivial destructor. + ValueTypeByHwMode VVT; + /// ApplyTypeConstraint - Given a node in a pattern, apply this type /// constraint to the nodes operands. This returns true if it makes a /// change, false otherwise. If a type contradiction is found, an error @@ -230,7 +206,8 @@ int NumOperands; std::vector TypeConstraints; public: - SDNodeInfo(Record *R); // Parse the specified record. + // Parse the specified record. + SDNodeInfo(Record *R, const CodeGenHwModes &CGH); unsigned getNumResults() const { return NumResults; } @@ -258,12 +235,7 @@ /// constraints for this node to the operands of the node. This returns /// true if it makes a change, false otherwise. If a type contradiction is /// found, an error is flagged. - bool ApplyTypeConstraints(TreePatternNode *N, TreePattern &TP) const { - bool MadeChange = false; - for (unsigned i = 0, e = TypeConstraints.size(); i != e; ++i) - MadeChange |= TypeConstraints[i].ApplyTypeConstraint(N, *this, TP); - return MadeChange; - } + bool ApplyTypeConstraints(TreePatternNode *N, TreePattern &TP) const; }; /// TreePredicateFn - This is an abstraction that represents the predicates on @@ -324,7 +296,7 @@ /// The type of each node result. Before and during type inference, each /// result may be a set of possible types. After (successful) type inference, /// each is a single concrete type. - SmallVector Types; + std::vector Types; /// Operator - The Record for the operator if this is an interior node (not /// a leaf). @@ -367,22 +339,24 @@ // Type accessors. unsigned getNumTypes() const { return Types.size(); } - MVT::SimpleValueType getType(unsigned ResNo) const { - return Types[ResNo].getConcrete(); + ValueTypeByHwMode getType(unsigned ResNo) const { + return Types[ResNo].getValueTypeByHwMode(); } - const SmallVectorImpl &getExtTypes() const { return Types; } - const EEVT::TypeSet &getExtType(unsigned ResNo) const { return Types[ResNo]; } - EEVT::TypeSet &getExtType(unsigned ResNo) { return Types[ResNo]; } - void setType(unsigned ResNo, const EEVT::TypeSet &T) { Types[ResNo] = T; } - - bool hasTypeSet(unsigned ResNo) const { - return Types[ResNo].isConcrete(); + const std::vector &getExtTypes() const { return Types; } + const TypeSetByHwMode &getExtType(unsigned ResNo) const { + return Types[ResNo]; } - bool isTypeCompletelyUnknown(unsigned ResNo) const { - return Types[ResNo].isCompletelyUnknown(); + TypeSetByHwMode &getExtType(unsigned ResNo) { return Types[ResNo]; } + void setType(unsigned ResNo, const TypeSetByHwMode &T) { Types[ResNo] = T; } + MVT::SimpleValueType getSimpleType(unsigned ResNo) const { + return Types[ResNo].getMachineValueType().SimpleTy; } - bool isTypeDynamicallyResolved(unsigned ResNo) const { - return Types[ResNo].isDynamicallyResolved(); + + bool hasConcreteType(unsigned ResNo) const { + return Types[ResNo].isValueTypeByHwMode(false); + } + bool isTypeCompletelyUnknown(unsigned ResNo, TreePattern &TP) const { + return Types[ResNo].empty(); } Init *getLeafValue() const { assert(isLeaf()); return Val; } @@ -401,6 +375,10 @@ return false; } + bool hasProperTypeByHwMode() const; + bool hasPossibleType() const; + bool setDefaultMode(unsigned Mode); + bool hasAnyPredicate() const { return !PredicateFns.empty(); } const std::vector &getPredicateFns() const { @@ -484,15 +462,12 @@ /// information. If N already contains a conflicting type, then flag an /// error. This returns true if any information was updated. /// - bool UpdateNodeType(unsigned ResNo, const EEVT::TypeSet &InTy, - TreePattern &TP) { - return Types[ResNo].MergeInTypeInfo(InTy, TP); - } - + bool UpdateNodeType(unsigned ResNo, const TypeSetByHwMode &InTy, + TreePattern &TP); bool UpdateNodeType(unsigned ResNo, MVT::SimpleValueType InTy, - TreePattern &TP) { - return Types[ResNo].MergeInTypeInfo(EEVT::TypeSet(InTy, TP), TP); - } + TreePattern &TP); + bool UpdateNodeType(unsigned ResNo, ValueTypeByHwMode InTy, + TreePattern &TP); // Update node type with types inferred from an instruction operand or result // def from the ins/outs lists. @@ -501,14 +476,7 @@ /// ContainsUnresolvedType - Return true if this tree contains any /// unresolved types. - bool ContainsUnresolvedType() const { - for (unsigned i = 0, e = Types.size(); i != e; ++i) - if (!Types[i].isConcrete()) return true; - - for (unsigned i = 0, e = getNumChildren(); i != e; ++i) - if (getChild(i)->ContainsUnresolvedType()) return true; - return false; - } + bool ContainsUnresolvedType(TreePattern &TP) const; /// canPatternMatch - If it is impossible for this pattern to match on this /// target, fill in Reason and return false. Otherwise, return true. @@ -560,6 +528,9 @@ /// number for each operand encountered in a ComplexPattern to aid in that /// check. StringMap> ComplexPatternOperands; + + TypeInfer Infer; + public: /// TreePattern constructor - Parse the specified DagInits into the @@ -625,6 +596,8 @@ HasError = false; } + TypeInfer &getInfer() { return Infer; } + void print(raw_ostream &OS) const; void dump() const; @@ -634,6 +607,32 @@ void ComputeNamedNodes(TreePatternNode *N); }; + +inline bool TreePatternNode::UpdateNodeType(unsigned ResNo, + const TypeSetByHwMode &InTy, + TreePattern &TP) { + TypeSetByHwMode VTS(InTy); + TP.getInfer().expandOverloads(VTS); + return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS); +} + +inline bool TreePatternNode::UpdateNodeType(unsigned ResNo, + MVT::SimpleValueType InTy, + TreePattern &TP) { + TypeSetByHwMode VTS(InTy); + TP.getInfer().expandOverloads(VTS); + return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS); +} + +inline bool TreePatternNode::UpdateNodeType(unsigned ResNo, + ValueTypeByHwMode InTy, + TreePattern &TP) { + TypeSetByHwMode VTS(InTy); + TP.getInfer().expandOverloads(VTS); + return TP.getInfer().MergeInTypeInfo(Types[ResNo], VTS); +} + + /// DAGDefaultOperand - One of these is created for each OperandWithDefaultOps /// that has a set ExecuteAlways / DefaultOps field. struct DAGDefaultOperand { @@ -680,31 +679,85 @@ TreePatternNode *getResultPattern() const { return ResultPattern; } }; +/// This class represents a condition that has to be satisfied for a pattern +/// to be tried. It is a generalization of a class "Pattern" from Target.td: +/// in addition to the Target.td's predicates, this class can also represent +/// conditions associated with hw modes. Both types will eventually become +/// strings containing C++ code to be executed, the difference is in how +/// these strings are generated. +class Predicate { +public: + Predicate(Record *R, bool C = true) : Def(R), IfCond(C), IsHwMode(false) { + assert(R->isSubClassOf("Predicate")); + } + Predicate(StringRef FS, bool C = true) : Def(nullptr), Features(FS.str()), + IfCond(C), IsHwMode(true) {} + + std::string getCondString() const { + // The string will excute in a subclass of SelectionDAGISel. + // Cast to std::string explicitly to avoid ambiguity with StringRef. + std::string C = IsHwMode + ? std::string("MF->getSubtarget().checkFeatures(\"" + Features + "\")") + : std::string(Def->getValueAsString("CondString")); + return IfCond ? C : "!("+C+')'; + } + bool operator==(const Predicate &P) const { + return IfCond == P.IfCond && IsHwMode == P.IsHwMode && Def == P.Def; + } + bool operator<(const Predicate &P) const { + if (IsHwMode != P.IsHwMode) + return IsHwMode < P.IsHwMode; + assert(!Def == !P.Def && "Inconsistency between Def and IsHwMode"); + if (IfCond != P.IfCond) + return IfCond < P.IfCond; + if (Def) + return LessRecord()(Def, P.Def); + return Features < P.Features; + } + Record *Def; ///< Predicate definition from .td file, null for + ///< hw modes. + std::string Features; ///< Feature string for hw mode. + bool IfCond; ///< The boolean value that the condition has to + ///< evaluate to for this predicate to be true. + bool IsHwMode; ///< Does this predicate correspond to a hw mode? +}; + /// PatternToMatch - Used by CodeGenDAGPatterns to keep tab of patterns /// processed to produce isel. class PatternToMatch { public: - PatternToMatch(Record *srcrecord, ListInit *preds, TreePatternNode *src, - TreePatternNode *dst, std::vector dstregs, - int complexity, unsigned uid) - : SrcRecord(srcrecord), Predicates(preds), SrcPattern(src), - DstPattern(dst), Dstregs(std::move(dstregs)), - AddedComplexity(complexity), ID(uid) {} + PatternToMatch(Record *srcrecord, const std::vector &preds, + TreePatternNode *src, TreePatternNode *dst, + const std::vector &dstregs, + int complexity, unsigned uid, unsigned setmode = 0) + : SrcRecord(srcrecord), SrcPattern(src), DstPattern(dst), + Predicates(preds), Dstregs(std::move(dstregs)), + AddedComplexity(complexity), ID(uid), ForceMode(setmode) {} + + PatternToMatch(Record *srcrecord, std::vector &&preds, + TreePatternNode *src, TreePatternNode *dst, + std::vector &&dstregs, + int complexity, unsigned uid, unsigned setmode = 0) + : SrcRecord(srcrecord), SrcPattern(src), DstPattern(dst), + Predicates(preds), Dstregs(std::move(dstregs)), + AddedComplexity(complexity), ID(uid), ForceMode(setmode) {} Record *SrcRecord; // Originating Record for the pattern. - ListInit *Predicates; // Top level predicate conditions to match. TreePatternNode *SrcPattern; // Source pattern to match. TreePatternNode *DstPattern; // Resulting pattern. + std::vector Predicates; // Top level predicate conditions + // to match. std::vector Dstregs; // Physical register defs being matched. int AddedComplexity; // Add to matching pattern complexity. unsigned ID; // Unique ID for the record. + unsigned ForceMode; // Force this mode in type inference when set. Record *getSrcRecord() const { return SrcRecord; } - ListInit *getPredicates() const { return Predicates; } TreePatternNode *getSrcPattern() const { return SrcPattern; } TreePatternNode *getDstPattern() const { return DstPattern; } const std::vector &getDstRegs() const { return Dstregs; } int getAddedComplexity() const { return AddedComplexity; } + const std::vector &getPredicates() const { return Predicates; } std::string getPredicateCheck() const; @@ -720,7 +773,8 @@ CodeGenIntrinsicTable TgtIntrinsics; std::map SDNodes; - std::map, LessRecordByID> SDNodeXForms; + std::map, LessRecordByID> + SDNodeXForms; std::map ComplexPatterns; std::map, LessRecordByID> PatternFragments; @@ -735,11 +789,15 @@ /// value is the pattern to match, the second pattern is the result to /// emit. std::vector PatternsToMatch; + + TypeSetByHwMode LegalVTS; + public: CodeGenDAGPatterns(RecordKeeper &R); CodeGenTarget &getTargetInfo() { return Target; } const CodeGenTarget &getTargetInfo() const { return Target; } + const TypeSetByHwMode &getLegalTypes() const { return LegalVTS; } Record *getSDNodeNamed(const std::string &Name) const; @@ -849,10 +907,13 @@ void ParseDefaultOperands(); void ParseInstructions(); void ParsePatterns(); + void ExpandHwModeBasedTypes(); void InferInstructionFlags(); void GenerateVariants(); void VerifyInstructionFlags(); + std::vector makePredList(ListInit *L); + void AddPatternToMatch(TreePattern *Pattern, PatternToMatch &&PTM); void FindPatternInputsAndOutputs(TreePattern *I, TreePatternNode *Pat, std::map &InstResults, std::vector &InstImpResults); }; + + +inline bool SDNodeInfo::ApplyTypeConstraints(TreePatternNode *N, + TreePattern &TP) const { + bool MadeChange = false; + for (unsigned i = 0, e = TypeConstraints.size(); i != e; ++i) + MadeChange |= TypeConstraints[i].ApplyTypeConstraint(N, *this, TP); + return MadeChange; + } } // end namespace llvm #endif Index: utils/TableGen/CodeGenDAGPatterns.cpp =================================================================== --- utils/TableGen/CodeGenDAGPatterns.cpp +++ utils/TableGen/CodeGenDAGPatterns.cpp @@ -14,6 +14,7 @@ #include "CodeGenDAGPatterns.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Twine.h" @@ -24,733 +25,801 @@ #include #include #include +#include using namespace llvm; #define DEBUG_TYPE "dag-patterns" -//===----------------------------------------------------------------------===// -// EEVT::TypeSet Implementation -//===----------------------------------------------------------------------===// - -static inline bool isInteger(MVT::SimpleValueType VT) { - return MVT(VT).isInteger(); +static inline bool isIntegerOrPtr(MVT VT) { + return VT.isInteger() || VT == MVT::iPTR; } -static inline bool isFloatingPoint(MVT::SimpleValueType VT) { - return MVT(VT).isFloatingPoint(); +static inline bool isFloatingPoint(MVT VT) { + return VT.isFloatingPoint(); } -static inline bool isVector(MVT::SimpleValueType VT) { - return MVT(VT).isVector(); +static inline bool isVector(MVT VT) { + return VT.isVector(); } -static inline bool isScalar(MVT::SimpleValueType VT) { - return !MVT(VT).isVector(); +static inline bool isScalar(MVT VT) { + return !VT.isVector(); } -EEVT::TypeSet::TypeSet(MVT::SimpleValueType VT, TreePattern &TP) { - if (VT == MVT::iAny) - EnforceInteger(TP); - else if (VT == MVT::fAny) - EnforceFloatingPoint(TP); - else if (VT == MVT::vAny) - EnforceVector(TP); - else { - assert((VT < MVT::LAST_VALUETYPE || VT == MVT::iPTR || - VT == MVT::iPTRAny || VT == MVT::Any) && "Not a concrete type!"); - TypeVec.push_back(VT); - } +template +static bool berase_if(std::set &S, Predicate P) { + bool Erased = false; + for (auto I = S.begin(); I != S.end(); ) { + if (P(*I)) { + Erased = true; + I = S.erase(I); + } else + ++I; + } + return Erased; } +// --- TypeSetByHwMode -EEVT::TypeSet::TypeSet(ArrayRef VTList) { - assert(!VTList.empty() && "empty list?"); - TypeVec.append(VTList.begin(), VTList.end()); - - if (!VTList.empty()) - assert(VTList[0] != MVT::iAny && VTList[0] != MVT::vAny && - VTList[0] != MVT::fAny); +// This is a parametrized type-set class. For each mode there is a list +// of types that are currently possible for a given tree node. Type +// inference will apply to each mode separately. - // Verify no duplicates. - array_pod_sort(TypeVec.begin(), TypeVec.end()); - assert(std::unique(TypeVec.begin(), TypeVec.end()) == TypeVec.end()); +TypeSetByHwMode::TypeSetByHwMode(ArrayRef VTList) { + for (const ValueTypeByHwMode &VVT : VTList) + insert(VVT); } -/// FillWithPossibleTypes - Set to all legal types and return true, only valid -/// on completely unknown type sets. -bool EEVT::TypeSet::FillWithPossibleTypes(TreePattern &TP, - bool (*Pred)(MVT::SimpleValueType), - const char *PredicateName) { - assert(isCompletelyUnknown()); - ArrayRef LegalTypes = - TP.getDAGPatterns().getTargetInfo().getLegalValueTypes(); +bool TypeSetByHwMode::isValueTypeByHwMode(bool AllowEmpty) const { + for (const auto &I : *this) { + if (I.second.size() > 1) + return false; + if (!AllowEmpty && I.second.empty()) + return false; + } + return true; +} - if (TP.hasError()) - return false; +ValueTypeByHwMode TypeSetByHwMode::getValueTypeByHwMode() const { + assert(isValueTypeByHwMode(true) && + "The type set has multiple types for at least one hw mode"); + ValueTypeByHwMode VVT; + for (const auto &I : *this) { + MVT T = I.second.empty() ? MVT::Other : *I.second.begin(); + VVT.getOrCreateTypeForMode(I.first, T); + } + return VVT; +} - for (MVT::SimpleValueType VT : LegalTypes) - if (!Pred || Pred(VT)) - TypeVec.push_back(VT); +bool TypeSetByHwMode::isPossible() const { + for (const auto &I : *this) + if (!I.second.empty()) + return true; + return false; +} - // If we have nothing that matches the predicate, bail out. - if (TypeVec.empty()) { - TP.error("Type inference contradiction found, no " + - std::string(PredicateName) + " types found"); - return false; +bool TypeSetByHwMode::insert(const ValueTypeByHwMode &VVT) { + bool Changed = false; + std::set Modes; + for (const auto &P : VVT) { + unsigned M = P.first; + Modes.insert(M); + // Make sure there exists a set for each specific mode from VVT. + Changed |= getOrCreate(M).insert(P.second).second; } - // No need to sort with one element. - if (TypeVec.size() == 1) return true; - // Remove duplicates. - array_pod_sort(TypeVec.begin(), TypeVec.end()); - TypeVec.erase(std::unique(TypeVec.begin(), TypeVec.end()), TypeVec.end()); + // If VVT has a default mode, add the corresponding type to all + // modes in "this" that do not exist in VVT. + if (Modes.count(DefaultMode)) { + MVT DT = VVT.getType(DefaultMode); + for (auto &I : *this) + if (!Modes.count(I.first)) + Changed |= I.second.insert(DT).second; + } - return true; + return Changed; } -/// hasIntegerTypes - Return true if this TypeSet contains iAny or an -/// integer value type. -bool EEVT::TypeSet::hasIntegerTypes() const { - return any_of(TypeVec, isInteger); -} +// Constrain the type set to be the intersection with VTS. +bool TypeSetByHwMode::constrain(const TypeSetByHwMode &VTS) { + bool Changed = false; + if (hasDefault()) { + for (const auto &I : VTS) { + unsigned M = I.first; + if (M == DefaultMode || hasMode(M)) + continue; + Map[M] = Map[DefaultMode]; + Changed = true; + } + } -/// hasFloatingPointTypes - Return true if this TypeSet contains an fAny or -/// a floating point value type. -bool EEVT::TypeSet::hasFloatingPointTypes() const { - return any_of(TypeVec, isFloatingPoint); + for (auto &I : *this) { + unsigned M = I.first; + SetType &S = I.second; + if (VTS.hasMode(M) || VTS.hasDefault()) { + Changed |= intersect(I.second, VTS.get(M)); + } else if (!S.empty()) { + S.clear(); + Changed = true; + } + } + return Changed; } -/// hasScalarTypes - Return true if this TypeSet contains a scalar value type. -bool EEVT::TypeSet::hasScalarTypes() const { - return any_of(TypeVec, isScalar); +template +bool TypeSetByHwMode::constrain(Predicate P) { + bool Changed = false; + for (auto &I : *this) + Changed |= berase_if(I.second, std::not1(std::ptr_fun(P))); + return Changed; } -/// hasVectorTypes - Return true if this TypeSet contains a vAny or a vector -/// value type. -bool EEVT::TypeSet::hasVectorTypes() const { - return any_of(TypeVec, isVector); +template +bool TypeSetByHwMode::assign_if(const TypeSetByHwMode &VTS, Predicate P) { + assert(empty()); + for (const auto &I : VTS) { + SetType &S = getOrCreate(I.first); + for (auto J : I.second) + if (P(J)) + S.insert(J); + } + return !empty(); } +std::string TypeSetByHwMode::getAsString() const { + std::stringstream str; + std::vector Modes; -std::string EEVT::TypeSet::getName() const { - if (TypeVec.empty()) return ""; + for (const auto &I : *this) + Modes.push_back(I.first); + if (Modes.empty()) + return "{}"; + array_pod_sort(Modes.begin(), Modes.end()); - std::string Result; - - for (unsigned i = 0, e = TypeVec.size(); i != e; ++i) { - std::string VTName = llvm::getEnumName(TypeVec[i]); - // Strip off MVT:: prefix if present. - if (VTName.substr(0,5) == "MVT::") - VTName = VTName.substr(5); - if (i) Result += ':'; - Result += VTName; + str << '{'; + for (unsigned M : Modes) { + const SetType &S = get(M); + str << ' ' << getModeName(M) << ':' << getAsString(S); } - - if (TypeVec.size() == 1) - return Result; - return "{" + Result + "}"; + str << " }"; + return str.str(); } -/// MergeInTypeInfo - This merges in type information from the specified -/// argument. If 'this' changes, it returns true. If the two types are -/// contradictory (e.g. merge f32 into i32) then this flags an error. -bool EEVT::TypeSet::MergeInTypeInfo(const EEVT::TypeSet &InVT, TreePattern &TP){ - if (InVT.isCompletelyUnknown() || *this == InVT || TP.hasError()) - return false; +std::string TypeSetByHwMode::getAsString(const SetType &S) { + std::vector Types(S.begin(), S.end()); + array_pod_sort(Types.begin(), Types.end()); - if (isCompletelyUnknown()) { - *this = InVT; - return true; + std::stringstream str; + str << '['; + for (unsigned i = 0, e = Types.size(); i != e; ++i) { + str << ValueTypeByHwMode::getMVTName(Types[i]); + if (i != e-1) + str << ' '; } + str << ']'; + return str.str(); +} - assert(!TypeVec.empty() && !InVT.TypeVec.empty() && "No unknowns"); - - // Handle the abstract cases, seeing if we can resolve them better. - switch (TypeVec[0]) { - default: break; - case MVT::iPTR: - case MVT::iPTRAny: - if (InVT.hasIntegerTypes()) { - EEVT::TypeSet InCopy(InVT); - InCopy.EnforceInteger(TP); - InCopy.EnforceScalar(TP); +bool TypeSetByHwMode::operator==(const TypeSetByHwMode &VTS) const { + bool HaveDefault = hasDefault(); + if (HaveDefault != VTS.hasDefault()) + return false; - if (InCopy.isConcrete()) { - // If the RHS has one integer type, upgrade iPTR to i32. - TypeVec[0] = InVT.TypeVec[0]; - return true; - } + std::set Modes; + for (auto &I : *this) + Modes.insert(I.first); + for (const auto &I : VTS) + Modes.insert(I.first); - // If the input has multiple scalar integers, this doesn't add any info. - if (!InCopy.isCompletelyUnknown()) + if (HaveDefault) { + // Both sets have default mode. + for (unsigned M : Modes) { + if (get(M) != VTS.get(M)) return false; } - break; + } else { + // Neither set has default mode. + for (unsigned M : Modes) { + // If there is no default mode, an empty set is equivalent to not having + // the corresponding mode. + bool NoModeThis = !hasMode(M) || get(M).empty(); + bool NoModeVTS = !VTS.hasMode(M) || VTS.get(M).empty(); + if (NoModeThis != NoModeVTS) + return false; + if (!NoModeThis) + if (get(M) != VTS.get(M)) + return false; + } } - // If the input constraint is iAny/iPTR and this is an integer type list, - // remove non-integer types from the list. - if ((InVT.TypeVec[0] == MVT::iPTR || InVT.TypeVec[0] == MVT::iPTRAny) && - hasIntegerTypes()) { - bool MadeChange = EnforceInteger(TP); + return true; +} - // If we're merging in iPTR/iPTRAny and the node currently has a list of - // multiple different integer types, replace them with a single iPTR. - if ((InVT.TypeVec[0] == MVT::iPTR || InVT.TypeVec[0] == MVT::iPTRAny) && - TypeVec.size() != 1) { - TypeVec.assign(1, InVT.TypeVec[0]); - MadeChange = true; - } +LLVM_DUMP_METHOD +void TypeSetByHwMode::dump() const { + dbgs() << getAsString() << '\n'; +} - return MadeChange; +bool TypeSetByHwMode::intersect(SetType &Out, const SetType &In) { + bool OutP = Out.count(MVT::iPTR), InP = In.count(MVT::iPTR); + auto Int = [&In](MVT T) -> bool { return !In.count(T); }; + + if (OutP == InP) + return berase_if(Out, Int); + + // Compute the intersection of scalars separately to account for only + // one set containing iPTR. + // The itersection of iPTR with a set of integer scalar types that does not + // include iPTR will result in the most specific scalar type: + // - iPTR is more specific than any set with two elements or more + // - iPTR is less specific than any single integer scalar type. + // For example + // { iPTR } * { i32 } -> { i32 } + // { iPTR } * { i32 i64 } -> { iPTR } + + SetType Diff; + if (InP) { + std::copy_if(Out.begin(), Out.end(), std::inserter(Diff, Diff.end()), + [&In](MVT T) { return !In.count(T); }); + berase_if(Out, [&Diff](MVT T) { return Diff.count(T); }); + } else { + std::copy_if(In.begin(), In.end(), std::inserter(Diff, Diff.end()), + [&Out](MVT T) { return !Out.count(T); }); + Out.erase(MVT::iPTR); } - // If this is a type list and the RHS is a typelist as well, eliminate entries - // from this list that aren't in the other one. - TypeSet InputSet(*this); - - TypeVec.clear(); - std::set_intersection(InputSet.TypeVec.begin(), InputSet.TypeVec.end(), - InVT.TypeVec.begin(), InVT.TypeVec.end(), - std::back_inserter(TypeVec)); - - // If the intersection is the same size as the original set then we're done. - if (TypeVec.size() == InputSet.TypeVec.size()) - return false; + bool Changed = berase_if(Out, Int); + unsigned NumD = Diff.size(); + if (NumD == 0) + return Changed; - // If we removed all of our types, we have a type contradiction. - if (!TypeVec.empty()) - return true; + if (NumD == 1) { + Out.insert(*Diff.begin()); + // This is a change only if Out was the one with iPTR (which is now + // being replaced). + Changed |= OutP; + } else { + Out.insert(MVT::iPTR); + Changed |= InP; + } + return Changed; +} - // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, merging '" + - InVT.getName() + "' into '" + InputSet.getName() + "'"); - return false; +void TypeSetByHwMode::validate() const { +#ifndef NDEBUG + if (empty()) + return; + bool AllEmpty = true; + for (const auto &I : *this) + AllEmpty &= I.second.empty(); + assert(!AllEmpty && + "type set is empty for each hw mode: type contradiction?"); +#endif } -/// EnforceInteger - Remove all non-integer types from this set. -bool EEVT::TypeSet::EnforceInteger(TreePattern &TP) { - if (TP.hasError()) - return false; - // If we know nothing, then get the full set. - if (TypeVec.empty()) - return FillWithPossibleTypes(TP, isInteger, "integer"); +// --- TypeInfer - if (!hasFloatingPointTypes()) +bool TypeInfer::MergeInTypeInfo(TypeSetByHwMode &Out, + const TypeSetByHwMode &In) { + ValidateOnExit _1(Out); + In.validate(); + if (In.empty() || Out == In || TP.hasError()) return false; + if (Out.empty()) { + Out = In; + return true; + } - TypeSet InputSet(*this); - - // Filter out all the fp types. - TypeVec.erase(remove_if(TypeVec, std::not1(std::ptr_fun(isInteger))), - TypeVec.end()); + bool Changed = Out.constrain(In); + if (Changed && Out.empty()) + TP.error("Type contradiction"); - if (TypeVec.empty()) { - TP.error("Type inference contradiction found, '" + - InputSet.getName() + "' needs to be integer"); - return false; - } - return true; + return Changed; } -/// EnforceFloatingPoint - Remove all integer types from this set. -bool EEVT::TypeSet::EnforceFloatingPoint(TreePattern &TP) { +bool TypeInfer::forceArbitrary(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out); if (TP.hasError()) return false; - // If we know nothing, then get the full set. - if (TypeVec.empty()) - return FillWithPossibleTypes(TP, isFloatingPoint, "floating point"); + assert(!Out.empty() && "cannot pick from an empty set"); - if (!hasIntegerTypes()) - return false; - - TypeSet InputSet(*this); - - // Filter out all the integer types. - TypeVec.erase(remove_if(TypeVec, std::not1(std::ptr_fun(isFloatingPoint))), - TypeVec.end()); - - if (TypeVec.empty()) { - TP.error("Type inference contradiction found, '" + - InputSet.getName() + "' needs to be floating point"); - return false; + bool Changed = false; + for (auto &I : Out) { + TypeSetByHwMode::SetType &S = I.second; + if (S.size() <= 1) + continue; + MVT T = *S.begin(); // Pick the first element. + S.clear(); + S.insert(T); + Changed = true; } - return true; + return Changed; } -/// EnforceScalar - Remove all vector types from this. -bool EEVT::TypeSet::EnforceScalar(TreePattern &TP) { +bool TypeInfer::EnforceInteger(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out); if (TP.hasError()) return false; + if (!Out.empty()) + return Out.constrain(isIntegerOrPtr); - // If we know nothing, then get the full set. - if (TypeVec.empty()) - return FillWithPossibleTypes(TP, isScalar, "scalar"); + return Out.assign_if(getLegalTypes(), isIntegerOrPtr); +} - if (!hasVectorTypes()) +bool TypeInfer::EnforceFloatingPoint(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out); + if (TP.hasError()) return false; + if (!Out.empty()) + return Out.constrain(isFloatingPoint); - TypeSet InputSet(*this); - - // Filter out all the vector types. - TypeVec.erase(remove_if(TypeVec, std::not1(std::ptr_fun(isScalar))), - TypeVec.end()); + return Out.assign_if(getLegalTypes(), isFloatingPoint); +} - if (TypeVec.empty()) { - TP.error("Type inference contradiction found, '" + - InputSet.getName() + "' needs to be scalar"); +bool TypeInfer::EnforceScalar(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out); + if (TP.hasError()) return false; - } - return true; + if (!Out.empty()) + return Out.constrain(isScalar); + + return Out.assign_if(getLegalTypes(), isScalar); } -/// EnforceVector - Remove all vector types from this. -bool EEVT::TypeSet::EnforceVector(TreePattern &TP) { +bool TypeInfer::EnforceVector(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out); if (TP.hasError()) return false; + if (!Out.empty()) + return Out.constrain(isVector); - // If we know nothing, then get the full set. - if (TypeVec.empty()) - return FillWithPossibleTypes(TP, isVector, "vector"); + return Out.assign_if(getLegalTypes(), isVector); +} - TypeSet InputSet(*this); - bool MadeChange = false; +bool TypeInfer::EnforceAny(TypeSetByHwMode &Out) { + ValidateOnExit _1(Out); + if (TP.hasError() || !Out.empty()) + return false; - // Filter out all the scalar types. - TypeVec.erase(remove_if(TypeVec, std::not1(std::ptr_fun(isVector))), - TypeVec.end()); + Out = getLegalTypes(); + return true; +} - if (TypeVec.empty()) { - TP.error("Type inference contradiction found, '" + - InputSet.getName() + "' needs to be a vector"); - return false; +template +static Iter min_if(Iter B, Iter E, Pred P, Less L) { + if (B == E) + return E; + Iter Min = E; + for (Iter I = B; I != E; ++I) { + if (!P(*I)) + continue; + if (Min == E || L(*I, *Min)) + Min = I; } - return MadeChange; + return Min; } +template +static Iter max_if(Iter B, Iter E, Pred P, Less L) { + if (B == E) + return E; + Iter Max = E; + for (Iter I = B; I != E; ++I) { + if (!P(*I)) + continue; + if (Max == E || L(*Max, *I)) + Max = I; + } + return Max; +} - -/// EnforceSmallerThan - 'this' must be a smaller VT than Other. For vectors -/// this should be based on the element type. Update this and other based on -/// this information. -bool EEVT::TypeSet::EnforceSmallerThan(EEVT::TypeSet &Other, TreePattern &TP) { - if (TP.hasError()) - return false; - - // Both operands must be integer or FP, but we don't care which. - bool MadeChange = false; - - if (isCompletelyUnknown()) - MadeChange = FillWithPossibleTypes(TP); - - if (Other.isCompletelyUnknown()) - MadeChange = Other.FillWithPossibleTypes(TP); - - // If one side is known to be integer or known to be FP but the other side has - // no information, get at least the type integrality info in there. - if (!hasFloatingPointTypes()) - MadeChange |= Other.EnforceInteger(TP); - else if (!hasIntegerTypes()) - MadeChange |= Other.EnforceFloatingPoint(TP); - if (!Other.hasFloatingPointTypes()) - MadeChange |= EnforceInteger(TP); - else if (!Other.hasIntegerTypes()) - MadeChange |= EnforceFloatingPoint(TP); - - assert(!isCompletelyUnknown() && !Other.isCompletelyUnknown() && - "Should have a type list now"); - - // If one contains vectors but the other doesn't pull vectors out. - if (!hasVectorTypes()) - MadeChange |= Other.EnforceScalar(TP); - else if (!hasScalarTypes()) - MadeChange |= Other.EnforceVector(TP); - if (!Other.hasVectorTypes()) - MadeChange |= EnforceScalar(TP); - else if (!Other.hasScalarTypes()) - MadeChange |= EnforceVector(TP); - - // This code does not currently handle nodes which have multiple types, - // where some types are integer, and some are fp. Assert that this is not - // the case. - assert(!(hasIntegerTypes() && hasFloatingPointTypes()) && - !(Other.hasIntegerTypes() && Other.hasFloatingPointTypes()) && - "SDTCisOpSmallerThanOp does not handle mixed int/fp types!"); - +/// Make sure that for each type in Small, there exists a larger type in Big. +bool TypeInfer::EnforceSmallerThan(TypeSetByHwMode &Small, + TypeSetByHwMode &Big) { + ValidateOnExit _1(Small), _2(Big); if (TP.hasError()) return false; - - // Okay, find the smallest type from current set and remove anything the - // same or smaller from the other set. We need to ensure that the scalar - // type size is smaller than the scalar size of the smallest type. For - // vectors, we also need to make sure that the total size is no larger than - // the size of the smallest type. - { - TypeSet InputSet(Other); - MVT Smallest = *std::min_element(TypeVec.begin(), TypeVec.end(), - [](MVT A, MVT B) { - return A.getScalarSizeInBits() < B.getScalarSizeInBits() || - (A.getScalarSizeInBits() == B.getScalarSizeInBits() && - A.getSizeInBits() < B.getSizeInBits()); - }); - - auto I = remove_if(Other.TypeVec, [Smallest](MVT OtherVT) { - // Don't compare vector and non-vector types. - if (OtherVT.isVector() != Smallest.isVector()) - return false; - // The getSizeInBits() check here is only needed for vectors, but is - // a subset of the scalar check for scalars so no need to qualify. - return OtherVT.getScalarSizeInBits() <= Smallest.getScalarSizeInBits() || - OtherVT.getSizeInBits() < Smallest.getSizeInBits(); - }); - MadeChange |= I != Other.TypeVec.end(); // If we're about to remove types. - Other.TypeVec.erase(I, Other.TypeVec.end()); - - if (Other.TypeVec.empty()) { - TP.error("Type inference contradiction found, '" + InputSet.getName() + - "' has nothing larger than '" + getName() +"'!"); - return false; + bool Changed = false; + + if (Small.empty()) + Changed |= EnforceAny(Small); + if (Big.empty()) + Changed |= EnforceAny(Big); + + assert(Small.hasDefault() && Big.hasDefault()); + + std::vector Modes = union_modes(Small, Big); + + // 1. Only allow integer or floating point types and make sure that + // both sides are both integer or both floating point. + // 2. Make sure that either both sides have vector types, or neither + // of them does. + for (unsigned M : Modes) { + TypeSetByHwMode::SetType &S = Small.get(M); + TypeSetByHwMode::SetType &B = Big.get(M); + + if (any_of(S, isIntegerOrPtr) && any_of(S, isIntegerOrPtr)) { + auto NotInt = std::not1(std::ptr_fun(isIntegerOrPtr)); + Changed |= berase_if(S, NotInt) | + berase_if(B, NotInt); + } else if (any_of(S, isFloatingPoint) && any_of(B, isFloatingPoint)) { + auto NotFP = std::not1(std::ptr_fun(isFloatingPoint)); + Changed |= berase_if(S, NotFP) | + berase_if(B, NotFP); + } else if (S.empty() || B.empty()) { + Changed = !S.empty() || !B.empty(); + S.clear(); + B.clear(); + } else { + TP.error("Incompatible types"); + return Changed; } - } - // Okay, find the largest type from the other set and remove anything the - // same or smaller from the current set. We need to ensure that the scalar - // type size is larger than the scalar size of the largest type. For - // vectors, we also need to make sure that the total size is no smaller than - // the size of the largest type. - { - TypeSet InputSet(*this); - MVT Largest = *std::max_element(Other.TypeVec.begin(), Other.TypeVec.end(), - [](MVT A, MVT B) { - return A.getScalarSizeInBits() < B.getScalarSizeInBits() || - (A.getScalarSizeInBits() == B.getScalarSizeInBits() && - A.getSizeInBits() < B.getSizeInBits()); - }); - auto I = remove_if(TypeVec, [Largest](MVT OtherVT) { - // Don't compare vector and non-vector types. - if (OtherVT.isVector() != Largest.isVector()) - return false; - return OtherVT.getScalarSizeInBits() >= Largest.getScalarSizeInBits() || - OtherVT.getSizeInBits() > Largest.getSizeInBits(); - }); - MadeChange |= I != TypeVec.end(); // If we're about to remove types. - TypeVec.erase(I, TypeVec.end()); - - if (TypeVec.empty()) { - TP.error("Type inference contradiction found, '" + InputSet.getName() + - "' has nothing smaller than '" + Other.getName() +"'!"); - return false; + if (none_of(S, isVector) || none_of(B, isVector)) { + Changed |= berase_if(S, isVector) | + berase_if(B, isVector); } } - return MadeChange; -} - -/// EnforceVectorEltTypeIs - 'this' is now constrained to be a vector type -/// whose element is specified by VTOperand. -bool EEVT::TypeSet::EnforceVectorEltTypeIs(MVT::SimpleValueType VT, - TreePattern &TP) { - bool MadeChange = false; - - MadeChange |= EnforceVector(TP); - - TypeSet InputSet(*this); + auto LT = [](MVT A, MVT B) -> bool { + return A.getScalarSizeInBits() < B.getScalarSizeInBits() || + (A.getScalarSizeInBits() == B.getScalarSizeInBits() && + A.getSizeInBits() < B.getSizeInBits()); + }; + auto LE = [](MVT A, MVT B) -> bool { + // This function is used when removing elements: when a vector is compared + // to a non-vector, it should return false (to avoid removal). + if (A.isVector() != B.isVector()) + return false; - // Filter out all the types which don't have the right element type. - auto I = remove_if(TypeVec, [VT](MVT VVT) { - return VVT.getVectorElementType().SimpleTy != VT; - }); - MadeChange |= I != TypeVec.end(); - TypeVec.erase(I, TypeVec.end()); + // Note on the < comparison below: + // X86 has patterns like + // (set VR128X:$dst, (v16i8 (X86vtrunc (v4i32 VR128X:$src1)))), + // where the truncated vector is given a type v16i8, while the source + // vector has type v4i32. They both have the same size in bits. + // The minimal type in the result is obviously v16i8, and when we remove + // all types from the source that are smaller-or-equal than v8i16, the + // only source type would also be removed (since it's equal in size). + return A.getScalarSizeInBits() <= B.getScalarSizeInBits() || + A.getSizeInBits() < B.getSizeInBits(); + }; + + for (unsigned M : Modes) { + TypeSetByHwMode::SetType &S = Small.get(M); + TypeSetByHwMode::SetType &B = Big.get(M); + // MinS = min scalar in Small, remove all scalars from Big that are + // smaller-or-equal than MinS. + auto MinS = min_if(S.begin(), S.end(), isScalar, LT); + if (MinS != S.end()) { + Changed |= berase_if(B, std::bind(LE, std::placeholders::_1, *MinS)); + if (B.empty()) { + TP.error("Type contradiction in " + + Twine(__func__) + ":" + Twine(__LINE__)); + return Changed; + } + } + // MaxS = max scalar in Big, remove all scalars from Small that are + // larger than MaxS. + auto MaxS = max_if(B.begin(), B.end(), isScalar, LT); + if (MaxS != B.end()) { + Changed |= berase_if(S, std::bind(LE, *MaxS, std::placeholders::_1)); + if (B.empty()) { + TP.error("Type contradiction in " + + Twine(__func__) + ":" + Twine(__LINE__)); + return Changed; + } + } - if (TypeVec.empty()) { // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, forcing '" + - InputSet.getName() + "' to have a vector element of type " + - getEnumName(VT)); - return false; + // MinV = min vector in Small, remove all vectors from Big that are + // smaller-or-equal than MinV. + auto MinV = min_if(S.begin(), S.end(), isVector, LT); + if (MinV != S.end()) { + Changed |= berase_if(B, std::bind(LE, std::placeholders::_1, *MinV)); + if (B.empty()) { + TP.error("Type contradiction in " + + Twine(__func__) + ":" + Twine(__LINE__)); + return Changed; + } + } + // MaxV = max vector in Big, remove all vectors from Small that are + // larger than MaxV. + auto MaxV = max_if(B.begin(), B.end(), isVector, LT); + if (MaxV != B.end()) { + Changed |= berase_if(S, std::bind(LE, *MaxV, std::placeholders::_1)); + if (B.empty()) { + TP.error("Type contradiction in " + + Twine(__func__) + ":" + Twine(__LINE__)); + return Changed; + } + } } - return MadeChange; + return Changed; } -/// EnforceVectorEltTypeIs - 'this' is now constrained to be a vector type -/// whose element is specified by VTOperand. -bool EEVT::TypeSet::EnforceVectorEltTypeIs(EEVT::TypeSet &VTOperand, - TreePattern &TP) { +/// 1. Ensure that for each type T in Vec, T is a vector type, and that +/// for each type U in Elem, U is a scalar type. +/// 2. Ensure that for each (scalar) type U in Elem, there exists a (vector) +/// type T in Vec, such that U is the element type of T. +bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, + TypeSetByHwMode &Elem) { + ValidateOnExit _1(Vec), _2(Elem); if (TP.hasError()) return false; - - // "This" must be a vector and "VTOperand" must be a scalar. - bool MadeChange = false; - MadeChange |= EnforceVector(TP); - MadeChange |= VTOperand.EnforceScalar(TP); - - // If we know the vector type, it forces the scalar to agree. - if (isConcrete()) { - MVT IVT = getConcrete(); - IVT = IVT.getVectorElementType(); - return MadeChange || VTOperand.MergeInTypeInfo(IVT.SimpleTy, TP); + bool Changed = false; + + if (Vec.empty()) + Changed |= EnforceVector(Vec); + if (Elem.empty()) + Changed |= EnforceScalar(Elem); + + for (unsigned M : union_modes(Vec, Elem)) { + TypeSetByHwMode::SetType &V = Vec.get(M); + TypeSetByHwMode::SetType &E = Elem.get(M); + + Changed |= berase_if(V, isScalar); // Scalar = !vector + Changed |= berase_if(E, isVector); // Vector = !scalar + assert(!V.empty() && !E.empty()); + + SmallSet VT, ST; + // Collect element types from the "vector" set. + for (MVT T : V) + VT.insert(T.getVectorElementType()); + // Collect scalar types from the "element" set. + for (MVT T : E) + ST.insert(T); + + // Remove from V all (vector) types whose element type is not in S. + Changed |= berase_if(V, [&ST](MVT T) -> bool { + return !ST.count(T.getVectorElementType()); + }); + // Remove from E all (scalar) types, for which there is no corresponding + // type in V. + Changed |= berase_if(E, [&VT](MVT T) -> bool { return !VT.count(T); }); + + if (V.empty() || E.empty()) { + TP.error("Type contradiction in " + + Twine(__func__) + ":" + Twine(__LINE__)); + return Changed; + } } - // If the scalar type is known, filter out vector types whose element types - // disagree. - if (!VTOperand.isConcrete()) - return MadeChange; - - MVT::SimpleValueType VT = VTOperand.getConcrete(); - - MadeChange |= EnforceVectorEltTypeIs(VT, TP); + return Changed; +} - return MadeChange; +bool TypeInfer::EnforceVectorEltTypeIs(TypeSetByHwMode &Vec, + const ValueTypeByHwMode &VVT) { + TypeSetByHwMode Tmp(VVT); + ValidateOnExit _1(Vec), _2(Tmp); + return EnforceVectorEltTypeIs(Vec, Tmp); } -/// EnforceVectorSubVectorTypeIs - 'this' is now constrained to be a -/// vector type specified by VTOperand. -bool EEVT::TypeSet::EnforceVectorSubVectorTypeIs(EEVT::TypeSet &VTOperand, - TreePattern &TP) { +/// Ensure that for each type T in Sub, T is a vector type, and there +/// exists a type U in Vec such that U is a vector type with the same +/// element type as T and at least as many elements as T. +bool TypeInfer::EnforceVectorSubVectorTypeIs(TypeSetByHwMode &Vec, + TypeSetByHwMode &Sub) { + ValidateOnExit _1(Vec), _2(Sub); if (TP.hasError()) return false; - // "This" must be a vector and "VTOperand" must be a vector. - bool MadeChange = false; - MadeChange |= EnforceVector(TP); - MadeChange |= VTOperand.EnforceVector(TP); - - // If one side is known to be integer or known to be FP but the other side has - // no information, get at least the type integrality info in there. - if (!hasFloatingPointTypes()) - MadeChange |= VTOperand.EnforceInteger(TP); - else if (!hasIntegerTypes()) - MadeChange |= VTOperand.EnforceFloatingPoint(TP); - if (!VTOperand.hasFloatingPointTypes()) - MadeChange |= EnforceInteger(TP); - else if (!VTOperand.hasIntegerTypes()) - MadeChange |= EnforceFloatingPoint(TP); - - assert(!isCompletelyUnknown() && !VTOperand.isCompletelyUnknown() && - "Should have a type list now"); - - // If we know the vector type, it forces the scalar types to agree. - // Also force one vector to have more elements than the other. - if (isConcrete()) { - MVT IVT = getConcrete(); - unsigned NumElems = IVT.getVectorNumElements(); - IVT = IVT.getVectorElementType(); - - EEVT::TypeSet EltTypeSet(IVT.SimpleTy, TP); - MadeChange |= VTOperand.EnforceVectorEltTypeIs(EltTypeSet, TP); - - // Only keep types that have less elements than VTOperand. - TypeSet InputSet(VTOperand); - - auto I = remove_if(VTOperand.TypeVec, [NumElems](MVT VVT) { - return VVT.getVectorNumElements() >= NumElems; - }); - MadeChange |= I != VTOperand.TypeVec.end(); - VTOperand.TypeVec.erase(I, VTOperand.TypeVec.end()); - - if (VTOperand.TypeVec.empty()) { // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, forcing '" + - InputSet.getName() + "' to have less vector elements than '" + - getName() + "'"); + /// Return true if B is a suB-vector of P, i.e. P is a suPer-vector of B. + auto IsSubVec = [](MVT B, MVT P) -> bool { + if (!B.isVector() || !P.isVector()) return false; - } - } else if (VTOperand.isConcrete()) { - MVT IVT = VTOperand.getConcrete(); - unsigned NumElems = IVT.getVectorNumElements(); - IVT = IVT.getVectorElementType(); - - EEVT::TypeSet EltTypeSet(IVT.SimpleTy, TP); - MadeChange |= EnforceVectorEltTypeIs(EltTypeSet, TP); - - // Only keep types that have more elements than 'this'. - TypeSet InputSet(*this); - - auto I = remove_if(TypeVec, [NumElems](MVT VVT) { - return VVT.getVectorNumElements() <= NumElems; - }); - MadeChange |= I != TypeVec.end(); - TypeVec.erase(I, TypeVec.end()); - - if (TypeVec.empty()) { // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, forcing '" + - InputSet.getName() + "' to have more vector elements than '" + - VTOperand.getName() + "'"); + if (B.getVectorElementType() != P.getVectorElementType()) return false; - } - } + return B.getVectorNumElements() < P.getVectorNumElements(); + }; + + /// Return true if S has no element (vector type) that T is a sub-vector of, + /// i.e. has the same element type as T and more elements. + auto NoSubV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { + for (const auto &I : S) + if (IsSubVec(T, I)) + return false; + return true; + }; - return MadeChange; -} + /// Return true if S has no element (vector type) that T is a super-vector + /// of, i.e. has the same element type as T and fewer elements. + auto NoSupV = [&IsSubVec](const TypeSetByHwMode::SetType &S, MVT T) -> bool { + for (const auto &I : S) + if (IsSubVec(I, T)) + return false; + return true; + }; -/// EnforceameNumElts - If VTOperand is a scalar, then 'this' is a scalar. If -/// VTOperand is a vector, then 'this' must have the same number of elements. -bool EEVT::TypeSet::EnforceSameNumElts(EEVT::TypeSet &VTOperand, - TreePattern &TP) { - if (TP.hasError()) - return false; + bool Changed = false; - bool MadeChange = false; + if (Vec.empty()) + Changed |= EnforceVector(Vec); + if (Sub.empty()) + Changed |= EnforceVector(Sub); - if (isCompletelyUnknown()) - MadeChange = FillWithPossibleTypes(TP); - - if (VTOperand.isCompletelyUnknown()) - MadeChange = VTOperand.FillWithPossibleTypes(TP); - - // If one contains vectors but the other doesn't pull vectors out. - if (!hasVectorTypes()) - MadeChange |= VTOperand.EnforceScalar(TP); - else if (!hasScalarTypes()) - MadeChange |= VTOperand.EnforceVector(TP); - if (!VTOperand.hasVectorTypes()) - MadeChange |= EnforceScalar(TP); - else if (!VTOperand.hasScalarTypes()) - MadeChange |= EnforceVector(TP); - - // If one type is a vector, make sure the other has the same element count. - // If this a scalar, then we are already done with the above. - if (isConcrete()) { - MVT IVT = getConcrete(); - if (IVT.isVector()) { - unsigned NumElems = IVT.getVectorNumElements(); - - // Only keep types that have same elements as 'this'. - TypeSet InputSet(VTOperand); - - auto I = remove_if(VTOperand.TypeVec, [NumElems](MVT VVT) { - return VVT.getVectorNumElements() != NumElems; - }); - MadeChange |= I != VTOperand.TypeVec.end(); - VTOperand.TypeVec.erase(I, VTOperand.TypeVec.end()); - - if (VTOperand.TypeVec.empty()) { // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, forcing '" + - InputSet.getName() + "' to have same number elements as '" + - getName() + "'"); - return false; - } - } - } else if (VTOperand.isConcrete()) { - MVT IVT = VTOperand.getConcrete(); - if (IVT.isVector()) { - unsigned NumElems = IVT.getVectorNumElements(); + for (unsigned M : union_modes(Vec, Sub)) { + TypeSetByHwMode::SetType &S = Sub.get(M); + TypeSetByHwMode::SetType &V = Vec.get(M); - // Only keep types that have same elements as VTOperand. - TypeSet InputSet(*this); + Changed |= berase_if(S, isScalar); + if (S.empty()) { + TP.error("Type contradiction in " + + Twine(__func__) + ":" + Twine(__LINE__)); + return Changed; + } - auto I = remove_if(TypeVec, [NumElems](MVT VVT) { - return VVT.getVectorNumElements() != NumElems; - }); - MadeChange |= I != TypeVec.end(); - TypeVec.erase(I, TypeVec.end()); + // Erase all types from S that are not sub-vectors of a type in V. + Changed |= berase_if(S, std::bind(NoSubV, V, std::placeholders::_1)); + if (S.empty()) { + TP.error("Type contradiction in " + + Twine(__func__) + ":" + Twine(__LINE__)); + return Changed; + } - if (TypeVec.empty()) { // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, forcing '" + - InputSet.getName() + "' to have same number elements than '" + - VTOperand.getName() + "'"); - return false; - } + // Erase all types from V that are not super-vectors of a type in S. + Changed |= berase_if(V, std::bind(NoSupV, S, std::placeholders::_1)); + if (V.empty()) { + TP.error("Type contradiction in " + + Twine(__func__) + ":" + Twine(__LINE__)); + return Changed; } } - return MadeChange; + return Changed; } -/// EnforceSameSize - 'this' is now constrained to be same size as VTOperand. -bool EEVT::TypeSet::EnforceSameSize(EEVT::TypeSet &VTOperand, - TreePattern &TP) { +/// 1. Ensure that V has a scalar type iff W has a scalar type. +/// 2. Ensure that for each vector type T in V, there exists a vector +/// type U in W, such that T and U have the same number of elements. +/// 3. Ensure that for each vector type U in W, there exists a vector +/// type T in V, such that T and U have the same number of elements +/// (reverse of 2). +bool TypeInfer::EnforceSameNumElts(TypeSetByHwMode &V, TypeSetByHwMode &W) { + ValidateOnExit _1(V), _2(W); if (TP.hasError()) return false; - bool MadeChange = false; - - if (isCompletelyUnknown()) - MadeChange = FillWithPossibleTypes(TP); - - if (VTOperand.isCompletelyUnknown()) - MadeChange = VTOperand.FillWithPossibleTypes(TP); - - // If we know one of the types, it forces the other type agree. - if (isConcrete()) { - MVT IVT = getConcrete(); - unsigned Size = IVT.getSizeInBits(); - - // Only keep types that have the same size as 'this'. - TypeSet InputSet(VTOperand); + bool Changed = false; + if (V.empty()) + Changed |= EnforceAny(V); + if (W.empty()) + Changed |= EnforceAny(W); + + // An actual vector type cannot have 0 elements, so we can treat scalars + // as zero-length vectors. This way both vectors and scalars can be + // processed identically. + auto NoLength = [](const SmallSet &Lengths, MVT T) -> bool { + return !Lengths.count(T.isVector() ? T.getVectorNumElements() : 0); + }; + + for (unsigned M : union_modes(V, W)) { + TypeSetByHwMode::SetType &VS = V.get(M); + TypeSetByHwMode::SetType &WS = W.get(M); + + SmallSet VN, WN; + for (MVT T : VS) + VN.insert(T.isVector() ? T.getVectorNumElements() : 0); + for (MVT T : WS) + WN.insert(T.isVector() ? T.getVectorNumElements() : 0); + + Changed |= berase_if(VS, std::bind(NoLength, WN, std::placeholders::_1)); + Changed |= berase_if(WS, std::bind(NoLength, VN, std::placeholders::_1)); + } + return Changed; +} - auto I = remove_if(VTOperand.TypeVec, - [&](MVT VT) { return VT.getSizeInBits() != Size; }); - MadeChange |= I != VTOperand.TypeVec.end(); - VTOperand.TypeVec.erase(I, VTOperand.TypeVec.end()); +/// 1. Ensure that for each type T in "this", there exists a type U in VTS, +/// such that T and U have equal size in bits. +/// 2. Ensure that for each type U in VTS, there exists a type T in "this" +/// such that T and U have equal size in bits (reverse of 1). +bool TypeInfer::EnforceSameSize(TypeSetByHwMode &A, TypeSetByHwMode &B) { + ValidateOnExit _1(A), _2(B); + if (TP.hasError()) + return false; + bool Changed = false; + if (A.empty()) + Changed |= EnforceAny(A); + if (B.empty()) + Changed |= EnforceAny(B); - if (VTOperand.TypeVec.empty()) { // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, forcing '" + - InputSet.getName() + "' to have same size as '" + - getName() + "'"); - return false; - } - } else if (VTOperand.isConcrete()) { - MVT IVT = VTOperand.getConcrete(); - unsigned Size = IVT.getSizeInBits(); + auto NoSize = [](const SmallSet &Sizes, MVT T) -> bool { + return !Sizes.count(T.getSizeInBits()); + }; - // Only keep types that have the same size as VTOperand. - TypeSet InputSet(*this); + for (unsigned M : union_modes(A, B)) { + TypeSetByHwMode::SetType &AS = A.get(M); + TypeSetByHwMode::SetType &BS = B.get(M); + SmallSet AN, BN; - auto I = - remove_if(TypeVec, [&](MVT VT) { return VT.getSizeInBits() != Size; }); - MadeChange |= I != TypeVec.end(); - TypeVec.erase(I, TypeVec.end()); + for (MVT T : AS) + AN.insert(T.getSizeInBits()); + for (MVT T : BS) + BN.insert(T.getSizeInBits()); - if (TypeVec.empty()) { // FIXME: Really want an SMLoc here! - TP.error("Type inference contradiction found, forcing '" + - InputSet.getName() + "' to have same size as '" + - VTOperand.getName() + "'"); - return false; - } + Changed |= berase_if(AS, std::bind(NoSize, BN, std::placeholders::_1)); + Changed |= berase_if(BS, std::bind(NoSize, AN, std::placeholders::_1)); } - return MadeChange; + return Changed; } -//===----------------------------------------------------------------------===// -// Helpers for working with extended types. +void TypeInfer::expandOverloads(TypeSetByHwMode &VTS) { + ValidateOnExit _1(VTS); + TypeSetByHwMode Legal = getLegalTypes(); + bool HaveLegalDef = Legal.hasDefault(); -/// Dependent variable map for CodeGenDAGPattern variant generation -typedef std::map DepVarMap; - -static void FindDepVarsOf(TreePatternNode *N, DepVarMap &DepMap) { - if (N->isLeaf()) { - if (isa(N->getLeafValue())) - DepMap[N->getName()]++; - } else { - for (size_t i = 0, e = N->getNumChildren(); i != e; ++i) - FindDepVarsOf(N->getChild(i), DepMap); + for (auto &I : VTS) { + unsigned M = I.first; + if (!Legal.hasMode(M) && !HaveLegalDef) { + TP.error("Invalid mode " + Twine(M)); + return; + } + expandOverloads(I.second, Legal.get(M)); } } - -/// Find dependent variables within child patterns -static void FindDepVars(TreePatternNode *N, MultipleUseVarSet &DepVars) { - DepVarMap depcounts; - FindDepVarsOf(N, depcounts); - for (const std::pair &Pair : depcounts) { - if (Pair.second > 1) - DepVars.insert(Pair.first); + +void TypeInfer::expandOverloads(TypeSetByHwMode::SetType &Out, + const TypeSetByHwMode::SetType &Legal) { + std::set Ovs; + for (auto I = Out.begin(); I != Out.end(); ) { + if (I->isOverloaded()) { + Ovs.insert(*I); + I = Out.erase(I); + continue; + } + ++I; + } + + for (MVT Ov : Ovs) { + switch (Ov.SimpleTy) { + case MVT::iPTRAny: + Out.insert(MVT::iPTR); + return; + case MVT::iAny: + for (MVT T : MVT::integer_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + for (MVT T : MVT::integer_vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + return; + case MVT::fAny: + for (MVT T : MVT::fp_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + for (MVT T : MVT::fp_vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + return; + case MVT::vAny: + for (MVT T : MVT::vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + return; + case MVT::Any: + for (MVT T : MVT::all_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + return; + default: + break; + } } } -#ifndef NDEBUG -/// Dump the dependent variable set: -static void DumpDepVars(MultipleUseVarSet &DepVars) { - if (DepVars.empty()) { - DEBUG(errs() << ""); + +TypeSetByHwMode TypeInfer::getLegalTypes() { + TypeSetByHwMode VTS; + TypeSetByHwMode::SetType &DS = VTS.getOrCreate(DefaultMode); + const TypeSetByHwMode <S = TP.getDAGPatterns().getLegalTypes(); + + if (!CodeGen) { + assert(LTS.hasDefault()); + const TypeSetByHwMode::SetType &S = LTS.get(DefaultMode); + DS.insert(S.begin(), S.end()); } else { - DEBUG(errs() << "[ "); - for (const std::string &DepVar : DepVars) { - DEBUG(errs() << DepVar << " "); - } - DEBUG(errs() << "]"); + for (const auto &I : LTS) + DS.insert(I.second.begin(), I.second.end()); } + return VTS; } -#endif - //===----------------------------------------------------------------------===// // TreePredicateFn Implementation @@ -817,7 +886,6 @@ // PatternToMatch implementation // - /// getPatternSize - Return the 'size' of this pattern. We want to match large /// patterns before small ones. This is used to determine the size of a /// pattern. @@ -845,10 +913,16 @@ // Count children in the count if they are also nodes. for (unsigned i = 0, e = P->getNumChildren(); i != e; ++i) { TreePatternNode *Child = P->getChild(i); - if (!Child->isLeaf() && Child->getNumTypes() && - Child->getType(0) != MVT::Other) - Size += getPatternSize(Child, CGP); - else if (Child->isLeaf()) { + if (!Child->isLeaf() && Child->getNumTypes()) { + const TypeSetByHwMode &T0 = Child->getType(0); + // At this point, all variable type sets should be simple, i.e. only + // have a default mode. + if (T0.getMachineValueType() != MVT::Other) { + Size += getPatternSize(Child, CGP); + continue; + } + } + if (Child->isLeaf()) { if (isa(Child->getLeafValue())) Size += 5; // Matches a ConstantSDNode (+3) and a specific value (+2). else if (Child->getComplexPatternInfo(CGP)) @@ -868,52 +942,37 @@ return getPatternSize(getSrcPattern(), CGP) + getAddedComplexity(); } - /// getPredicateCheck - Return a single string containing all of this /// pattern's predicates concatenated with "&&" operators. /// std::string PatternToMatch::getPredicateCheck() const { - SmallVector PredicateRecs; - for (Init *I : Predicates->getValues()) { - if (DefInit *Pred = dyn_cast(I)) { - Record *Def = Pred->getDef(); - if (!Def->isSubClassOf("Predicate")) { -#ifndef NDEBUG - Def->dump(); -#endif - llvm_unreachable("Unknown predicate type!"); - } - PredicateRecs.push_back(Def); - } - } - // Sort so that different orders get canonicalized to the same string. - std::sort(PredicateRecs.begin(), PredicateRecs.end(), LessRecord()); - - SmallString<128> PredicateCheck; - for (Record *Pred : PredicateRecs) { - if (!PredicateCheck.empty()) - PredicateCheck += " && "; - PredicateCheck += "("; - PredicateCheck += Pred->getValueAsString("CondString"); - PredicateCheck += ")"; - } - - return PredicateCheck.str(); + SmallVector PredList; + for (const Predicate &P : Predicates) + PredList.push_back(&P); + std::sort(PredList.begin(), PredList.end(), deref()); + + std::string Check; + for (unsigned i = 0, e = PredList.size(); i != e; ++i) { + if (i != 0) + Check += " && "; + Check += '(' + PredList[i]->getCondString() + ')'; + } + return Check; } //===----------------------------------------------------------------------===// // SDTypeConstraint implementation // -SDTypeConstraint::SDTypeConstraint(Record *R) { +SDTypeConstraint::SDTypeConstraint(Record *R, const CodeGenHwModes &CGH) { OperandNo = R->getValueAsInt("OperandNum"); if (R->isSubClassOf("SDTCisVT")) { ConstraintType = SDTCisVT; - x.SDTCisVT_Info.VT = getValueType(R->getValueAsDef("VT")); - if (x.SDTCisVT_Info.VT == MVT::isVoid) - PrintFatalError(R->getLoc(), "Cannot use 'Void' as type to SDTCisVT"); - + VVT = getValueTypeByHwMode(R->getValueAsDef("VT"), CGH); + for (const auto &P : VVT) + if (P.second == MVT::isVoid) + PrintFatalError(R->getLoc(), "Cannot use 'Void' as type to SDTCisVT"); } else if (R->isSubClassOf("SDTCisPtrTy")) { ConstraintType = SDTCisPtrTy; } else if (R->isSubClassOf("SDTCisInt")) { @@ -942,13 +1001,16 @@ R->getValueAsInt("OtherOpNum"); } else if (R->isSubClassOf("SDTCVecEltisVT")) { ConstraintType = SDTCVecEltisVT; - x.SDTCVecEltisVT_Info.VT = getValueType(R->getValueAsDef("VT")); - if (MVT(x.SDTCVecEltisVT_Info.VT).isVector()) - PrintFatalError(R->getLoc(), "Cannot use vector type as SDTCVecEltisVT"); - if (!MVT(x.SDTCVecEltisVT_Info.VT).isInteger() && - !MVT(x.SDTCVecEltisVT_Info.VT).isFloatingPoint()) - PrintFatalError(R->getLoc(), "Must use integer or floating point type " - "as SDTCVecEltisVT"); + VVT = getValueTypeByHwMode(R->getValueAsDef("VT"), CGH); + for (const auto &P : VVT) { + MVT T = P.second; + if (T.isVector()) + PrintFatalError(R->getLoc(), + "Cannot use vector type as SDTCVecEltisVT"); + if (!T.isInteger() && !T.isFloatingPoint()) + PrintFatalError(R->getLoc(), "Must use integer or floating point type " + "as SDTCVecEltisVT"); + } } else if (R->isSubClassOf("SDTCisSameNumEltsAs")) { ConstraintType = SDTCisSameNumEltsAs; x.SDTCisSameNumEltsAs_Info.OtherOperandNum = @@ -998,23 +1060,24 @@ unsigned ResNo = 0; // The result number being referenced. TreePatternNode *NodeToApply = getOperandNum(OperandNo, N, NodeInfo, ResNo); + TypeInfer &TI = TP.getInfer(); switch (ConstraintType) { case SDTCisVT: // Operand must be a particular type. - return NodeToApply->UpdateNodeType(ResNo, x.SDTCisVT_Info.VT, TP); + return NodeToApply->UpdateNodeType(ResNo, VVT, TP); case SDTCisPtrTy: // Operand must be same as target pointer type. return NodeToApply->UpdateNodeType(ResNo, MVT::iPTR, TP); case SDTCisInt: // Require it to be one of the legal integer VTs. - return NodeToApply->getExtType(ResNo).EnforceInteger(TP); + return TI.EnforceInteger(NodeToApply->getExtType(ResNo)); case SDTCisFP: // Require it to be one of the legal fp VTs. - return NodeToApply->getExtType(ResNo).EnforceFloatingPoint(TP); + return TI.EnforceFloatingPoint(NodeToApply->getExtType(ResNo)); case SDTCisVec: // Require it to be one of the legal vector VTs. - return NodeToApply->getExtType(ResNo).EnforceVector(TP); + return TI.EnforceVector(NodeToApply->getExtType(ResNo)); case SDTCisSameAs: { unsigned OResNo = 0; TreePatternNode *OtherNode = @@ -1032,36 +1095,35 @@ TP.error(N->getOperator()->getName() + " expects a VT operand!"); return false; } - MVT::SimpleValueType VT = - getValueType(static_cast(NodeToApply->getLeafValue())->getDef()); - - EEVT::TypeSet TypeListTmp(VT, TP); + DefInit *DI = static_cast(NodeToApply->getLeafValue()); + const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); + auto VVT = getValueTypeByHwMode(DI->getDef(), T.getHwModes()); + TypeSetByHwMode TypeListTmp(VVT); unsigned OResNo = 0; TreePatternNode *OtherNode = getOperandNum(x.SDTCisVTSmallerThanOp_Info.OtherOperandNum, N, NodeInfo, OResNo); - return TypeListTmp.EnforceSmallerThan(OtherNode->getExtType(OResNo), TP); + return TI.EnforceSmallerThan(TypeListTmp, OtherNode->getExtType(OResNo)); } case SDTCisOpSmallerThanOp: { unsigned BResNo = 0; TreePatternNode *BigOperand = getOperandNum(x.SDTCisOpSmallerThanOp_Info.BigOperandNum, N, NodeInfo, BResNo); - return NodeToApply->getExtType(ResNo). - EnforceSmallerThan(BigOperand->getExtType(BResNo), TP); + return TI.EnforceSmallerThan(NodeToApply->getExtType(ResNo), + BigOperand->getExtType(BResNo)); } case SDTCisEltOfVec: { unsigned VResNo = 0; TreePatternNode *VecOperand = getOperandNum(x.SDTCisEltOfVec_Info.OtherOperandNum, N, NodeInfo, VResNo); - // Filter vector types out of VecOperand that don't have the right element // type. - return VecOperand->getExtType(VResNo). - EnforceVectorEltTypeIs(NodeToApply->getExtType(ResNo), TP); + return TI.EnforceVectorEltTypeIs(VecOperand->getExtType(VResNo), + NodeToApply->getExtType(ResNo)); } case SDTCisSubVecOfVec: { unsigned VResNo = 0; @@ -1071,28 +1133,27 @@ // Filter vector types out of BigVecOperand that don't have the // right subvector type. - return BigVecOperand->getExtType(VResNo). - EnforceVectorSubVectorTypeIs(NodeToApply->getExtType(ResNo), TP); + return TI.EnforceVectorSubVectorTypeIs(BigVecOperand->getExtType(VResNo), + NodeToApply->getExtType(ResNo)); } case SDTCVecEltisVT: { - return NodeToApply->getExtType(ResNo). - EnforceVectorEltTypeIs(x.SDTCVecEltisVT_Info.VT, TP); + return TI.EnforceVectorEltTypeIs(NodeToApply->getExtType(ResNo), VVT); } case SDTCisSameNumEltsAs: { unsigned OResNo = 0; TreePatternNode *OtherNode = getOperandNum(x.SDTCisSameNumEltsAs_Info.OtherOperandNum, N, NodeInfo, OResNo); - return OtherNode->getExtType(OResNo). - EnforceSameNumElts(NodeToApply->getExtType(ResNo), TP); + return TI.EnforceSameNumElts(OtherNode->getExtType(OResNo), + NodeToApply->getExtType(ResNo)); } case SDTCisSameSizeAs: { unsigned OResNo = 0; TreePatternNode *OtherNode = getOperandNum(x.SDTCisSameSizeAs_Info.OtherOperandNum, N, NodeInfo, OResNo); - return OtherNode->getExtType(OResNo). - EnforceSameSize(NodeToApply->getExtType(ResNo), TP); + return TI.EnforceSameSize(OtherNode->getExtType(OResNo), + NodeToApply->getExtType(ResNo)); } } llvm_unreachable("Invalid ConstraintType!"); @@ -1110,9 +1171,11 @@ return false; // The Operand class specifies a type directly. - if (Operand->isSubClassOf("Operand")) - return UpdateNodeType(ResNo, getValueType(Operand->getValueAsDef("Type")), - TP); + if (Operand->isSubClassOf("Operand")) { + Record *R = Operand->getValueAsDef("Type"); + const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); + return UpdateNodeType(ResNo, getValueTypeByHwMode(R, T.getHwModes()), TP); + } // PointerLikeRegClass has a type that is determined at runtime. if (Operand->isSubClassOf("PointerLikeRegClass")) @@ -1131,11 +1194,53 @@ return UpdateNodeType(ResNo, Tgt.getRegisterClass(RC).getValueTypes(), TP); } +bool TreePatternNode::ContainsUnresolvedType(TreePattern &TP) const { + for (unsigned i = 0, e = Types.size(); i != e; ++i) + if (!TP.getInfer().isConcrete(Types[i], true)) + return true; + for (unsigned i = 0, e = getNumChildren(); i != e; ++i) + if (getChild(i)->ContainsUnresolvedType(TP)) + return true; + return false; +} + +bool TreePatternNode::hasProperTypeByHwMode() const { + for (const TypeSetByHwMode &S : Types) + if (!S.isDefaultOnly()) + return true; + for (TreePatternNode *C : Children) + if (C->hasProperTypeByHwMode()) + return true; + return false; +} + +bool TreePatternNode::hasPossibleType() const { + for (const TypeSetByHwMode &S : Types) + if (!S.isPossible()) + return false; + for (TreePatternNode *C : Children) + if (!C->hasPossibleType()) + return false; + return true; +} + +bool TreePatternNode::setDefaultMode(unsigned Mode) { + for (TypeSetByHwMode &S : Types) { + S.makeSimple(Mode); + // Check if the selected mode had a type conflict. + if (S.get(DefaultMode).empty()) + return false; + } + for (TreePatternNode *C : Children) + if (!C->setDefaultMode(Mode)) + return false; + return true; +} //===----------------------------------------------------------------------===// // SDNodeInfo implementation // -SDNodeInfo::SDNodeInfo(Record *R) : Def(R) { +SDNodeInfo::SDNodeInfo(Record *R, const CodeGenHwModes &CGH) : Def(R) { EnumName = R->getValueAsString("Opcode"); SDClassName = R->getValueAsString("SDClass"); Record *TypeProfile = R->getValueAsDef("TypeProfile"); @@ -1178,7 +1283,8 @@ // Parse the type constraints. std::vector ConstraintList = TypeProfile->getValueAsListOfDefs("Constraints"); - TypeConstraints.assign(ConstraintList.begin(), ConstraintList.end()); + for (Record *R : ConstraintList) + TypeConstraints.emplace_back(R, CGH); } /// getKnownType - If the type constraints on this node imply a fixed type @@ -1198,7 +1304,9 @@ switch (Constraint.ConstraintType) { default: break; case SDTypeConstraint::SDTCisVT: - return Constraint.x.SDTCisVT_Info.VT; + if (Constraint.VVT.isSimple()) + return Constraint.VVT.getSimple().SimpleTy; + break; case SDTypeConstraint::SDTCisPtrTy: return MVT::iPTR; } @@ -1285,7 +1393,7 @@ OS << '(' << getOperator()->getName(); for (unsigned i = 0, e = Types.size(); i != e; ++i) - OS << ':' << getExtType(i).getName(); + OS << ':' << getExtType(i).getAsString(); if (!isLeaf()) { if (getNumChildren() != 0) { @@ -1368,7 +1476,7 @@ /// RemoveAllTypes - Recursively strip all the types of this tree. void TreePatternNode::RemoveAllTypes() { // Reset to unknown type. - std::fill(Types.begin(), Types.end(), EEVT::TypeSet()); + std::fill(Types.begin(), Types.end(), TypeSetByHwMode()); if (isLeaf()) return; for (unsigned i = 0, e = getNumChildren(); i != e; ++i) getChild(i)->RemoveAllTypes(); @@ -1485,18 +1593,20 @@ /// When Unnamed is false, return the type of a named DAG operand such as the /// GPR:$src operand above. /// -static EEVT::TypeSet getImplicitType(Record *R, unsigned ResNo, - bool NotRegisters, - bool Unnamed, - TreePattern &TP) { +static TypeSetByHwMode getImplicitType(Record *R, unsigned ResNo, + bool NotRegisters, + bool Unnamed, + TreePattern &TP) { + CodeGenDAGPatterns &CDP = TP.getDAGPatterns(); + // Check to see if this is a register operand. if (R->isSubClassOf("RegisterOperand")) { assert(ResNo == 0 && "Regoperand ref only has one result!"); if (NotRegisters) - return EEVT::TypeSet(); // Unknown. + return TypeSetByHwMode(); // Unknown. Record *RegClass = R->getValueAsDef("RegClass"); const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); - return EEVT::TypeSet(T.getRegisterClass(RegClass).getValueTypes()); + return TypeSetByHwMode(T.getRegisterClass(RegClass).getValueTypes()); } // Check to see if this is a register or a register class. @@ -1505,33 +1615,33 @@ // An unnamed register class represents itself as an i32 immediate, for // example on a COPY_TO_REGCLASS instruction. if (Unnamed) - return EEVT::TypeSet(MVT::i32, TP); + return TypeSetByHwMode(MVT::i32); // In a named operand, the register class provides the possible set of // types. if (NotRegisters) - return EEVT::TypeSet(); // Unknown. + return TypeSetByHwMode(); // Unknown. const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); - return EEVT::TypeSet(T.getRegisterClass(R).getValueTypes()); + return TypeSetByHwMode(T.getRegisterClass(R).getValueTypes()); } if (R->isSubClassOf("PatFrag")) { assert(ResNo == 0 && "FIXME: PatFrag with multiple results?"); // Pattern fragment types will be resolved when they are inlined. - return EEVT::TypeSet(); // Unknown. + return TypeSetByHwMode(); // Unknown. } if (R->isSubClassOf("Register")) { assert(ResNo == 0 && "Registers only produce one result!"); if (NotRegisters) - return EEVT::TypeSet(); // Unknown. + return TypeSetByHwMode(); // Unknown. const CodeGenTarget &T = TP.getDAGPatterns().getTargetInfo(); - return EEVT::TypeSet(T.getRegisterVTs(R)); + return TypeSetByHwMode(T.getRegisterVTs(R)); } if (R->isSubClassOf("SubRegIndex")) { assert(ResNo == 0 && "SubRegisterIndices only produce one result!"); - return EEVT::TypeSet(MVT::i32, TP); + return TypeSetByHwMode(MVT::i32); } if (R->isSubClassOf("ValueType")) { @@ -1541,46 +1651,51 @@ // (sext_inreg GPR:$src, i16) // ~~~ if (Unnamed) - return EEVT::TypeSet(MVT::Other, TP); + return TypeSetByHwMode(MVT::Other); // With a name, the ValueType simply provides the type of the named // variable. // // (sext_inreg i32:$src, i16) // ~~~~~~~~ if (NotRegisters) - return EEVT::TypeSet(); // Unknown. - return EEVT::TypeSet(getValueType(R), TP); + return TypeSetByHwMode(); // Unknown. + const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); + return TypeSetByHwMode(getValueTypeByHwMode(R, CGH)); } if (R->isSubClassOf("CondCode")) { assert(ResNo == 0 && "This node only has one result!"); // Using a CondCodeSDNode. - return EEVT::TypeSet(MVT::Other, TP); + return TypeSetByHwMode(MVT::Other); } if (R->isSubClassOf("ComplexPattern")) { assert(ResNo == 0 && "FIXME: ComplexPattern with multiple results?"); if (NotRegisters) - return EEVT::TypeSet(); // Unknown. - return EEVT::TypeSet(TP.getDAGPatterns().getComplexPattern(R).getValueType(), - TP); + return TypeSetByHwMode(); // Unknown. + return TypeSetByHwMode(CDP.getComplexPattern(R).getValueType()); } if (R->isSubClassOf("PointerLikeRegClass")) { assert(ResNo == 0 && "Regclass can only have one result!"); - return EEVT::TypeSet(MVT::iPTR, TP); + TypeSetByHwMode VTS(MVT::iPTR); + TP.getInfer().expandOverloads(VTS); + return VTS; } if (R->getName() == "node" || R->getName() == "srcvalue" || R->getName() == "zero_reg") { // Placeholder. - return EEVT::TypeSet(); // Unknown. + return TypeSetByHwMode(); // Unknown. } - if (R->isSubClassOf("Operand")) - return EEVT::TypeSet(getValueType(R->getValueAsDef("Type"))); + if (R->isSubClassOf("Operand")) { + const CodeGenHwModes &CGH = CDP.getTargetInfo().getHwModes(); + Record *T = R->getValueAsDef("Type"); + return TypeSetByHwMode(getValueTypeByHwMode(T, CGH)); + } TP.error("Unknown node flavor used in pattern: " + R->getName()); - return EEVT::TypeSet(MVT::Other, TP); + return TypeSetByHwMode(MVT::Other); } @@ -1722,29 +1837,34 @@ assert(Types.size() == 1 && "Invalid IntInit"); // Int inits are always integers. :) - bool MadeChange = Types[0].EnforceInteger(TP); + bool MadeChange = TP.getInfer().EnforceInteger(Types[0]); - if (!Types[0].isConcrete()) + if (!TP.getInfer().isConcrete(Types[0], false)) return MadeChange; - MVT::SimpleValueType VT = getType(0); - if (VT == MVT::iPTR || VT == MVT::iPTRAny) - return MadeChange; - - unsigned Size = MVT(VT).getSizeInBits(); - // Make sure that the value is representable for this type. - if (Size >= 32) return MadeChange; - - // Check that the value doesn't use more bits than we have. It must either - // be a sign- or zero-extended equivalent of the original. - int64_t SignBitAndAbove = II->getValue() >> (Size - 1); - if (SignBitAndAbove == -1 || SignBitAndAbove == 0 || SignBitAndAbove == 1) - return MadeChange; + ValueTypeByHwMode VVT = TP.getInfer().getConcrete(Types[0], false); + for (auto &P : VVT) { + MVT::SimpleValueType VT = P.second.SimpleTy; + if (VT == MVT::iPTR || VT == MVT::iPTRAny) + continue; + unsigned Size = MVT(VT).getSizeInBits(); + // Make sure that the value is representable for this type. + if (Size >= 32) + continue; + // Check that the value doesn't use more bits than we have. It must + // either be a sign- or zero-extended equivalent of the original. + int64_t SignBitAndAbove = II->getValue() >> (Size - 1); + if (SignBitAndAbove == -1 || SignBitAndAbove == 0 || + SignBitAndAbove == 1) + continue; - TP.error("Integer value '" + itostr(II->getValue()) + - "' is out of range for type '" + getEnumName(getType(0)) + "'!"); - return false; + TP.error("Integer value '" + itostr(II->getValue()) + + "' is out of range for type '" + getEnumName(VT) + "'!"); + break; + } + return MadeChange; } + return false; } @@ -1773,7 +1893,7 @@ bool MadeChange = false; for (unsigned i = 0; i < getNumChildren(); ++i) - MadeChange = getChild(i)->ApplyTypeConstraints(TP, NotRegisters); + MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); return MadeChange; } @@ -1810,7 +1930,6 @@ if (getOperator()->isSubClassOf("SDNode")) { const SDNodeInfo &NI = CDP.getSDNodeInfo(getOperator()); - // Check that the number of operands is sane. Negative operands -> varargs. if (NI.getNumOperands() >= 0 && getNumChildren() != (unsigned)NI.getNumOperands()) { TP.error(getOperator()->getName() + " node requires exactly " + @@ -1818,9 +1937,10 @@ return false; } - bool MadeChange = NI.ApplyTypeConstraints(this, TP); + bool MadeChange = false; for (unsigned i = 0, e = getNumChildren(); i != e; ++i) MadeChange |= getChild(i)->ApplyTypeConstraints(TP, NotRegisters); + MadeChange |= NI.ApplyTypeConstraints(this, TP); return MadeChange; } @@ -1981,7 +2101,7 @@ // type info. We assume they must be the same. Otherwise, it is perfectly // legal to transform from one type to a completely different type. #if 0 - if (!hasTypeSet() || !getChild(0)->hasTypeSet()) { + if (!hasConcreteType() || !getChild(0)->hasConcreteType()) { bool MadeChange = UpdateNodeType(getChild(0)->getExtType(), TP); MadeChange |= getChild(0)->UpdateNodeType(getExtType(), TP); return MadeChange; @@ -2050,20 +2170,23 @@ TreePattern::TreePattern(Record *TheRec, ListInit *RawPat, bool isInput, CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp), - isInputPattern(isInput), HasError(false) { + isInputPattern(isInput), HasError(false), + Infer(*this) { for (Init *I : RawPat->getValues()) Trees.push_back(ParseTreePattern(I, "")); } TreePattern::TreePattern(Record *TheRec, DagInit *Pat, bool isInput, CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp), - isInputPattern(isInput), HasError(false) { + isInputPattern(isInput), HasError(false), + Infer(*this) { Trees.push_back(ParseTreePattern(Pat, "")); } TreePattern::TreePattern(Record *TheRec, TreePatternNode *Pat, bool isInput, CodeGenDAGPatterns &cdp) : TheRecord(TheRec), CDP(cdp), - isInputPattern(isInput), HasError(false) { + isInputPattern(isInput), HasError(false), + Infer(*this) { Trees.push_back(Pat); } @@ -2158,7 +2281,8 @@ // Apply the type cast. assert(New->getNumTypes() == 1 && "FIXME: Unhandled"); - New->UpdateNodeType(0, getValueType(Operator), *this); + const CodeGenHwModes &CGH = getDAGPatterns().getTargetInfo().getHwModes(); + New->UpdateNodeType(0, getValueTypeByHwMode(Operator, CGH), *this); if (!OpName.empty()) error("ValueType cast should not have a name!"); @@ -2273,7 +2397,7 @@ // If we have a bitconvert with a resolved type and if the source and // destination types are the same, then the bitconvert is useless, remove it. if (N->getOperator()->getName() == "bitconvert" && - N->getExtType(0).isConcrete() && + N->getExtType(0).isValueTypeByHwMode(false) && N->getExtType(0) == N->getChild(0)->getExtType(0) && N->getName().empty()) { N = N->getChild(0); @@ -2364,7 +2488,7 @@ bool HasUnresolvedTypes = false; for (const TreePatternNode *Tree : Trees) - HasUnresolvedTypes |= Tree->ContainsUnresolvedType(); + HasUnresolvedTypes |= Tree->ContainsUnresolvedType(*this); return !HasUnresolvedTypes; } @@ -2397,7 +2521,7 @@ // CodeGenDAGPatterns::CodeGenDAGPatterns(RecordKeeper &R) : - Records(R), Target(R) { + Records(R), Target(R), LegalVTS(Target.getLegalValueTypes()) { Intrinsics = CodeGenIntrinsicTable(Records, false); TgtIntrinsics = CodeGenIntrinsicTable(Records, true); @@ -2410,6 +2534,11 @@ ParsePatternFragments(/*OutFrags*/true); ParsePatterns(); + // Break patterns with parametrized types into a series of patterns, + // where each one has a fixed type and is predicated on the conditions + // of the associated hw mode. + ExpandHwModeBasedTypes(); + // Generate variants. For example, commutative patterns can match // multiple ways. Add them to PatternsToMatch as well. GenerateVariants(); @@ -2434,8 +2563,11 @@ // Parse all of the SDNode definitions for the target, populating SDNodes. void CodeGenDAGPatterns::ParseNodeInfo() { std::vector Nodes = Records.getAllDerivedDefinitions("SDNode"); + const CodeGenHwModes &CGH = getTargetInfo().getHwModes(); + while (!Nodes.empty()) { - SDNodes.insert(std::make_pair(Nodes.back(), Nodes.back())); + Record *R = Nodes.back(); + SDNodes.insert(std::make_pair(R, SDNodeInfo(R, CGH))); Nodes.pop_back(); } @@ -2589,7 +2721,7 @@ while (TPN->ApplyTypeConstraints(P, false)) /* Resolve all types */; - if (TPN->ContainsUnresolvedType()) { + if (TPN->ContainsUnresolvedType(P)) { PrintFatalError("Value #" + Twine(i) + " of OperandWithDefaultOps '" + DefaultOps[i]->getName() + "' doesn't have a concrete type!"); @@ -2988,7 +3120,7 @@ for (unsigned k = 0, ke = Pat->getNumTypes(); k != ke; ++k) { if (k > 0) Types += ", "; - Types += Pat->getExtType(k).getName(); + Types += Pat->getExtType(k).getAsString(); } I->error("Top-level forms in instruction pattern should have" " void types, has types " + Types); @@ -3186,14 +3318,13 @@ } Record *Instr = Entry.first; - AddPatternToMatch(I, - PatternToMatch(Instr, - Instr->getValueAsListInit("Predicates"), - SrcPattern, - TheInst.getResultPattern(), - TheInst.getImpResults(), - Instr->getValueAsInt("AddedComplexity"), - Instr->getID())); + ListInit *Preds = Instr->getValueAsListInit("Predicates"); + int Complexity = Instr->getValueAsInt("AddedComplexity"); + AddPatternToMatch( + I, + PatternToMatch(Instr, makePredList(Preds), SrcPattern, + TheInst.getResultPattern(), TheInst.getImpResults(), + Complexity, Instr->getID())); } } @@ -3219,6 +3350,20 @@ } } +std::vector CodeGenDAGPatterns::makePredList(ListInit *L) { + std::vector Preds; + for (Init *I : L->getValues()) { + if (DefInit *Pred = dyn_cast(I)) + Preds.push_back(Pred->getDef()); + else + llvm_unreachable("Non-def on the list"); + } + + // Sort so that different orders get canonicalized to the same string. + std::sort(Preds.begin(), Preds.end()); + return Preds; +} + void CodeGenDAGPatterns::AddPatternToMatch(TreePattern *Pattern, PatternToMatch &&PTM) { // Do some sanity checking on the pattern we're about to match. @@ -3262,8 +3407,6 @@ PatternsToMatch.push_back(std::move(PTM)); } - - void CodeGenDAGPatterns::InferInstructionFlags() { ArrayRef Instructions = Target.getInstructionsByEnumValue(); @@ -3425,12 +3568,13 @@ // If this type is already concrete or completely unknown we can't do // anything. + TypeInfer &TI = TP.getInfer(); for (unsigned i = 0, e = N->getNumTypes(); i != e; ++i) { - if (N->getExtType(i).isCompletelyUnknown() || N->getExtType(i).isConcrete()) + if (N->getExtType(i).empty() || TI.isConcrete(N->getExtType(i), false)) continue; - // Otherwise, force its type to the first possibility (an arbitrary choice). - if (N->getExtType(i).MergeInTypeInfo(N->getExtType(i).getTypeList()[0], TP)) + // Otherwise, force its type to an arbitrary choice. + if (TI.forceArbitrary(N->getExtType(i))) return true; } @@ -3551,15 +3695,154 @@ TreePattern Temp(Result.getRecord(), DstPattern, false, *this); Temp.InferAllTypes(); - AddPatternToMatch( - Pattern, - PatternToMatch( - CurPattern, CurPattern->getValueAsListInit("Predicates"), - Pattern->getTree(0), Temp.getOnlyTree(), std::move(InstImpResults), - CurPattern->getValueAsInt("AddedComplexity"), CurPattern->getID())); + // A pattern may end up with an "impossible" type, i.e. a situation + // where all types have been eliminated for some node in this pattern. + // This could occur for intrinsics that only make sense for a specific + // value type, and use a specific register class. If, for some mode, + // that register class does not accept that type, the type inference + // will lead to a contradiction, which is not an error however, but + // a sign that this pattern will simply never match. + if (Pattern->getTree(0)->hasPossibleType() && + Temp.getOnlyTree()->hasPossibleType()) { + ListInit *Preds = CurPattern->getValueAsListInit("Predicates"); + int Complexity = CurPattern->getValueAsInt("AddedComplexity"); + AddPatternToMatch( + Pattern, + PatternToMatch( + CurPattern, makePredList(Preds), Pattern->getTree(0), + Temp.getOnlyTree(), std::move(InstImpResults), Complexity, + CurPattern->getID())); + } + } +} + +static void collectModes(std::set &Modes, const TreePatternNode *N) { + for (const TypeSetByHwMode &VTS : N->getExtTypes()) + for (const auto &I : VTS) + Modes.insert(I.first); + + for (unsigned i = 0, e = N->getNumChildren(); i != e; ++i) + collectModes(Modes, N->getChild(i)); +} + +void CodeGenDAGPatterns::ExpandHwModeBasedTypes() { + const CodeGenHwModes &CGH = getTargetInfo().getHwModes(); + std::map> ModeChecks; + std::vector Copy = PatternsToMatch; + PatternsToMatch.clear(); + + auto AppendPattern = [this,&ModeChecks](PatternToMatch &P, unsigned Mode) { + TreePatternNode *NewSrc = P.SrcPattern->clone(); + TreePatternNode *NewDst = P.DstPattern->clone(); + if (!NewSrc->setDefaultMode(Mode) || !NewDst->setDefaultMode(Mode)) { + delete NewSrc; + delete NewDst; + return; + } + + std::vector Preds = P.Predicates; + const std::vector &MC = ModeChecks[Mode]; + Preds.insert(Preds.end(), MC.begin(), MC.end()); + PatternsToMatch.emplace_back(P.getSrcRecord(), Preds, NewSrc, NewDst, + P.getDstRegs(), P.getAddedComplexity(), + Record::getNewUID(), Mode); + }; + + for (PatternToMatch &P : Copy) { + TreePatternNode *SrcP = nullptr, *DstP = nullptr; + if (P.SrcPattern->hasProperTypeByHwMode()) + SrcP = P.SrcPattern; + if (P.DstPattern->hasProperTypeByHwMode()) + DstP = P.DstPattern; + if (!SrcP && !DstP) { + PatternsToMatch.push_back(P); + continue; + } + + std::set Modes; + if (SrcP) + collectModes(Modes, SrcP); + if (DstP) + collectModes(Modes, DstP); + + // The predicate for the default mode needs to be constructed for each + // pattern separately. + // Since not all modes must be present in each pattern, if a mode m is + // absent, then there is no point in constructing a check for m. If such + // a check was created, it would be equivalent to checking the default + // mode, except not all modes' predicates would be a part of the checking + // code. The subsequently generated check for the default mode would then + // have the exact same patterns, but a different predicate code. To avoid + // duplicated patterns with different predicate checks, construct the + // default check as a negation of all predicates that are actually present + // in the source/destination patterns. + std::vector DefaultPred; + + for (unsigned M : Modes) { + if (M == DefaultMode) + continue; + if (ModeChecks.find(M) != ModeChecks.end()) + continue; + + // Fill the map entry for this mode. + const HwMode &HM = CGH.getMode(M); + ModeChecks[M].emplace_back(Predicate(HM.Features, true)); + + // Add negations of the HM's predicates to the default predicate. + DefaultPred.emplace_back(Predicate(HM.Features, false)); + } + + for (unsigned M : Modes) { + if (M == DefaultMode) + continue; + AppendPattern(P, M); + } + + bool HasDefault = Modes.count(DefaultMode); + if (HasDefault) + AppendPattern(P, DefaultMode); + } +} + +/// Dependent variable map for CodeGenDAGPattern variant generation +typedef std::map DepVarMap; + +static void FindDepVarsOf(TreePatternNode *N, DepVarMap &DepMap) { + if (N->isLeaf()) { + if (isa(N->getLeafValue())) + DepMap[N->getName()]++; + } else { + for (size_t i = 0, e = N->getNumChildren(); i != e; ++i) + FindDepVarsOf(N->getChild(i), DepMap); } } +/// Find dependent variables within child patterns +static void FindDepVars(TreePatternNode *N, MultipleUseVarSet &DepVars) { + DepVarMap depcounts; + FindDepVarsOf(N, depcounts); + for (const std::pair &Pair : depcounts) { + if (Pair.second > 1) + DepVars.insert(Pair.first); + } +} + +#ifndef NDEBUG +/// Dump the dependent variable set: +static void DumpDepVars(MultipleUseVarSet &DepVars) { + if (DepVars.empty()) { + DEBUG(errs() << ""); + } else { + DEBUG(errs() << "[ "); + for (const std::string &DepVar : DepVars) { + DEBUG(errs() << DepVar << " "); + } + DEBUG(errs() << "]"); + } +} +#endif + + /// CombineChildVariants - Given a bunch of permutations of each child of the /// 'operator' node, put them together in all possible ways. static void CombineChildVariants(TreePatternNode *Orig, Index: utils/TableGen/CodeGenHwModes.h =================================================================== --- /dev/null +++ utils/TableGen/CodeGenHwModes.h @@ -0,0 +1,64 @@ +//===--- CodeGenHwModes.h ---------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Classes to parse and store HW mode information for instruction selection +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_CODEGENHWMODES_H +#define LLVM_UTILS_TABLEGEN_CODEGENHWMODES_H + +#include "llvm/ADT/StringMap.h" +#include +#include +#include + +// HwModeId -> list of predicates (definition) + +namespace llvm { + class Record; + class RecordKeeper; + + struct CodeGenHwModes; + + struct HwMode { + HwMode(Record *R); + StringRef Name; + std::string Features; + void dump() const; + }; + + struct HwModeSelect { + HwModeSelect(Record *R, CodeGenHwModes &CGH); + typedef std::pair PairType; + std::vector Items; + void dump() const; + }; + + struct CodeGenHwModes { + enum : unsigned { DefaultMode = 0 }; + static StringRef DefaultModeName; + + CodeGenHwModes(RecordKeeper &R); + unsigned getHwModeId(StringRef Name) const; + const HwMode &getMode(unsigned Id) const { + assert(Id != 0); + return Modes[Id-1]; + } + const HwModeSelect &getHwModeSelect(Record *R) const; + unsigned getNumModeIds() const { return Modes.size()+1; } + void dump() const; + + private: + RecordKeeper &Records; + StringMap ModeIds; // HwMode (string) -> HwModeId + std::vector Modes; + std::map ModeSelects; + }; +} + +#endif // LLVM_UTILS_TABLEGEN_CODEGENHWMODES_H Index: utils/TableGen/CodeGenHwModes.cpp =================================================================== --- /dev/null +++ utils/TableGen/CodeGenHwModes.cpp @@ -0,0 +1,111 @@ +//===--- CodeGenHwModes.cpp -----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Classes to parse and store HW mode information for instruction selection +//===----------------------------------------------------------------------===// + +#include "CodeGenHwModes.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TableGen/Error.h" +#include "llvm/TableGen/Record.h" + +using namespace llvm; + +StringRef CodeGenHwModes::DefaultModeName = "DefaultMode"; + +HwMode::HwMode(Record *R) { + Name = R->getName(); + Features = R->getValueAsString("Features"); +} + +void HwMode::dump() const { + dbgs() << Name << ": " << Features << '\n'; +} + +HwModeSelect::HwModeSelect(Record *R, CodeGenHwModes &CGH) { + std::vector Modes = R->getValueAsListOfDefs("Modes"); + std::vector Objects = R->getValueAsListOfDefs("Objects"); + if (Modes.size() != Objects.size()) { + PrintError(R->getLoc(), "in record " + R->getName() + + " derived from HwModeSelect: the lists Modes and Objects should " + "have the same size"); + report_fatal_error("error in target description."); + } + for (unsigned i = 0, e = Modes.size(); i != e; ++i) { + unsigned ModeId = CGH.getHwModeId(Modes[i]->getName()); + Items.push_back(std::make_pair(ModeId, Objects[i])); + } +} + +void HwModeSelect::dump() const { + dbgs() << '{'; + for (const PairType &P : Items) + dbgs() << " (" << P.first << ',' << P.second->getName() << ')'; + dbgs() << " }\n"; +} + +CodeGenHwModes::CodeGenHwModes(RecordKeeper &RK) : Records(RK) { + std::vector MRs = Records.getAllDerivedDefinitions("HwMode"); + // The default mode needs a definition in the .td sources for TableGen + // to accept references to it. We need to ignore the definition here. + for (auto I = MRs.begin(), E = MRs.end(); I != E; ++I) { + if ((*I)->getName() != DefaultModeName) + continue; + MRs.erase(I); + break; + } + + for (Record *R : MRs) { + Modes.emplace_back(R); + unsigned NewId = Modes.size(); + ModeIds.insert(std::make_pair(Modes[NewId-1].Name, NewId)); + } + + std::vector MSs = Records.getAllDerivedDefinitions("HwModeSelect"); + for (Record *R : MSs) { + auto P = ModeSelects.emplace(std::make_pair(R, HwModeSelect(R, *this))); + assert(P.second); + (void)P; + } +} + +unsigned CodeGenHwModes::getHwModeId(StringRef Name) const { + if (Name == DefaultModeName) + return DefaultMode; + auto F = ModeIds.find(Name); + assert(F != ModeIds.end()); + return F->second; +} + +const HwModeSelect &CodeGenHwModes::getHwModeSelect(Record *R) const { + auto F = ModeSelects.find(R); + assert(F != ModeSelects.end()); + return F->second; +} + +void CodeGenHwModes::dump() const { + dbgs() << "Modes: {\n"; + for (const HwMode &M : Modes) { + dbgs() << " "; + M.dump(); + } + dbgs() << "}\n"; + + dbgs() << "ModeIds: {\n"; + for (const auto &P : ModeIds) + dbgs() << " " << P.first() << " -> " << P.second << '\n'; + dbgs() << "}\n"; + + dbgs() << "ModeSelects: {\n"; + for (const auto &P : ModeSelects) { + dbgs() << " " << P.first->getName() << " -> "; + P.second.dump(); + } + dbgs() << "}\n"; +} Index: utils/TableGen/CodeGenInstruction.cpp =================================================================== --- utils/TableGen/CodeGenInstruction.cpp +++ utils/TableGen/CodeGenInstruction.cpp @@ -375,10 +375,10 @@ // Check to see if the first implicit def has a resolvable type. Record *FirstImplicitDef = ImplicitDefs[0]; assert(FirstImplicitDef->isSubClassOf("Register")); - const std::vector &RegVTs = + const std::vector &RegVTs = TargetInfo.getRegisterVTs(FirstImplicitDef); - if (RegVTs.size() == 1) - return RegVTs[0]; + if (RegVTs.size() == 1 && RegVTs[0].isSimple()) + return RegVTs[0].getSimple().SimpleTy; return MVT::Other; } Index: utils/TableGen/CodeGenRegisters.h =================================================================== --- utils/TableGen/CodeGenRegisters.h +++ utils/TableGen/CodeGenRegisters.h @@ -15,6 +15,7 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENREGISTERS_H #define LLVM_UTILS_TABLEGEN_CODEGENREGISTERS_H +#include "InfoByHwMode.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" @@ -309,9 +310,8 @@ public: unsigned EnumValue; StringRef Namespace; - SmallVector VTs; - unsigned SpillSize; - unsigned SpillAlignment; + SmallVector VTs; + RegSizeInfoByHwMode RSI; int CopyCost; bool Allocatable; StringRef AltOrderSelect; @@ -328,13 +328,10 @@ const std::string &getName() const { return Name; } std::string getQualifiedName() const; - ArrayRef getValueTypes() const {return VTs;} - bool hasValueType(MVT::SimpleValueType VT) const { - return std::find(VTs.begin(), VTs.end(), VT) != VTs.end(); - } + ArrayRef getValueTypes() const { return VTs; } unsigned getNumValueTypes() const { return VTs.size(); } - MVT::SimpleValueType getValueTypeNum(unsigned VTNum) const { + ValueTypeByHwMode getValueTypeNum(unsigned VTNum) const { if (VTNum < VTs.size()) return VTs[VTNum]; llvm_unreachable("VTNum greater than number of ValueTypes in RegClass!"); @@ -429,18 +426,15 @@ // the topological order used for the EnumValues. struct Key { const CodeGenRegister::Vec *Members; - unsigned SpillSize; - unsigned SpillAlignment; + RegSizeInfoByHwMode RSI; - Key(const CodeGenRegister::Vec *M, unsigned S = 0, unsigned A = 0) - : Members(M), SpillSize(S), SpillAlignment(A) {} + Key(const CodeGenRegister::Vec *M, const RegSizeInfoByHwMode &I) + : Members(M), RSI(I) {} Key(const CodeGenRegisterClass &RC) - : Members(&RC.getMembers()), - SpillSize(RC.SpillSize), - SpillAlignment(RC.SpillAlignment) {} + : Members(&RC.getMembers()), RSI(RC.RSI) {} - // Lexicographical order of (Members, SpillSize, SpillAlignment). + // Lexicographical order of (Members, RegSizeInfoByHwMode). bool operator<(const Key&) const; }; @@ -503,6 +497,8 @@ class CodeGenRegBank { SetTheory Sets; + const CodeGenHwModes &CGH; + std::deque SubRegIndices; DenseMap Def2SubRegIdx; @@ -586,10 +582,12 @@ void computeRegUnitLaneMasks(); public: - CodeGenRegBank(RecordKeeper&); + CodeGenRegBank(RecordKeeper&, const CodeGenHwModes&); SetTheory &getSets() { return Sets; } + const CodeGenHwModes &getHwModes() const { return CGH; } + // Sub-register indices. The first NumNamedIndices are defined by the user // in the .td files. The rest are synthesized such that all sub-registers // have a unique name. Index: utils/TableGen/CodeGenRegisters.cpp =================================================================== --- utils/TableGen/CodeGenRegisters.cpp +++ utils/TableGen/CodeGenRegisters.cpp @@ -686,7 +686,7 @@ if (!Type->isSubClassOf("ValueType")) PrintFatalError("RegTypes list member '" + Type->getName() + "' does not derive from the ValueType class!"); - VTs.push_back(getValueType(Type)); + VTs.push_back(getValueTypeByHwMode(Type, RegBank.getHwModes())); } assert(!VTs.empty() && "RegisterClass must contain at least one ValueType!"); @@ -719,12 +719,22 @@ } } - // Allow targets to override the size in bits of the RegisterClass. + Namespace = R->getValueAsString("Namespace"); + + if (const RecordVal *RV = R->getValue("RegInfos")) + if (DefInit *DI = dyn_cast_or_null(RV->getValue())) + RSI = RegSizeInfoByHwMode(DI->getDef(), RegBank.getHwModes()); unsigned Size = R->getValueAsInt("Size"); + assert((RSI.hasDefault() || Size != 0 || VTs[0].isSimple()) && + "Impossible to determine register size"); + if (!RSI.hasDefault()) { + RegSizeInfo RI; + RI.RegSize = RI.SpillSize = Size ? Size + : VTs[0].getSimple().getSizeInBits(); + RI.SpillAlignment = R->getValueAsInt("Alignment"); + RSI.Map.insert({DefaultMode, RI}); + } - Namespace = R->getValueAsString("Namespace"); - SpillSize = Size ? Size : MVT(VTs[0]).getSizeInBits(); - SpillAlignment = R->getValueAsInt("Alignment"); CopyCost = R->getValueAsInt("CopyCost"); Allocatable = R->getValueAsBit("isAllocatable"); AltOrderSelect = R->getValueAsString("AltOrderSelect"); @@ -744,8 +754,7 @@ Name(Name), TopoSigs(RegBank.getNumTopoSigs()), EnumValue(-1), - SpillSize(Props.SpillSize), - SpillAlignment(Props.SpillAlignment), + RSI(Props.RSI), CopyCost(0), Allocatable(true), AllocationPriority(0) { @@ -787,7 +796,7 @@ namespace llvm { raw_ostream &operator<<(raw_ostream &OS, const CodeGenRegisterClass::Key &K) { - OS << "{ S=" << K.SpillSize << ", A=" << K.SpillAlignment; + OS << "{ " << K.RSI.getAsString(); for (const auto R : *K.Members) OS << ", " << R->getName(); return OS << " }"; @@ -800,8 +809,7 @@ bool CodeGenRegisterClass::Key:: operator<(const CodeGenRegisterClass::Key &B) const { assert(Members && B.Members); - return std::tie(*Members, SpillSize, SpillAlignment) < - std::tie(*B.Members, B.SpillSize, B.SpillAlignment); + return std::tie(*Members, RSI) < std::tie(*B.Members, B.RSI); } // Returns true if RC is a strict subclass. @@ -815,8 +823,7 @@ // static bool testSubClass(const CodeGenRegisterClass *A, const CodeGenRegisterClass *B) { - return A->SpillAlignment && B->SpillAlignment % A->SpillAlignment == 0 && - A->SpillSize <= B->SpillSize && + return A->RSI.isSubClassOf(B->RSI) && std::includes(A->getMembers().begin(), A->getMembers().end(), B->getMembers().begin(), B->getMembers().end(), deref()); @@ -835,16 +842,9 @@ if (A == B) return false; - // Order by ascending spill size. - if (A->SpillSize < B->SpillSize) - return true; - if (A->SpillSize > B->SpillSize) - return false; - - // Order by ascending spill alignment. - if (A->SpillAlignment < B->SpillAlignment) + if (A->RSI < B->RSI) return true; - if (A->SpillAlignment > B->SpillAlignment) + if (A->RSI != B->RSI) return false; // Order by descending set size. Note that the classes' allocation order may @@ -1017,7 +1017,8 @@ // CodeGenRegBank //===----------------------------------------------------------------------===// -CodeGenRegBank::CodeGenRegBank(RecordKeeper &Records) { +CodeGenRegBank::CodeGenRegBank(RecordKeeper &Records, + const CodeGenHwModes &Modes) : CGH(Modes) { // Configure register Sets to understand register classes and tuples. Sets.addFieldExpander("RegisterClass", "MemberList"); Sets.addFieldExpander("CalleeSavedRegs", "SaveList"); @@ -1147,7 +1148,7 @@ const CodeGenRegister::Vec *Members, StringRef Name) { // Synthetic sub-class has the same size and alignment as RC. - CodeGenRegisterClass::Key K(Members, RC->SpillSize, RC->SpillAlignment); + CodeGenRegisterClass::Key K(Members, RC->RSI); RCKeyMap::const_iterator FoundI = Key2RC.find(K); if (FoundI != Key2RC.end()) return FoundI->second; @@ -1989,10 +1990,8 @@ continue; // If RC1 and RC2 have different spill sizes or alignments, use the - // larger size for sub-classing. If they are equal, prefer RC1. - if (RC2->SpillSize > RC1->SpillSize || - (RC2->SpillSize == RC1->SpillSize && - RC2->SpillAlignment > RC1->SpillAlignment)) + // stricter one for sub-classing. If they are equal, prefer RC1. + if (RC2->RSI.hasStricterSpillThan(RC1->RSI)) std::swap(RC1, RC2); getOrCreateSubClass(RC1, &Intersection, Index: utils/TableGen/CodeGenTarget.h =================================================================== --- utils/TableGen/CodeGenTarget.h +++ utils/TableGen/CodeGenTarget.h @@ -17,8 +17,10 @@ #ifndef LLVM_UTILS_TABLEGEN_CODEGENTARGET_H #define LLVM_UTILS_TABLEGEN_CODEGENTARGET_H +#include "CodeGenHwModes.h" #include "CodeGenInstruction.h" #include "CodeGenRegisters.h" +#include "InfoByHwMode.h" #include "llvm/Support/raw_ostream.h" #include "llvm/TableGen/Record.h" #include @@ -69,7 +71,8 @@ std::unique_ptr> Instructions; mutable std::unique_ptr RegBank; mutable std::vector RegAltNameIndices; - mutable SmallVector LegalValueTypes; + mutable SmallVector LegalValueTypes; + CodeGenHwModes CGH; void ReadRegAltNameIndices() const; void ReadInstructions() const; void ReadLegalValueTypes() const; @@ -128,22 +131,18 @@ /// getRegisterVTs - Find the union of all possible SimpleValueTypes for the /// specified physical register. - std::vector getRegisterVTs(Record *R) const; + std::vector getRegisterVTs(Record *R) const; - ArrayRef getLegalValueTypes() const { - if (LegalValueTypes.empty()) ReadLegalValueTypes(); + ArrayRef getLegalValueTypes() const { + if (LegalValueTypes.empty()) + ReadLegalValueTypes(); return LegalValueTypes; } - /// isLegalValueType - Return true if the specified value type is natively - /// supported by the target (i.e. there are registers that directly hold it). - bool isLegalValueType(MVT::SimpleValueType VT) const { - ArrayRef LegalVTs = getLegalValueTypes(); - return is_contained(LegalVTs, VT); - } - CodeGenSchedModels &getSchedModels() const; + const CodeGenHwModes &getHwModes() const { return CGH; } + private: DenseMap> & getInstructions() const { Index: utils/TableGen/CodeGenTarget.cpp =================================================================== --- utils/TableGen/CodeGenTarget.cpp +++ utils/TableGen/CodeGenTarget.cpp @@ -191,7 +191,7 @@ /// getTarget - Return the current instance of the Target class. /// CodeGenTarget::CodeGenTarget(RecordKeeper &records) - : Records(records) { + : Records(records), CGH(records) { std::vector Targets = Records.getAllDerivedDefinitions("Target"); if (Targets.size() == 0) PrintFatalError("ERROR: No 'Target' subclasses defined!"); @@ -266,7 +266,7 @@ CodeGenRegBank &CodeGenTarget::getRegBank() const { if (!RegBank) - RegBank = llvm::make_unique(Records); + RegBank = llvm::make_unique(Records, getHwModes()); return *RegBank; } @@ -285,19 +285,19 @@ return I->second; } -std::vector CodeGenTarget:: -getRegisterVTs(Record *R) const { +std::vector CodeGenTarget::getRegisterVTs(Record *R) + const { const CodeGenRegister *Reg = getRegBank().getReg(R); - std::vector Result; + std::vector Result; for (const auto &RC : getRegBank().getRegClasses()) { if (RC.contains(Reg)) { - ArrayRef InVTs = RC.getValueTypes(); + ArrayRef InVTs = RC.getValueTypes(); Result.insert(Result.end(), InVTs.begin(), InVTs.end()); } } // Remove duplicates. - array_pod_sort(Result.begin(), Result.end()); + std::sort(Result.begin(), Result.end()); Result.erase(std::unique(Result.begin(), Result.end()), Result.end()); return Result; } @@ -308,7 +308,7 @@ LegalValueTypes.insert(LegalValueTypes.end(), RC.VTs.begin(), RC.VTs.end()); // Remove duplicates. - array_pod_sort(LegalValueTypes.begin(), LegalValueTypes.end()); + std::sort(LegalValueTypes.begin(), LegalValueTypes.end()); LegalValueTypes.erase(std::unique(LegalValueTypes.begin(), LegalValueTypes.end()), LegalValueTypes.end()); Index: utils/TableGen/DAGISelEmitter.cpp =================================================================== --- utils/TableGen/DAGISelEmitter.cpp +++ utils/TableGen/DAGISelEmitter.cpp @@ -80,11 +80,11 @@ CodeGenDAGPatterns &CGP; bool operator()(const PatternToMatch *LHS, const PatternToMatch *RHS) { - const TreePatternNode *LHSSrc = LHS->getSrcPattern(); - const TreePatternNode *RHSSrc = RHS->getSrcPattern(); + const TreePatternNode *LT = LHS->getSrcPattern(); + const TreePatternNode *RT = RHS->getSrcPattern(); - MVT LHSVT = (LHSSrc->getNumTypes() != 0 ? LHSSrc->getType(0) : MVT::Other); - MVT RHSVT = (RHSSrc->getNumTypes() != 0 ? RHSSrc->getType(0) : MVT::Other); + MVT LHSVT = LT->getNumTypes() != 0 ? LT->getSimpleType(0) : MVT::Other; + MVT RHSVT = RT->getNumTypes() != 0 ? RT->getSimpleType(0) : MVT::Other; if (LHSVT.isVector() != RHSVT.isVector()) return RHSVT.isVector(); Index: utils/TableGen/DAGISelMatcherGen.cpp =================================================================== --- utils/TableGen/DAGISelMatcherGen.cpp +++ utils/TableGen/DAGISelMatcherGen.cpp @@ -33,12 +33,17 @@ if (!FoundRC) { FoundRC = true; - VT = RC.getValueTypeNum(0); + ValueTypeByHwMode VVT = RC.getValueTypeNum(0); + if (VVT.isSimple()) + VT = VVT.getSimple().SimpleTy; continue; } // If this occurs in multiple register classes, they all have to agree. - assert(VT == RC.getValueTypeNum(0)); +#ifndef NDEBUG + ValueTypeByHwMode T = RC.getValueTypeNum(0); + assert(!T.isSimple() || T.getSimple().SimpleTy == VT); +#endif } return VT; } @@ -105,13 +110,15 @@ Matcher *GetMatcher() const { return TheMatcher; } private: void AddMatcher(Matcher *NewNode); - void InferPossibleTypes(); + void InferPossibleTypes(unsigned ForceMode); // Matcher Generation. - void EmitMatchCode(const TreePatternNode *N, TreePatternNode *NodeNoTypes); + void EmitMatchCode(const TreePatternNode *N, TreePatternNode *NodeNoTypes, + unsigned ForceMode); void EmitLeafMatchCode(const TreePatternNode *N); void EmitOperatorMatchCode(const TreePatternNode *N, - TreePatternNode *NodeNoTypes); + TreePatternNode *NodeNoTypes, + unsigned ForceMode); /// If this is the first time a node with unique identifier Name has been /// seen, record it. Otherwise, emit a check to make sure this is the same @@ -164,17 +171,19 @@ PatWithNoTypes->RemoveAllTypes(); // If there are types that are manifestly known, infer them. - InferPossibleTypes(); + InferPossibleTypes(Pattern.ForceMode); } /// InferPossibleTypes - As we emit the pattern, we end up generating type /// checks and applying them to the 'PatWithNoTypes' tree. As we do this, we /// want to propagate implied types as far throughout the tree as possible so /// that we avoid doing redundant type checks. This does the type propagation. -void MatcherGen::InferPossibleTypes() { +void MatcherGen::InferPossibleTypes(unsigned ForceMode) { // TP - Get *SOME* tree pattern, we don't care which. It is only used for // diagnostics, which we know are impossible at this point. TreePattern &TP = *CGP.pf_begin()->second; + TP.getInfer().CodeGen = true; + TP.getInfer().ForceMode = ForceMode; bool MadeChange = true; while (MadeChange) @@ -281,7 +290,8 @@ } void MatcherGen::EmitOperatorMatchCode(const TreePatternNode *N, - TreePatternNode *NodeNoTypes) { + TreePatternNode *NodeNoTypes, + unsigned ForceMode) { assert(!N->isLeaf() && "Not an operator?"); if (N->getOperator()->isSubClassOf("ComplexPattern")) { @@ -334,7 +344,7 @@ // Match the LHS of the AND as appropriate. AddMatcher(new MoveChildMatcher(0)); - EmitMatchCode(N->getChild(0), NodeNoTypes->getChild(0)); + EmitMatchCode(N->getChild(0), NodeNoTypes->getChild(0), ForceMode); AddMatcher(new MoveParentMatcher()); return; } @@ -433,7 +443,7 @@ // Get the code suitable for matching this child. Move to the child, check // it then move back to the parent. AddMatcher(new MoveChildMatcher(OpNo)); - EmitMatchCode(N->getChild(i), NodeNoTypes->getChild(i)); + EmitMatchCode(N->getChild(i), NodeNoTypes->getChild(i), ForceMode); AddMatcher(new MoveParentMatcher()); } } @@ -456,7 +466,8 @@ } void MatcherGen::EmitMatchCode(const TreePatternNode *N, - TreePatternNode *NodeNoTypes) { + TreePatternNode *NodeNoTypes, + unsigned ForceMode) { // If N and NodeNoTypes don't agree on a type, then this is a case where we // need to do a type check. Emit the check, apply the type to NodeNoTypes and // reinfer any correlated types. @@ -465,7 +476,7 @@ for (unsigned i = 0, e = NodeNoTypes->getNumTypes(); i != e; ++i) { if (NodeNoTypes->getExtType(i) == N->getExtType(i)) continue; NodeNoTypes->setType(i, N->getExtType(i)); - InferPossibleTypes(); + InferPossibleTypes(ForceMode); ResultsToTypeCheck.push_back(i); } @@ -478,14 +489,14 @@ if (N->isLeaf()) EmitLeafMatchCode(N); else - EmitOperatorMatchCode(N, NodeNoTypes); + EmitOperatorMatchCode(N, NodeNoTypes, ForceMode); // If there are node predicates for this node, generate their checks. for (unsigned i = 0, e = N->getPredicateFns().size(); i != e; ++i) AddMatcher(new CheckPredicateMatcher(N->getPredicateFns()[i])); for (unsigned i = 0, e = ResultsToTypeCheck.size(); i != e; ++i) - AddMatcher(new CheckTypeMatcher(N->getType(ResultsToTypeCheck[i]), + AddMatcher(new CheckTypeMatcher(N->getSimpleType(ResultsToTypeCheck[i]), ResultsToTypeCheck[i])); } @@ -509,7 +520,7 @@ } // Emit the matcher for the pattern structure and types. - EmitMatchCode(Pattern.getSrcPattern(), PatWithNoTypes); + EmitMatchCode(Pattern.getSrcPattern(), PatWithNoTypes, Pattern.ForceMode); // If the pattern has a predicate on it (e.g. only enabled when a subtarget // feature is around, do the check). @@ -606,7 +617,7 @@ assert(N->isLeaf() && "Must be a leaf"); if (IntInit *II = dyn_cast(N->getLeafValue())) { - AddMatcher(new EmitIntegerMatcher(II->getValue(), N->getType(0))); + AddMatcher(new EmitIntegerMatcher(II->getValue(), N->getSimpleType(0))); ResultOps.push_back(NextRecordedOperandNo++); return; } @@ -617,13 +628,13 @@ if (Def->isSubClassOf("Register")) { const CodeGenRegister *Reg = CGP.getTargetInfo().getRegBank().getReg(Def); - AddMatcher(new EmitRegisterMatcher(Reg, N->getType(0))); + AddMatcher(new EmitRegisterMatcher(Reg, N->getSimpleType(0))); ResultOps.push_back(NextRecordedOperandNo++); return; } if (Def->getName() == "zero_reg") { - AddMatcher(new EmitRegisterMatcher(nullptr, N->getType(0))); + AddMatcher(new EmitRegisterMatcher(nullptr, N->getSimpleType(0))); ResultOps.push_back(NextRecordedOperandNo++); return; } @@ -834,7 +845,7 @@ // Determine the result types. SmallVector ResultVTs; for (unsigned i = 0, e = N->getNumTypes(); i != e; ++i) - ResultVTs.push_back(N->getType(i)); + ResultVTs.push_back(N->getSimpleType(i)); // If this is the root instruction of a pattern that has physical registers in // its result pattern, add output VTs for them. For example, X86 has: Index: utils/TableGen/FastISelEmitter.cpp =================================================================== --- utils/TableGen/FastISelEmitter.cpp +++ utils/TableGen/FastISelEmitter.cpp @@ -159,10 +159,11 @@ TreePredicateFn PredFn = ImmPredicates.getPredicate(Code-1); // Emit the type check. - OS << "VT == " - << getEnumName(PredFn.getOrigPatFragRecord()->getTree(0)->getType(0)) - << " && "; - + TreePattern *TP = PredFn.getOrigPatFragRecord(); + ValueTypeByHwMode VVT = TP->getTree(0)->getType(0); + assert(VVT.isSimple() && + "Cannot use variable value types with fast isel"); + OS << "VT == " << getEnumName(VVT.getSimple().SimpleTy) << " && "; OS << PredFn.getFnName() << "(imm" << i <<')'; EmittedAnything = true; @@ -218,7 +219,7 @@ } // Handle unmatched immediate sizes here. - //if (Op->getType(0) != VT) + //if (Op->getSimpleType(0) != VT) // return false; Operands.push_back(OpKind::getImm(PredNo)); @@ -240,12 +241,12 @@ return false; } - assert(Op->hasTypeSet(0) && "Type infererence not done?"); + assert(Op->hasConcreteType(0) && "Type infererence not done?"); // For now, all the operands must have the same type (if they aren't // immediates). Note that this causes us to reject variable sized shifts // on X86. - if (Op->getType(0) != VT) + if (Op->getSimpleType(0) != VT) return false; DefInit *OpDI = dyn_cast(Op->getLeafValue()); @@ -506,11 +507,11 @@ Record *InstPatOp = InstPatNode->getOperator(); std::string OpcodeName = getOpcodeName(InstPatOp, CGP); MVT::SimpleValueType RetVT = MVT::isVoid; - if (InstPatNode->getNumTypes()) RetVT = InstPatNode->getType(0); + if (InstPatNode->getNumTypes()) RetVT = InstPatNode->getSimpleType(0); MVT::SimpleValueType VT = RetVT; if (InstPatNode->getNumChildren()) { assert(InstPatNode->getChild(0)->getNumTypes() == 1); - VT = InstPatNode->getChild(0)->getType(0); + VT = InstPatNode->getChild(0)->getSimpleType(0); } // For now, filter out any instructions with predicates. Index: utils/TableGen/GlobalISelEmitter.cpp =================================================================== --- utils/TableGen/GlobalISelEmitter.cpp +++ utils/TableGen/GlobalISelEmitter.cpp @@ -1811,7 +1811,7 @@ void gatherNodeEquivs(); const CodeGenInstruction *findNodeEquiv(Record *N) const; - Error importRulePredicates(RuleMatcher &M, ArrayRef Predicates); + Error importRulePredicates(RuleMatcher &M, ArrayRef Predicates); Expected createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, const TreePatternNode *Src, @@ -1858,17 +1858,19 @@ } GlobalISelEmitter::GlobalISelEmitter(RecordKeeper &RK) - : RK(RK), CGP(RK), Target(CGP.getTargetInfo()), CGRegs(RK) {} + : RK(RK), CGP(RK), Target(CGP.getTargetInfo()), + CGRegs(RK, Target.getHwModes()) {} //===- Emitter ------------------------------------------------------------===// Error GlobalISelEmitter::importRulePredicates(RuleMatcher &M, - ArrayRef Predicates) { - for (const Init *Predicate : Predicates) { - const DefInit *PredicateDef = static_cast(Predicate); - declareSubtargetFeature(PredicateDef->getDef()); - M.addRequiredFeature(PredicateDef->getDef()); + ArrayRef Predicates) { + for (const Predicate &P : Predicates) { + if (!P.Def) + continue; + declareSubtargetFeature(P.Def); + M.addRequiredFeature(P.Def); } return Error::success(); @@ -1904,9 +1906,10 @@ } unsigned OpIdx = 0; - for (const EEVT::TypeSet &Ty : Src->getExtTypes()) { - auto OpTyOrNone = MVTToLLT(Ty.getConcrete()); - + for (const TypeSetByHwMode &VTy : Src->getExtTypes()) { + auto OpTyOrNone = VTy.isMachineValueType() + ? MVTToLLT(VTy.getMachineValueType().SimpleTy) + : None; if (!OpTyOrNone) return failedImport( "Result of Src pattern operator has an unsupported type"); @@ -1973,7 +1976,7 @@ return failedImport("Src pattern child has predicate (" + explainPredicates(SrcChild) + ")"); - ArrayRef ChildTypes = SrcChild->getExtTypes(); + ArrayRef ChildTypes = SrcChild->getExtTypes(); if (ChildTypes.size() != 1) return failedImport("Src pattern child has multiple results"); @@ -1988,7 +1991,9 @@ } } - auto OpTyOrNone = MVTToLLT(ChildTypes.front().getConcrete()); + Optional OpTyOrNone = None; + if (ChildTypes.front().isMachineValueType()) + OpTyOrNone = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy); if (!OpTyOrNone) return failedImport("Src operand has an unsupported type (" + to_string(*SrcChild) + ")"); OM.addPredicate(*OpTyOrNone); @@ -2085,11 +2090,13 @@ if (auto *ChildDefInit = dyn_cast(DstChild->getLeafValue())) { auto *ChildRec = ChildDefInit->getDef(); - ArrayRef ChildTypes = DstChild->getExtTypes(); + ArrayRef ChildTypes = DstChild->getExtTypes(); if (ChildTypes.size() != 1) return failedImport("Dst pattern child has multiple results"); - auto OpTyOrNone = MVTToLLT(ChildTypes.front().getConcrete()); + Optional OpTyOrNone = None; + if (ChildTypes.front().isMachineValueType()) + OpTyOrNone = MVTToLLT(ChildTypes.front().getMachineValueType().SimpleTy); if (!OpTyOrNone) return failedImport("Dst operand has an unsupported type"); @@ -2268,7 +2275,7 @@ RuleMatcher M; M.addAction(P); - if (auto Error = importRulePredicates(M, P.getPredicates()->getValues())) + if (auto Error = importRulePredicates(M, P.getPredicates())) return std::move(Error); // Next, analyze the pattern operators. @@ -2334,8 +2341,8 @@ // The root of the match also has constraints on the register bank so that it // matches the result instruction. unsigned OpIdx = 0; - for (const EEVT::TypeSet &Ty : Src->getExtTypes()) { - (void)Ty; + for (const TypeSetByHwMode &VTy : Src->getExtTypes()) { + (void)VTy; const auto &DstIOperand = DstI.Operands[OpIdx]; Record *DstIOpRec = DstIOperand.Rec; Index: utils/TableGen/InfoByHwMode.h =================================================================== --- /dev/null +++ utils/TableGen/InfoByHwMode.h @@ -0,0 +1,168 @@ +//===--- InfoByHwMode.h -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Classes that implement data parametrized by HW modes for instruction +// selection. Currently it is ValueTypeByHwMode (parametrized ValueType), +// and RegSizeInfoByHwMode (parametrized register/spill size and alignment +// data). +//===----------------------------------------------------------------------===// + +#ifndef LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H +#define LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H + +#include "CodeGenHwModes.h" +#include "llvm/CodeGen/MachineValueType.h" + +#include +#include +#include +#include + +namespace llvm { + +struct CodeGenHwModes; +class Record; + +template struct InfoByHwMode; + +std::string getModeName(unsigned Mode); + +enum : unsigned { + DefaultMode = CodeGenHwModes::DefaultMode, +}; + +template +std::vector union_modes(const InfoByHwMode &A, + const InfoByHwMode &B) { + std::vector V; + std::set U; + for (const auto &P : A) + U.insert(P.first); + for (const auto &P : B) + U.insert(P.first); + // Make sure that the default mode is last on the list. + bool HasDefault = U.count(DefaultMode); + for (unsigned M : U) + if (M != DefaultMode) + V.push_back(M); + if (HasDefault) + V.push_back(DefaultMode); + return V; +} + +template +struct InfoByHwMode { + typedef std::map MapType; + typedef typename MapType::value_type PairType; + typedef typename MapType::iterator iterator; + typedef typename MapType::const_iterator const_iterator; + + InfoByHwMode() = default; + InfoByHwMode(const MapType &&M) : Map(M) {} + + iterator begin() { return Map.begin(); } + iterator end() { return Map.end(); } + const_iterator begin() const { return Map.begin(); } + const_iterator end() const { return Map.end(); } + bool empty() const { return Map.empty(); } + + bool hasMode(unsigned M) const { return Map.find(M) != Map.end(); } + bool hasDefault() const { return hasMode(DefaultMode); } + + InfoT &get(unsigned Mode) { + if (!hasMode(Mode)) { + assert(hasMode(DefaultMode)); + Map[Mode] = Map[DefaultMode]; + } + return Map[Mode]; + } + const InfoT &get(unsigned Mode) const { + auto F = Map.find(Mode); + if (Mode != DefaultMode && F == Map.end()) + F = Map.find(DefaultMode); + assert(F != Map.end()); + return F->second; + } + + bool isSimple() const { + return Map.size() == 1 && Map.begin()->first == DefaultMode; + } + InfoT getSimple() const { + assert(isSimple()); + return Map.begin()->second; + } + void makeSimple(unsigned Mode) { + assert(hasMode(Mode) || hasDefault()); + InfoT I = get(Mode); + Map.clear(); + Map.insert(std::make_pair(DefaultMode, I)); + } + + MapType Map; +}; + +struct ValueTypeByHwMode : public InfoByHwMode { + ValueTypeByHwMode(Record *R, const CodeGenHwModes &CGH); + ValueTypeByHwMode(MVT T) { Map.insert({DefaultMode,T}); } + ValueTypeByHwMode() = default; + + bool operator== (const ValueTypeByHwMode &T) const; + bool operator< (const ValueTypeByHwMode &T) const; + + bool isValid() const { + return !Map.empty(); + } + MVT getType(unsigned Mode) const { return get(Mode); } + MVT &getOrCreateTypeForMode(unsigned Mode, MVT Type); + + static std::string getMVTName(MVT T); + std::string getAsString() const; + void dump() const; +}; + +ValueTypeByHwMode getValueTypeByHwMode(Record *Rec, + const CodeGenHwModes &CGH); + + +struct RegSizeInfo { + unsigned RegSize; + unsigned SpillSize; + unsigned SpillAlignment; + + RegSizeInfo(Record *R, const CodeGenHwModes &CGH); + RegSizeInfo() = default; + bool operator< (const RegSizeInfo &I) const; + bool operator== (const RegSizeInfo &I) const { + return std::tie(RegSize, SpillSize, SpillAlignment) == + std::tie(I.RegSize, I.SpillSize, I.SpillAlignment); + } + bool operator!= (const RegSizeInfo &I) const { + return !(*this == I); + } + + bool isSubClassOf(const RegSizeInfo &I) const; + std::string getAsString() const; +}; + +struct RegSizeInfoByHwMode : public InfoByHwMode { + RegSizeInfoByHwMode(Record *R, const CodeGenHwModes &CGH); + RegSizeInfoByHwMode() = default; + bool operator< (const RegSizeInfoByHwMode &VI) const; + bool operator== (const RegSizeInfoByHwMode &VI) const; + bool operator!= (const RegSizeInfoByHwMode &VI) const { + return !(*this == VI); + } + + bool isSubClassOf(const RegSizeInfoByHwMode &I) const; + bool hasStricterSpillThan(const RegSizeInfoByHwMode &I) const; + + std::string getAsString() const; +}; +} // namespace llvm + +#endif // LLVM_UTILS_TABLEGEN_INFOBYHWMODE_H Index: utils/TableGen/InfoByHwMode.cpp =================================================================== --- /dev/null +++ utils/TableGen/InfoByHwMode.cpp @@ -0,0 +1,198 @@ +//===--- InfoByHwMode.cpp -------------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// Classes that implement data parametrized by HW modes for instruction +// selection. Currently it is ValueTypeByHwMode (parametrized ValueType), +// and RegSizeInfoByHwMode (parametrized register/spill size and alignment +// data). +//===----------------------------------------------------------------------===// + +#include "CodeGenTarget.h" +#include "InfoByHwMode.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/Twine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +#include +#include +#include + +using namespace llvm; + +std::string llvm::getModeName(unsigned Mode) { + if (Mode == DefaultMode) + return "*"; + return (Twine('m') + Twine(Mode)).str(); +} + +ValueTypeByHwMode::ValueTypeByHwMode(Record *R, const CodeGenHwModes &CGH) { + const HwModeSelect &MS = CGH.getHwModeSelect(R); + for (const HwModeSelect::PairType &P : MS.Items) { + auto I = Map.insert({P.first, MVT(llvm::getValueType(P.second))}); + assert(I.second && "Duplicate entry?"); + (void)I; + } +} + +bool ValueTypeByHwMode::operator== (const ValueTypeByHwMode &T) const { + assert(isValid() && T.isValid() && "Invalid type in assignment"); + bool Simple = isSimple(); + if (Simple != T.isSimple()) + return false; + if (Simple) + return getSimple() == T.getSimple(); + + return Map == T.Map; +} + +bool ValueTypeByHwMode::operator< (const ValueTypeByHwMode &T) const { + assert(isValid() && T.isValid() && "Invalid type in comparison"); + // Default order for maps. + return Map < T.Map; +} + +MVT &ValueTypeByHwMode::getOrCreateTypeForMode(unsigned Mode, MVT Type) { + auto F = Map.find(Mode); + if (F != Map.end()) + return F->second; + // If Mode is not in the map, look up the default mode. If it exists, + // make a copy of it for Mode and return it. + auto D = Map.find(DefaultMode); + if (D != Map.end()) + return Map.insert(std::make_pair(Mode, D->second)).first->second; + // If default mode is not present either, use provided Type. + return Map.insert(std::make_pair(Mode, Type)).first->second; +} + +std::string ValueTypeByHwMode::getMVTName(MVT T) { + std::string N = llvm::getEnumName(T.SimpleTy); + if (N.substr(0,5) == "MVT::") + N = N.substr(5); + return N; +} + +std::string ValueTypeByHwMode::getAsString() const { + if (isSimple()) + return getMVTName(getSimple()); + + std::vector Pairs; + for (const auto &P : Map) + Pairs.push_back(&P); + std::sort(Pairs.begin(), Pairs.end(), deref>()); + + std::stringstream str; + str << '{'; + for (unsigned i = 0, e = Pairs.size(); i != e; ++i) { + const PairType *P = Pairs[i]; + str << '(' << getModeName(P->first) + << ':' << getMVTName(P->second) << ')'; + if (i != e-1) + str << ','; + } + str << '}'; + return str.str(); +} + +LLVM_DUMP_METHOD +void ValueTypeByHwMode::dump() const { + dbgs() << "size=" << Map.size() << '\n'; + for (const auto &P : Map) + dbgs() << " " << P.first << " -> " + << llvm::getEnumName(P.second.SimpleTy) << '\n'; +} + +ValueTypeByHwMode llvm::getValueTypeByHwMode(Record *Rec, + const CodeGenHwModes &CGH) { +#ifndef NDEBUG + if (!Rec->isSubClassOf("ValueType")) + Rec->dump(); +#endif + assert(Rec->isSubClassOf("ValueType")); + if (Rec->isSubClassOf("HwModeSelect")) + return ValueTypeByHwMode(Rec, CGH); + return ValueTypeByHwMode(llvm::getValueType(Rec)); +} + +RegSizeInfo::RegSizeInfo(Record *R, const CodeGenHwModes &CGH) { + RegSize = R->getValueAsInt("RegSize"); + SpillSize = R->getValueAsInt("SpillSize"); + SpillAlignment = R->getValueAsInt("SpillAlignment"); +} + +bool RegSizeInfo::operator< (const RegSizeInfo &I) const { + return std::tie(RegSize, SpillSize, SpillAlignment) < + std::tie(I.RegSize, I.SpillSize, I.SpillAlignment); +} + +bool RegSizeInfo::isSubClassOf(const RegSizeInfo &I) const { + return RegSize <= I.RegSize && + SpillAlignment && I.SpillAlignment % SpillAlignment == 0 && + SpillSize <= I.SpillSize; +} + +std::string RegSizeInfo::getAsString() const { + std::stringstream str; + str << "[R=" << RegSize << ",S=" << SpillSize + << ",A=" << SpillAlignment << ']'; + return str.str(); +} + +RegSizeInfoByHwMode::RegSizeInfoByHwMode(Record *R, + const CodeGenHwModes &CGH) { + const HwModeSelect &MS = CGH.getHwModeSelect(R); + for (const HwModeSelect::PairType &P : MS.Items) { + auto I = Map.insert({P.first, RegSizeInfo(P.second, CGH)}); + assert(I.second && "Duplicate entry?"); + (void)I; + } +} + +bool RegSizeInfoByHwMode::operator< (const RegSizeInfoByHwMode &I) const { + unsigned M0 = Map.begin()->first; + return get(M0) < I.get(M0); +} + +bool RegSizeInfoByHwMode::operator== (const RegSizeInfoByHwMode &I) const { + unsigned M0 = Map.begin()->first; + return get(M0) == I.get(M0); +} + +bool RegSizeInfoByHwMode::isSubClassOf(const RegSizeInfoByHwMode &I) const { + unsigned M0 = Map.begin()->first; + return get(M0).isSubClassOf(I.get(M0)); +} + +bool RegSizeInfoByHwMode::hasStricterSpillThan(const RegSizeInfoByHwMode &I) + const { + unsigned M0 = Map.begin()->first; + const RegSizeInfo &A0 = get(M0); + const RegSizeInfo &B0 = I.get(M0); + return std::tie(A0.SpillSize, A0.SpillAlignment) > + std::tie(B0.SpillSize, B0.SpillAlignment); +} + +std::string RegSizeInfoByHwMode::getAsString() const { + typedef typename decltype(Map)::value_type PairType; + std::vector Pairs; + for (const auto &P : Map) + Pairs.push_back(&P); + std::sort(Pairs.begin(), Pairs.end(), deref>()); + + std::stringstream str; + str << '{'; + for (unsigned i = 0, e = Pairs.size(); i != e; ++i) { + const PairType *P = Pairs[i]; + str << '(' << getModeName(P->first) + << ':' << P->second.getAsString() << ')'; + if (i != e-1) + str << ','; + } + str << '}'; + return str.str(); +} Index: utils/TableGen/RegisterBankEmitter.cpp =================================================================== --- utils/TableGen/RegisterBankEmitter.cpp +++ utils/TableGen/RegisterBankEmitter.cpp @@ -18,6 +18,7 @@ #include "llvm/TableGen/Record.h" #include "llvm/TableGen/TableGenBackend.h" +#include "CodeGenHwModes.h" #include "CodeGenRegisters.h" #define DEBUG_TYPE "register-bank-emitter" @@ -84,7 +85,8 @@ // the VT's reliably due to Untyped. if (RCWithLargestRegsSize == nullptr) RCWithLargestRegsSize = RC; - else if (RCWithLargestRegsSize->SpillSize < RC->SpillSize) + else if (RCWithLargestRegsSize->RSI.get(DefaultMode).SpillSize < + RC->RSI.get(DefaultMode).SpillSize) RCWithLargestRegsSize = RC; assert(RCWithLargestRegsSize && "RC was nullptr?"); @@ -115,7 +117,7 @@ public: RegisterBankEmitter(RecordKeeper &R) - : Records(R), RegisterClassHierarchy(Records) {} + : Records(R), RegisterClassHierarchy(Records, CodeGenHwModes(R)) {} void run(raw_ostream &OS); }; @@ -241,7 +243,8 @@ for (const auto &Bank : Banks) { std::string QualifiedBankID = (TargetName + "::" + Bank.getEnumeratorName()).str(); - unsigned Size = Bank.getRCWithLargestRegsSize()->SpillSize; + const CodeGenRegisterClass &RC = *Bank.getRCWithLargestRegsSize(); + unsigned Size = RC.RSI.get(DefaultMode).SpillSize; OS << "RegisterBank " << Bank.getInstanceVarName() << "(/* ID */ " << QualifiedBankID << ", /* Name */ \"" << Bank.getName() << "\", /* Size */ " << Size << ", " Index: utils/TableGen/RegisterInfoEmitter.cpp =================================================================== --- utils/TableGen/RegisterInfoEmitter.cpp +++ utils/TableGen/RegisterInfoEmitter.cpp @@ -1023,13 +1023,14 @@ for (const auto &RC : RegisterClasses) { assert(isInt<8>(RC.CopyCost) && "Copy cost too large."); - // Register size and spill size will become independent, but are not at - // the moment. For now use SpillSize as the register size. + uint32_t RegSize = 0; + if (RC.RSI.isSimple()) + RegSize = RC.RSI.getSimple().RegSize; OS << " { " << RC.getName() << ", " << RC.getName() << "Bits, " << RegClassStrings.get(RC.getName()) << ", " << RC.getOrder().size() << ", sizeof(" << RC.getName() << "Bits), " << RC.getQualifiedName() + "RegClassID" << ", " - << RC.SpillSize/8 << ", " + << RegSize/8 << ", " << RC.CopyCost << ", " << ( RC.Allocatable ? "true" : "false" ) << " },\n"; } @@ -1097,7 +1098,8 @@ OS << "struct " << ClassName << " : public TargetRegisterInfo {\n" << " explicit " << ClassName - << "(unsigned RA, unsigned D = 0, unsigned E = 0, unsigned PC = 0);\n"; + << "(unsigned RA, unsigned D = 0, unsigned E = 0,\n" + << " unsigned PC = 0, unsigned HwMode = 0);\n"; if (!RegBank.getSubRegIndices().empty()) { OS << " unsigned composeSubRegIndicesImpl" << "(unsigned, unsigned) const override;\n" @@ -1176,10 +1178,19 @@ AllocatableRegs.insert(Order.begin(), Order.end()); } + const CodeGenHwModes &CGH = Target.getHwModes(); + unsigned NumModes = CGH.getNumModeIds(); + // Build a shared array of value types. - SequenceToOffsetTable > VTSeqs; - for (const auto &RC : RegisterClasses) - VTSeqs.add(RC.VTs); + SequenceToOffsetTable> VTSeqs; + for (unsigned M = 0; M < NumModes; ++M) { + for (const auto &RC : RegisterClasses) { + std::vector S; + for (const ValueTypeByHwMode &VVT : RC.VTs) + S.push_back(VVT.get(M).SimpleTy); + VTSeqs.add(S); + } + } VTSeqs.layout(); OS << "\nstatic const MVT::SimpleValueType VTLists[] = {\n"; VTSeqs.emit(OS, printSimpleValueType, "MVT::Other"); @@ -1207,6 +1218,31 @@ // Now that all of the structs have been emitted, emit the instances. if (!RegisterClasses.empty()) { + OS << "\nstatic const TargetRegisterInfo::RegClassInfo RegClassInfos[]" + << " = {\n"; + for (unsigned M = 0; M < NumModes; ++M) { + unsigned EV = 0; + OS << " // Mode = " << M << " ("; + if (M == 0) + OS << "Default"; + else + OS << CGH.getMode(M).Name; + OS << ")\n"; + for (const auto &RC : RegisterClasses) { + assert(RC.EnumValue == EV++ && "Unexpected order of register classes"); + const RegSizeInfo &RI = RC.RSI.get(M); + OS << " { " << RI.RegSize << ", " << RI.SpillSize << ", " + << RI.SpillAlignment; + std::vector VTs; + for (const ValueTypeByHwMode &VVT : RC.VTs) + VTs.push_back(VVT.get(M).SimpleTy); + OS << ", VTLists+" << VTSeqs.get(VTs) << " }, // " + << RC.getName() << '\n'; + } + } + OS << "};\n"; + + OS << "\nstatic const TargetRegisterClass *const " << "NullRegClasses[] = { nullptr };\n\n"; @@ -1313,15 +1349,10 @@ << " { // Register class instances\n"; for (const auto &RC : RegisterClasses) { - assert(isUInt<16>(RC.SpillSize/8) && "SpillSize too large."); - assert(isUInt<16>(RC.SpillAlignment/8) && "SpillAlignment too large."); OS << " extern const TargetRegisterClass " << RC.getName() << "RegClass = {\n " << '&' << Target.getName() << "MCRegisterClasses[" << RC.getName() << "RegClassID],\n " - << RC.SpillSize/8 << ", /* SpillSize */\n " - << RC.SpillAlignment/8 << ", /* SpillAlignment */\n " - << "VTLists + " << VTSeqs.get(RC.VTs) << ",\n " << RC.getName() - << "SubClassMask,\n SuperRegIdxSeqs + " + << RC.getName() << "SubClassMask,\n SuperRegIdxSeqs + " << SuperRegIdxSeqs.get(SuperRegIdxLists[RC.EnumValue]) << ",\n "; printMask(OS, RC.LaneMask); OS << ",\n " << (unsigned)RC.AllocationPriority << ",\n " @@ -1425,12 +1456,14 @@ EmitRegMappingTables(OS, Regs, true); OS << ClassName << "::\n" << ClassName - << "(unsigned RA, unsigned DwarfFlavour, unsigned EHFlavour, unsigned PC)\n" + << "(unsigned RA, unsigned DwarfFlavour, unsigned EHFlavour,\n" + " unsigned PC, unsigned HwMode)\n" << " : TargetRegisterInfo(" << TargetName << "RegInfoDesc" - << ", RegisterClasses, RegisterClasses+" << RegisterClasses.size() <<",\n" - << " SubRegIndexNameTable, SubRegIndexLaneMaskTable, "; + << ", RegisterClasses, RegisterClasses+" << RegisterClasses.size() << ",\n" + << " SubRegIndexNameTable, SubRegIndexLaneMaskTable,\n" + << " "; printMask(OS, RegBank.CoveringLanes); - OS << ") {\n" + OS << ", RegClassInfos, HwMode) {\n" << " InitMCRegisterInfo(" << TargetName << "RegDesc, " << Regs.size() + 1 << ", RA, PC,\n " << TargetName << "MCRegisterClasses, " << RegisterClasses.size() << ",\n"