Index: llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp =================================================================== --- llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -3591,21 +3591,22 @@ }; const auto &InstInfo = getVOPDInstInfo(Opcode, &MII); - auto InvalidOperandInfo = InstInfo.getInvalidOperandIndex(getVRegIdx); - if (!InvalidOperandInfo) + auto InvalidCompOprIdx = InstInfo.getInvalidCompOperandIndex(getVRegIdx); + if (!InvalidCompOprIdx) return true; - auto OprIdx = *InvalidOperandInfo; - auto ParsedIdx = std::max(InstInfo[VOPD::X].getParsedOperandIndex(OprIdx), - InstInfo[VOPD::Y].getParsedOperandIndex(OprIdx)); + auto CompOprIdx = *InvalidCompOprIdx; + auto ParsedIdx = + std::max(InstInfo[VOPD::X].getIndexInParsedOperands(CompOprIdx), + InstInfo[VOPD::Y].getIndexInParsedOperands(CompOprIdx)); assert(ParsedIdx > 0 && ParsedIdx < Operands.size()); auto Loc = ((AMDGPUOperand &)*Operands[ParsedIdx]).getStartLoc(); - if (OprIdx == VOPD::Component::DST) { + if (CompOprIdx == VOPD::Component::DST) { Error(Loc, "one dst register must be even and the other odd"); } else { - auto SrcIdx = OprIdx - VOPD::Component::DST_NUM; - Error(Loc, Twine("src") + Twine(SrcIdx) + + auto CompSrcIdx = CompOprIdx - VOPD::Component::DST_NUM; + Error(Loc, Twine("src") + Twine(CompSrcIdx) + " operands must use different VGPR banks"); } @@ -8500,8 +8501,8 @@ // Create VOPD MCInst operands using parsed assembler operands. void AMDGPUAsmParser::cvtVOPD(MCInst &Inst, const OperandVector &Operands) { - auto addOp = [&](uint16_t i) { // NOLINT:function pointer - AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[i]); + auto addOp = [&](uint16_t ParsedOprIdx) { // NOLINT:function pointer + AMDGPUOperand &Op = ((AMDGPUOperand &)*Operands[ParsedOprIdx]); if (Op.isReg()) { Op.addRegOperands(Inst, 1); return; @@ -8519,16 +8520,16 @@ // dstX, dstY, src0X [, other OpX operands], src0Y [, other OpY operands] for (auto CompIdx : VOPD::COMPONENTS) { - addOp(InstInfo[CompIdx].getParsedDstIndex()); + addOp(InstInfo[CompIdx].getDstIndexInParsedOperands()); } for (auto CompIdx : VOPD::COMPONENTS) { const auto &CInfo = InstInfo[CompIdx]; - auto ParsedSrcOperandsNum = InstInfo[CompIdx].getParsedSrcOperandsNum(); - for (unsigned SrcIdx = 0; SrcIdx < ParsedSrcOperandsNum; ++SrcIdx) - addOp(CInfo.getParsedSrcIndex(SrcIdx)); + auto CompSrcOperandsNum = InstInfo[CompIdx].getCompParsedSrcOperandsNum(); + for (unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOperandsNum; ++CompSrcIdx) + addOp(CInfo.getSrcIndexInParsedOperands(CompSrcIdx)); if (CInfo.hasSrc2Acc()) - addOp(CInfo.getParsedDstIndex()); + addOp(CInfo.getDstIndexInParsedOperands()); } } Index: llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp =================================================================== --- llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp +++ llvm/lib/Target/AMDGPU/GCNCreateVOPD.cpp @@ -77,14 +77,17 @@ auto InstInfo = AMDGPU::getVOPDInstInfo(FirstMI->getDesc(), SecondMI->getDesc()); - for (auto CompIdx : VOPD::COMPONENTS) - VOPDInst.add(MI[CompIdx]->getOperand(InstInfo[CompIdx].getDstIndex())); + for (auto CompIdx : VOPD::COMPONENTS) { + auto MCOprIdx = InstInfo[CompIdx].getDstIndexInMCOperands(); + VOPDInst.add(MI[CompIdx]->getOperand(MCOprIdx)); + } for (auto CompIdx : VOPD::COMPONENTS) { - auto SrcOperandsNum = InstInfo[CompIdx].getSrcOperandsNum(); - for (unsigned SrcIdx = 0; SrcIdx < SrcOperandsNum; ++SrcIdx) - VOPDInst.add( - MI[CompIdx]->getOperand(InstInfo[CompIdx].getSrcIndex(SrcIdx))); + auto CompSrcOprNum = InstInfo[CompIdx].getCompSrcOperandsNum(); + for (unsigned CompSrcIdx = 0; CompSrcIdx < CompSrcOprNum; ++CompSrcIdx) { + auto MCOprIdx = InstInfo[CompIdx].getSrcIndexInMCOperands(CompSrcIdx); + VOPDInst.add(MI[CompIdx]->getOperand(MCOprIdx)); + } } for (auto CompIdx : VOPD::COMPONENTS) Index: llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp =================================================================== --- llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp +++ llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp @@ -91,8 +91,10 @@ addLiteral(Src0); } - if (InstInfo[CompIdx].hasMandatoryLiteral()) - addLiteral(MI.getOperand(InstInfo[CompIdx].getMandatoryLiteralIndex())); + if (InstInfo[CompIdx].hasMandatoryLiteral()) { + auto CompOprIdx = InstInfo[CompIdx].getMandatoryLiteralCompOperandIndex(); + addLiteral(MI.getOperand(CompOprIdx)); + } if (MI.getDesc().hasImplicitUseOfPhysReg(AMDGPU::VCC)) UniqueScalarRegs.push_back(AMDGPU::VCC_LO); } Index: llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h =================================================================== --- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -540,26 +540,39 @@ ComponentProps() = default; ComponentProps(const MCInstrDesc &OpDesc); - unsigned getSrcOperandsNum() const { return SrcOperandsNum; } - unsigned getParsedSrcOperandsNum() const { + // Return the total number of src operands this component has. + unsigned getCompSrcOperandsNum() const { return SrcOperandsNum; } + + // Return the number of src operands of this component visible to the parser. + unsigned getCompParsedSrcOperandsNum() const { return SrcOperandsNum - HasSrc2Acc; } + + // Return true iif this component has a mandatory literal. bool hasMandatoryLiteral() const { return MandatoryLiteralIdx.has_value(); } - unsigned getMandatoryLiteralIndex() const { + + // If this component has a mandatory literal, return component operand + // index of this literal (i.e. either Component::SRC1 or Component::SRC2). + unsigned getMandatoryLiteralCompOperandIndex() const { assert(hasMandatoryLiteral()); return *MandatoryLiteralIdx; } - bool hasRegSrcOperand(unsigned SrcIdx) const { - assert(SrcIdx < Component::MAX_SRC_NUM); - return SrcOperandsNum > SrcIdx && !hasMandatoryLiteralAt(SrcIdx); + + // Return true iif this component has operand + // with component index CompSrcIdx and this operand may be a register. + bool hasRegSrcOperand(unsigned CompSrcIdx) const { + assert(CompSrcIdx < Component::MAX_SRC_NUM); + return SrcOperandsNum > CompSrcIdx && !hasMandatoryLiteralAt(CompSrcIdx); } + + // Return true iif this component has tied src2. bool hasSrc2Acc() const { return HasSrc2Acc; } private: - bool hasMandatoryLiteralAt(unsigned SrcIdx) const { - assert(SrcIdx < Component::MAX_SRC_NUM); + bool hasMandatoryLiteralAt(unsigned CompSrcIdx) const { + assert(CompSrcIdx < Component::MAX_SRC_NUM); return hasMandatoryLiteral() && - *MandatoryLiteralIdx == Component::DST_NUM + SrcIdx; + *MandatoryLiteralIdx == Component::DST_NUM + CompSrcIdx; } }; @@ -570,8 +583,36 @@ MAX = COMPONENT_Y }; -// Location of operands in a MachineInstr/MCInst -// and position of operands in parsed operands array. +// Interface functions of this class map VOPD component operand indices +// to indices of operands in MachineInstr/MCInst or parsed operands array. +// +// Note that this class operates with 3 kinds of indices: +// - VOPD component operand indices (Component::DST, Component::SRC0, etc.); +// - MC operand indices (they refer operands in a MachineInstr/MCInst); +// - parsed operand indices (they refer operands in parsed operands array). +// +// For SINGLE components mapping between these indices is trivial. +// But things get more complicated for COMPONENT_X and +// COMPONENT_Y because these components share the same +// MachineInstr/MCInst and the same parsed operands array. +// Below is an example of component operand to parsed operand +// mapping for the following instruction: +// +// v_dual_add_f32 v255, v4, v5 :: v_dual_mov_b32 v6, v1 +// +// PARSED COMPONENT PARSED +// COMPONENT OPERANDS OPERAND INDEX OPERAND INDEX +// ------------------------------------------------------------------- +// "v_dual_add_f32" 0 +// v_dual_add_f32 v255 0 (DST) --> 1 +// v4 1 (SRC0) --> 2 +// v5 2 (SRC1) --> 3 +// "::" 4 +// "v_dual_mov_b32" 5 +// v_dual_mov_b32 v6 0 (DST) --> 6 +// v1 1 (SRC0) --> 7 +// ------------------------------------------------------------------- +// class ComponentLayout { private: // Regular MachineInstr/MCInst operands are ordered as follows: @@ -580,7 +621,7 @@ // dstX, dstY, src0X [, other OpX operands], src0Y [, other OpY operands] // Each ComponentKind has operand indices defined below. static constexpr unsigned MC_DST_IDX[] = {0, 0, 1}; - static constexpr unsigned FIRST_MC_SRC_IDX[] = {1, 2, 2 /* + OpX.SrcNum */}; + static constexpr unsigned FIRST_MC_SRC_IDX[] = {1, 2, 2 /* + OpX.MCSrcNum */}; // Parsed operands of regular instructions are ordered as follows: // Mnemo dst src0 [vsrc1 ...] @@ -595,56 +636,63 @@ private: const ComponentKind Kind; - const ComponentProps PrevOp; + const ComponentProps PrevComp; public: - // Create layout for COMPONENT_X or SINGLE component + // Create layout for COMPONENT_X or SINGLE component. ComponentLayout(ComponentKind Kind) : Kind(Kind) { assert(Kind == ComponentKind::SINGLE || Kind == ComponentKind::COMPONENT_X); } - // Create layout for COMPONENT_Y which depends on COMPONENT_X layout + // Create layout for COMPONENT_Y which depends on COMPONENT_X layout. ComponentLayout(const ComponentProps &OpXProps) - : Kind(ComponentKind::COMPONENT_Y), PrevOp(OpXProps) {} + : Kind(ComponentKind::COMPONENT_Y), PrevComp(OpXProps) {} public: - unsigned getDstIndex() const { return MC_DST_IDX[Kind]; } - unsigned getSrcIndex(unsigned SrcIdx) const { - assert(SrcIdx < Component::MAX_SRC_NUM); - return FIRST_MC_SRC_IDX[Kind] + getPrevOpSrcNum() + SrcIdx; + // Return the index of dst operand in MCInst operands. + unsigned getDstIndexInMCOperands() const { return MC_DST_IDX[Kind]; } + + // Return the index of the specified src operand in MCInst operands. + unsigned getSrcIndexInMCOperands(unsigned CompSrcIdx) const { + assert(CompSrcIdx < Component::MAX_SRC_NUM); + return FIRST_MC_SRC_IDX[Kind] + getPrevCompSrcNum() + CompSrcIdx; } - unsigned getParsedDstIndex() const { - return PARSED_DST_IDX[Kind] + getPrevOpParsedSrcNum(); + // Return the index of dst operand in the parsed operands array. + unsigned getDstIndexInParsedOperands() const { + return PARSED_DST_IDX[Kind] + getPrevCompParsedSrcNum(); } - unsigned getParsedSrcIndex(unsigned SrcIdx) const { - assert(SrcIdx < Component::MAX_SRC_NUM); - return FIRST_PARSED_SRC_IDX[Kind] + getPrevOpParsedSrcNum() + SrcIdx; + + // Return the index of the specified src operand in the parsed operands array. + unsigned getSrcIndexInParsedOperands(unsigned CompSrcIdx) const { + assert(CompSrcIdx < Component::MAX_SRC_NUM); + return FIRST_PARSED_SRC_IDX[Kind] + getPrevCompParsedSrcNum() + CompSrcIdx; } private: - unsigned getPrevOpSrcNum() const { return PrevOp.getSrcOperandsNum(); } - unsigned getPrevOpParsedSrcNum() const { - return PrevOp.getParsedSrcOperandsNum(); + unsigned getPrevCompSrcNum() const { + return PrevComp.getCompSrcOperandsNum(); + } + unsigned getPrevCompParsedSrcNum() const { + return PrevComp.getCompParsedSrcOperandsNum(); } }; // Layout and properties of VOPD components. class ComponentInfo : public ComponentLayout, public ComponentProps { public: - // Create ComponentInfo for COMPONENT_X or SINGLE component + // Create ComponentInfo for COMPONENT_X or SINGLE component. ComponentInfo(const MCInstrDesc &OpDesc, ComponentKind Kind = ComponentKind::SINGLE) : ComponentLayout(Kind), ComponentProps(OpDesc) {} - // Create ComponentInfo for COMPONENT_Y which depends on COMPONENT_X layout - ComponentInfo(const MCInstrDesc &OpDesc, - const ComponentProps &OpXProps) + // Create ComponentInfo for COMPONENT_Y which depends on COMPONENT_X layout. + ComponentInfo(const MCInstrDesc &OpDesc, const ComponentProps &OpXProps) : ComponentLayout(OpXProps), ComponentProps(OpDesc) {} - // Map MC operand index to parsed operand index. + // Map component operand index to parsed operand index. // Return 0 if the specified operand does not exist. - unsigned getParsedOperandIndex(unsigned OprIdx) const; + unsigned getIndexInParsedOperands(unsigned CompOprIdx) const; }; // Properties of VOPD instructions. @@ -667,17 +715,17 @@ } // Check VOPD operands constraints. - // GetRegIdx(Component, OperandIdx) must return a VGPR register index - // for the specified component and operand. The callback must return 0 + // GetRegIdx(Component, MCOperandIdx) must return a VGPR register index + // for the specified component and MC operand. The callback must return 0 // if the operand is not a register or not a VGPR. bool hasInvalidOperand( std::function GetRegIdx) const { - return getInvalidOperandIndex(GetRegIdx).has_value(); + return getInvalidCompOperandIndex(GetRegIdx).has_value(); } // Check VOPD operands constraints. // Return the index of an invalid component operand, if any. - Optional getInvalidOperandIndex( + Optional getInvalidCompOperandIndex( std::function GetRegIdx) const; private: Index: llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp =================================================================== --- llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -484,39 +484,41 @@ assert(SrcOperandsNum <= Component::MAX_SRC_NUM); auto OperandsNum = OpDesc.getNumOperands(); - for (unsigned OprIdx = Component::SRC1; OprIdx < OperandsNum; ++OprIdx) { - if (OpDesc.OpInfo[OprIdx].OperandType == AMDGPU::OPERAND_KIMM32) { - MandatoryLiteralIdx = OprIdx; + unsigned CompOprIdx; + for (CompOprIdx = Component::SRC1; CompOprIdx < OperandsNum; ++CompOprIdx) { + if (OpDesc.OpInfo[CompOprIdx].OperandType == AMDGPU::OPERAND_KIMM32) { + MandatoryLiteralIdx = CompOprIdx; break; } } } -unsigned ComponentInfo::getParsedOperandIndex(unsigned OprIdx) const { - assert(OprIdx < Component::MAX_OPR_NUM); +unsigned ComponentInfo::getIndexInParsedOperands(unsigned CompOprIdx) const { + assert(CompOprIdx < Component::MAX_OPR_NUM); - if (OprIdx == Component::DST) - return getParsedDstIndex(); + if (CompOprIdx == Component::DST) + return getDstIndexInParsedOperands(); - auto SrcIdx = OprIdx - Component::DST_NUM; - if (SrcIdx < getParsedSrcOperandsNum()) - return getParsedSrcIndex(SrcIdx); + auto CompSrcIdx = CompOprIdx - Component::DST_NUM; + if (CompSrcIdx < getCompParsedSrcOperandsNum()) + return getSrcIndexInParsedOperands(CompSrcIdx); // The specified operand does not exist. return 0; } -Optional InstInfo::getInvalidOperandIndex( +Optional InstInfo::getInvalidCompOperandIndex( std::function GetRegIdx) const { auto OpXRegs = getRegIndices(ComponentIndex::X, GetRegIdx); auto OpYRegs = getRegIndices(ComponentIndex::Y, GetRegIdx); - for (unsigned OprIdx = 0; OprIdx < Component::MAX_OPR_NUM; ++OprIdx) { - unsigned BanksNum = BANKS_NUM[OprIdx]; - if (OpXRegs[OprIdx] && OpYRegs[OprIdx] && - (OpXRegs[OprIdx] % BanksNum == OpYRegs[OprIdx] % BanksNum)) - return OprIdx; + unsigned CompOprIdx; + for (CompOprIdx = 0; CompOprIdx < Component::MAX_OPR_NUM; ++CompOprIdx) { + unsigned BanksNum = BANKS_NUM[CompOprIdx]; + if (OpXRegs[CompOprIdx] && OpYRegs[CompOprIdx] && + (OpXRegs[CompOprIdx] % BanksNum == OpYRegs[CompOprIdx] % BanksNum)) + return CompOprIdx; } return {}; @@ -526,8 +528,8 @@ // by the specified component. If an operand is unused // or is not a VGPR, the corresponding value is 0. // -// GetRegIdx(Component, OperandIdx) must return a VGPR register index -// for the specified component and operand. The callback must return 0 +// GetRegIdx(Component, MCOperandIdx) must return a VGPR register index +// for the specified component and MC operand. The callback must return 0 // if the operand is not a register or not a VGPR. InstInfo::RegIndices InstInfo::getRegIndices( unsigned ComponentIdx, @@ -537,13 +539,14 @@ auto Comp = CompInfo[ComponentIdx]; InstInfo::RegIndices RegIndices; - RegIndices[DST] = GetRegIdx(ComponentIdx, Comp.getDstIndex()); + RegIndices[DST] = GetRegIdx(ComponentIdx, Comp.getDstIndexInMCOperands()); - for (unsigned OprIdx : {SRC0, SRC1, SRC2}) { - unsigned SrcIdx = OprIdx - DST_NUM; - RegIndices[OprIdx] = Comp.hasRegSrcOperand(SrcIdx) - ? GetRegIdx(ComponentIdx, Comp.getSrcIndex(SrcIdx)) - : 0; + for (unsigned CompOprIdx : {SRC0, SRC1, SRC2}) { + unsigned CompSrcIdx = CompOprIdx - DST_NUM; + RegIndices[CompOprIdx] = + Comp.hasRegSrcOperand(CompSrcIdx) + ? GetRegIdx(ComponentIdx, Comp.getSrcIndexInMCOperands(CompSrcIdx)) + : 0; } return RegIndices; }