Index: llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp +++ llvm/lib/Target/SPIRV/SPIRVAsmPrinter.cpp @@ -57,6 +57,7 @@ void outputMCInst(MCInst &Inst); void outputInstruction(const MachineInstr *MI); void outputModuleSection(SPIRV::ModuleSectionType MSType); + void outputGlobalRequirements(); void outputEntryPoints(); void outputDebugSourceAndStrings(const Module &M); void outputOpExtInstImports(const Module &M); @@ -318,6 +319,30 @@ } } +// Create global OpCapability instructions for the required capabilities. +void SPIRVAsmPrinter::outputGlobalRequirements() { + // Abort here if not all requirements can be satisfied. + MAI->Reqs.checkSatisfiable(*ST); + + for (const auto &Cap : MAI->Reqs.getMinimalCapabilities()) { + MCInst Inst; + Inst.setOpcode(SPIRV::OpCapability); + Inst.addOperand(MCOperand::createImm(Cap)); + outputMCInst(Inst); + } + + // Generate the final OpExtensions with strings instead of enums. + for (const auto &Ext : MAI->Reqs.getExtensions()) { + MCInst Inst; + Inst.setOpcode(SPIRV::OpExtension); + addStringImm(getSymbolicOperandMnemonic( + SPIRV::OperandCategory::ExtensionOperand, Ext), + Inst); + outputMCInst(Inst); + } + // TODO add a pseudo instr for version number. +} + void SPIRVAsmPrinter::outputExtFuncDecls() { // Insert OpFunctionEnd after each declaration. SmallVectorImpl::iterator @@ -467,8 +492,8 @@ MAI = &SPIRVModuleAnalysis::MAI; assert(ST && TII && MAI && M && "Module analysis is required"); // Output instructions according to the Logical Layout of a Module: - // TODO: 1,2. All OpCapability instructions, then optional OpExtension - // instructions. + // 1,2. All OpCapability instructions, then optional OpExtension instructions. + outputGlobalRequirements(); // 3. Optional OpExtInstImport instructions. outputOpExtInstImports(*M); // 4. The single required OpMemoryModel instruction. Index: llvm/lib/Target/SPIRV/SPIRVInstrInfo.h =================================================================== --- llvm/lib/Target/SPIRV/SPIRVInstrInfo.h +++ llvm/lib/Target/SPIRV/SPIRVInstrInfo.h @@ -32,6 +32,9 @@ bool isConstantInstr(const MachineInstr &MI) const; bool isTypeDeclInstr(const MachineInstr &MI) const; bool isDecorationInstr(const MachineInstr &MI) const; + bool canUseFastMathFlags(const MachineInstr &MI) const; + bool canUseNSW(const MachineInstr &MI) const; + bool canUseNUW(const MachineInstr &MI) const; bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, Index: llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp +++ llvm/lib/Target/SPIRV/SPIRVInstrInfo.cpp @@ -91,6 +91,56 @@ } } +bool SPIRVInstrInfo::canUseFastMathFlags(const MachineInstr &MI) const { + switch (MI.getOpcode()) { + case SPIRV::OpFAddS: + case SPIRV::OpFSubS: + case SPIRV::OpFMulS: + case SPIRV::OpFDivS: + case SPIRV::OpFRemS: + case SPIRV::OpFAddV: + case SPIRV::OpFSubV: + case SPIRV::OpFMulV: + case SPIRV::OpFDivV: + case SPIRV::OpFRemV: + case SPIRV::OpFMod: + return true; + default: + return false; + } +} + +bool SPIRVInstrInfo::canUseNSW(const MachineInstr &MI) const { + switch (MI.getOpcode()) { + case SPIRV::OpIAddS: + case SPIRV::OpIAddV: + case SPIRV::OpISubS: + case SPIRV::OpISubV: + case SPIRV::OpIMulS: + case SPIRV::OpIMulV: + case SPIRV::OpShiftLeftLogicalS: + case SPIRV::OpShiftLeftLogicalV: + case SPIRV::OpSNegate: + return true; + default: + return false; + } +} + +bool SPIRVInstrInfo::canUseNUW(const MachineInstr &MI) const { + switch (MI.getOpcode()) { + case SPIRV::OpIAddS: + case SPIRV::OpIAddV: + case SPIRV::OpISubS: + case SPIRV::OpISubV: + case SPIRV::OpIMulS: + case SPIRV::OpIMulV: + return true; + default: + return false; + } +} + // Analyze the branching code at the end of MBB, returning // true if it cannot be understood (e.g. it's a switch dispatch or isn't // implemented for a target). Upon success, this returns false and returns Index: llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h =================================================================== --- llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h +++ llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.h @@ -15,13 +15,15 @@ #define LLVM_LIB_TARGET_SPIRV_SPIRVMODULEANALYSIS_H #include "MCTargetDesc/SPIRVBaseInfo.h" -#include "SPIRVDuplicatesTracker.h" -#include "SPIRVSubtarget.h" +#include "SPIRVGlobalRegistry.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" namespace llvm { +class SPIRVSubtarget; class MachineFunction; class MachineModuleInfo; @@ -39,6 +41,77 @@ NUM_MODULE_SECTIONS // Total number of sections requiring basic blocks. }; +struct Requirements { + const bool IsSatisfiable; + const Optional Cap; + const ExtensionList Exts; + const unsigned MinVer; // 0 if no min version is required. + const unsigned MaxVer; // 0 if no max version is required. + + Requirements(bool IsSatisfiable = false, + Optional Cap = {}, + ExtensionList Exts = {}, unsigned MinVer = 0, + unsigned MaxVer = 0) + : IsSatisfiable(IsSatisfiable), Cap(Cap), Exts(Exts), MinVer(MinVer), + MaxVer(MaxVer) {} + Requirements(Capability::Capability Cap) : Requirements(true, {Cap}) {} +}; + +struct RequirementHandler { +private: + CapabilityList MinimalCaps; + SmallSet AllCaps; + SmallSet AllExtensions; + unsigned MinVersion; // 0 if no min version is defined. + unsigned MaxVersion; // 0 if no max version is defined. + DenseSet AvailableCaps; + // Remove a list of capabilities from dedupedCaps and add them to AllCaps, + // recursing through their implicitly declared capabilities too. + void pruneCapabilities(const CapabilityList &ToPrune); + +public: + RequirementHandler() : MinVersion(0), MaxVersion(0) {} + void clear() { + MinimalCaps.clear(); + AllCaps.clear(); + AvailableCaps.clear(); + AllExtensions.clear(); + MinVersion = 0; + MaxVersion = 0; + } + unsigned getMinVersion() const { return MinVersion; } + unsigned getMaxVersion() const { return MaxVersion; } + const CapabilityList &getMinimalCapabilities() const { return MinimalCaps; } + const SmallSet &getExtensions() const { + return AllExtensions; + } + // Add a list of capabilities, ensuring AllCaps captures all the implicitly + // declared capabilities, and MinimalCaps has the minimal set of required + // capabilities (so all implicitly declared ones are removed). + void addCapabilities(const CapabilityList &ToAdd); + void addCapability(Capability::Capability ToAdd) { addCapabilities({ToAdd}); } + void addExtensions(const ExtensionList &ToAdd) { + AllExtensions.insert(ToAdd.begin(), ToAdd.end()); + } + void addExtension(Extension::Extension ToAdd) { AllExtensions.insert(ToAdd); } + // Add the given requirements to the lists. If constraints conflict, or these + // requirements cannot be satisfied, then abort the compilation. + void addRequirements(const Requirements &Req); + // Get requirement and add it to the list. + void getAndAddRequirements(SPIRV::OperandCategory::OperandCategory Category, + uint32_t i, const SPIRVSubtarget &ST); + // Check if all the requirements can be satisfied for the given subtarget, and + // if not abort compilation. + void checkSatisfiable(const SPIRVSubtarget &ST) const; + void initAvailableCapabilities(const SPIRVSubtarget &ST); + // Add the given capabilities to available and all their implicitly defined + // capabilities too. + void addAvailableCaps(const CapabilityList &ToAdd); + bool isCapabilityAvailable(Capability::Capability Cap) const { + return AvailableCaps.contains(Cap); + } +}; + using InstrList = SmallVector; // Maps a local register to the corresponding global alias. using LocalToGlobalRegTable = std::map; @@ -48,9 +121,10 @@ // The struct contains results of the module analysis and methods // to access them. struct ModuleAnalysisInfo { - SPIRV::MemoryModel::MemoryModel Mem; - SPIRV::AddressingModel::AddressingModel Addr; - SPIRV::SourceLanguage::SourceLanguage SrcLang; + RequirementHandler Reqs; + MemoryModel::MemoryModel Mem; + AddressingModel::AddressingModel Addr; + SourceLanguage::SourceLanguage SrcLang; unsigned SrcLangVersion; StringSet<> SrcExt; // Maps ExtInstSet to corresponding ID register. Index: llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -16,7 +16,6 @@ #include "SPIRVModuleAnalysis.h" #include "SPIRV.h" -#include "SPIRVGlobalRegistry.h" #include "SPIRVSubtarget.h" #include "SPIRVTargetMachine.h" #include "SPIRVUtils.h" @@ -52,6 +51,41 @@ return DefaultVal; } +static SPIRV::Requirements +getSymbolicOperandRequirements(SPIRV::OperandCategory::OperandCategory Category, + unsigned i, const SPIRVSubtarget &ST, + SPIRV::RequirementHandler &Reqs) { + unsigned ReqMinVer = getSymbolicOperandMinVersion(Category, i); + unsigned ReqMaxVer = getSymbolicOperandMaxVersion(Category, i); + unsigned TargetVer = ST.getSPIRVVersion(); + bool MinVerOK = !ReqMinVer || !TargetVer || TargetVer >= ReqMinVer; + bool MaxVerOK = !ReqMaxVer || !TargetVer || TargetVer <= ReqMaxVer; + CapabilityList ReqCaps = getSymbolicOperandCapabilities(Category, i); + ExtensionList ReqExts = getSymbolicOperandExtensions(Category, i); + if (ReqCaps.empty()) { + if (ReqExts.empty()) { + if (MinVerOK && MaxVerOK) + return {true, {}, {}, ReqMinVer, ReqMaxVer}; + return {false, {}, {}, 0, 0}; + } + } else if (MinVerOK && MaxVerOK) { + for (auto Cap : ReqCaps) { // Only need 1 of the capabilities to work. + if (Reqs.isCapabilityAvailable(Cap)) + return {true, {Cap}, {}, ReqMinVer, ReqMaxVer}; + } + } + // If there are no capabilities, or we can't satisfy the version or + // capability requirements, use the list of extensions (if the subtarget + // can handle them all). + if (std::all_of(ReqExts.begin(), ReqExts.end(), + [&ST](const SPIRV::Extension::Extension &Ext) { + return ST.canUseExtension(Ext); + })) { + return {true, {}, ReqExts, 0, 0}; // TODO: add versions to extensions. + } + return {false, {}, {}, 0, 0}; +} + void SPIRVModuleAnalysis::setBaseInfo(const Module &M) { MAI.MaxID = 0; for (int i = 0; i < SPIRV::NUM_MODULE_SECTIONS; i++) @@ -61,6 +95,8 @@ MAI.FuncNameMap.clear(); MAI.GlobalVarList.clear(); MAI.ExtInstSetMap.clear(); + MAI.Reqs.clear(); + MAI.Reqs.initAvailableCapabilities(*ST); // TODO: determine memory model and source language from the configuratoin. if (auto MemModel = M.getNamedMetadata("spirv.MemoryModel")) { @@ -103,6 +139,15 @@ } } + // Update required capabilities for this memory model, addressing model and + // source language. + MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::MemoryModelOperand, + MAI.Mem, *ST); + MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::SourceLanguageOperand, + MAI.SrcLang, *ST); + MAI.Reqs.getAndAddRequirements(SPIRV::OperandCategory::AddressingModelOperand, + MAI.Addr, *ST); + // TODO: check if it's required by default. MAI.ExtInstSetMap[static_cast(SPIRV::InstructionSet::OpenCL_std)] = Register::index2VirtReg(MAI.getNextID()); @@ -180,6 +225,27 @@ DepsGraph, SPIRV::MB_TypeConstVars, [](const SPIRV::DTSortableEntry *E) { return !E->getIsFunc(); }); + for (auto F = M.begin(), E = M.end(); F != E; ++F) { + MachineFunction *MF = MMI->getMachineFunction(*F); + if (!MF) + continue; + // Iterate through and collect OpExtension/OpCapability instructions. + for (MachineBasicBlock &MBB : *MF) { + for (MachineInstr &MI : MBB) { + if (MI.getOpcode() == SPIRV::OpExtension) { + // Here, OpExtension just has a single enum operand, not a string. + auto Ext = SPIRV::Extension::Extension(MI.getOperand(0).getImm()); + MAI.Reqs.addExtension(Ext); + MAI.setSkipEmission(&MI); + } else if (MI.getOpcode() == SPIRV::OpCapability) { + auto Cap = SPIRV::Capability::Capability(MI.getOperand(0).getImm()); + MAI.Reqs.addCapability(Cap); + MAI.setSkipEmission(&MI); + } + } + } + } + collectGlobalEntities( DepsGraph, SPIRV::MB_ExtFuncDecls, [](const SPIRV::DTSortableEntry *E) { return E->getIsFunc(); }, true); @@ -339,7 +405,14 @@ assert(MI.getOperand(0).isReg()); SwitchRegs.insert(MI.getOperand(0).getReg()); } - if (MI.getOpcode() != SPIRV::OpIEqual || !MI.getOperand(2).isReg() || + if (MI.getOpcode() == SPIRV::OpISubS && + SwitchRegs.contains(MI.getOperand(2).getReg())) { + SwitchRegs.insert(MI.getOperand(0).getReg()); + MAI.setSkipEmission(&MI); + } + if ((MI.getOpcode() != SPIRV::OpIEqual && + MI.getOpcode() != SPIRV::OpULessThanEqual) || + !MI.getOperand(2).isReg() || !SwitchRegs.contains(MI.getOperand(2).getReg())) continue; Register CmpReg = MI.getOperand(0).getReg(); @@ -355,6 +428,576 @@ } } +// RequirementHandler implementations. +void SPIRV::RequirementHandler::getAndAddRequirements( + SPIRV::OperandCategory::OperandCategory Category, uint32_t i, + const SPIRVSubtarget &ST) { + addRequirements(getSymbolicOperandRequirements(Category, i, ST, *this)); +} + +void SPIRV::RequirementHandler::pruneCapabilities( + const CapabilityList &ToPrune) { + for (const auto &Cap : ToPrune) { + AllCaps.insert(Cap); + auto FoundIndex = std::find(MinimalCaps.begin(), MinimalCaps.end(), Cap); + if (FoundIndex != MinimalCaps.end()) + MinimalCaps.erase(FoundIndex); + CapabilityList ImplicitDecls = + getSymbolicOperandCapabilities(OperandCategory::CapabilityOperand, Cap); + pruneCapabilities(ImplicitDecls); + } +} + +void SPIRV::RequirementHandler::addCapabilities(const CapabilityList &ToAdd) { + for (const auto &Cap : ToAdd) { + bool IsNewlyInserted = AllCaps.insert(Cap).second; + if (!IsNewlyInserted) // Don't re-add if it's already been declared. + continue; + CapabilityList ImplicitDecls = + getSymbolicOperandCapabilities(OperandCategory::CapabilityOperand, Cap); + pruneCapabilities(ImplicitDecls); + MinimalCaps.push_back(Cap); + } +} + +void SPIRV::RequirementHandler::addRequirements( + const SPIRV::Requirements &Req) { + if (!Req.IsSatisfiable) + report_fatal_error("Adding SPIR-V requirements this target can't satisfy."); + + if (Req.Cap.hasValue()) + addCapabilities({Req.Cap.getValue()}); + + addExtensions(Req.Exts); + + if (Req.MinVer) { + if (MaxVersion && Req.MinVer > MaxVersion) { + LLVM_DEBUG(dbgs() << "Conflicting version requirements: >= " << Req.MinVer + << " and <= " << MaxVersion << "\n"); + report_fatal_error("Adding SPIR-V requirements that can't be satisfied."); + } + + if (MinVersion == 0 || Req.MinVer > MinVersion) + MinVersion = Req.MinVer; + } + + if (Req.MaxVer) { + if (MinVersion && Req.MaxVer < MinVersion) { + LLVM_DEBUG(dbgs() << "Conflicting version requirements: <= " << Req.MaxVer + << " and >= " << MinVersion << "\n"); + report_fatal_error("Adding SPIR-V requirements that can't be satisfied."); + } + + if (MaxVersion == 0 || Req.MaxVer < MaxVersion) + MaxVersion = Req.MaxVer; + } +} + +void SPIRV::RequirementHandler::checkSatisfiable( + const SPIRVSubtarget &ST) const { + // Report as many errors as possible before aborting the compilation. + bool IsSatisfiable = true; + auto TargetVer = ST.getSPIRVVersion(); + + if (MaxVersion && TargetVer && MaxVersion < TargetVer) { + LLVM_DEBUG( + dbgs() << "Target SPIR-V version too high for required features\n" + << "Required max version: " << MaxVersion << " target version " + << TargetVer << "\n"); + IsSatisfiable = false; + } + + if (MinVersion && TargetVer && MinVersion > TargetVer) { + LLVM_DEBUG(dbgs() << "Target SPIR-V version too low for required features\n" + << "Required min version: " << MinVersion + << " target version " << TargetVer << "\n"); + IsSatisfiable = false; + } + + if (MinVersion && MaxVersion && MinVersion > MaxVersion) { + LLVM_DEBUG( + dbgs() + << "Version is too low for some features and too high for others.\n" + << "Required SPIR-V min version: " << MinVersion + << " required SPIR-V max version " << MaxVersion << "\n"); + IsSatisfiable = false; + } + + for (auto Cap : MinimalCaps) { + if (AvailableCaps.contains(Cap)) + continue; + LLVM_DEBUG(dbgs() << "Capability not supported: " + << getSymbolicOperandMnemonic( + OperandCategory::CapabilityOperand, Cap) + << "\n"); + IsSatisfiable = false; + } + + for (auto Ext : AllExtensions) { + if (ST.canUseExtension(Ext)) + continue; + LLVM_DEBUG(dbgs() << "Extension not suported: " + << getSymbolicOperandMnemonic( + OperandCategory::ExtensionOperand, Ext) + << "\n"); + IsSatisfiable = false; + } + + if (!IsSatisfiable) + report_fatal_error("Unable to meet SPIR-V requirements for this target."); +} + +// Add the given capabilities and all their implicitly defined capabilities too. +void SPIRV::RequirementHandler::addAvailableCaps(const CapabilityList &ToAdd) { + for (const auto Cap : ToAdd) + if (AvailableCaps.insert(Cap).second) + addAvailableCaps(getSymbolicOperandCapabilities( + SPIRV::OperandCategory::CapabilityOperand, Cap)); +} + +namespace llvm { +namespace SPIRV { +void RequirementHandler::initAvailableCapabilities(const SPIRVSubtarget &ST) { + // TODO: Implemented for other targets other then OpenCL. + if (!ST.isOpenCLEnv()) + return; + // Add the min requirements for different OpenCL and SPIR-V versions. + addAvailableCaps({Capability::Addresses, Capability::Float16Buffer, + Capability::Int16, Capability::Int8, Capability::Kernel, + Capability::Linkage, Capability::Vector16, + Capability::Groups, Capability::GenericPointer, + Capability::Shader}); + if (ST.hasOpenCLFullProfile()) + addAvailableCaps({Capability::Int64, Capability::Int64Atomics}); + if (ST.hasOpenCLImageSupport()) { + addAvailableCaps({Capability::ImageBasic, Capability::LiteralSampler, + Capability::Image1D, Capability::SampledBuffer, + Capability::ImageBuffer}); + if (ST.isAtLeastOpenCLVer(20)) + addAvailableCaps({Capability::ImageReadWrite}); + } + if (ST.isAtLeastSPIRVVer(11) && ST.isAtLeastOpenCLVer(22)) + addAvailableCaps({Capability::SubgroupDispatch, Capability::PipeStorage}); + if (ST.isAtLeastSPIRVVer(13)) + addAvailableCaps({Capability::GroupNonUniform, + Capability::GroupNonUniformVote, + Capability::GroupNonUniformArithmetic, + Capability::GroupNonUniformBallot, + Capability::GroupNonUniformClustered, + Capability::GroupNonUniformShuffle, + Capability::GroupNonUniformShuffleRelative}); + if (ST.isAtLeastSPIRVVer(14)) + addAvailableCaps({Capability::DenormPreserve, Capability::DenormFlushToZero, + Capability::SignedZeroInfNanPreserve, + Capability::RoundingModeRTE, + Capability::RoundingModeRTZ}); + // TODO: verify if this needs some checks. + addAvailableCaps({Capability::Float16, Capability::Float64}); + + // TODO: add OpenCL extensions. +} +} // namespace SPIRV +} // namespace llvm + +// Add the required capabilities from a decoration instruction (including +// BuiltIns). +static void addOpDecorateReqs(const MachineInstr &MI, unsigned DecIndex, + SPIRV::RequirementHandler &Reqs, + const SPIRVSubtarget &ST) { + int64_t DecOp = MI.getOperand(DecIndex).getImm(); + auto Dec = static_cast(DecOp); + Reqs.addRequirements(getSymbolicOperandRequirements( + SPIRV::OperandCategory::DecorationOperand, Dec, ST, Reqs)); + + if (Dec == SPIRV::Decoration::BuiltIn) { + int64_t BuiltInOp = MI.getOperand(DecIndex + 1).getImm(); + auto BuiltIn = static_cast(BuiltInOp); + Reqs.addRequirements(getSymbolicOperandRequirements( + SPIRV::OperandCategory::BuiltInOperand, BuiltIn, ST, Reqs)); + } +} + +// Add requirements for image handling. +static void addOpTypeImageReqs(const MachineInstr &MI, + SPIRV::RequirementHandler &Reqs, + const SPIRVSubtarget &ST) { + assert(MI.getNumOperands() >= 8 && "Insufficient operands for OpTypeImage"); + // The operand indices used here are based on the OpTypeImage layout, which + // the MachineInstr follows as well. + int64_t ImgFormatOp = MI.getOperand(7).getImm(); + auto ImgFormat = static_cast(ImgFormatOp); + Reqs.getAndAddRequirements(SPIRV::OperandCategory::ImageFormatOperand, + ImgFormat, ST); + + bool IsArrayed = MI.getOperand(4).getImm() == 1; + bool IsMultisampled = MI.getOperand(5).getImm() == 1; + bool NoSampler = MI.getOperand(6).getImm() == 2; + // Add dimension requirements. + assert(MI.getOperand(2).isImm()); + switch (MI.getOperand(2).getImm()) { + case SPIRV::Dim::DIM_1D: + Reqs.addRequirements(NoSampler ? SPIRV::Capability::Image1D + : SPIRV::Capability::Sampled1D); + break; + case SPIRV::Dim::DIM_2D: + if (IsMultisampled && NoSampler) + Reqs.addRequirements(SPIRV::Capability::ImageMSArray); + break; + case SPIRV::Dim::DIM_Cube: + Reqs.addRequirements(SPIRV::Capability::Shader); + if (IsArrayed) + Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageCubeArray + : SPIRV::Capability::SampledCubeArray); + break; + case SPIRV::Dim::DIM_Rect: + Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageRect + : SPIRV::Capability::SampledRect); + break; + case SPIRV::Dim::DIM_Buffer: + Reqs.addRequirements(NoSampler ? SPIRV::Capability::ImageBuffer + : SPIRV::Capability::SampledBuffer); + break; + case SPIRV::Dim::DIM_SubpassData: + Reqs.addRequirements(SPIRV::Capability::InputAttachment); + break; + } + + // Has optional access qualifier. + // TODO: check if it's OpenCL's kernel. + if (MI.getNumOperands() > 8 && + MI.getOperand(8).getImm() == SPIRV::AccessQualifier::ReadWrite) + Reqs.addRequirements(SPIRV::Capability::ImageReadWrite); + else + Reqs.addRequirements(SPIRV::Capability::ImageBasic); +} + +void addInstrRequirements(const MachineInstr &MI, + SPIRV::RequirementHandler &Reqs, + const SPIRVSubtarget &ST) { + switch (MI.getOpcode()) { + case SPIRV::OpMemoryModel: { + int64_t Addr = MI.getOperand(0).getImm(); + Reqs.getAndAddRequirements(SPIRV::OperandCategory::AddressingModelOperand, + Addr, ST); + int64_t Mem = MI.getOperand(1).getImm(); + Reqs.getAndAddRequirements(SPIRV::OperandCategory::MemoryModelOperand, Mem, + ST); + break; + } + case SPIRV::OpEntryPoint: { + int64_t Exe = MI.getOperand(0).getImm(); + Reqs.getAndAddRequirements(SPIRV::OperandCategory::ExecutionModelOperand, + Exe, ST); + break; + } + case SPIRV::OpExecutionMode: + case SPIRV::OpExecutionModeId: { + int64_t Exe = MI.getOperand(1).getImm(); + Reqs.getAndAddRequirements(SPIRV::OperandCategory::ExecutionModeOperand, + Exe, ST); + break; + } + case SPIRV::OpTypeMatrix: + Reqs.addCapability(SPIRV::Capability::Matrix); + break; + case SPIRV::OpTypeInt: { + unsigned BitWidth = MI.getOperand(1).getImm(); + if (BitWidth == 64) + Reqs.addCapability(SPIRV::Capability::Int64); + else if (BitWidth == 16) + Reqs.addCapability(SPIRV::Capability::Int16); + else if (BitWidth == 8) + Reqs.addCapability(SPIRV::Capability::Int8); + break; + } + case SPIRV::OpTypeFloat: { + unsigned BitWidth = MI.getOperand(1).getImm(); + if (BitWidth == 64) + Reqs.addCapability(SPIRV::Capability::Float64); + else if (BitWidth == 16) + Reqs.addCapability(SPIRV::Capability::Float16); + break; + } + case SPIRV::OpTypeVector: { + unsigned NumComponents = MI.getOperand(2).getImm(); + if (NumComponents == 8 || NumComponents == 16) + Reqs.addCapability(SPIRV::Capability::Vector16); + break; + } + case SPIRV::OpTypePointer: { + auto SC = MI.getOperand(1).getImm(); + Reqs.getAndAddRequirements(SPIRV::OperandCategory::StorageClassOperand, SC, + ST); + // If it's a type of pointer to float16, add Float16Buffer capability. + assert(MI.getOperand(2).isReg()); + const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); + SPIRVType *TypeDef = MRI.getVRegDef(MI.getOperand(2).getReg()); + if (TypeDef->getOpcode() == SPIRV::OpTypeFloat && + TypeDef->getOperand(1).getImm() == 16) + Reqs.addCapability(SPIRV::Capability::Float16Buffer); + break; + } + case SPIRV::OpBitReverse: + case SPIRV::OpTypeRuntimeArray: + Reqs.addCapability(SPIRV::Capability::Shader); + break; + case SPIRV::OpTypeOpaque: + case SPIRV::OpTypeEvent: + Reqs.addCapability(SPIRV::Capability::Kernel); + break; + case SPIRV::OpTypePipe: + case SPIRV::OpTypeReserveId: + Reqs.addCapability(SPIRV::Capability::Pipes); + break; + case SPIRV::OpTypeDeviceEvent: + case SPIRV::OpTypeQueue: + Reqs.addCapability(SPIRV::Capability::DeviceEnqueue); + break; + case SPIRV::OpDecorate: + case SPIRV::OpDecorateId: + case SPIRV::OpDecorateString: + addOpDecorateReqs(MI, 1, Reqs, ST); + break; + case SPIRV::OpMemberDecorate: + case SPIRV::OpMemberDecorateString: + addOpDecorateReqs(MI, 2, Reqs, ST); + break; + case SPIRV::OpInBoundsPtrAccessChain: + Reqs.addCapability(SPIRV::Capability::Addresses); + break; + case SPIRV::OpConstantSampler: + Reqs.addCapability(SPIRV::Capability::LiteralSampler); + break; + case SPIRV::OpTypeImage: + addOpTypeImageReqs(MI, Reqs, ST); + break; + case SPIRV::OpTypeSampler: + Reqs.addCapability(SPIRV::Capability::ImageBasic); + break; + case SPIRV::OpTypeForwardPointer: + // TODO: check if it's OpenCL's kernel. + Reqs.addCapability(SPIRV::Capability::Addresses); + break; + case SPIRV::OpAtomicFlagTestAndSet: + case SPIRV::OpAtomicLoad: + case SPIRV::OpAtomicStore: + case SPIRV::OpAtomicExchange: + case SPIRV::OpAtomicCompareExchange: + case SPIRV::OpAtomicIIncrement: + case SPIRV::OpAtomicIDecrement: + case SPIRV::OpAtomicIAdd: + case SPIRV::OpAtomicISub: + case SPIRV::OpAtomicUMin: + case SPIRV::OpAtomicUMax: + case SPIRV::OpAtomicSMin: + case SPIRV::OpAtomicSMax: + case SPIRV::OpAtomicAnd: + case SPIRV::OpAtomicOr: + case SPIRV::OpAtomicXor: { + const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo(); + const MachineInstr *InstrPtr = &MI; + if (MI.getOpcode() == SPIRV::OpAtomicStore) { + assert(MI.getOperand(3).isReg()); + InstrPtr = MRI.getVRegDef(MI.getOperand(3).getReg()); + assert(InstrPtr && "Unexpected type instruction for OpAtomicStore"); + } + assert(InstrPtr->getOperand(1).isReg() && "Unexpected operand in atomic"); + Register TypeReg = InstrPtr->getOperand(1).getReg(); + SPIRVType *TypeDef = MRI.getVRegDef(TypeReg); + if (TypeDef->getOpcode() == SPIRV::OpTypeInt) { + unsigned BitWidth = TypeDef->getOperand(1).getImm(); + if (BitWidth == 64) + Reqs.addCapability(SPIRV::Capability::Int64Atomics); + } + break; + } + case SPIRV::OpGroupNonUniformIAdd: + case SPIRV::OpGroupNonUniformFAdd: + case SPIRV::OpGroupNonUniformIMul: + case SPIRV::OpGroupNonUniformFMul: + case SPIRV::OpGroupNonUniformSMin: + case SPIRV::OpGroupNonUniformUMin: + case SPIRV::OpGroupNonUniformFMin: + case SPIRV::OpGroupNonUniformSMax: + case SPIRV::OpGroupNonUniformUMax: + case SPIRV::OpGroupNonUniformFMax: + case SPIRV::OpGroupNonUniformBitwiseAnd: + case SPIRV::OpGroupNonUniformBitwiseOr: + case SPIRV::OpGroupNonUniformBitwiseXor: + case SPIRV::OpGroupNonUniformLogicalAnd: + case SPIRV::OpGroupNonUniformLogicalOr: + case SPIRV::OpGroupNonUniformLogicalXor: { + assert(MI.getOperand(3).isImm()); + int64_t GroupOp = MI.getOperand(3).getImm(); + switch (GroupOp) { + case SPIRV::GroupOperation::Reduce: + case SPIRV::GroupOperation::InclusiveScan: + case SPIRV::GroupOperation::ExclusiveScan: + Reqs.addCapability(SPIRV::Capability::Kernel); + Reqs.addCapability(SPIRV::Capability::GroupNonUniformArithmetic); + Reqs.addCapability(SPIRV::Capability::GroupNonUniformBallot); + break; + case SPIRV::GroupOperation::ClusteredReduce: + Reqs.addCapability(SPIRV::Capability::GroupNonUniformClustered); + break; + case SPIRV::GroupOperation::PartitionedReduceNV: + case SPIRV::GroupOperation::PartitionedInclusiveScanNV: + case SPIRV::GroupOperation::PartitionedExclusiveScanNV: + Reqs.addCapability(SPIRV::Capability::GroupNonUniformPartitionedNV); + break; + } + break; + } + case SPIRV::OpGroupNonUniformShuffle: + case SPIRV::OpGroupNonUniformShuffleXor: + Reqs.addCapability(SPIRV::Capability::GroupNonUniformShuffle); + break; + case SPIRV::OpGroupNonUniformShuffleUp: + case SPIRV::OpGroupNonUniformShuffleDown: + Reqs.addCapability(SPIRV::Capability::GroupNonUniformShuffleRelative); + break; + case SPIRV::OpGroupAll: + case SPIRV::OpGroupAny: + case SPIRV::OpGroupBroadcast: + case SPIRV::OpGroupIAdd: + case SPIRV::OpGroupFAdd: + case SPIRV::OpGroupFMin: + case SPIRV::OpGroupUMin: + case SPIRV::OpGroupSMin: + case SPIRV::OpGroupFMax: + case SPIRV::OpGroupUMax: + case SPIRV::OpGroupSMax: + Reqs.addCapability(SPIRV::Capability::Groups); + break; + case SPIRV::OpGroupNonUniformElect: + Reqs.addCapability(SPIRV::Capability::GroupNonUniform); + break; + case SPIRV::OpGroupNonUniformAll: + case SPIRV::OpGroupNonUniformAny: + case SPIRV::OpGroupNonUniformAllEqual: + Reqs.addCapability(SPIRV::Capability::GroupNonUniformVote); + break; + case SPIRV::OpGroupNonUniformBroadcast: + case SPIRV::OpGroupNonUniformBroadcastFirst: + case SPIRV::OpGroupNonUniformBallot: + case SPIRV::OpGroupNonUniformInverseBallot: + case SPIRV::OpGroupNonUniformBallotBitExtract: + case SPIRV::OpGroupNonUniformBallotBitCount: + case SPIRV::OpGroupNonUniformBallotFindLSB: + case SPIRV::OpGroupNonUniformBallotFindMSB: + Reqs.addCapability(SPIRV::Capability::GroupNonUniformBallot); + break; + default: + break; + } +} + +static void collectReqs(const Module &M, SPIRV::ModuleAnalysisInfo &MAI, + MachineModuleInfo *MMI, const SPIRVSubtarget &ST) { + // Collect requirements for existing instructions. + for (auto F = M.begin(), E = M.end(); F != E; ++F) { + MachineFunction *MF = MMI->getMachineFunction(*F); + if (!MF) + continue; + for (const MachineBasicBlock &MBB : *MF) + for (const MachineInstr &MI : MBB) + addInstrRequirements(MI, MAI.Reqs, ST); + } + // Collect requirements for OpExecutionMode instructions. + auto Node = M.getNamedMetadata("spirv.ExecutionMode"); + if (Node) { + for (unsigned i = 0; i < Node->getNumOperands(); i++) { + MDNode *MDN = cast(Node->getOperand(i)); + const MDOperand &MDOp = MDN->getOperand(1); + if (auto *CMeta = dyn_cast(MDOp)) { + Constant *C = CMeta->getValue(); + if (ConstantInt *Const = dyn_cast(C)) { + auto EM = Const->getZExtValue(); + MAI.Reqs.getAndAddRequirements( + SPIRV::OperandCategory::ExecutionModeOperand, EM, ST); + } + } + } + } + for (auto FI = M.begin(), E = M.end(); FI != E; ++FI) { + const Function &F = *FI; + if (F.isDeclaration()) + continue; + if (F.getMetadata("reqd_work_group_size")) + MAI.Reqs.getAndAddRequirements( + SPIRV::OperandCategory::ExecutionModeOperand, + SPIRV::ExecutionMode::LocalSize, ST); + if (F.getMetadata("work_group_size_hint")) + MAI.Reqs.getAndAddRequirements( + SPIRV::OperandCategory::ExecutionModeOperand, + SPIRV::ExecutionMode::LocalSizeHint, ST); + if (F.getMetadata("intel_reqd_sub_group_size")) + MAI.Reqs.getAndAddRequirements( + SPIRV::OperandCategory::ExecutionModeOperand, + SPIRV::ExecutionMode::SubgroupSize, ST); + if (F.getMetadata("vec_type_hint")) + MAI.Reqs.getAndAddRequirements( + SPIRV::OperandCategory::ExecutionModeOperand, + SPIRV::ExecutionMode::VecTypeHint, ST); + } +} + +static unsigned getFastMathFlags(const MachineInstr &I) { + unsigned Flags = SPIRV::FPFastMathMode::None; + if (I.getFlag(MachineInstr::MIFlag::FmNoNans)) + Flags |= SPIRV::FPFastMathMode::NotNaN; + if (I.getFlag(MachineInstr::MIFlag::FmNoInfs)) + Flags |= SPIRV::FPFastMathMode::NotInf; + if (I.getFlag(MachineInstr::MIFlag::FmNsz)) + Flags |= SPIRV::FPFastMathMode::NSZ; + if (I.getFlag(MachineInstr::MIFlag::FmArcp)) + Flags |= SPIRV::FPFastMathMode::AllowRecip; + if (I.getFlag(MachineInstr::MIFlag::FmReassoc)) + Flags |= SPIRV::FPFastMathMode::Fast; + return Flags; +} + +static void handleMIFlagDecoration(MachineInstr &I, const SPIRVSubtarget &ST, + const SPIRVInstrInfo &TII, + SPIRV::RequirementHandler &Reqs) { + if (I.getFlag(MachineInstr::MIFlag::NoSWrap) && TII.canUseNSW(I) && + getSymbolicOperandRequirements(SPIRV::OperandCategory::DecorationOperand, + SPIRV::Decoration::NoSignedWrap, ST, Reqs) + .IsSatisfiable) { + buildOpDecorate(I.getOperand(0).getReg(), I, TII, + SPIRV::Decoration::NoSignedWrap, {}); + } + if (I.getFlag(MachineInstr::MIFlag::NoUWrap) && TII.canUseNUW(I) && + getSymbolicOperandRequirements(SPIRV::OperandCategory::DecorationOperand, + SPIRV::Decoration::NoUnsignedWrap, ST, + Reqs) + .IsSatisfiable) { + buildOpDecorate(I.getOperand(0).getReg(), I, TII, + SPIRV::Decoration::NoUnsignedWrap, {}); + } + if (!TII.canUseFastMathFlags(I)) + return; + unsigned FMFlags = getFastMathFlags(I); + if (FMFlags == SPIRV::FPFastMathMode::None) + return; + Register DstReg = I.getOperand(0).getReg(); + buildOpDecorate(DstReg, I, TII, SPIRV::Decoration::FPFastMathMode, {FMFlags}); +} + +// Walk all functions and add decorations related to MI flags. +static void addDecorations(const Module &M, const SPIRVInstrInfo &TII, + MachineModuleInfo *MMI, const SPIRVSubtarget &ST, + SPIRV::ModuleAnalysisInfo &MAI) { + for (auto F = M.begin(), E = M.end(); F != E; ++F) { + MachineFunction *MF = MMI->getMachineFunction(*F); + if (!MF) + continue; + for (auto &MBB : *MF) + for (auto &MI : MBB) + handleMIFlagDecoration(MI, ST, TII, MAI.Reqs); + } +} + struct SPIRV::ModuleAnalysisInfo SPIRVModuleAnalysis::MAI; void SPIRVModuleAnalysis::getAnalysisUsage(AnalysisUsage &AU) const { @@ -373,6 +1016,10 @@ setBaseInfo(M); + addDecorations(M, *TII, MMI, *ST, MAI); + + collectReqs(M, MAI, MMI, *ST); + processSwitches(M, MAI, MMI); // Process type/const/global var/func decl instructions, number their @@ -385,5 +1032,9 @@ // Collect OpName, OpEntryPoint, OpDecorate etc, process other instructions. processOtherInstrs(M); + // If there are no entry points, we need the Linkage capability. + if (MAI.MS[SPIRV::MB_EntryPoints].empty()) + MAI.Reqs.addCapability(SPIRV::Capability::Linkage); + return false; } Index: llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp +++ llvm/lib/Target/SPIRV/SPIRVPreLegalizer.cpp @@ -13,7 +13,6 @@ //===----------------------------------------------------------------------===// #include "SPIRV.h" -#include "SPIRVGlobalRegistry.h" #include "SPIRVSubtarget.h" #include "SPIRVUtils.h" #include "llvm/ADT/PostOrderIterator.h" @@ -208,10 +207,14 @@ // This is to make it convenient for Legalizer to get the SPIRVType // when processing the actual MI (i.e. not pseudo one). GR->assignSPIRVTypeToVReg(SpirvTy, NewReg, MIB.getMF()); + // Copy MIFlags from Def to ASSIGN_TYPE instruction. It's required to keep + // the flags after instruction selection. + const uint16_t Flags = Def->getFlags(); MIB.buildInstr(SPIRV::ASSIGN_TYPE) .addDef(Reg) .addUse(NewReg) - .addUse(GR->getSPIRVTypeID(SpirvTy)); + .addUse(GR->getSPIRVTypeID(SpirvTy)) + .setMIFlags(Flags); Def->getOperand(0).setReg(NewReg); MRI.setRegClass(Reg, &SPIRV::ANYIDRegClass); return NewReg; @@ -409,6 +412,15 @@ // Set the first successor as default MBB to support empty switches. DefaultMBBs[Reg] = *MBB.succ_begin(); } + // Process G_SUB coming from switch range-compare lowering. + if (MI.getOpcode() == TargetOpcode::G_SUB && MI.getOperand(1).isReg() && + SwitchRegs.contains(MI.getOperand(1).getReg())) { + assert(MI.getOperand(0).isReg() && MI.getOperand(1).isReg()); + Register Dst = MI.getOperand(0).getReg(); + SwitchRegs.insert(Dst); + SPIRVType *Ty = GR->getSPIRVTypeForVReg(MI.getOperand(1).getReg()); + insertAssignInstr(Dst, nullptr, Ty, GR, MIB, MRI); + } // Process only ICMPs that relate to spv_switches. if (MI.getOpcode() == TargetOpcode::G_ICMP && MI.getOperand(2).isReg() && SwitchRegs.contains(MI.getOperand(2).getReg())) { @@ -426,8 +438,8 @@ Register CmpReg = MI.getOperand(2).getReg(); MachineOperand &PredOp = MI.getOperand(1); const auto CC = static_cast(PredOp.getPredicate()); - assert(CC == CmpInst::ICMP_EQ && MRI.hasOneUse(Dst) && - MRI.hasOneDef(CmpReg)); + assert((CC == CmpInst::ICMP_EQ || CC == CmpInst::ICMP_ULE) && + MRI.hasOneUse(Dst) && MRI.hasOneDef(CmpReg)); uint64_t Val = getIConstVal(MI.getOperand(3).getReg(), &MRI); MachineInstr *CBr = MRI.use_begin(Dst)->getParent(); assert(CBr->getOpcode() == SPIRV::G_BRCOND && @@ -465,6 +477,8 @@ Register CReg = MI.getOperand(i).getReg(); uint64_t Val = getIConstVal(CReg, &MRI); MachineInstr *ConstInstr = getDefInstrMaybeConstant(CReg, &MRI); + if (!SwitchRegToMBB[Reg][Val]) + continue; Vals.push_back(ConstInstr->getOperand(1).getCImm()); MBBs.push_back(SwitchRegToMBB[Reg][Val]); } @@ -489,8 +503,8 @@ foldConstantsIntoIntrinsics(MF); insertBitcasts(MF, GR, MIB); generateAssignInstrs(MF, GR, MIB); - processInstrsWithTypeFolding(MF, GR, MIB); processSwitches(MF, GR, MIB); + processInstrsWithTypeFolding(MF, GR, MIB); return true; } Index: llvm/lib/Target/SPIRV/SPIRVSubtarget.h =================================================================== --- llvm/lib/Target/SPIRV/SPIRVSubtarget.h +++ llvm/lib/Target/SPIRV/SPIRVSubtarget.h @@ -30,14 +30,15 @@ namespace llvm { class StringRef; -class SPIRVGlobalRegistry; class SPIRVTargetMachine; class SPIRVSubtarget : public SPIRVGenSubtargetInfo { private: const unsigned PointerSize; uint32_t SPIRVVersion; + uint32_t OpenCLVersion; + SmallSet AvailableExtensions; std::unique_ptr GR; SPIRVInstrInfo InstrInfo; @@ -50,6 +51,10 @@ std::unique_ptr Legalizer; std::unique_ptr InstSelector; + // TODO: Initialise the available extensions based on + // the environment settings. + void initAvailableExtensions(); + public: // This constructor initializes the data members to match that // of the specified triple. @@ -62,7 +67,18 @@ void ParseSubtargetFeatures(StringRef CPU, StringRef TuneCPU, StringRef FS); unsigned getPointerSize() const { return PointerSize; } bool canDirectlyComparePointers() const; + // TODO: this environment is not implemented in Triple, we need to decide + // how to standartize its support. For now, let's assume that we always + // operate with OpenCL. + bool isOpenCLEnv() const { return true; } uint32_t getSPIRVVersion() const { return SPIRVVersion; }; + bool isAtLeastSPIRVVer(uint32_t VerToCompareTo) const; + bool isAtLeastOpenCLVer(uint32_t VerToCompareTo) const; + // TODO: implement command line args or other ways to determine this. + bool hasOpenCLFullProfile() const { return true; } + bool hasOpenCLImageSupport() const { return true; } + bool canUseExtension(SPIRV::Extension::Extension E) const; + SPIRVGlobalRegistry *getSPIRVGlobalRegistry() const { return GR.get(); } const CallLowering *getCallLowering() const override { Index: llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp +++ llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp @@ -43,8 +43,10 @@ const std::string &FS, const SPIRVTargetMachine &TM) : SPIRVGenSubtargetInfo(TT, CPU, /*TuneCPU=*/CPU, FS), - PointerSize(computePointerSize(TT)), SPIRVVersion(0), InstrInfo(), - FrameLowering(initSubtargetDependencies(CPU, FS)), TLInfo(TM, *this) { + PointerSize(computePointerSize(TT)), SPIRVVersion(0), OpenCLVersion(0), + InstrInfo(), FrameLowering(initSubtargetDependencies(CPU, FS)), + TLInfo(TM, *this) { + initAvailableExtensions(); GR = std::make_unique(PointerSize); CallLoweringInfo = std::make_unique(TLInfo, GR.get()); Legalizer = std::make_unique(*this); @@ -58,10 +60,34 @@ ParseSubtargetFeatures(CPU, /*TuneCPU=*/CPU, FS); if (SPIRVVersion == 0) SPIRVVersion = 14; + if (OpenCLVersion == 0) + OpenCLVersion = 22; return *this; } +bool SPIRVSubtarget::canUseExtension(SPIRV::Extension::Extension E) const { + return AvailableExtensions.contains(E); +} + +bool SPIRVSubtarget::isAtLeastSPIRVVer(uint32_t VerToCompareTo) const { + return isAtLeastVer(SPIRVVersion, VerToCompareTo); +} + +bool SPIRVSubtarget::isAtLeastOpenCLVer(uint32_t VerToCompareTo) const { + return isAtLeastVer(OpenCLVersion, VerToCompareTo); +} + // If the SPIR-V version is >= 1.4 we can call OpPtrEqual and OpPtrNotEqual. bool SPIRVSubtarget::canDirectlyComparePointers() const { return isAtLeastVer(SPIRVVersion, 14); } + +// TODO: use command line args for this rather than defaults. +void SPIRVSubtarget::initAvailableExtensions() { + AvailableExtensions.clear(); + if (!isOpenCLEnv()) + return; + // A default extension for testing. + AvailableExtensions.insert( + SPIRV::Extension::SPV_KHR_no_integer_wrap_decoration); +}