Index: llvm/include/llvm/IR/IntrinsicsSPIRV.td =================================================================== --- llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -29,4 +29,6 @@ def int_spv_bitcast : Intrinsic<[llvm_any_ty], [llvm_any_ty]>; def int_spv_switch : Intrinsic<[], [llvm_any_ty, llvm_vararg_ty]>; def int_spv_cmpxchg : Intrinsic<[llvm_i32_ty], [llvm_any_ty, llvm_vararg_ty]>; + def int_spv_unreachable : Intrinsic<[], []>; + def int_spv_alloca : Intrinsic<[llvm_any_ty], []>; } Index: llvm/lib/Target/SPIRV/SPIRVBuiltins.h =================================================================== --- llvm/lib/Target/SPIRV/SPIRVBuiltins.h +++ llvm/lib/Target/SPIRV/SPIRVBuiltins.h @@ -22,19 +22,33 @@ /// Lowers a builtin funtion call using the provided \p DemangledCall skeleton /// and external instruction \p Set. /// -/// \return a pair of boolean values, the first true means the call recognized +/// \return A pair of boolean values, the first true means the call recognized /// as a builtin, the second one indicates the successful lowering. /// /// \p DemangledCall is the skeleton of the lowered builtin function call. /// \p Set is the external instruction set containing the given builtin. /// \p OrigRet is the single original virtual return register if defined, -/// Register(0) otherwise. \p OrigRetTy is the type of the \p OrigRet. \p Args -/// are the arguments of the lowered builtin call. +/// Register(0) otherwise. +/// \p OrigRetTy is the type of the \p OrigRet. +/// \p Args are the arguments of the lowered builtin call. std::pair lowerBuiltin(const StringRef DemangledCall, InstructionSet::InstructionSet Set, MachineIRBuilder &MIRBuilder, const Register OrigRet, const Type *OrigRetTy, const SmallVectorImpl &Args, SPIRVGlobalRegistry *GR); + +/// Handles the translation of the provided special opaque/builtin type \p Type +/// to SPIR-V type. Generates the corresponding machine instructions for the +/// target type or gets the already existing OpType<...> register from the +/// global registry \p GR. +/// +/// \return A machine instruction representing the OpType<...> SPIR-V type. +/// +/// \p Type is the special opaque/builtin type to be lowered. +SPIRVType *lowerBuiltinType(const StructType *Type, + AccessQualifier::AccessQualifier AccessQual, + MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR); } // namespace SPIRV } // namespace llvm #endif // LLVM_LIB_TARGET_SPIRV_SPIRVBUILTINS_H Index: llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp +++ llvm/lib/Target/SPIRV/SPIRVBuiltins.cpp @@ -1613,5 +1613,236 @@ } return {true, false}; } + +struct DemangledType { + StringRef Name; + uint32_t Opcode; +}; + +#define GET_DemangledTypes_DECL +#define GET_DemangledTypes_IMPL + +struct ImageType { + StringRef Name; + StringRef SampledType; + AccessQualifier::AccessQualifier Qualifier; + Dim::Dim Dimensionality; + bool Arrayed; + bool Depth; + bool Multisampled; + bool Sampled; + ImageFormat::ImageFormat Format; +}; + +struct PipeType { + StringRef Name; + AccessQualifier::AccessQualifier Qualifier; +}; + +using namespace AccessQualifier; +using namespace Dim; +using namespace ImageFormat; +#define GET_ImageTypes_DECL +#define GET_ImageTypes_IMPL +#define GET_PipeTypes_DECL +#define GET_PipeTypes_IMPL +#include "SPIRVGenTables.inc" +} // namespace SPIRV + +//===----------------------------------------------------------------------===// +// Misc functions for parsing builtin types and looking up implementation +// details in TableGenerated tables. +//===----------------------------------------------------------------------===// + +static const SPIRV::DemangledType *findBuiltinType(StringRef Name) { + if (Name.startswith("opencl.")) + return SPIRV::lookupBuiltinType(Name); + if (Name.startswith("spirv.")) { + // Some SPIR-V builtin types have a complex list of parameters as part of + // their name (e.g. spirv.Image._void_1_0_0_0_0_0_0). Those parameters often + // are numeric literals which cannot be easily represented by TableGen + // records and should be parsed instead. + unsigned BaseTypeNameLength = + Name.contains('_') ? Name.find('_') - 1 : Name.size(); + return SPIRV::lookupBuiltinType(Name.substr(0, BaseTypeNameLength).str()); + } + return nullptr; +} + +static std::unique_ptr +lookupOrParseBuiltinImageType(StringRef Name) { + if (Name.startswith("opencl.")) { + // Lookup OpenCL builtin image type lowering details in TableGen records. + const SPIRV::ImageType *Record = SPIRV::lookupImageType(Name); + return std::unique_ptr(new SPIRV::ImageType(*Record)); + } + if (Name.startswith("spirv.")) { + // Parse the literals of SPIR-V image builtin parameters. The name should + // have the following format: + // spirv.Image._Type_Dim_Depth_Arrayed_MS_Sampled_ImageFormat_AccessQualifier + // e.g. %spirv.Image._void_1_0_0_0_0_0_0 + StringRef TypeParametersString = Name.substr(strlen("spirv.Image.")); + SmallVector TypeParameters; + SplitString(TypeParametersString, TypeParameters, "_"); + assert(TypeParameters.size() == 8 && + "Wrong number of literals in SPIR-V builtin image type"); + + StringRef SampledType = TypeParameters[0]; + unsigned Dim, Depth, Arrayed, Multisampled, Sampled, Format, AccessQual; + bool AreParameterLiteralsValid = + !(TypeParameters[1].getAsInteger(10, Dim) || + TypeParameters[2].getAsInteger(10, Depth) || + TypeParameters[3].getAsInteger(10, Arrayed) || + TypeParameters[4].getAsInteger(10, Multisampled) || + TypeParameters[5].getAsInteger(10, Sampled) || + TypeParameters[6].getAsInteger(10, Format) || + TypeParameters[7].getAsInteger(10, AccessQual)); + assert(AreParameterLiteralsValid && + "Invalid format of SPIR-V image type parameter literals."); + + return std::unique_ptr(new SPIRV::ImageType{ + Name, SampledType, SPIRV::AccessQualifier::AccessQualifier(AccessQual), + SPIRV::Dim::Dim(Dim), static_cast(Arrayed), + static_cast(Depth), static_cast(Multisampled), + static_cast(Sampled), SPIRV::ImageFormat::ImageFormat(Format)}); + } + llvm_unreachable("Unknown builtin image type name/literal"); +} + +static std::unique_ptr +lookupOrParseBuiltinPipeType(StringRef Name) { + if (Name.startswith("opencl.")) { + // Lookup OpenCL builtin pipe type lowering details in TableGen records. + const SPIRV::PipeType *Record = SPIRV::lookupPipeType(Name); + return std::unique_ptr(new SPIRV::PipeType(*Record)); + } + if (Name.startswith("spirv.")) { + // Parse the access qualifier literal in the name of the SPIR-V pipe type. + // The name should have the following format: + // spirv.Pipe._AccessQualifier + // e.g. %spirv.Pipe._1 + if (Name.endswith("_0")) + return std::unique_ptr( + new SPIRV::PipeType{Name, SPIRV::AccessQualifier::ReadOnly}); + if (Name.endswith("_1")) + return std::unique_ptr( + new SPIRV::PipeType{Name, SPIRV::AccessQualifier::WriteOnly}); + if (Name.endswith("_2")) + return std::unique_ptr( + new SPIRV::PipeType{Name, SPIRV::AccessQualifier::ReadWrite}); + llvm_unreachable("Unknown pipe type access qualifier literal"); + } + llvm_unreachable("Unknown builtin pipe type name/literal"); +} + +//===----------------------------------------------------------------------===// +// Implementation functions for builtin types. +//===----------------------------------------------------------------------===// + +SPIRVType *getNonParametrizedType(const StructType *OpaqueType, + const SPIRV::DemangledType *TypeRecord, + MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR) { + unsigned Opcode = TypeRecord->Opcode; + // Create or get an existing type from GlobalRegistry. + return GR->getOrCreateOpTypeByOpcode(OpaqueType, MIRBuilder, Opcode); +} + +SPIRVType *getSamplerType(MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR) { + // Create or get an existing type from GlobalRegistry. + return GR->getOrCreateOpTypeSampler(MIRBuilder); +} + +SPIRVType *getPipeType(const StructType *OpaqueType, + MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { + // Lookup pipe type lowering details in TableGen records or parse the + // name/literal for details. + std::unique_ptr Record = + lookupOrParseBuiltinPipeType(OpaqueType->getName()); + // Create or get an existing type from GlobalRegistry. + return GR->getOrCreateOpTypePipe(MIRBuilder, Record.get()->Qualifier); +} + +SPIRVType *getImageType(const StructType *OpaqueType, + SPIRV::AccessQualifier::AccessQualifier AccessQual, + MachineIRBuilder &MIRBuilder, SPIRVGlobalRegistry *GR) { + // Lookup image type lowering details in TableGen records or parse the + // name/literal for details. + std::unique_ptr Record = + lookupOrParseBuiltinImageType(OpaqueType->getName()); + + SPIRVType *SampledType = + GR->getOrCreateSPIRVTypeByName(Record.get()->SampledType, MIRBuilder); + return GR->getOrCreateOpTypeImage( + MIRBuilder, SampledType, Record.get()->Dimensionality, + Record.get()->Depth, Record.get()->Arrayed, Record.get()->Multisampled, + Record.get()->Sampled, Record.get()->Format, + AccessQual == SPIRV::AccessQualifier::WriteOnly + ? SPIRV::AccessQualifier::WriteOnly + : Record.get()->Qualifier); +} + +SPIRVType *getSampledImageType(const StructType *OpaqueType, + MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR) { + StringRef TypeParametersString = + OpaqueType->getName().substr(strlen("spirv.SampledImage.")); + LLVMContext &Context = MIRBuilder.getMF().getFunction().getContext(); + Type *ImageOpaqueType = StructType::getTypeByName( + Context, "spirv.Image." + TypeParametersString.str()); + SPIRVType *TargetImageType = + GR->getOrCreateSPIRVType(ImageOpaqueType, MIRBuilder); + return GR->getOrCreateOpTypeSampledImage(TargetImageType, MIRBuilder); +} + +namespace SPIRV { +SPIRVType *lowerBuiltinType(const StructType *OpaqueType, + AccessQualifier::AccessQualifier AccessQual, + MachineIRBuilder &MIRBuilder, + SPIRVGlobalRegistry *GR) { + assert(OpaqueType->hasName() && + "Structs representing builtin types must have a parsable name"); + unsigned NumStartingVRegs = MIRBuilder.getMRI()->getNumVirtRegs(); + + const StringRef Name = OpaqueType->getName(); + LLVM_DEBUG(dbgs() << "Lowering builtin type: " << Name << "\n"); + + // Lookup the demangled builtin type in the TableGen records. + const SPIRV::DemangledType *TypeRecord = findBuiltinType(Name); + if (!TypeRecord) + report_fatal_error("Missing TableGen record for builtin type: " + Name); + + // "Lower" the BuiltinType into TargetType. The following get<...>Type methods + // use the implementation details from TableGen records to either create a new + // OpType<...> machine instruction or get an existing equivalent SPIRVType + // from GlobalRegistry. + SPIRVType *TargetType; + switch (TypeRecord->Opcode) { + case SPIRV::OpTypeImage: + TargetType = getImageType(OpaqueType, AccessQual, MIRBuilder, GR); + break; + case SPIRV::OpTypePipe: + TargetType = getPipeType(OpaqueType, MIRBuilder, GR); + break; + case SPIRV::OpTypeSampler: + TargetType = getSamplerType(MIRBuilder, GR); + break; + case SPIRV::OpTypeSampledImage: + TargetType = getSampledImageType(OpaqueType, MIRBuilder, GR); + break; + default: + TargetType = getNonParametrizedType(OpaqueType, TypeRecord, MIRBuilder, GR); + break; + } + + // Emit OpName instruction if a new OpType<...> instruction was added + // (equivalent type was not found in GlobalRegistry). + if (NumStartingVRegs < MIRBuilder.getMRI()->getNumVirtRegs()) + buildOpName(GR->getSPIRVTypeID(TargetType), OpaqueType->getName(), + MIRBuilder); + + return TargetType; +} } // namespace SPIRV } // namespace llvm Index: llvm/lib/Target/SPIRV/SPIRVBuiltins.td =================================================================== --- llvm/lib/Target/SPIRV/SPIRVBuiltins.td +++ llvm/lib/Target/SPIRV/SPIRVBuiltins.td @@ -1104,22 +1104,36 @@ } // Function to lookup builtin types by their demangled name. -def lookupType : SearchIndex { +def lookupBuiltinType : SearchIndex { let Table = DemangledTypes; let Key = ["Name"]; } -// OpenCL builtin types: def : DemangledType<"opencl.reserve_id_t", OpTypeReserveId>; def : DemangledType<"opencl.event_t", OpTypeEvent>; def : DemangledType<"opencl.queue_t", OpTypeQueue>; def : DemangledType<"opencl.sampler_t", OpTypeSampler>; def : DemangledType<"opencl.clk_event_t", OpTypeDeviceEvent>; -def : DemangledType<"opencl.clk_event_t", OpTypeDeviceEvent>; + +def : DemangledType<"spirv.ReserveId", OpTypeReserveId>; +def : DemangledType<"spirv.PipeStorage", OpTypePipeStorage>; +def : DemangledType<"spirv.Queue", OpTypeQueue>; +def : DemangledType<"spirv.Event", OpTypeEvent>; +def : DemangledType<"spirv.Sampler", OpTypeSampler>; +def : DemangledType<"spirv.DeviceEvent", OpTypeDeviceEvent>; + +// Some SPIR-V builtin types (e.g. spirv.Image) have a complex list of +// parameters as part of their name. Some of those parameters should be treated +// as numeric literals and therefore they cannot be represented in TableGen and +// should be parsed instead. +def : DemangledType<"spirv.Image", OpTypeImage>; +def : DemangledType<"spirv.SampledImage", OpTypeSampledImage>; +def : DemangledType<"spirv.Pipe", OpTypePipe>; // Class definining lowering details for various variants of image type indentifiers. class ImageType { string Name = name; + string Type = "void"; AccessQualifier Qualifier = !cond(!not(!eq(!find(name, "_ro_t"), -1)) : ReadOnly, !not(!eq(!find(name, "_wo_t"), -1)) : WriteOnly, !not(!eq(!find(name, "_rw_t"), -1)) : ReadWrite, @@ -1130,14 +1144,19 @@ !not(!eq(!find(name, "image3"), -1)) : DIM_3D); bit Arrayed = !not(!eq(!find(name, "array"), -1)); bit Depth = !not(!eq(!find(name, "depth"), -1)); + bit Multisampled = false; + bit Sampled = false; + ImageFormat Format = Unknown; } // Table gathering all the image type records. def ImageTypes : GenericTable { let FilterClass = "ImageType"; - let Fields = ["Name", "Qualifier", "Dimensionality", "Arrayed", "Depth"]; + let Fields = ["Name", "Type", "Qualifier", "Dimensionality", "Arrayed", + "Depth", "Multisampled", "Sampled", "Format"]; string TypeOf_Qualifier = "AccessQualifier"; string TypeOf_Dimensionality = "Dim"; + string TypeOf_Format = "ImageFormat"; } // Function to lookup builtin image types by their demangled name. Index: llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -88,6 +88,7 @@ Instruction *visitStoreInst(StoreInst &I); Instruction *visitAllocaInst(AllocaInst &I); Instruction *visitAtomicCmpXchgInst(AtomicCmpXchgInst &I); + Instruction *visitUnreachableInst(UnreachableInst &I); bool runOnFunction(Function &F) override; }; } // namespace @@ -313,7 +314,13 @@ Instruction *SPIRVEmitIntrinsics::visitAllocaInst(AllocaInst &I) { TrackConstants = false; - return &I; + Type *PtrTy = I.getType(); + auto *NewI = IRB->CreateIntrinsic(Intrinsic::spv_alloca, {PtrTy}, {}); + std::string InstName = I.hasName() ? I.getName().str() : ""; + I.replaceAllUsesWith(NewI); + I.eraseFromParent(); + NewI->setName(InstName); + return NewI; } Instruction *SPIRVEmitIntrinsics::visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { @@ -332,6 +339,12 @@ return NewI; } +Instruction *SPIRVEmitIntrinsics::visitUnreachableInst(UnreachableInst &I) { + IRB->SetInsertPoint(&I); + IRB->CreateIntrinsic(Intrinsic::spv_unreachable, {}, {}); + return &I; +} + void SPIRVEmitIntrinsics::processGlobalValue(GlobalVariable &GV) { // Skip special artifical variable llvm.global.annotations. if (GV.getName() == "llvm.global.annotations") @@ -368,7 +381,7 @@ if (isa(Op) || isa(Op) || // Check GetElementPtrConstantExpr case. (isa(Op) && isa(Op))) { - IRB->SetInsertPoint(I); + setInsertPointSkippingPhis(*IRB, I); if (isa(Op) && Op->getType()->isAggregateType()) buildIntrWithMD(Intrinsic::spv_assign_type, {IRB->getInt32Ty()}, Op, UndefValue::get(IRB->getInt32Ty())); Index: llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h =================================================================== --- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h +++ llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.h @@ -208,6 +208,11 @@ SPIRVType *getOpTypeFunction(SPIRVType *RetType, const SmallVectorImpl &ArgTypes, MachineIRBuilder &MIRBuilder); + + SPIRVType * + getOrCreateSpecialType(const Type *Ty, MachineIRBuilder &MIRBuilder, + SPIRV::AccessQualifier::AccessQualifier AccQual); + std::tuple getOrCreateConstIntReg( uint64_t Val, SPIRVType *SpvType, MachineIRBuilder *MIRBuilder, MachineInstr *I = nullptr, const SPIRVInstrInfo *TII = nullptr); @@ -240,7 +245,6 @@ SPIRVType *SpvType, bool EmitIR = true); Register getOrCreateConsIntArray(uint64_t Val, MachineIRBuilder &MIRBuilder, SPIRVType *SpvType, bool EmitIR = true); - Register buildConstantSampler(Register Res, unsigned AddrMode, unsigned Param, unsigned FilerMode, MachineIRBuilder &MIRBuilder, @@ -270,19 +274,39 @@ SPIRVType *getOrCreateSPIRVVectorType(SPIRVType *BaseType, unsigned NumElements, MachineInstr &I, const SPIRVInstrInfo &TII); + SPIRVType *getOrCreateSPIRVArrayType(SPIRVType *BaseType, + unsigned NumElements, MachineInstr &I, + const SPIRVInstrInfo &TII); + SPIRVType *getOrCreateSPIRVPointerType( SPIRVType *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SClass = SPIRV::StorageClass::Function); SPIRVType *getOrCreateSPIRVPointerType( SPIRVType *BaseType, MachineInstr &I, const SPIRVInstrInfo &TII, SPIRV::StorageClass::StorageClass SClass = SPIRV::StorageClass::Function); + + SPIRVType * + getOrCreateOpTypeImage(MachineIRBuilder &MIRBuilder, SPIRVType *SampledType, + SPIRV::Dim::Dim Dim, uint32_t Depth, uint32_t Arrayed, + uint32_t Multisampled, uint32_t Sampled, + SPIRV::ImageFormat::ImageFormat ImageFormat, + SPIRV::AccessQualifier::AccessQualifier AccQual); + + SPIRVType *getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder); + SPIRVType *getOrCreateOpTypeSampledImage(SPIRVType *ImageType, MachineIRBuilder &MIRBuilder); + SPIRVType * + getOrCreateOpTypePipe(MachineIRBuilder &MIRBuilder, + SPIRV::AccessQualifier::AccessQualifier AccQual); SPIRVType *getOrCreateOpTypeFunctionWithArgs( const Type *Ty, SPIRVType *RetType, const SmallVectorImpl &ArgTypes, MachineIRBuilder &MIRBuilder); + SPIRVType *getOrCreateOpTypeByOpcode(const Type *Ty, + MachineIRBuilder &MIRBuilder, + unsigned Opcode); }; } // end namespace llvm #endif // LLLVM_LIB_TARGET_SPIRV_SPIRVTYPEMANAGER_H Index: llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -16,6 +16,7 @@ #include "SPIRVGlobalRegistry.h" #include "SPIRV.h" +#include "SPIRVBuiltins.h" #include "SPIRVSubtarget.h" #include "SPIRVTargetMachine.h" #include "SPIRVUtils.h" @@ -479,10 +480,10 @@ if (IsConst) buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Constant, {}); - if (GVar && GVar->getAlign().valueOrOne().value() != 1) - buildOpDecorate( - Reg, MIRBuilder, SPIRV::Decoration::Alignment, - {static_cast(GVar->getAlign().valueOrOne().value())}); + if (GVar && GVar->getAlign().valueOrOne().value() != 1) { + unsigned Alignment = (unsigned)GVar->getAlign().valueOrOne().value(); + buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::Alignment, {Alignment}); + } if (HasLinkageTy) buildOpDecorate(Reg, MIRBuilder, SPIRV::Decoration::LinkageAttributes, @@ -563,6 +564,20 @@ return false; } +SPIRVType *SPIRVGlobalRegistry::getOrCreateSpecialType( + const Type *Ty, MachineIRBuilder &MIRBuilder, + SPIRV::AccessQualifier::AccessQualifier AccQual) { + // Some OpenCL and SPIRV builtins like image2d_t are passed in as + // pointers, but should be treated as custom types like OpTypeImage. + if (auto PType = dyn_cast(Ty)) { + assert(!PType->isOpaque()); + Ty = PType->getNonOpaquePointerElementType(); + } + auto SType = cast(Ty); + assert(isOpenCLBuiltinType(SType) || isSPIRVBuiltinType(SType)); + return SPIRV::lowerBuiltinType(SType, AccQual, MIRBuilder, this); +} + SPIRVType *SPIRVGlobalRegistry::getOpTypePointer( SPIRV::StorageClass::StorageClass SC, SPIRVType *ElemType, MachineIRBuilder &MIRBuilder, Register Reg) { @@ -624,7 +639,8 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType( const Type *Ty, MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AccQual, bool EmitIR) { - assert(!isSpecialType(Ty)); + if (isSpecialType(Ty)) + return getOrCreateSpecialType(Ty, MIRBuilder, AccQual); auto &TypeToSPIRVTypeMap = DT.getTypes()->getAllUses(); auto t = TypeToSPIRVTypeMap.find(Ty); if (t != TypeToSPIRVTypeMap.end()) { @@ -729,7 +745,7 @@ const Type *Ty, MachineIRBuilder &MIRBuilder, SPIRV::AccessQualifier::AccessQualifier AccessQual, bool EmitIR) { Register Reg = DT.find(Ty, &MIRBuilder.getMF()); - if (Reg.isValid()) + if (Reg.isValid() && !isSpecialType(Ty)) return getSPIRVTypeForVReg(Reg); TypesInProcessing.clear(); SPIRVType *STy = restOfCreateSPIRVType(Ty, MIRBuilder, AccessQual, EmitIR); @@ -804,6 +820,53 @@ Type->getOperand(1).getImm()); } +SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeImage( + MachineIRBuilder &MIRBuilder, SPIRVType *SampledType, SPIRV::Dim::Dim Dim, + uint32_t Depth, uint32_t Arrayed, uint32_t Multisampled, uint32_t Sampled, + SPIRV::ImageFormat::ImageFormat ImageFormat, + SPIRV::AccessQualifier::AccessQualifier AccessQual) { + SPIRV::ImageTypeDescriptor TD(SPIRVToLLVMType.lookup(SampledType), Dim, Depth, + Arrayed, Multisampled, Sampled, ImageFormat, + AccessQual); + if (auto *Res = checkSpecialInstr(TD, MIRBuilder)) + return Res; + Register ResVReg = createTypeVReg(MIRBuilder); + DT.add(TD, &MIRBuilder.getMF(), ResVReg); + return MIRBuilder.buildInstr(SPIRV::OpTypeImage) + .addDef(ResVReg) + .addUse(getSPIRVTypeID(SampledType)) + .addImm(Dim) + .addImm(Depth) // Depth (whether or not it is a Depth image). + .addImm(Arrayed) // Arrayed. + .addImm(Multisampled) // Multisampled (0 = only single-sample). + .addImm(Sampled) // Sampled (0 = usage known at runtime). + .addImm(ImageFormat) + .addImm(AccessQual); +} + +SPIRVType * +SPIRVGlobalRegistry::getOrCreateOpTypeSampler(MachineIRBuilder &MIRBuilder) { + SPIRV::SamplerTypeDescriptor TD; + if (auto *Res = checkSpecialInstr(TD, MIRBuilder)) + return Res; + Register ResVReg = createTypeVReg(MIRBuilder); + DT.add(TD, &MIRBuilder.getMF(), ResVReg); + return MIRBuilder.buildInstr(SPIRV::OpTypeSampler).addDef(ResVReg); +} + +SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypePipe( + MachineIRBuilder &MIRBuilder, + SPIRV::AccessQualifier::AccessQualifier AccessQual) { + SPIRV::PipeTypeDescriptor TD(AccessQual); + if (auto *Res = checkSpecialInstr(TD, MIRBuilder)) + return Res; + Register ResVReg = createTypeVReg(MIRBuilder); + DT.add(TD, &MIRBuilder.getMF(), ResVReg); + return MIRBuilder.buildInstr(SPIRV::OpTypePipe) + .addDef(ResVReg) + .addImm(AccessQual); +} + SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeSampledImage( SPIRVType *ImageType, MachineIRBuilder &MIRBuilder) { SPIRV::SampledImageTypeDescriptor TD( @@ -813,11 +876,20 @@ if (auto *Res = checkSpecialInstr(TD, MIRBuilder)) return Res; Register ResVReg = createTypeVReg(MIRBuilder); - auto MIB = MIRBuilder.buildInstr(SPIRV::OpTypeSampledImage) - .addDef(ResVReg) - .addUse(getSPIRVTypeID(ImageType)); DT.add(TD, &MIRBuilder.getMF(), ResVReg); - return MIB; + return MIRBuilder.buildInstr(SPIRV::OpTypeSampledImage) + .addDef(ResVReg) + .addUse(getSPIRVTypeID(ImageType)); +} + +SPIRVType *SPIRVGlobalRegistry::getOrCreateOpTypeByOpcode( + const Type *Ty, MachineIRBuilder &MIRBuilder, unsigned Opcode) { + Register ResVReg = DT.find(Ty, &MIRBuilder.getMF()); + if (ResVReg.isValid()) + return MIRBuilder.getMF().getRegInfo().getUniqueVRegDef(ResVReg); + ResVReg = createTypeVReg(MIRBuilder); + DT.add(Ty, &MIRBuilder.getMF(), ResVReg); + return MIRBuilder.buildInstr(Opcode).addDef(ResVReg); } const MachineInstr * @@ -942,6 +1014,24 @@ return finishCreatingSPIRVType(LLVMTy, MIB); } +SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVArrayType( + SPIRVType *BaseType, unsigned NumElements, MachineInstr &I, + const SPIRVInstrInfo &TII) { + Type *LLVMTy = ArrayType::get( + const_cast(getTypeForSPIRVType(BaseType)), NumElements); + Register Reg = DT.find(LLVMTy, CurMF); + if (Reg.isValid()) + return getSPIRVTypeForVReg(Reg); + MachineBasicBlock &BB = *I.getParent(); + SPIRVType *SpirvType = getOrCreateSPIRVIntegerType(32, I, TII); + Register Len = getOrCreateConstInt(NumElements, I, SpirvType, TII); + auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpTypeArray)) + .addDef(createTypeVReg(CurMF->getRegInfo())) + .addUse(getSPIRVTypeID(BaseType)) + .addUse(Len); + return finishCreatingSPIRVType(LLVMTy, MIB); +} + SPIRVType *SPIRVGlobalRegistry::getOrCreateSPIRVPointerType( SPIRVType *BaseType, MachineIRBuilder &MIRBuilder, SPIRV::StorageClass::StorageClass SClass) { Index: llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -30,6 +30,11 @@ #define DEBUG_TYPE "spirv-isel" using namespace llvm; +namespace CL = SPIRV::OpenCLExtInst; +namespace GL = SPIRV::GLSLExtInst; + +using ExtInstList = + std::vector>; namespace { @@ -132,9 +137,8 @@ bool selectTrunc(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; - bool selectIntToBool(Register IntReg, Register ResVReg, - const SPIRVType *intTy, const SPIRVType *boolTy, - MachineInstr &I) const; + bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I, + const SPIRVType *intTy, const SPIRVType *boolTy) const; bool selectOpUndef(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; @@ -160,6 +164,14 @@ bool selectPhi(Register ResVReg, const SPIRVType *ResType, MachineInstr &I) const; + bool selectExtInst(Register ResVReg, const SPIRVType *ResType, + MachineInstr &I, CL::OpenCLExtInst CLInst) const; + bool selectExtInst(Register ResVReg, const SPIRVType *ResType, + MachineInstr &I, CL::OpenCLExtInst CLInst, + GL::GLSLExtInst GLInst) const; + bool selectExtInst(Register ResVReg, const SPIRVType *ResType, + MachineInstr &I, const ExtInstList &ExtInsts) const; + Register buildI32Constant(uint32_t Val, MachineInstr &I, const SPIRVType *ResType = nullptr) const; @@ -283,6 +295,7 @@ } case TargetOpcode::G_MEMMOVE: case TargetOpcode::G_MEMCPY: + case TargetOpcode::G_MEMSET: return selectMemOperation(ResVReg, I); case TargetOpcode::G_ICMP: @@ -318,6 +331,85 @@ case TargetOpcode::G_CTPOP: return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount); + case TargetOpcode::G_SMIN: + return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin); + case TargetOpcode::G_UMIN: + return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin); + + case TargetOpcode::G_SMAX: + return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax); + case TargetOpcode::G_UMAX: + return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax); + + case TargetOpcode::G_FMA: + return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma); + + case TargetOpcode::G_FPOW: + return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow); + case TargetOpcode::G_FPOWI: + return selectExtInst(ResVReg, ResType, I, CL::pown); + + case TargetOpcode::G_FEXP: + return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp); + case TargetOpcode::G_FEXP2: + return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2); + + case TargetOpcode::G_FLOG: + return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log); + case TargetOpcode::G_FLOG2: + return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2); + case TargetOpcode::G_FLOG10: + return selectExtInst(ResVReg, ResType, I, CL::log10); + + case TargetOpcode::G_FABS: + return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs); + case TargetOpcode::G_ABS: + return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs); + + case TargetOpcode::G_FMINNUM: + case TargetOpcode::G_FMINIMUM: + return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::FMin); + case TargetOpcode::G_FMAXNUM: + case TargetOpcode::G_FMAXIMUM: + return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::FMax); + + case TargetOpcode::G_FCOPYSIGN: + return selectExtInst(ResVReg, ResType, I, CL::copysign); + + case TargetOpcode::G_FCEIL: + return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil); + case TargetOpcode::G_FFLOOR: + return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor); + + case TargetOpcode::G_FCOS: + return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos); + case TargetOpcode::G_FSIN: + return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin); + + case TargetOpcode::G_FSQRT: + return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt); + + case TargetOpcode::G_CTTZ: + case TargetOpcode::G_CTTZ_ZERO_UNDEF: + return selectExtInst(ResVReg, ResType, I, CL::ctz); + case TargetOpcode::G_CTLZ: + case TargetOpcode::G_CTLZ_ZERO_UNDEF: + return selectExtInst(ResVReg, ResType, I, CL::clz); + + case TargetOpcode::G_INTRINSIC_ROUND: + return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round); + case TargetOpcode::G_INTRINSIC_ROUNDEVEN: + return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); + case TargetOpcode::G_INTRINSIC_TRUNC: + return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc); + case TargetOpcode::G_FRINT: + case TargetOpcode::G_FNEARBYINT: + return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven); + + case TargetOpcode::G_SMULH: + return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi); + case TargetOpcode::G_UMULH: + return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi); case TargetOpcode::G_SEXT: return selectExt(ResVReg, ResType, I, true); @@ -394,6 +486,48 @@ } } +bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, + const SPIRVType *ResType, + MachineInstr &I, + CL::OpenCLExtInst CLInst) const { + return selectExtInst(ResVReg, ResType, I, + {{SPIRV::InstructionSet::OpenCL_std, CLInst}}); +} + +bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, + const SPIRVType *ResType, + MachineInstr &I, + CL::OpenCLExtInst CLInst, + GL::GLSLExtInst GLInst) const { + ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst}, + {SPIRV::InstructionSet::GLSL_std_450, GLInst}}; + return selectExtInst(ResVReg, ResType, I, ExtInsts); +} + +bool SPIRVInstructionSelector::selectExtInst(Register ResVReg, + const SPIRVType *ResType, + MachineInstr &I, + const ExtInstList &Insts) const { + + for (const auto &Ex : Insts) { + SPIRV::InstructionSet::InstructionSet Set = Ex.first; + uint32_t Opcode = Ex.second; + if (STI.canUseExtInstSet(Set)) { + MachineBasicBlock &BB = *I.getParent(); + auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst)) + .addDef(ResVReg) + .addUse(GR.getSPIRVTypeID(ResType)) + .addImm(static_cast(Set)) + .addImm(Opcode); + const unsigned NumOps = I.getNumOperands(); + for (unsigned i = 1; i < NumOps; ++i) + MIB.add(I.getOperand(i)); + return MIB.constrainAllUses(TII, TRI, RBI); + } + } + return false; +} + bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType, MachineInstr &I, @@ -493,9 +627,39 @@ bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg, MachineInstr &I) const { MachineBasicBlock &BB = *I.getParent(); + Register SrcReg = I.getOperand(1).getReg(); + if (I.getOpcode() == TargetOpcode::G_MEMSET) { + assert(I.getOperand(1).isReg() && I.getOperand(2).isReg()); + unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI); + unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI); + SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII); + SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII); + Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII); + SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType( + ArrTy, I, TII, SPIRV::StorageClass::UniformConstant); + // TODO: check if we have such GV, add init, use buildGlobalVariable. + Type *LLVMArrTy = ArrayType::get( + IntegerType::get(GR.CurMF->getFunction().getContext(), 8), Num); + GlobalVariable *GV = + new GlobalVariable(LLVMArrTy, true, GlobalValue::InternalLinkage); + Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); + GR.add(GV, GR.CurMF, VarReg); + + buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {}); + BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable)) + .addDef(VarReg) + .addUse(GR.getSPIRVTypeID(VarTy)) + .addImm(SPIRV::StorageClass::UniformConstant) + .addUse(Const) + .constrainAllUses(TII, TRI, RBI); + SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType( + ValTy, I, TII, SPIRV::StorageClass::UniformConstant); + SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32)); + selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast); + } auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized)) .addUse(I.getOperand(0).getReg()) - .addUse(I.getOperand(1).getReg()) + .addUse(SrcReg) .addUse(I.getOperand(2).getReg()); if (I.getNumMemOperands()) addMemoryOperands(*I.memoperands_begin(), MIB); @@ -974,9 +1138,9 @@ bool SPIRVInstructionSelector::selectIntToBool(Register IntReg, Register ResVReg, + MachineInstr &I, const SPIRVType *IntTy, - const SPIRVType *BoolTy, - MachineInstr &I) const { + const SPIRVType *BoolTy) const { // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero. Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass); bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector; @@ -1004,7 +1168,7 @@ if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) { Register IntReg = I.getOperand(1).getReg(); const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg); - return selectIntToBool(IntReg, ResVReg, ArgType, ResType, I); + return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType); } bool IsSigned = GR.isScalarOrVectorSigned(ResType); unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert; @@ -1223,6 +1387,12 @@ case Intrinsic::spv_cmpxchg: return selectAtomicCmpXchg(ResVReg, ResType, I); break; + case Intrinsic::spv_unreachable: + BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable)); + break; + case Intrinsic::spv_alloca: + return selectFrameIndex(ResVReg, ResType, I); + break; default: llvm_unreachable("Intrinsic selection not implemented"); } Index: llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp =================================================================== --- llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp +++ llvm/lib/Target/SPIRV/SPIRVLegalizerInfo.cpp @@ -255,6 +255,18 @@ getActionDefinitionsBuilder(G_FPOWI).legalForCartesianProduct( allFloatScalarsAndVectors, allIntScalarsAndVectors); + if (ST.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) { + getActionDefinitionsBuilder(G_FLOG10).legalFor(allFloatScalarsAndVectors); + + getActionDefinitionsBuilder( + {G_CTTZ, G_CTTZ_ZERO_UNDEF, G_CTLZ, G_CTLZ_ZERO_UNDEF}) + .legalForCartesianProduct(allIntScalarsAndVectors, + allIntScalarsAndVectors); + + // Struct return types become a single scalar, so cannot easily legalize. + getActionDefinitionsBuilder({G_SMULH, G_UMULH}).alwaysLegal(); + } + getLegacyLegalizerInfo().computeTables(); verify(*ST.getInstrInfo()); } Index: llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td =================================================================== --- llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -734,7 +734,7 @@ } multiclass ImageFormatOperand value, list reqCapabilities> { - def : ImageFormat; + def NAME : ImageFormat; defm : SymbolicOperandWithRequirements; }