Index: include/llvm/IR/IntrinsicsAMDGPU.td =================================================================== --- include/llvm/IR/IntrinsicsAMDGPU.td +++ include/llvm/IR/IntrinsicsAMDGPU.td @@ -590,7 +590,7 @@ AMDGPUDimProps dim, AMDGPUSampleVariant sample> : AMDGPUDimProfile { let IsSample = 1; - let RetTypes = [llvm_anyfloat_ty]; + let RetTypes = [llvm_any_ty]; let ExtraAddrArgs = sample.ExtraAddrArgs; let Gradients = sample.Gradients; let LodClampMip = sample.LodOrClamp; @@ -683,11 +683,11 @@ } defm int_amdgcn_image_load - : AMDGPUImageDimIntrinsicsAll<"LOAD", [llvm_anyfloat_ty], [], [IntrReadMem], + : AMDGPUImageDimIntrinsicsAll<"LOAD", [llvm_any_ty], [], [IntrReadMem], [SDNPMemOperand]>, AMDGPUImageDMaskIntrinsic; defm int_amdgcn_image_load_mip - : AMDGPUImageDimIntrinsicsNoMsaa<"LOAD_MIP", [llvm_anyfloat_ty], [], + : AMDGPUImageDimIntrinsicsNoMsaa<"LOAD_MIP", [llvm_any_ty], [], [IntrReadMem], [SDNPMemOperand], 1>, AMDGPUImageDMaskIntrinsic; Index: lib/Target/AMDGPU/AMDGPU.h =================================================================== --- lib/Target/AMDGPU/AMDGPU.h +++ lib/Target/AMDGPU/AMDGPU.h @@ -41,6 +41,7 @@ FunctionPass *createSIFoldOperandsPass(); FunctionPass *createSIPeepholeSDWAPass(); FunctionPass *createSILowerI1CopiesPass(); +FunctionPass *createSIAddIMGInitPass(); FunctionPass *createSIShrinkInstructionsPass(); FunctionPass *createSILoadStoreOptimizerPass(); FunctionPass *createSIWholeQuadModePass(); @@ -145,6 +146,9 @@ void initializeAMDGPUUseNativeCallsPass(PassRegistry &); extern char &AMDGPUUseNativeCallsID; +void initializeSIAddIMGInitPass(PassRegistry &); +extern char &SIAddIMGInitID; + void initializeAMDGPUPerfHintAnalysisPass(PassRegistry &); extern char &AMDGPUPerfHintAnalysisID; Index: lib/Target/AMDGPU/AMDGPU.td =================================================================== --- lib/Target/AMDGPU/AMDGPU.td +++ lib/Target/AMDGPU/AMDGPU.td @@ -376,6 +376,16 @@ "Use ds_{read|write}_b128" >; +// Sparse texture support requires that all result registers are zeroed when +// PRTStrictNull is set to true. This feature is turned on for all architectures +// but is enabled as a feature in case there are situations where PRTStrictNull +// is disabled by the driver. +def FeatureEnablePRTStrictNull : SubtargetFeature<"enable-prt-strict-null", + "EnablePRTStrictNull", + "true", + "Enable zeroing of result registers for sparse texture fetches" +>; + // Unless +-flat-for-global is specified, turn on FlatForGlobal for // all OS-es on VI and newer hardware to avoid assertion failures due // to missing ADDR64 variants of MUBUF instructions. Index: lib/Target/AMDGPU/AMDGPUSubtarget.h =================================================================== --- lib/Target/AMDGPU/AMDGPUSubtarget.h +++ lib/Target/AMDGPU/AMDGPUSubtarget.h @@ -326,6 +326,7 @@ bool EnableUnsafeDSOffsetFolding; bool EnableSIScheduler; bool EnableDS128; + bool EnablePRTStrictNull; bool DumpCode; // Subtarget statically properties set by tablegen @@ -572,6 +573,12 @@ return getGeneration() < AMDGPUSubtarget::GFX9; } + /// \returns If target requires PRT Struct NULL support (zero result registers + /// for sparse texture support). + bool usePRTStrictNull() const { + return EnablePRTStrictNull; + } + bool hasAutoWaitcntBeforeBarrier() const { return AutoWaitcntBeforeBarrier; } Index: lib/Target/AMDGPU/AMDGPUSubtarget.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -74,6 +74,9 @@ // We want to be able to turn these off, but making this a subtarget feature // for SI has the unhelpful behavior that it unsets everything else if you // disable it. + // + // Similarly we want enable-prt-strict-null to be on by default and not to + // unset everything else if it is disabled SmallString<256> FullFS("+promote-alloca,+dx10-clamp,+load-store-opt,"); @@ -89,6 +92,8 @@ FullFS += "-fp32-denormals,"; } + FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS + FullFS += FS; ParseSubtargetFeatures(GPU, FullFS); @@ -176,6 +181,7 @@ EnableUnsafeDSOffsetFolding(false), EnableSIScheduler(false), EnableDS128(false), + EnablePRTStrictNull(false), DumpCode(false), FP64(false), Index: lib/Target/AMDGPU/AMDGPUTargetMachine.cpp =================================================================== --- lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -812,6 +812,7 @@ bool GCNPassConfig::addInstSelector() { AMDGPUPassConfig::addInstSelector(); addPass(createSILowerI1CopiesPass()); + addPass(createSIAddIMGInitPass()); addPass(&SIFixSGPRCopiesID); return false; } Index: lib/Target/AMDGPU/CMakeLists.txt =================================================================== --- lib/Target/AMDGPU/CMakeLists.txt +++ lib/Target/AMDGPU/CMakeLists.txt @@ -92,6 +92,7 @@ R600OptimizeVectorRegisters.cpp R600Packetizer.cpp R600RegisterInfo.cpp + SIAddIMGInit.cpp SIAnnotateControlFlow.cpp SIDebuggerInsertNops.cpp SIFixSGPRCopies.cpp Index: lib/Target/AMDGPU/MIMGInstructions.td =================================================================== --- lib/Target/AMDGPU/MIMGInstructions.td +++ lib/Target/AMDGPU/MIMGInstructions.td @@ -179,6 +179,8 @@ defm _V3 : MIMG_NoSampler_Src_Helper ; let VDataDwords = 4 in defm _V4 : MIMG_NoSampler_Src_Helper ; + let VDataDwords = 8 in + defm _V8 : MIMG_NoSampler_Src_Helper ; } } @@ -411,6 +413,8 @@ defm _V3 : MIMG_Sampler_Src_Helper; let VDataDwords = 4 in defm _V4 : MIMG_Sampler_Src_Helper; + let VDataDwords = 8 in + defm _V8 : MIMG_Sampler_Src_Helper; } } @@ -429,6 +433,8 @@ defm _V2 : MIMG_Sampler_Src_Helper; /* for packed D16 only */ let VDataDwords = 4 in defm _V4 : MIMG_Sampler_Src_Helper; + let VDataDwords = 8 in + defm _V8 : MIMG_Sampler_Src_Helper; } } Index: lib/Target/AMDGPU/SIAddIMGInit.cpp =================================================================== --- /dev/null +++ lib/Target/AMDGPU/SIAddIMGInit.cpp @@ -0,0 +1,179 @@ +//===-- SIAddIMGInit.cpp - Add any required IMG inits ---------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +/// Any MIMG instructions that use tfe or lwe require an initialization of the +/// result register that will be written in the case of a memory access failure +/// The required code is also added to tie this init code to the result of the +/// img instruction +/// +//===----------------------------------------------------------------------===// +// + +#include "AMDGPU.h" +#include "AMDGPUSubtarget.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "SIInstrInfo.h" +#include "Utils/AMDGPULaneDominator.h" +#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/IR/Function.h" +#include "llvm/Support/Debug.h" +#include "llvm/Target/TargetMachine.h" + +#define DEBUG_TYPE "si-img-init" + +using namespace llvm; + +namespace { + +class SIAddIMGInit : public MachineFunctionPass { +public: + static char ID; + +public: + SIAddIMGInit() : MachineFunctionPass(ID) { + initializeSIAddIMGInitPass(*PassRegistry::getPassRegistry()); + } + + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } +}; + +} // End anonymous namespace. + +INITIALIZE_PASS(SIAddIMGInit, DEBUG_TYPE, "SI Add IMG Init", false, false) + +char SIAddIMGInit::ID = 0; + +char &llvm::SIAddIMGInitID = SIAddIMGInit::ID; + +FunctionPass *llvm::createSIAddIMGInitPass() { return new SIAddIMGInit(); } + +bool SIAddIMGInit::runOnMachineFunction(MachineFunction &MF) { + MachineRegisterInfo &MRI = MF.getRegInfo(); + const GCNSubtarget &ST = MF.getSubtarget(); + const SIInstrInfo *TII = ST.getInstrInfo(); + const SIRegisterInfo *RI = ST.getRegisterInfo(); + bool Changed = false; + + for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE; + ++BI) { + MachineBasicBlock &MBB = *BI; + MachineBasicBlock::iterator I, Next; + for (I = MBB.begin(); I != MBB.end(); I = Next) { + Next = std::next(I); + MachineInstr &MI = *I; + + auto Opcode = MI.getOpcode(); + if (TII->isMIMG(Opcode) && !MI.mayStore()) { + MachineOperand *TFE = TII->getNamedOperand(MI, AMDGPU::OpName::tfe); + MachineOperand *LWE = TII->getNamedOperand(MI, AMDGPU::OpName::lwe); + MachineOperand *D16 = TII->getNamedOperand(MI, AMDGPU::OpName::d16); + + // Abandon attempts for instructions that don't have tfe or lwe fields + // Shouldn't be any at this point, but this will allow for future + // variants. + if (!TFE && !LWE) + continue; + + unsigned TFEVal = TFE->getImm(); + unsigned LWEVal = LWE->getImm(); + unsigned D16Val = D16 ? D16->getImm() : 0; + + if (TFEVal || LWEVal) { + // At least one of TFE or LWE are non-zero + // We have to insert a suitable initialization of the result value and + // tie this to the dest of the image instruction. + + const DebugLoc &DL = MI.getDebugLoc(); + + int DstIdx = + AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + + // Calculate which dword we have to initialize to 0. + MachineOperand *MO_Dmask = + TII->getNamedOperand(MI, AMDGPU::OpName::dmask); + + // Abandon attempt if no dmask operand is found. + if (!MO_Dmask) + continue; + + unsigned dmask = MO_Dmask->getImm(); + // Determine the number of active lanes taking into account the + // Gather4 special case + unsigned ActiveLanes = + TII->isGather4(Opcode) ? 4 : countPopulation(dmask); + // Subreg indices are counted from 1 + // When D16 then we want next whole VGPR after write data. + bool Packed = !ST.hasUnpackedD16VMem(); + unsigned InitIdx = + D16Val && Packed ? ((ActiveLanes + 1) >> 1) + 1 : ActiveLanes + 1; + + // Abandon attempt if the dst size isn't large enough + // - this is in fact an error but this is picked up elsewhere and + // reported correctly. + uint32_t DstSize = + RI->getRegSizeInBits(*TII->getOpRegClass(MI, DstIdx)) / 32; + if (DstSize < InitIdx) + continue; + + // Create a register for the intialization value. + unsigned PrevDst = + MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx)); + unsigned NewDst = 0; // Final initialized value will be in here + + // If PRTStrictNull feature is enabled (the default) then initialize + // all the result registers to 0, otherwise just the error indication + // register (VGPRn+1) + unsigned SizeLeft = ST.usePRTStrictNull() ? InitIdx : 1; + unsigned CurrIdx = ST.usePRTStrictNull() ? 1 : InitIdx; + + if (DstSize == 1) { + // In this case we can just initialize the result directly + BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), PrevDst) + .addImm(0); + NewDst = PrevDst; + } else { + BuildMI(MBB, MI, DL, TII->get(AMDGPU::IMPLICIT_DEF), PrevDst); + for (; SizeLeft; SizeLeft--, CurrIdx++) { + NewDst = + MRI.createVirtualRegister(TII->getOpRegClass(MI, DstIdx)); + // Initialize dword + unsigned SubReg = + MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); + BuildMI(MBB, MI, DL, TII->get(AMDGPU::V_MOV_B32_e32), SubReg) + .addImm(0); + // Insert into the super-reg + BuildMI(MBB, I, DL, TII->get(TargetOpcode::INSERT_SUBREG), NewDst) + .addReg(PrevDst) + .addReg(SubReg) + .addImm(CurrIdx); + + PrevDst = NewDst; + } + } + + // Add as an implicit operand + MachineInstrBuilder(MF, MI).addReg(NewDst, RegState::Implicit); + + // Tie the just added implicit operand to the dst + MI.tieOperands(DstIdx, MI.getNumOperands() - 1); + + Changed = true; + } + } + } + } + + return Changed; +} Index: lib/Target/AMDGPU/SIISelLowering.cpp =================================================================== --- lib/Target/AMDGPU/SIISelLowering.cpp +++ lib/Target/AMDGPU/SIISelLowering.cpp @@ -216,6 +216,7 @@ setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v2f16, Custom); setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v4f16, Custom); + setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::v8f16, Custom); setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); @@ -813,6 +814,44 @@ Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT); } +static MVT memVTFromAggregate(Type *Ty) { + // Only limited forms of aggregate type currently expected. + assert(Ty->isStructTy() && "Expected struct type"); + + VectorType *VecComponent = cast(Ty->getContainedType(0)); + Type *ElementType = nullptr; + if (VecComponent) + ElementType = VecComponent->getElementType(); + else + ElementType = Ty->getContainedType(0); + + Type *FlagComponent = Ty->getContainedType(1); + assert(FlagComponent->isIntegerTy(32) && "Expected int32 type"); + + // Calculate the size of the memVT type from the aggregate + unsigned Pow2Elts = 0; + unsigned ElementSize; + switch (ElementType->getTypeID()) { + default: + llvm_unreachable("Unknown type!"); + case Type::IntegerTyID: + ElementSize = cast(ElementType)->getBitWidth(); + break; + case Type::HalfTyID: + ElementSize = 16; + break; + case Type::FloatTyID: + ElementSize = 32; + break; + } + unsigned AdditionalElts = ElementSize == 16 ? 2 : 1; + unsigned NumElts = VecComponent->getNumElements(); + Pow2Elts = 1 << Log2_32_Ceil(NumElts + AdditionalElts); + + return MVT::getVectorVT(MVT::getVT(ElementType, false), + Pow2Elts); +} + bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &CI, MachineFunction &MF, @@ -840,7 +879,12 @@ Info.flags = MachineMemOperand::MODereferenceable; if (Attr.hasFnAttribute(Attribute::ReadOnly)) { Info.opc = ISD::INTRINSIC_W_CHAIN; - Info.memVT = MVT::getVT(CI.getType()); + Info.memVT = MVT::getVT(CI.getType(),true); + if (Info.memVT == MVT::Other) { + // Some intrinsics return an aggregate type - special case to work out + // the correct memVT + Info.memVT = memVTFromAggregate(CI.getType()); + } Info.flags |= MachineMemOperand::MOLoad; } else if (Attr.hasFnAttribute(Attribute::WriteOnly)) { Info.opc = ISD::INTRINSIC_VOID; @@ -4613,6 +4657,84 @@ return Value == 0; } +static bool adjustRetType(SelectionDAG &DAG, SmallVector &ResultTypes, + unsigned &TexFailStart, EVT &OrigVT, EVT &CastVT, + EVT &NewVT, LLVMContext &Context, bool &IsTexFail, + bool UnpackedD16) { + if (ResultTypes.size() == 3) { + // The current return type is an aggregate. This is used for anything that + // requires TFE or LWE support + + // Calculate the return vector type required + TexFailStart = (ResultTypes[0].getSizeInBits() + 31) / 32; + unsigned NumVDataDwords = TexFailStart + 1; + + // Work out how many elements that number of dwords gives + unsigned EltBitSize = ResultTypes[0].getVectorElementType().getSizeInBits(); + unsigned NewVecBitSize = NumVDataDwords * 32; + + assert(((NewVecBitSize % EltBitSize) == 0) && + "Unexpected vector element size"); + unsigned NElts = NewVecBitSize / EltBitSize; + + // Round the size to the nearest power of 2 + unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts); + unsigned Pow2NEltsTFF = (Pow2NElts * EltBitSize) / + 32; // Should also be pow2 in all cases I think + + NewVT = EVT::getVectorVT(Context, ResultTypes[0].getVectorElementType(), + Pow2NElts); + + CastVT = EVT::getVectorVT(Context, MVT::i32, Pow2NEltsTFF); + + OrigVT = ResultTypes[0]; + if (NewVT == MVT::v8f16 && !UnpackedD16) { + // Special case - this will be turned into MVT::v4f32 so we need to ensure + // that the OrigVT is a compatible sub vector type + assert(OrigVT == MVT::v4f16 && "Encountered unexpected vector result for " + "image load with TFE/LWE enabled"); + OrigVT = MVT::v2i32; + } + if (UnpackedD16 && OrigVT.getScalarType() == MVT::f16) { + OrigVT = (OrigVT == MVT::v2f16) + ? MVT::v2i32 + : (OrigVT == MVT::v4f16) ? MVT::v4i32 : MVT::v8i32; + CastVT = (CastVT == MVT::v2i32) ? MVT::v4i32 : MVT::v8i32; + // Based on the unpacking need to re-calc the TexFailStart value + TexFailStart = ((OrigVT.getSizeInBits() + 31) / 32); + } + + ResultTypes[0] = NewVT; + ResultTypes.erase(&ResultTypes[1]); + + IsTexFail = true; + } + return true; +} + +static bool parseTexFail(SDValue TexFailCtrl, SelectionDAG &DAG, SDValue *TFE, + SDValue *LWE, bool IsTexFail) { + auto TexFailCtrlConst = dyn_cast(TexFailCtrl.getNode()); + if (!TexFailCtrlConst) + return false; + + uint64_t Value = TexFailCtrlConst->getZExtValue(); + SDLoc DL(TexFailCtrlConst); + if (Value) { + // One of the texture fail control bits is set + // Check that we've already detected an aggregate return type + if (!IsTexFail) + return false; + } + + *TFE = DAG.getTargetConstant((Value & 0x1) ? 1 : 0, DL, MVT::i32); + Value &= ~(uint64_t)0x1; + *LWE = DAG.getTargetConstant((Value & 0x2) ? 1 : 0, DL, MVT::i32); + Value &= ~(uint64_t)0x2; + + return Value == 0; +} + SDValue SITargetLowering::lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr, SelectionDAG &DAG) const { @@ -4626,13 +4748,29 @@ AMDGPU::getMIMGLZMappingInfo(Intr->BaseOpcode); unsigned IntrOpcode = Intr->BaseOpcode; - SmallVector ResultTypes(Op->value_begin(), Op->value_end()); + SmallVector ResultTypes(Op->value_begin(), Op->value_end()); bool IsD16 = false; bool IsA16 = false; + bool IsTexFail = false; SDValue VData; int NumVDataDwords; unsigned AddrIdx; // Index of first address argument unsigned DMask; + unsigned TexFailStart; + // The following extra VTs are used for any TFE/LWE adjustment + // OrigVT is the original VT before any adjustment - altered for packed D16 + // CastVT is the VT used to cast the result to extract the TFE/LWE flag if + // required NewVT is the adjusted VT in its raw state - not altered for packed + // D16 + EVT OrigVT, CastVT, NewVT; + + // Make appropriate adjustments to the return values if the intrinsic is using + // an aggregate type - this in an indication that TFE or LWE is expected to be + // used + if (!adjustRetType(DAG, ResultTypes, TexFailStart, OrigVT, CastVT, NewVT, + *DAG.getContext(), IsTexFail, + Subtarget->hasUnpackedD16VMem())) + return Op; if (BaseOpcode->Atomic) { VData = Op.getOperand(2); @@ -4673,7 +4811,7 @@ NumVDataDwords = (VData.getValueType().getSizeInBits() + 31) / 32; DMaskIdx = 3; } else { - MVT LoadVT = Op.getSimpleValueType(); + MVT LoadVT = ResultTypes[0].getSimpleVT(); if (LoadVT.getScalarType() == MVT::f16) { if (Subtarget->getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS || !BaseOpcode->HasD16) @@ -4681,7 +4819,18 @@ IsD16 = true; if (LoadVT.isVector() && Subtarget->hasUnpackedD16VMem()) - ResultTypes[0] = (LoadVT == MVT::v2f16) ? MVT::v2i32 : MVT::v4i32; + ResultTypes[0] = + (LoadVT == MVT::v2f16) + ? MVT::v2i32 + : (LoadVT == MVT::v4f16) ? MVT::v4i32 : MVT::v8i32; + else if (LoadVT == MVT::v8f16) + // Rather than add lots of code to handle v8f16 for this case, just + // treat it as v4i32 - this is reasonable since this is only done for + // TFE/LWE support anyway and the result is a mixture of 4 packed + // 16 bit values and a 32 bit error condition (plus an unused 32 bit + // value) : + // [ Res0 : Res1 ][ Res2 : Res3 ][ ErrCode ][ unused ] + ResultTypes[0] = MVT::v4i32; } NumVDataDwords = (ResultTypes[0].getSizeInBits() + 31) / 32; @@ -4778,9 +4927,10 @@ CtrlIdx = AddrIdx + NumVAddrs + 3; } + SDValue TFE; + SDValue LWE; SDValue TexFail = Op.getOperand(CtrlIdx); - auto TexFailConst = dyn_cast(TexFail.getNode()); - if (!TexFailConst || TexFailConst->getZExtValue() != 0) + if (!parseTexFail(TexFail, DAG, &TFE, &LWE, IsTexFail)) return Op; SDValue GLC; @@ -4807,8 +4957,8 @@ Ops.push_back(SLC); Ops.push_back(IsA16 && // a16 or r128 ST->hasFeature(AMDGPU::FeatureR128A16) ? True : False); - Ops.push_back(False); // tfe - Ops.push_back(False); // lwe + Ops.push_back(TFE); // tfe + Ops.push_back(LWE); // lwe Ops.push_back(DimInfo->DA ? True : False); if (BaseOpcode->HasD16) Ops.push_back(IsD16 ? True : False); @@ -4836,11 +4986,26 @@ SmallVector Elt; DAG.ExtractVectorElements(SDValue(NewNode, 0), Elt, 0, 1); return DAG.getMergeValues({Elt[0], SDValue(NewNode, 1)}, DL); - } else if (IsD16 && !BaseOpcode->Store) { + } else if (IsD16 && !BaseOpcode->Store && !IsTexFail) { MVT LoadVT = Op.getSimpleValueType(); SDValue Adjusted = adjustLoadValueTypeImpl( SDValue(NewNode, 0), LoadVT, DL, DAG, Subtarget->hasUnpackedD16VMem()); return DAG.getMergeValues({Adjusted, SDValue(NewNode, 1)}, DL); + } else if (IsTexFail) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SmallVector Elt; + SDValue N(NewNode, 0); + SDValue NewRes = DAG.getNode( + ISD::EXTRACT_SUBVECTOR, DL, OrigVT, N, + DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); + if (IsD16 && !BaseOpcode->Store) { + EVT LoadVT = Op.getSimpleValueType(); + NewRes = adjustLoadValueTypeImpl(NewRes, LoadVT, DL, DAG, + Subtarget->hasUnpackedD16VMem()); + } + SDValue CastNewRes = DAG.getNode(ISD::BITCAST, DL, CastVT, N); + DAG.ExtractVectorElements(CastNewRes, Elt, TexFailStart, 1); + return DAG.getMergeValues({NewRes, Elt[0], SDValue(NewNode, 1)}, DL); } return SDValue(NewNode, 0); @@ -8797,6 +8962,7 @@ case AMDGPU::sub1: return 1; case AMDGPU::sub2: return 2; case AMDGPU::sub3: return 3; + case AMDGPU::sub4: return 4; // Possible with TFE/LWE } } @@ -8810,11 +8976,16 @@ if (D16Idx >= 0 && Node->getConstantOperandVal(D16Idx)) return Node; // not implemented for D16 - SDNode *Users[4] = { nullptr }; + SDNode *Users[5] = { nullptr }; unsigned Lane = 0; unsigned DmaskIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::dmask) - 1; unsigned OldDmask = Node->getConstantOperandVal(DmaskIdx); unsigned NewDmask = 0; + unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::tfe) - 1; + unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::lwe) - 1; + bool UsesTFC = (Node->getConstantOperandVal(TFEIdx) || + Node->getConstantOperandVal(LWEIdx)) ? 1 : 0; + unsigned TFCLane = 0; bool HasChain = Node->getNumValues() > 1; if (OldDmask == 0) { @@ -8822,6 +8993,12 @@ return Node; } + // Work out which is the TFE/LWE lane if that is enabled. + if (UsesTFC) { + unsigned OldBitsSet = countPopulation(OldDmask); + TFCLane = OldBitsSet; + } + // Try to figure out the used register components for (SDNode::use_iterator I = Node->use_begin(), E = Node->use_end(); I != E; ++I) { @@ -8841,28 +9018,44 @@ // set, etc. Lane = SubIdx2Lane(I->getConstantOperandVal(1)); - // Set which texture component corresponds to the lane. - unsigned Comp; - for (unsigned i = 0, Dmask = OldDmask; i <= Lane; i++) { - Comp = countTrailingZeros(Dmask); - Dmask &= ~(1 << Comp); - } + // Check if the use is for the TFE/LWE generated result at VGPRn+1. + if (UsesTFC && Lane == TFCLane) { + Users[Lane] = *I; + } else { + // Set which texture component corresponds to the lane. + unsigned Comp; + for (unsigned i = 0, Dmask = OldDmask; i <= Lane; i++) { + Comp = countTrailingZeros(Dmask); + Dmask &= ~(1 << Comp); + } - // Abort if we have more than one user per component - if (Users[Lane]) - return Node; + // Abort if we have more than one user per component. + if (Users[Lane]) + return Node; - Users[Lane] = *I; - NewDmask |= 1 << Comp; + Users[Lane] = *I; + NewDmask |= 1 << Comp; + } } + // Don't allow 0 dmask, as hardware assumes one channel enabled. + bool NoChannels = !NewDmask; + if (NoChannels) + NewDmask = 1; // Abort if there's no change if (NewDmask == OldDmask) return Node; unsigned BitsSet = countPopulation(NewDmask); - int NewOpcode = AMDGPU::getMaskedMIMGOp(Node->getMachineOpcode(), BitsSet); + // Check for TFE or LWE - increase the number of channels by one to account + // for the extra return value + // This will need adjustment for D16 if this is also included in + // adjustWriteMask (this function) but at present D16 are excluded. + unsigned NewChannels = BitsSet + UsesTFC; + + int NewOpcode = + AMDGPU::getMaskedMIMGOp(Node->getMachineOpcode(), NewChannels); assert(NewOpcode != -1 && NewOpcode != static_cast(Node->getMachineOpcode()) && "failed to find equivalent MIMG op"); @@ -8875,8 +9068,9 @@ MVT SVT = Node->getValueType(0).getVectorElementType().getSimpleVT(); - MVT ResultVT = BitsSet == 1 ? - SVT : MVT::getVectorVT(SVT, BitsSet == 3 ? 4 : BitsSet); + MVT ResultVT = NewChannels == 1 ? + SVT : MVT::getVectorVT(SVT, NewChannels == 3 ? 4 : + NewChannels == 5 ? 8 : NewChannels); SDVTList NewVTList = HasChain ? DAG.getVTList(ResultVT, MVT::Other) : DAG.getVTList(ResultVT); @@ -8890,7 +9084,7 @@ DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(NewNode, 1)); } - if (BitsSet == 1) { + if (NewChannels == 1) { assert(Node->hasNUsesOfValue(1, 0)); SDNode *Copy = DAG.getMachineNode(TargetOpcode::COPY, SDLoc(Node), Users[Lane]->getValueType(0), @@ -8900,19 +9094,24 @@ } // Update the users of the node with the new indices - for (unsigned i = 0, Idx = AMDGPU::sub0; i < 4; ++i) { + for (unsigned i = 0, Idx = AMDGPU::sub0; i < 5; ++i) { SDNode *User = Users[i]; - if (!User) - continue; - - SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32); - DAG.UpdateNodeOperands(User, SDValue(NewNode, 0), Op); + if (!User) { + // Handle the special case of NoChannels. We set NewDmask to 1 above, but + // Users[0] is still nullptr because channel 0 doesn't really have a use. + if (i || !NoChannels) + continue; + } else { + SDValue Op = DAG.getTargetConstant(Idx, SDLoc(User), MVT::i32); + DAG.UpdateNodeOperands(User, SDValue(NewNode, 0), Op); + } switch (Idx) { default: break; case AMDGPU::sub0: Idx = AMDGPU::sub1; break; case AMDGPU::sub1: Idx = AMDGPU::sub2; break; case AMDGPU::sub2: Idx = AMDGPU::sub3; break; + case AMDGPU::sub3: Idx = AMDGPU::sub4; break; } } Index: lib/Target/AMDGPU/SIInstrInfo.cpp =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.cpp +++ lib/Target/AMDGPU/SIInstrInfo.cpp @@ -2944,6 +2944,42 @@ } } + // Verify MIMG + if (isMIMG(MI.getOpcode()) && !MI.mayStore()) { + // Ensure that the return type used is large enough for all the options + // being used TFE/LWE require an extra result register. + const MachineOperand *DMask = getNamedOperand(MI, AMDGPU::OpName::dmask); + if (DMask) { + uint64_t DMaskImm = DMask->getImm(); + uint32_t RegCount = + isGather4(MI.getOpcode()) ? 4 : countPopulation(DMaskImm); + const MachineOperand *TFE = getNamedOperand(MI, AMDGPU::OpName::tfe); + const MachineOperand *LWE = getNamedOperand(MI, AMDGPU::OpName::lwe); + const MachineOperand *D16 = getNamedOperand(MI, AMDGPU::OpName::d16); + + // Adjust for packed 16 bit values + if (D16 && D16->getImm() && !ST.hasUnpackedD16VMem()) + RegCount >>= 1; + + // Adjust if using LWE or TFE + if ((LWE && LWE->getImm()) || (TFE && TFE->getImm())) + RegCount += 1; + + const uint32_t DstIdx = + AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vdata); + const MachineOperand &Dst = MI.getOperand(DstIdx); + if (Dst.isReg()) { + const TargetRegisterClass *DstRC = getOpRegClass(MI, DstIdx); + uint32_t DstSize = RI.getRegSizeInBits(*DstRC) / 32; + if (RegCount > DstSize) { + ErrInfo = "MIMG instruction returns too many registers for dst " + "register class"; + return false; + } + } + } + } + // Verify VOP*. Ignore multiple sgpr operands on writelane. if (Desc.getOpcode() != AMDGPU::V_WRITELANE_B32 && (isVOP1(MI) || isVOP2(MI) || isVOP3(MI) || isVOPC(MI) || isSDWA(MI))) { Index: lib/Transforms/InstCombine/InstCombineInternal.h =================================================================== --- lib/Transforms/InstCombine/InstCombineInternal.h +++ lib/Transforms/InstCombine/InstCombineInternal.h @@ -799,7 +799,8 @@ Value *simplifyAMDGCNMemoryIntrinsicDemanded(IntrinsicInst *II, APInt DemandedElts, - int DmaskIdx = -1); + int DmaskIdx = -1, + int TFEIdx = -1); Value *SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, APInt &UndefElts, unsigned Depth = 0); Index: lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp =================================================================== --- lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -945,11 +945,24 @@ /// Implement SimplifyDemandedVectorElts for amdgcn buffer and image intrinsics. Value *InstCombiner::simplifyAMDGCNMemoryIntrinsicDemanded(IntrinsicInst *II, APInt DemandedElts, - int DMaskIdx) { + int DMaskIdx, + int TFCIdx) { unsigned VWidth = II->getType()->getVectorNumElements(); if (VWidth == 1) return nullptr; + // Need to change to new instruction format + ConstantInt *TFC = nullptr; + bool TFELWEEnabled = false; + if (TFCIdx > 0) { + TFC = dyn_cast(II->getArgOperand(TFCIdx)); + TFELWEEnabled = TFC->getZExtValue() & 0x1 // TFE + || TFC->getZExtValue() & 0x2; // LWE + } + + if (TFELWEEnabled) + return nullptr; // TFE not yet supported + ConstantInt *NewDMask = nullptr; if (DMaskIdx < 0) { @@ -1598,7 +1611,8 @@ return simplifyAMDGCNMemoryIntrinsicDemanded(II, DemandedElts); default: { if (getAMDGPUImageDMaskIntrinsic(II->getIntrinsicID())) - return simplifyAMDGCNMemoryIntrinsicDemanded(II, DemandedElts, 0); + return simplifyAMDGCNMemoryIntrinsicDemanded( + II, DemandedElts, 0, II->getNumArgOperands() - 2); break; } Index: test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll =================================================================== --- test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll +++ test/CodeGen/AMDGPU/llvm.amdgcn.image.dim.ll @@ -1,6 +1,7 @@ -; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI %s -; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s -; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s +; RUN: llc -march=amdgcn -mcpu=verde -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,SI,SIVI,PRT %s +; RUN: llc -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,SIVI,PRT %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900,PRT %s +; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=-enable-prt-strict-null -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX900,NOPRT %s ; GCN-LABEL: {{^}}load_1d: ; GCN: image_load v[0:3], v0, s[0:7] dmask:0xf unorm{{$}} @@ -10,6 +11,52 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_1d_tfe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load v[0:7], v{{[0-9]+}}, s[0:7] dmask:0xf unorm tfe{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 1, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + +; GCN-LABEL: {{^}}load_1d_lwe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load v[0:7], v{{[0-9]+}}, s[0:7] dmask:0xf unorm lwe{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_1d_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s) { +main_body: + %v = call {<4 x float>, i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32 15, i32 %s, <8 x i32> %rsrc, i32 2, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}load_2d: ; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm{{$}} define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t) { @@ -18,6 +65,29 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_2d_tfe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2d.v4f32i32.i32(i32 15, i32 %s, i32 %t, <8 x i32> %rsrc, i32 1, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}load_3d: ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm{{$}} define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r) { @@ -26,6 +96,29 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_3d_tfe_lwe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe lwe{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_3d_tfe_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %r) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.3d.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %r, <8 x i32> %rsrc, i32 3, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}load_cube: ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da{{$}} define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice) { @@ -34,6 +127,29 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_cube_lwe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_cube_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.cube.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}load_1darray: ; GCN: image_load v[0:3], v[0:1], s[0:7] dmask:0xf unorm da{{$}} define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, i32 %s, i32 %slice) { @@ -42,6 +158,29 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_1darray_tfe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe da{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_1darray_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %slice) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.1darray.v4f32i32.i32(i32 15, i32 %s, i32 %slice, <8 x i32> %rsrc, i32 1, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}load_2darray: ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da{{$}} define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice) { @@ -50,6 +189,29 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_2darray_lwe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe da{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_2darray_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2darray.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, <8 x i32> %rsrc, i32 2, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}load_2dmsaa: ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm{{$}} define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %fragid) { @@ -58,6 +220,29 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_2dmsaa_both: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe lwe{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_2dmsaa_both(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %fragid) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2dmsaa.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %fragid, <8 x i32> %rsrc, i32 3, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}load_2darraymsaa: ; GCN: image_load v[0:3], v[0:3], s[0:7] dmask:0xf unorm da{{$}} define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %slice, i32 %fragid) { @@ -66,6 +251,29 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_2darraymsaa_tfe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe da{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %slice, i32 %fragid) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.2darraymsaa.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %slice, i32 %fragid, <8 x i32> %rsrc, i32 1, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}load_mip_1d: ; GCN: image_load_mip v[0:3], v[0:1], s[0:7] dmask:0xf unorm{{$}} define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, i32 %s, i32 %mip) { @@ -74,6 +282,29 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_mip_1d_lwe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load_mip v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm lwe{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_mip_1d_lwe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %mip) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.mip.1d.v4f32i32.i32(i32 15, i32 %s, i32 %mip, <8 x i32> %rsrc, i32 2, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}load_mip_2d: ; GCN: image_load_mip v[0:3], v[0:3], s[0:7] dmask:0xf unorm{{$}} define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %mip) { @@ -82,6 +313,44 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}load_mip_2d_tfe: +; PRT: v_mov_b32_e32 v0, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v0 +; NOPRT: v_mov_b32_e32 v4, 0 +; NOPRT-NOT: v_mov_b32_e32 v0 +; NOPRT-NOT: v_mov_b32_e32 v1 +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT-NOT: v_mov_b32_e32 v3 +; GCN: image_load_mip v[0:7], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0xf unorm tfe{{$}} +; SIVI: buffer_store_dword v4, off, s[8:11], 0 +; GFX900: global_store_dword v[{{[0-9]+:[0-9]+}}], v4 +define amdgpu_ps <4 x float> @load_mip_2d_tfe(<8 x i32> inreg %rsrc, i32 addrspace(1)* inreg %out, i32 %s, i32 %t, i32 %mip) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %mip, <8 x i32> %rsrc, i32 1, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + +; Do not make dmask 0 even if no result (other than tfe) is used. +; GCN-LABEL: {{^}}load_mip_2d_tfe_nouse: +; GCN: v_mov_b32_e32 v3, 0 +; PRT-DAG: v_mov_b32_e32 v{{[0-9]+}}, v3 +; PRT: image_load_mip v[3:4], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x1 unorm tfe{{$}} +; NOPRT-NOT: v_mov_b32_e32 v2 +; NOPRT: image_load_mip v[2:3], v[{{[0-9]+:[0-9]+}}], s[0:7] dmask:0x1 unorm tfe{{$}} +define amdgpu_ps float @load_mip_2d_tfe_nouse(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %mip) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v4f32i32.i32(i32 15, i32 %s, i32 %t, i32 %mip, <8 x i32> %rsrc, i32 1, i32 0) + %v.err = extractvalue {<4 x float>, i32} %v, 1 + %vv = bitcast i32 %v.err to float + ret float %vv +} + ; GCN-LABEL: {{^}}load_mip_3d: ; GCN: image_load_mip v[0:3], v[0:3], s[0:7] dmask:0xf unorm{{$}} define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, i32 %s, i32 %t, i32 %r, i32 %mip) { @@ -404,23 +673,33 @@ store float 0.000000e+00, float addrspace(3)* %lds %c0 = extractelement <2 x i32> %c, i32 0 %c1 = extractelement <2 x i32> %c, i32 1 - %tex = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 15, i32 %c0, i32 %c1, <8 x i32> %rsrc, i32 0, i32 0) + %tex = call float @llvm.amdgcn.image.load.2d.f32.i32(i32 1, i32 %c0, i32 %c1, <8 x i32> %rsrc, i32 0, i32 0) %tmp2 = getelementptr float, float addrspace(3)* %lds, i32 4 store float 0.000000e+00, float addrspace(3)* %tmp2 ret float %tex } declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.1d.v4f32i32.i32(i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.2d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.2d.v4f32i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.3d.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.3d.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.cube.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.cube.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.1darray.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.1darray.v4f32i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.2darray.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.2darray.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.2dmsaa.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.2dmsaa.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.2darraymsaa.v4f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.2darraymsaa.v4f32i32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.mip.1d.v4f32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.mip.2d.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.mip.1d.v4f32i32.i32(i32, i32, i32, <8 x i32>, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.load.mip.2d.v4f32i32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.mip.3d.v4f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.mip.cube.v4f32.i32(i32, i32, i32, i32, i32, <8 x i32>, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.load.mip.1darray.v4f32.i32(i32, i32, i32, i32, <8 x i32>, i32, i32) #1 Index: test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll =================================================================== --- test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll +++ test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll @@ -10,6 +10,22 @@ ret half %tex } +; GCN-LABEL: {{^}}image_sample_2d_f16_tfe: +; GCN: v_mov_b32_e32 v{{[0-9]+}}, 0 +; PACKED: image_sample v[{{[0-9]+:[0-9]+}}], v[0:1], s[0:7], s[8:11] dmask:0x1 tfe d16{{$}} +; UNPACKED: image_sample v[{{[0-9]+:[0-9]+}}], v[0:1], s[0:7], s[8:11] dmask:0x1 tfe d16{{$}} +define amdgpu_ps <2 x float> @image_sample_2d_f16_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) { +main_body: + %tex = call {<2 x half>,i32} @llvm.amdgcn.image.sample.2d.v2f16i32.f32(i32 1, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0) + %tex.vec = extractvalue {<2 x half>, i32} %tex, 0 + %tex.err = extractvalue {<2 x half>, i32} %tex, 1 + %tex.vecf = bitcast <2 x half> %tex.vec to float + %r.0 = insertelement <2 x float> undef, float %tex.vecf, i32 0 + %tex.errf = bitcast i32 %tex.err to float + %r = insertelement <2 x float> %r.0, float %tex.errf, i32 1 + ret <2 x float> %r +} + ; GCN-LABEL: {{^}}image_sample_c_d_1d_v2f16: ; UNPACKED: image_sample_c_d v[0:1], v[0:3], s[0:7], s[8:11] dmask:0x3 d16{{$}} ; PACKED: image_sample_c_d v0, v[0:3], s[0:7], s[8:11] dmask:0x3 d16{{$}} @@ -20,6 +36,22 @@ ret float %r } +; GCN-LABEL: {{^}}image_sample_c_d_1d_v2f16_tfe: +; GCN: v_mov_b32_e32 v{{[0-9]+}}, 0 +; UNPACKED: image_sample_c_d v[{{[0-9]+:[0-9]+}}], v[0:3], s[0:7], s[8:11] dmask:0x3 tfe d16{{$}} +; PACKED: image_sample_c_d v[{{[0-9]+:[0-9]+}}], v[0:3], s[0:7], s[8:11] dmask:0x3 tfe d16{{$}} +define amdgpu_ps <2 x float> @image_sample_c_d_1d_v2f16_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %zcompare, float %dsdh, float %dsdv, float %s) { +main_body: + %tex = call {<2 x half>,i32} @llvm.amdgcn.image.sample.c.d.1d.v2f16i32.f32.f32(i32 3, float %zcompare, float %dsdh, float %dsdv, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0) + %tex.vec = extractvalue {<2 x half>, i32} %tex, 0 + %tex.err = extractvalue {<2 x half>, i32} %tex, 1 + %tex.vecf = bitcast <2 x half> %tex.vec to float + %r.0 = insertelement <2 x float> undef, float %tex.vecf, i32 0 + %tex.errf = bitcast i32 %tex.err to float + %r = insertelement <2 x float> %r.0, float %tex.errf, i32 1 + ret <2 x float> %r +} + ; GCN-LABEL: {{^}}image_sample_b_2d_v4f16: ; UNPACKED: image_sample_b v[0:3], v[0:3], s[0:7], s[8:11] dmask:0xf d16{{$}} ; PACKED: image_sample_b v[0:1], v[0:3], s[0:7], s[8:11] dmask:0xf d16{{$}} @@ -30,9 +62,32 @@ ret <2 x float> %r } +; GCN-LABEL: {{^}}image_sample_b_2d_v4f16_tfe: +; GCN: v_mov_b32_e32 v{{[0-9]+}}, 0 +; UNPACKED: image_sample_b v[{{[0-9]+:[0-9]+}}], v[0:3], s[0:7], s[8:11] dmask:0xf tfe d16{{$}} +; PACKED: image_sample_b v[{{[0-9]+:[0-9]+}}], v[0:3], s[0:7], s[8:11] dmask:0xf tfe d16{{$}} +define amdgpu_ps <4 x float> @image_sample_b_2d_v4f16_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %bias, float %s, float %t) { +main_body: + %tex = call {<4 x half>,i32} @llvm.amdgcn.image.sample.b.2d.v4f16i32.f32.f32(i32 15, float %bias, float %s, float %t, <8 x i32> %rsrc, <4 x i32> %samp, i1 false, i32 1, i32 0) + %tex.vec = extractvalue {<4 x half>, i32} %tex, 0 + %tex.err = extractvalue {<4 x half>, i32} %tex, 1 + %tex.vecf = bitcast <4 x half> %tex.vec to <2 x float> + %tex.vecf.0 = extractelement <2 x float> %tex.vecf, i32 0 + %tex.vecf.1 = extractelement <2 x float> %tex.vecf, i32 1 + %r.0 = insertelement <4 x float> undef, float %tex.vecf.0, i32 0 + %r.1 = insertelement <4 x float> %r.0, float %tex.vecf.1, i32 1 + %tex.errf = bitcast i32 %tex.err to float + %r = insertelement <4 x float> %r.1, float %tex.errf, i32 2 + ret <4 x float> %r +} + declare half @llvm.amdgcn.image.sample.2d.f16.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 +declare <4 x half> @llvm.amdgcn.image.sample.2d.v4f16.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 +declare {<2 x half>,i32} @llvm.amdgcn.image.sample.2d.v2f16i32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 declare <2 x half> @llvm.amdgcn.image.sample.c.d.1d.v2f16.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 +declare {<2 x half>,i32} @llvm.amdgcn.image.sample.c.d.1d.v2f16i32.f32.f32(i32, float, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 declare <4 x half> @llvm.amdgcn.image.sample.b.2d.v4f16.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 +declare {<4 x half>,i32} @llvm.amdgcn.image.sample.b.2d.v4f16i32.f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 attributes #0 = { nounwind } attributes #1 = { nounwind readonly } Index: test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll =================================================================== --- test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll +++ test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.dim.ll @@ -9,6 +9,162 @@ ret <4 x float> %v } +; GCN-LABEL: {{^}}sample_1d_tfe: +; GCN: v_mov_b32_e32 v0, 0 +; GCN: v_mov_b32_e32 v1, v0 +; GCN: v_mov_b32_e32 v2, v0 +; GCN: v_mov_b32_e32 v3, v0 +; GCN: v_mov_b32_e32 v4, v0 +; GCN: image_sample v[0:7], v5, s[0:7], s[8:11] dmask:0xf tfe{{$}} +define amdgpu_ps <4 x float> @sample_1d_tfe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 addrspace(1)* inreg %out, float %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + +; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_1: +; GCN: v_mov_b32_e32 v0, 0 +; GCN: v_mov_b32_e32 v1, v0 +; GCN: image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x1 tfe{{$}} +define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_1(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 addrspace(1)* inreg %out, float %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0) + %res.vec = extractvalue {<4 x float>,i32} %v, 0 + %res.f = extractelement <4 x float> %res.vec, i32 0 + %res.err = extractvalue {<4 x float>,i32} %v, 1 + %res.errf = bitcast i32 %res.err to float + %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0 + %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1 + ret <2 x float> %res +} + +; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_2: +; GCN: v_mov_b32_e32 v0, 0 +; GCN: v_mov_b32_e32 v1, v0 +; GCN: image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x2 tfe{{$}} +define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_2(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0) + %res.vec = extractvalue {<4 x float>,i32} %v, 0 + %res.f = extractelement <4 x float> %res.vec, i32 1 + %res.err = extractvalue {<4 x float>,i32} %v, 1 + %res.errf = bitcast i32 %res.err to float + %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0 + %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1 + ret <2 x float> %res +} + +; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_3: +; GCN: v_mov_b32_e32 v0, 0 +; GCN: v_mov_b32_e32 v1, v0 +; GCN: image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x4 tfe{{$}} +define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_3(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0) + %res.vec = extractvalue {<4 x float>,i32} %v, 0 + %res.f = extractelement <4 x float> %res.vec, i32 2 + %res.err = extractvalue {<4 x float>,i32} %v, 1 + %res.errf = bitcast i32 %res.err to float + %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0 + %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1 + ret <2 x float> %res +} + +; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_4: +; GCN: v_mov_b32_e32 v0, 0 +; GCN: v_mov_b32_e32 v1, v0 +; GCN: image_sample v[0:1], v2, s[0:7], s[8:11] dmask:0x8 tfe{{$}} +define amdgpu_ps <2 x float> @sample_1d_tfe_adjust_writemask_4(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0) + %res.vec = extractvalue {<4 x float>,i32} %v, 0 + %res.f = extractelement <4 x float> %res.vec, i32 3 + %res.err = extractvalue {<4 x float>,i32} %v, 1 + %res.errf = bitcast i32 %res.err to float + %res.tmp = insertelement <2 x float> undef, float %res.f, i32 0 + %res = insertelement <2 x float> %res.tmp, float %res.errf, i32 1 + ret <2 x float> %res +} + +; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_12: +; GCN: v_mov_b32_e32 v0, 0 +; GCN: v_mov_b32_e32 v1, v0 +; GCN: v_mov_b32_e32 v2, v0 +; GCN: image_sample v[0:2], v3, s[0:7], s[8:11] dmask:0x3 tfe{{$}} +define amdgpu_ps <4 x float> @sample_1d_tfe_adjust_writemask_12(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0) + %res.vec = extractvalue {<4 x float>,i32} %v, 0 + %res.f1 = extractelement <4 x float> %res.vec, i32 0 + %res.f2 = extractelement <4 x float> %res.vec, i32 1 + %res.err = extractvalue {<4 x float>,i32} %v, 1 + %res.errf = bitcast i32 %res.err to float + %res.tmp1 = insertelement <4 x float> undef, float %res.f1, i32 0 + %res.tmp2 = insertelement <4 x float> %res.tmp1, float %res.f2, i32 1 + %res = insertelement <4 x float> %res.tmp2, float %res.errf, i32 2 + ret <4 x float> %res +} + +; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_24: +; GCN: v_mov_b32_e32 v0, 0 +; GCN: v_mov_b32_e32 v1, v0 +; GCN: v_mov_b32_e32 v2, v0 +; GCN: image_sample v[0:2], v3, s[0:7], s[8:11] dmask:0xa tfe{{$}} +define amdgpu_ps <4 x float> @sample_1d_tfe_adjust_writemask_24(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0) + %res.vec = extractvalue {<4 x float>,i32} %v, 0 + %res.f1 = extractelement <4 x float> %res.vec, i32 1 + %res.f2 = extractelement <4 x float> %res.vec, i32 3 + %res.err = extractvalue {<4 x float>,i32} %v, 1 + %res.errf = bitcast i32 %res.err to float + %res.tmp1 = insertelement <4 x float> undef, float %res.f1, i32 0 + %res.tmp2 = insertelement <4 x float> %res.tmp1, float %res.f2, i32 1 + %res = insertelement <4 x float> %res.tmp2, float %res.errf, i32 2 + ret <4 x float> %res +} + +; GCN-LABEL: {{^}}sample_1d_tfe_adjust_writemask_134: +; GCN: v_mov_b32_e32 v0, 0 +; GCN: v_mov_b32_e32 v1, v0 +; GCN: v_mov_b32_e32 v2, v0 +; GCN: v_mov_b32_e32 v3, v0 +; GCN: image_sample v[0:3], v4, s[0:7], s[8:11] dmask:0xd tfe{{$}} +define amdgpu_ps <4 x float> @sample_1d_tfe_adjust_writemask_134(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 1, i32 0) + %res.vec = extractvalue {<4 x float>,i32} %v, 0 + %res.f1 = extractelement <4 x float> %res.vec, i32 0 + %res.f2 = extractelement <4 x float> %res.vec, i32 2 + %res.f3 = extractelement <4 x float> %res.vec, i32 3 + %res.err = extractvalue {<4 x float>,i32} %v, 1 + %res.errf = bitcast i32 %res.err to float + %res.tmp1 = insertelement <4 x float> undef, float %res.f1, i32 0 + %res.tmp2 = insertelement <4 x float> %res.tmp1, float %res.f2, i32 1 + %res.tmp3 = insertelement <4 x float> %res.tmp2, float %res.f3, i32 2 + %res = insertelement <4 x float> %res.tmp3, float %res.errf, i32 3 + ret <4 x float> %res +} + +; GCN-LABEL: {{^}}sample_1d_lwe: +; GCN: v_mov_b32_e32 v0, 0 +; GCN: v_mov_b32_e32 v1, v0 +; GCN: v_mov_b32_e32 v2, v0 +; GCN: v_mov_b32_e32 v3, v0 +; GCN: v_mov_b32_e32 v4, v0 +; GCN: image_sample v[0:7], v5, s[0:7], s[8:11] dmask:0xf lwe{{$}} +define amdgpu_ps <4 x float> @sample_1d_lwe(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, i32 addrspace(1)* inreg %out, float %s) { +main_body: + %v = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32 15, float %s, <8 x i32> %rsrc, <4 x i32> %samp, i1 0, i32 2, i32 0) + %v.vec = extractvalue {<4 x float>, i32} %v, 0 + %v.err = extractvalue {<4 x float>, i32} %v, 1 + store i32 %v.err, i32 addrspace(1)* %out, align 4 + ret <4 x float> %v.vec +} + ; GCN-LABEL: {{^}}sample_2d: ; GCN: image_sample v[0:3], v[0:1], s[0:7], s[8:11] dmask:0xf{{$}} define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, float %s, float %t) { @@ -491,6 +647,7 @@ } declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.v4f32i32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.sample.cube.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 Index: test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll =================================================================== --- test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll +++ test/Transforms/InstCombine/AMDGPU/amdgcn-demanded-vector-elts.ll @@ -328,6 +328,28 @@ ret float %elt0 } +; Check that the intrinsic remains unchanged in the presence of TFE or LWE +; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_tfe( +; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0) +; CHECK: ret float %elt0 +define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_tfe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 { + %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 1, i32 0) + %data.vec = extractvalue {<4 x float>,i32} %data, 0 + %elt0 = extractelement <4 x float> %data.vec, i32 0 + ret float %elt0 +} + +; Check that the intrinsic remains unchanged in the presence of TFE or LWE +; CHECK-LABEL: @extract_elt0_image_sample_1d_v4f32_f32_lwe( +; CHECK-NEXT: %data = call { <4 x float>, i32 } @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0) +; CHECK: ret float %elt0 +define amdgpu_ps float @extract_elt0_image_sample_1d_v4f32_f32_lwe(float %vaddr, <8 x i32> inreg %sampler, <4 x i32> inreg %rsrc) #0 { + %data = call {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32 15, float %vaddr, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 2, i32 0) + %data.vec = extractvalue {<4 x float>,i32} %data, 0 + %elt0 = extractelement <4 x float> %data.vec, i32 0 + ret float %elt0 +} + ; CHECK-LABEL: @extract_elt0_image_sample_2d_v4f32_f32( ; CHECK-NEXT: %data = call float @llvm.amdgcn.image.sample.2d.f32.f32(i32 1, float %s, float %t, <8 x i32> %sampler, <4 x i32> %rsrc, i1 false, i32 0, i32 0) ; CHECK-NEXT: ret float %data @@ -506,6 +528,7 @@ } declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 +declare {<4 x float>,i32} @llvm.amdgcn.image.sample.1d.sl_v4f32i32s.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.sample.3d.v4f32.f32(i32, float, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1 declare <4 x float> @llvm.amdgcn.image.sample.1darray.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #1