diff --git a/llvm/include/llvm/IR/IntrinsicsSPIRV.td b/llvm/include/llvm/IR/IntrinsicsSPIRV.td --- a/llvm/include/llvm/IR/IntrinsicsSPIRV.td +++ b/llvm/include/llvm/IR/IntrinsicsSPIRV.td @@ -32,4 +32,8 @@ def int_spv_unreachable : Intrinsic<[], []>; def int_spv_alloca : Intrinsic<[llvm_any_ty], []>; def int_spv_undef : Intrinsic<[llvm_i32_ty], []>; -} + + // Expect, Assume Intrinsics + def int_spv_assume : Intrinsic<[], [llvm_i1_ty]>; + def int_spv_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>]>; +} \ No newline at end of file diff --git a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td --- a/llvm/lib/Target/SPIRV/SPIRVBuiltins.td +++ b/llvm/lib/Target/SPIRV/SPIRVBuiltins.td @@ -54,6 +54,7 @@ def AsyncCopy : BuiltinGroup; def VectorLoadStore : BuiltinGroup; def LoadStore : BuiltinGroup; +def ExpectAssume : BuiltinGroup; //===----------------------------------------------------------------------===// // Class defining a demangled builtin record. The information in the record diff --git a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td --- a/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td +++ b/llvm/lib/Target/SPIRV/SPIRVInstrInfo.td @@ -757,3 +757,7 @@ def OpGroupNonUniformLogicalAnd: OpGroupNUGroup<"LogicalAnd", 362>; def OpGroupNonUniformLogicalOr: OpGroupNUGroup<"LogicalOr", 363>; def OpGroupNonUniformLogicalXor: OpGroupNUGroup<"LogicalXor", 364>; + +// SPV_KHR_expect_assume : Expect assume instructions +def OpAssumeTrueKHR: Op<5630, (outs), (ins ID:$cond), "OpAssumeTrueKHR $cond">; +def OpExpectKHR: Op<5631, (outs ID:$res), (ins TYPE:$ty, ID:$val, ID:$expected), "$res = OpExpectKHR $ty $val $expected">; diff --git a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp --- a/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVInstructionSelector.cpp @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +#include "MCTargetDesc/SPIRVMCTargetDesc.h" #include "SPIRV.h" #include "SPIRVGlobalRegistry.h" #include "SPIRVInstrInfo.h" @@ -1395,6 +1396,17 @@ break; case Intrinsic::spv_alloca: return selectFrameIndex(ResVReg, ResType, I); + case Intrinsic::spv_assume: + BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR)) + .addUse(I.getOperand(1).getReg()); + break; + case Intrinsic::spv_expect: + BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR)) + .addDef(ResVReg) + .addUse(GR.getSPIRVTypeID(ResType)) + .addUse(I.getOperand(2).getReg()) + .addUse(I.getOperand(3).getReg()); + break; default: llvm_unreachable("Intrinsic selection not implemented"); } diff --git a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp --- a/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVModuleAnalysis.cpp @@ -15,6 +15,8 @@ //===----------------------------------------------------------------------===// #include "SPIRVModuleAnalysis.h" +#include "MCTargetDesc/SPIRVBaseInfo.h" +#include "MCTargetDesc/SPIRVMCTargetDesc.h" #include "SPIRV.h" #include "SPIRVSubtarget.h" #include "SPIRVTargetMachine.h" @@ -555,6 +557,7 @@ // Add cap for SPV_INTEL_optnone. // FIXME: this should be added only if the target has the extension. addAvailableCaps({Capability::OptNoneINTEL}); + addAvailableCaps({Capability::ExpectAssumeKHR}); // TODO: add OpenCL extensions. } @@ -850,6 +853,11 @@ case SPIRV::OpGroupNonUniformBallotFindMSB: Reqs.addCapability(SPIRV::Capability::GroupNonUniformBallot); break; + case SPIRV::OpAssumeTrueKHR: + case SPIRV::OpExpectKHR: + Reqs.addExtension(SPIRV::Extension::SPV_KHR_expect_assume); + Reqs.addCapability(SPIRV::Capability::ExpectAssumeKHR); + break; default: break; } diff --git a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp --- a/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp @@ -24,6 +24,8 @@ #include "llvm/CodeGen/IntrinsicLowering.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsSPIRV.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" @@ -233,6 +235,32 @@ IRB.CreateRet(Res); } +static void lowerExpectAssume(IntrinsicInst *II) { + // If we cannot use the SPV_KHR_expect_assume extension, then we need to + // ignore the intrinsic and move on. It should be removed later on by LLVM. + // Otherwise we should lower the intrinsic to the corresponding SPIR-V + // instruction. + // For @llvm.assume we have OpAssumeTrueKHR. + // For @llvm.expect we have OpExpectKHR. + // + // We need to lower this into a builtin and then the builtin into a SPIR-V + // instruction. + if (II->getIntrinsicID() == Intrinsic::assume) { + Function *F = Intrinsic::getDeclaration( + II->getModule(), Intrinsic::SPVIntrinsics::spv_assume); + II->setCalledFunction(F); + } else if (II->getIntrinsicID() == Intrinsic::expect) { + Function *F = Intrinsic::getDeclaration( + II->getModule(), Intrinsic::SPVIntrinsics::spv_expect, + {II->getOperand(0)->getType()}); + II->setCalledFunction(F); + } else { + llvm_unreachable("Unknown intrinsic"); + } + + return; +} + static void lowerUMulWithOverflow(IntrinsicInst *UMulIntrinsic) { // Get a separate function - otherwise, we'd have to rework the CFG of the // current one. Then simply replace the intrinsic uses with a call to the new @@ -270,6 +298,10 @@ } else if (II->getIntrinsicID() == Intrinsic::umul_with_overflow) { lowerUMulWithOverflow(II); Changed = true; + } else if (II->getIntrinsicID() == Intrinsic::assume || + II->getIntrinsicID() == Intrinsic::expect) { + lowerExpectAssume(II); + Changed = true; } } } diff --git a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp --- a/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVSubtarget.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "SPIRVSubtarget.h" +#include "MCTargetDesc/SPIRVBaseInfo.h" #include "SPIRV.h" #include "SPIRVGlobalRegistry.h" #include "SPIRVLegalizerInfo.h" @@ -101,6 +102,7 @@ AvailableExtensions.insert( SPIRV::Extension::SPV_KHR_no_integer_wrap_decoration); AvailableExtensions.insert(SPIRV::Extension::SPV_INTEL_optnone); + AvailableExtensions.insert(SPIRV::Extension::SPV_KHR_expect_assume); } // TODO: use command line args for this rather than just defaults. diff --git a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td --- a/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td +++ b/llvm/lib/Target/SPIRV/SPIRVSymbolicOperands.td @@ -244,6 +244,7 @@ defm SPV_EXT_demote_to_helper_invocation : ExtensionOperand<56>; defm SPV_INTEL_fpga_reg : ExtensionOperand<57>; defm SPV_INTEL_optnone : ExtensionOperand<58>; +defm SPV_KHR_expect_assume : ExtensionOperand<60>; //===----------------------------------------------------------------------===// // Multiclass used to define Capabilities enum values and at the same time @@ -398,6 +399,7 @@ defm PhysicalStorageBufferAddressesEXT : CapabilityOperand<5347, 0, 0, [], [Shader]>; defm CooperativeMatrixNV : CapabilityOperand<5357, 0, 0, [], [Shader]>; defm OptNoneINTEL : CapabilityOperand<6094, 0, 0, [SPV_INTEL_optnone], []>; +defm ExpectAssumeKHR : CapabilityOperand<5629, 0, 0, [SPV_KHR_expect_assume], []>; //===----------------------------------------------------------------------===// // Multiclass used to define SourceLanguage enum values and at the same time diff --git a/llvm/test/CodeGen/SPIRV/assume.ll b/llvm/test/CodeGen/SPIRV/assume.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/assume.ll @@ -0,0 +1,15 @@ +; RUN: llc -mtriple=spirv32-unknown-unknown < %s | FileCheck %s +; RUN: llc -mtriple=spirv64-unknown-unknown < %s | FileCheck %s + +; CHECK: OpCapability ExpectAssumeKHR +; CHECK-NEXT: OpExtension "SPV_KHR_expect_assume" + +declare void @llvm.assume(i1) + +; CHECK-DAG: %9 = OpIEqual %5 %6 %7 +; CHECK-NEXT: OpAssumeTrueKHR %9 +define void @assumeeq(i32 %x, i32 %y) { + %cmp = icmp eq i32 %x, %y + call void @llvm.assume(i1 %cmp) + ret void +} \ No newline at end of file diff --git a/llvm/test/CodeGen/SPIRV/expect.ll b/llvm/test/CodeGen/SPIRV/expect.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/SPIRV/expect.ll @@ -0,0 +1,20 @@ +; RUN: llc -mtriple=spirv32-unknown-unknown < %s | FileCheck %s +; RUN: llc -mtriple=spirv64-unknown-unknown < %s | FileCheck %s + +; CHECK: OpCapability ExpectAssumeKHR +; CHECK-NEXT: OpExtension "SPV_KHR_expect_assume" + +declare i32 @llvm.expect.i32(i32, i32) +declare i32 @getOne() + +; CHECK-DAG: %2 = OpTypeInt 32 0 +; CHECK-DAG: %6 = OpFunctionParameter %2 +; CHECK-DAG: %9 = OpIMul %2 %6 %8 +; CHECK-DAG: %10 = OpExpectKHR %2 %9 %6 + +define i32 @test(i32 %x) { + %one = call i32 @getOne() + %val = mul i32 %x, %one + %v = call i32 @llvm.expect.i32(i32 %val, i32 %x) + ret i32 %v +}