Index: cfe/trunk/include/clang/Basic/BuiltinsNVPTX.def =================================================================== --- cfe/trunk/include/clang/Basic/BuiltinsNVPTX.def +++ cfe/trunk/include/clang/Basic/BuiltinsNVPTX.def @@ -378,6 +378,9 @@ BUILTIN(__nvvm_bar0_and, "ii", "") BUILTIN(__nvvm_bar0_or, "ii", "") BUILTIN(__nvvm_bar_sync, "vi", "n") +TARGET_BUILTIN(__nvvm_bar_warp_sync, "vUi", "n", "ptx60") +TARGET_BUILTIN(__nvvm_barrier_sync, "vUi", "n", "ptx60") +TARGET_BUILTIN(__nvvm_barrier_sync_cnt, "vUiUi", "n", "ptx60") // Shuffle @@ -399,6 +402,17 @@ TARGET_BUILTIN(__nvvm_shfl_sync_idx_i32, "iUiiii", "", "ptx60") TARGET_BUILTIN(__nvvm_shfl_sync_idx_f32, "fUifii", "", "ptx60") +// Vote +BUILTIN(__nvvm_vote_all, "bb", "") +BUILTIN(__nvvm_vote_any, "bb", "") +BUILTIN(__nvvm_vote_uni, "bb", "") +BUILTIN(__nvvm_vote_ballot, "Uib", "") + +TARGET_BUILTIN(__nvvm_vote_all_sync, "bUib", "", "ptx60") +TARGET_BUILTIN(__nvvm_vote_any_sync, "bUib", "", "ptx60") +TARGET_BUILTIN(__nvvm_vote_uni_sync, "bUib", "", "ptx60") +TARGET_BUILTIN(__nvvm_vote_ballot_sync, "UiUib", "", "ptx60") + // Membar BUILTIN(__nvvm_membar_cta, "v", "") Index: cfe/trunk/lib/Headers/__clang_cuda_intrinsics.h =================================================================== --- cfe/trunk/lib/Headers/__clang_cuda_intrinsics.h +++ cfe/trunk/lib/Headers/__clang_cuda_intrinsics.h @@ -157,6 +157,37 @@ #pragma pop_macro("__MAKE_SYNC_SHUFFLES") +inline __device__ void __syncwarp(unsigned int mask = 0xffffffff) { + return __nvvm_bar_warp_sync(mask); +} + +inline __device__ void __barrier_sync(unsigned int id) { + __nvvm_barrier_sync(id); +} + +inline __device__ void __barrier_sync_count(unsigned int id, + unsigned int count) { + __nvvm_barrier_sync_cnt(id, count); +} + +inline __device__ int __all_sync(unsigned int mask, int pred) { + return __nvvm_vote_sync_all(mask, pred); +} + +inline __device__ int __any_sync(unsigned int mask, int pred) { + return __nvvm_vote_sync_any(mask, pred); +} + +inline __device__ int __uni_sync(unsigned int mask, int pred) { + return __nvvm_vote_sync_uni(mask, pred); +} + +inline __device__ unsigned int __ballot_sync(unsigned int mask, int pred) { + return __nvvm_vote_sync_ballot(mask, pred); +} + +inline __device__ activemask() { return __nvvm_vote.ballot(1); } + #endif // __CUDA_VERSION >= 9000 && (!defined(__CUDA_ARCH__) || // __CUDA_ARCH__ >= 300) Index: cfe/trunk/test/CodeGen/builtins-nvptx-ptx60.cu =================================================================== --- cfe/trunk/test/CodeGen/builtins-nvptx-ptx60.cu +++ cfe/trunk/test/CodeGen/builtins-nvptx-ptx60.cu @@ -10,8 +10,27 @@ #define __shared__ __attribute__((shared)) #define __constant__ __attribute__((constant)) -// CHECK-LABEL: nvvm_shfl_sync -__device__ void nvvm_shfl_sync(unsigned mask, int i, float f, int a, int b) { +// We have to keep all builtins that depend on particular target feature in the +// same function, because the codegen will stop after the very first function +// that encounters an error, so -verify will not be able to find errors in +// subsequent functions. + +// CHECK-LABEL: nvvm_sync +__device__ void nvvm_sync(unsigned mask, int i, float f, int a, int b, + bool pred) { + // CHECK: call void @llvm.nvvm.bar.warp.sync(i32 + // expected-error@+1 {{'__nvvm_bar_warp_sync' needs target feature ptx60}} + __nvvm_bar_warp_sync(mask); + // CHECK: call void @llvm.nvvm.barrier.sync(i32 + // expected-error@+1 {{'__nvvm_barrier_sync' needs target feature ptx60}} + __nvvm_barrier_sync(mask); + // CHECK: call void @llvm.nvvm.barrier.sync.cnt(i32 + // expected-error@+1 {{'__nvvm_barrier_sync_cnt' needs target feature ptx60}} + __nvvm_barrier_sync_cnt(mask, i); + + // + // SHFL.SYNC + // // CHECK: call i32 @llvm.nvvm.shfl.sync.down.i32(i32 {{%[0-9]+}}, i32 // expected-error@+1 {{'__nvvm_shfl_sync_down_i32' needs target feature ptx60}} __nvvm_shfl_sync_down_i32(mask, i, a, b); @@ -36,5 +55,23 @@ // CHECK: call float @llvm.nvvm.shfl.sync.idx.f32(i32 {{%[0-9]+}}, float // expected-error@+1 {{'__nvvm_shfl_sync_idx_f32' needs target feature ptx60}} __nvvm_shfl_sync_idx_f32(mask, f, a, b); + + // + // VOTE.SYNC + // + + // CHECK: call i1 @llvm.nvvm.vote.all.sync(i32 + // expected-error@+1 {{'__nvvm_vote_all_sync' needs target feature ptx60}} + __nvvm_vote_all_sync(mask, pred); + // CHECK: call i1 @llvm.nvvm.vote.any.sync(i32 + // expected-error@+1 {{'__nvvm_vote_any_sync' needs target feature ptx60}} + __nvvm_vote_any_sync(mask, pred); + // CHECK: call i1 @llvm.nvvm.vote.uni.sync(i32 + // expected-error@+1 {{'__nvvm_vote_uni_sync' needs target feature ptx60}} + __nvvm_vote_uni_sync(mask, pred); + // CHECK: call i32 @llvm.nvvm.vote.ballot.sync(i32 + // expected-error@+1 {{'__nvvm_vote_ballot_sync' needs target feature ptx60}} + __nvvm_vote_ballot_sync(mask, pred); + // CHECK: ret void } Index: cfe/trunk/test/CodeGen/builtins-nvptx.c =================================================================== --- cfe/trunk/test/CodeGen/builtins-nvptx.c +++ cfe/trunk/test/CodeGen/builtins-nvptx.c @@ -657,3 +657,15 @@ __nvvm_shfl_idx_f32(f, a, b); // CHECK: ret void } + +__device__ void nvvm_vote(int pred) { + // CHECK: call i1 @llvm.nvvm.vote.all(i1 + __nvvm_vote_all(pred); + // CHECK: call i1 @llvm.nvvm.vote.any(i1 + __nvvm_vote_any(pred); + // CHECK: call i1 @llvm.nvvm.vote.uni(i1 + __nvvm_vote_uni(pred); + // CHECK: call i32 @llvm.nvvm.vote.ballot(i1 + __nvvm_vote_ballot(pred); + // CHECK: ret void +} Index: llvm/trunk/include/llvm/IR/IntrinsicsNVVM.td =================================================================== --- llvm/trunk/include/llvm/IR/IntrinsicsNVVM.td +++ llvm/trunk/include/llvm/IR/IntrinsicsNVVM.td @@ -750,6 +750,17 @@ def int_nvvm_bar_sync : Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, GCCBuiltin<"__nvvm_bar_sync">; + def int_nvvm_bar_warp_sync : + Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, + GCCBuiltin<"__nvvm_bar_warp_sync">; + + // barrier.sync id[, cnt] + def int_nvvm_barrier_sync : + Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>, + GCCBuiltin<"__nvvm_barrier_sync">; + def int_nvvm_barrier_sync_cnt : + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent]>, + GCCBuiltin<"__nvvm_barrier_sync_cnt">; // Membar def int_nvvm_membar_cta : GCCBuiltin<"__nvvm_membar_cta">, @@ -3780,4 +3791,55 @@ Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.sync.idx.f32">, GCCBuiltin<"__nvvm_shfl_sync_idx_f32">; -} + +// +// VOTE +// + +// vote.all pred +def int_nvvm_vote_all : + Intrinsic<[llvm_i1_ty], [llvm_i1_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.all">, + GCCBuiltin<"__nvvm_vote_all">; +// vote.any pred +def int_nvvm_vote_any : + Intrinsic<[llvm_i1_ty], [llvm_i1_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.any">, + GCCBuiltin<"__nvvm_vote_any">; +// vote.uni pred +def int_nvvm_vote_uni : + Intrinsic<[llvm_i1_ty], [llvm_i1_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.uni">, + GCCBuiltin<"__nvvm_vote_uni">; +// vote.ballot pred +def int_nvvm_vote_ballot : + Intrinsic<[llvm_i32_ty], [llvm_i1_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.ballot">, + GCCBuiltin<"__nvvm_vote_ballot">; + +// +// VOTE.SYNC +// + +// vote.sync.all mask, pred +def int_nvvm_vote_all_sync : + Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.all.sync">, + GCCBuiltin<"__nvvm_vote_all_sync">; +// vote.sync.any mask, pred +def int_nvvm_vote_any_sync : + Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.any.sync">, + GCCBuiltin<"__nvvm_vote_any_sync">; +// vote.sync.uni mask, pred +def int_nvvm_vote_uni_sync : + Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.uni.sync">, + GCCBuiltin<"__nvvm_vote_uni_sync">; +// vote.sync.ballot mask, pred +def int_nvvm_vote_ballot_sync : + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i1_ty], + [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.ballot.sync">, + GCCBuiltin<"__nvvm_vote_ballot_sync">; + +} // let TargetPrefix = "nvvm" Index: llvm/trunk/lib/Target/NVPTX/NVPTXInstrInfo.td =================================================================== --- llvm/trunk/lib/Target/NVPTX/NVPTXInstrInfo.td +++ llvm/trunk/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -155,6 +155,9 @@ def true : Predicate<"true">; def hasPTX31 : Predicate<"Subtarget->getPTXVersion() >= 31">; +def hasPTX60 : Predicate<"Subtarget->getPTXVersion() >= 60">; + +def hasSM30 : Predicate<"Subtarget->getSmVersion() >= 30">; def useFP16Math: Predicate<"Subtarget->allowFP16Math()">; Index: llvm/trunk/lib/Target/NVPTX/NVPTXIntrinsics.td =================================================================== --- llvm/trunk/lib/Target/NVPTX/NVPTXIntrinsics.td +++ llvm/trunk/lib/Target/NVPTX/NVPTXIntrinsics.td @@ -71,6 +71,38 @@ def INT_BAR_SYNC : NVPTXInst<(outs), (ins i32imm:$i), "bar.sync \t$i;", [(int_nvvm_bar_sync imm:$i)]>; +def INT_BAR_WARP_SYNC_I : NVPTXInst<(outs), (ins i32imm:$i), "bar.warp.sync \t$i;", + [(int_nvvm_bar_warp_sync imm:$i)]>, + Requires<[hasPTX60, hasSM30]>; +def INT_BAR_WARP_SYNC_R : NVPTXInst<(outs), (ins Int32Regs:$i), "bar.warp.sync \t$i;", + [(int_nvvm_bar_warp_sync Int32Regs:$i)]>, + Requires<[hasPTX60, hasSM30]>; + +def INT_BARRIER_SYNC_I : NVPTXInst<(outs), (ins i32imm:$i), "barrier.sync \t$i;", + [(int_nvvm_barrier_sync imm:$i)]>, + Requires<[hasPTX60, hasSM30]>; +def INT_BARRIER_SYNC_R : NVPTXInst<(outs), (ins Int32Regs:$i), "barrier.sync \t$i;", + [(int_nvvm_barrier_sync Int32Regs:$i)]>, + Requires<[hasPTX60, hasSM30]>; + +def INT_BARRIER_SYNC_CNT_RR : NVPTXInst<(outs), (ins Int32Regs:$id, Int32Regs:$cnt), + "barrier.sync \t$id, $cnt;", + [(int_nvvm_barrier_sync_cnt Int32Regs:$id, Int32Regs:$cnt)]>, + Requires<[hasPTX60, hasSM30]>; +def INT_BARRIER_SYNC_CNT_RI : NVPTXInst<(outs), (ins Int32Regs:$id, i32imm:$cnt), + "barrier.sync \t$id, $cnt;", + [(int_nvvm_barrier_sync_cnt Int32Regs:$id, imm:$cnt)]>, + Requires<[hasPTX60, hasSM30]>; +def INT_BARRIER_SYNC_CNT_IR : NVPTXInst<(outs), (ins i32imm:$id, Int32Regs:$cnt), + "barrier.sync \t$id, $cnt;", + [(int_nvvm_barrier_sync_cnt imm:$id, Int32Regs:$cnt)]>, + Requires<[hasPTX60, hasSM30]>; +def INT_BARRIER_SYNC_CNT_II : NVPTXInst<(outs), (ins i32imm:$id, i32imm:$cnt), + "barrier.sync \t$id, $cnt;", + [(int_nvvm_barrier_sync_cnt imm:$id, imm:$cnt)]>, + Requires<[hasPTX60, hasSM30]>; + + // shfl.{up,down,bfly,idx}.b32 multiclass SHFL { // The last two parameters to shfl can be regs or imms. ptxas is smart @@ -184,6 +216,37 @@ defm INT_SHFL_SYNC_IDX_I32 : SHFL_SYNC; defm INT_SHFL_SYNC_IDX_F32 : SHFL_SYNC; + +// vote.{all,any,uni,ballot} +multiclass VOTE { + def : NVPTXInst<(outs regclass:$dest), (ins Int1Regs:$pred), + "vote." # mode # " \t$dest, $pred;", + [(set regclass:$dest, (IntOp Int1Regs:$pred))]>, + Requires<[hasPTX60, hasSM30]>; +} + +defm VOTE_ALL : VOTE; +defm VOTE_ANY : VOTE; +defm VOTE_UNI : VOTE; +defm VOTE_BALLOT : VOTE; + +// vote.sync.{all,any,uni,ballot} +multiclass VOTE_SYNC { + def i : NVPTXInst<(outs regclass:$dest), (ins i32imm:$mask, Int1Regs:$pred), + "vote.sync." # mode # " \t$dest, $pred, $mask;", + [(set regclass:$dest, (IntOp imm:$mask, Int1Regs:$pred))]>, + Requires<[hasPTX60, hasSM30]>; + def r : NVPTXInst<(outs regclass:$dest), (ins Int32Regs:$mask, Int1Regs:$pred), + "vote.sync." # mode #" \t$dest, $pred, $mask;", + [(set regclass:$dest, (IntOp Int32Regs:$mask, Int1Regs:$pred))]>, + Requires<[hasPTX60, hasSM30]>; +} + +defm VOTE_SYNC_ALL : VOTE_SYNC; +defm VOTE_SYNC_ANY : VOTE_SYNC; +defm VOTE_SYNC_UNI : VOTE_SYNC; +defm VOTE_SYNC_BALLOT : VOTE_SYNC; + } // isConvergent = 1 //----------------------------------- Index: llvm/trunk/test/CodeGen/NVPTX/barrier.ll =================================================================== --- llvm/trunk/test/CodeGen/NVPTX/barrier.ll +++ llvm/trunk/test/CodeGen/NVPTX/barrier.ll @@ -0,0 +1,32 @@ +; RUN: llc < %s -march=nvptx64 -mcpu=sm_30 -mattr=+ptx60 | FileCheck %s + +declare void @llvm.nvvm.bar.warp.sync(i32) +declare void @llvm.nvvm.barrier.sync(i32) +declare void @llvm.nvvm.barrier.sync.cnt(i32, i32) + +; CHECK-LABEL: .func{{.*}}barrier.sync +define void @barrier.sync(i32 %id, i32 %cnt) { + ; CHECK: ld.param.u32 [[ID:%r[0-9]+]], [barrier.sync_param_0]; + ; CHECK: ld.param.u32 [[CNT:%r[0-9]+]], [barrier.sync_param_1]; + + ; CHECK: barrier.sync [[ID]], [[CNT]]; + call void @llvm.nvvm.barrier.sync.cnt(i32 %id, i32 %cnt) + ; CHECK: barrier.sync [[ID]], 2; + call void @llvm.nvvm.barrier.sync.cnt(i32 %id, i32 2) + ; CHECK: barrier.sync 3, [[CNT]]; + call void @llvm.nvvm.barrier.sync.cnt(i32 3, i32 %cnt) + ; CHECK: barrier.sync 4, 5; + call void @llvm.nvvm.barrier.sync.cnt(i32 4, i32 5) + + ; CHECK: barrier.sync [[ID]]; + call void @llvm.nvvm.barrier.sync(i32 %id) + ; CHECK: barrier.sync 1; + call void @llvm.nvvm.barrier.sync(i32 1) + + ; CHECK: bar.warp.sync [[ID]]; + call void @llvm.nvvm.bar.warp.sync(i32 %id) + ; CHECK: bar.warp.sync 6; + call void @llvm.nvvm.bar.warp.sync(i32 6) + ret void; +} + Index: llvm/trunk/test/CodeGen/NVPTX/vote.ll =================================================================== --- llvm/trunk/test/CodeGen/NVPTX/vote.ll +++ llvm/trunk/test/CodeGen/NVPTX/vote.ll @@ -0,0 +1,65 @@ +; RUN: llc < %s -march=nvptx64 -mcpu=sm_30 -mattr=+ptx60 | FileCheck %s + +declare i1 @llvm.nvvm.vote.all(i1) +; CHECK-LABEL: .func{{.*}}vote.all +define i1 @vote.all(i1 %pred) { + ; CHECK: vote.all.pred + %val = call i1 @llvm.nvvm.vote.all(i1 %pred) + ret i1 %val +} + +declare i1 @llvm.nvvm.vote.any(i1) +; CHECK-LABEL: .func{{.*}}vote.any +define i1 @vote.any(i1 %pred) { + ; CHECK: vote.any.pred + %val = call i1 @llvm.nvvm.vote.any(i1 %pred) + ret i1 %val +} + +declare i1 @llvm.nvvm.vote.uni(i1) +; CHECK-LABEL: .func{{.*}}vote.uni +define i1 @vote.uni(i1 %pred) { + ; CHECK: vote.uni.pred + %val = call i1 @llvm.nvvm.vote.uni(i1 %pred) + ret i1 %val +} + +declare i32 @llvm.nvvm.vote.ballot(i1) +; CHECK-LABEL: .func{{.*}}vote.ballot +define i32 @vote.ballot(i1 %pred) { + ; CHECK: vote.ballot.b32 + %val = call i32 @llvm.nvvm.vote.ballot(i1 %pred) + ret i32 %val +} + +declare i1 @llvm.nvvm.vote.all.sync(i32, i1) +; CHECK-LABEL: .func{{.*}}vote.sync.all +define i1 @vote.sync.all(i32 %mask, i1 %pred) { + ; CHECK: vote.sync.all.pred + %val = call i1 @llvm.nvvm.vote.all.sync(i32 %mask, i1 %pred) + ret i1 %val +} + +declare i1 @llvm.nvvm.vote.any.sync(i32, i1) +; CHECK-LABEL: .func{{.*}}vote.sync.any +define i1 @vote.sync.any(i32 %mask, i1 %pred) { + ; CHECK: vote.sync.any.pred + %val = call i1 @llvm.nvvm.vote.any.sync(i32 %mask, i1 %pred) + ret i1 %val +} + +declare i1 @llvm.nvvm.vote.uni.sync(i32, i1) +; CHECK-LABEL: .func{{.*}}vote.sync.uni +define i1 @vote.sync.uni(i32 %mask, i1 %pred) { + ; CHECK: vote.sync.uni.pred + %val = call i1 @llvm.nvvm.vote.uni.sync(i32 %mask, i1 %pred) + ret i1 %val +} + +declare i32 @llvm.nvvm.vote.ballot.sync(i32, i1) +; CHECK-LABEL: .func{{.*}}vote.sync.ballot +define i32 @vote.sync.ballot(i32 %mask, i1 %pred) { + ; CHECK: vote.sync.ballot.b32 + %val = call i32 @llvm.nvvm.vote.ballot.sync(i32 %mask, i1 %pred) + ret i32 %val +}