[NVPTX] Implemented bar.warp.sync, barrier.sync, and vote{.sync} instructions/intrinsics/builtins.

[tra]

Fixed a typo in one test.

cfe/trunk/include/clang/Basic/BuiltinsNVPTX.def

 
 // Sync
 
 BUILTIN(__syncthreads, "v", "")
 BUILTIN(__nvvm_bar0_popc, "ii", "")
 BUILTIN(__nvvm_bar0_and, "ii", "")
 BUILTIN(__nvvm_bar0_or, "ii", "")
 BUILTIN(__nvvm_bar_sync, "vi", "n")
+TARGET_BUILTIN(__nvvm_bar_warp_sync, "vUi", "n", "ptx60")
+TARGET_BUILTIN(__nvvm_barrier_sync, "vUi", "n", "ptx60")
+TARGET_BUILTIN(__nvvm_barrier_sync_cnt, "vUiUi", "n", "ptx60")
 
 // Shuffle
 
 BUILTIN(__nvvm_shfl_down_i32, "iiii", "")
 BUILTIN(__nvvm_shfl_down_f32, "ffii", "")
 BUILTIN(__nvvm_shfl_up_i32, "iiii", "")
 BUILTIN(__nvvm_shfl_up_f32, "ffii", "")
 BUILTIN(__nvvm_shfl_bfly_i32, "iiii", "")
 BUILTIN(__nvvm_shfl_bfly_f32, "ffii", "")
 BUILTIN(__nvvm_shfl_idx_i32, "iiii", "")
 BUILTIN(__nvvm_shfl_idx_f32, "ffii", "")
 
 TARGET_BUILTIN(__nvvm_shfl_sync_down_i32, "iUiiii", "", "ptx60")
 TARGET_BUILTIN(__nvvm_shfl_sync_down_f32, "fUifii", "", "ptx60")
 TARGET_BUILTIN(__nvvm_shfl_sync_up_i32, "iUiiii", "", "ptx60")
 TARGET_BUILTIN(__nvvm_shfl_sync_up_f32, "fUifii", "", "ptx60")
 TARGET_BUILTIN(__nvvm_shfl_sync_bfly_i32, "iUiiii", "", "ptx60")
 TARGET_BUILTIN(__nvvm_shfl_sync_bfly_f32, "fUifii", "", "ptx60")
 TARGET_BUILTIN(__nvvm_shfl_sync_idx_i32, "iUiiii", "", "ptx60")
 TARGET_BUILTIN(__nvvm_shfl_sync_idx_f32, "fUifii", "", "ptx60")
 
+// Vote
+BUILTIN(__nvvm_vote_all, "bb", "")
+BUILTIN(__nvvm_vote_any, "bb", "")
+BUILTIN(__nvvm_vote_uni, "bb", "")
+BUILTIN(__nvvm_vote_ballot, "Uib", "")
+
+TARGET_BUILTIN(__nvvm_vote_all_sync, "bUib", "", "ptx60")
+TARGET_BUILTIN(__nvvm_vote_any_sync, "bUib", "", "ptx60")
+TARGET_BUILTIN(__nvvm_vote_uni_sync, "bUib", "", "ptx60")
+TARGET_BUILTIN(__nvvm_vote_ballot_sync, "UiUib", "", "ptx60")
+
 // Membar
 
 BUILTIN(__nvvm_membar_cta, "v", "")
 BUILTIN(__nvvm_membar_gl, "v", "")
 BUILTIN(__nvvm_membar_sys, "v", "")
 
 // Memcpy, Memset
 

cfe/trunk/lib/Headers/__clang_cuda_intrinsics.h

__MAKE_SYNC_SHUFFLES(__shfl_sync_up, __nvvm_shfl_sync_up_i32,
                      __nvvm_shfl_sync_up_f32, 0);
 __MAKE_SYNC_SHUFFLES(__shfl_sync_down, __nvvm_shfl_sync_down_i32,
                      __nvvm_shfl_sync_down_f32, 0x1f);
 __MAKE_SYNC_SHUFFLES(__shfl_sync_xor, __nvvm_shfl_sync_bfly_i32,
                      __nvvm_shfl_sync_bfly_f32, 0x1f);
 
 #pragma pop_macro("__MAKE_SYNC_SHUFFLES")
 
+inline __device__ void __syncwarp(unsigned int mask = 0xffffffff) {
+  return __nvvm_bar_warp_sync(mask);
+}
+
+inline __device__ void __barrier_sync(unsigned int id) {
+  __nvvm_barrier_sync(id);
+}
+
+inline __device__ void __barrier_sync_count(unsigned int id,
+                                            unsigned int count) {
+  __nvvm_barrier_sync_cnt(id, count);
+}
+
+inline __device__ int __all_sync(unsigned int mask, int pred) {
+  return __nvvm_vote_sync_all(mask, pred);
+}
+
+inline __device__ int __any_sync(unsigned int mask, int pred) {
+  return __nvvm_vote_sync_any(mask, pred);
+}
+
+inline __device__ int __uni_sync(unsigned int mask, int pred) {
+  return __nvvm_vote_sync_uni(mask, pred);
+}
+
+inline __device__ unsigned int __ballot_sync(unsigned int mask, int pred) {
+  return __nvvm_vote_sync_ballot(mask, pred);
+}
+
+inline __device__ activemask() { return __nvvm_vote.ballot(1); }
+
 #endif // __CUDA_VERSION >= 9000 && (!defined(__CUDA_ARCH__) ||
        // __CUDA_ARCH__ >= 300)
 
 // sm_32 intrinsics: __ldg and __funnelshift_{l,lc,r,rc}.
 
 // Prevent the vanilla sm_32 intrinsics header from being included.
 #define __SM_32_INTRINSICS_H__
 #define __SM_32_INTRINSICS_HPP__

cfe/trunk/test/CodeGen/builtins-nvptx-ptx60.cu

 // RUN: %clang_cc1 -triple nvptx64-unknown-unknown -target-cpu sm_60 \
 // RUN:            -fcuda-is-device -target-feature +ptx60 \
 // RUN:            -S -emit-llvm -o - -x cuda %s \
 // RUN:   | FileCheck -check-prefix=CHECK %s
 // RUN: %clang_cc1 -triple nvptx-unknown-unknown -target-cpu sm_60 \
 // RUN:   -fcuda-is-device -S -o /dev/null -x cuda -verify %s
 
 #define __device__ __attribute__((device))
 #define __global__ __attribute__((global))
 #define __shared__ __attribute__((shared))
 #define __constant__ __attribute__((constant))
 
-// CHECK-LABEL: nvvm_shfl_sync
-__device__ void nvvm_shfl_sync(unsigned mask, int i, float f, int a, int b) {
+// We have to keep all builtins that depend on particular target feature in the
+// same function, because the codegen will stop after the very first function
+// that encounters an error, so -verify will not be able to find errors in
+// subsequent functions.
+
+// CHECK-LABEL: nvvm_sync
+__device__ void nvvm_sync(unsigned mask, int i, float f, int a, int b,
+                          bool pred) {
+  // CHECK: call void @llvm.nvvm.bar.warp.sync(i32
+  // expected-error@+1 {{'__nvvm_bar_warp_sync' needs target feature ptx60}}
+  __nvvm_bar_warp_sync(mask);
+  // CHECK: call void @llvm.nvvm.barrier.sync(i32
+  // expected-error@+1 {{'__nvvm_barrier_sync' needs target feature ptx60}}
+  __nvvm_barrier_sync(mask);
+  // CHECK: call void @llvm.nvvm.barrier.sync.cnt(i32
+  // expected-error@+1 {{'__nvvm_barrier_sync_cnt' needs target feature ptx60}}
+  __nvvm_barrier_sync_cnt(mask, i);
+
+  //
+  // SHFL.SYNC
+  //
   // CHECK: call i32 @llvm.nvvm.shfl.sync.down.i32(i32 {{%[0-9]+}}, i32
   // expected-error@+1 {{'__nvvm_shfl_sync_down_i32' needs target feature ptx60}}
   __nvvm_shfl_sync_down_i32(mask, i, a, b);
   // CHECK: call float @llvm.nvvm.shfl.sync.down.f32(i32 {{%[0-9]+}}, float
   // expected-error@+1 {{'__nvvm_shfl_sync_down_f32' needs target feature ptx60}}
   __nvvm_shfl_sync_down_f32(mask, f, a, b);
   // CHECK: call i32 @llvm.nvvm.shfl.sync.up.i32(i32 {{%[0-9]+}}, i32
   // expected-error@+1 {{'__nvvm_shfl_sync_up_i32' needs target feature ptx60}}
   __nvvm_shfl_sync_up_i32(mask, i, a, b);
   // CHECK: call float @llvm.nvvm.shfl.sync.up.f32(i32 {{%[0-9]+}}, float
   // expected-error@+1 {{'__nvvm_shfl_sync_up_f32' needs target feature ptx60}}
   __nvvm_shfl_sync_up_f32(mask, f, a, b);
   // CHECK: call i32 @llvm.nvvm.shfl.sync.bfly.i32(i32 {{%[0-9]+}}, i32
   // expected-error@+1 {{'__nvvm_shfl_sync_bfly_i32' needs target feature ptx60}}
   __nvvm_shfl_sync_bfly_i32(mask, i, a, b);
   // CHECK: call float @llvm.nvvm.shfl.sync.bfly.f32(i32 {{%[0-9]+}}, float
   // expected-error@+1 {{'__nvvm_shfl_sync_bfly_f32' needs target feature ptx60}}
   __nvvm_shfl_sync_bfly_f32(mask, f, a, b);
   // CHECK: call i32 @llvm.nvvm.shfl.sync.idx.i32(i32 {{%[0-9]+}}, i32
   // expected-error@+1 {{'__nvvm_shfl_sync_idx_i32' needs target feature ptx60}}
   __nvvm_shfl_sync_idx_i32(mask, i, a, b);
   // CHECK: call float @llvm.nvvm.shfl.sync.idx.f32(i32 {{%[0-9]+}}, float
   // expected-error@+1 {{'__nvvm_shfl_sync_idx_f32' needs target feature ptx60}}
   __nvvm_shfl_sync_idx_f32(mask, f, a, b);
+
+  //
+  // VOTE.SYNC
+  //
+
+  // CHECK: call i1 @llvm.nvvm.vote.all.sync(i32
+  // expected-error@+1 {{'__nvvm_vote_all_sync' needs target feature ptx60}}
+  __nvvm_vote_all_sync(mask, pred);
+  // CHECK: call i1 @llvm.nvvm.vote.any.sync(i32
+  // expected-error@+1 {{'__nvvm_vote_any_sync' needs target feature ptx60}}
+  __nvvm_vote_any_sync(mask, pred);
+  // CHECK: call i1 @llvm.nvvm.vote.uni.sync(i32
+  // expected-error@+1 {{'__nvvm_vote_uni_sync' needs target feature ptx60}}
+  __nvvm_vote_uni_sync(mask, pred);
+  // CHECK: call i32 @llvm.nvvm.vote.ballot.sync(i32
+  // expected-error@+1 {{'__nvvm_vote_ballot_sync' needs target feature ptx60}}
+  __nvvm_vote_ballot_sync(mask, pred);
+
   // CHECK: ret void
 }

cfe/trunk/test/CodeGen/builtins-nvptx.c

__device__ void nvvm_shfl(int i, float f, int a, int b) {
   // CHECK: call float @llvm.nvvm.shfl.bfly.f32(float
   __nvvm_shfl_bfly_f32(f, a, b);
   // CHECK: call i32 @llvm.nvvm.shfl.idx.i32(i32
   __nvvm_shfl_idx_i32(i, a, b);
   // CHECK: call float @llvm.nvvm.shfl.idx.f32(float
   __nvvm_shfl_idx_f32(f, a, b);
   // CHECK: ret void
 }
+
+__device__ void nvvm_vote(int pred) {
+  // CHECK: call i1 @llvm.nvvm.vote.all(i1
+  __nvvm_vote_all(pred);
+  // CHECK: call i1 @llvm.nvvm.vote.any(i1
+  __nvvm_vote_any(pred);
+  // CHECK: call i1 @llvm.nvvm.vote.uni(i1
+  __nvvm_vote_uni(pred);
+  // CHECK: call i32 @llvm.nvvm.vote.ballot(i1
+  __nvvm_vote_ballot(pred);
+  // CHECK: ret void
+}

llvm/trunk/include/llvm/IR/IntrinsicsNVVM.td

// Bar.Sync
   def int_nvvm_barrier0_and : GCCBuiltin<"__nvvm_bar0_and">,
       Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>;
   def int_nvvm_barrier0_or : GCCBuiltin<"__nvvm_bar0_or">,
       Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrConvergent]>;
 
   def int_nvvm_bar_sync :
       Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>,
       GCCBuiltin<"__nvvm_bar_sync">;
+  def int_nvvm_bar_warp_sync :
+      Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>,
+      GCCBuiltin<"__nvvm_bar_warp_sync">;
+
+  // barrier.sync id[, cnt]
+  def int_nvvm_barrier_sync :
+      Intrinsic<[], [llvm_i32_ty], [IntrConvergent]>,
+      GCCBuiltin<"__nvvm_barrier_sync">;
+  def int_nvvm_barrier_sync_cnt :
+      Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrConvergent]>,
+      GCCBuiltin<"__nvvm_barrier_sync_cnt">;
 
   // Membar
   def int_nvvm_membar_cta : GCCBuiltin<"__nvvm_membar_cta">,
       Intrinsic<[], [], []>;
   def int_nvvm_membar_gl : GCCBuiltin<"__nvvm_membar_gl">,
       Intrinsic<[], [], []>;
   def int_nvvm_membar_sys : GCCBuiltin<"__nvvm_membar_sys">,
       Intrinsic<[], [], []>;

 def int_nvvm_shfl_sync_idx_i32 :
   Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty],
             [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.sync.idx.i32">,
   GCCBuiltin<"__nvvm_shfl_sync_idx_i32">;
 def int_nvvm_shfl_sync_idx_f32 :
   Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty],
             [IntrNoMem, IntrConvergent], "llvm.nvvm.shfl.sync.idx.f32">,
   GCCBuiltin<"__nvvm_shfl_sync_idx_f32">;
-}
+
+//
+// VOTE
+//
+
+// vote.all pred
+def int_nvvm_vote_all :
+  Intrinsic<[llvm_i1_ty], [llvm_i1_ty],
+            [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.all">,
+  GCCBuiltin<"__nvvm_vote_all">;
+// vote.any pred
+def int_nvvm_vote_any :
+  Intrinsic<[llvm_i1_ty], [llvm_i1_ty],
+            [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.any">,
+  GCCBuiltin<"__nvvm_vote_any">;
+// vote.uni pred
+def int_nvvm_vote_uni :
+  Intrinsic<[llvm_i1_ty], [llvm_i1_ty],
+            [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.uni">,
+  GCCBuiltin<"__nvvm_vote_uni">;
+// vote.ballot pred
+def int_nvvm_vote_ballot :
+  Intrinsic<[llvm_i32_ty], [llvm_i1_ty],
+            [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.ballot">,
+  GCCBuiltin<"__nvvm_vote_ballot">;
+
+//
+// VOTE.SYNC
+//
+
+// vote.sync.all mask, pred
+def int_nvvm_vote_all_sync :
+  Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty],
+            [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.all.sync">,
+  GCCBuiltin<"__nvvm_vote_all_sync">;
+// vote.sync.any mask, pred
+def int_nvvm_vote_any_sync :
+  Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty],
+            [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.any.sync">,
+  GCCBuiltin<"__nvvm_vote_any_sync">;
+// vote.sync.uni mask, pred
+def int_nvvm_vote_uni_sync :
+  Intrinsic<[llvm_i1_ty], [llvm_i32_ty, llvm_i1_ty],
+            [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.uni.sync">,
+  GCCBuiltin<"__nvvm_vote_uni_sync">;
+// vote.sync.ballot mask, pred
+def int_nvvm_vote_ballot_sync :
+  Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i1_ty],
+            [IntrNoMem, IntrConvergent], "llvm.nvvm.vote.ballot.sync">,
+  GCCBuiltin<"__nvvm_vote_ballot_sync">;
+
+} // let TargetPrefix = "nvvm"

llvm/trunk/lib/Target/NVPTX/NVPTXInstrInfo.td

 def do_SQRTF32_RN : Predicate<"usePrecSqrtF32()">;
 
 def hasHWROT32 : Predicate<"Subtarget->hasHWROT32()">;
 def noHWROT32 : Predicate<"!Subtarget->hasHWROT32()">;
 
 def true : Predicate<"true">;
 
 def hasPTX31 : Predicate<"Subtarget->getPTXVersion() >= 31">;
+def hasPTX60 : Predicate<"Subtarget->getPTXVersion() >= 60">;
+
+def hasSM30 : Predicate<"Subtarget->getSmVersion() >= 30">;
 
 def useFP16Math: Predicate<"Subtarget->allowFP16Math()">;
 
 //===----------------------------------------------------------------------===//
 // Some Common Instruction Class Templates
 //===----------------------------------------------------------------------===//
 
 // Template for instructions which take three int64, int32, or int16 args.

llvm/trunk/lib/Target/NVPTX/NVPTXIntrinsics.td

!strconcat("{{ \n\t",
              "bar.red.or.pred \t%p2, 0, %p1; \n\t",
              "selp.u32 \t$dst, 1, 0, %p2; \n\t",
              "}}"),
       [(set Int32Regs:$dst, (int_nvvm_barrier0_or Int32Regs:$pred))]>;
 
 def INT_BAR_SYNC : NVPTXInst<(outs), (ins i32imm:$i), "bar.sync \t$i;",
                              [(int_nvvm_bar_sync imm:$i)]>;
 
+def INT_BAR_WARP_SYNC_I : NVPTXInst<(outs), (ins i32imm:$i), "bar.warp.sync \t$i;",
+                             [(int_nvvm_bar_warp_sync imm:$i)]>,
+        Requires<[hasPTX60, hasSM30]>;
+def INT_BAR_WARP_SYNC_R : NVPTXInst<(outs), (ins Int32Regs:$i), "bar.warp.sync \t$i;",
+                             [(int_nvvm_bar_warp_sync Int32Regs:$i)]>,
+        Requires<[hasPTX60, hasSM30]>;
+
+def INT_BARRIER_SYNC_I : NVPTXInst<(outs), (ins i32imm:$i), "barrier.sync \t$i;",
+                                   [(int_nvvm_barrier_sync imm:$i)]>,
+        Requires<[hasPTX60, hasSM30]>;
+def INT_BARRIER_SYNC_R : NVPTXInst<(outs), (ins Int32Regs:$i), "barrier.sync \t$i;",
+                                   [(int_nvvm_barrier_sync Int32Regs:$i)]>,
+        Requires<[hasPTX60, hasSM30]>;
+
+def INT_BARRIER_SYNC_CNT_RR : NVPTXInst<(outs), (ins Int32Regs:$id, Int32Regs:$cnt),
+                 "barrier.sync \t$id, $cnt;",
+                 [(int_nvvm_barrier_sync_cnt Int32Regs:$id, Int32Regs:$cnt)]>,
+        Requires<[hasPTX60, hasSM30]>;
+def INT_BARRIER_SYNC_CNT_RI : NVPTXInst<(outs), (ins Int32Regs:$id, i32imm:$cnt),
+                 "barrier.sync \t$id, $cnt;",
+                 [(int_nvvm_barrier_sync_cnt Int32Regs:$id, imm:$cnt)]>,
+        Requires<[hasPTX60, hasSM30]>;
+def INT_BARRIER_SYNC_CNT_IR : NVPTXInst<(outs), (ins i32imm:$id, Int32Regs:$cnt),
+                 "barrier.sync \t$id, $cnt;",
+                 [(int_nvvm_barrier_sync_cnt imm:$id, Int32Regs:$cnt)]>,
+        Requires<[hasPTX60, hasSM30]>;
+def INT_BARRIER_SYNC_CNT_II : NVPTXInst<(outs), (ins i32imm:$id, i32imm:$cnt),
+                 "barrier.sync \t$id, $cnt;",
+                 [(int_nvvm_barrier_sync_cnt imm:$id, imm:$cnt)]>,
+        Requires<[hasPTX60, hasSM30]>;
+
+
 // shfl.{up,down,bfly,idx}.b32
 multiclass SHFL<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
   // The last two parameters to shfl can be regs or imms.  ptxas is smart
   // enough to inline constant registers, so strictly speaking we don't need to
   // handle immediates here.  But it's easy enough, and it makes our ptx more
   // readable.
   def reg : NVPTXInst<
       (outs regclass:$dst),

 defm INT_SHFL_SYNC_DOWN_F32 : SHFL_SYNC<Float32Regs, "down", int_nvvm_shfl_sync_down_f32>;
 defm INT_SHFL_SYNC_UP_I32 : SHFL_SYNC<Int32Regs, "up", int_nvvm_shfl_sync_up_i32>;
 defm INT_SHFL_SYNC_UP_F32 : SHFL_SYNC<Float32Regs, "up", int_nvvm_shfl_sync_up_f32>;
 defm INT_SHFL_SYNC_BFLY_I32 : SHFL_SYNC<Int32Regs, "bfly", int_nvvm_shfl_sync_bfly_i32>;
 defm INT_SHFL_SYNC_BFLY_F32 : SHFL_SYNC<Float32Regs, "bfly", int_nvvm_shfl_sync_bfly_f32>;
 defm INT_SHFL_SYNC_IDX_I32 : SHFL_SYNC<Int32Regs, "idx", int_nvvm_shfl_sync_idx_i32>;
 defm INT_SHFL_SYNC_IDX_F32 : SHFL_SYNC<Float32Regs, "idx", int_nvvm_shfl_sync_idx_f32>;
 
+
+// vote.{all,any,uni,ballot}
+multiclass VOTE<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
+  def : NVPTXInst<(outs regclass:$dest), (ins Int1Regs:$pred),
+              "vote." # mode # " \t$dest, $pred;",
+              [(set regclass:$dest, (IntOp Int1Regs:$pred))]>,
+        Requires<[hasPTX60, hasSM30]>;
+}
+
+defm VOTE_ALL : VOTE<Int1Regs, "all.pred", int_nvvm_vote_all>;
+defm VOTE_ANY : VOTE<Int1Regs, "any.pred", int_nvvm_vote_any>;
+defm VOTE_UNI : VOTE<Int1Regs, "uni.pred", int_nvvm_vote_uni>;
+defm VOTE_BALLOT : VOTE<Int32Regs, "ballot.b32", int_nvvm_vote_ballot>;
+
+// vote.sync.{all,any,uni,ballot}
+multiclass VOTE_SYNC<NVPTXRegClass regclass, string mode, Intrinsic IntOp> {
+  def i : NVPTXInst<(outs regclass:$dest), (ins i32imm:$mask, Int1Regs:$pred),
+              "vote.sync." # mode # " \t$dest, $pred, $mask;",
+              [(set regclass:$dest, (IntOp imm:$mask, Int1Regs:$pred))]>,
+          Requires<[hasPTX60, hasSM30]>;
+  def r : NVPTXInst<(outs regclass:$dest), (ins Int32Regs:$mask, Int1Regs:$pred),
+              "vote.sync." # mode #" \t$dest, $pred, $mask;",
+              [(set regclass:$dest, (IntOp Int32Regs:$mask, Int1Regs:$pred))]>,
+          Requires<[hasPTX60, hasSM30]>;
+}
+
+defm VOTE_SYNC_ALL : VOTE_SYNC<Int1Regs, "all.pred", int_nvvm_vote_all_sync>;
+defm VOTE_SYNC_ANY : VOTE_SYNC<Int1Regs, "any.pred", int_nvvm_vote_any_sync>;
+defm VOTE_SYNC_UNI : VOTE_SYNC<Int1Regs, "uni.pred", int_nvvm_vote_uni_sync>;
+defm VOTE_SYNC_BALLOT : VOTE_SYNC<Int32Regs, "ballot.b32", int_nvvm_vote_ballot_sync>;
+
 } // isConvergent = 1
 
 //-----------------------------------
 // Explicit Memory Fence Functions
 //-----------------------------------
 class MEMBAR<string StrOp, Intrinsic IntOP> :
               NVPTXInst<(outs), (ins),
             StrOp, [(IntOP)]>;

llvm/trunk/test/CodeGen/NVPTX/barrier.ll

+; RUN: llc < %s -march=nvptx64 -mcpu=sm_30 -mattr=+ptx60 | FileCheck %s
+
+declare void @llvm.nvvm.bar.warp.sync(i32)
+declare void @llvm.nvvm.barrier.sync(i32)
+declare void @llvm.nvvm.barrier.sync.cnt(i32, i32)
+
+; CHECK-LABEL: .func{{.*}}barrier.sync
+define void @barrier.sync(i32 %id, i32 %cnt) {
+  ; CHECK: ld.param.u32   [[ID:%r[0-9]+]], [barrier.sync_param_0];
+  ; CHECK: ld.param.u32   [[CNT:%r[0-9]+]], [barrier.sync_param_1];
+
+  ; CHECK:  barrier.sync [[ID]], [[CNT]];
+  call void @llvm.nvvm.barrier.sync.cnt(i32 %id, i32 %cnt)
+  ; CHECK:  barrier.sync [[ID]], 2;
+  call void @llvm.nvvm.barrier.sync.cnt(i32 %id, i32 2)
+  ; CHECK:  barrier.sync 3, [[CNT]];
+  call void @llvm.nvvm.barrier.sync.cnt(i32 3, i32 %cnt)
+  ; CHECK:  barrier.sync 4, 5;
+  call void @llvm.nvvm.barrier.sync.cnt(i32 4, i32 5)
+
+  ; CHECK: barrier.sync [[ID]];
+  call void @llvm.nvvm.barrier.sync(i32 %id)
+  ; CHECK: barrier.sync 1;
+  call void @llvm.nvvm.barrier.sync(i32 1)
+
+  ; CHECK: bar.warp.sync [[ID]];
+  call void @llvm.nvvm.bar.warp.sync(i32 %id)
+  ; CHECK: bar.warp.sync 6;
+  call void @llvm.nvvm.bar.warp.sync(i32 6)
+  ret void;
+}
+

llvm/trunk/test/CodeGen/NVPTX/vote.ll

+; RUN: llc < %s -march=nvptx64 -mcpu=sm_30 -mattr=+ptx60 | FileCheck %s
+
+declare i1 @llvm.nvvm.vote.all(i1)
+; CHECK-LABEL: .func{{.*}}vote.all
+define i1 @vote.all(i1 %pred) {
+  ; CHECK: vote.all.pred
+  %val = call i1 @llvm.nvvm.vote.all(i1 %pred)
+  ret i1 %val
+}
+
+declare i1 @llvm.nvvm.vote.any(i1)
+; CHECK-LABEL: .func{{.*}}vote.any
+define i1 @vote.any(i1 %pred) {
+  ; CHECK: vote.any.pred
+  %val = call i1 @llvm.nvvm.vote.any(i1 %pred)
+  ret i1 %val
+}
+
+declare i1 @llvm.nvvm.vote.uni(i1)
+; CHECK-LABEL: .func{{.*}}vote.uni
+define i1 @vote.uni(i1 %pred) {
+  ; CHECK: vote.uni.pred
+  %val = call i1 @llvm.nvvm.vote.uni(i1 %pred)
+  ret i1 %val
+}
+
+declare i32 @llvm.nvvm.vote.ballot(i1)
+; CHECK-LABEL: .func{{.*}}vote.ballot
+define i32 @vote.ballot(i1 %pred) {
+  ; CHECK: vote.ballot.b32
+  %val = call i32 @llvm.nvvm.vote.ballot(i1 %pred)
+  ret i32 %val
+}
+
+declare i1 @llvm.nvvm.vote.all.sync(i32, i1)
+; CHECK-LABEL: .func{{.*}}vote.sync.all
+define i1 @vote.sync.all(i32 %mask, i1 %pred) {
+  ; CHECK: vote.sync.all.pred
+  %val = call i1 @llvm.nvvm.vote.all.sync(i32 %mask, i1 %pred)
+  ret i1 %val
+}
+
+declare i1 @llvm.nvvm.vote.any.sync(i32, i1)
+; CHECK-LABEL: .func{{.*}}vote.sync.any
+define i1 @vote.sync.any(i32 %mask, i1 %pred) {
+  ; CHECK: vote.sync.any.pred
+  %val = call i1 @llvm.nvvm.vote.any.sync(i32 %mask, i1 %pred)
+  ret i1 %val
+}
+
+declare i1 @llvm.nvvm.vote.uni.sync(i32, i1)
+; CHECK-LABEL: .func{{.*}}vote.sync.uni
+define i1 @vote.sync.uni(i32 %mask, i1 %pred) {
+  ; CHECK: vote.sync.uni.pred
+  %val = call i1 @llvm.nvvm.vote.uni.sync(i32 %mask, i1 %pred)
+  ret i1 %val
+}
+
+declare i32 @llvm.nvvm.vote.ballot.sync(i32, i1)
+; CHECK-LABEL: .func{{.*}}vote.sync.ballot
+define i32 @vote.sync.ballot(i32 %mask, i1 %pred) {
+  ; CHECK: vote.sync.ballot.b32
+  %val = call i32 @llvm.nvvm.vote.ballot.sync(i32 %mask, i1 %pred)
+  ret i32 %val
+}