AMDGPU: mark llvm.amdgcn.image.atomic.* as a source of divergence

[nhaehnle]

Summary

When multiple threads perform an atomic op with the same arguments, they
will usually see different return values.

[arsenm]

LGTM, although your test would be greatly simplified by just running the divergence analysis like is done in test/Analysis/DivergenceAnalysis/AMDGPU

Are the standard atomic instructions also reported as divergent?

[nhaehnle]

r263440 - committed with a simplified test.

From a brief check it looks like atomic instructions aren't marked as divergent.

lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

static bool isIntrinsicSourceOfDivergence(const TargetIntrinsicInfo *TII,
   case Intrinsic::amdgcn_workitem_id_z:
   case Intrinsic::amdgcn_interp_p1:
   case Intrinsic::amdgcn_interp_p2:
   case Intrinsic::amdgcn_mbcnt_hi:
   case Intrinsic::amdgcn_mbcnt_lo:
   case Intrinsic::r600_read_tidig_x:
   case Intrinsic::r600_read_tidig_y:
   case Intrinsic::r600_read_tidig_z:
+  case Intrinsic::amdgcn_image_atomic_swap:
+  case Intrinsic::amdgcn_image_atomic_add:
+  case Intrinsic::amdgcn_image_atomic_sub:
+  case Intrinsic::amdgcn_image_atomic_smin:
+  case Intrinsic::amdgcn_image_atomic_umin:
+  case Intrinsic::amdgcn_image_atomic_smax:
+  case Intrinsic::amdgcn_image_atomic_umax:
+  case Intrinsic::amdgcn_image_atomic_and:
+  case Intrinsic::amdgcn_image_atomic_or:
+  case Intrinsic::amdgcn_image_atomic_xor:
+  case Intrinsic::amdgcn_image_atomic_inc:
+  case Intrinsic::amdgcn_image_atomic_dec:
+  case Intrinsic::amdgcn_image_atomic_cmpswap:
     return true;
   }
 
   StringRef Name = I->getCalledFunction()->getName();
   switch (TII->lookupName((const char *)Name.bytes_begin(), Name.size())) {
   default:
     return false;
   case AMDGPUIntrinsic::SI_tid:

test/CodeGen/AMDGPU/llvm.amdgcn.image.atomic.divergence.ll

+;RUN: llc -march=amdgcn -mcpu=verde < %s | FileCheck %s --check-prefix=CHECK
+
+;CHECK-LABEL: {{^}}image_atomic_swap:
+;CHECK: image_atomic_swap [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_swap(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.swap.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_add:
+;CHECK: image_atomic_add [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_add(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.add.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_sub:
+;CHECK: image_atomic_sub [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_sub(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.sub.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_smin:
+;CHECK: image_atomic_smin [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_smin(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.smin.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_umin:
+;CHECK: image_atomic_umin [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_umin(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.umin.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_smax:
+;CHECK: image_atomic_smax [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_smax(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.smax.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_umax:
+;CHECK: image_atomic_umax [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_umax(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.umax.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_and:
+;CHECK: image_atomic_and [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_and(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.and.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_or:
+;CHECK: image_atomic_or [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_or(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.or.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_xor:
+;CHECK: image_atomic_xor [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_xor(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.xor.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_inc:
+;CHECK: image_atomic_inc [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_inc(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.inc.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_dec:
+;CHECK: image_atomic_dec [[ORIG:v[0-9]+]],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, [[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_dec(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.dec.i32(i32 %data, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+;CHECK-LABEL: {{^}}image_atomic_cmpswap:
+;CHECK: image_atomic_cmpswap v{{\[}}[[ORIG:[0-9]+]]:{{[0-9]+}}],
+;CHECK: v_cmp_eq_i32_e32 vcc, 0, v[[ORIG]]
+;CHECK: s_and_saveexec_b64 s[{{[0-9]+}}:{{[0-9]+}}], vcc
+define void @image_atomic_cmpswap(<8 x i32> inreg %rsrc, i32 inreg %addr, i32 inreg %data, i32 inreg %cmp) #0 {
+main_body:
+  %orig = call i32 @llvm.amdgcn.image.atomic.cmpswap.i32(i32 %data, i32 %cmp, i32 %addr, <8 x i32> %rsrc, i1 0, i1 0, i1 0)
+  %cc = icmp eq i32 %orig, 0
+  br i1 %cc, label %A, label %B
+
+A:
+  call void @llvm.amdgcn.image.store.i32(<4 x float> undef, i32 %addr, <8 x i32> %rsrc, i32 15, i1 0, i1 0, i1 0, i1 0)
+  br label %B
+
+B:
+  ret void
+}
+
+declare void @llvm.amdgcn.image.store.i32(<4 x float>, i32, <8 x i32>, i32, i1, i1, i1, i1) #1
+
+declare i32 @llvm.amdgcn.image.atomic.swap.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.add.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.sub.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.smin.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.umin.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.smax.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.umax.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.and.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.or.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.xor.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.inc.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.dec.i32(i32, i32, <8 x i32>, i1, i1, i1) #1
+declare i32 @llvm.amdgcn.image.atomic.cmpswap.i32(i32, i32, i32, <8 x i32>,i1, i1, i1) #1
+
+attributes #0 = { "ShaderType"="0" }
+attributes #1 = { nounwind }