AMDGPU: Fix DivergenceAnalysis for llvm.read_register

[arsenm]

Summary

This was treating all calls as uniform by default, which
is wrong if used to read a VGPR.

[rampitec]

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

876

You could just check the first symbol of the name, it is divergent if it is 'v' or 'a'. Same effect at the end, but much less work.

[arsenm]

Don't fully parse register names. This does change the behavior for invalid register names from divergent to not divergent

[rampitec]

LGTM

[arsenm]

583450fa0988d1ac088d36ec840cec4f84e013c4

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h

InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind,
                                  const Instruction *I = nullptr);
 
   bool isInlineAsmSourceOfDivergence(const CallInst *CI,
                                      ArrayRef<unsigned> Indices = {}) const;
 
   using BaseT::getVectorInstrCost;
   InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy,
                                      unsigned Index);
+
+  bool isReadRegisterSourceOfDivergence(const IntrinsicInst *ReadReg) const;
   bool isSourceOfDivergence(const Value *V) const;
   bool isAlwaysUniform(const Value *V) const;
 
   unsigned getFlatAddressSpace() const {
     // Don't bother running InferAddressSpaces pass on graphics shaders which
     // don't use flat addressing.
     if (IsGraphics)
       return -1;

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

bool GCNTTIImpl::isInlineAsmSourceOfDivergence(
   return false;
 }
 
 /// \returns true if the new GPU divergence analysis is enabled.
 bool GCNTTIImpl::useGPUDivergenceAnalysis() const {
   return !UseLegacyDA;
 }
 
+bool GCNTTIImpl::isReadRegisterSourceOfDivergence(
+    const IntrinsicInst *ReadReg) const {
+  Metadata *MD =
+      cast<MetadataAsValue>(ReadReg->getArgOperand(0))->getMetadata();
+  StringRef RegName =
+      cast<MDString>(cast<MDNode>(MD)->getOperand(0))->getString();
+
+  // Special case registers that look like VCC.
+  MVT VT = MVT::getVT(ReadReg->getType());
+  if (VT == MVT::i1)
+    return true;
+
+  // Special case scalar registers that start with 'v'.
+  if (RegName.startswith("vcc") || RegName.empty())
+    return false;
+
+  // VGPR or AGPR is divergent. There aren't any specially named vector
+  // registers.
+  return RegName[0] == 'v' || RegName[0] == 'a';
+}
+
 /// \returns true if the result of the value could potentially be
 /// different across workitems in a wavefront.
 bool GCNTTIImpl::isSourceOfDivergence(const Value *V) const {
   if (const Argument *A = dyn_cast<Argument>(V))
     return !AMDGPU::isArgPassedInSGPR(A);
 
   // Loads from the private and flat address spaces are divergent, because
   // threads can execute the load instruction with the same inputs and get
   // different results.
   //
   // All other loads are not divergent, because if threads issue loads with the
   // same arguments, they will always get the same result.
   if (const LoadInst *Load = dyn_cast<LoadInst>(V))
     return Load->getPointerAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS ||
            Load->getPointerAddressSpace() == AMDGPUAS::FLAT_ADDRESS;
 
   // Atomics are divergent because they are executed sequentially: when an
   // atomic operation refers to the same address in each thread, then each
   // thread after the first sees the value written by the previous thread as
   // original value.
   if (isa<AtomicRMWInst>(V) || isa<AtomicCmpXchgInst>(V))
     return true;
 
-  if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V))
+  if (const IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(V)) {
+    if (Intrinsic->getIntrinsicID() == Intrinsic::read_register)
+      return isReadRegisterSourceOfDivergence(Intrinsic);
+
     return AMDGPU::isIntrinsicSourceOfDivergence(Intrinsic->getIntrinsicID());
+  }
 
   // Assume all function calls are a source of divergence.
   if (const CallInst *CI = dyn_cast<CallInst>(V)) {
     if (CI->isInlineAsm())
       return isInlineAsmSourceOfDivergence(CI);
     return true;
   }
 

llvm/test/Analysis/DivergenceAnalysis/AMDGPU/read_register.ll

+; RUN: opt -mtriple amdgcn-unknown-amdhsa -mcpu=gfx90a -passes='print<divergence>' -disable-output %s 2>&1 | FileCheck %s
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_exec':
+; CHECK-NOT: DIVERGENT
+define i64 @read_register_exec() {
+  %reg = call i64 @llvm.read_register.i64(metadata !0)
+  ret i64 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_m0':
+; CHECK-NOT: DIVERGENT
+define i32 @read_register_m0() {
+  %reg = call i32 @llvm.read_register.i32(metadata !1)
+  ret i32 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_s17':
+; CHECK-NOT: DIVERGENT
+define i32 @read_register_s17() {
+  %reg = call i32 @llvm.read_register.i32(metadata !2)
+  ret i32 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_s17_i17':
+; CHECK-NOT: DIVERGENT
+define i17 @read_register_s17_i17() {
+  %reg = call i17 @llvm.read_register.i17(metadata !2)
+  ret i17 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_v0':
+; CHECK: DIVERGENT
+define i32 @read_register_v0() {
+  %reg = call i32 @llvm.read_register.i32(metadata !3)
+  ret i32 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_v0_v1':
+; CHECK: DIVERGENT
+define i64 @read_register_v0_v1() {
+  %reg = call i64 @llvm.read_register.i64(metadata !4)
+  ret i64 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_a0':
+; CHECK: DIVERGENT
+define i32 @read_register_a0() {
+  %reg = call i32 @llvm.read_register.i32(metadata !5)
+  ret i32 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_a0_a1':
+; CHECK: DIVERGENT
+define i64 @read_register_a0_a1() {
+  %reg = call i64 @llvm.read_register.i64(metadata !6)
+  ret i64 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_vcc_i64':
+; CHECK-NOT: DIVERGENT
+define i64 @read_register_vcc_i64() {
+  %reg = call i64 @llvm.read_register.i64(metadata !7)
+  ret i64 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_vcc_i1':
+; CHECK: DIVERGENT
+define i1 @read_register_vcc_i1() {
+  %reg = call i1 @llvm.read_register.i1(metadata !7)
+  ret i1 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_invalid_reg':
+; CHECK-NOT: DIVERGENT
+define i64 @read_register_invalid_reg() {
+  %reg = call i64 @llvm.read_register.i64(metadata !8)
+  ret i64 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_flat_scratch':
+; CHECK-NOT: DIVERGENT
+define i32 @read_register_flat_scratch() {
+  %reg = call i32 @llvm.read_register.i32(metadata !9)
+  ret i32 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_vcc_lo_i32':
+; CHECK-NOT: DIVERGENT
+define i32 @read_register_vcc_lo_i32() {
+  %reg = call i32 @llvm.read_register.i32(metadata !10)
+  ret i32 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_vcc_hi_i32':
+; CHECK-NOT: DIVERGENT
+define i32 @read_register_vcc_hi_i32() {
+  %reg = call i32 @llvm.read_register.i32(metadata !11)
+  ret i32 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_exec_lo_i32':
+; CHECK-NOT: DIVERGENT
+define i32 @read_register_exec_lo_i32() {
+  %reg = call i32 @llvm.read_register.i32(metadata !12)
+  ret i32 %reg
+}
+
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_exec_hi_i32':
+; CHECK-NOT: DIVERGENT
+define i32 @read_register_exec_hi_i32() {
+  %reg = call i32 @llvm.read_register.i32(metadata !13)
+  ret i32 %reg
+}
+
+; FIXME: Why does the verifier allow this?
+; CHECK-LABEL: Divergence Analysis' for function 'read_register_empty_str_i32':
+; CHECK-NOT: DIVERGENT
+define i32 @read_register_empty_str_i32() {
+  %reg = call i32 @llvm.read_register.i32(metadata !14)
+  ret i32 %reg
+}
+
+declare i64 @llvm.read_register.i64(metadata)
+declare i32 @llvm.read_register.i32(metadata)
+declare i17 @llvm.read_register.i17(metadata)
+declare i1 @llvm.read_register.i1(metadata)
+
+!0 = !{!"exec"}
+!1 = !{!"m0"}
+!2 = !{!"s17"}
+!3 = !{!"v0"}
+!4 = !{!"v[0:1]"}
+!5 = !{!"a0"}
+!6 = !{!"a[0:1]"}
+!7 = !{!"vcc"}
+!8 = !{!"not a register"}
+!9 = !{!"flat_scratch"}
+!10 = !{!"vcc_lo"}
+!11 = !{!"vcc_hi"}
+!12 = !{!"exec_lo"}
+!13 = !{!"exec_hi"}
+!14 = !{!""}