[AMDGPU] Avoid using divergent value in mubuf addr64 descriptor

[tpr]

Summary

This fixes a problem where a load from global+idx generated incorrect
code on <=gfx7 when the index is divergent.

Change-Id: Ib4d177d6254b1dd3f8ec0203fdddec94bd8bc5ed

[arsenm]

The test seems pretty lacking. There are a lot more combinations of divergent and non-divergent changed here that aren't tested

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1002

SReg_64_XEXEC

[arsenm]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1002

Probably should also create a helper that the constant lowering can use as well since this is basically the same

[tpr]

V2: s_mov_b64 helper func; improved test

[tpr]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

1002

I left the SGPR_64RegClassID as that's what the existing code for a 64 bit constant load has. I have done the suggested helper func.

[nhaehnle]

Thanks. I feel like some of this could perhaps be improved with a computeKnownBits if there is a 64-bit uniform base and a 32-bit non-uniform offset, but that doesn't have to be part of this change.

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

363–367

Have you run clang-format on this? It looks a bit off.

996

Spurious whitespace change.

1087–1102

This has a lot of redundancy with the isBaseWithConstantOffset case above. Perhaps the cases can be combined?

[tpr]

V3: minor formatting; commoned up two legs of fix

[tpr]

V4: fixed bugs in commoning two legs

[tpr]

Ping.

[nhaehnle]

test/CodeGen/AMDGPU/shader-addr64-nonuniform.ll

27–35

Could you please add a similar test-case, with a non-uniform i64 %arg18 and %offset a constant? I don't think this case is covered by tests, and I'm not sure that the code would do the right thing for that case, where I think Addr64 would also be needed.

[tpr]

V5: Added extra test suggested in review comment, and fixed the bug it showed up.

[tpr]

test/CodeGen/AMDGPU/shader-addr64-nonuniform.ll

27–35

Good spot; that case was not covered.

[nhaehnle]

Great, thanks!

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
      Ops.push_back(N->getOperand(i));
   }
   Ops.push_back(Glue);
   return CurDAG->MorphNodeTo(N, N->getOpcode(), N->getVTList(), Ops);
 }
 
 static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) {
   switch (NumVectorElts) {
   case 1:
     return AMDGPU::SReg_32_XM0RegClassID;
   case 2:
     return AMDGPU::SReg_64RegClassID;
   case 4:
     return AMDGPU::SReg_128RegClassID;
   case 8:
     return AMDGPU::SReg_256RegClassID;
   case 16:
     return AMDGPU::SReg_512RegClassID;
   }
 
   llvm_unreachable("invalid vector size");
bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base,
   // FIXME: This is broken on SI where we still need to check if the base
   // pointer is positive here.
   Base = Addr;
   Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8);
   Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8);
   return true;
 }
 
+
+static MachineSDNode *buildSMovImm64(SelectionDAG &DAG, const SDLoc &DL,
+                              uint64_t Val) {
+  SDValue ValLo = DAG.getTargetConstant(Val & 0xffffffffU, DL, MVT::i32);
+  SDValue ValHi = DAG.getTargetConstant(Val >> 32, DL, MVT::i32);
+  const SDValue Ops0[] = {
+    DAG.getTargetConstant(AMDGPU::SGPR_64RegClassID, DL, MVT::i32),
+    SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, ValLo), 0),
+    DAG.getTargetConstant(AMDGPU::sub0, DL, MVT::i32),
+    SDValue(DAG.getMachineNode(AMDGPU::S_MOV_B32, DL, MVT::i32, ValHi), 0),
+    DAG.getTargetConstant(AMDGPU::sub1, DL, MVT::i32)
+  };
+  return DAG.getMachineNode(AMDGPU::REG_SEQUENCE, DL, MVT::v2i32, Ops0);
+}
+
 bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr,
                                      SDValue &VAddr, SDValue &SOffset,
                                      SDValue &Offset, SDValue &Offen,
                                      SDValue &Idxen, SDValue &Addr64,
                                      SDValue &GLC, SDValue &SLC,
                                      SDValue &TFE) const {
   // Subtarget prefers to use flat instruction
   if (Subtarget->useFlatForGlobal())
if (CurDAG->isBaseWithConstantOffset(Addr)) {
     SDValue N1 = Addr.getOperand(1);
     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
 
     if (N0.getOpcode() == ISD::ADD) {
       // (add (add N2, N3), C1) -> addr64
       SDValue N2 = N0.getOperand(0);
       SDValue N3 = N0.getOperand(1);
       Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
+      if (N2->isDivergent()) {
+        if (N3->isDivergent()) {
+          // Both N2 and N3 are divergent. Keep the add and use N2+N3 as the
+          // vaddr, and construct the resource out of 0.
+          Ptr = SDValue(buildSMovImm64(*CurDAG, DL, 0), 0);
+          VAddr = N0;
+        } else {
+          // N2 is divergent, N3 is not.
+          Ptr = N3;
+          VAddr = N2;
+        }
+      } else {
+        // N2 is not divergent.
         Ptr = N2;
         VAddr = N3;
+      }
     } else {
       // (add N0, C1) -> offset
       VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
       Ptr = N0;
     }
 
     if (SIInstrInfo::isLegalMUBUFImmOffset(C1->getZExtValue())) {
       Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
if (CurDAG->isBaseWithConstantOffset(Addr)) {
     }
   }
 
   if (Addr.getOpcode() == ISD::ADD) {
     // (add N0, N1) -> addr64
     SDValue N0 = Addr.getOperand(0);
     SDValue N1 = Addr.getOperand(1);
     Addr64 = CurDAG->getTargetConstant(1, DL, MVT::i1);
+
+    if (N0->isDivergent()) {
+      if (N1->isDivergent()) {
+        // Both N0 and N1 are divergent. Use the result of the add as the
+        // addr64, and construct the resource from a 0 address.
+        Ptr = SDValue(buildSMovImm64(*CurDAG, DL, 0), 0);
+        VAddr = Addr;
+      } else {
+        // N0 is divergent, N1 is not.
+        Ptr = N1;
+        VAddr = N0;
+      }
+    } else {
+      // N0 is not divergent.
       Ptr = N0;
       VAddr = N1;
+    }
     Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);
     return true;
   }
 
   // default case -> offset
   VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32);
   Ptr = Addr;
   Offset = CurDAG->getTargetConstant(0, DL, MVT::i16);

test/CodeGen/AMDGPU/shader-addr64-nonuniform.ll

+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx700 -verify-machineinstrs <%s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SICI %s
+
+; GCN-LABEL: {{^}}main:
+; GCN-NOT: readfirstlane
+; SICI: buffer_load_dwordx4 {{.*}} addr64
+
+@indexable = internal unnamed_addr addrspace(1) constant [6 x <3 x float>] [<3 x float> <float 1.000000e+00, float 0.000000e+00, float 0.000000e+00>, <3 x float> <float 0.000000e+00, float 1.000000e+00, float 0.000000e+00>, <3 x float> <float 0.000000e+00, float 0.000000e+00, float 1.000000e+00>, <3 x float> <float 0.000000e+00, float 1.000000e+00, float 1.000000e+00>, <3 x float> <float 1.000000e+00, float 0.000000e+00, float 1.000000e+00>, <3 x float> <float 1.000000e+00, float 1.000000e+00, float 0.000000e+00>]
+
+define amdgpu_ps float @main(i32 %arg18) {
+.entry:
+  %tmp31 = sext i32 %arg18 to i64
+  %tmp32 = getelementptr [6 x <3 x float>], [6 x <3 x float>] addrspace(1)* @indexable, i64 0, i64 %tmp31
+  %tmp33 = load <3 x float>, <3 x float> addrspace(1)* %tmp32, align 16
+  %tmp34 = extractelement <3 x float> %tmp33, i32 0
+  ret float %tmp34
+}
+