AMDGPU/SI: Add offset field to ds_permute/ds_bpermute instructions

[• tstellarAMD]

Summary

These instructions can add an immediate offset to the address, like other
ds instructions.

[• tstellarAMD]

Ping.

[arsenm]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

825

Indentation

833

How was this working before?

lib/Target/AMDGPU/SIInstrInfo.td

2752–2753

Why is offset1 not set to 0 now? It's called offset but I thought this was always called offset0 for the single offset instructions

[• tstellarAMD]

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

833

All of the ds patterns are stand-alone patterns and they use an SDNodeXForm to convert the immediate to a target immediate. I couldn't get SDNodeXForm to work with the instruction's pattern, so I had to change this here.

lib/Target/AMDGPU/SIInstrInfo.td

2752–2753

The operand name is 'offset' for the single offset instructions.

[• tstellarAMD]

Fix indentation.

[arsenm]

LGTM

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

833

So should the xform be removed now?

lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

bool AMDGPUDAGToDAGISel::isDSOffsetLegal(const SDValue &Base, unsigned Offset,
 
   // On Southern Islands instruction with a negative base value and an offset
   // don't seem to work.
   return CurDAG->SignBitIsZero(Base);
 }
 
 bool AMDGPUDAGToDAGISel::SelectDS1Addr1Offset(SDValue Addr, SDValue &Base,
                                               SDValue &Offset) const {
+  SDLoc DL(Addr);
   if (CurDAG->isBaseWithConstantOffset(Addr)) {
     SDValue N0 = Addr.getOperand(0);
     SDValue N1 = Addr.getOperand(1);
     ConstantSDNode *C1 = cast<ConstantSDNode>(N1);
     if (isDSOffsetLegal(N0, C1->getSExtValue(), 16)) {
       // (add n0, c0)
       Base = N0;
-      Offset = N1;
+      Offset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16);
       return true;
     }
   } else if (Addr.getOpcode() == ISD::SUB) {
     // sub C, x -> add (sub 0, x), C
     if (const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Addr.getOperand(0))) {
       int64_t ByteOffset = C->getSExtValue();
       if (isUInt<16>(ByteOffset)) {
-        SDLoc DL(Addr);
         SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i32);
 
         // XXX - This is kind of hacky. Create a dummy sub node so we can check
         // the known bits in isDSOffsetLegal. We need to emit the selected node
         // here, so this is thrown away.
         SDValue Sub = CurDAG->getNode(ISD::SUB, DL, MVT::i32,
                                       Zero, Addr.getOperand(1));
 

lib/Target/AMDGPU/SIInstrInfo.cpp

if (OffsetImm) {
       return true;
     }
 
     // The 2 offset instructions use offset0 and offset1 instead. We can treat
     // these as a load with a single offset if the 2 offsets are consecutive. We
     // will use this for some partially aligned loads.
     const MachineOperand *Offset0Imm = getNamedOperand(*LdSt,
                                                        AMDGPU::OpName::offset0);
-    // DS_PERMUTE does not have Offset0Imm (and Offset1Imm).
-    if (!Offset0Imm)
-      return false;
-
     const MachineOperand *Offset1Imm = getNamedOperand(*LdSt,
                                                        AMDGPU::OpName::offset1);
 
     uint8_t Offset0 = Offset0Imm->getImm();
     uint8_t Offset1 = Offset1Imm->getImm();
 
     if (Offset1 > Offset0 && Offset1 - Offset0 == 1) {
       // Each of these offsets is in element sized units, so we need to convert

lib/Target/AMDGPU/SIInstrInfo.td

let data1 = 0 in {
       def _vi : DS_Off16_Real_vi <op, opName, outs, ins, asm>;
     }
   }
 }
 
 multiclass DS_1A1D_PERMUTE <bits<8> op, string opName, RegisterClass rc,
                             SDPatternOperator node = null_frag,
   dag outs = (outs rc:$vdst),
-  dag ins = (ins VGPR_32:$addr, rc:$data0),
-  string asm = opName#" $vdst, $addr, $data0"> {
+  dag ins = (ins VGPR_32:$addr, rc:$data0, ds_offset:$offset),
+  string asm = opName#" $vdst, $addr, $data0"#"$offset"> {
 
   let mayLoad = 0, mayStore = 0, isConvergent = 1 in {
     def "" : DS_Pseudo <opName, outs, ins,
-     [(set (i32 rc:$vdst),
-         (node (i32 VGPR_32:$addr), (i32 rc:$data0)))]>;
+     [(set i32:$vdst,
+         (node (DS1Addr1Offset i32:$addr, i16:$offset), i32:$data0))]>;
 
-    let data1 = 0, offset0 = 0, offset1 = 0, gds = 0  in {
-      def "_vi" : DS_Real_vi <op, opName, outs, ins, asm>;
+    let data1 = 0, gds = 0  in {
+      def "_vi" : DS_Off16_Real_vi <op, opName, outs, ins, asm>;
     }
   }
 }
 
 multiclass DS_1A2D_RET_m <bits<8> op, string opName, RegisterClass rc,
                           string noRetOp = "", dag ins,
   dag outs = (outs rc:$vdst),
   string asm = opName#" $vdst, $addr, $data0, $data1"#"$offset"#"$gds"> {

test/CodeGen/AMDGPU/llvm.amdgcn.ds.bpermute.ll

 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s
 
 declare i32 @llvm.amdgcn.ds.bpermute(i32, i32) #0
 
 ; FUNC-LABEL: {{^}}ds_bpermute:
 ; CHECK: ds_bpermute_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; CHECK: s_waitcnt lgkmcnt
 define void @ds_bpermute(i32 addrspace(1)* %out, i32 %index, i32 %src) nounwind {
   %bpermute = call i32 @llvm.amdgcn.ds.bpermute(i32 %index, i32 %src) #0
   store i32 %bpermute, i32 addrspace(1)* %out, align 4
   ret void
 }
 
+; CHECK-LABEL: {{^}}ds_bpermute_imm_offset:
+; CHECK: ds_bpermute_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:4
+; CHECK: s_waitcnt lgkmcnt
+define void @ds_bpermute_imm_offset(i32 addrspace(1)* %out, i32 %base_index, i32 %src) nounwind {
+  %index = add i32 %base_index, 4
+  %bpermute = call i32 @llvm.amdgcn.ds.bpermute(i32 %index, i32 %src) #0
+  store i32 %bpermute, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
 attributes #0 = { nounwind readnone convergent }

test/CodeGen/AMDGPU/llvm.amdgcn.ds.permute.ll

 ; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck %s
 
 declare i32 @llvm.amdgcn.ds.permute(i32, i32) #0
 
-; FUNC-LABEL: {{^}}ds_permute:
+; CHECK-LABEL: {{^}}ds_permute:
 ; CHECK: ds_permute_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}}
 ; CHECK: s_waitcnt lgkmcnt
 define void @ds_permute(i32 addrspace(1)* %out, i32 %index, i32 %src) nounwind {
   %bpermute = call i32 @llvm.amdgcn.ds.permute(i32 %index, i32 %src) #0
   store i32 %bpermute, i32 addrspace(1)* %out, align 4
   ret void
 }
 
+; CHECK-LABEL: {{^}}ds_permute_imm_offset:
+; CHECK: ds_permute_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset:4
+; CHECK: s_waitcnt lgkmcnt
+define void @ds_permute_imm_offset(i32 addrspace(1)* %out, i32 %base_index, i32 %src) nounwind {
+  %index = add i32 %base_index, 4
+  %bpermute = call i32 @llvm.amdgcn.ds.permute(i32 %index, i32 %src) #0
+  store i32 %bpermute, i32 addrspace(1)* %out, align 4
+  ret void
+}
+
 attributes #0 = { nounwind readnone convergent }