AMDGPU: Use carry-less adds in FI elimination

[rampitec]

LGTM

[arsenm]

r319501

lib/Target/AMDGPU/SIInstrInfo.cpp

return !MI.isTerminator() && MI.getOpcode() != AMDGPU::COPY &&
          MI.modifiesRegister(AMDGPU::EXEC, &RI);
 }
 
 MachineInstrBuilder
 SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB,
                            MachineBasicBlock::iterator I,
                            const DebugLoc &DL,
                            unsigned DestReg) const {
-  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
+  if (ST.hasAddNoCarry())
+    return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e64), DestReg);
 
+  MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
   unsigned UnusedCarry = MRI.createVirtualRegister(&AMDGPU::SReg_64RegClass);
+  MRI.setRegAllocationHint(UnusedCarry, 0, AMDGPU::VCC);
 
   return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_I32_e64), DestReg)
            .addReg(UnusedCarry, RegState::Define | RegState::Dead);
 }
 
 bool SIInstrInfo::isKillTerminator(unsigned Opcode) {
   switch (Opcode) {
   case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:

lib/Target/AMDGPU/SIRegisterInfo.cpp

default: {
 
         int64_t Offset = FrameInfo.getObjectOffset(Index);
         if (Offset == 0) {
           // XXX - This never happens because of emergency scavenging slot at 0?
           BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ResultReg)
             .addImm(Log2_32(ST.getWavefrontSize()))
             .addReg(DiffReg);
         } else {
-          unsigned CarryOut
-            = MRI.createVirtualRegister(&AMDGPU::SReg_64_XEXECRegClass);
           unsigned ScaledReg
             = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
 
           BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_LSHRREV_B32_e64), ScaledReg)
             .addImm(Log2_32(ST.getWavefrontSize()))
             .addReg(DiffReg, RegState::Kill);
 
           // TODO: Fold if use instruction is another add of a constant.
           if (AMDGPU::isInlinableLiteral32(Offset, ST.hasInv2PiInlineImm())) {
-            BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ADD_I32_e64), ResultReg)
-              .addReg(CarryOut, RegState::Define | RegState::Dead)
+            TII->getAddNoCarry(*MBB, MI, DL, ResultReg)
               .addImm(Offset)
               .addReg(ScaledReg, RegState::Kill);
           } else {
             unsigned ConstOffsetReg
               = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
 
             BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg)
               .addImm(Offset);
-            BuildMI(*MBB, MI, DL, TII->get(AMDGPU::V_ADD_I32_e64), ResultReg)
-              .addReg(CarryOut, RegState::Define | RegState::Dead)
+            TII->getAddNoCarry(*MBB, MI, DL, ResultReg)
               .addReg(ConstOffsetReg, RegState::Kill)
               .addReg(ScaledReg, RegState::Kill);
           }
-
-          MRI.setRegAllocationHint(CarryOut, 0, AMDGPU::VCC);
         }
 
         // Don't introduce an extra copy if we're just materializing in a mov.
         if (IsCopy)
           MI->eraseFromParent();
         else
           FIOp.ChangeToRegister(ResultReg, false, false, true);
         return;

test/CodeGen/AMDGPU/frame-index-elimination.ll

-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=kaveri -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,CI %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-promote-alloca -amdgpu-sroa=0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9 %s
 
 ; Test that non-entry function frame indices are expanded properly to
 ; give an index relative to the scratch wave offset register
 
 ; Materialize into a mov. Make sure there isn't an unnecessary copy.
 ; GCN-LABEL: {{^}}func_mov_fi_i32:
 ; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN: s_sub_u32 s6, s5, s4
-; GCN-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
-; GCN-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+
+; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
+; CI-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+
+; GFX9-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
+; GFX9-NEXT: v_add_u32_e32 v0, 4, [[SCALED]]
+
 ; GCN-NOT: v_mov
 ; GCN: ds_write_b32 v0, v0
 define void @func_mov_fi_i32() #0 {
   %alloca = alloca i32
   store volatile i32* %alloca, i32* addrspace(3)* undef
   ret void
 }
 
 ; Materialize into an add of a constant offset from the FI.
 ; FIXME: Should be able to merge adds
 
 ; GCN-LABEL: {{^}}func_add_constant_to_fi_i32:
 ; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN: s_sub_u32 s6, s5, s4
-; GCN-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
-; GCN-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
-; GCN-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+
+; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
+; CI-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+; CI-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+
+; GFX9-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
+; GFX9-NEXT: v_add_u32_e32 v0, 4, [[SCALED]]
+; GFX9-NEXT: v_add_u32_e32 v0, 4, v0
+
+
 ; GCN-NOT: v_mov
 ; GCN: ds_write_b32 v0, v0
 define void @func_add_constant_to_fi_i32() #0 {
   %alloca = alloca [2 x i32], align 4
   %gep0 = getelementptr inbounds [2 x i32], [2 x i32]* %alloca, i32 0, i32 1
   store volatile i32* %gep0, i32* addrspace(3)* undef
   ret void
 }
 
 ; A user the materialized frame index can't be meaningfully folded
 ; into.
 
 ; GCN-LABEL: {{^}}func_other_fi_user_i32:
 ; GCN: s_sub_u32 s6, s5, s4
-; GCN-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
-; GCN-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+
+; CI-NEXT: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
+; CI-NEXT: v_add_i32_e64 v0, s[6:7], 4, [[SCALED]]
+
+; GFX9-NEXT: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
+; GFX9-NEXT: v_add_u32_e32 v0, 4, [[SCALED]]
+
 ; GCN-NEXT: v_mul_lo_i32 v0, v0, 9
 ; GCN-NOT: v_mov
 ; GCN: ds_write_b32 v0, v0
 define void @func_other_fi_user_i32() #0 {
   %alloca = alloca [2 x i32], align 4
   %ptrtoint = ptrtoint [2 x i32]* %alloca to i32
   %mul = mul i32 %ptrtoint, 9
   store volatile i32 %mul, i32 addrspace(3)* undef
define void @func_load_private_arg_i32_ptr(i32* %ptr) #0 {
   %val = load volatile i32, i32* %ptr
   ret void
 }
 
 ; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32_ptr:
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_mov_b32 s5, s32
 ; GCN-NEXT: s_sub_u32 [[SUB_OFFSET:s[0-9]+]], s5, s4
-; GCN-NEXT: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], [[SUB_OFFSET]], 6
-; GCN-NEXT: v_add_i32_e64 [[ADD:v[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 4, [[SHIFT]]
-; GCN-NEXT: v_add_i32_e32 v0, vcc, 4, [[ADD]]
+
+; CI-NEXT: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], [[SUB_OFFSET]], 6
+; CI-NEXT: v_add_i32_e64 [[ADD:v[0-9]+]], {{s\[[0-9]+:[0-9]+\]}}, 4, [[SHIFT]]
+; CI-NEXT: v_add_i32_e32 v0, vcc, 4, [[ADD]]
+
+; GFX9-NEXT: v_lshrrev_b32_e64 [[SHIFT:v[0-9]+]], 6, [[SUB_OFFSET]]
+; GFX9-NEXT: v_add_u32_e32 [[ADD:v[0-9]+]], 4, [[SHIFT]]
+; GFX9-NEXT: v_add_u32_e32 v0, 4, [[ADD]]
+
 ; GCN-NOT: v_mov
 ; GCN: ds_write_b32 v0, v0
 define void @void_func_byval_struct_i8_i32_ptr({ i8, i32 }* byval %arg0) #0 {
   %gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %arg0, i32 0, i32 0
   %gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %arg0, i32 0, i32 1
   %load1 = load i32, i32* %gep1
   store volatile i32* %gep1, i32* addrspace(3)* undef
   ret void
define void @void_func_byval_struct_i8_i32_ptr_value({ i8, i32 }* byval %arg0) #0 {
   ret void
 }
 
 ; FIXME: Should be able to see that this can use vaddr, but the
 ; FrameIndex is hidden behind a CopyFromReg in the second block.
 
 ; GCN-LABEL: {{^}}void_func_byval_struct_i8_i32_ptr_nonentry_block:
 ; GCN: s_sub_u32 [[SUB_OFFSET:s[0-9]+]], s5, s4
-; GCN: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], [[SUB_OFFSET]], 6
-; GCN: v_add_i32_e64 [[ADD:v[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 4, [[SHIFT]]
+
+; CI: v_lshr_b32_e64 [[SHIFT:v[0-9]+]], [[SUB_OFFSET]], 6
+; CI: v_add_i32_e64 [[ADD:v[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 4, [[SHIFT]]
+
+; GFX9: v_lshrrev_b32_e64 [[SHIFT:v[0-9]+]], 6, [[SUB_OFFSET]]
+; GFX9: v_add_u32_e32 [[ADD:v[0-9]+]], 4, [[SHIFT]]
+
 ; GCN: s_and_saveexec_b64
 
-; GCN: v_add_i32_e32 v0, vcc, 4, [[ADD]]
+; CI: v_add_i32_e32 v0, vcc, 4, [[ADD]]
+; GFX9: v_add_u32_e32 v0, 4, [[ADD]]
+
+
 ; GCN: buffer_load_dword v1, v0, s[0:3], s4 offen{{$}}
 ; GCN: ds_write_b32
 define void @void_func_byval_struct_i8_i32_ptr_nonentry_block({ i8, i32 }* byval %arg0, i32 %arg2) #0 {
   %cmp = icmp eq i32 %arg2, 0
   br i1 %cmp, label %bb, label %ret
 
 bb:
   %gep0 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %arg0, i32 0, i32 0
   %gep1 = getelementptr inbounds { i8, i32 }, { i8, i32 }* %arg0, i32 0, i32 1
   %load1 = load volatile i32, i32* %gep1
   store volatile i32* %gep1, i32* addrspace(3)* undef
   br label %ret
 
 ret:
   ret void
 }
 
 ; Added offset can't be used with VOP3 add
 ; GCN-LABEL: {{^}}func_other_fi_user_non_inline_imm_offset_i32:
 ; GCN: s_sub_u32 s6, s5, s4
-; GCN-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
 ; GCN-DAG: s_movk_i32 s6, 0x204
-; GCN: v_add_i32_e64 v0, s[6:7], s6, [[SCALED]]
+
+; CI-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], s6, 6
+; CI: v_add_i32_e64 v0, s[6:7], s6, [[SCALED]]
+
+; GFX9-DAG: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, s6
+; GFX9: v_add_u32_e32 v0, s6, [[SCALED]]
+
 ; GCN: v_mul_lo_i32 v0, v0, 9
 ; GCN: ds_write_b32 v0, v0
 define void @func_other_fi_user_non_inline_imm_offset_i32() #0 {
   %alloca0 = alloca [128 x i32], align 4
   %alloca1 = alloca [8 x i32], align 4
   %gep0 = getelementptr inbounds [128 x i32], [128 x i32]* %alloca0, i32 0, i32 65
   %gep1 = getelementptr inbounds [8 x i32], [8 x i32]* %alloca1, i32 0, i32 0
   store volatile i32 7, i32* %gep0
   %ptrtoint = ptrtoint i32* %gep1 to i32
   %mul = mul i32 %ptrtoint, 9
   store volatile i32 %mul, i32 addrspace(3)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}func_other_fi_user_non_inline_imm_offset_i32_vcc_live:
 ; GCN: s_sub_u32 [[DIFF:s[0-9]+]], s5, s4
-; GCN-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[DIFF]], 6
 ; GCN-DAG: s_movk_i32 [[OFFSET:s[0-9]+]], 0x204
-; GCN: v_add_i32_e64 v0, s{{\[[0-9]+:[0-9]+\]}}, [[OFFSET]], [[SCALED]]
+
+; CI-DAG: v_lshr_b32_e64 [[SCALED:v[0-9]+]], [[DIFF]], 6
+; CI: v_add_i32_e64 v0, s{{\[[0-9]+:[0-9]+\]}}, [[OFFSET]], [[SCALED]]
+
+; GFX9-DAG: v_lshrrev_b32_e64 [[SCALED:v[0-9]+]], 6, [[DIFF]]
+; GFX9: v_add_u32_e32 v0, [[OFFSET]], [[SCALED]]
+
 ; GCN: v_mul_lo_i32 v0, v0, 9
 ; GCN: ds_write_b32 v0, v0
 define void @func_other_fi_user_non_inline_imm_offset_i32_vcc_live() #0 {
   %alloca0 = alloca [128 x i32], align 4
   %alloca1 = alloca [8 x i32], align 4
   %vcc = call i64 asm sideeffect "; def $0", "={VCC}"()
   %gep0 = getelementptr inbounds [128 x i32], [128 x i32]* %alloca0, i32 0, i32 65
   %gep1 = getelementptr inbounds [8 x i32], [8 x i32]* %alloca1, i32 0, i32 0