[AMDGPU] Created a sub-register class for the return address operand in the return instruction.

[cdevadas]

Summary

Function return instruction lowering, currently uses the fixed register pair, s[30:31] for holding the return address.
It can be any SGPR pair other than the CSRs. Created an SGPR pair sub-register class exclusive of the CSRs, and used this register class while lowering the return instruction.

[cdevadas]

Hi Matt,

The codegen is different now and the scheduler & RA introduce most changes in the test cases.

[arsenm]

lib/Target/AMDGPU/SOPInstructions.td

133–137

The name doesn't match what this is for. I would rather not introduce a separate instruction class for this. You can instead parameterize the existing class with the (ins) for the instruction, defaulting to (ins SReg_64:$src0)

test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll

32

It looks like you manually added these checks instead of using update_llc_test_checks?

test/CodeGen/AMDGPU/nested-calls.ll

33

Are the results actually better looking? Can you add explicit checks for the register s_setpc_b64 is using?

[cdevadas]

Parameterized the existing operand class 'SOP1_1' to accommodate different register classes.
Added explicit check for the register pair in return instruction, 's_setpc_b64' (for nested-calls.ll test)

[arsenm]

lib/Target/AMDGPU/SIISelLowering.cpp

2237

This line looks too long? Run clang-format?

lib/Target/AMDGPU/SIRegisterInfo.td

492

You should bee able to use SGPR_64.RegTypes to avoid repeating the type list

493–494

You can also avoid repeating these with SGPR_64.CopyCost, and AllocationPriority

[cdevadas]

reused the existing register class' parameters for the new class.
Also ran clang-format to fix the long lines.

[arsenm]

LGTM

[jfb]

is this the cause of http://green.lab.llvm.org/green/job/clang-stage1-cmake-RA-incremental/62962/testReport/junit/LLVM/CodeGen_AMDGPU/spill_before_exec_mir/ ?

[jfb]

In D63924#1576560, @jfb wrote:

is this the cause of http://green.lab.llvm.org/green/job/clang-stage1-cmake-RA-incremental/62962/testReport/junit/LLVM/CodeGen_AMDGPU/spill_before_exec_mir/ ?

Matt says it's fixed in llvm.org/r365521. Thanks!

lib/Target/AMDGPU/SIISelLowering.cpp

SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
   RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
 
   // Add return address for callable functions.
   if (!Info->isEntryFunction()) {
     const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo();
     SDValue ReturnAddrReg = CreateLiveInRegister(
       DAG, &AMDGPU::SReg_64RegClass, TRI->getReturnAddressReg(MF), MVT::i64);
 
-    // FIXME: Should be able to use a vreg here, but need a way to prevent it
-    // from being allcoated to a CSR.
-
-    SDValue PhysReturnAddrReg = DAG.getRegister(TRI->getReturnAddressReg(MF),
-                                                MVT::i64);
-
-    Chain = DAG.getCopyToReg(Chain, DL, PhysReturnAddrReg, ReturnAddrReg, Flag);
+    SDValue ReturnAddrVirtualReg = DAG.getRegister(
+      MF.getRegInfo().createVirtualRegister(
+        &AMDGPU::CCR_SGPR_64RegClass), MVT::i64);
+    Chain = DAG.getCopyToReg(Chain, DL, ReturnAddrVirtualReg, ReturnAddrReg, Flag);
     Flag = Chain.getValue(1);
-
-    RetOps.push_back(PhysReturnAddrReg);
+    RetOps.push_back(ReturnAddrVirtualReg);
   }
 
   // Copy the result values into the output registers.
   for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E;
        ++I, ++RealRVLocIdx) {
     CCValAssign &VA = RVLocs[I];
     assert(VA.isRegLoc() && "Can only return in registers!");
     // TODO: Partially return in registers if return values don't fit.

lib/Target/AMDGPU/SIRegisterInfo.td

def SRegOrLds_32 : RegisterClass<"AMDGPU", [i32, f32, i16, f16, v2i16, v2f16, i1], 32,
   let isAllocatable = 0;
 }
 
 def SGPR_64 : RegisterClass<"AMDGPU", [v2i32, i64, v2f32, f64, v4i16, v4f16], 32, (add SGPR_64Regs)> {
   let CopyCost = 1;
   let AllocationPriority = 9;
 }
 
+// CCR (call clobbered registers) SGPR 64-bit registers
+def CCR_SGPR_64 : RegisterClass<"AMDGPU", [v2i32, i64, v2f32, f64, v4i16, v4f16], 32, (add (trunc SGPR_64, 16))> {
+  let CopyCost = 1;
+  let AllocationPriority = 9;
+}
+
 def TTMP_64 : RegisterClass<"AMDGPU", [v2i32, i64, f64, v4i16, v4f16], 32, (add TTMP_64Regs)> {
   let isAllocatable = 0;
 }
 
 def SReg_64_XEXEC : RegisterClass<"AMDGPU", [v2i32, i64, v2f32, f64, i1, v4i16, v4f16], 32,
   (add SGPR_64, VCC, FLAT_SCR, XNACK_MASK, TTMP_64, TBA, TMA)> {
   let CopyCost = 1;
   let AllocationPriority = 9;

lib/Target/AMDGPU/SOPInstructions.td

 }
 
 // 64-bit input, no output
 class SOP1_1 <string opName, list<dag> pattern=[]> : SOP1_Pseudo <
   opName, (outs), (ins SReg_64:$src0), "$src0", pattern> {
   let has_sdst = 0;
 }
 
+// 64-bit input (subset of SGPR_64) , no output
+class SOP1_64_2 <string opName, list<dag> pattern=[]> : SOP1_Pseudo <
+  opName, (outs), (ins CCR_SGPR_64:$src0), "$src0", pattern> {
+  let has_sdst = 0;
+}
+
 
 let isMoveImm = 1 in {
   let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
     def S_MOV_B32 : SOP1_32 <"s_mov_b32">;
     def S_MOV_B64 : SOP1_64 <"s_mov_b64">;
   } // End isRematerializeable = 1
 
   let Uses = [SCC] in {

 let isTerminator = 1, isBarrier = 1, SchedRW = [WriteBranch] in {
 
 let isBranch = 1, isIndirectBranch = 1 in {
 def S_SETPC_B64 : SOP1_1  <"s_setpc_b64">;
 } // End isBranch = 1, isIndirectBranch = 1
 
 let isReturn = 1 in {
 // Define variant marked as return rather than branch.
-def S_SETPC_B64_return : SOP1_1<"", [(AMDGPUret_flag i64:$src0)]>;
+def S_SETPC_B64_return : SOP1_64_2<"", [(AMDGPUret_flag i64:$src0)]>;
 }
 } // End isTerminator = 1, isBarrier = 1
 
 let isCall = 1 in {
 def S_SWAPPC_B64 : SOP1_64 <"s_swappc_b64"
 >;
 }
 

test/CodeGen/AMDGPU/call-graph-register-usage.ll

 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,CI %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=fiji -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,VI-NOBUG %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mattr=-code-object-v3 -mcpu=iceland -enable-ipra=0 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,VI-BUG %s
 
 ; Make sure to run a GPU with the SGPR allocation bug.
 
 ; GCN-LABEL: {{^}}use_vcc:
 ; GCN: ; NumSgprs: 34
 ; GCN: ; NumVgprs: 0
 define void @use_vcc() #1 {
   call void asm sideeffect "", "~{vcc}" () #0
   ret void
 }
 
 ; GCN-LABEL: {{^}}indirect_use_vcc:
 ; GCN: v_writelane_b32 v32, s34, 0
-; GCN: v_writelane_b32 v32, s35, 1
-; GCN: v_writelane_b32 v32, s36, 2
+; GCN: v_writelane_b32 v32, s30, 1
+; GCN: v_writelane_b32 v32, s31, 2
 ; GCN: s_swappc_b64
-; GCN: v_readlane_b32 s36, v32, 2
-; GCN: v_readlane_b32 s35, v32, 1
+; GCN: v_readlane_b32 s6, v32, 1
+; GCN: v_readlane_b32 s7, v32, 2
 ; GCN: v_readlane_b32 s34, v32, 0
-; GCN: ; NumSgprs: 39
+; GCN: ; NumSgprs: 37
 ; GCN: ; NumVgprs: 33
 define void @indirect_use_vcc() #1 {
   call void @use_vcc()
   ret void
 }
 
 ; GCN-LABEL: {{^}}indirect_2level_use_vcc_kernel:
 ; GCN: is_dynamic_callstack = 0
-; CI: ; NumSgprs: 41
-; VI-NOBUG: ; NumSgprs: 43
+; CI: ; NumSgprs: 39
+; VI-NOBUG: ; NumSgprs: 41
 ; VI-BUG: ; NumSgprs: 96
 ; GCN: ; NumVgprs: 33
 define amdgpu_kernel void @indirect_2level_use_vcc_kernel(i32 addrspace(1)* %out) #0 {
   call void @indirect_use_vcc()
   ret void
 }
 
 ; GCN-LABEL: {{^}}use_flat_scratch:
 ; CI: ; NumSgprs: 36
 ; VI: ; NumSgprs: 38
 ; GCN: ; NumVgprs: 0
 define void @use_flat_scratch() #1 {
   call void asm sideeffect "", "~{flat_scratch}" () #0
   ret void
 }
 
 ; GCN-LABEL: {{^}}indirect_use_flat_scratch:
-; CI: ; NumSgprs: 41
-; VI: ; NumSgprs: 43
+; CI: ; NumSgprs: 39
+; VI: ; NumSgprs: 41
 ; GCN: ; NumVgprs: 33
 define void @indirect_use_flat_scratch() #1 {
   call void @use_flat_scratch()
   ret void
 }
 
 ; GCN-LABEL: {{^}}indirect_2level_use_flat_scratch_kernel:
 ; GCN: is_dynamic_callstack = 0
-; CI: ; NumSgprs: 41
-; VI-NOBUG: ; NumSgprs: 43
+; CI: ; NumSgprs: 39
+; VI-NOBUG: ; NumSgprs: 41
 ; VI-BUG: ; NumSgprs: 96
 ; GCN: ; NumVgprs: 33
 define amdgpu_kernel void @indirect_2level_use_flat_scratch_kernel(i32 addrspace(1)* %out) #0 {
   call void @indirect_use_flat_scratch()
   ret void
 }
 
 ; GCN-LABEL: {{^}}use_10_vgpr:

test/CodeGen/AMDGPU/call-preserved-registers.ll

define amdgpu_kernel void @test_kernel_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void() #0 {
   call void @external_void_func_void()
   ret void
 }
 
 ; GCN-LABEL: {{^}}test_func_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void:
 ; GCN: v_writelane_b32 v32, s34, 0
 ; GCN: v_writelane_b32 v32, s35, 1
 ; GCN: v_writelane_b32 v32, s36, 2
-; GCN: v_writelane_b32 v32, s37, 3
-; GCN: v_writelane_b32 v32, s38, 4
+; GCN: v_writelane_b32 v32, s30, 3
+; GCN: v_writelane_b32 v32, s31, 4
 
 ; GCN: s_mov_b32 [[COPY_FP:s[0-9]+]], s5
 ; GCN-NEXT: s_swappc_b64
 ; GCN-NEXT: s_mov_b32 s5, [[COPY_FP]]
 ; GCN-NEXT: s_mov_b32 [[COPY_FP]], s5
 ; GCN-NEXT: ;;#ASMSTART
 ; GCN-NEXT: ;;#ASMEND
 ; GCN-NEXT: s_swappc_b64
+; GCN-DAG: v_readlane_b32 s6, v32, 3
 ; GCN-DAG: s_mov_b32 s5, [[COPY_FP]]
-; GCN-DAG: v_readlane_b32 s38, v32, 4
-; GCN: v_readlane_b32 s37, v32, 3
+; GCN: v_readlane_b32 s7, v32, 4
 ; GCN: v_readlane_b32 s36, v32, 2
 ; GCN: v_readlane_b32 s35, v32, 1
 ; GCN: v_readlane_b32 s34, v32, 0
 ; GCN: s_setpc_b64
 define void @test_func_call_external_void_func_void_clobber_s30_s31_call_external_void_func_void() #0 {
   call void @external_void_func_void()
   call void asm sideeffect "", ""() #0
   call void @external_void_func_void()

 }
 
 ; GCN-LABEL: {{^}}void_func_void_clobber_s30_s31:
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_mov_b64 [[SAVEPC:s\[[0-9]+:[0-9]+\]]], s[30:31]
 ; GCN-NEXT: #ASMSTART
 ; GCN: ; clobber
 ; GCN-NEXT: #ASMEND
-; GCN-NEXT: s_mov_b64 s[30:31], [[SAVEPC]]
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; GCN-NEXT: s_setpc_b64 [[SAVEPC]]
 define void @void_func_void_clobber_s30_s31() #2 {
   call void asm sideeffect "; clobber", "~{s[30:31]}"() #0
   ret void
 }
 
 ; GCN-LABEL: {{^}}void_func_void_clobber_vcc:
 ; GCN: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT: ;;#ASMSTART

test/CodeGen/AMDGPU/callee-frame-setup.ll

 ; GCN-LABEL: {{^}}callee_with_stack_and_call:
 ; GCN: ; %bb.0:
 ; GCN-NEXT: s_waitcnt
 ; GCN: s_mov_b32 s5, s32
 ; GCN-DAG: s_add_u32 s32, s32, 0x400{{$}}
 ; GCN: buffer_store_dword v32, off, s[0:3], s5 offset:4
 
 ; GCN-DAG: v_writelane_b32 v32, s34,
-; GCN-DAG: v_writelane_b32 v32, s35,
-; GCN-DAG: v_writelane_b32 v32, s36,
+; GCN-DAG: v_writelane_b32 v32, s30,
+; GCN-DAG: v_writelane_b32 v32, s31,
 ; GCN-DAG: v_mov_b32_e32 v0, 0{{$}}
 ; GCN-DAG: buffer_store_dword v0, off, s[0:3], s5{{$}}
 ; GCN-DAG: s_mov_b32 [[COPY_FP:s[0-9]+]], s5
 
 
 ; GCN: s_swappc_b64
+; GCN-DAG: v_readlane_b32 s6,
 ; GCN-DAG: s_mov_b32 s5, [[COPY_FP]]
+; GCN-DAG: v_readlane_b32 s7,
 ; GCN-DAG: v_readlane_b32 s34,
-; GCN-DAG: v_readlane_b32 s35,
-; GCN-DAG: v_readlane_b32 s36,
 ; GCN: buffer_load_dword v32, off, s[0:3], s5 offset:4
 ; GCN: s_waitcnt
 ; GCN-NEXT: s_setpc_b64
 define void @callee_with_stack_and_call() #0 {
   %alloca = alloca i32, addrspace(5)
   store volatile i32 0, i32 addrspace(5)* %alloca
   call void @external_void_func_void()
   ret void
 }
 
 ; Should be able to copy incoming stack pointer directly to inner
 ; call's stack pointer argument.
 
 ; There is stack usage only because of the need to evict a VGPR for
 ; spilling CSR SGPRs.
 
 ; GCN-LABEL: {{^}}callee_no_stack_with_call:
 ; GCN: s_waitcnt
 ; GCN: s_mov_b32 s5, s32
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT: buffer_store_dword v32, off, s[0:3], s5 ; 4-byte Folded Spill
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN-DAG: v_writelane_b32 v32, s34, 0
-; GCN-DAG: v_writelane_b32 v32, s35, 1
-; GCN-DAG: v_writelane_b32 v32, s36, 2
+; GCN-DAG: v_writelane_b32 v32, s30, 1
+; GCN-DAG: v_writelane_b32 v32, s31, 2
 ; GCN-DAG: s_mov_b32 [[COPY_FP:s[0-9]+]], s5
 ; GCN: s_swappc_b64
-; GCN: s_mov_b32 s5, [[COPY_FP]]
+; GCN: v_readlane_b32 s6, v32, 1
 
+; GCN-DAG: s_mov_b32 s5, [[COPY_FP]]
+; GCN-DAG: v_readlane_b32 s7, v32, 2
 ; GCN-DAG: v_readlane_b32 s34, v32, 0
-; GCN-DAG: v_readlane_b32 s35, v32, 1
-; GCN-DAG: v_readlane_b32 s36, v32, 2
 
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s5 ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
 
 ; GCN: s_sub_u32 s32, s32, 0x400
 ; GCN: s_setpc_b64
 define void @callee_no_stack_with_call() #0 {

test/CodeGen/AMDGPU/callee-special-input-sgprs.ll

 ; GCN-NOT: s10
 ; GCN-NOT: s11
 ; GCN-NOT: s12
 ; GCN-NOT: s13
 ; GCN-NOT: s[6:7]
 ; GCN-NOT: s[8:9]
 ; GCN-NOT: s[10:11]
 ; GCN-NOT: s[12:13]
-; GCN: s_or_saveexec_b64 s[6:7], -1
+; GCN: v_readlane_b32 s6, v32, 1
+; GCN: s_or_saveexec_b64 s[8:9], -1
 define hidden void @func_indirect_use_every_sgpr_input() #1 {
   call void @use_every_sgpr_input()
   ret void
 }
 
 ; GCN-LABEL: {{^}}func_use_every_sgpr_input_call_use_workgroup_id_xyz:
 ; GCN-NOT: s_mov_b32 s4
 ; GCN: s_mov_b32 s6, s14

test/CodeGen/AMDGPU/chain-hi-to-lo.ll

bb:
   %to.hi = insertelement <2 x i16> undef, i16 %load_hi, i32 1
   %op.hi = add <2 x i16> %to.hi, <i16 12, i16 12>
   %result = insertelement <2 x i16> %op.hi, i16 %load_lo, i32 0
   ret <2 x i16> %result
 }
 
 ; GCN-LABEL: {{^}}chain_hi_to_lo_group_may_alias_store:
 ; GFX900: v_mov_b32_e32 [[K:v[0-9]+]], 0x7b
-; GFX900-NEXT: ds_read_u16 v3, v0
+; GFX900-NEXT: ds_read_u16 v2, v0
 ; GFX900-NEXT: ds_write_b16 v1, [[K]]
 ; GFX900-NEXT: ds_read_u16 v0, v0 offset:2
 ; GFX900-NEXT: s_waitcnt lgkmcnt(0)
 ; GFX900-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX900-NEXT: v_lshl_or_b32 v0, v3, 16, v0
+; GFX900-NEXT: v_lshl_or_b32 v0, v2, 16, v0
 ; GFX900-NEXT: s_setpc_b64
 define <2 x i16> @chain_hi_to_lo_group_may_alias_store(i16 addrspace(3)* %ptr, i16 addrspace(3)* %may.alias) {
 bb:
   %gep_lo = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i64 1
   %gep_hi = getelementptr inbounds i16, i16 addrspace(3)* %ptr, i64 0
   %load_hi = load i16, i16 addrspace(3)* %gep_hi
   store i16 123, i16 addrspace(3)* %may.alias
   %load_lo = load i16, i16 addrspace(3)* %gep_lo
 
   %to.hi = insertelement <2 x i16> undef, i16 %load_hi, i32 1
   %result = insertelement <2 x i16> %to.hi, i16 %load_lo, i32 0
   ret <2 x i16> %result
 }

test/CodeGen/AMDGPU/cross-block-use-is-not-abi-copy.ll

 
 
 define float @call_split_type_used_outside_block_v2f32() #0 {
 ; GCN-LABEL: call_split_type_used_outside_block_v2f32:
 ; GCN:       ; %bb.0: ; %bb0
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_mov_b32 s5, s32
 ; GCN-NEXT:    s_add_u32 s32, s32, 0x400
-; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s5 ; 4-byte Folded Spill
-; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN-NEXT:    v_writelane_b32 v32, s34, 0
-; GCN-NEXT:    v_writelane_b32 v32, s35, 1
-; GCN-NEXT:    v_writelane_b32 v32, s36, 2
+; GCN-NEXT:    v_writelane_b32 v32, s30, 1
+; GCN-NEXT:    v_writelane_b32 v32, s31, 2
 ; GCN-NEXT:    s_getpc_b64 s[6:7]
 ; GCN-NEXT:    s_add_u32 s6, s6, func_v2f32@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s7, s7, func_v2f32@rel32@hi+4
-; GCN-NEXT:    s_mov_b64 s[34:35], s[30:31]
-; GCN-NEXT:    s_mov_b32 s36, s5
+; GCN-NEXT:    s_mov_b32 [[COPY_FP:s[0-9]+]], s5
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[6:7]
-; GCN-NEXT:    s_mov_b32 s5, s36
-; GCN-NEXT:    s_mov_b64 s[30:31], s[34:35]
-; GCN-NEXT:    v_readlane_b32 s36, v32, 2
-; GCN-NEXT:    v_readlane_b32 s35, v32, 1
+; GCN-NEXT:    v_readlane_b32 s6, v32, 1
+; GCN-NEXT:    s_mov_b32 s5, [[COPY_FP]]
+; GCN-NEXT:    v_readlane_b32 s7, v32, 2
 ; GCN-NEXT:    v_readlane_b32 s34, v32, 0
-; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s5 ; 4-byte Folded Reload
-; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT:    s_sub_u32 s32, s32, 0x400
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:    s_setpc_b64 s[6:7]
 bb0:
   %split.ret.type = call <2 x float> @func_v2f32()
   br label %bb1
 
 bb1:
   %extract = extractelement <2 x float> %split.ret.type, i32 0
   ret float %extract
 }
 
 define float @call_split_type_used_outside_block_v3f32() #0 {
 ; GCN-LABEL: call_split_type_used_outside_block_v3f32:
 ; GCN:       ; %bb.0: ; %bb0
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_mov_b32 s5, s32
 ; GCN-NEXT:    s_add_u32 s32, s32, 0x400
-; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s5 ; 4-byte Folded Spill
-; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN-NEXT:    v_writelane_b32 v32, s34, 0
-; GCN-NEXT:    v_writelane_b32 v32, s35, 1
-; GCN-NEXT:    v_writelane_b32 v32, s36, 2
+; GCN-NEXT:    v_writelane_b32 v32, s30, 1
+; GCN-NEXT:    v_writelane_b32 v32, s31, 2
 ; GCN-NEXT:    s_getpc_b64 s[6:7]
 ; GCN-NEXT:    s_add_u32 s6, s6, func_v3f32@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s7, s7, func_v3f32@rel32@hi+4
-; GCN-NEXT:    s_mov_b64 s[34:35], s[30:31]
-; GCN-NEXT:    s_mov_b32 s36, s5
+; GCN-NEXT:    s_mov_b32 [[COPY_FP:s[0-9]+]], s5
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[6:7]
-; GCN-NEXT:    s_mov_b32 s5, s36
-; GCN-NEXT:    s_mov_b64 s[30:31], s[34:35]
-; GCN-NEXT:    v_readlane_b32 s36, v32, 2
-; GCN-NEXT:    v_readlane_b32 s35, v32, 1
+; GCN-NEXT:    v_readlane_b32 s6, v32, 1
+; GCN-NEXT:    s_mov_b32 s5, [[COPY_FP]]
+; GCN-NEXT:    v_readlane_b32 s7, v32, 2
 ; GCN-NEXT:    v_readlane_b32 s34, v32, 0
-; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s5 ; 4-byte Folded Reload
-; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT:    s_sub_u32 s32, s32, 0x400
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:    s_setpc_b64 s[6:7]
 bb0:
   %split.ret.type = call <3 x float> @func_v3f32()
   br label %bb1
 
 bb1:
   %extract = extractelement <3 x float> %split.ret.type, i32 0
   ret float %extract
 }
 
 define half @call_split_type_used_outside_block_v4f16() #0 {
 ; GCN-LABEL: call_split_type_used_outside_block_v4f16:
 ; GCN:       ; %bb.0: ; %bb0
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_mov_b32 s5, s32
 ; GCN-NEXT:    s_add_u32 s32, s32, 0x400
-; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s5 ; 4-byte Folded Spill
-; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN-NEXT:    v_writelane_b32 v32, s34, 0
-; GCN-NEXT:    v_writelane_b32 v32, s35, 1
-; GCN-NEXT:    v_writelane_b32 v32, s36, 2
+; GCN-NEXT:    v_writelane_b32 v32, s30, 1
+; GCN-NEXT:    v_writelane_b32 v32, s31, 2
 ; GCN-NEXT:    s_getpc_b64 s[6:7]
 ; GCN-NEXT:    s_add_u32 s6, s6, func_v4f16@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s7, s7, func_v4f16@rel32@hi+4
-; GCN-NEXT:    s_mov_b64 s[34:35], s[30:31]
-; GCN-NEXT:    s_mov_b32 s36, s5
+; GCN-NEXT:    s_mov_b32 [[COPY_FP:s[0-9]+]], s5
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[6:7]
-; GCN-NEXT:    s_mov_b32 s5, s36
-; GCN-NEXT:    s_mov_b64 s[30:31], s[34:35]
-; GCN-NEXT:    v_readlane_b32 s36, v32, 2
-; GCN-NEXT:    v_readlane_b32 s35, v32, 1
+; GCN-NEXT:    v_readlane_b32 s6, v32, 1
+; GCN-NEXT:    s_mov_b32 s5, [[COPY_FP]]
+; GCN-NEXT:    v_readlane_b32 s7, v32, 2
 ; GCN-NEXT:    v_readlane_b32 s34, v32, 0
-; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s5 ; 4-byte Folded Reload
-; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT:    s_sub_u32 s32, s32, 0x400
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:    s_setpc_b64 s[6:7]
 bb0:
   %split.ret.type = call <4 x half> @func_v4f16()
   br label %bb1
 
 bb1:
   %extract = extractelement <4 x half> %split.ret.type, i32 0
   ret half %extract
 }
 
 define { i32, half } @call_split_type_used_outside_block_struct() #0 {
 ; GCN-LABEL: call_split_type_used_outside_block_struct:
 ; GCN:       ; %bb.0: ; %bb0
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    s_mov_b32 s5, s32
 ; GCN-NEXT:    s_add_u32 s32, s32, 0x400
-; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT:    buffer_store_dword v32, off, s[0:3], s5 ; 4-byte Folded Spill
-; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b64 exec, [[COPY_EXEC0]]
 ; GCN-NEXT:    v_writelane_b32 v32, s34, 0
-; GCN-NEXT:    v_writelane_b32 v32, s35, 1
-; GCN-NEXT:    v_writelane_b32 v32, s36, 2
+; GCN-NEXT:    v_writelane_b32 v32, s30, 1
+; GCN-NEXT:    v_writelane_b32 v32, s31, 2
 ; GCN-NEXT:    s_getpc_b64 s[6:7]
 ; GCN-NEXT:    s_add_u32 s6, s6, func_struct@rel32@lo+4
 ; GCN-NEXT:    s_addc_u32 s7, s7, func_struct@rel32@hi+4
-; GCN-NEXT:    s_mov_b64 s[34:35], s[30:31]
-; GCN-NEXT:    s_mov_b32 s36, s5
+; GCN-NEXT:    s_mov_b32 [[COPY_FP:s[0-9]+]], s5
 ; GCN-NEXT:    s_swappc_b64 s[30:31], s[6:7]
-; GCN-NEXT:    s_mov_b32 s5, s36
+; GCN-NEXT:    v_readlane_b32 s6, v32, 1
+; GCN-NEXT:    s_mov_b32 s5, [[COPY_FP]]
+; GCN-NEXT:    v_readlane_b32 s7, v32, 2
 ; GCN-NEXT:    v_mov_b32_e32 v1, v4
-; GCN-NEXT:    s_mov_b64 s[30:31], s[34:35]
-; GCN-NEXT:    v_readlane_b32 s36, v32, 2
-; GCN-NEXT:    v_readlane_b32 s35, v32, 1
 ; GCN-NEXT:    v_readlane_b32 s34, v32, 0
-; GCN-NEXT:    s_or_saveexec_b64 s[6:7], -1
+; GCN-NEXT:    s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT:    buffer_load_dword v32, off, s[0:3], s5 ; 4-byte Folded Reload
-; GCN-NEXT:    s_mov_b64 exec, s[6:7]
+; GCN-NEXT:    s_mov_b64 exec, [[COPY_EXEC1]]
 ; GCN-NEXT:    s_sub_u32 s32, s32, 0x400
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GCN-NEXT:    s_setpc_b64 s[6:7]
 bb0:
   %split.ret.type = call { <4 x i32>, <4 x half> } @func_struct()
   br label %bb1
 
 bb1:
   %val0 = extractvalue { <4 x i32>, <4 x half> } %split.ret.type, 0
   %val1 = extractvalue { <4 x i32>, <4 x half> } %split.ret.type, 1
   %extract0 = extractelement <4 x i32> %val0, i32 0

test/CodeGen/AMDGPU/llvm.log.f16.ll

 
 ; FUNC-LABEL: {{^}}log_v2f16
 ; SI:     buffer_load_dword v[[A_F16_0:[0-9]+]]
 ; VI:     flat_load_dword v[[A_F16_0:[0-9]+]]
 ; GFX9:   global_load_dword v[[A_F16_0:[0-9]+]]
 ; SI:     s_mov_b32 [[A_F32_2:s[0-9]+]], 0x3f317218
 ; VIGFX9: s_movk_i32 [[A_F32_2:s[0-9]+]], 0x398c
 ; VI:     v_mov_b32_e32 [[A_F32_2_V:v[0-9]+]], [[A_F32_2]]
-; SI:     v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_0]]
 ; SI:     v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_F16_0]]
-; SI:     v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
-; SI:     v_log_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]]
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_1]]
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_0]]
 ; SI:     v_log_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]]
-; SI:     v_mul_f32_e32 v[[R_F32_6:[0-9]+]], [[A_F32_2]], v[[R_F32_1]]
-; SI:     v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_6]]
+; SI:     v_log_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]]
 ; SI:     v_mul_f32_e32 v[[R_F32_5:[0-9]+]], [[A_F32_2]], v[[R_F32_0]]
 ; SI:     v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_5]]
+; SI:     v_mul_f32_e32 v[[R_F32_6:[0-9]+]], [[A_F32_2]], v[[R_F32_1]]
+; SI:     v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_6]]
 ; GFX9:   v_log_f16_e32 v[[R_F16_2:[0-9]+]], v[[A_F16_0]]
 ; VIGFX9: v_log_f16_sdwa v[[R_F16_1:[0-9]+]], v[[A_F16_0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
 ; VI:     v_log_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_F16_0]]
 ; VI:     v_mul_f16_sdwa v[[R_F16_2:[0-9]+]], v[[R_F16_1]], [[A_F32_2_V]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9:   v_mul_f16_e32 v[[R_F32_3:[0-9]+]], [[A_F32_2]], v[[R_F16_2]]
 ; VIGFX9: v_mul_f16_e32 v[[R_F32_2:[0-9]+]], [[A_F32_2]], v[[R_F16_0]]
 ; SI:     v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_0]]
 ; SI-NOT: v_and_b32_e32

test/CodeGen/AMDGPU/llvm.log10.f16.ll

 
 ; GCN-LABEL: {{^}}log10_v2f16
 ; SI:     buffer_load_dword v[[A_F16_0:[0-9]+]]
 ; VI:     flat_load_dword v[[A_F16_0:[0-9]+]]
 ; GFX9:   global_load_dword v[[A_F16_0:[0-9]+]]
 ; SI:     s_mov_b32 [[A_F32_2:s[0-9]+]], 0x3e9a209a
 ; VIGFX9: s_movk_i32 [[A_F32_2:s[0-9]+]], 0x34d1
 ; VI:     v_mov_b32_e32 [[A_F32_2_V:v[0-9]+]], [[A_F32_2]]
-; SI:     v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_0]]
 ; SI:     v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_F16_0]]
-; SI:     v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_0]]
-; SI:     v_log_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]]
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_F16_1]]
+; SI:     v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_0]]
 ; SI:     v_log_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]]
-; SI:     v_mul_f32_e32 v[[R_F32_6:[0-9]+]], [[A_F32_2]], v[[R_F32_1]]
-; SI:     v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_6]]
+; SI:     v_log_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]]
 ; SI:     v_mul_f32_e32 v[[R_F32_5:[0-9]+]], [[A_F32_2]], v[[R_F32_0]]
 ; SI:     v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_5]]
+; SI:     v_mul_f32_e32 v[[R_F32_6:[0-9]+]], [[A_F32_2]], v[[R_F32_1]]
+; SI:     v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_6]]
 ; GFX9:   v_log_f16_e32 v[[R_F16_2:[0-9]+]], v[[A_F16_0]]
 ; VIGFX9: v_log_f16_sdwa v[[R_F16_1:[0-9]+]], v[[A_F16_0]] dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
 ; VI:     v_log_f16_e32 v[[R_F16_0:[0-9]+]], v[[A_F16_0]]
 ; VI:     v_mul_f16_sdwa v[[R_F16_2:[0-9]+]], v[[R_F16_1]], [[A_F32_2_V]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX9:   v_mul_f16_e32 v[[R_F32_3:[0-9]+]], [[A_F32_2]], v[[R_F16_2]]
 ; VIGFX9: v_mul_f16_e32 v[[R_F32_2:[0-9]+]], [[A_F32_2]], v[[R_F16_0]]
 ; SI:     v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_0]]
 ; SI-NOT: v_and_b32_e32

test/CodeGen/AMDGPU/load-lo16.ll

entry:
   %build1 = insertelement <2 x i16> %reg, i16 %load, i32 0
   store <2 x i16> %build1, <2 x i16> addrspace(1)* undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}load_local_lo_v2i16_reghi_vreg_multi_use_lohi:
 ; GFX900: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX900: ds_read_u16 v0, v0
-; GFX900: v_lshrrev_b32_e32 v4, 16, v1
+; GFX900: v_lshrrev_b32_e32 v[[A_F16:[0-9]+]], 16, v1
+; GFX900: v_mov_b32_e32 v[[A_F32:[0-9]+]], 0xffff
 ; GFX900: s_waitcnt lgkmcnt(0)
 ; GFX900: ds_write_b16 v2, v0
-; GFX900: ds_write_b16 v3, v4
-; GFX900: v_mov_b32_e32 v2, 0xffff
-; GFX900: v_bfi_b32 v0, v2, v0, v1
+; GFX900: ds_write_b16 v3, v[[A_F16]]
+; GFX900: v_bfi_b32 v0, v[[A_F32]], v0, v1
 ; GFX900: global_store_dword v[0:1], v0, off
 ; GFX900: s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX900: s_setpc_b64 s[30:31]
 
 ; NO-D16-HI: ds_read_u16 v
 define void @load_local_lo_v2i16_reghi_vreg_multi_use_lohi(i16 addrspace(3)* noalias %in, <2 x i16> %reg, i16 addrspace(3)* noalias %out0, i16 addrspace(3)* noalias %out1) #0 {
 entry:
   %load = load i16, i16 addrspace(3)* %in

test/CodeGen/AMDGPU/nested-calls.ll

 ; GCN: s_mov_b32 s5, s32
 ; GCN-DAG: s_add_u32 s32, s32, 0x400
 ; Spill CSR VGPR used for SGPR spilling
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC0:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT: buffer_store_dword v32, off, s[0:3], s5 ; 4-byte Folded Spill
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 
 ; GCN-DAG: v_writelane_b32 v32, s34, 0
-; GCN-DAG: v_writelane_b32 v32, s35, 1
-; GCN-DAG: v_writelane_b32 v32, s36, 2
+; GCN-DAG: v_writelane_b32 v32, s30, 1
+; GCN-DAG: v_writelane_b32 v32, s31, 2
 
 ; GCN: s_swappc_b64
 
-; GCN: v_readlane_b32 s36, v32, 2
-; GCN: v_readlane_b32 s35, v32, 1
+; GCN: v_readlane_b32 s6, v32, 1
+; GCN: v_readlane_b32 s7, v32, 2
 ; GCN: v_readlane_b32 s34, v32, 0
 ; GCN: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s5 ; 4-byte Folded Reload
 ; GCN-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
 
 ; GCN: s_sub_u32 s32, s32, 0x400
 ; GCN: s_setpc_b64
 define void @test_func_call_external_void_func_i32_imm() #0 {
   call void @external_void_func_i32(i32 42)
   ret void
 }
 
 ; GCN-LABEL: {{^}}test_func_call_external_void_func_i32_imm_stack_use:
 ; GCN: s_waitcnt
 ; GCN: s_mov_b32 s5, s32

test/CodeGen/AMDGPU/wave32.ll

 ; GFX1032: s_or_saveexec_b32 [[COPY_EXEC0:s[0-9]]], -1{{$}}
 
 ; GCN-NEXT: buffer_store_dword v32, off, s[0:3], s5 ; 4-byte Folded Spill
 
 ; GFX1064-NEXT: s_mov_b64 exec, [[COPY_EXEC0]]
 ; GFX1032-NEXT: s_mov_b32 exec_lo, [[COPY_EXEC0]]
 
 ; GCN-DAG: v_writelane_b32 v32, s34, 0
-; GCN-DAG: v_writelane_b32 v32, s35, 1
+; GCN-DAG: v_writelane_b32 v32, s30, 1
+; GCN-DAG: v_writelane_b32 v32, s31, 2
 ; GCN-DAG: s_mov_b32 [[COPY_FP:s[0-9]+]], s5
 ; GCN: s_swappc_b64
+; GCN-DAG: v_readlane_b32 s6, v32, 1
 ; GCN-DAG: s_mov_b32 s5, [[COPY_FP]]
-; GCN-DAG: v_readlane_b32 s35, v32, 1
+; GCN-DAG: v_readlane_b32 s7, v32, 2
 ; GCN-DAG: v_readlane_b32 s34, v32, 0
 
 ; GFX1064: s_or_saveexec_b64 [[COPY_EXEC1:s\[[0-9]+:[0-9]+\]]], -1{{$}}
 ; GFX1032: s_or_saveexec_b32 [[COPY_EXEC1:s[0-9]]], -1{{$}}
 ; GCN-NEXT: buffer_load_dword v32, off, s[0:3], s5 ; 4-byte Folded Reload
 ; GFX1064-NEXT: s_mov_b64 exec, [[COPY_EXEC1]]
 ; GFX1032-NEXT: s_mov_b32 exec_lo, [[COPY_EXEC1]]