GlobalISel: Fix artifact combine value finder look through copy

[Petar.Avramovic]

Summary

Search for COPY source in instruction we get from look through (not copy dst).

llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h

       return Register();
     }
 
     /// Internal implementation for findValueFromDef(). findValueFromDef()
     /// initializes some data like the CurrentBest register, which this method
     /// and its callees rely upon.
     Register findValueFromDefImpl(Register DefReg, unsigned StartBit,
                                   unsigned Size) {
-      MachineInstr *Def = getDefIgnoringCopies(DefReg, MRI);
+      Optional<DefinitionAndSourceRegister> DefSrcReg =
+          getDefSrcRegIgnoringCopies(DefReg, MRI);
+      MachineInstr *Def = DefSrcReg->MI;
+      DefReg = DefSrcReg->Reg;
       // If the instruction has a single def, then simply delegate the search.
       // For unmerge however with multiple defs, we need to compute the offset
       // into the source of the unmerge.
       switch (Def->getOpcode()) {
       case TargetOpcode::G_CONCAT_VECTORS:
         return findValueFromConcat(cast<GConcatVectors>(*Def), StartBit, Size);
       case TargetOpcode::G_UNMERGE_VALUES: {
         unsigned DefStartBit = 0;

llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-build-vector.mir

     %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
     %1:_(<2 x s32>) = COPY $vgpr2_vgpr3
     %2:_(s32) = COPY $vgpr4
     %3:_(s32), %4:_(s32) = G_UNMERGE_VALUES %0(<2 x s32>)
     %5:_(s32), %6:_(s32) = G_UNMERGE_VALUES %1(<2 x s32>)
     %7:_(<5 x s32>) = G_BUILD_VECTOR %3, %4, %5, %6, %2
     $vgpr5_vgpr6_vgpr7_vgpr8_vgpr9= COPY %7
 ...
+
+---
+name: value_finder_look_through_copy
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+
+    ; GFX9-LABEL: name: value_finder_look_through_copy
+    ; GFX9: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3
+    ; GFX9-NEXT: {{  $}}
+    ; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    ; GFX9-NEXT: $vgpr2_vgpr3 = COPY [[COPY]](<2 x s32>)
+    %0:_(<2 x s32>) = COPY $vgpr0_vgpr1
+    %1:_(s32), %2:_(s32) = G_UNMERGE_VALUES %0(<2 x s32>)
+    %3:_(s32) = COPY %1
+    %4:_(<2 x s32>) = G_BUILD_VECTOR %3, %2
+    $vgpr2_vgpr3= COPY %4
+...

llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.i128.ll

   %element = extractelement <4 x i128> %vector, i32 0
   ret i128 %element
 }
 
 define i128 @extractelement_vgpr_v4i128_idx1(<4 x i128> addrspace(1)* %ptr) {
 ; GFX9-LABEL: extractelement_vgpr_v4i128_idx1:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx4 v[4:7], v[0:1], off offset:16
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off offset:16
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, v4
-; GFX9-NEXT:    v_mov_b32_e32 v1, v5
-; GFX9-NEXT:    v_mov_b32_e32 v2, v6
-; GFX9-NEXT:    v_mov_b32_e32 v3, v7
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: extractelement_vgpr_v4i128_idx1:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 16, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dwordx4 v[4:7], v[0:1]
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, v4
-; GFX8-NEXT:    v_mov_b32_e32 v1, v5
-; GFX8-NEXT:    v_mov_b32_e32 v2, v6
-; GFX8-NEXT:    v_mov_b32_e32 v3, v7
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: extractelement_vgpr_v4i128_idx1:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 s6, 0
 ; GFX7-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[4:7], 0 addr64 offset:16
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64 offset:16
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, v4
-; GFX7-NEXT:    v_mov_b32_e32 v1, v5
-; GFX7-NEXT:    v_mov_b32_e32 v2, v6
-; GFX7-NEXT:    v_mov_b32_e32 v3, v7
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: extractelement_vgpr_v4i128_idx1:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    global_load_dwordx4 v[4:7], v[0:1], off offset:16
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off offset:16
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, v4
-; GFX10-NEXT:    v_mov_b32_e32 v1, v5
-; GFX10-NEXT:    v_mov_b32_e32 v2, v6
-; GFX10-NEXT:    v_mov_b32_e32 v3, v7
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: extractelement_vgpr_v4i128_idx1:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    global_load_b128 v[4:7], v[0:1], off offset:16
+; GFX11-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:16
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-NEXT:    v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %vector = load <4 x i128>, <4 x i128> addrspace(1)* %ptr
   %element = extractelement <4 x i128> %vector, i32 1
   ret i128 %element
 }
 
 define i128 @extractelement_vgpr_v4i128_idx2(<4 x i128> addrspace(1)* %ptr) {
 ; GFX9-LABEL: extractelement_vgpr_v4i128_idx2:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx4 v[8:11], v[0:1], off offset:32
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off offset:32
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, v8
-; GFX9-NEXT:    v_mov_b32_e32 v1, v9
-; GFX9-NEXT:    v_mov_b32_e32 v2, v10
-; GFX9-NEXT:    v_mov_b32_e32 v3, v11
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: extractelement_vgpr_v4i128_idx2:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 32, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dwordx4 v[8:11], v[0:1]
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, v8
-; GFX8-NEXT:    v_mov_b32_e32 v1, v9
-; GFX8-NEXT:    v_mov_b32_e32 v2, v10
-; GFX8-NEXT:    v_mov_b32_e32 v3, v11
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: extractelement_vgpr_v4i128_idx2:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 s6, 0
 ; GFX7-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[4:7], 0 addr64 offset:32
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64 offset:32
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, v8
-; GFX7-NEXT:    v_mov_b32_e32 v1, v9
-; GFX7-NEXT:    v_mov_b32_e32 v2, v10
-; GFX7-NEXT:    v_mov_b32_e32 v3, v11
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: extractelement_vgpr_v4i128_idx2:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    global_load_dwordx4 v[8:11], v[0:1], off offset:32
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off offset:32
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, v8
-; GFX10-NEXT:    v_mov_b32_e32 v1, v9
-; GFX10-NEXT:    v_mov_b32_e32 v2, v10
-; GFX10-NEXT:    v_mov_b32_e32 v3, v11
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: extractelement_vgpr_v4i128_idx2:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    global_load_b128 v[8:11], v[0:1], off offset:32
+; GFX11-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:32
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
-; GFX11-NEXT:    v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %vector = load <4 x i128>, <4 x i128> addrspace(1)* %ptr
   %element = extractelement <4 x i128> %vector, i32 2
   ret i128 %element
 }
 
 define i128 @extractelement_vgpr_v4i128_idx3(<4 x i128> addrspace(1)* %ptr) {
 ; GFX9-LABEL: extractelement_vgpr_v4i128_idx3:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    global_load_dwordx4 v[12:15], v[0:1], off offset:48
+; GFX9-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off offset:48
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_mov_b32_e32 v0, v12
-; GFX9-NEXT:    v_mov_b32_e32 v1, v13
-; GFX9-NEXT:    v_mov_b32_e32 v2, v14
-; GFX9-NEXT:    v_mov_b32_e32 v3, v15
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX8-LABEL: extractelement_vgpr_v4i128_idx3:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 48, v0
 ; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT:    flat_load_dwordx4 v[12:15], v[0:1]
+; GFX8-NEXT:    flat_load_dwordx4 v[0:3], v[0:1]
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_mov_b32_e32 v0, v12
-; GFX8-NEXT:    v_mov_b32_e32 v1, v13
-; GFX8-NEXT:    v_mov_b32_e32 v2, v14
-; GFX8-NEXT:    v_mov_b32_e32 v3, v15
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX7-LABEL: extractelement_vgpr_v4i128_idx3:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX7-NEXT:    s_mov_b32 s6, 0
 ; GFX7-NEXT:    s_mov_b32 s7, 0xf000
 ; GFX7-NEXT:    s_mov_b64 s[4:5], 0
-; GFX7-NEXT:    buffer_load_dwordx4 v[12:15], v[0:1], s[4:7], 0 addr64 offset:48
+; GFX7-NEXT:    buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64 offset:48
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
-; GFX7-NEXT:    v_mov_b32_e32 v0, v12
-; GFX7-NEXT:    v_mov_b32_e32 v1, v13
-; GFX7-NEXT:    v_mov_b32_e32 v2, v14
-; GFX7-NEXT:    v_mov_b32_e32 v3, v15
 ; GFX7-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: extractelement_vgpr_v4i128_idx3:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    global_load_dwordx4 v[12:15], v[0:1], off offset:48
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v[0:1], off offset:48
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v0, v12
-; GFX10-NEXT:    v_mov_b32_e32 v1, v13
-; GFX10-NEXT:    v_mov_b32_e32 v2, v14
-; GFX10-NEXT:    v_mov_b32_e32 v3, v15
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: extractelement_vgpr_v4i128_idx3:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    global_load_b128 v[12:15], v[0:1], off offset:48
+; GFX11-NEXT:    global_load_b128 v[0:3], v[0:1], off offset:48
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_dual_mov_b32 v0, v12 :: v_dual_mov_b32 v1, v13
-; GFX11-NEXT:    v_dual_mov_b32 v2, v14 :: v_dual_mov_b32 v3, v15
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %vector = load <4 x i128>, <4 x i128> addrspace(1)* %ptr
   %element = extractelement <4 x i128> %vector, i32 3
   ret i128 %element
 }

llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll

 ; GPRIDX-NEXT:    s_mov_b32 s15, 0x40180000
 ; GPRIDX-NEXT:    s_mov_b32 s14, s18
 ; GPRIDX-NEXT:    s_mov_b32 s13, 0x40140000
 ; GPRIDX-NEXT:    s_mov_b32 s12, s18
 ; GPRIDX-NEXT:    s_mov_b64 s[10:11], 4.0
 ; GPRIDX-NEXT:    s_mov_b32 s9, 0x40080000
 ; GPRIDX-NEXT:    s_mov_b32 s8, s18
 ; GPRIDX-NEXT:    s_mov_b64 s[6:7], 2.0
-; GPRIDX-NEXT:    v_mov_b32_e32 v4, s4
-; GPRIDX-NEXT:    v_mov_b32_e32 v5, s5
-; GPRIDX-NEXT:    v_mov_b32_e32 v6, s6
-; GPRIDX-NEXT:    v_mov_b32_e32 v7, s7
-; GPRIDX-NEXT:    v_mov_b32_e32 v8, s8
-; GPRIDX-NEXT:    v_mov_b32_e32 v9, s9
-; GPRIDX-NEXT:    v_mov_b32_e32 v10, s10
-; GPRIDX-NEXT:    v_mov_b32_e32 v11, s11
-; GPRIDX-NEXT:    v_mov_b32_e32 v12, s12
-; GPRIDX-NEXT:    v_mov_b32_e32 v13, s13
-; GPRIDX-NEXT:    v_mov_b32_e32 v14, s14
-; GPRIDX-NEXT:    v_mov_b32_e32 v15, s15
-; GPRIDX-NEXT:    v_mov_b32_e32 v16, s16
-; GPRIDX-NEXT:    v_mov_b32_e32 v17, s17
-; GPRIDX-NEXT:    v_mov_b32_e32 v18, s18
-; GPRIDX-NEXT:    v_mov_b32_e32 v19, s19
+; GPRIDX-NEXT:    v_mov_b32_e32 v3, s4
+; GPRIDX-NEXT:    v_mov_b32_e32 v4, s5
+; GPRIDX-NEXT:    v_mov_b32_e32 v5, s6
+; GPRIDX-NEXT:    v_mov_b32_e32 v6, s7
+; GPRIDX-NEXT:    v_mov_b32_e32 v7, s8
+; GPRIDX-NEXT:    v_mov_b32_e32 v8, s9
+; GPRIDX-NEXT:    v_mov_b32_e32 v9, s10
+; GPRIDX-NEXT:    v_mov_b32_e32 v10, s11
+; GPRIDX-NEXT:    v_mov_b32_e32 v11, s12
+; GPRIDX-NEXT:    v_mov_b32_e32 v12, s13
+; GPRIDX-NEXT:    v_mov_b32_e32 v13, s14
+; GPRIDX-NEXT:    v_mov_b32_e32 v14, s15
+; GPRIDX-NEXT:    v_mov_b32_e32 v15, s16
+; GPRIDX-NEXT:    v_mov_b32_e32 v16, s17
+; GPRIDX-NEXT:    v_mov_b32_e32 v17, s18
+; GPRIDX-NEXT:    v_mov_b32_e32 v18, s19
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[16:17], 0, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 2, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 3, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 4, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 5, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[12:13], 6, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[14:15], 7, v2
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v4, v0, s[16:17]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v6, v0, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v5, v1, s[16:17]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v7, v1, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v8, v0, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v10, v0, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v12, v0, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v14, v0, s[10:11]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v16, v0, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v16, v18, v0, s[14:15]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v9, v1, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v11, v1, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v13, v1, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v15, v1, s[10:11]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v17, v1, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v17, v19, v1, s[14:15]
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v3, v0, s[16:17]
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v4, v1, s[16:17]
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v6, v1, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v0, s[4:5]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s[10:11]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v15, v0, s[12:13]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s[14:15]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s[4:5]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s[10:11]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v16, v16, v1, s[12:13]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v18, v18, v1, s[14:15]
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[10:13], off
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[11:14], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[14:17], off
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[15:18], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
 ; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: dyn_insertelement_v8f64_const_s_v_v:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    s_mov_b32 s18, 0

 ; GPRIDX-NEXT:    s_mov_b32 s0, s2
 ; GPRIDX-NEXT:    s_mov_b32 s2, s4
 ; GPRIDX-NEXT:    s_mov_b32 s4, s6
 ; GPRIDX-NEXT:    s_mov_b32 s6, s8
 ; GPRIDX-NEXT:    s_mov_b32 s8, s10
 ; GPRIDX-NEXT:    s_mov_b32 s10, s12
 ; GPRIDX-NEXT:    s_mov_b32 s12, s14
 ; GPRIDX-NEXT:    s_mov_b32 s14, s16
-; GPRIDX-NEXT:    v_mov_b32_e32 v17, s15
-; GPRIDX-NEXT:    v_mov_b32_e32 v16, s14
-; GPRIDX-NEXT:    v_mov_b32_e32 v15, s13
-; GPRIDX-NEXT:    v_mov_b32_e32 v14, s12
-; GPRIDX-NEXT:    v_mov_b32_e32 v13, s11
-; GPRIDX-NEXT:    v_mov_b32_e32 v12, s10
-; GPRIDX-NEXT:    v_mov_b32_e32 v11, s9
-; GPRIDX-NEXT:    v_mov_b32_e32 v10, s8
-; GPRIDX-NEXT:    v_mov_b32_e32 v9, s7
-; GPRIDX-NEXT:    v_mov_b32_e32 v8, s6
-; GPRIDX-NEXT:    v_mov_b32_e32 v7, s5
-; GPRIDX-NEXT:    v_mov_b32_e32 v6, s4
-; GPRIDX-NEXT:    v_mov_b32_e32 v5, s3
-; GPRIDX-NEXT:    v_mov_b32_e32 v4, s2
-; GPRIDX-NEXT:    v_mov_b32_e32 v3, s1
-; GPRIDX-NEXT:    v_mov_b32_e32 v2, s0
-; GPRIDX-NEXT:    v_mov_b32_e32 v1, s18
+; GPRIDX-NEXT:    v_mov_b32_e32 v16, s15
+; GPRIDX-NEXT:    v_mov_b32_e32 v15, s14
+; GPRIDX-NEXT:    v_mov_b32_e32 v14, s13
+; GPRIDX-NEXT:    v_mov_b32_e32 v13, s12
+; GPRIDX-NEXT:    v_mov_b32_e32 v12, s11
+; GPRIDX-NEXT:    v_mov_b32_e32 v11, s10
+; GPRIDX-NEXT:    v_mov_b32_e32 v10, s9
+; GPRIDX-NEXT:    v_mov_b32_e32 v9, s8
+; GPRIDX-NEXT:    v_mov_b32_e32 v8, s7
+; GPRIDX-NEXT:    v_mov_b32_e32 v7, s6
+; GPRIDX-NEXT:    v_mov_b32_e32 v6, s5
+; GPRIDX-NEXT:    v_mov_b32_e32 v5, s4
+; GPRIDX-NEXT:    v_mov_b32_e32 v4, s3
+; GPRIDX-NEXT:    v_mov_b32_e32 v3, s2
+; GPRIDX-NEXT:    v_mov_b32_e32 v2, s1
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v17, s18
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], 2, v0
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[2:3], 3, v0
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 4, v0
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 5, v0
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 6, v0
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v0
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v0
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v0, v2, v1, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v4, v1, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v6, v1, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v8, v1, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v10, v1, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v12, v1, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v14, v1, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v16, v1, s[10:11]
-; GPRIDX-NEXT:    v_mov_b32_e32 v16, s19
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v1, v3, v16, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v5, v16, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v7, v16, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v9, v16, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v11, v16, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v13, v16, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v15, v16, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v17, v16, s[10:11]
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s19
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v1, v1, v17, s[12:13]
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v17, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v2, v0, s[12:13]
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v5, v17, s[0:1]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s[2:3]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s[4:5]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v13, v17, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s[10:11]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v0, s[0:1]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v0, s[2:3]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v0, s[4:5]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v0, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v14, v0, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v16, v16, v0, s[10:11]
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[1:4], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[5:8], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[8:11], off
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[9:12], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[12:15], off
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[13:16], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
 ; GPRIDX-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: dyn_insertelement_v8f64_s_s_v:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7

 ; GPRIDX-NEXT:    s_mov_b32 s0, s2
 ; GPRIDX-NEXT:    s_mov_b32 s2, s4
 ; GPRIDX-NEXT:    s_mov_b32 s4, s6
 ; GPRIDX-NEXT:    s_mov_b32 s6, s8
 ; GPRIDX-NEXT:    s_mov_b32 s8, s10
 ; GPRIDX-NEXT:    s_mov_b32 s10, s12
 ; GPRIDX-NEXT:    s_mov_b32 s12, s14
 ; GPRIDX-NEXT:    s_mov_b32 s14, s16
-; GPRIDX-NEXT:    v_mov_b32_e32 v19, s15
-; GPRIDX-NEXT:    v_mov_b32_e32 v18, s14
-; GPRIDX-NEXT:    v_mov_b32_e32 v17, s13
-; GPRIDX-NEXT:    v_mov_b32_e32 v16, s12
-; GPRIDX-NEXT:    v_mov_b32_e32 v15, s11
-; GPRIDX-NEXT:    v_mov_b32_e32 v14, s10
-; GPRIDX-NEXT:    v_mov_b32_e32 v13, s9
-; GPRIDX-NEXT:    v_mov_b32_e32 v12, s8
-; GPRIDX-NEXT:    v_mov_b32_e32 v11, s7
-; GPRIDX-NEXT:    v_mov_b32_e32 v10, s6
-; GPRIDX-NEXT:    v_mov_b32_e32 v9, s5
-; GPRIDX-NEXT:    v_mov_b32_e32 v8, s4
-; GPRIDX-NEXT:    v_mov_b32_e32 v7, s3
-; GPRIDX-NEXT:    v_mov_b32_e32 v6, s2
-; GPRIDX-NEXT:    v_mov_b32_e32 v5, s1
-; GPRIDX-NEXT:    v_mov_b32_e32 v4, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v18, s15
+; GPRIDX-NEXT:    v_mov_b32_e32 v17, s14
+; GPRIDX-NEXT:    v_mov_b32_e32 v16, s13
+; GPRIDX-NEXT:    v_mov_b32_e32 v15, s12
+; GPRIDX-NEXT:    v_mov_b32_e32 v14, s11
+; GPRIDX-NEXT:    v_mov_b32_e32 v13, s10
+; GPRIDX-NEXT:    v_mov_b32_e32 v12, s9
+; GPRIDX-NEXT:    v_mov_b32_e32 v11, s8
+; GPRIDX-NEXT:    v_mov_b32_e32 v10, s7
+; GPRIDX-NEXT:    v_mov_b32_e32 v9, s6
+; GPRIDX-NEXT:    v_mov_b32_e32 v8, s5
+; GPRIDX-NEXT:    v_mov_b32_e32 v7, s4
+; GPRIDX-NEXT:    v_mov_b32_e32 v6, s3
+; GPRIDX-NEXT:    v_mov_b32_e32 v5, s2
+; GPRIDX-NEXT:    v_mov_b32_e32 v4, s1
+; GPRIDX-NEXT:    v_mov_b32_e32 v3, s0
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], 2, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[2:3], 3, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 4, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 5, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 6, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v2
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v4, v0, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v6, v0, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v5, v1, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v7, v1, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v8, v0, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v10, v0, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v12, v0, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v14, v0, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v16, v0, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v16, v18, v0, s[10:11]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v9, v1, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v11, v1, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v13, v1, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v15, v1, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v17, v1, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v17, v19, v1, s[10:11]
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[2:5], off
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v3, v0, s[12:13]
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v4, v1, s[12:13]
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v6, v1, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v0, s[0:1]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s[2:3]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s[4:5]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v15, v0, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s[10:11]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s[0:1]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s[2:3]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s[4:5]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v16, v16, v1, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v18, v18, v1, s[10:11]
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[6:9], off
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[10:13], off
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[11:14], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
-; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[14:17], off
+; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[15:18], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
 ; GPRIDX-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: dyn_insertelement_v8f64_s_v_v:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_mov_b32 s1, s3
 ; GFX10-NEXT:    s_mov_b32 s3, s5
 ; GFX10-NEXT:    s_mov_b32 s5, s7