[AMDGPU][GISel] Smaller code for scalar 32 to 64-bit extensions

[arsenm]

I thought these got up split before selection

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

2100

s/0/AMDGPU::NoSubRegister

2101

Braces

2105

Braces

[foad]

Add braces, use NoSubRegister.

[foad]

In D107639#2930963, @arsenm wrote:

I thought these got up split before selection

There is code in AMDGPURegisterBankInfo::applyMappingImpl to do that for VALU, but not for SALU. Would it be better to do it there for SALU too?

[arsenm]

In D107639#2931000, @foad wrote:

In D107639#2930963, @arsenm wrote:

I thought these got up split before selection

There is code in AMDGPURegisterBankInfo::applyMappingImpl to do that for VALU, but not for SALU. Would it be better to do it there for SALU too?

Maybe this should just be a post-regbankselect combine. We don't want to have any other combines that need to create extensions to worry about this too

[arsenm]

reverse ping

[foad]

Do you still think this should be done as a post-regbankselect combine instead? I really have no idea.

[foad]

Rebase.

[arsenm]

In D107639#3924314, @foad wrote:

Do you still think this should be done as a post-regbankselect combine instead? I really have no idea.

I do think that could potentially enable other combines, but I'm not sure it needs to be exclusionary to also doing it in selection

[foad]

Then I'm just waiting for review. Ping.

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

if (Signed && DstSize == 32 && (SrcSize == 8 || SrcSize == 16)) {
       const unsigned SextOpc = SrcSize == 8 ?
         AMDGPU::S_SEXT_I32_I8 : AMDGPU::S_SEXT_I32_I16;
       BuildMI(MBB, I, DL, TII.get(SextOpc), DstReg)
         .addReg(SrcReg);
       I.eraseFromParent();
       return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
     }
 
+    // Using a single 32-bit SALU to calculate the high half is smaller than
+    // S_BFE with a literal constant operand.
+    if (DstSize > 32 && SrcSize == 32) {
+      Register HiReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
+      unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
+      if (Signed) {
+        BuildMI(MBB, I, DL, TII.get(AMDGPU::S_ASHR_I32), HiReg)
+          .addReg(SrcReg, 0, SubReg)
+          .addImm(31);
+      } else {
+        BuildMI(MBB, I, DL, TII.get(AMDGPU::S_MOV_B32), HiReg)
+          .addImm(0);
+      }
+      BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), DstReg)
+        .addReg(SrcReg, 0, SubReg)
+        .addImm(AMDGPU::sub0)
+        .addReg(HiReg)
+        .addImm(AMDGPU::sub1);
+      I.eraseFromParent();
+      return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_64RegClass,
+                                          *MRI);
+    }
+
     const unsigned BFE64 = Signed ? AMDGPU::S_BFE_I64 : AMDGPU::S_BFE_U64;
     const unsigned BFE32 = Signed ? AMDGPU::S_BFE_I32 : AMDGPU::S_BFE_U32;
 
     // Scalar BFE is encoded as S1[5:0] = offset, S1[22:16]= width.
     if (DstSize > 32 && (SrcSize <= 32 || InReg)) {
       // We need a 64-bit register source, but the high bits don't matter.
       Register ExtReg = MRI->createVirtualRegister(&AMDGPU::SReg_64RegClass);
       Register UndefReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
-      unsigned SubReg = InReg ? AMDGPU::sub0 : 0;
+      unsigned SubReg = InReg ? AMDGPU::sub0 : AMDGPU::NoSubRegister;
 
       BuildMI(MBB, I, DL, TII.get(AMDGPU::IMPLICIT_DEF), UndefReg);
       BuildMI(MBB, I, DL, TII.get(AMDGPU::REG_SEQUENCE), ExtReg)
         .addReg(SrcReg, 0, SubReg)
         .addImm(AMDGPU::sub0)
         .addReg(UndefReg)
         .addImm(AMDGPU::sub1);
 

llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-sext-inreg.mir

 legalized:       true
 regBankSelected: true
 body: |
   bb.0:
     liveins: $sgpr0_sgpr1
 
     ; GCN-LABEL: name: sext_inreg_sgpr_s64_32
     ; GCN: [[COPY:%[0-9]+]]:sreg_64 = COPY $sgpr0_sgpr1
-    ; GCN: [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[DEF]], %subreg.sub1
-    ; GCN: [[S_BFE_I64_:%[0-9]+]]:sreg_64 = S_BFE_I64 [[REG_SEQUENCE]], 2097152, implicit-def $scc
-    ; GCN: $sgpr0_sgpr1 = COPY [[S_BFE_I64_]]
+    ; GCN: [[S_ASHR_I32_:%[0-9]+]]:sreg_32 = S_ASHR_I32 [[COPY]].sub0, 31, implicit-def $scc
+    ; GCN: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY]].sub0, %subreg.sub0, [[S_ASHR_I32_]], %subreg.sub1
+    ; GCN: $sgpr0_sgpr1 = COPY [[REG_SEQUENCE]]
     %0:sgpr(s64) = COPY $sgpr0_sgpr1
     %1:sgpr(s64) = G_SEXT_INREG %0, 32
     $sgpr0_sgpr1 = COPY %1
 ...
 
 ---
 
 name: sext_inreg_sgpr_s64_63

llvm/test/CodeGen/AMDGPU/GlobalISel/mubuf-global.ll

; GFX7-NEXT:    s_endpgm
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i64 4096
   store i32 0, i32 addrspace(1)* %gep
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_sgpr_ptr_sgpr_offset(i32 addrspace(1)* inreg %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_store_sgpr_ptr_sgpr_offset:
 ; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_ashr_i32 s5, s4, 31
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX6-NEXT:    s_mov_b32 s0, s2
 ; GFX6-NEXT:    s_mov_b32 s1, s3
-; GFX6-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
-; GFX6-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
-; GFX6-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_sgpr_ptr_sgpr_offset:
 ; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_ashr_i32 s5, s4, 31
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-NEXT:    s_mov_b32 s0, s2
 ; GFX7-NEXT:    s_mov_b32 s1, s3
-; GFX7-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_endpgm
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %soffset
   store i32 0, i32 addrspace(1)* %gep
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset(i32 addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX6-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX7-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_endpgm
   %gep = getelementptr i32, i32 addrspace(1)* %ptr, i32 %soffset
   store i32 0, i32 addrspace(1)* %gep
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset_offset256(i32 addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset_offset256:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX6-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:1024
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset_offset256:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX7-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 offset:1024
 ; GFX7-NEXT:    s_endpgm
   %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 %soffset
   %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 256
   store i32 0, i32 addrspace(1)* %gep1
   ret void
 }
 
 define amdgpu_ps void @mubuf_store_vgpr_ptr_sgpr_offset256_offset(i32 addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_store_vgpr_ptr_sgpr_offset256_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_mov_b64 s[4:5], 0x400
 ; GFX6-NEXT:    v_mov_b32_e32 v2, s4
-; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GFX6-NEXT:    v_mov_b32_e32 v3, s5
+; GFX6-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX6-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, s5
 ; GFX6-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: mubuf_store_vgpr_ptr_sgpr_offset256_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_mov_b64 s[4:5], 0x400
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
+; GFX7-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX7-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, s5
 ; GFX7-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
 ; GFX7-NEXT:    s_endpgm
   %gep0 = getelementptr i32, i32 addrspace(1)* %ptr, i32 256
   %gep1 = getelementptr i32, i32 addrspace(1)* %gep0, i32 %soffset
   store i32 0, i32 addrspace(1)* %gep1
; GFX7-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, float addrspace(1)* %ptr, i64 4096
   %val = load volatile float, float addrspace(1)* %gep
   ret float %val
 }
 
 define amdgpu_ps float @mubuf_load_sgpr_ptr_sgpr_offset(float addrspace(1)* inreg %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_load_sgpr_ptr_sgpr_offset:
 ; GFX6:       ; %bb.0:
+; GFX6-NEXT:    s_ashr_i32 s5, s4, 31
+; GFX6-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
+; GFX6-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX6-NEXT:    s_mov_b32 s0, s2
 ; GFX6-NEXT:    s_mov_b32 s1, s3
-; GFX6-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
-; GFX6-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
-; GFX6-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_sgpr_ptr_sgpr_offset:
 ; GFX7:       ; %bb.0:
+; GFX7-NEXT:    s_ashr_i32 s5, s4, 31
+; GFX7-NEXT:    s_lshl_b64 s[4:5], s[4:5], 2
+; GFX7-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-NEXT:    s_mov_b32 s0, s2
 ; GFX7-NEXT:    s_mov_b32 s1, s3
-; GFX7-NEXT:    s_bfe_i64 s[2:3], s[4:5], 0x200000
-; GFX7-NEXT:    s_lshl_b64 s[4:5], s[2:3], 2
-; GFX7-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, float addrspace(1)* %ptr, i32 %soffset
   %val = load volatile float, float addrspace(1)* %gep
   ret float %val
 }
 
 define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset(float addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX6-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX7-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep = getelementptr float, float addrspace(1)* %ptr, i32 %soffset
   %val = load volatile float, float addrspace(1)* %gep
   ret float %val
 }
 
 define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset_offset256(float addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset_offset256:
 ; GFX6:       ; %bb.0:
-; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX6-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX6-NEXT:    s_mov_b32 s2, 0
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024 glc
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset_offset256:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
-; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
+; GFX7-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX7-NEXT:    s_mov_b32 s2, 0
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 offset:1024 glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, float addrspace(1)* %ptr, i32 %soffset
   %gep1 = getelementptr float, float addrspace(1)* %gep0, i32 256
   %val = load volatile float, float addrspace(1)* %gep1
   ret float %val
 }
 
 define amdgpu_ps float @mubuf_load_vgpr_ptr_sgpr_offset256_offset(float addrspace(1)* %ptr, i32 inreg %soffset) {
 ; GFX6-LABEL: mubuf_load_vgpr_ptr_sgpr_offset256_offset:
 ; GFX6:       ; %bb.0:
 ; GFX6-NEXT:    s_mov_b64 s[4:5], 0x400
 ; GFX6-NEXT:    v_mov_b32_e32 v2, s4
-; GFX6-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
+; GFX6-NEXT:    s_ashr_i32 s3, s2, 31
 ; GFX6-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX6-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX6-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX6-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NEXT:    s_mov_b32 s2, s5
 ; GFX6-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
 ; GFX6-NEXT:    ; return to shader part epilog
 ;
 ; GFX7-LABEL: mubuf_load_vgpr_ptr_sgpr_offset256_offset:
 ; GFX7:       ; %bb.0:
 ; GFX7-NEXT:    s_mov_b64 s[4:5], 0x400
 ; GFX7-NEXT:    v_mov_b32_e32 v2, s4
-; GFX7-NEXT:    s_bfe_i64 s[0:1], s[2:3], 0x200000
+; GFX7-NEXT:    s_ashr_i32 s3, s2, 31
 ; GFX7-NEXT:    v_mov_b32_e32 v3, s5
 ; GFX7-NEXT:    v_add_i32_e32 v0, vcc, v0, v2
+; GFX7-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GFX7-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX7-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX7-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX7-NEXT:    s_mov_b32 s2, s5
 ; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[0:3], 0 addr64 glc
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
 ; GFX7-NEXT:    ; return to shader part epilog
   %gep0 = getelementptr float, float addrspace(1)* %ptr, i64 256
   %gep1 = getelementptr float, float addrspace(1)* %gep0, i32 %soffset
   %val = load volatile float, float addrspace(1)* %gep1

llvm/test/CodeGen/AMDGPU/GlobalISel/shl-ext-reduce.ll

; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shl = shl i64 %ext, 2
   ret i64 %shl
 }
 
 define amdgpu_ps i64 @s_shl_i64_zext_i32_overflow(i32 inreg %x) {
 ; GCN-LABEL: s_shl_i64_zext_i32_overflow:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_bitset0_b32 s0, 31
-; GCN-NEXT:    s_bfe_u64 s[0:1], s[0:1], 0x200000
+; GCN-NEXT:    s_mov_b32 s1, 0
 ; GCN-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GCN-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_shl_i64_zext_i32_overflow:
 ; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_mov_b32 s1, 0
 ; GFX10-NEXT:    s_bitset0_b32 s0, 31
-; GFX10-NEXT:    s_bfe_u64 s[0:1], s[0:1], 0x200000
 ; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX10-NEXT:    ; return to shader part epilog
   %and = and i32 %x, 2147483647
   %ext = zext i32 %and to i64
   %shl = shl i64 %ext, 2
   ret i64 %shl
 }
 
; GFX10-NEXT:    s_setpc_b64 s[30:31]
   %shl = shl i64 %ext, 2
   ret i64 %shl
 }
 
 define amdgpu_ps i64 @s_shl_i64_sext_i32_overflow(i32 inreg %x) {
 ; GCN-LABEL: s_shl_i64_sext_i32_overflow:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_bitset0_b32 s0, 31
-; GCN-NEXT:    s_bfe_i64 s[0:1], s[0:1], 0x200000
+; GCN-NEXT:    s_ashr_i32 s1, s0, 31
 ; GCN-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GCN-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_shl_i64_sext_i32_overflow:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_bitset0_b32 s0, 31
-; GFX10-NEXT:    s_bfe_i64 s[0:1], s[0:1], 0x200000
+; GFX10-NEXT:    s_ashr_i32 s1, s0, 31
 ; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
 ; GFX10-NEXT:    ; return to shader part epilog
   %and = and i32 %x, 2147483647
   %ext = sext i32 %and to i64
   %shl = shl i64 %ext, 2
   ret i64 %shl
 }
 

 }
 
 define amdgpu_ps <2 x i64> @s_shl_v2i64_zext_v2i32(<2 x i32> inreg %x) {
 ; GCN-LABEL: s_shl_v2i64_zext_v2i32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_brev_b32 s2, -4
 ; GCN-NEXT:    s_mov_b32 s3, s2
 ; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], s[2:3]
-; GCN-NEXT:    s_bfe_u64 s[2:3], s[0:1], 0x200000
-; GCN-NEXT:    s_mov_b32 s0, s1
-; GCN-NEXT:    s_bfe_u64 s[4:5], s[0:1], 0x200000
+; GCN-NEXT:    s_mov_b32 s3, 0
+; GCN-NEXT:    s_mov_b32 s2, s0
+; GCN-NEXT:    s_mov_b32 s4, s1
+; GCN-NEXT:    s_mov_b32 s5, s3
 ; GCN-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GCN-NEXT:    s_lshl_b64 s[2:3], s[4:5], 2
 ; GCN-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_shl_v2i64_zext_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_brev_b32 s2, -4
 ; GFX10-NEXT:    s_mov_b32 s3, s2
 ; GFX10-NEXT:    s_and_b64 s[0:1], s[0:1], s[2:3]
-; GFX10-NEXT:    s_mov_b32 s2, s1
-; GFX10-NEXT:    s_bfe_u64 s[0:1], s[0:1], 0x200000
-; GFX10-NEXT:    s_bfe_u64 s[2:3], s[2:3], 0x200000
-; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
-; GFX10-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX10-NEXT:    s_mov_b32 s3, 0
+; GFX10-NEXT:    s_mov_b32 s2, s0
+; GFX10-NEXT:    s_mov_b32 s4, s1
+; GFX10-NEXT:    s_mov_b32 s5, s3
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[4:5], 2
 ; GFX10-NEXT:    ; return to shader part epilog
   %and = and <2 x i32> %x, <i32 1073741823, i32 1073741823>
   %ext = zext <2 x i32> %and to <2 x i64>
   %shl = shl <2 x i64> %ext, <i64 2, i64 2>
   ret <2 x i64> %shl
 }
 
 define <2 x i64> @v_shl_v2i64_zext_v2i32(<2 x i32> %x) {

 }
 
 define amdgpu_ps <2 x i64> @s_shl_v2i64_sext_v2i32(<2 x i32> inreg %x) {
 ; GCN-LABEL: s_shl_v2i64_sext_v2i32:
 ; GCN:       ; %bb.0:
 ; GCN-NEXT:    s_brev_b32 s2, -8
 ; GCN-NEXT:    s_mov_b32 s3, s2
 ; GCN-NEXT:    s_and_b64 s[0:1], s[0:1], s[2:3]
-; GCN-NEXT:    s_bfe_i64 s[2:3], s[0:1], 0x200000
-; GCN-NEXT:    s_mov_b32 s0, s1
-; GCN-NEXT:    s_bfe_i64 s[4:5], s[0:1], 0x200000
+; GCN-NEXT:    s_ashr_i32 s3, s0, 31
+; GCN-NEXT:    s_mov_b32 s2, s0
+; GCN-NEXT:    s_ashr_i32 s5, s1, 31
+; GCN-NEXT:    s_mov_b32 s4, s1
 ; GCN-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
 ; GCN-NEXT:    s_lshl_b64 s[2:3], s[4:5], 2
 ; GCN-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: s_shl_v2i64_sext_v2i32:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_brev_b32 s2, -8
 ; GFX10-NEXT:    s_mov_b32 s3, s2
 ; GFX10-NEXT:    s_and_b64 s[0:1], s[0:1], s[2:3]
-; GFX10-NEXT:    s_mov_b32 s2, s1
-; GFX10-NEXT:    s_bfe_i64 s[0:1], s[0:1], 0x200000
-; GFX10-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x200000
-; GFX10-NEXT:    s_lshl_b64 s[0:1], s[0:1], 2
-; GFX10-NEXT:    s_lshl_b64 s[2:3], s[2:3], 2
+; GFX10-NEXT:    s_ashr_i32 s3, s0, 31
+; GFX10-NEXT:    s_mov_b32 s2, s0
+; GFX10-NEXT:    s_ashr_i32 s5, s1, 31
+; GFX10-NEXT:    s_mov_b32 s4, s1
+; GFX10-NEXT:    s_lshl_b64 s[0:1], s[2:3], 2
+; GFX10-NEXT:    s_lshl_b64 s[2:3], s[4:5], 2
 ; GFX10-NEXT:    ; return to shader part epilog
   %and = and <2 x i32> %x, <i32 536870911, i32 536870911>
   %ext = sext <2 x i32> %and to <2 x i64>
   %shl = shl <2 x i64> %ext, <i64 2, i64 2>
   ret <2 x i64> %shl
 }
 
 define <2 x i64> @v_shl_v2i64_sext_v2i32(<2 x i32> %x) {

llvm/test/CodeGen/AMDGPU/ctlz.ll

 ; GFX10-NEXT:    global_store_dwordx2 v1, v[0:1], s[4:5]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: s_ctlz_i64:
 ; GFX10-GISEL:       ; %bb.0:
 ; GFX10-GISEL-NEXT:    s_clause 0x1
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX10-GISEL-NEXT:    s_mov_b32 s1, 0
 ; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-GISEL-NEXT:    s_flbit_i32_b64 s0, s[2:3]
 ; GFX10-GISEL-NEXT:    s_min_u32 s0, s0, 64
-; GFX10-GISEL-NEXT:    s_bfe_u64 s[0:1], s[0:1], 0x200000
 ; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX10-GISEL-NEXT:    s_endpgm
   %ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 false)
   store i64 %ctlz, i64 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll

 ; EG-NEXT:     MOV T0.Y, 0.0,
 ; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i64:
 ; GFX9-GISEL:       ; %bb.0:
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x4c
+; GFX9-GISEL-NEXT:    s_mov_b32 s1, 0
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-GISEL-NEXT:    s_flbit_i32_b64 s0, s[4:5]
-; GFX9-GISEL-NEXT:    s_bfe_u64 s[0:1], s[0:1], 0x200000
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-GISEL-NEXT:    s_endpgm
   %ctlz = call i64 @llvm.ctlz.i64(i64 %val, i1 true)
   store i64 %ctlz, i64 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/cttz.ll

 ; GFX10-NEXT:    global_store_dwordx2 v1, v[0:1], s[4:5]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX10-GISEL-LABEL: s_cttz_i64:
 ; GFX10-GISEL:       ; %bb.0:
 ; GFX10-GISEL-NEXT:    s_clause 0x1
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x4c
 ; GFX10-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x24
+; GFX10-GISEL-NEXT:    s_mov_b32 s1, 0
 ; GFX10-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-GISEL-NEXT:    s_ff1_i32_b64 s0, s[2:3]
 ; GFX10-GISEL-NEXT:    s_min_u32 s0, s0, 64
-; GFX10-GISEL-NEXT:    s_bfe_u64 s[0:1], s[0:1], 0x200000
 ; GFX10-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX10-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX10-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[4:5]
 ; GFX10-GISEL-NEXT:    s_endpgm
   %cttz = call i64 @llvm.cttz.i64(i64 %val, i1 false)
   store i64 %cttz, i64 addrspace(1)* %out
   ret void
 }

llvm/test/CodeGen/AMDGPU/cttz_zero_undef.ll

 ; EG-NEXT:     MOV T0.Y, 0.0,
 ; EG-NEXT:     LSHR * T1.X, KC0[2].Y, literal.x,
 ; EG-NEXT:    2(2.802597e-45), 0(0.000000e+00)
 ;
 ; GFX9-GISEL-LABEL: s_cttz_zero_undef_i64_with_select:
 ; GFX9-GISEL:       ; %bb.0:
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x24
 ; GFX9-GISEL-NEXT:    s_load_dwordx2 s[4:5], s[0:1], 0x2c
+; GFX9-GISEL-NEXT:    s_mov_b32 s1, 0
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-GISEL-NEXT:    s_ff1_i32_b64 s0, s[4:5]
-; GFX9-GISEL-NEXT:    s_bfe_u64 s[0:1], s[0:1], 0x200000
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-GISEL-NEXT:    s_endpgm
   %cttz = tail call i64 @llvm.cttz.i64(i64 %val, i1 true) nounwind readnone
   %cttz_ret = icmp ne i64 %val, 0
   %ret = select i1 %cttz_ret, i64 %cttz, i64 32
   store i64 %cttz, i64 addrspace(1)* %out, align 4