AMDGPU/GlobalISel: Partially move constant selection to patterns

[arsenm]

Summary

This is still relying on the manual code for splitting 64-bit
constants, and handling pointers.

[foad]

Looks OK to me, but I don't know this stuff well enough to approve it.

llvm/lib/Target/AMDGPU/AMDGPUGISel.td

365–369

Can't you call renderBitcastFPImm directly and avoid the 32/64 wrapper functions?

llvm/lib/Target/AMDGPU/SIInstrInfo.td

864

Why is this needed? Perhaps add a comment? In other cases where you've done this it's because there's a PredicateCode that might return false, but I don't see that here.

[arsenm]

llvm/lib/Target/AMDGPU/AMDGPUGISel.td

365–369

No, the emitter doesn't handle reusing the same function

llvm/lib/Target/AMDGPU/SIInstrInfo.td

864

Because patterns with custom predicate code that don't have the gisel equivalent aren't imported. We need an IgnoreCodePredicate field or something to avoid this. I'm also pretty sure we don't need this specific predicate in SelectionDAG either and don't remember why we have it

[foad]

llvm/lib/Target/AMDGPU/AMDGPUGISel.td

365–369

D96587

[arsenm]

Rebase

llvm/lib/Target/AMDGPU/AMDGPUGISel.td

def gi_as_i8timm : GICustomOperandRenderer<"renderTruncTImm">,
   GISDNodeXFormEquiv<as_i8timm>;
 
 def gi_as_i1timm : GICustomOperandRenderer<"renderTruncTImm">,
   GISDNodeXFormEquiv<as_i1timm>;
 
 def gi_NegateImm : GICustomOperandRenderer<"renderNegateImm">,
   GISDNodeXFormEquiv<NegateImm>;
 
-def gi_bitcast_fpimm_to_i32 : GICustomOperandRenderer<"renderBitcastImm">,
+def gi_bitcast_fpimm_to_i32 : GICustomOperandRenderer<"renderBitcastFPImm">,
   GISDNodeXFormEquiv<bitcast_fpimm_to_i32>;
+def gi_bitcast_fpimm_to_i64 : GICustomOperandRenderer<"renderBitcastFPImm">,
+  GISDNodeXFormEquiv<bitcast_fpimm_to_i64>;
 
 def gi_IMMPopCount : GICustomOperandRenderer<"renderPopcntImm">,
   GISDNodeXFormEquiv<IMMPopCount>;
 
 def gi_extract_cpol : GICustomOperandRenderer<"renderExtractCPol">,
   GISDNodeXFormEquiv<extract_cpol>;
 
 def gi_extract_swz : GICustomOperandRenderer<"renderExtractSWZ">,
   GISDNodeXFormEquiv<extract_swz>;

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h

void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                        int OpIdx) const;
 
   void renderOpSelTImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                        int OpIdx) const;
 
   void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                        int OpIdx) const;
 
-  void renderBitcastImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
+  void renderBitcastFPImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                           int OpIdx) const;
 
   void renderPopcntImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
                        int OpIdx) const;
   void renderExtractCPol(MachineInstrBuilder &MIB, const MachineInstr &MI,
                          int OpIdx) const;
   void renderExtractSWZ(MachineInstrBuilder &MIB, const MachineInstr &MI,
                         int OpIdx) const;
   void renderSetGLC(MachineInstrBuilder &MIB, const MachineInstr &MI,

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

if (SrcBank->getID() == AMDGPU::SGPRRegBankID && DstSize <= 64) {
     I.eraseFromParent();
     return RBI.constrainGenericRegister(DstReg, AMDGPU::SReg_32RegClass, *MRI);
   }
 
   return false;
 }
 
 bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const {
+  if (selectImpl(I, *CoverageInfo))
+    return true;
+
+  // FIXME: Relying on manual selection for 64-bit case, and pointer typed
+  // constants.
   MachineBasicBlock *BB = I.getParent();
   MachineOperand &ImmOp = I.getOperand(1);
   Register DstReg = I.getOperand(0).getReg();
-  unsigned Size = MRI->getType(DstReg).getSizeInBits();
+  LLT Ty = MRI->getType(DstReg);
+  unsigned Size = Ty.getSizeInBits();
+  assert((Size == 64 || Ty.isPointer()) &&
+         "patterns should have selected this");
 
   // The AMDGPU backend only supports Imm operands and not CImm or FPImm.
   if (ImmOp.isFPImm()) {
     const APInt &Imm = ImmOp.getFPImm()->getValueAPF().bitcastToAPInt();
     ImmOp.ChangeToImmediate(Imm.getZExtValue());
   } else if (ImmOp.isCImm()) {
     ImmOp.ChangeToImmediate(ImmOp.getCImm()->getSExtValue());
   } else {
     llvm_unreachable("Not supported by g_constants");
   }
 
   const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI);
   const bool IsSgpr = DstRB->getID() == AMDGPU::SGPRRegBankID;
-
-  unsigned Opcode;
-  if (DstRB->getID() == AMDGPU::VCCRegBankID) {
-    Opcode = STI.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
-  } else {
-    Opcode = IsSgpr ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
-
-    // We should never produce s1 values on banks other than VCC. If the user of
-    // this already constrained the register, we may incorrectly think it's VCC
-    // if it wasn't originally.
-    if (Size == 1)
-      return false;
-  }
+  unsigned Opcode = IsSgpr ? AMDGPU::S_MOV_B32 : AMDGPU::V_MOV_B32_e32;
 
   if (Size != 64) {
     I.setDesc(TII.get(Opcode));
     I.addImplicitDefUseOperands(*MF);
     return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
   }
 
   const DebugLoc &DL = I.getDebugLoc();
-
   APInt Imm(Size, I.getOperand(1).getImm());
 
   MachineInstr *ResInst;
   if (IsSgpr && TII.isInlineConstant(Imm)) {
     ResInst = BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_MOV_B64), DstReg)
       .addImm(I.getOperand(1).getImm());
   } else {
     const TargetRegisterClass *RC = IsSgpr ?

 void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB,
                                                 const MachineInstr &MI,
                                                 int OpIdx) const {
   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
          "Expected G_CONSTANT");
   MIB.addImm(-MI.getOperand(1).getCImm()->getSExtValue());
 }
 
-void AMDGPUInstructionSelector::renderBitcastImm(MachineInstrBuilder &MIB,
+void AMDGPUInstructionSelector::renderBitcastFPImm(MachineInstrBuilder &MIB,
                                                    const MachineInstr &MI,
                                                    int OpIdx) const {
-  assert(OpIdx == -1);
-
   const MachineOperand &Op = MI.getOperand(1);
-  if (MI.getOpcode() == TargetOpcode::G_FCONSTANT)
-    MIB.addImm(Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue());
-  else {
-    assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
-    MIB.addImm(Op.getCImm()->getSExtValue());
-  }
+  assert(MI.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1);
 }
 
 void AMDGPUInstructionSelector::renderPopcntImm(MachineInstrBuilder &MIB,
                                                 const MachineInstr &MI,
                                                 int OpIdx) const {
   assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
          "Expected G_CONSTANT");
   MIB.addImm(MI.getOperand(1).getCImm()->getValue().popcount());

llvm/lib/Target/AMDGPU/SIInstrInfo.td

 
 def InlineImmFP64 : FPImmLeaf<f64, [{
   return isInlineImmediate(Imm);
 }]>;
 
 
 class VGPRImm <dag frag> : PatLeaf<frag, [{
   return isVGPRImm(N);
-}]>;
+}]> {
+  let GISelPredicateCode = [{return true;}];
+}
 
 def NegateImm : SDNodeXForm<imm, [{
   return CurDAG->getConstant(-N->getSExtValue(), SDLoc(N), MVT::i32);
 }]>;
 
 // TODO: When FP inline imm values work?
 def NegSubInlineConst32 : ImmLeaf<i32, [{
   return Imm < -16 && Imm >= -64;

llvm/lib/Target/AMDGPU/SIInstructions.td

(V_BFI_B32_e64 (S_MOV_B32 (i32 0x00007fff)), $src0,
              (V_LSHRREV_B32_e64 (i32 16), (EXTRACT_SUBREG $src1, sub1)))
 >;
 
 /********** ================== **********/
 /********** Immediate Patterns **********/
 /********** ================== **********/
 
 def : GCNPat <
+  (i32 imm:$imm),
+  (S_MOV_B32 imm:$imm)
+>;
+
+// FIXME: Remove VGPRImm
+def : GCNPat <
   (VGPRImm<(i32 imm)>:$imm),
   (V_MOV_B32_e32 imm:$imm)
 >;
 
 def : GCNPat <
   (VGPRImm<(f32 fpimm)>:$imm),
   (V_MOV_B32_e32 (f32 (bitcast_fpimm_to_i32 $imm)))
 >;
 
 def : GCNPat <
-  (i32 imm:$imm),
-  (S_MOV_B32 imm:$imm)
->;
-
-def : GCNPat <
   (VGPRImm<(SIlds tglobaladdr:$ga)>),
   (V_MOV_B32_e32 $ga)
 >;
 
 def : GCNPat <
   (SIlds tglobaladdr:$ga),
   (S_MOV_B32 $ga)
 >;
 
 // FIXME: Workaround for ordering issue with peephole optimizer where
 // a register class copy interferes with immediate folding.  Should
 // use s_mov_b32, which can be shrunk to s_movk_i32
 def : GCNPat <
   (VGPRImm<(f16 fpimm)>:$imm),
   (V_MOV_B32_e32 (f16 (bitcast_fpimm_to_i32 $imm)))
 >;
 
 def : GCNPat <
+  (VGPRImm<(i16 imm)>:$imm),
+  (V_MOV_B32_e32 imm:$imm)
+>;
+
+def : GCNPat <
   (f32 fpimm:$imm),
   (S_MOV_B32 (f32 (bitcast_fpimm_to_i32 $imm)))
 >;
 
 def : GCNPat <
+  (f32 fpimm:$imm),
+  (V_MOV_B32_e32 (f32 (bitcast_fpimm_to_i32 $imm)))
+>;
+
+def : GCNPat <
   (f16 fpimm:$imm),
   (S_MOV_B32 (i32 (bitcast_fpimm_to_i32 $imm)))
 >;
 
 def : GCNPat <
+  (f16 fpimm:$imm),
+  (V_MOV_B32_e32 (i32 (bitcast_fpimm_to_i32 $imm)))
+>;
+
+def : GCNPat <
   (p5 frameindex:$fi),
   (V_MOV_B32_e32 (p5 (frameindex_to_targetframeindex $fi)))
 >;
 
 def : GCNPat <
   (p5 frameindex:$fi),
   (S_MOV_B32 (p5 (frameindex_to_targetframeindex $fi)))
 >;
 
 def : GCNPat <
   (i64 InlineImm64:$imm),
   (S_MOV_B64 InlineImm64:$imm)
 >;
 
-// XXX - Should this use a s_cmp to set SCC?
+def : GCNPat <
+  (f64 InlineImmFP64:$imm),
+  (S_MOV_B64 (i64 (bitcast_fpimm_to_i64 $imm)))
+>;
 
 // Set to sign-extended 64-bit value (true = -1, false = 0)
-def : GCNPat <
-  (i1 imm:$imm),
-  (S_MOV_B64 (i64 (as_i64imm $imm)))
-> {
+def : GCNPat <(i1 imm:$imm),
+              (S_MOV_B64 imm:$imm)> {
   let WaveSizePredicate = isWave64;
 }
 
-def : GCNPat <
-  (i1 imm:$imm),
-  (S_MOV_B32 (i32 (as_i32imm $imm)))
-> {
+def : GCNPat <(i1 imm:$imm),
+              (S_MOV_B32 imm:$imm)> {
   let WaveSizePredicate = isWave32;
 }
 
-def : GCNPat <
-  (f64 InlineImmFP64:$imm),
-  (S_MOV_B64 (f64 (bitcast_fpimm_to_i64 InlineImmFP64:$imm)))
->;
-
 /********** ================== **********/
 /********** Intrinsic Patterns **********/
 /********** ================== **********/
 
 def : GCNPat <
   (f32 (fpow (VOP3Mods f32:$src0, i32:$src0_mods), (VOP3Mods f32:$src1, i32:$src1_mods))),
   (V_EXP_F32_e64 SRCMODS.NONE, (V_MUL_LEGACY_F32_e64 $src1_mods, $src1, SRCMODS.NONE, (V_LOG_F32_e64 $src0_mods, $src0), 0, 0))
 >;

llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-constant.mir

bb.0:
     %1:vgpr(p3) = G_CONSTANT i32 1
     %2:vgpr(p3) = G_CONSTANT i32 -1
     %3:vgpr(p3) = G_CONSTANT i32 -54
     %4:vgpr(p3) = G_CONSTANT i32 27
     S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
 ...
 
 ---
+name:            constant_s_p2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; WAVE64-LABEL: name: constant_s_p2
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE64-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; WAVE64-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 -54
+    ; WAVE64-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
+    ; WAVE64-NEXT: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
+    ;
+    ; WAVE32-LABEL: name: constant_s_p2
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE32-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; WAVE32-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 -54
+    ; WAVE32-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
+    ; WAVE32-NEXT: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
+    %0:sgpr(p2) = G_CONSTANT i32 0
+    %1:sgpr(p2) = G_CONSTANT i32 1
+    %2:sgpr(p2) = G_CONSTANT i32 -1
+    %3:sgpr(p2) = G_CONSTANT i32 -54
+    %4:sgpr(p2) = G_CONSTANT i32 27
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
+...
+
+---
+name:            constant_v_p2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    ; WAVE64-LABEL: name: constant_v_p2
+    ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -54, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
+    ; WAVE64-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
+    ;
+    ; WAVE32-LABEL: name: constant_v_p2
+    ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -54, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
+    ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
+    %0:vgpr(p2) = G_CONSTANT i32 0
+    %1:vgpr(p2) = G_CONSTANT i32 1
+    %2:vgpr(p2) = G_CONSTANT i32 -1
+    %3:vgpr(p2) = G_CONSTANT i32 -54
+    %4:vgpr(p2) = G_CONSTANT i32 27
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
+...
+
+---
+name:            constant_s_p5
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; WAVE64-LABEL: name: constant_s_p5
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE64-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; WAVE64-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 -54
+    ; WAVE64-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
+    ; WAVE64-NEXT: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
+    ;
+    ; WAVE32-LABEL: name: constant_s_p5
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE32-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; WAVE32-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 -54
+    ; WAVE32-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
+    ; WAVE32-NEXT: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
+    %0:sgpr(p5) = G_CONSTANT i32 0
+    %1:sgpr(p5) = G_CONSTANT i32 1
+    %2:sgpr(p5) = G_CONSTANT i32 -1
+    %3:sgpr(p5) = G_CONSTANT i32 -54
+    %4:sgpr(p5) = G_CONSTANT i32 27
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
+...
+
+---
+name:            constant_v_p5
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    ; WAVE64-LABEL: name: constant_v_p5
+    ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -54, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
+    ; WAVE64-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
+    ;
+    ; WAVE32-LABEL: name: constant_v_p5
+    ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -54, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
+    ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
+    %0:vgpr(p5) = G_CONSTANT i32 0
+    %1:vgpr(p5) = G_CONSTANT i32 1
+    %2:vgpr(p5) = G_CONSTANT i32 -1
+    %3:vgpr(p5) = G_CONSTANT i32 -54
+    %4:vgpr(p5) = G_CONSTANT i32 27
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
+...
+
+---
+name:            constant_s_p6
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+
+    ; WAVE64-LABEL: name: constant_s_p6
+    ; WAVE64: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE64-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE64-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; WAVE64-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 -54
+    ; WAVE64-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
+    ; WAVE64-NEXT: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
+    ;
+    ; WAVE32-LABEL: name: constant_s_p6
+    ; WAVE32: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; WAVE32-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+    ; WAVE32-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
+    ; WAVE32-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 -54
+    ; WAVE32-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 27
+    ; WAVE32-NEXT: S_ENDPGM 0, implicit [[S_MOV_B32_]], implicit [[S_MOV_B32_1]], implicit [[S_MOV_B32_2]], implicit [[S_MOV_B32_3]], implicit [[S_MOV_B32_4]]
+    %0:sgpr(p6) = G_CONSTANT i32 0
+    %1:sgpr(p6) = G_CONSTANT i32 1
+    %2:sgpr(p6) = G_CONSTANT i32 -1
+    %3:sgpr(p6) = G_CONSTANT i32 -54
+    %4:sgpr(p6) = G_CONSTANT i32 27
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
+...
+
+---
+name:            constant_v_p6
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+
+body: |
+  bb.0:
+    ; WAVE64-LABEL: name: constant_v_p6
+    ; WAVE64: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -54, implicit $exec
+    ; WAVE64-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
+    ; WAVE64-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
+    ;
+    ; WAVE32-LABEL: name: constant_v_p6
+    ; WAVE32: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -1, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 -54, implicit $exec
+    ; WAVE32-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 27, implicit $exec
+    ; WAVE32-NEXT: S_ENDPGM 0, implicit [[V_MOV_B32_e32_]], implicit [[V_MOV_B32_e32_1]], implicit [[V_MOV_B32_e32_2]], implicit [[V_MOV_B32_e32_3]], implicit [[V_MOV_B32_e32_4]]
+    %0:vgpr(p6) = G_CONSTANT i32 0
+    %1:vgpr(p6) = G_CONSTANT i32 1
+    %2:vgpr(p6) = G_CONSTANT i32 -1
+    %3:vgpr(p6) = G_CONSTANT i32 -54
+    %4:vgpr(p6) = G_CONSTANT i32 27
+    S_ENDPGM 0, implicit %0 , implicit %1 , implicit %2, implicit %3, implicit %4
+...
+
+---
 name:            constant_s_p1
 legalized:       true
 regBankSelected: true
 tracksRegLiveness: true
 
 body: |
   bb.0:
     ; WAVE64-LABEL: name: constant_s_p1

llvm/test/CodeGen/AMDGPU/frame-index-elimination.ll

 
 %struct0 = type { [4224 x %type.i16] }
 %type.i16 = type { i16 }
 @_ZZN0 = external hidden addrspace(3) global %struct0, align 8
 
 ; GFX11-LABEL: tied_operand_test:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-DAG:     scratch_load_u16 [[LDRESULT:v[0-9]+]], off, off offset:4
-; GFX11-DAG:     v_mov_b32_e32 [[C:v[0-9]+]], 0x7b
+; GFX11-DAG:     v_dual_mov_b32 [[C:v[0-9]+]], 0x7b
 ; GFX11-DAG:     ds_store_b16 v{{[0-9]+}}, [[LDRESULT]]  offset:10
 ; GFX11-DAG:     ds_store_b16 v{{[0-9]+}}, [[C]]  offset:8
 ; GFX11-NEXT:    s_endpgm
 define protected amdgpu_kernel void @tied_operand_test(i1 %c1, i1 %c2, i32 %val) {
 entry:
   %scratch0 = alloca i16, align 4, addrspace(5)
   %scratch1 = alloca i16, align 4, addrspace(5)
   %first = select i1 %c1, ptr addrspace(5) %scratch0, ptr addrspace(5) %scratch1

llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll

 ; GCN-NEXT:    s_mov_b32 s4, SCRATCH_RSRC_DWORD0
 ; GCN-NEXT:    s_mov_b32 s5, SCRATCH_RSRC_DWORD1
 ; GCN-NEXT:    s_mov_b32 s6, -1
 ; GCN-NEXT:    s_mov_b32 s7, 0xe80000
 ; GCN-NEXT:    s_add_u32 s4, s4, s3
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GCN-NEXT:    s_addc_u32 s5, s5, 0
 ; GCN-NEXT:    v_mov_b32_e32 v0, 4
+; GCN-NEXT:    v_mov_b32_e32 v1, 1
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    s_and_b32 s3, s3, 3
-; GCN-NEXT:    v_mov_b32_e32 v1, s2
 ; GCN-NEXT:    v_lshrrev_b16_e64 v2, 1, s2
-; GCN-NEXT:    v_lshrrev_b16_e64 v3, 2, s2
-; GCN-NEXT:    v_lshrrev_b16_e64 v4, 3, s2
+; GCN-NEXT:    v_mov_b32_e32 v3, s2
+; GCN-NEXT:    v_lshrrev_b16_e64 v4, 2, s2
+; GCN-NEXT:    v_lshrrev_b16_e64 v5, 3, s2
 ; GCN-NEXT:    v_or_b32_e32 v0, s3, v0
 ; GCN-NEXT:    v_and_b32_e32 v2, 1, v2
-; GCN-NEXT:    v_and_b32_e32 v3, 3, v3
-; GCN-NEXT:    v_and_b32_e32 v4, 1, v4
-; GCN-NEXT:    buffer_store_byte v1, off, s[4:7], 0 offset:4
-; GCN-NEXT:    buffer_store_byte v4, off, s[4:7], 0 offset:7
-; GCN-NEXT:    buffer_store_byte v3, off, s[4:7], 0 offset:6
+; GCN-NEXT:    v_and_b32_e32 v4, 3, v4
+; GCN-NEXT:    v_and_b32_e32 v5, 1, v5
+; GCN-NEXT:    buffer_store_byte v3, off, s[4:7], 0 offset:4
+; GCN-NEXT:    buffer_store_byte v5, off, s[4:7], 0 offset:7
+; GCN-NEXT:    buffer_store_byte v4, off, s[4:7], 0 offset:6
 ; GCN-NEXT:    buffer_store_byte v2, off, s[4:7], 0 offset:5
-; GCN-NEXT:    v_mov_b32_e32 v1, 1
 ; GCN-NEXT:    buffer_store_byte v1, v0, s[4:7], 0 offen
 ; GCN-NEXT:    buffer_load_ubyte v0, off, s[4:7], 0 offset:4
 ; GCN-NEXT:    buffer_load_ubyte v1, off, s[4:7], 0 offset:5
 ; GCN-NEXT:    buffer_load_ubyte v2, off, s[4:7], 0 offset:6
 ; GCN-NEXT:    buffer_load_ubyte v3, off, s[4:7], 0 offset:7
 ; GCN-NEXT:    s_waitcnt vmcnt(3)
 ; GCN-NEXT:    v_and_b32_e32 v0, 1, v0
 ; GCN-NEXT:    s_waitcnt vmcnt(2)

llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll

 ; module_variable used/not-used
 ; kernel variable normal/overaligned
 ; extern variable normal/overaligned
 
 define amdgpu_kernel void @module_0_kernel_normal_extern_normal(i32 %idx) {
 ; CHECK-LABEL: module_0_kernel_normal_extern_normal:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2
+; CHECK-NEXT:    v_mov_b32_e32 v0, 2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_lshl_b32 s0, s0, 2
 ; CHECK-NEXT:    s_add_i32 s0, s0, 4
 ; CHECK-NEXT:    v_mov_b32_e32 v2, s0
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b32 v2, v0
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b32 v2, v1
 ; CHECK-NEXT:    s_endpgm
   store i16 2, ptr addrspace(3) @kernel_normal
 
   %arrayidx1 = getelementptr inbounds [0 x float], ptr addrspace(3) @extern_normal, i32 0, i32 %idx
   store float 0.0, ptr addrspace(3) %arrayidx1
   ret void
 }
 

 ; CHECK-NEXT:    s_add_u32 s8, s8, use_module@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s9, s9, use_module@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[10:11], s[8:9], 0x0
 ; CHECK-NEXT:    s_load_dword s12, s[6:7], 0x0
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[10:11]
 ; CHECK-NEXT:    s_lshl_b32 s4, s12, 2
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 1
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_add_i32 s4, s4, 4
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 2
 ; CHECK-NEXT:    v_mov_b32_e32 v3, s4
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b16 v0, v2 offset:2
-; CHECK-NEXT:    ds_write_b32 v3, v0
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b16 v1, v2 offset:2
+; CHECK-NEXT:    ds_write_b32 v3, v1
 ; CHECK-NEXT:    s_endpgm
   call void @use_module()
   store i16 1, ptr addrspace(3) @module_variable
 
   store i16 2, ptr addrspace(3) @kernel_normal
 
   %arrayidx1 = getelementptr inbounds [0 x float], ptr addrspace(3) @extern_normal, i32 0, i32 %idx
   store float 0.0, ptr addrspace(3) %arrayidx1
   ret void
 }
 
 define amdgpu_kernel void @module_0_kernel_overalign_extern_normal(i32 %idx) {
 ; CHECK-LABEL: module_0_kernel_overalign_extern_normal:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2
+; CHECK-NEXT:    v_mov_b32_e32 v0, 2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_lshl_b32 s0, s0, 2
 ; CHECK-NEXT:    s_add_i32 s0, s0, 4
 ; CHECK-NEXT:    v_mov_b32_e32 v2, s0
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b32 v2, v0
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b32 v2, v1
 ; CHECK-NEXT:    s_endpgm
   store i16 2, ptr addrspace(3) @kernel_overalign
 
   %arrayidx1 = getelementptr inbounds [0 x float], ptr addrspace(3) @extern_normal, i32 0, i32 %idx
   store float 0.0, ptr addrspace(3) %arrayidx1
   ret void
 }
 

 ; CHECK-NEXT:    s_add_u32 s8, s8, use_module@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s9, s9, use_module@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[10:11], s[8:9], 0x0
 ; CHECK-NEXT:    s_load_dword s12, s[6:7], 0x0
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[10:11]
 ; CHECK-NEXT:    s_lshl_b32 s4, s12, 2
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 1
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_add_i32 s4, s4, 8
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 2
 ; CHECK-NEXT:    v_mov_b32_e32 v3, s4
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b16 v0, v2 offset:4
-; CHECK-NEXT:    ds_write_b32 v3, v0
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b16 v1, v2 offset:4
+; CHECK-NEXT:    ds_write_b32 v3, v1
 ; CHECK-NEXT:    s_endpgm
   call void @use_module()
   store i16 1, ptr addrspace(3) @module_variable
 
   store i16 2, ptr addrspace(3) @kernel_overalign
 
   %arrayidx1 = getelementptr inbounds [0 x float], ptr addrspace(3) @extern_normal, i32 0, i32 %idx
   store float 0.0, ptr addrspace(3) %arrayidx1
   ret void
 }
 
 define amdgpu_kernel void @module_0_kernel_normal_extern_overalign(i32 %idx) {
 ; CHECK-LABEL: module_0_kernel_normal_extern_overalign:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2
+; CHECK-NEXT:    v_mov_b32_e32 v0, 2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_lshl_b32 s0, s0, 2
 ; CHECK-NEXT:    s_add_i32 s0, s0, 8
 ; CHECK-NEXT:    v_mov_b32_e32 v2, s0
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b32 v2, v0
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b32 v2, v1
 ; CHECK-NEXT:    s_endpgm
   store i16 2, ptr addrspace(3) @kernel_normal
 
   %arrayidx1 = getelementptr inbounds [0 x float], ptr addrspace(3) @extern_overalign, i32 0, i32 %idx
   store float 0.0, ptr addrspace(3) %arrayidx1
   ret void
 }
 

 ; CHECK-NEXT:    s_add_u32 s8, s8, use_module@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s9, s9, use_module@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[10:11], s[8:9], 0x0
 ; CHECK-NEXT:    s_load_dword s12, s[6:7], 0x0
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[10:11]
 ; CHECK-NEXT:    s_lshl_b32 s4, s12, 2
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 1
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_add_i32 s4, s4, 8
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 2
 ; CHECK-NEXT:    v_mov_b32_e32 v3, s4
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b16 v0, v2 offset:2
-; CHECK-NEXT:    ds_write_b32 v3, v0
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b16 v1, v2 offset:2
+; CHECK-NEXT:    ds_write_b32 v3, v1
 ; CHECK-NEXT:    s_endpgm
   call void @use_module()
   store i16 1, ptr addrspace(3) @module_variable
 
   store i16 2, ptr addrspace(3) @kernel_normal
 
   %arrayidx1 = getelementptr inbounds [0 x float], ptr addrspace(3) @extern_overalign, i32 0, i32 %idx
   store float 0.0, ptr addrspace(3) %arrayidx1
   ret void
 }
 
 define amdgpu_kernel void @module_0_kernel_overalign_extern_overalign(i32 %idx) {
 ; CHECK-LABEL: module_0_kernel_overalign_extern_overalign:
 ; CHECK:       ; %bb.0:
 ; CHECK-NEXT:    s_load_dword s0, s[4:5], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2
+; CHECK-NEXT:    v_mov_b32_e32 v0, 2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_lshl_b32 s0, s0, 2
 ; CHECK-NEXT:    s_add_i32 s0, s0, 8
 ; CHECK-NEXT:    v_mov_b32_e32 v2, s0
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b32 v2, v0
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b32 v2, v1
 ; CHECK-NEXT:    s_endpgm
   store i16 2, ptr addrspace(3) @kernel_overalign
 
   %arrayidx1 = getelementptr inbounds [0 x float], ptr addrspace(3) @extern_overalign, i32 0, i32 %idx
   store float 0.0, ptr addrspace(3) %arrayidx1
   ret void
 }
 

 ; CHECK-NEXT:    s_add_u32 s8, s8, use_module@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s9, s9, use_module@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[10:11], s[8:9], 0x0
 ; CHECK-NEXT:    s_load_dword s12, s[6:7], 0x0
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[10:11]
 ; CHECK-NEXT:    s_lshl_b32 s4, s12, 2
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 1
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_add_i32 s4, s4, 8
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 2
 ; CHECK-NEXT:    v_mov_b32_e32 v3, s4
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b16 v0, v2 offset:4
-; CHECK-NEXT:    ds_write_b32 v3, v0
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b16 v1, v2 offset:4
+; CHECK-NEXT:    ds_write_b32 v3, v1
 ; CHECK-NEXT:    s_endpgm
   call void @use_module()
   store i16 1, ptr addrspace(3) @module_variable
 
   store i16 2, ptr addrspace(3) @kernel_overalign
 
   %arrayidx1 = getelementptr inbounds [0 x float], ptr addrspace(3) @extern_overalign, i32 0, i32 %idx
   store float 0.0, ptr addrspace(3) %arrayidx1

 ; CHECK-NEXT:    s_addc_u32 s9, s9, 0
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s8
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
 ; CHECK-NEXT:    s_add_u32 s0, s0, s11
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
 ; CHECK-NEXT:    s_getpc_b64 s[6:7]
 ; CHECK-NEXT:    s_add_u32 s6, s6, use_extern_normal@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s7, s7, use_extern_normal@gotpcrel32@hi+12
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 2
 ; CHECK-NEXT:    s_load_dwordx2 s[8:9], s[6:7], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; CHECK-NEXT:    s_mov_b32 s15, 0
-; CHECK-NEXT:    ds_write_b16 v0, v1
+; CHECK-NEXT:    ds_write_b16 v1, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; CHECK-NEXT:    s_endpgm
   store i16 2, ptr addrspace(3) @kernel_normal
 
   call void @use_extern_normal()
   ret void
 }

 ; CHECK-NEXT:    s_add_u32 s4, s4, use_module@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, use_module@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, use_extern_normal@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, use_extern_normal@gotpcrel32@hi+12
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 2
 ; CHECK-NEXT:    s_mov_b32 s15, 4
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b16 v0, v2 offset:2
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b16 v1, v2 offset:2
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
   call void @use_module()
   store i16 1, ptr addrspace(3) @module_variable
 
   store i16 2, ptr addrspace(3) @kernel_normal
 

 ; CHECK-NEXT:    s_addc_u32 s9, s9, 0
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s8
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
 ; CHECK-NEXT:    s_add_u32 s0, s0, s11
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
 ; CHECK-NEXT:    s_getpc_b64 s[6:7]
 ; CHECK-NEXT:    s_add_u32 s6, s6, use_extern_normal@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s7, s7, use_extern_normal@gotpcrel32@hi+12
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 2
 ; CHECK-NEXT:    s_load_dwordx2 s[8:9], s[6:7], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; CHECK-NEXT:    s_mov_b32 s15, 2
-; CHECK-NEXT:    ds_write_b16 v0, v1
+; CHECK-NEXT:    ds_write_b16 v1, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; CHECK-NEXT:    s_endpgm
   store i16 2, ptr addrspace(3) @kernel_overalign
 
   call void @use_extern_normal()
   ret void
 }

 ; CHECK-NEXT:    s_add_u32 s4, s4, use_module@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, use_module@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, use_extern_normal@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, use_extern_normal@gotpcrel32@hi+12
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 2
 ; CHECK-NEXT:    s_mov_b32 s15, 6
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b16 v0, v2 offset:4
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b16 v1, v2 offset:4
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
   call void @use_module()
   store i16 1, ptr addrspace(3) @module_variable
 
   store i16 2, ptr addrspace(3) @kernel_overalign
 

 ; CHECK-NEXT:    s_addc_u32 s9, s9, 0
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s8
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
 ; CHECK-NEXT:    s_add_u32 s0, s0, s11
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
 ; CHECK-NEXT:    s_getpc_b64 s[6:7]
 ; CHECK-NEXT:    s_add_u32 s6, s6, use_extern_overalign@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s7, s7, use_extern_overalign@gotpcrel32@hi+12
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 2
 ; CHECK-NEXT:    s_load_dwordx2 s[8:9], s[6:7], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; CHECK-NEXT:    s_mov_b32 s15, 1
-; CHECK-NEXT:    ds_write_b16 v0, v1
+; CHECK-NEXT:    ds_write_b16 v1, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; CHECK-NEXT:    s_endpgm
   store i16 2, ptr addrspace(3) @kernel_normal
 
   call void @use_extern_overalign()
   ret void
 }

 ; CHECK-NEXT:    s_add_u32 s4, s4, use_module@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, use_module@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, use_extern_overalign@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, use_extern_overalign@gotpcrel32@hi+12
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 2
 ; CHECK-NEXT:    s_mov_b32 s15, 5
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b16 v0, v2 offset:2
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b16 v1, v2 offset:2
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
   call void @use_module()
   store i16 1, ptr addrspace(3) @module_variable
 
   store i16 2, ptr addrspace(3) @kernel_normal
 

 ; CHECK-NEXT:    s_addc_u32 s9, s9, 0
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s8
 ; CHECK-NEXT:    s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s9
 ; CHECK-NEXT:    s_add_u32 s0, s0, s11
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
 ; CHECK-NEXT:    s_getpc_b64 s[6:7]
 ; CHECK-NEXT:    s_add_u32 s6, s6, use_extern_overalign@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s7, s7, use_extern_overalign@gotpcrel32@hi+12
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 2
 ; CHECK-NEXT:    s_load_dwordx2 s[8:9], s[6:7], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 2
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_mov_b64 s[6:7], s[4:5]
 ; CHECK-NEXT:    s_mov_b32 s15, 3
-; CHECK-NEXT:    ds_write_b16 v0, v1
+; CHECK-NEXT:    ds_write_b16 v1, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[8:9]
 ; CHECK-NEXT:    s_endpgm
   store i16 2, ptr addrspace(3) @kernel_overalign
 
   call void @use_extern_overalign()
   ret void
 }

 ; CHECK-NEXT:    s_add_u32 s4, s4, use_module@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, use_module@gotpcrel32@hi+12
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_getpc_b64 s[4:5]
 ; CHECK-NEXT:    s_add_u32 s4, s4, use_extern_overalign@gotpcrel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s5, s5, use_extern_overalign@gotpcrel32@hi+12
-; CHECK-NEXT:    v_mov_b32_e32 v0, 0
+; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[4:5], 0x0
-; CHECK-NEXT:    v_mov_b32_e32 v1, 1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 2
 ; CHECK-NEXT:    s_mov_b32 s15, 7
-; CHECK-NEXT:    ds_write_b16 v0, v1
-; CHECK-NEXT:    ds_write_b16 v0, v2 offset:4
+; CHECK-NEXT:    ds_write_b16 v1, v0
+; CHECK-NEXT:    ds_write_b16 v1, v2 offset:4
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; CHECK-NEXT:    s_endpgm
   call void @use_module()
   store i16 1, ptr addrspace(3) @module_variable
 
   store i16 2, ptr addrspace(3) @kernel_overalign
 
   call void @use_extern_overalign()
   ret void
 }
 
 
 attributes #0 = { noinline }

llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa-memcpy.ll

 @_f2 = linkonce_odr hidden local_unnamed_addr addrspace(3) global %vec_type undef, comdat, align 1
 
 ;.
 ; CHECK: @[[LLVM_AMDGCN_KERNEL_TEST_LDS:[a-zA-Z0-9_$"\\.-]+]] = internal addrspace(3) global [[LLVM_AMDGCN_KERNEL_TEST_LDS_T:%.*]] undef, align 4, !absolute_symbol !0
 ;.
 define protected amdgpu_kernel void @test(ptr addrspace(1) nocapture %ptr.coerce) local_unnamed_addr #0 {
 ; GCN-LABEL: test:
 ; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    v_mov_b32_e32 v0, 0
-; GCN-NEXT:    v_mov_b32_e32 v1, 2
-; GCN-NEXT:    ds_write_b8 v0, v1
-; GCN-NEXT:    ds_read_u8 v2, v0 offset:2
-; GCN-NEXT:    ds_read_u16 v3, v0
+; GCN-NEXT:    v_mov_b32_e32 v0, 2
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    ds_write_b8 v1, v0
+; GCN-NEXT:    ds_read_u8 v2, v1 offset:2
+; GCN-NEXT:    ds_read_u16 v3, v1
 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    ds_write_b8 v0, v2 offset:6
-; GCN-NEXT:    ds_write_b16 v0, v3 offset:4
-; GCN-NEXT:    v_cmp_eq_u16_sdwa s[2:3], v3, v1 src0_sel:BYTE_0 src1_sel:DWORD
-; GCN-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s[2:3]
-; GCN-NEXT:    global_store_byte v0, v1, s[0:1]
+; GCN-NEXT:    ds_write_b8 v1, v2 offset:6
+; GCN-NEXT:    ds_write_b16 v1, v3 offset:4
+; GCN-NEXT:    v_cmp_eq_u16_sdwa s[2:3], v3, v0 src0_sel:BYTE_0 src1_sel:DWORD
+; GCN-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s[2:3]
+; GCN-NEXT:    global_store_byte v1, v0, s[0:1]
 ; GCN-NEXT:    s_endpgm
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    store i8 3, ptr addrspace(3) @llvm.amdgcn.kernel.test.lds, align 4, !alias.scope !1, !noalias !4
 ; CHECK-NEXT:    tail call void @llvm.memcpy.p3.p3.i64(ptr addrspace(3) noundef align 1 dereferenceable(3) getelementptr inbounds (%llvm.amdgcn.kernel.test.lds.t, ptr addrspace(3) @llvm.amdgcn.kernel.test.lds, i32 0, i32 2), ptr addrspace(3) noundef align 1 dereferenceable(3) @llvm.amdgcn.kernel.test.lds, i64 3, i1 false), !alias.scope !6, !noalias !7
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i8, ptr addrspace(3) getelementptr inbounds (%llvm.amdgcn.kernel.test.lds.t, ptr addrspace(3) @llvm.amdgcn.kernel.test.lds, i32 0, i32 2), align 4, !alias.scope !4, !noalias !1
 ; CHECK-NEXT:    [[CMP_I_I:%.*]] = icmp eq i8 [[TMP4]], 3
 ; CHECK-NEXT:    store i8 2, ptr addrspace(3) @llvm.amdgcn.kernel.test.lds, align 4, !alias.scope !1, !noalias !4

llvm/test/CodeGen/AMDGPU/scalar_to_vector_v2x16.ll

-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-OPT %s
-; RUN: llc -march=amdgcn -mcpu=fiji -O0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GCN-NOOPT %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=fiji -O0 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 ; GCN-LABEL: {{^}}scalar_to_vector_i16:
-; GCN-NOOPT: s_mov_b32 [[S:s[0-9]+]], 42
-; GCN-NOOPT: v_mov_b32_e32 [[V:v[0-9]+]], [[S]]
-; GCN-OPT:   v_mov_b32_e32 [[V:v[0-9]+]], 42
+; GCN:   v_mov_b32_e32 [[V:v[0-9]+]], 42
 ; GCN: buffer_store_short [[V]],
 define void @scalar_to_vector_i16() {
   %tmp = load <2 x i16>, ptr addrspace(5) undef
   %tmp1 = insertelement <2 x i16> %tmp, i16 42, i64 0
   store <2 x i16> %tmp1, ptr addrspace(5) undef
   ret void
 }
 
 ; GCN-LABEL: {{^}}scalar_to_vector_f16:
-; GCN-NOOPT: s_mov_b32 [[S:s[0-9]+]], 0x3c00
-; GCN-NOOPT: v_mov_b32_e32 [[V:v[0-9]+]], [[S]]
-; GCN-OPT:   v_mov_b32_e32 [[V:v[0-9]+]], 0x3c00
+; GCN:   v_mov_b32_e32 [[V:v[0-9]+]], 0x3c00
 ; GCN: buffer_store_short [[V]],
 define void @scalar_to_vector_f16() {
   %tmp = load <2 x half>, ptr addrspace(5) undef
   %tmp1 = insertelement <2 x half> %tmp, half 1.0, i64 0
   store <2 x half> %tmp1, ptr addrspace(5) undef
   ret void
 }

llvm/test/CodeGen/AMDGPU/vgpr-spill-placement-issue61083.ll

 ; CHECK-NEXT:    s_mov_b64 exec, s[8:9]
 ; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    global_load_ushort v3, v1, s[4:5] offset:4
 ; CHECK-NEXT:    s_waitcnt vmcnt(0)
 ; CHECK-NEXT:    buffer_store_dword v3, off, s[0:3], 0 offset:8 ; 4-byte Folded Spill
 ; CHECK-NEXT:    ; implicit-def: $sgpr4
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:    v_cmp_eq_u32_e64 s[6:7], v2, s4
-; CHECK-NEXT:    s_mov_b32 s4, 0
-; CHECK-NEXT:    v_mov_b32_e32 v2, s4
+; CHECK-NEXT:    v_mov_b32_e32 v2, 0
 ; CHECK-NEXT:    ds_write_b8 v1, v2
 ; CHECK-NEXT:    s_mov_b64 s[4:5], exec
 ; CHECK-NEXT:    v_writelane_b32 v0, s4, 0
 ; CHECK-NEXT:    v_writelane_b32 v0, s5, 1
 ; CHECK-NEXT:    s_or_saveexec_b64 s[8:9], -1
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:4 ; 4-byte Folded Spill
 ; CHECK-NEXT:    s_mov_b64 exec, s[8:9]
 ; CHECK-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]