[AMDGPU] Enable scheduling around FP MODE-setting instructions

[foad]

Summary

Pre-gfx10 all MODE-setting instructions were S_SETREG_B32 which is
marked as having unmodeled side effects, which makes the machine
scheduler treat it as a barrier. Now that we have proper implicit $mode
operands we can use a no-side-effects S_SETMODE_B32 pseudo instead for
setregs that only touch the FP MODE bits, to give the scheduler more
freedom.

[arsenm]

llvm/lib/Target/AMDGPU/SOPInstructions.td

847

I don't like defining pseudos that look like real instruction names. I would prefer to call this something like

S_SETREG_B32_mode or something like that

[foad]

Rename the pseudo-instructions.

[arsenm]

llvm/lib/Target/AMDGPU/SOPInstructions.td

869

I'm not sure this will be encoded correctly since it's repeating the same instruction definition and not adding the physical aliases. Can you add tests checking the encoding? I think this needs to use PseudoInstExpansion

[foad]

llvm/lib/Target/AMDGPU/SOPInstructions.td

869

I'm not sure this will be encoded correctly since it's repeating the same instruction definition and not adding the physical aliases. Can you add tests checking the encoding? I think this needs to use PseudoInstExpansion

869

I'm not sure this will be encoded correctly since it's repeating the same instruction definition and not adding the physical aliases. Can you add tests checking the encoding?

You mean something like adding -show-mc-encoding to test/CodeGen/AMDGPU/llvm.amdgcn.s.setreg.ll ?

I think this needs to use PseudoInstExpansion

I'm not sure how to do that, because at the place where I want to define S_SETREG_IMM32_B32_mode, I don't know what real instruction it should expand to (S_SETREG_B32_gfx6_gfx7 vs S_SETREG_B32_vi vs S_SETREG_B32_gfx10). Is there an example I could follow?

[arsenm]

llvm/lib/Target/AMDGPU/SOPInstructions.td

869

Yes.

Oh right, this mechanism doesn't have a way handle multiple alternative encodings. I thought this wasn't an issue for scalar instructions, which I thought had a stable encoding. You could hack around this in emitInstruction. I do think we need better infrastructure for dealing with this problem though.

[foad]

Add -show-mc-encoding.

[foad]

The encoding seems OK. I can pre-commit the -show-mc-encoding change to make that clearer if you want.

[arsenm]

LGTM, but it would be slightly nicer to factor the instruction definition into a class to avoid repeating the operand definition and asm string

[foad]

Factor instruction definitions into a class.

llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp

static bool isDivFMas(unsigned Opcode) {
   return Opcode == AMDGPU::V_DIV_FMAS_F32 || Opcode == AMDGPU::V_DIV_FMAS_F64;
 }
 
 static bool isSGetReg(unsigned Opcode) {
   return Opcode == AMDGPU::S_GETREG_B32;
 }
 
 static bool isSSetReg(unsigned Opcode) {
-  return Opcode == AMDGPU::S_SETREG_B32 || Opcode == AMDGPU::S_SETREG_IMM32_B32;
+  switch (Opcode) {
+  case AMDGPU::S_SETREG_B32:
+  case AMDGPU::S_SETREG_B32_mode:
+  case AMDGPU::S_SETREG_IMM32_B32:
+  case AMDGPU::S_SETREG_IMM32_B32_mode:
+    return true;
+  }
+  return false;
 }
 
 static bool isRWLane(unsigned Opcode) {
   return Opcode == AMDGPU::V_READLANE_B32 || Opcode == AMDGPU::V_WRITELANE_B32;
 }
 
 static bool isRFE(unsigned Opcode) {
   return Opcode == AMDGPU::S_RFE_B64;

llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

if ((Opc == AMDGPU::V_MAC_F32_e64 || Opc == AMDGPU::V_MAC_F16_e64 ||
       if (FoldAsMAD) {
         MI->untieRegOperand(OpNo);
         return true;
       }
       MI->setDesc(TII->get(Opc));
     }
 
     // Special case for s_setreg_b32
-    if (Opc == AMDGPU::S_SETREG_B32 && OpToFold->isImm()) {
-      MI->setDesc(TII->get(AMDGPU::S_SETREG_IMM32_B32));
+    if (OpToFold->isImm()) {
+      unsigned ImmOpc = 0;
+      if (Opc == AMDGPU::S_SETREG_B32)
+        ImmOpc = AMDGPU::S_SETREG_IMM32_B32;
+      else if (Opc == AMDGPU::S_SETREG_B32_mode)
+        ImmOpc = AMDGPU::S_SETREG_IMM32_B32_mode;
+      if (ImmOpc) {
+        MI->setDesc(TII->get(ImmOpc));
         appendFoldCandidate(FoldList, MI, OpNo, OpToFold);
         return true;
       }
+    }
 
     // If we are already folding into another operand of MI, then
     // we can't commute the instruction, otherwise we risk making the
     // other fold illegal.
     if (isUseMIInFoldList(FoldList, MI))
       return false;
 
     unsigned CommuteOpNo = OpNo;

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

case AMDGPU::DS_GWS_BARRIER:
     // A s_waitcnt 0 is required to be the instruction immediately following.
     if (getSubtarget()->hasGWSAutoReplay()) {
       bundleInstWithWaitcnt(MI);
       return BB;
     }
 
     return emitGWSMemViolTestLoop(MI, BB);
   case AMDGPU::S_SETREG_B32: {
-    if (!getSubtarget()->hasDenormModeInst())
-      return BB;
-
     // Try to optimize cases that only set the denormal mode or rounding mode.
     //
     // If the s_setreg_b32 fully sets all of the bits in the rounding mode or
     // denormal mode to a constant, we can use s_round_mode or s_denorm_mode
     // instead.
     //
     // FIXME: This could be predicates on the immediate, but tablegen doesn't
     // allow you to have a no side effect instruction in the output of a
     // sideeffecting pattern.
-
-    // TODO: Should also emit a no side effects pseudo if only FP bits are
-    // touched, even if not all of them or to a variable.
     unsigned ID, Offset, Width;
     AMDGPU::Hwreg::decodeHwreg(MI.getOperand(1).getImm(), ID, Offset, Width);
     if (ID != AMDGPU::Hwreg::ID_MODE)
       return BB;
 
     const unsigned WidthMask = maskTrailingOnes<unsigned>(Width);
     const unsigned SetMask = WidthMask << Offset;
+
+    if (getSubtarget()->hasDenormModeInst()) {
       unsigned SetDenormOp = 0;
       unsigned SetRoundOp = 0;
 
-    // The dedicated instructions can only set the whole denorm or round mode at
-    // once, not a subset of bits in either.
+      // The dedicated instructions can only set the whole denorm or round mode
+      // at once, not a subset of bits in either.
       if (SetMask ==
           (AMDGPU::Hwreg::FP_ROUND_MASK | AMDGPU::Hwreg::FP_DENORM_MASK)) {
         // If this fully sets both the round and denorm mode, emit the two
         // dedicated instructions for these.
         SetRoundOp = AMDGPU::S_ROUND_MODE;
         SetDenormOp = AMDGPU::S_DENORM_MODE;
       } else if (SetMask == AMDGPU::Hwreg::FP_ROUND_MASK) {
         SetRoundOp = AMDGPU::S_ROUND_MODE;
       } else if (SetMask == AMDGPU::Hwreg::FP_DENORM_MASK) {
         SetDenormOp = AMDGPU::S_DENORM_MODE;
       }
 
       if (SetRoundOp || SetDenormOp) {
         MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
         MachineInstr *Def = MRI.getVRegDef(MI.getOperand(0).getReg());
         if (Def && Def->isMoveImmediate() && Def->getOperand(1).isImm()) {
           unsigned ImmVal = Def->getOperand(1).getImm();
           if (SetRoundOp) {
             BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetRoundOp))
                 .addImm(ImmVal & 0xf);
 
             // If we also have the denorm mode, get just the denorm mode bits.
             ImmVal >>= 4;
           }
 
           if (SetDenormOp) {
             BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(SetDenormOp))
                 .addImm(ImmVal & 0xf);
           }
 
           MI.eraseFromParent();
+          return BB;
+        }
       }
     }
 
+    // If only FP bits are touched, used the no side effects pseudo.
+    if ((SetMask & (AMDGPU::Hwreg::FP_ROUND_MASK |
+                    AMDGPU::Hwreg::FP_DENORM_MASK)) == SetMask)
+      MI.setDesc(TII->get(AMDGPU::S_SETREG_B32_mode));
+
     return BB;
   }
   default:
     return AMDGPUTargetLowering::EmitInstrWithCustomInserter(MI, BB);
   }
 }
 
 bool SITargetLowering::hasBitPreservingFPLogic(EVT VT) const {

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

bool SIInstrInfo::isSchedulingBoundary(const MachineInstr &MI,
 
   // INLINEASM_BR can jump to another block
   if (MI.getOpcode() == TargetOpcode::INLINEASM_BR)
     return true;
 
   // Target-independent instructions do not have an implicit-use of EXEC, even
   // when they operate on VGPRs. Treating EXEC modifications as scheduling
   // boundaries prevents incorrect movements of such instructions.
-
-  // TODO: Don't treat setreg with known constant that only changes MODE as
-  // barrier.
   return MI.modifiesRegister(AMDGPU::EXEC, &RI) ||
          MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
          MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
          changesVGPRIndexingMode(MI);
 }
 
 bool SIInstrInfo::isAlwaysGDS(uint16_t Opcode) const {
   return Opcode == AMDGPU::DS_ORDERED_COUNT ||

llvm/lib/Target/AMDGPU/SIModeRegister.cpp

void SIModeRegister::processBlockPhase1(MachineBasicBlock &MBB,
   // requirements for the block, and need to defer the first InsertionPoint to
   // Phase 3. It is set to false once we have set FirstInsertionPoint, or when
   // we discover an explict setreg that means this block doesn't have any
   // initial requirements.
   bool RequirePending = true;
   Status IPChange;
   for (MachineInstr &MI : MBB) {
     Status InstrMode = getInstructionMode(MI, TII);
-    if ((MI.getOpcode() == AMDGPU::S_SETREG_B32) ||
-        (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32)) {
+    if (MI.getOpcode() == AMDGPU::S_SETREG_B32 ||
+        MI.getOpcode() == AMDGPU::S_SETREG_B32_mode ||
+        MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
+        MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_mode) {
       // We preserve any explicit mode register setreg instruction we encounter,
       // as we assume it has been inserted by a higher authority (this is
       // likely to be a very rare occurrence).
       unsigned Dst = TII->getNamedOperand(MI, AMDGPU::OpName::simm16)->getImm();
       if (((Dst & AMDGPU::Hwreg::ID_MASK_) >> AMDGPU::Hwreg::ID_SHIFT_) !=
           AMDGPU::Hwreg::ID_MODE)
         continue;
 
       unsigned Width = ((Dst & AMDGPU::Hwreg::WIDTH_M1_MASK_) >>
                         AMDGPU::Hwreg::WIDTH_M1_SHIFT_) +
                        1;
       unsigned Offset =
           (Dst & AMDGPU::Hwreg::OFFSET_MASK_) >> AMDGPU::Hwreg::OFFSET_SHIFT_;
       unsigned Mask = ((1 << Width) - 1) << Offset;
 
       // If an InsertionPoint is set we will insert a setreg there.
       if (InsertionPoint) {
         insertSetreg(MBB, InsertionPoint, TII, IPChange.delta(NewInfo->Change));
         InsertionPoint = nullptr;
       }
       // If this is an immediate then we know the value being set, but if it is
       // not an immediate then we treat the modified bits of the mode register
       // as unknown.
-      if (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32) {
+      if (MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32 ||
+          MI.getOpcode() == AMDGPU::S_SETREG_IMM32_B32_mode) {
         unsigned Val = TII->getNamedOperand(MI, AMDGPU::OpName::imm)->getImm();
         unsigned Mode = (Val << Offset) & Mask;
         Status Setreg = Status(Mask, Mode);
         // If we haven't already set the initial requirements for the block we
         // don't need to as the requirements start from this explicit setreg.
         RequirePending = false;
         NewInfo->Change = NewInfo->Change.merge(Setreg);
       } else {

llvm/lib/Target/AMDGPU/SOPInstructions.td

 
 let SubtargetPredicate = isGFX6GFX7GFX8GFX9 in
 def S_CBRANCH_I_FORK : SOPK_Pseudo <
   "s_cbranch_i_fork",
   (outs), (ins SReg_64:$sdst, sopp_brtarget:$simm16),
   "$sdst, $simm16"
 >;
 
-let hasSideEffects = 1 in {
-
 let mayLoad = 1 in {
 // s_getreg_b32 should use hasSideEffects = 1 for tablegen to allow
 // its use in the readcyclecounter selection.
 // FIXME: Need to truncate immediate to 16-bits.
 def S_GETREG_B32 : SOPK_Pseudo <
   "s_getreg_b32",
   (outs SReg_32:$sdst), (ins hwreg:$simm16),
   "$sdst, $simm16",
   [(set i32:$sdst, (int_amdgcn_s_getreg (i32 timm:$simm16)))]> {
   let SOPKZext = 1;
+  let hasSideEffects = 1;
 }
-}
+} // End mayLoad = 1
 
-let mayLoad = 0, mayStore =0 in {
+let mayLoad = 0, mayStore = 0, Defs = [MODE], Uses = [MODE] in {
 
 // FIXME: Need to truncate immediate to 16-bits.
 def S_SETREG_B32 : SOPK_Pseudo <
   "s_setreg_b32",
   (outs), (ins SReg_32:$sdst, hwreg:$simm16),
   "$simm16, $sdst",
   [(int_amdgcn_s_setreg (i32 timm:$simm16), i32:$sdst)]> {
 
   // Use custom inserter to optimize some cases to
-  // S_DENORM_MODE/S_ROUND_MODE.
+  // S_DENORM_MODE/S_ROUND_MODE/S_SETREG_B32_mode.
   let usesCustomInserter = 1;
-  let Defs = [MODE];
-  let Uses = [MODE];
+  let hasSideEffects = 1;
+}
+
+// Variant of SETREG that is guaranteed to only touch FP bits in the MODE
+// register, so doesn't have unmodeled side effects.
+def S_SETREG_B32_mode : SOPK_Pseudo <
+  "s_setreg_b32",
+  (outs), (ins SReg_32:$sdst, hwreg:$simm16),
+  "$simm16, $sdst"> {
+
+  let hasSideEffects = 0;
 }
 
 // FIXME: Not on SI?
 //def S_GETREG_REGRD_B32 : SOPK_32 <sopk<0x14, 0x13>, "s_getreg_regrd_b32">;
 
 def S_SETREG_IMM32_B32 : SOPK_Pseudo <
   "s_setreg_imm32_b32",
   (outs), (ins i32imm:$imm, hwreg:$simm16),
   "$simm16, $imm"> {
   let Size = 8; // Unlike every other SOPK instruction.
   let has_sdst = 0;
-  let Defs = [MODE];
-  let Uses = [MODE];
+  let hasSideEffects = 1;
 }
 
+// Variant of SETREG_IMM32 that is guaranteed to only touch FP bits in the MODE
+// register, so doesn't have unmodeled side effects.
+def S_SETREG_IMM32_B32_mode : SOPK_Pseudo <
+  "s_setreg_imm32_b32",
+  (outs), (ins i32imm:$imm, hwreg:$simm16),
+  "$simm16, $imm"> {
+  let Size = 8; // Unlike every other SOPK instruction.
+  let has_sdst = 0;
+  let hasSideEffects = 0;
 }
-} // End hasSideEffects = 1
+
+} // End mayLoad = 0, mayStore = 0, Defs = [MODE], Uses = [MODE]
 
 class SOPK_WAITCNT<string opName, list<dag> pat=[]> :
     SOPK_Pseudo<
         opName,
         (outs),
         (ins SReg_32:$sdst, s16imm:$simm16),
         "$sdst, $simm16",
         pat> {

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.s.setreg.ll

 ; GFX6789-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: test_setreg_roundingmode_var_vgpr:
 ; GFX10:       ; %bb.0:
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX10-NEXT:    ; implicit-def: $vcc_hi
-; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 0, 3), s4
 ; GFX10-NEXT:    ;;#ASMSTART
 ; GFX10-NEXT:    ;;#ASMEND
+; GFX10-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 0, 3), s4
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
   call void @llvm.amdgcn.s.setreg(i32 4097, i32 %var.mode)
   call void asm sideeffect "", ""()
   ret void
 }
 
 declare void @llvm.amdgcn.s.setreg(i32 immarg, i32) #0
 
 attributes #0 = { nounwind }

llvm/test/CodeGen/AMDGPU/fdiv-nofpexcept.ll

define float @fdiv_f32(float %a, float %b) #0 {
   ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GCN:   %6:vgpr_32, %7:sreg_64 = nofpexcept V_DIV_SCALE_F32 [[COPY2]], [[COPY1]], [[COPY2]], implicit $mode, implicit $exec
   ; GCN:   %8:vgpr_32, %9:sreg_64 = nofpexcept V_DIV_SCALE_F32 [[COPY1]], [[COPY1]], [[COPY2]], implicit $mode, implicit $exec
   ; GCN:   %10:vgpr_32 = nofpexcept V_RCP_F32_e64 0, %8, 0, 0, implicit $mode, implicit $exec
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 3
   ; GCN:   [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 1065353216
   ; GCN:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GCN:   S_SETREG_B32 killed [[S_MOV_B32_]], 2305, implicit-def $mode, implicit $mode
+  ; GCN:   S_SETREG_B32_mode killed [[S_MOV_B32_]], 2305, implicit-def $mode, implicit $mode
   ; GCN:   %14:vgpr_32 = nofpexcept V_FMA_F32 1, %8, 0, %10, 0, killed [[S_MOV_B32_1]], 0, 0, implicit $mode, implicit $exec
   ; GCN:   %15:vgpr_32 = nofpexcept V_FMA_F32 0, killed %14, 0, %10, 0, %10, 0, 0, implicit $mode, implicit $exec
   ; GCN:   %16:vgpr_32 = nofpexcept V_MUL_F32_e64 0, %6, 0, %15, 0, 0, implicit $mode, implicit $exec
   ; GCN:   %17:vgpr_32 = nofpexcept V_FMA_F32 1, %8, 0, %16, 0, %6, 0, 0, implicit $mode, implicit $exec
   ; GCN:   %18:vgpr_32 = nofpexcept V_FMA_F32 0, killed %17, 0, %15, 0, %16, 0, 0, implicit $mode, implicit $exec
   ; GCN:   %19:vgpr_32 = nofpexcept V_FMA_F32 1, %8, 0, %18, 0, %6, 0, 0, implicit $mode, implicit $exec
-  ; GCN:   S_SETREG_B32 killed [[S_MOV_B32_2]], 2305, implicit-def dead $mode, implicit $mode
+  ; GCN:   S_SETREG_B32_mode killed [[S_MOV_B32_2]], 2305, implicit-def dead $mode, implicit $mode
   ; GCN:   $vcc = COPY %7
   ; GCN:   %20:vgpr_32 = nofpexcept V_DIV_FMAS_F32 0, killed %19, 0, %15, 0, %18, 0, 0, implicit $mode, implicit $vcc, implicit $exec
   ; GCN:   %21:vgpr_32 = nofpexcept V_DIV_FIXUP_F32 0, killed %20, 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $mode, implicit $exec
   ; GCN:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
   ; GCN:   $vgpr0 = COPY %21
   ; GCN:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY3]]
   ; GCN:   S_SETPC_B64_return [[COPY4]], implicit $vgpr0
 entry:
define float @fdiv_nnan_f32(float %a, float %b) #0 {
   ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
   ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
   ; GCN:   %6:vgpr_32, %7:sreg_64 = nnan nofpexcept V_DIV_SCALE_F32 [[COPY2]], [[COPY1]], [[COPY2]], implicit $mode, implicit $exec
   ; GCN:   %8:vgpr_32, %9:sreg_64 = nnan nofpexcept V_DIV_SCALE_F32 [[COPY1]], [[COPY1]], [[COPY2]], implicit $mode, implicit $exec
   ; GCN:   %10:vgpr_32 = nnan nofpexcept V_RCP_F32_e64 0, %8, 0, 0, implicit $mode, implicit $exec
   ; GCN:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 3
   ; GCN:   [[S_MOV_B32_1:%[0-9]+]]:sgpr_32 = S_MOV_B32 1065353216
   ; GCN:   [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
-  ; GCN:   S_SETREG_B32 killed [[S_MOV_B32_]], 2305, implicit-def $mode, implicit $mode
+  ; GCN:   S_SETREG_B32_mode killed [[S_MOV_B32_]], 2305, implicit-def $mode, implicit $mode
   ; GCN:   %14:vgpr_32 = nnan nofpexcept V_FMA_F32 1, %8, 0, %10, 0, killed [[S_MOV_B32_1]], 0, 0, implicit $mode, implicit $exec
   ; GCN:   %15:vgpr_32 = nnan nofpexcept V_FMA_F32 0, killed %14, 0, %10, 0, %10, 0, 0, implicit $mode, implicit $exec
   ; GCN:   %16:vgpr_32 = nnan nofpexcept V_MUL_F32_e64 0, %6, 0, %15, 0, 0, implicit $mode, implicit $exec
   ; GCN:   %17:vgpr_32 = nnan nofpexcept V_FMA_F32 1, %8, 0, %16, 0, %6, 0, 0, implicit $mode, implicit $exec
   ; GCN:   %18:vgpr_32 = nnan nofpexcept V_FMA_F32 0, killed %17, 0, %15, 0, %16, 0, 0, implicit $mode, implicit $exec
   ; GCN:   %19:vgpr_32 = nnan nofpexcept V_FMA_F32 1, %8, 0, %18, 0, %6, 0, 0, implicit $mode, implicit $exec
-  ; GCN:   S_SETREG_B32 killed [[S_MOV_B32_2]], 2305, implicit-def dead $mode, implicit $mode
+  ; GCN:   S_SETREG_B32_mode killed [[S_MOV_B32_2]], 2305, implicit-def dead $mode, implicit $mode
   ; GCN:   $vcc = COPY %7
   ; GCN:   %20:vgpr_32 = nnan nofpexcept V_DIV_FMAS_F32 0, killed %19, 0, %15, 0, %18, 0, 0, implicit $mode, implicit $vcc, implicit $exec
   ; GCN:   %21:vgpr_32 = nnan nofpexcept V_DIV_FIXUP_F32 0, killed %20, 0, [[COPY1]], 0, [[COPY2]], 0, 0, implicit $mode, implicit $exec
   ; GCN:   [[COPY3:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY]]
   ; GCN:   $vgpr0 = COPY %21
   ; GCN:   [[COPY4:%[0-9]+]]:ccr_sgpr_64 = COPY [[COPY3]]
   ; GCN:   S_SETPC_B64_return [[COPY4]], implicit $vgpr0
 entry:
   %fdiv = fdiv nnan float %a, %b
   ret float %fdiv
 }
 
 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" }

llvm/test/CodeGen/AMDGPU/frem.ll

 ; CI-NEXT:    v_fma_f32 v7, v8, v6, v7
 ; CI-NEXT:    v_fma_f32 v4, -v5, v7, v4
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; CI-NEXT:    v_div_fmas_f32 v4, v4, v6, v7
 ; CI-NEXT:    v_div_fixup_f32 v4, v4, v2, v0
 ; CI-NEXT:    v_trunc_f32_e32 v4, v4
 ; CI-NEXT:    v_fma_f32 v0, -v4, v2, v0
 ; CI-NEXT:    v_div_scale_f32 v4, s[4:5], v3, v3, v1
+; CI-NEXT:    v_div_scale_f32 v2, vcc, v1, v3, v1
 ; CI-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
 ; CI-NEXT:    v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT:    v_div_scale_f32 v2, vcc, v1, v3, v1
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; CI-NEXT:    v_rcp_f32_e32 v5, v4
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; CI-NEXT:    v_fma_f32 v6, -v4, v5, 1.0
 ; CI-NEXT:    v_fma_f32 v5, v6, v5, v5
 ; CI-NEXT:    v_mul_f32_e32 v6, v2, v5
 ; CI-NEXT:    v_fma_f32 v7, -v4, v6, v2
 ; CI-NEXT:    v_fma_f32 v6, v7, v5, v6

 ; CI-NEXT:    v_fma_f32 v11, v12, v10, v11
 ; CI-NEXT:    v_fma_f32 v8, -v9, v11, v8
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; CI-NEXT:    v_div_fmas_f32 v8, v8, v10, v11
 ; CI-NEXT:    v_div_fixup_f32 v8, v8, v1, v5
 ; CI-NEXT:    v_trunc_f32_e32 v8, v8
 ; CI-NEXT:    v_fma_f32 v1, -v8, v1, v5
 ; CI-NEXT:    v_div_scale_f32 v8, s[4:5], v7, v7, v4
+; CI-NEXT:    v_div_scale_f32 v5, vcc, v4, v7, v4
 ; CI-NEXT:    s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
 ; CI-NEXT:    v_cvt_f16_f32_e32 v1, v1
-; CI-NEXT:    v_div_scale_f32 v5, vcc, v4, v7, v4
 ; CI-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
 ; CI-NEXT:    v_rcp_f32_e32 v9, v8
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; CI-NEXT:    v_fma_f32 v10, -v8, v9, 1.0
 ; CI-NEXT:    v_fma_f32 v9, v10, v9, v9
 ; CI-NEXT:    v_mul_f32_e32 v10, v5, v9
 ; CI-NEXT:    v_fma_f32 v11, -v8, v10, v5
 ; CI-NEXT:    v_fma_f32 v10, v11, v9, v10

 ; CI-NEXT:    v_fma_f32 v8, v9, v7, v8
 ; CI-NEXT:    v_fma_f32 v4, -v5, v8, v4
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s7
 ; CI-NEXT:    v_div_fmas_f32 v4, v4, v7, v8
 ; CI-NEXT:    v_div_fixup_f32 v4, v4, v0, v3
 ; CI-NEXT:    v_trunc_f32_e32 v4, v4
 ; CI-NEXT:    v_fma_f32 v0, -v4, v0, v3
 ; CI-NEXT:    v_div_scale_f32 v4, s[4:5], v6, v6, v2
-; CI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; CI-NEXT:    v_div_scale_f32 v3, vcc, v2, v6, v2
+; CI-NEXT:    v_cvt_f16_f32_e32 v0, v0
 ; CI-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
 ; CI-NEXT:    v_rcp_f32_e32 v5, v4
 ; CI-NEXT:    s_setreg_b32 hwreg(HW_REG_MODE, 4, 2), s6
 ; CI-NEXT:    v_fma_f32 v7, -v4, v5, 1.0
 ; CI-NEXT:    v_fma_f32 v5, v7, v5, v5
 ; CI-NEXT:    v_mul_f32_e32 v7, v3, v5
 ; CI-NEXT:    v_fma_f32 v8, -v4, v7, v3
 ; CI-NEXT:    v_fma_f32 v7, v8, v5, v7