[AMDGPU] Introduce Strict WQM mode

[piotr]

Summary

• Add amdgcn_strict_wqm intrinsic.
• Add a corresponding STRICT_WQM machine instruction.
• The semantic is similar to amdgcn_strict_wwm with a notable difference that not all threads will be forcibly enabled during the computations of the intrinsic's argument, but only all threads in quads that have at least one thread active.
• The difference between amdgc_wqm and amdgcn_strict_wqm, is that in the strict mode an inactive lane will always be enabled irrespective of control flow decisions.

[critson]

llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp

14

I am not sure if this is really the clearest description of strict mode and control flow.
Technically strict modes are disabled and reenabled through control flow.
Both strict and non-strict modes "respect" control-flow, but with different semantics.
Perhaps something like:
The "strict" prefix indicates that inactive lanes do not take part in control flow, specifically an inactive lane enabled by a strict WQM/WWM will always be enabled irrespective of control flow decisions.
Conversely in non-strict WQM inactive lanes may control flow decisions.

203

I wonder if these should now just be called toStrictMode and fromStrictMode.

llvm/test/CodeGen/AMDGPU/wqm.ll

422

It's not clear to me how this test proves the "coalesced" part?

489

Might be worth clarify that this exit is in the %if branch.

524

Same comment as previous test.

[piotr]

llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp

14

Thanks, I will incorporate your description in the code.

203

Agreed, that would be clearer - I will rename it.

llvm/test/CodeGen/AMDGPU/wqm.ll

422

I patterned this test on the existing one for WWM (test_strictwwm4), which follows the same structure. It seems the comment is wrong.

[piotr]

Rebased, changed strictwqm to strict_wqm and updated tests.

[piotr]

llvm/test/CodeGen/AMDGPU/wqm.ll

422

I looked closely at the original test and I believe the check for "endif" was missing. These two cases make sure that a WWM feeding a phi node is handled properly and the mov/add happens in the same block as WWM computations.

[arsenm]

llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp

180

Printing MI already includes the newline

llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp

107

static const, const char*?

[piotr]

Fixed newline and added static/const.

[critson]

LGTM

llvm/include/llvm/IR/IntrinsicsAMDGPU.td

def int_amdgcn_strictwwm : Intrinsic<[llvm_any_ty],
   [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable,
                        IntrConvergent, IntrWillReturn]
 >;
 // Deprecated. Use int_amdgcn_strictwwm instead.
 def int_amdgcn_wwm : Intrinsic<[llvm_any_ty],
   [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable,
                        IntrConvergent, IntrWillReturn]
 >;
+def int_amdgcn_strictwqm : Intrinsic<[llvm_any_ty],
+  [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable,
+                       IntrConvergent, IntrWillReturn]
+>;
 
 // Given a value, copies it while setting all the inactive lanes to a given
 // value. Note that OpenGL helper lanes are considered active, so if the
 // program ever uses WQM, then the instruction and the first source will be
 // computed in WQM.
 def int_amdgcn_set_inactive :
   Intrinsic<[llvm_anyint_ty],
             [LLVMMatchType<0>, // value to be copied

llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp

case Intrinsic::amdgcn_wqm:
     break;
   case Intrinsic::amdgcn_softwqm:
     Opcode = AMDGPU::SOFT_WQM;
     break;
   case Intrinsic::amdgcn_wwm:
   case Intrinsic::amdgcn_strictwwm:
     Opcode = AMDGPU::STRICTWWM;
     break;
+  case Intrinsic::amdgcn_strictwqm:
+    Opcode = AMDGPU::STRICTWQM;
+    break;
   case Intrinsic::amdgcn_interp_p1_f16:
     SelectInterpP1F16(N);
     return;
   default:
     SelectCode(N);
     return;
   }
 

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

case Intrinsic::amdgcn_interp_p1_f16:
     return selectInterpP1F16(I);
   case Intrinsic::amdgcn_wqm:
     return constrainCopyLikeIntrin(I, AMDGPU::WQM);
   case Intrinsic::amdgcn_softwqm:
     return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM);
   case Intrinsic::amdgcn_strictwwm:
   case Intrinsic::amdgcn_wwm:
     return constrainCopyLikeIntrin(I, AMDGPU::STRICTWWM);
+  case Intrinsic::amdgcn_strictwqm:
+    return constrainCopyLikeIntrin(I, AMDGPU::STRICTWQM);
   case Intrinsic::amdgcn_writelane:
     return selectWritelane(I);
   case Intrinsic::amdgcn_div_scale:
     return selectDivScale(I);
   case Intrinsic::amdgcn_icmp:
     return selectIntrinsicIcmp(I);
   case Intrinsic::amdgcn_ballot:
     return selectBallot(I);

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

case AMDGPU::G_INTRINSIC: {
     case Intrinsic::amdgcn_ds_swizzle:
     case Intrinsic::amdgcn_ds_permute:
     case Intrinsic::amdgcn_ds_bpermute:
     case Intrinsic::amdgcn_update_dpp:
     case Intrinsic::amdgcn_mov_dpp8:
     case Intrinsic::amdgcn_mov_dpp:
     case Intrinsic::amdgcn_strictwwm:
     case Intrinsic::amdgcn_wwm:
+    case Intrinsic::amdgcn_strictwqm:
     case Intrinsic::amdgcn_wqm:
     case Intrinsic::amdgcn_softwqm:
     case Intrinsic::amdgcn_set_inactive:
       return getDefaultMappingAllVGPR(MI);
     case Intrinsic::amdgcn_kernarg_segment_ptr:
     case Intrinsic::amdgcn_s_getpc:
     case Intrinsic::amdgcn_groupstaticsize:
     case Intrinsic::amdgcn_reloc_constant:

llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp

for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;
          ++I) {
       MachineInstr &MI = *I;
 
       switch (MI.getOpcode()) {
       default:
         continue;
       case AMDGPU::COPY:
       case AMDGPU::WQM:
+      case AMDGPU::STRICTWQM:
       case AMDGPU::SOFT_WQM:
       case AMDGPU::STRICTWWM: {
         Register DstReg = MI.getOperand(0).getReg();
 
         const TargetRegisterClass *SrcRC, *DstRC;
         std::tie(SrcRC, DstRC) = getCopyRegClasses(MI, *TRI, *MRI);
 
         if (!DstReg.isVirtual()) {

llvm/lib/Target/AMDGPU/SIInstrInfo.cpp

 //===- SIInstrInfo.cpp - SI Instruction Information  ----------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file
bool SIInstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
   }
   case AMDGPU::ENTER_STRICTWWM: {
     // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
     // Whole Wave Mode is entered.
     MI.setDesc(get(ST.isWave32() ? AMDGPU::S_OR_SAVEEXEC_B32
                                  : AMDGPU::S_OR_SAVEEXEC_B64));
     break;
   }
-  case AMDGPU::EXIT_STRICTWWM: {
+  case AMDGPU::ENTER_STRICTWQM: {
     // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
-    // Whole Wave Mode is exited.
+    // STRICTWQM is entered.
+    const unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
+    const unsigned WQMOp = ST.isWave32() ? AMDGPU::S_WQM_B32 : AMDGPU::S_WQM_B64;
+    const unsigned MovOp = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
+    BuildMI(MBB, MI, DL, get(MovOp), MI.getOperand(0).getReg()).addReg(Exec);
+    BuildMI(MBB, MI, DL, get(WQMOp), Exec).addReg(Exec);
+
+    MI.eraseFromParent();
+    break;
+  }
+  case AMDGPU::EXIT_STRICTWWM:
+  case AMDGPU::EXIT_STRICTWQM: {
+    // This only gets its own opcode so that SIPreAllocateWWMRegs can tell when
+    // WWM/STICT_WQM is exited.
     MI.setDesc(get(ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64));
     break;
   }
   }
   return true;
 }
 
 std::pair<MachineInstr*, MachineInstr*>
unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
   default: return AMDGPU::INSTRUCTION_LIST_END;
   case AMDGPU::REG_SEQUENCE: return AMDGPU::REG_SEQUENCE;
   case AMDGPU::COPY: return AMDGPU::COPY;
   case AMDGPU::PHI: return AMDGPU::PHI;
   case AMDGPU::INSERT_SUBREG: return AMDGPU::INSERT_SUBREG;
   case AMDGPU::WQM: return AMDGPU::WQM;
   case AMDGPU::SOFT_WQM: return AMDGPU::SOFT_WQM;
   case AMDGPU::STRICTWWM: return AMDGPU::STRICTWWM;
+  case AMDGPU::STRICTWQM: return AMDGPU::STRICTWQM;
   case AMDGPU::S_MOV_B32: {
     const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
     return MI.getOperand(1).isReg() ||
            RI.isAGPR(MRI, MI.getOperand(0).getReg()) ?
            AMDGPU::COPY : AMDGPU::V_MOV_B32_e32;
   }
   case AMDGPU::S_ADD_I32:
     return ST.hasAddNoCarry() ? AMDGPU::V_ADD_U32_e64 : AMDGPU::V_ADD_CO_U32_e32;
for (MachineRegisterInfo::use_iterator I = MRI.use_begin(DstReg),
 
     unsigned OpNo = 0;
 
     switch (UseMI.getOpcode()) {
     case AMDGPU::COPY:
     case AMDGPU::WQM:
     case AMDGPU::SOFT_WQM:
     case AMDGPU::STRICTWWM:
+    case AMDGPU::STRICTWQM:
     case AMDGPU::REG_SEQUENCE:
     case AMDGPU::PHI:
     case AMDGPU::INSERT_SUBREG:
       break;
     default:
       OpNo = I.getOperandNo();
       break;
     }
const TargetRegisterClass *SIInstrInfo::getDestEquivalentVGPRClass(
   // class associated with the operand, so we need to find an equivalent VGPR
   // register class in order to move the instruction to the VALU.
   case AMDGPU::COPY:
   case AMDGPU::PHI:
   case AMDGPU::REG_SEQUENCE:
   case AMDGPU::INSERT_SUBREG:
   case AMDGPU::WQM:
   case AMDGPU::SOFT_WQM:
-  case AMDGPU::STRICTWWM: {
+  case AMDGPU::STRICTWWM:
+  case AMDGPU::STRICTWQM: {
     const TargetRegisterClass *SrcRC = getOpRegClass(Inst, 1);
     if (RI.hasAGPRs(SrcRC)) {
       if (RI.hasAGPRs(NewDstRC))
         return nullptr;
 
       switch (Inst.getOpcode()) {
       case AMDGPU::PHI:
       case AMDGPU::REG_SEQUENCE:

llvm/lib/Target/AMDGPU/SIInstructions.td

 // Pseudoinstruction for @llvm.amdgcn.wqm. It is turned into a copy after the
 // WQM pass processes it.
 def WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>;
 
 // Pseudoinstruction for @llvm.amdgcn.softwqm. Like @llvm.amdgcn.wqm it is
 // turned into a copy by WQM pass, but does not seed WQM requirements.
 def SOFT_WQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>;
 
-// Pseudoinstruction for @llvm.amdgcn.strictwwm. It is turned into a copy post-RA, so
-// that the @earlyclobber is respected. The @earlyclobber is to make sure that
-// the instruction that defines $src0 (which is run in Whole Wave Mode) doesn't
+// Pseudoinstructions for @llvm.amdgcn.strictwwm and @llvm.amdgcn.strictwqm.
+// It is turned into a copy post-RA, so that the @earlyclobber is respected.
+// The @earlyclobber is to make sure that the instruction that defines $src0
+// (which is run in Strict Whole Wave Mode or Strict Whole Quad Mode) doesn't
 // accidentally clobber inactive channels of $vdst.
 let Constraints = "@earlyclobber $vdst" in {
 def STRICTWWM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>;
+def STRICTWQM : PseudoInstSI <(outs unknown:$vdst), (ins unknown:$src0)>;
 }
 
 } // End let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Uses = [EXEC]
 
 def ENTER_STRICTWWM : SPseudoInstSI <(outs SReg_1:$sdst), (ins i64imm:$src0)> {
   let Uses = [EXEC];
   let Defs = [EXEC, SCC];
   let hasSideEffects = 0;
   let mayLoad = 0;
   let mayStore = 0;
 }
 
 def EXIT_STRICTWWM : SPseudoInstSI <(outs SReg_1:$sdst), (ins SReg_1:$src0)> {
   let hasSideEffects = 0;
   let mayLoad = 0;
   let mayStore = 0;
 }
 
+def ENTER_STRICTWQM : SPseudoInstSI <(outs SReg_1:$sdst), (ins i64imm:$src0)> {
+  let Uses = [EXEC];
+  let Defs = [EXEC, SCC];
+  let hasSideEffects = 0;
+  let mayLoad = 0;
+  let mayStore = 0;
+}
+
+def EXIT_STRICTWQM : SPseudoInstSI <(outs SReg_1:$sdst), (ins SReg_1:$src0)> {
+  let hasSideEffects = 0;
+  let mayLoad = 0;
+  let mayStore = 0;
+}
+
 // Invert the exec mask and overwrite the inactive lanes of dst with inactive,
 // restoring it after we're done.
 let Defs = [SCC] in {
 def V_SET_INACTIVE_B32 : VPseudoInstSI <(outs VGPR_32:$vdst),
   (ins VGPR_32: $src, VSrc_b32:$inactive),
   [(set i32:$vdst, (int_amdgcn_set_inactive i32:$src, i32:$inactive))]> {
   let Constraints = "$src = $vdst";
 }

llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp

private:
   const SIRegisterInfo *TRI;
   MachineRegisterInfo *MRI;
   LiveIntervals *LIS;
   LiveRegMatrix *Matrix;
   VirtRegMap *VRM;
   RegisterClassInfo RegClassInfo;
 
   std::vector<unsigned> RegsToRewrite;
+#ifndef NDEBUG
+  void printWWMInfo(const MachineInstr &MI);
+#endif
 
 public:
   static char ID;
 
   SIPreAllocateWWMRegs() : MachineFunctionPass(ID) {
     initializeSIPreAllocateWWMRegsPass(*PassRegistry::getPassRegistry());
   }
 
void SIPreAllocateWWMRegs::rewriteRegs(MachineFunction &MF) {
   }
 
   RegsToRewrite.clear();
 
   // Update the set of reserved registers to include WWM ones.
   MRI->freezeReservedRegs(MF);
 }
 
+#ifndef NDEBUG
+LLVM_DUMP_METHOD void
+SIPreAllocateWWMRegs::printWWMInfo(const MachineInstr &MI) {
+
+  unsigned Opc = MI.getOpcode();
+
+  if (Opc == AMDGPU::ENTER_STRICTWWM || Opc == AMDGPU::ENTER_STRICTWQM) {
+    dbgs() << "Entering ";
+  } else {
+    assert(Opc == AMDGPU::EXIT_STRICTWWM || Opc == AMDGPU::EXIT_STRICTWQM);
+    dbgs() << "Exiting ";
+  }
+
+  if (Opc == AMDGPU::ENTER_STRICTWWM || Opc == AMDGPU::EXIT_STRICTWWM) {
+    dbgs() << "Strict WWM ";
+  } else {
+    assert(Opc == AMDGPU::ENTER_STRICTWQM || Opc == AMDGPU::EXIT_STRICTWQM);
+    dbgs() << "Strict WQM ";
+  }
+
+  dbgs() << "region: " << MI << "\n";
+}
+
+#endif
+
 bool SIPreAllocateWWMRegs::runOnMachineFunction(MachineFunction &MF) {
   LLVM_DEBUG(dbgs() << "SIPreAllocateWWMRegs: function " << MF.getName() << "\n");
 
   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 
   TII = ST.getInstrInfo();
   TRI = &TII->getRegisterInfo();
   MRI = &MF.getRegInfo();
bool SIPreAllocateWWMRegs::runOnMachineFunction(MachineFunction &MF) {
 
   for (MachineBasicBlock *MBB : RPOT) {
     bool InWWM = false;
     for (MachineInstr &MI : *MBB) {
       if (MI.getOpcode() == AMDGPU::V_SET_INACTIVE_B32 ||
           MI.getOpcode() == AMDGPU::V_SET_INACTIVE_B64)
         RegsAssigned |= processDef(MI.getOperand(0));
 
-      if (MI.getOpcode() == AMDGPU::ENTER_STRICTWWM) {
-        LLVM_DEBUG(dbgs() << "entering WWM region: " << MI << "\n");
+      if (MI.getOpcode() == AMDGPU::ENTER_STRICTWWM ||
+          MI.getOpcode() == AMDGPU::ENTER_STRICTWQM) {
+        LLVM_DEBUG(printWWMInfo(MI));
         InWWM = true;
         continue;
       }
 
-      if (MI.getOpcode() == AMDGPU::EXIT_STRICTWWM) {
-        LLVM_DEBUG(dbgs() << "exiting WWM region: " << MI << "\n");
+      if (MI.getOpcode() == AMDGPU::EXIT_STRICTWWM ||
+          MI.getOpcode() == AMDGPU::EXIT_STRICTWQM) {
+        LLVM_DEBUG(printWWMInfo(MI));
         InWWM = false;
       }
 
       if (!InWWM)
         continue;
 
-      LLVM_DEBUG(dbgs() << "processing " << MI << "\n");
+      LLVM_DEBUG(dbgs() << "Processing " << MI << "\n");
 
       for (MachineOperand &DefOpnd : MI.defs()) {
         RegsAssigned |= processDef(DefOpnd);
       }
     }
   }
 
   if (!RegsAssigned)
     return false;
 
   rewriteRegs(MF);
   return true;
 }

llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp

 //===-- SIWholeQuadMode.cpp - enter and suspend whole quad mode -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 /// \file
-/// This pass adds instructions to enable whole quad mode for pixel
-/// shaders, and whole wavefront mode for all programs.
+/// This pass adds instructions to enable whole quad mode (strict or non-strict)
+/// for pixel shaders, and strict whole wavefront mode for all programs.
+///
+/// The "strict" prefix denotes that the inactive lanes are enabled irrespective
+/// of the control flow.
 ///
 /// Whole quad mode is required for derivative computations, but it interferes
 /// with shader side effects (stores and atomics). It ensures that WQM is
 /// enabled when necessary, but disabled around stores and atomics.
 ///
 /// When necessary, this pass creates a function prolog
 ///
 ///   S_MOV_B64 LiveMask, EXEC
 ///   S_WQM_B64 EXEC, EXEC
 ///
 /// to enter WQM at the top of the function and surrounds blocks of Exact
 /// instructions by
 ///
 ///   S_AND_SAVEEXEC_B64 Tmp, LiveMask
 ///   ...
 ///   S_MOV_B64 EXEC, Tmp
 ///
-/// We also compute when a sequence of instructions requires Whole Wavefront
-/// Mode (StrictWWM) and insert instructions to save and restore it:
+/// We also compute when a sequence of instructions requires strict whole
+/// wavefront mode (StrictWWM) and insert instructions to save and restore it:
 ///
-/// S_OR_SAVEEXEC_B64 Tmp, -1
-/// ...
-/// S_MOV_B64 EXEC, Tmp
+///   S_OR_SAVEEXEC_B64 Tmp, -1
+///   ...
+///   S_MOV_B64 EXEC, Tmp
 ///
+/// When a sequence of instructions requires strict whole quad mode (StrictWQM)
+/// we use a similar save and restore mechanism and force whole quad mode for
+/// those instructions:
+///
+///  S_MOV_B64 Tmp, EXEC
+///  S_WQM_B64 EXEC, EXEC
+///  ...
+///  S_MOV_B64 EXEC, Tmp
+///
 /// In order to avoid excessive switching during sequences of Exact
 /// instructions, the pass first analyzes which instructions must be run in WQM
 /// (aka which instructions produce values that lead to derivative
 /// computations).
 ///
 /// Basic blocks are always exited in WQM as long as some successor needs WQM.
 ///
 /// There is room for improvement given better control flow analysis:

 
 #define DEBUG_TYPE "si-wqm"
 
 namespace {
 
 enum {
   StateWQM = 0x1,
   StateStrictWWM = 0x2,
-  StateExact = 0x4,
+  StateStrictWQM = 0x4,
+  StateExact = 0x8,
+  StateStrict = StateStrictWWM | StateStrictWQM,
 };
 
 struct PrintState {
 public:
   int State;
 
   explicit PrintState(int State) : State(State) {}
 };
 
 #ifndef NDEBUG
 static raw_ostream &operator<<(raw_ostream &OS, const PrintState &PS) {
-  if (PS.State & StateWQM)
-    OS << "WQM";
-  if (PS.State & StateStrictWWM) {
-    if (PS.State & StateWQM)
+
+  std::pair<char, std::string> Mapping[] = {
+      std::make_pair(StateWQM, "WQM"),
+      std::make_pair(StateStrictWWM, "StrictWWM"),
+      std::make_pair(StateStrictWQM, "StrictWQM"),
+      std::make_pair(StateExact, "Exact")};
+  char State = PS.State;
+  for (auto M : Mapping) {
+    if (State & M.first) {
+      OS << M.second;
+      State &= ~M.first;
+
+      if (State)
         OS << '|';
-    OS << "StrictWWM";
     }
-  if (PS.State & StateExact) {
-    if (PS.State & (StateWQM | StateStrictWWM))
-      OS << '|';
-    OS << "Exact";
   }
-
+  assert(State == 0);
   return OS;
 }
 #endif
 
 struct InstrInfo {
   char Needs = 0;
   char Disabled = 0;
   char OutNeeds = 0;
private:
   unsigned OrSaveExecOpc;
   unsigned WQMOpc;
   Register Exec;
   Register LiveMaskReg;
 
   DenseMap<const MachineInstr *, InstrInfo> Instructions;
   MapVector<MachineBasicBlock *, BlockInfo> Blocks;
 
-  // Tracks state (WQM/StrictWWM/Exact) after a given instruction
+  // Tracks state (WQM/StrictWWM/StrictWQM/Exact) after a given instruction
   DenseMap<const MachineInstr *, char> StateTransition;
 
   SmallVector<MachineInstr *, 2> LiveMaskQueries;
   SmallVector<MachineInstr *, 4> LowerToMovInstrs;
   SmallVector<MachineInstr *, 4> LowerToCopyInstrs;
   SmallVector<MachineInstr *, 4> KillInstrs;
 
   void printInfo();
private:
   MachineBasicBlock::iterator
   prepareInsertion(MachineBasicBlock &MBB, MachineBasicBlock::iterator First,
                    MachineBasicBlock::iterator Last, bool PreferLast,
                    bool SaveSCC);
   void toExact(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
                Register SaveWQM);
   void toWQM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
              Register SavedWQM);
   void toStrictWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
-             Register SaveOrig);
+                   Register SaveOrig, char StrictStateNeeded);
   void fromStrictWWM(MachineBasicBlock &MBB, MachineBasicBlock::iterator Before,
-               Register SavedOrig, char NonStrictWWMState);
+                     Register SavedOrig, char NonStrictState,
+                     char CurrentStrictState);
 
   MachineBasicBlock *splitBlock(MachineBasicBlock *BB, MachineInstr *TermMI);
 
   MachineInstr *lowerKillI1(MachineBasicBlock &MBB, MachineInstr &MI,
                             bool IsWQM);
   MachineInstr *lowerKillF32(MachineBasicBlock &MBB, MachineInstr &MI);
 
   void lowerBlock(MachineBasicBlock &MBB);
for (auto II = MBB.begin(), IE = MBB.end(); II != IE; ++II) {
         // correct, so we need it to be in WQM.
         Flags = StateWQM;
         LowerToCopyInstrs.push_back(&MI);
       } else if (Opcode == AMDGPU::SOFT_WQM) {
         LowerToCopyInstrs.push_back(&MI);
         SoftWQMInstrs.push_back(&MI);
         continue;
       } else if (Opcode == AMDGPU::STRICTWWM) {
-        // The STRICTWWM intrinsic doesn't make the same guarantee, and plus
-        // it needs to be executed in WQM or Exact so that its copy doesn't clobber
+        // The STRICTWWM intrinsic doesn't make the same guarantee, and plus it
+        // needs to be executed in WQM or Exact so that its copy doesn't clobber
         // inactive lanes.
         markInstructionUses(MI, StateStrictWWM, Worklist);
         GlobalFlags |= StateStrictWWM;
         LowerToMovInstrs.push_back(&MI);
         continue;
+      } else if (Opcode == AMDGPU::STRICTWQM) {
+        // STRICTWQM is similar to STRICTWWM, but instead of enabling all
+        // threads of the wave like STRICTWWM, STRICTWQM enables all threads in
+        // quads that have at least one active thread.
+        markInstructionUses(MI, StateStrictWQM, Worklist);
+        GlobalFlags |= StateStrictWQM;
+        LowerToMovInstrs.push_back(&MI);
+        continue;
       } else if (Opcode == AMDGPU::V_SET_INACTIVE_B32 ||
                  Opcode == AMDGPU::V_SET_INACTIVE_B64) {
-        III.Disabled = StateStrictWWM;
+        III.Disabled = StateStrict;
         MachineOperand &Inactive = MI.getOperand(2);
         if (Inactive.isReg()) {
           if (Inactive.isUndef()) {
             LowerToCopyInstrs.push_back(&MI);
           } else {
             Register Reg = Inactive.getReg();
             if (Reg.isVirtual()) {
               for (MachineInstr &DefMI : MRI->def_instructions(Reg))
-                markInstruction(DefMI, StateStrictWWM, Worklist);
+                markInstruction(DefMI, StateStrict, Worklist);
             }
           }
         }
         SetInactiveInstrs.push_back(&MI);
         continue;
       } else if (TII->isDisableWQM(MI)) {
         BBI.Needs |= StateExact;
         if (!(BBI.InNeeds & StateExact)) {
           BBI.InNeeds |= StateExact;
           Worklist.push_back(&MBB);
         }
         GlobalFlags |= StateExact;
-        III.Disabled = StateWQM | StateStrictWWM;
+        III.Disabled = StateWQM | StateStrict;
         continue;
       } else {
         if (Opcode == AMDGPU::SI_PS_LIVE || Opcode == AMDGPU::SI_LIVE_MASK) {
           LiveMaskQueries.push_back(&MI);
         } else if (Opcode == AMDGPU::SI_KILL_I1_TERMINATOR ||
                    Opcode == AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR ||
                    Opcode == AMDGPU::SI_DEMOTE_I1) {
           KillInstrs.push_back(&MI);
if (II.Needs & StateWQM) {
     if (!(BI.InNeeds & StateWQM)) {
       BI.InNeeds |= StateWQM;
       Worklist.push_back(MBB);
     }
   }
 
   // Propagate backwards within block
   if (MachineInstr *PrevMI = MI.getPrevNode()) {
-    char InNeeds = (II.Needs & ~StateStrictWWM) | II.OutNeeds;
+    char InNeeds = (II.Needs & ~StateStrict) | II.OutNeeds;
     if (!PrevMI->isPHI()) {
       InstrInfo &PrevII = Instructions[PrevMI];
       if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
         PrevII.OutNeeds |= InNeeds;
         Worklist.push_back(PrevMI);
       }
     }
   }
 
   // Propagate WQM flag to instruction inputs
   assert(!(II.Needs & StateExact));
 
   if (II.Needs != 0)
     markInstructionUses(MI, II.Needs, Worklist);
 
-  // Ensure we process a block containing StrictWWM, even if it does not require any
-  // WQM transitions.
+  // Ensure we process a block containing StrictWWM/StrictWQM, even if it does
+  // not require any WQM transitions.
   if (II.Needs & StateStrictWWM)
     BI.Needs |= StateStrictWWM;
+  if (II.Needs & StateStrictWQM)
+    BI.Needs |= StateStrictWQM;
 }
 
 void SIWholeQuadMode::propagateBlock(MachineBasicBlock &MBB,
                                      std::vector<WorkItem>& Worklist) {
   BlockInfo BI = Blocks[&MBB]; // Make a copy to prevent dangling references.
 
   // Propagate through instructions
   if (!MBB.empty()) {
if (SavedWQM) {
     MI = BuildMI(MBB, Before, DebugLoc(), TII->get(WQMOpc), Exec).addReg(Exec);
   }
 
   LIS->InsertMachineInstrInMaps(*MI);
   StateTransition[MI] = StateWQM;
 }
 
 void SIWholeQuadMode::toStrictWWM(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator Before,
-                            Register SaveOrig) {
+                                  Register SaveOrig, char StrictStateNeeded) {
   MachineInstr *MI;
-
   assert(SaveOrig);
-  MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICTWWM), SaveOrig)
+  assert(StrictStateNeeded == StateStrictWWM ||
+         StrictStateNeeded == StateStrictWQM);
+
+  if (StrictStateNeeded == StateStrictWWM) {
+    MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICTWWM),
+                 SaveOrig)
+             .addImm(-1);
+  } else {
+    MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::ENTER_STRICTWQM),
+                 SaveOrig)
              .addImm(-1);
+  }
   LIS->InsertMachineInstrInMaps(*MI);
   StateTransition[MI] = StateStrictWWM;
 }
 
 void SIWholeQuadMode::fromStrictWWM(MachineBasicBlock &MBB,
                                     MachineBasicBlock::iterator Before,
-                              Register SavedOrig, char NonStrictWWMState) {
+                                    Register SavedOrig, char NonStrictState,
+                                    char CurrentStrictState) {
   MachineInstr *MI;
 
   assert(SavedOrig);
-  MI = BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICTWWM), Exec)
+  assert(CurrentStrictState == StateStrictWWM ||
+         CurrentStrictState == StateStrictWQM);
+
+  if (CurrentStrictState == StateStrictWWM) {
+    MI =
+        BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICTWWM), Exec)
             .addReg(SavedOrig);
+  } else {
+    MI =
+        BuildMI(MBB, Before, DebugLoc(), TII->get(AMDGPU::EXIT_STRICTWQM), Exec)
+            .addReg(SavedOrig);
+  }
   LIS->InsertMachineInstrInMaps(*MI);
-  StateTransition[MI] = NonStrictWWMState;
+  StateTransition[MI] = NonStrictState;
 }
 
 void SIWholeQuadMode::processBlock(MachineBasicBlock &MBB, bool IsEntry) {
   auto BII = Blocks.find(&MBB);
   if (BII == Blocks.end())
     return;
 
   BlockInfo &BI = BII->second;
 
   // This is a non-entry block that is WQM throughout, so no need to do
   // anything.
   if (!IsEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact) {
     BI.InitialState = StateWQM;
     return;
   }
 
   LLVM_DEBUG(dbgs() << "\nProcessing block " << printMBBReference(MBB)
                     << ":\n");
 
   Register SavedWQMReg;
-  Register SavedNonStrictWWMReg;
+  Register SavedNonStrictReg;
   bool WQMFromExec = IsEntry;
   char State = (IsEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM;
-  char NonStrictWWMState = 0;
+  char NonStrictState = 0;
   const TargetRegisterClass *BoolRC = TRI->getBoolRC();
 
   auto II = MBB.getFirstNonPHI(), IE = MBB.end();
   if (IsEntry) {
     // Skip the instruction that saves LiveMask
     if (II != IE && II->getOpcode() == AMDGPU::COPY)
       ++II;
   }
 
   // This stores the first instruction where it's safe to switch from WQM to
   // Exact or vice versa.
   MachineBasicBlock::iterator FirstWQM = IE;
 
-  // This stores the first instruction where it's safe to switch from StrictWWM to
-  // Exact/WQM or to switch to StrictWWM. It must always be the same as, or after,
-  // FirstWQM since if it's safe to switch to/from StrictWWM, it must be safe to
-  // switch to/from WQM as well.
-  MachineBasicBlock::iterator FirstStrictWWM = IE;
+  // This stores the first instruction where it's safe to switch from Strict
+  // mode to Exact/WQM or to switch to Strict mode. It must always be the same
+  // as, or after, FirstWQM since if it's safe to switch to/from Strict, it must
+  // be safe to switch to/from WQM as well.
+  MachineBasicBlock::iterator FirstStrict = IE;
 
   // Record initial state is block information.
   BI.InitialState = State;
 
   for (;;) {
     MachineBasicBlock::iterator Next = II;
-    char Needs = StateExact | StateWQM; // StrictWWM is disabled by default
+    char Needs = StateExact | StateWQM; // Strict mode is disabled by default.
     char OutNeeds = 0;
 
     if (FirstWQM == IE)
       FirstWQM = II;
 
-    if (FirstStrictWWM == IE)
-      FirstStrictWWM = II;
+    if (FirstStrict == IE)
+      FirstStrict = II;
 
     // First, figure out the allowed states (Needs) based on the propagated
     // flags.
     if (II != IE) {
       MachineInstr &MI = *II;
 
       if (MI.isTerminator() || TII->mayReadEXEC(*MRI, MI)) {
         auto III = Instructions.find(&MI);
         if (III != Instructions.end()) {
           if (III->second.Needs & StateStrictWWM)
             Needs = StateStrictWWM;
+          else if (III->second.Needs & StateStrictWQM)
+            Needs = StateStrictWQM;
           else if (III->second.Needs & StateWQM)
             Needs = StateWQM;
           else
             Needs &= ~III->second.Disabled;
           OutNeeds = III->second.OutNeeds;
         }
       } else {
         // If the instruction doesn't actually need a correct EXEC, then we can
-        // safely leave StrictWWM enabled.
-        Needs = StateExact | StateWQM | StateStrictWWM;
+        // safely leave Strict mode enabled.
+        Needs = StateExact | StateWQM | StateStrict;
       }
 
       if (MI.isTerminator() && OutNeeds == StateExact)
         Needs = StateExact;
 
       ++Next;
     } else {
       // End of basic block
       if (BI.OutNeeds & StateWQM)
         Needs = StateWQM;
       else if (BI.OutNeeds == StateExact)
         Needs = StateExact;
       else
         Needs = StateWQM | StateExact;
     }
 
     // Now, transition if necessary.
     if (!(Needs & State)) {
       MachineBasicBlock::iterator First;
-      if (State == StateStrictWWM || Needs == StateStrictWWM) {
-        // We must switch to or from StrictWWM
-        First = FirstStrictWWM;
+      if (State == StateStrictWWM || Needs == StateStrictWWM ||
+          State == StateStrictWQM || Needs == StateStrictWQM) {
+        // We must switch to or from Strict mode.
+        First = FirstStrict;
       } else {
-        // We only need to switch to/from WQM, so we can use FirstWQM
+        // We only need to switch to/from WQM, so we can use FirstWQM.
         First = FirstWQM;
       }
 
-      // Whether we need to save SCC depends on start and end states
+      // Whether we need to save SCC depends on start and end states.
       bool SaveSCC = false;
       switch (State) {
       case StateExact:
       case StateStrictWWM:
-        // Exact/WWM -> WWM: save SCC
-        // Exact/WWM -> WQM: save SCC if WQM mask is generated from exec
-        // Exact/WWM -> Exact: no save
-        SaveSCC = (Needs & StateStrictWWM) || ((Needs & StateWQM) && WQMFromExec);
+      case StateStrictWQM:
+        // Exact/Strict -> Strict: save SCC
+        // Exact/Strict -> WQM: save SCC if WQM mask is generated from exec
+        // Exact/Strict -> Exact: no save
+        SaveSCC = (Needs & StateStrict) || ((Needs & StateWQM) && WQMFromExec);
         break;
       case StateWQM:
-        // WQM -> Exact/WMM: save SCC
+        // WQM -> Exact/Strict: save SCC
         SaveSCC = !(Needs & StateWQM);
         break;
       default:
         llvm_unreachable("Unknown state");
         break;
       }
       MachineBasicBlock::iterator Before =
           prepareInsertion(MBB, First, II, Needs == StateWQM, SaveSCC);
 
-      if (State == StateStrictWWM) {
-        assert(SavedNonStrictWWMReg);
-        fromStrictWWM(MBB, Before, SavedNonStrictWWMReg, NonStrictWWMState);
-        LIS->createAndComputeVirtRegInterval(SavedNonStrictWWMReg);
-        SavedNonStrictWWMReg = 0;
-        State = NonStrictWWMState;
-      }
+      if (State & StateStrict) {
+        assert(State == StateStrictWWM || State == StateStrictWQM);
+        assert(SavedNonStrictReg);
+        fromStrictWWM(MBB, Before, SavedNonStrictReg, NonStrictState, State);
+
+        LIS->createAndComputeVirtRegInterval(SavedNonStrictReg);
+        SavedNonStrictReg = 0;
+        State = NonStrictState;
+      }
+
+      if (Needs & StateStrict) {
+        NonStrictState = State;
+        assert(Needs == StateStrictWWM || Needs == StateStrictWQM);
+        assert(!SavedNonStrictReg);
+        SavedNonStrictReg = MRI->createVirtualRegister(BoolRC);
+
+        toStrictWWM(MBB, Before, SavedNonStrictReg, Needs);
+        State = Needs;
 
-      if (Needs == StateStrictWWM) {
-        NonStrictWWMState = State;
-        assert(!SavedNonStrictWWMReg);
-        SavedNonStrictWWMReg = MRI->createVirtualRegister(BoolRC);
-        toStrictWWM(MBB, Before, SavedNonStrictWWMReg);
-        State = StateStrictWWM;
       } else {
         if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) {
           if (!WQMFromExec && (OutNeeds & StateWQM)) {
             assert(!SavedWQMReg);
             SavedWQMReg = MRI->createVirtualRegister(BoolRC);
           }
 
           toExact(MBB, Before, SavedWQMReg);
           State = StateExact;
         } else if (State == StateExact && (Needs & StateWQM) &&
                    !(Needs & StateExact)) {
           assert(WQMFromExec == (SavedWQMReg == 0));
 
           toWQM(MBB, Before, SavedWQMReg);
 
           if (SavedWQMReg) {
             LIS->createAndComputeVirtRegInterval(SavedWQMReg);
             SavedWQMReg = 0;
           }
           State = StateWQM;
         } else {
-          // We can get here if we transitioned from StrictWWM to a non-StrictWWM state that
-          // already matches our needs, but we shouldn't need to do anything.
+          // We can get here if we transitioned from StrictWWM to a
+          // non-StrictWWM state that already matches our needs, but we
+          // shouldn't need to do anything.
           assert(Needs & State);
         }
       }
     }
 
-    if (Needs != (StateExact | StateWQM | StateStrictWWM)) {
+    if (Needs != (StateExact | StateWQM | StateStrict)) {
       if (Needs != (StateExact | StateWQM))
         FirstWQM = IE;
-      FirstStrictWWM = IE;
+      FirstStrict = IE;
     }
 
     if (II == IE)
       break;
 
     II = Next;
   }
   assert(!SavedWQMReg);
-  assert(!SavedNonStrictWWMReg);
+  assert(!SavedNonStrictReg);
 }
 
 void SIWholeQuadMode::lowerLiveMaskQueries() {
   for (MachineInstr *MI : LiveMaskQueries) {
     const DebugLoc &DL = MI->getDebugLoc();
     Register Dest = MI->getOperand(0).getReg();
 
     MachineInstr *Copy =
default:
       continue;
     }
     if (SplitPoint)
       splitBlock(MBB, SplitPoint);
   }
 }
 
 bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) {
+  LLVM_DEBUG(dbgs() << "SI Whole Quad Mode on " << MF.getName()
+                    << " ------------- \n");
+  LLVM_DEBUG(MF.dump(););
+
   Instructions.clear();
   Blocks.clear();
   LiveMaskQueries.clear();
   LowerToCopyInstrs.clear();
   LowerToMovInstrs.clear();
   KillInstrs.clear();
   StateTransition.clear();
 
if (ST->isWave32()) {
     Exec = AMDGPU::EXEC;
   }
 
   const char GlobalFlags = analyzeFunction(MF);
   const bool NeedsLiveMask = !(KillInstrs.empty() && LiveMaskQueries.empty());
 
   LiveMaskReg = Exec;
 
-  // Shader is simple does not need WQM/StrictWWM or any complex lowering
-  if (!(GlobalFlags & (StateWQM | StateStrictWWM)) && LowerToCopyInstrs.empty() &&
+  // Shader is simple does not need any state changes or any complex lowering
+  if (!(GlobalFlags & (StateWQM | StateStrict)) && LowerToCopyInstrs.empty() &&
       LowerToMovInstrs.empty() && KillInstrs.empty()) {
     lowerLiveMaskQueries();
     return !LiveMaskQueries.empty();
   }
 
   MachineBasicBlock &Entry = MF.front();
   MachineBasicBlock::iterator EntryMI = Entry.getFirstNonPHI();
 

llvm/test/CodeGen/AMDGPU/wqm.ll

main_body:
   %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0)
   %out = fadd float %src0, %src1
   %out.0 = bitcast float %out to i32
   %out.1 = call i32 @llvm.amdgcn.wqm.i32(i32 %out.0)
   %out.2 = bitcast i32 %out.1 to float
   ret float %out.2
 }
 
+
 ; Check that WWM is triggered by the strictwwm intrinsic.
 ;
 ;CHECK-LABEL: {{^}}test_strictwwm1:
 ;CHECK: s_or_saveexec_b64 s{{\[[0-9]+:[0-9]+\]}}, -1
 ;CHECK: buffer_load_dword
 ;CHECK: buffer_load_dword
 ;CHECK: v_add_f32_e32
 define amdgpu_ps float @test_strictwwm1(i32 inreg %idx0, i32 inreg %idx1) {
main_body:
   %src.1 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %src.0, i32 0)
   %out = add i32 %src.1, %src.1
   %out.0 = call i32 @llvm.amdgcn.strictwwm.i32(i32 %out)
   %out.1 = bitcast i32 %out.0 to float
   call void @llvm.amdgcn.struct.buffer.store.f32(float %out.1, <4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0)
   ret void
 }
 
+; Check that Strict WQM is triggered by the strictwqm intrinsic.
+;
+;CHECK-LABEL: {{^}}test_strictwqm1:
+;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test_strictwqm1(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0)
+  %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0)
+  %out = fadd float %src0, %src1
+  %out.0 = call float @llvm.amdgcn.strictwqm.f32(float %out)
+  ret float %out.0
+}
+
+; Same as above, but with an integer type.
+;
+;CHECK-LABEL: {{^}}test_strictwqm2:
+;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: buffer_load_dword
+;CHECK: v_add_{{[iu]}}32_e32
+define amdgpu_ps float @test_strictwqm2(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0)
+  %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0)
+  %src0.0 = bitcast float %src0 to i32
+  %src1.0 = bitcast float %src1 to i32
+  %out = add i32 %src0.0, %src1.0
+  %out.0 = call i32 @llvm.amdgcn.strictwqm.i32(i32 %out)
+  %out.1 = bitcast i32 %out.0 to float
+  ret float %out.1
+}
+
+; Check that we don't leave Strict WQM on for computations that don't require it,
+; since that will lead clobbering things that aren't supposed to be clobbered
+; in cases like this.
+;
+;CHECK-LABEL: {{^}}test_strictwqm3:
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: v_add_f32_e32
+define amdgpu_ps float @test_strictwqm3(i32 inreg %idx) {
+main_body:
+  ; use mbcnt to make sure the branch is divergent
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+  %cc = icmp uge i32 %hi, 32
+  br i1 %cc, label %endif, label %if
+
+if:
+  %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0)
+  %out = fadd float %src, %src
+  %out.0 = call float @llvm.amdgcn.strictwqm.f32(float %out)
+  %out.1 = fadd float %src, %out.0
+  br label %endif
+
+endif:
+  %out.2 = phi float [ %out.1, %if ], [ 0.0, %main_body ]
+  ret float %out.2
+}
+
+; Check that Strict WQM writes aren't coalesced with non-strict writes, since
+; the Strict WQM write could clobber disabled channels in the non-strict one.
+;
+;CHECK-LABEL: {{^}}test_strictwqm4:
+;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK-NEXT: v_mov_b32_e32
+define amdgpu_ps float @test_strictwqm4(i32 inreg %idx) {
+main_body:
+  ; use mbcnt to make sure the branch is divergent
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+  %cc = icmp uge i32 %hi, 32
+  br i1 %cc, label %endif, label %if
+
+if:
+  %src = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx, i32 0, i32 0, i32 0)
+  %out = fadd float %src, %src
+  %out.0 = call float @llvm.amdgcn.strictwqm.f32(float %out)
+  br label %endif
+
+endif:
+  %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]
+  ret float %out.1
+}
+
+; Make sure the transition from Exact to Strict WQM then WQM works properly.
+;
+;CHECK-LABEL: {{^}}test_strictwqm5:
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: buffer_store_dword
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: s_wqm_b64 exec, exec
+define amdgpu_ps float @test_strictwqm5(i32 inreg %idx0, i32 inreg %idx1) {
+main_body:
+  %src0 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0)
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %src0, <4 x i32> undef, i32 %idx0, i32 0, i32 0, i32 0)
+  %src1 = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> undef, i32 %idx1, i32 0, i32 0, i32 0)
+  %temp = fadd float %src1, %src1
+  %temp.0 = call float @llvm.amdgcn.strictwqm.f32(float %temp)
+  %out = fadd float %temp.0, %temp.0
+  %out.0 = call float @llvm.amdgcn.wqm.f32(float %out)
+  ret float %out.0
+}
+
+; Check that Strict WQM is turned on correctly across basic block boundaries.
+; if..then..endif version
+;
+;CHECK-LABEL: {{^}}test_strictwqm6_then:
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: %if
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: v_add_f32_e32
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+define amdgpu_ps float @test_strictwqm6_then() {
+main_body:
+  %src0 = load volatile float, float addrspace(1)* undef
+  ; use mbcnt to make sure the branch is divergent
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+  %cc = icmp uge i32 %hi, 32
+  br i1 %cc, label %endif, label %if
+
+if:
+  %src1 = load volatile float, float addrspace(1)* undef
+  %out = fadd float %src0, %src1
+  %out.0 = call float @llvm.amdgcn.strictwqm.f32(float %out)
+  br label %endif
+
+endif:
+  %out.1 = phi float [ %out.0, %if ], [ 0.0, %main_body ]
+  ret float %out.1
+}
+
+; Check that Strict WQM is turned on correctly across basic block boundaries.
+; loop version
+;
+;CHECK-LABEL: {{^}}test_strictwqm6_loop:
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG]]
+;CHECK: %loop
+;CHECK: s_mov_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;SI-CHECK: buffer_load_dword
+;VI-CHECK: flat_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+define amdgpu_ps float @test_strictwqm6_loop() {
+main_body:
+  %src0 = load volatile float, float addrspace(1)* undef
+  ; use mbcnt to make sure the branch is divergent
+  %lo = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %hi = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %lo)
+  br label %loop
+
+loop:
+  %counter = phi i32 [ %lo, %main_body ], [ %counter.1, %loop ]
+  %src1 = load volatile float, float addrspace(1)* undef
+  %out = fadd float %src0, %src1
+  %out.0 = call float @llvm.amdgcn.strictwqm.f32(float %out)
+  %counter.1 = sub i32 %counter, 1
+  %cc = icmp ne i32 %counter.1, 0
+  br i1 %cc, label %loop, label %endloop
+
+endloop:
+  ret float %out.0
+}
+
 ; Check that enabling WQM anywhere enables WQM for the set.inactive source.
 ;
 ;CHECK-LABEL: {{^}}test_set_inactive2:
 ;CHECK: s_wqm_b64 exec, exec
 ;CHECK: buffer_load_dword
 ;CHECK: buffer_load_dword
 define amdgpu_ps void @test_set_inactive2(i32 inreg %idx0, i32 inreg %idx1) {
 main_body:
IF:
   %data4f = sitofp i32 %data4 to float
   br label %ENDIF
 
 ENDIF:
   %r = phi float [ %data4f, %IF ], [ 0.0, %main_body ]
   ret float %r
 }
 
+; Check a case of a block being entirely WQM except for a bit of STRICT WQM.
+;
+;CHECK-LABEL: {{^}}test_strictwqm_within_wqm:
+;CHECK: %IF
+;CHECK: s_mov_b64 s{{\[[0-9]+:[0-9]+\]}}, exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: ds_swizzle
+;
+define amdgpu_ps float @test_strictwqm_within_wqm(<8 x i32> inreg %rsrc, <4 x i32> inreg %sampler, i32 %c, i32 %z, float %data) {
+main_body:
+  %c.bc = bitcast i32 %c to float
+  %tex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %c.bc, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0
+  %tex0 = extractelement <4 x float> %tex, i32 0
+  %dtex = call <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32 15, float %tex0, <8 x i32> %rsrc, <4 x i32> %sampler, i1 false, i32 0, i32 0) #0
+  %cmp = icmp eq i32 %z, 0
+  br i1 %cmp, label %IF, label %ENDIF
+
+IF:
+  %dataf = extractelement <4 x float> %dtex, i32 0
+  %data1 = fptosi float %dataf to i32
+  %data2 = call i32 @llvm.amdgcn.ds.swizzle(i32 %data1, i32 2079)
+  %data3 = call i32 @llvm.amdgcn.strictwqm.i32(i32 %data2)
+  %data3f = sitofp i32 %data3 to float
+  br label %ENDIF
+
+ENDIF:
+  %r = phi float [ %data3f, %IF ], [ 0.0, %main_body ]
+  ret float %r
+}
+
+;CHECK-LABEL: {{^}}test_strictwqm_strictwwm_wqm:
+;CHECK: buffer_store_dword
+
+;CHECK: s_mov_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG]]
+
+;CHECK: s_or_saveexec_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+
+;CHECK: s_mov_b64 [[ORIG3:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: v_add
+;CHECK: s_mov_b64 exec, [[ORIG3]]
+
+;TODO: StrictWQM -> WQM transition could be improved. WQM could use the exec from the previous state instead of calling s_wqm again.
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: image_sample
+
+define amdgpu_ps float @test_strictwqm_strictwwm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, <4 x i32> inreg %res2, float %inp, <8 x i32> inreg %res3) {
+main_body:
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+  %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+  %temp = fadd float %reload, %reload
+  %temp2 = call float @llvm.amdgcn.strictwqm.f32(float %temp)
+  %temp3 = fadd float %temp2, %temp2
+  %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res2, i32 %idx0, i32 0, i32 0, i32 0)
+  %temp4 = call float @llvm.amdgcn.strictwwm.f32(float %reload_wwm)
+  %temp5 = fadd float %temp3, %temp4
+  %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res3, <4 x i32> %res, i1 false, i32 0, i32 0)
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+  %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+  ret float %out
+}
+
+;CHECK-LABEL: {{^}}test_strictwwm_strictwqm_wqm:
+;CHECK: buffer_store_dword
+
+;CHECK: s_or_saveexec_b64 [[ORIG:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG]]
+
+;CHECK: s_mov_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+
+;CHECK: s_or_saveexec_b64 [[ORIG3:s\[[0-9]+:[0-9]+\]]], -1
+;CHECK: v_add
+;CHECK: s_mov_b64 exec, [[ORIG3]]
+
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: image_sample
+define amdgpu_ps float @test_strictwwm_strictwqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) {
+main_body:
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+  %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+  %temp = fadd float %reload, %reload
+  %temp2 = call float @llvm.amdgcn.strictwwm.f32(float %temp)
+  %temp3 = fadd float %temp2, %temp2
+  %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+  %temp4 = call float @llvm.amdgcn.strictwqm.f32(float %reload_wwm)
+  %temp5 = fadd float %temp3, %temp4
+  %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp5, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0)
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %tex, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+  %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+  ret float %out
+}
+
+;CHECK-LABEL: {{^}}test_wqm_strictwqm_wqm:
+;CHECK: buffer_store_dword
+
+;CHECK: s_wqm_b64 exec, exec
+
+;TODO: WQM -> StrictWQM transition could be improved. StrictWQM could use the exec from the previous state instead of calling s_wqm again.
+;CHECK: s_mov_b64 [[ORIG2:s\[[0-9]+:[0-9]+\]]], exec
+;CHECK: s_wqm_b64 exec, exec
+;CHECK: buffer_load_dword
+;CHECK: s_mov_b64 exec, [[ORIG2]]
+
+;CHECK: image_sample
+
+define amdgpu_ps float @test_wqm_strictwqm_wqm(i32 inreg %idx0, i32 inreg %idx1, <4 x i32> inreg %res, float %inp, <8 x i32> inreg %res2) {
+main_body:
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %inp, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+  %reload = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx1, i32 0, i32 0, i32 0)
+  %temp = fadd float %reload, %reload
+  %tex = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0)
+  %temp2 = fadd float %tex, %tex
+  %reload_wwm = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+  %temp3 = call float @llvm.amdgcn.strictwqm.f32(float %reload_wwm)
+  %temp4 = fadd float %temp2, %temp3
+  %tex2 = call float @llvm.amdgcn.image.sample.1d.f32.f32(i32 1, float %temp4, <8 x i32> %res2, <4 x i32> %res, i1 false, i32 0, i32 0)
+  call void @llvm.amdgcn.struct.buffer.store.f32(float %tex2, <4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+  %out = call float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32> %res, i32 %idx0, i32 0, i32 0, i32 0)
+  ret float %out
+}
+
 declare void @llvm.amdgcn.exp.f32(i32, i32, float, float, float, float, i1, i1) #1
 declare void @llvm.amdgcn.image.store.1d.v4f32.i32(<4 x float>, i32, i32, <8 x i32>, i32, i32) #1
 
 declare void @llvm.amdgcn.struct.buffer.store.f32(float, <4 x i32>, i32, i32, i32, i32 immarg) #2
 declare void @llvm.amdgcn.struct.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32, i32 immarg) #2
 declare void @llvm.amdgcn.raw.buffer.store.v4f32(<4 x float>, <4 x i32>, i32, i32, i32 immarg) #2
 declare void @llvm.amdgcn.raw.buffer.store.f32(float, <4 x i32>, i32, i32, i32 immarg) #2
 declare float @llvm.amdgcn.raw.buffer.load.f32(<4 x i32>, i32, i32, i32) #3
 declare float @llvm.amdgcn.struct.buffer.load.f32(<4 x i32>, i32, i32, i32, i32) #3
 
 declare <4 x float> @llvm.amdgcn.image.load.1d.v4f32.i32(i32, i32, <8 x i32>, i32, i32) #3
 declare <4 x float> @llvm.amdgcn.image.sample.1d.v4f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
 declare <4 x float> @llvm.amdgcn.image.sample.2d.v4f32.f32(i32, float, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
+declare float @llvm.amdgcn.image.sample.1d.f32.f32(i32, float, <8 x i32>, <4 x i32>, i1, i32, i32) #3
 declare void @llvm.amdgcn.kill(i1) #1
 declare float @llvm.amdgcn.wqm.f32(float) #3
 declare i32 @llvm.amdgcn.wqm.i32(i32) #3
 declare float @llvm.amdgcn.strictwwm.f32(float) #3
 declare i32 @llvm.amdgcn.strictwwm.i32(i32) #3
+declare float @llvm.amdgcn.strictwqm.f32(float) #3
+declare i32 @llvm.amdgcn.strictwqm.i32(i32) #3
 declare i32 @llvm.amdgcn.set.inactive.i32(i32, i32) #4
 declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32) #3
 declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32) #3
 declare <2 x half> @llvm.amdgcn.cvt.pkrtz(float, float) #3
 declare void @llvm.amdgcn.exp.compr.v2f16(i32, i32, <2 x half>, <2 x half>, i1, i1) #1
 declare float @llvm.amdgcn.interp.p1(float, i32, i32, i32) #2
 declare float @llvm.amdgcn.interp.p2(float, float, i32, i32, i32) #2
 declare i32 @llvm.amdgcn.ds.swizzle(i32, i32)
 
 attributes #1 = { nounwind }
 attributes #2 = { nounwind readonly }
 attributes #3 = { nounwind readnone }
 attributes #4 = { nounwind readnone convergent }
 attributes #5 = { "amdgpu-ps-wqm-outputs" }
 attributes #6 = { nounwind "InitialPSInputAddr"="2" }