AMDGPU: Allow some control flow intrinsics to be CSEd

[arsenm]

Summary

These clean up some unnecessary or instructions in
cases with complex loops.

In the original testcase I noticed this, the same
or with exec was repeated 5 or 6 times in a row. With
this only one is emitted or sometimes a copy.

[• tstellarAMD]

LGTM.

[arsenm]

r281786

lib/Target/AMDGPU/SIAnnotateControlFlow.cpp

bool SIAnnotateControlFlow::doInitialization(Module &M) {
   If = M.getOrInsertFunction(
     IfIntrinsic, ReturnStruct, Boolean, (Type *)nullptr);
 
   Else = M.getOrInsertFunction(
     ElseIntrinsic, ReturnStruct, Int64, (Type *)nullptr);
 
   Break = M.getOrInsertFunction(
     BreakIntrinsic, Int64, Int64, (Type *)nullptr);
+  cast<Function>(Break)->setDoesNotAccessMemory();
 
   IfBreak = M.getOrInsertFunction(
     IfBreakIntrinsic, Int64, Boolean, Int64, (Type *)nullptr);
+  cast<Function>(IfBreak)->setDoesNotAccessMemory();;
 
   ElseBreak = M.getOrInsertFunction(
     ElseBreakIntrinsic, Int64, Int64, Int64, (Type *)nullptr);
+  cast<Function>(ElseBreak)->setDoesNotAccessMemory();
 
   Loop = M.getOrInsertFunction(
     LoopIntrinsic, Boolean, Int64, (Type *)nullptr);
 
   EndCf = M.getOrInsertFunction(
     EndCfIntrinsic, Void, Int64, (Type *)nullptr);
 
   return false;

lib/Target/AMDGPU/SIISelLowering.cpp

SDValue SITargetLowering::LowerFrameIndex(SDValue Op, SelectionDAG &DAG) const {
   EVT ExtVT = EVT::getIntegerVT(*DAG.getContext(), KnownBits);
 
   SDValue TFI = DAG.getTargetFrameIndex(FrameIndex, MVT::i32);
   return DAG.getNode(ISD::AssertZext, SL, MVT::i32, TFI,
                      DAG.getValueType(ExtVT));
 }
 
 bool SITargetLowering::isCFIntrinsic(const SDNode *Intr) const {
-  if (Intr->getOpcode() != ISD::INTRINSIC_W_CHAIN)
-    return false;
-
+  if (Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
     switch (cast<ConstantSDNode>(Intr->getOperand(1))->getZExtValue()) {
-  default: return false;
     case AMDGPUIntrinsic::amdgcn_if:
     case AMDGPUIntrinsic::amdgcn_else:
+    case AMDGPUIntrinsic::amdgcn_end_cf:
+    case AMDGPUIntrinsic::amdgcn_loop:
+      return true;
+    default:
+      return false;
+    }
+  }
+
+  if (Intr->getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
+    switch (cast<ConstantSDNode>(Intr->getOperand(0))->getZExtValue()) {
     case AMDGPUIntrinsic::amdgcn_break:
     case AMDGPUIntrinsic::amdgcn_if_break:
     case AMDGPUIntrinsic::amdgcn_else_break:
-  case AMDGPUIntrinsic::amdgcn_loop:
-  case AMDGPUIntrinsic::amdgcn_end_cf:
       return true;
+    default:
+      return false;
+    }
   }
+
+  return false;
 }
 
 void SITargetLowering::createDebuggerPrologueStackObjects(
     MachineFunction &MF) const {
   // Create stack objects that are used for emitting debugger prologue.
   //
   // Debugger prologue writes work group IDs and work item IDs to scratch memory
   // at fixed location in the following format:
if (Intr->getOpcode() == ISD::SETCC) {
     Intr = SetCC->getOperand(0).getNode();
 
   } else {
     // Get the target from BR if we don't negate the condition
     BR = findUser(BRCOND, ISD::BR);
     Target = BR->getOperand(1);
   }
 
+  // FIXME: This changes the types of the intrinsics instead of introducing new
+  // nodes with the correct types.
+  // e.g. llvm.amdgcn.loop
+
+  // eg: i1,ch = llvm.amdgcn.loop t0, TargetConstant:i32<6271>, t3
+  // =>     t9: ch = llvm.amdgcn.loop t0, TargetConstant:i32<6271>, t3, BasicBlock:ch<bb1 0x7fee5286d088>
+
   if (!isCFIntrinsic(Intr)) {
     // This is a uniform branch so we don't need to legalize.
     return BRCOND;
   }
 
+  bool HaveChain = Intr->getOpcode() == ISD::INTRINSIC_VOID ||
+                   Intr->getOpcode() == ISD::INTRINSIC_W_CHAIN;
+
   assert(!SetCC ||
         (SetCC->getConstantOperandVal(1) == 1 &&
          cast<CondCodeSDNode>(SetCC->getOperand(2).getNode())->get() ==
                                                              ISD::SETNE));
 
-  // Build the result and
-  ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
-
   // operands of the new intrinsic call
   SmallVector<SDValue, 4> Ops;
+  if (HaveChain)
     Ops.push_back(BRCOND.getOperand(0));
-  Ops.append(Intr->op_begin() + 1, Intr->op_end());
+
+  Ops.append(Intr->op_begin() + (HaveChain ?  1 : 0), Intr->op_end());
   Ops.push_back(Target);
 
+  ArrayRef<EVT> Res(Intr->value_begin() + 1, Intr->value_end());
+
   // build the new intrinsic call
   SDNode *Result = DAG.getNode(
     Res.size() > 1 ? ISD::INTRINSIC_W_CHAIN : ISD::INTRINSIC_VOID, DL,
     DAG.getVTList(Res), Ops).getNode();
 
+  if (!HaveChain) {
+    SDValue Ops[] =  {
+      SDValue(Result, 0),
+      BRCOND.getOperand(0)
+    };
+
+    Result = DAG.getMergeValues(Ops, DL).getNode();
+  }
+
   if (BR) {
     // Give the branch instruction our target
     SDValue Ops[] = {
       BR->getOperand(0),
       BRCOND.getOperand(2)
     };
     SDValue NewBR = DAG.getNode(ISD::BR, DL, BR->getVTList(), Ops);
     DAG.ReplaceAllUsesWith(BR, NewBR.getNode());

lib/Target/AMDGPU/SIInstrFormats.td

 }
 
 class CFPseudoInstSI<dag outs, dag ins, list<dag> pattern = [],
   bit UseExec = 0, bit DefExec = 0> :
   SPseudoInstSI<outs, ins, pattern> {
 
   let Uses = !if(UseExec, [EXEC], []);
   let Defs = !if(DefExec, [EXEC, SCC], [SCC]);
+  let mayLoad = 0;
+  let mayStore = 0;
+  let hasSideEffects = 0;
 }
 
 class Enc32 {
   field bits<32> Inst;
   int Size = 4;
 }
 
 class Enc64 {

lib/Target/AMDGPU/SIInstructions.td

 let usesCustomInserter = 1, SALU = 1 in {
 def GET_GROUPSTATICSIZE : PseudoInstSI <(outs SReg_32:$sdst), (ins),
   [(set SReg_32:$sdst, (int_amdgcn_groupstaticsize))]>;
 } // End let usesCustomInserter = 1, SALU = 1
 
 // SI pseudo instructions. These are used by the CFG structurizer pass
 // and should be lowered to ISA instructions prior to codegen.
 
-let hasSideEffects = 1 in {
-
 // Dummy terminator instruction to use after control flow instructions
 // replaced with exec mask operations.
 def SI_MASK_BRANCH : PseudoInstSI <
   (outs), (ins brtarget:$target)> {
   let isBranch = 0;
   let isTerminator = 1;
   let isBarrier = 0;
   let SALU = 1;
   let Uses = [EXEC];
 }
 
 let isTerminator = 1 in {
 
 def SI_IF: CFPseudoInstSI <
   (outs SReg_64:$dst), (ins SReg_64:$vcc, brtarget:$target),
   [(set i64:$dst, (int_amdgcn_if i1:$vcc, bb:$target))], 1, 1> {
   let Constraints = "";
   let Size = 8;
+  let mayStore = 1;
+  let mayLoad = 1;
+  let hasSideEffects = 1;
 }
 
 def SI_ELSE : CFPseudoInstSI <
   (outs SReg_64:$dst), (ins SReg_64:$src, brtarget:$target, i1imm:$execfix), [], 1, 1> {
   let Constraints = "$src = $dst";
   let Size = 12;
+  let mayStore = 1;
+  let mayLoad = 1;
+  let hasSideEffects = 1;
 }
 
 def SI_LOOP : CFPseudoInstSI <
   (outs), (ins SReg_64:$saved, brtarget:$target),
   [(int_amdgcn_loop i64:$saved, bb:$target)], 1, 1> {
   let Size = 8;
   let isBranch = 1;
+  let hasSideEffects = 1;
+  let mayLoad = 1;
+  let mayStore = 1;
 }
 
 } // End isBranch = 1, isTerminator = 1
 
 def SI_END_CF : CFPseudoInstSI <
   (outs), (ins SReg_64:$saved),
   [(int_amdgcn_end_cf i64:$saved)], 1, 1> {
   let Size = 4;
+  let isAsCheapAsAMove = 1;
+  let isReMaterializable = 1;
+  let mayLoad = 1;
+  let mayStore = 1;
+  let hasSideEffects = 1;
 }
 
 def SI_BREAK : CFPseudoInstSI <
   (outs SReg_64:$dst), (ins SReg_64:$src),
   [(set i64:$dst, (int_amdgcn_break i64:$src))], 1> {
   let Size = 4;
+  let isAsCheapAsAMove = 1;
+  let isReMaterializable = 1;
 }
 
 def SI_IF_BREAK : CFPseudoInstSI <
   (outs SReg_64:$dst), (ins SReg_64:$vcc, SReg_64:$src),
   [(set i64:$dst, (int_amdgcn_if_break i1:$vcc, i64:$src))]> {
   let Size = 4;
+  let isAsCheapAsAMove = 1;
+  let isReMaterializable = 1;
 }
 
 def SI_ELSE_BREAK : CFPseudoInstSI <
   (outs SReg_64:$dst), (ins SReg_64:$src0, SReg_64:$src1),
   [(set i64:$dst, (int_amdgcn_else_break i64:$src0, i64:$src1))]> {
   let Size = 4;
+  let isAsCheapAsAMove = 1;
+  let isReMaterializable = 1;
 }
 
 let Uses = [EXEC], Defs = [EXEC,VCC] in {
 def SI_KILL : PseudoInstSI <
   (outs), (ins VSrc_32:$src),
   [(AMDGPUkill i32:$src)]> {
   let isConvergent = 1;
   let usesCustomInserter = 1;
 }
 
 def SI_KILL_TERMINATOR : SPseudoInstSI <
   (outs), (ins VSrc_32:$src)> {
   let isTerminator = 1;
 }
 
 } // End Uses = [EXEC], Defs = [EXEC,VCC]
 
-} // End mayLoad = 1, mayStore = 1, hasSideEffects = 1
 
 def SI_PS_LIVE : PseudoInstSI <
   (outs SReg_64:$dst), (ins),
   [(set i1:$dst, (int_amdgcn_ps_live))]> {
   let SALU = 1;
 }
 
 // Used as an isel pseudo to directly emit initialization with an

lib/Target/AMDGPU/SIIntrinsics.td

let TargetPrefix = "amdgcn", isTarget = 1 in {
   // Emit 2.5 ulp, no denormal division. Should only be inserted by
   // pass based on !fpmath metadata.
   def int_amdgcn_fdiv_fast : Intrinsic<
     [llvm_float_ty], [llvm_float_ty], [IntrNoMem]
   >;
 
   /* Control flow Intrinsics */
 
-  def int_amdgcn_if : Intrinsic<[llvm_i64_ty], [llvm_i1_ty, llvm_empty_ty], []>;
-  def int_amdgcn_else : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_empty_ty], []>;
-  def int_amdgcn_break : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], []>;
-  def int_amdgcn_if_break : Intrinsic<[llvm_i64_ty], [llvm_i1_ty, llvm_i64_ty], []>;
-  def int_amdgcn_else_break : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], []>;
-  def int_amdgcn_loop : Intrinsic<[], [llvm_i64_ty, llvm_empty_ty], []>;
-  def int_amdgcn_end_cf : Intrinsic<[], [llvm_i64_ty], []>;
+  def int_amdgcn_if : Intrinsic<[llvm_i64_ty], [llvm_i1_ty, llvm_empty_ty], [IntrConvergent]>;
+  def int_amdgcn_else : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_empty_ty], [IntrConvergent]>;
+  def int_amdgcn_break : Intrinsic<[llvm_i64_ty], [llvm_i64_ty], [IntrNoMem, IntrConvergent]>;
+  def int_amdgcn_if_break : Intrinsic<[llvm_i64_ty], [llvm_i1_ty, llvm_i64_ty], [IntrNoMem, IntrConvergent]>;
+  def int_amdgcn_else_break : Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem, IntrConvergent]>;
+  def int_amdgcn_loop : Intrinsic<[], [llvm_i64_ty, llvm_empty_ty], [IntrConvergent]>;
+  def int_amdgcn_end_cf : Intrinsic<[], [llvm_i64_ty], [IntrConvergent]>;
 }

test/CodeGen/AMDGPU/multilevel-break.ll

-; RUN: opt -S -mtriple=amdgcn-- -structurizecfg -si-annotate-control-flow < %s | FileCheck %s
+; RUN: opt -S -mtriple=amdgcn-- -structurizecfg -si-annotate-control-flow < %s | FileCheck -check-prefix=OPT %s
+; RUN: llc -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
 
-; CHECK-LABEL: {{^}}define amdgpu_vs void @main
-; CHECK: main_body:
-; CHECK: LOOP.outer:
-; CHECK: LOOP:
-; CHECK:     [[if:%[0-9]+]] = call { i1, i64 } @llvm.amdgcn.if(
-; CHECK:     [[if_exec:%[0-9]+]] = extractvalue { i1, i64 } [[if]], 1
+; OPT-LABEL: {{^}}define amdgpu_vs void @multi_else_break(
+; OPT: main_body:
+; OPT: LOOP.outer:
+; OPT: LOOP:
+; OPT:     [[if:%[0-9]+]] = call { i1, i64 } @llvm.amdgcn.if(
+; OPT:     [[if_exec:%[0-9]+]] = extractvalue { i1, i64 } [[if]], 1
 ;
-; CHECK: Flow:
+; OPT: Flow:
 ;
 ; Ensure two else.break calls, for both the inner and outer loops
+
+; OPT:        call i64 @llvm.amdgcn.else.break(i64 [[if_exec]],
+; OPT-NEXT:   call i64 @llvm.amdgcn.else.break(i64 [[if_exec]],
+; OPT-NEXT:   call void @llvm.amdgcn.end.cf
 ;
-; CHECK:        call i64 @llvm.amdgcn.else.break(i64 [[if_exec]],
-; CHECK-NEXT:   call i64 @llvm.amdgcn.else.break(i64 [[if_exec]],
-; CHECK-NEXT:   call void @llvm.amdgcn.end.cf
-;
-; CHECK: Flow1:
-define amdgpu_vs void @main(<4 x float> %vec, i32 %ub, i32 %cont) {
+; OPT: Flow1:
+
+; GCN-LABEL: {{^}}multi_else_break:
+
+; GCN: [[OUTER_LOOP:BB[0-9]+_[0-9]+]]: ; %LOOP.outer{{$}}
+
+; GCN: [[INNER_LOOP:BB[0-9]+_[0-9]+]]: ; %LOOP{{$}}
+; GCN: s_and_saveexec_b64 [[SAVE_BREAK:s\[[0-9]+:[0-9]+\]]], vcc
+
+; GCN: BB{{[0-9]+}}_{{[0-9]+}}: ; %Flow{{$}}
+; GCN-NEXT: ; in Loop: Header=[[INNER_LOOP]] Depth=2
+
+; Ensure extra or eliminated
+; GCN-NEXT: s_or_b64 exec, exec, [[SAVE_BREAK]]
+; GCN-NEXT: s_or_b64 [[OR_BREAK:s\[[0-9]+:[0-9]+\]]], [[SAVE_BREAK]], s{{\[[0-9]+:[0-9]+\]}}
+; GCN-NEXT: s_andn2_b64 exec, exec, [[OR_BREAK]]
+; GCN-NEXT: s_cbranch_execnz [[INNER_LOOP]]
+
+; GCN: ; BB#{{[0-9]+}}: ; %Flow1{{$}}
+; GCN-NEXT: ; in Loop: Header=[[OUTER_LOOP]] Depth=1
+
+; Ensure copy is eliminated
+; GCN-NEXT: s_or_b64 exec, exec, [[OR_BREAK]]
+; GCN-NEXT: s_or_b64 [[OUTER_OR_BREAK:s\[[0-9]+:[0-9]+\]]], [[SAVE_BREAK]], s{{\[[0-9]+:[0-9]+\]}}
+; GCN-NEXT: s_andn2_b64 exec, exec, [[OUTER_OR_BREAK]]
+; GCN-NEXT: s_cbranch_execnz [[OUTER_LOOP]]
+define amdgpu_vs void @multi_else_break(<4 x float> %vec, i32 %ub, i32 %cont) {
 main_body:
   br label %LOOP.outer
 
 LOOP.outer:                                       ; preds = %ENDIF, %main_body
   %tmp43 = phi i32 [ 0, %main_body ], [ %tmp47, %ENDIF ]
   br label %LOOP
 
 LOOP:                                             ; preds = %ENDIF, %LOOP.outer
   %tmp45 = phi i32 [ %tmp43, %LOOP.outer ], [ %tmp47, %ENDIF ]
   %tmp47 = add i32 %tmp45, 1
   %tmp48 = icmp slt i32 %tmp45, %ub
   br i1 %tmp48, label %ENDIF, label %IF
 
 IF:                                               ; preds = %LOOP
   ret void
 
 ENDIF:                                            ; preds = %LOOP
   %tmp51 = icmp eq i32 %tmp47, %cont
   br i1 %tmp51, label %LOOP, label %LOOP.outer
 }
 
-attributes #0 = { nounwind readnone }
+; OPT-LABEL: define void @multi_if_break_loop(
+; OPT: llvm.amdgcn.break
+; OPT: llvm.amdgcn.loop
+; OPT: llvm.amdgcn.if.break
+; OPT: llvm.amdgcn.if.break
+; OPT: llvm.amdgcn.end.cf
+
+; GCN-LABEL: {{^}}multi_if_break_loop:
+; GCN: s_mov_b64 [[BREAK_REG:s\[[0-9]+:[0-9]+\]]], 0{{$}}
+
+; GCN: [[LOOP:BB[0-9]+_[0-9]+]]: ; %bb1{{$}}
+
+; Uses a copy intsead of an or
+; GCN: s_mov_b64 [[COPY:s\[[0-9]+:[0-9]+\]]], [[BREAK_REG]]
+; GCN: s_or_b64 [[BREAK_REG]], exec, [[COPY]]
+define void @multi_if_break_loop(i32 %arg) #0 {
+bb:
+  %id = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp = sub i32 %id, %arg
+  br label %bb1
+
+bb1:
+  %lsr.iv = phi i32 [ undef, %bb ], [ %lsr.iv.next, %case0 ], [ %lsr.iv.next, %case1 ]
+  %lsr.iv.next = add i32 %lsr.iv, 1
+  %cmp0 = icmp slt i32 %lsr.iv.next, 0
+  %load0 = load volatile i32, i32 addrspace(1)* undef, align 4
+  switch i32 %load0, label %bb9 [
+    i32 0, label %case0
+    i32 1, label %case1
+  ]
+
+case0:
+  %load1 = load volatile i32, i32 addrspace(1)* undef, align 4
+  %cmp1 = icmp slt i32 %tmp, %load1
+  br i1 %cmp1, label %bb1, label %bb9
+
+case1:
+  %load2 = load volatile i32, i32 addrspace(1)* undef, align 4
+  %cmp2 = icmp slt i32 %tmp, %load2
+  br i1 %cmp2, label %bb1, label %bb9
+
+bb9:
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }