[AMDGPU] Expand not pattern according to the XOR node divergence

[alex-t]

Summary

The "not" is defined as XOR $src -1.
We need to transform this pattern to either S_NOT_B32 or V_NOT_B32_e32
dependent on the "xor" node divergence.

[foad]

llvm/lib/Target/AMDGPU/SIInstructions.td

2120

There are already patterns on the definitions of S_NOT_* in SOPInstructions.td. Can you update those, instead of adding new standalone patterns?

[rampitec]

llvm/lib/Target/AMDGPU/SIInstructions.td

2120

+1.

[alex-t]

llvm/lib/Target/AMDGPU/SIInstructions.td

2120

The pattern in SOPInstructions.td works with the explicit "not" PatFrag. What if there is no "not" but the "xor src, -1" that was produced by the earlier DAG transformations?
Are you quite sure that this scenario never happens?

[foad]

llvm/lib/Target/AMDGPU/SIInstructions.td

2120

'not' is not a separate DAG node, it is just a named pattern that matches an xor with -1.

[alex-t]

llvm/lib/Target/AMDGPU/SIInstructions.td

2120

Ah... yes. :) "not" is a PatFrag so it is always considered as "not" during selection.

[alex-t]

Uniform XOR src, -1 pattern removed and existing S_NOT_B32/64 patterns equipped with the divergence predicates

[foad]

LGTM but maybe wait a bit in case @rampitec has any further comments.

[rampitec]

LGTM

llvm/lib/Target/AMDGPU/SIInstructions.td

 def : GCNPat <
   (i1 (sub i1:$src0, (i1 -1))),
   (S_NOT_B32 $src0)
 >;
 }
 } // end isWave32
 
 def : GCNPat <
+  (i32 (DivergentBinFrag<xor> i32:$src0, (i32 -1))),
+  (V_NOT_B32_e32 $src0)
+>;
+
+def : GCNPat <
+  (i64 (DivergentBinFrag<xor> i64:$src0, (i64 -1))),
+    (REG_SEQUENCE VReg_64,
+      (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub0))), sub0,
+      (V_NOT_B32_e32 (i32 (EXTRACT_SUBREG i64:$src0, sub1))), sub1
+    )
+>;
+
+def : GCNPat <
   (f16 (sint_to_fp i1:$src)),
   (V_CVT_F16_F32_e32 (
       V_CNDMASK_B32_e64 /*src0mod*/(i32 0), /*src0*/(i32 0),
                         /*src1mod*/(i32 0), /*src1*/(i32 CONST.FP32_NEG_ONE),
                         SSrc_i1:$src))
 >;
 
 def : GCNPat <

llvm/lib/Target/AMDGPU/SOPInstructions.td

 }
 
 // 64-bit input, no output
 class SOP1_1 <string opName, RegisterClass rc = SReg_64, list<dag> pattern=[]> : SOP1_Pseudo <
   opName, (outs), (ins rc:$src0), "$src0", pattern> {
   let has_sdst = 0;
 }
 
+class UniformUnaryFrag<SDPatternOperator Op> : PatFrag <
+  (ops node:$src0),
+  (Op $src0),
+  [{ return !N->isDivergent(); }]> {
+  // This check is unnecessary as it's captured by the result register
+  // bank constraint.
+  //
+  // FIXME: Should add a way for the emitter to recognize this is a
+  // trivially true predicate to eliminate the check.
+  let GISelPredicateCode = [{return true;}];
+}
+
+class UniformBinFrag<SDPatternOperator Op> : PatFrag <
+  (ops node:$src0, node:$src1),
+  (Op $src0, $src1),
+  [{ return !N->isDivergent(); }]> {
+  // This check is unnecessary as it's captured by the result register
+  // bank constraint.
+  //
+  // FIXME: Should add a way for the emitter to recognize this is a
+  // trivially true predicate to eliminate the check.
+  let GISelPredicateCode = [{return true;}];
+}
+
+class DivergentBinFrag<SDPatternOperator Op> : PatFrag <
+  (ops node:$src0, node:$src1),
+  (Op $src0, $src1),
+  [{ return N->isDivergent(); }]> {
+  // This check is unnecessary as it's captured by the result register
+  // bank constraint.
+  //
+  // FIXME: Should add a way for the emitter to recognize this is a
+  // trivially true predicate to eliminate the check.
+  let GISelPredicateCode = [{return true;}];
+}
+
 
 let isMoveImm = 1 in {
   let isReMaterializable = 1, isAsCheapAsAMove = 1 in {
     def S_MOV_B32 : SOP1_32 <"s_mov_b32">;
     def S_MOV_B64 : SOP1_64 <"s_mov_b64">;
   } // End isReMaterializable = 1
 
   let Uses = [SCC] in {
     def S_CMOV_B32 : SOP1_32 <"s_cmov_b32">;
     def S_CMOV_B64 : SOP1_64 <"s_cmov_b64">;
   } // End Uses = [SCC]
 } // End isMoveImm = 1
 
 let Defs = [SCC] in {
   def S_NOT_B32 : SOP1_32 <"s_not_b32",
-    [(set i32:$sdst, (not i32:$src0))]
+    [(set i32:$sdst, (UniformUnaryFrag<not> i32:$src0))]
   >;
 
   def S_NOT_B64 : SOP1_64 <"s_not_b64",
-    [(set i64:$sdst, (not i64:$src0))]
+    [(set i64:$sdst, (UniformUnaryFrag<not> i64:$src0))]
   >;
   def S_WQM_B32 : SOP1_32 <"s_wqm_b32">;
   def S_WQM_B64 : SOP1_64 <"s_wqm_b64">;
 } // End Defs = [SCC]
 
 
 let WaveSizePredicate = isWave32 in {
 def : GCNPat <
class SOP2_64_32 <string opName, list<dag> pattern=[]> : SOP2_Pseudo <
   "$sdst, $src0, $src1", pattern
 >;
 
 class SOP2_64_32_32 <string opName, list<dag> pattern=[]> : SOP2_Pseudo <
   opName, (outs SReg_64:$sdst), (ins SSrc_b32:$src0, SSrc_b32:$src1),
   "$sdst, $src0, $src1", pattern
 >;
 
-class UniformUnaryFrag<SDPatternOperator Op> : PatFrag <
-  (ops node:$src0),
-  (Op $src0),
-  [{ return !N->isDivergent(); }]> {
-  // This check is unnecessary as it's captured by the result register
-  // bank constraint.
-  //
-  // FIXME: Should add a way for the emitter to recognize this is a
-  // trivially true predicate to eliminate the check.
-  let GISelPredicateCode = [{return true;}];
-}
-
-class UniformBinFrag<SDPatternOperator Op> : PatFrag <
-  (ops node:$src0, node:$src1),
-  (Op $src0, $src1),
-  [{ return !N->isDivergent(); }]> {
-  // This check is unnecessary as it's captured by the result register
-  // bank constraint.
-  //
-  // FIXME: Should add a way for the emitter to recognize this is a
-  // trivially true predicate to eliminate the check.
-  let GISelPredicateCode = [{return true;}];
-}
-
-class DivergentBinFrag<SDPatternOperator Op> : PatFrag <
-  (ops node:$src0, node:$src1),
-  (Op $src0, $src1),
-  [{ return N->isDivergent(); }]> {
-  // This check is unnecessary as it's captured by the result register
-  // bank constraint.
-  //
-  // FIXME: Should add a way for the emitter to recognize this is a
-  // trivially true predicate to eliminate the check.
-  let GISelPredicateCode = [{return true;}];
-}
 
 let Defs = [SCC] in { // Carry out goes to SCC
 let isCommutable = 1 in {
 def S_ADD_U32 : SOP2_32 <"s_add_u32">;
 def S_ADD_I32 : SOP2_32 <"s_add_i32",
   [(set i32:$sdst, (UniformBinFrag<add> SSrc_b32:$src0, SSrc_b32:$src1))]
 >;
 } // End isCommutable = 1

llvm/test/CodeGen/AMDGPU/divergence-driven-not-isel.ll

+; RUN: llc -march=amdgcn -stop-after=amdgpu-isel < %s | FileCheck -check-prefix=GCN %s
+
+; GCN-LABEL: name:            scalar_not_i32
+; GCN: S_NOT_B32
+define amdgpu_kernel void @scalar_not_i32(i32 addrspace(1)* %out, i32 %val) {
+  %not.val = xor i32 %val, -1
+  store i32 %not.val, i32 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: name:            scalar_not_i64
+; GCN: S_NOT_B64
+define amdgpu_kernel void @scalar_not_i64(i64 addrspace(1)* %out, i64 %val) {
+  %not.val = xor i64 %val, -1
+  store i64 %not.val, i64 addrspace(1)* %out
+  ret void
+}
+
+; GCN-LABEL: name:            vector_not_i32
+; GCN: V_NOT_B32_e32
+define i32 @vector_not_i32(i32 %val) {
+  %not.val = xor i32 %val, -1
+  ret i32 %not.val
+}
+
+; GCN-LABEL: name:            vector_not_i64
+; GCN: V_NOT_B32_e32
+; GCN: V_NOT_B32_e32
+define i64 @vector_not_i64(i64 %val) {
+  %not.val = xor i64 %val, -1
+  ret i64 %not.val
+}
+
+

llvm/test/CodeGen/AMDGPU/fused-bitlogic.ll

 ; GCN-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GCN-NEXT:    v_lshlrev_b32_e32 v6, 5, v0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NEXT:    global_load_dwordx2 v[4:5], v6, s[0:1] offset:16
 ; GCN-NEXT:    global_load_dwordx4 v[0:3], v6, s[0:1]
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_or3_b32 v1, v3, v1, v5
 ; GCN-NEXT:    v_or3_b32 v0, v2, v0, v4
-; GCN-NEXT:    v_not_b32_e32 v0, v0
 ; GCN-NEXT:    v_not_b32_e32 v1, v1
+; GCN-NEXT:    v_not_b32_e32 v0, v0
 ; GCN-NEXT:    global_store_dwordx2 v6, v[0:1], s[0:1]
 ; GCN-NEXT:    s_endpgm
 bb:
   %i = tail call i32 @llvm.amdgcn.workitem.id.x()
   %i1 = zext i32 %i to i64
   %i2 = getelementptr inbounds <3 x i64>, <3 x i64> addrspace(1)* %arg, i64 %i1
   %i3 = load <3 x i64>, <3 x i64> addrspace(1)* %i2, align 32
   %i4 = extractelement <3 x i64> %i3, i64 0

 ; GCN-NEXT:    global_load_dwordx4 v[0:3], v6, s[0:1]
 ; GCN-NEXT:    global_load_dwordx2 v[4:5], v6, s[0:1] offset:16
 ; GCN-NEXT:    s_waitcnt vmcnt(1)
 ; GCN-NEXT:    v_and_b32_e32 v1, v3, v1
 ; GCN-NEXT:    v_and_b32_e32 v0, v2, v0
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NEXT:    v_and_b32_e32 v1, v1, v5
 ; GCN-NEXT:    v_and_b32_e32 v0, v0, v4
-; GCN-NEXT:    v_not_b32_e32 v0, v0
 ; GCN-NEXT:    v_not_b32_e32 v1, v1
+; GCN-NEXT:    v_not_b32_e32 v0, v0
 ; GCN-NEXT:    global_store_dwordx2 v6, v[0:1], s[0:1]
 ; GCN-NEXT:    s_endpgm
 bb:
   %i = tail call i32 @llvm.amdgcn.workitem.id.x()
   %i1 = zext i32 %i to i64
   %i2 = getelementptr inbounds <3 x i64>, <3 x i64> addrspace(1)* %arg, i64 %i1
   %i3 = load <3 x i64>, <3 x i64> addrspace(1)* %i2, align 32
   %i4 = extractelement <3 x i64> %i3, i64 0

llvm/test/CodeGen/AMDGPU/llvm.round.f64.ll

 ; SI-NEXT:    s_mov_b32 s3, 0xfffff
 ; SI-NEXT:    s_brev_b32 s5, -2
 ; SI-NEXT:    v_mov_b32_e32 v8, 0x3ff00000
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_bfe_u32 v4, v3, 20, 11
 ; SI-NEXT:    v_add_i32_e32 v6, vcc, s4, v4
 ; SI-NEXT:    v_lshr_b64 v[4:5], s[2:3], v6
 ; SI-NEXT:    v_and_b32_e32 v7, 0x80000000, v3
-; SI-NEXT:    v_not_b32_e32 v4, v4
 ; SI-NEXT:    v_not_b32_e32 v5, v5
+; SI-NEXT:    v_not_b32_e32 v4, v4
 ; SI-NEXT:    v_and_b32_e32 v5, v3, v5
 ; SI-NEXT:    v_and_b32_e32 v4, v2, v4
 ; SI-NEXT:    v_cmp_gt_i32_e32 vcc, 0, v6
 ; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v7, vcc
 ; SI-NEXT:    v_cndmask_b32_e64 v4, v4, 0, vcc
 ; SI-NEXT:    v_cmp_lt_i32_e32 vcc, 51, v6
 ; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v3, vcc
 ; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v2, vcc

llvm/test/CodeGen/AMDGPU/urem64.ll

 ; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB1_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v14, vcc, -1, v2
 ; GCN-IR-NEXT:    v_addc_u32_e32 v15, vcc, -1, v3, vcc
-; GCN-IR-NEXT:    v_not_b32_e32 v6, v8
-; GCN-IR-NEXT:    v_not_b32_e32 v7, v9
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, v6, v10
+; GCN-IR-NEXT:    v_not_b32_e32 v7, v8
+; GCN-IR-NEXT:    v_not_b32_e32 v6, v9
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, v7, v10
 ; GCN-IR-NEXT:    v_lshr_b64 v[12:13], v[0:1], v12
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, v7, v11, vcc
+; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, v6, v11, vcc
 ; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v11, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
 ; GCN-IR-NEXT:  .LBB1_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GCN-IR-NEXT:    v_lshl_b64 v[12:13], v[12:13], 1
 ; GCN-IR-NEXT:    v_lshrrev_b32_e32 v6, 31, v5
 ; GCN-IR-NEXT:    v_or_b32_e32 v12, v12, v6