AMDGPU: Force sign operand of f64 fcopysign to f32

[arsenm]

Summary

The fcopysign DAG operation, unlike the IR one, allows
different types for the sign and magnitude. We can reduce
the bitwidth of the high operand since only the sign bit matters.

The default combine only introduces mixed fcopysign
operand types from fpext/fptrunc. We effectively do this
already during selection, but doing it earlier in the combiner
should expose new combine opportunities (e.g. the existing tests
now eliminate the load of the low half of the double). Unfortunately
this isn't enough to handle the case I'm interested in just yet.

[foad]

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

9445

Can't you bitcast f64 to v2f32, to avoid the second bitcast?

[arsenm]

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

9445

Yes, but surprisingly this loses the load width reduction optimization

[arsenm]

Avoid second bitcast

[foad]

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

9445

Ugh.

[arsenm]

606a62ce27e602cfc12381dd9d1ec2d065aa075e

llvm/lib/Target/AMDGPU/SIISelLowering.h

private:
   SDValue lowerTrapHsaQueuePtr(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerTrapHsa(SDValue Op, SelectionDAG &DAG) const;
   SDValue lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const;
 
   SDNode *adjustWritemask(MachineSDNode *&N, SelectionDAG &DAG) const;
 
   SDValue performUCharToFloatCombine(SDNode *N,
                                      DAGCombinerInfo &DCI) const;
+  SDValue performFCopySignCombine(SDNode *N, DAGCombinerInfo &DCI) const;
+
   SDValue performSHLPtrCombine(SDNode *N,
                                unsigned AS,
                                EVT MemVT,
                                DAGCombinerInfo &DCI) const;
 
   SDValue performMemSDNodeCombine(MemSDNode *N, DAGCombinerInfo &DCI) const;
 
   SDValue splitBinaryBitConstantOp(DAGCombinerInfo &DCI, const SDLoc &SL,

llvm/lib/Target/AMDGPU/SIISelLowering.cpp

setTargetDAGCombine({ISD::ADD,
                        ISD::XOR,
                        ISD::SINT_TO_FP,
                        ISD::UINT_TO_FP,
                        ISD::FCANONICALIZE,
                        ISD::SCALAR_TO_VECTOR,
                        ISD::ZERO_EXTEND,
                        ISD::SIGN_EXTEND_INREG,
                        ISD::EXTRACT_VECTOR_ELT,
-                       ISD::INSERT_VECTOR_ELT});
+                       ISD::INSERT_VECTOR_ELT,
+                       ISD::FCOPYSIGN});
 
   // All memory operations. Some folding on the pointer operand is done to help
   // matching the constant offsets in the addressing modes.
   setTargetDAGCombine({ISD::LOAD,
                        ISD::STORE,
                        ISD::ATOMIC_LOAD,
                        ISD::ATOMIC_STORE,
                        ISD::ATOMIC_CMP_SWAP,
if (DAG.MaskedValueIsZero(Src, APInt::getHighBitsSet(32, 24))) {
       }
       return Cvt;
     }
   }
 
   return SDValue();
 }
 
+SDValue SITargetLowering::performFCopySignCombine(SDNode *N,
+                                                  DAGCombinerInfo &DCI) const {
+  SDValue SignOp = N->getOperand(1);
+  if (SignOp.getValueType() != MVT::f64)
+    return SDValue();
+
+  SelectionDAG &DAG = DCI.DAG;
+  SDLoc DL(N);
+
+  // Reduce width of sign operand, we only need the highest bit.
+  //
+  // fcopysign f64:x, f64:y ->
+  //   fcopysign f64:x, (extract_vector_elt (bitcast f64:y to v2f32), 1)
+  // TODO: In some cases it might make sense to go all the way to f16.
+  SDValue SignAsVector = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, SignOp);
+  SDValue SignAsF32 =
+      DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, SignAsVector,
+                  DAG.getConstant(1, DL, MVT::i32));
+
+  return DAG.getNode(ISD::FCOPYSIGN, DL, N->getValueType(0), N->getOperand(0),
+                     SignAsF32);
+}
+
 // (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2)
 
 // This is a variant of
 // (mul (add x, c1), c2) -> add (mul x, c2), (mul c1, c2),
 //
 // The normal DAG combiner will do this, but only if the add has one use since
 // that would increase the number of instructions.
 //
case AMDGPUISD::LDEXP: {
     SDValue Src = N->getOperand(0);
     if (Src.isUndef())
       return Src;
     break;
   }
   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:
     return performUCharToFloatCombine(N, DCI);
+  case ISD::FCOPYSIGN:
+    return performFCopySignCombine(N, DCI);
   case AMDGPUISD::CVT_F32_UBYTE0:
   case AMDGPUISD::CVT_F32_UBYTE1:
   case AMDGPUISD::CVT_F32_UBYTE2:
   case AMDGPUISD::CVT_F32_UBYTE3:
     return performCvtF32UByteNCombine(N, DCI);
   case AMDGPUISD::FMED3:
     return performFMed3Combine(N, DCI);
   case AMDGPUISD::CVT_PKRTZ_F16_F32:

llvm/test/CodeGen/AMDGPU/fnearbyint.ll

 ; SI-NEXT:    s_brev_b32 s8, -2
 ; SI-NEXT:    v_mov_b32_e32 v1, 0x43300000
 ; SI-NEXT:    v_mov_b32_e32 v0, 0
 ; SI-NEXT:    v_mov_b32_e32 v2, -1
 ; SI-NEXT:    v_mov_b32_e32 v3, 0x432fffff
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
 ; SI-NEXT:    s_mov_b32 s4, s0
 ; SI-NEXT:    s_mov_b32 s5, s1
-; SI-NEXT:    v_mov_b32_e32 v4, s3
-; SI-NEXT:    v_bfi_b32 v1, s8, v1, v4
 ; SI-NEXT:    v_mov_b32_e32 v6, s3
+; SI-NEXT:    v_bfi_b32 v1, s8, v1, v6
 ; SI-NEXT:    v_mov_b32_e32 v7, s2
 ; SI-NEXT:    v_add_f64 v[4:5], s[2:3], v[0:1]
 ; SI-NEXT:    v_add_f64 v[0:1], v[4:5], -v[0:1]
 ; SI-NEXT:    v_cmp_gt_f64_e64 vcc, |s[2:3]|, v[2:3]
 ; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v6, vcc
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v7, vcc
 ; SI-NEXT:    buffer_store_dwordx2 v[0:1], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm

 ; SI-NEXT:    s_brev_b32 s10, -2
 ; SI-NEXT:    v_mov_b32_e32 v6, 0x43300000
 ; SI-NEXT:    s_mov_b32 s9, 0x432fffff
 ; SI-NEXT:    v_mov_b32_e32 v0, 0
 ; SI-NEXT:    s_mov_b32 s8, s6
 ; SI-NEXT:    v_mov_b32_e32 v4, s8
 ; SI-NEXT:    v_mov_b32_e32 v5, s9
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v1, s3
-; SI-NEXT:    v_bfi_b32 v1, s10, v6, v1
 ; SI-NEXT:    v_mov_b32_e32 v7, s3
+; SI-NEXT:    v_bfi_b32 v1, s10, v6, v7
 ; SI-NEXT:    v_mov_b32_e32 v8, s2
 ; SI-NEXT:    v_mov_b32_e32 v9, s1
-; SI-NEXT:    v_mov_b32_e32 v10, s1
-; SI-NEXT:    v_mov_b32_e32 v11, s0
+; SI-NEXT:    v_mov_b32_e32 v10, s0
 ; SI-NEXT:    v_add_f64 v[2:3], s[2:3], v[0:1]
 ; SI-NEXT:    v_add_f64 v[2:3], v[2:3], -v[0:1]
 ; SI-NEXT:    v_bfi_b32 v1, s10, v6, v9
 ; SI-NEXT:    v_cmp_gt_f64_e64 vcc, |s[2:3]|, v[4:5]
 ; SI-NEXT:    v_cndmask_b32_e32 v3, v3, v7, vcc
 ; SI-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
 ; SI-NEXT:    v_add_f64 v[6:7], s[0:1], v[0:1]
 ; SI-NEXT:    v_add_f64 v[0:1], v[6:7], -v[0:1]
 ; SI-NEXT:    v_cmp_gt_f64_e64 vcc, |s[0:1]|, v[4:5]
-; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v10, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v11, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[4:7], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: nearbyint_v2f64:
 ; CI:       ; %bb.0: ; %entry
 ; CI-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0xd
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
 ; CI-NEXT:    s_mov_b32 s3, 0xf000

 ; SI-NEXT:    s_brev_b32 s14, -2
 ; SI-NEXT:    v_mov_b32_e32 v10, 0x43300000
 ; SI-NEXT:    s_mov_b32 s13, 0x432fffff
 ; SI-NEXT:    v_mov_b32_e32 v4, 0
 ; SI-NEXT:    s_mov_b32 s12, s10
 ; SI-NEXT:    v_mov_b32_e32 v8, s12
 ; SI-NEXT:    v_mov_b32_e32 v9, s13
 ; SI-NEXT:    s_waitcnt lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v0, s3
-; SI-NEXT:    v_bfi_b32 v5, s14, v10, v0
 ; SI-NEXT:    v_mov_b32_e32 v2, s3
+; SI-NEXT:    v_bfi_b32 v5, s14, v10, v2
 ; SI-NEXT:    v_mov_b32_e32 v6, s2
-; SI-NEXT:    v_mov_b32_e32 v3, s1
 ; SI-NEXT:    v_mov_b32_e32 v7, s1
 ; SI-NEXT:    v_mov_b32_e32 v11, s0
 ; SI-NEXT:    v_mov_b32_e32 v12, s7
-; SI-NEXT:    v_mov_b32_e32 v13, s7
-; SI-NEXT:    v_mov_b32_e32 v14, s6
-; SI-NEXT:    v_mov_b32_e32 v15, s5
-; SI-NEXT:    v_mov_b32_e32 v16, s5
-; SI-NEXT:    v_mov_b32_e32 v17, s4
+; SI-NEXT:    v_mov_b32_e32 v13, s6
+; SI-NEXT:    v_mov_b32_e32 v14, s5
+; SI-NEXT:    v_mov_b32_e32 v15, s4
 ; SI-NEXT:    v_add_f64 v[0:1], s[2:3], v[4:5]
 ; SI-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
-; SI-NEXT:    v_bfi_b32 v5, s14, v10, v3
+; SI-NEXT:    v_bfi_b32 v5, s14, v10, v7
 ; SI-NEXT:    v_cmp_gt_f64_e64 vcc, |s[2:3]|, v[8:9]
 ; SI-NEXT:    v_cndmask_b32_e32 v3, v1, v2, vcc
 ; SI-NEXT:    v_cndmask_b32_e32 v2, v0, v6, vcc
 ; SI-NEXT:    v_add_f64 v[0:1], s[0:1], v[4:5]
 ; SI-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
 ; SI-NEXT:    v_bfi_b32 v5, s14, v10, v12
 ; SI-NEXT:    v_cmp_gt_f64_e64 vcc, |s[0:1]|, v[8:9]
 ; SI-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; SI-NEXT:    v_cndmask_b32_e32 v0, v0, v11, vcc
 ; SI-NEXT:    v_add_f64 v[6:7], s[6:7], v[4:5]
 ; SI-NEXT:    v_add_f64 v[6:7], v[6:7], -v[4:5]
-; SI-NEXT:    v_bfi_b32 v5, s14, v10, v15
+; SI-NEXT:    v_bfi_b32 v5, s14, v10, v14
 ; SI-NEXT:    v_cmp_gt_f64_e64 vcc, |s[6:7]|, v[8:9]
-; SI-NEXT:    v_cndmask_b32_e32 v7, v7, v13, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v7, v7, v12, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v6, v6, v13, vcc
 ; SI-NEXT:    v_add_f64 v[10:11], s[4:5], v[4:5]
 ; SI-NEXT:    v_add_f64 v[4:5], v[10:11], -v[4:5]
 ; SI-NEXT:    v_cmp_gt_f64_e64 vcc, |s[4:5]|, v[8:9]
-; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v16, vcc
-; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v17, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v5, v5, v14, vcc
+; SI-NEXT:    v_cndmask_b32_e32 v4, v4, v15, vcc
 ; SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[8:11], 0 offset:16
 ; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[8:11], 0
 ; SI-NEXT:    s_endpgm
 ;
 ; CI-LABEL: nearbyint_v4f64:
 ; CI:       ; %bb.0: ; %entry
 ; CI-NEXT:    s_load_dwordx8 s[4:11], s[0:1], 0x11
 ; CI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9