[AMDGPU] Add new llvm.amdgcn.fma.legacy intrinsic

llvm/include/llvm/IR/IntrinsicsAMDGPU.td

 >;
 
 def int_amdgcn_log_clamp : Intrinsic<
   [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn]
 >;
 
 def int_amdgcn_fmul_legacy : GCCBuiltin<"__builtin_amdgcn_fmul_legacy">,
   Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty],
-  [IntrNoMem, IntrSpeculatable, IntrWillReturn]
+  [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]
+>;
+
+// Fused single-precision multiply-add with legacy behaviour for the multiply,
+// which is that +/- 0.0 * anything (even NaN or infinity) is +0.0. This is
+// intended for use on subtargets that have the v_fma_legacy_f32 and/or
+// v_fmac_legacy_f32 instructions. (Note that v_fma_legacy_f16 is unrelated and
+// has a completely different kind of legacy behaviour.)
+def int_amdgcn_fma_legacy :
+  Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty],
+  [IntrNoMem, IntrSpeculatable, IntrWillReturn, Commutative]
 >;
 
 def int_amdgcn_rcp : Intrinsic<
   [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrWillReturn]
 >;
 
 def int_amdgcn_rcp_legacy : GCCBuiltin<"__builtin_amdgcn_rcp_legacy">,
   Intrinsic<[llvm_float_ty], [llvm_float_ty],

llvm/lib/Analysis/ConstantFolding.cpp

bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) {
   case Intrinsic::convert_from_fp16:
   case Intrinsic::convert_to_fp16:
   case Intrinsic::amdgcn_cos:
   case Intrinsic::amdgcn_cubeid:
   case Intrinsic::amdgcn_cubema:
   case Intrinsic::amdgcn_cubesc:
   case Intrinsic::amdgcn_cubetc:
   case Intrinsic::amdgcn_fmul_legacy:
+  case Intrinsic::amdgcn_fma_legacy:
   case Intrinsic::amdgcn_fract:
   case Intrinsic::amdgcn_ldexp:
   case Intrinsic::amdgcn_sin:
   // The intrinsics below depend on rounding mode in MXCSR.
   case Intrinsic::x86_sse_cvtss2si:
   case Intrinsic::x86_sse_cvtss2si64:
   case Intrinsic::x86_sse_cvttss2si:
   case Intrinsic::x86_sse_cvttss2si64:
if (auto *Op2 = dyn_cast<ConstantFP>(Operands[1])) {
         const APFloat &C1 = Op1->getValueAPF();
         const APFloat &C2 = Op2->getValueAPF();
         return ConstantFP::get(Ty->getContext(), maximum(C1, C2));
       }
 
       if (IntrinsicID == Intrinsic::amdgcn_fmul_legacy) {
         const APFloat &C1 = Op1->getValueAPF();
         const APFloat &C2 = Op2->getValueAPF();
-        // The legacy behaviour is that multiplying zero by anything, even NaN
-        // or infinity, gives +0.0.
+        // The legacy behaviour is that multiplying +/- 0.0 by anything, even
+        // NaN or infinity, gives +0.0.
         if (C1.isZero() || C2.isZero())
           return ConstantFP::getNullValue(Ty);
         return ConstantFP::get(Ty->getContext(), C1 * C2);
       }
 
       if (!TLI)
         return nullptr;
 
static Constant *ConstantFoldScalarCall3(StringRef Name,
                                          const CallBase *Call) {
   assert(Operands.size() == 3 && "Wrong number of operands.");
 
   if (const auto *Op1 = dyn_cast<ConstantFP>(Operands[0])) {
     if (const auto *Op2 = dyn_cast<ConstantFP>(Operands[1])) {
       if (const auto *Op3 = dyn_cast<ConstantFP>(Operands[2])) {
         switch (IntrinsicID) {
         default: break;
+        case Intrinsic::amdgcn_fma_legacy: {
+          const APFloat &C1 = Op1->getValueAPF();
+          const APFloat &C2 = Op2->getValueAPF();
+          // The legacy behaviour is that multiplying +/- 0.0 by anything, even
+          // NaN or infinity, gives +0.0.
+          if (C1.isZero() || C2.isZero()) {
+            const APFloat &C3 = Op3->getValueAPF();
+            // It's tempting to just return C3 here, but that would give the
+            // wrong result if C3 was -0.0.
+            return ConstantFP::get(Ty->getContext(), APFloat(0.0f) + C3);
+          }
+          LLVM_FALLTHROUGH;
+        }
         case Intrinsic::fma:
         case Intrinsic::fmuladd: {
           APFloat V = Op1->getValueAPF();
           V.fusedMultiplyAdd(Op2->getValueAPF(), Op3->getValueAPF(),
                              APFloat::rmNearestTiesToEven);
           return ConstantFP::get(Ty->getContext(), V);
         }
         case Intrinsic::amdgcn_cubeid:

llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp

static bool fnegFoldsIntoOp(unsigned Opc) {
   case AMDGPUISD::RCP:
   case AMDGPUISD::RCP_LEGACY:
   case AMDGPUISD::RCP_IFLAG:
   case AMDGPUISD::SIN_HW:
   case AMDGPUISD::FMUL_LEGACY:
   case AMDGPUISD::FMIN_LEGACY:
   case AMDGPUISD::FMAX_LEGACY:
   case AMDGPUISD::FMED3:
+    // TODO: handle llvm.amdgcn.fma.legacy
     return true;
   default:
     return false;
   }
 }
 
 /// \p returns true if the operation will definitely need to use a 64-bit
 /// encoding, and thus will use a VOP3 encoding regardless of the source
case AMDGPUISD::FMUL_LEGACY: {
     if (Res.getOpcode() != Opc)
       return SDValue(); // Op got folded away.
     if (!N0.hasOneUse())
       DAG.ReplaceAllUsesWith(N0, DAG.getNode(ISD::FNEG, SL, VT, Res));
     return Res;
   }
   case ISD::FMA:
   case ISD::FMAD: {
+    // TODO: handle llvm.amdgcn.fma.legacy
     if (!mayIgnoreSignedZero(N0))
       return SDValue();
 
     // (fneg (fma x, y, z)) -> (fma x, (fneg y), (fneg z))
     SDValue LHS = N0.getOperand(0);
     SDValue MHS = N0.getOperand(1);
     SDValue RHS = N0.getOperand(2);
 
case Intrinsic::amdgcn_rsq_clamp: {
 
       // TODO: Need is known positive check.
       return false;
     }
     case Intrinsic::amdgcn_trig_preop:
     case Intrinsic::amdgcn_fdot2:
       // TODO: Refine on operand
       return SNaN;
+    case Intrinsic::amdgcn_fma_legacy:
+      if (SNaN)
+        return true;
+      return DAG.isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
+             DAG.isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1) &&
+             DAG.isKnownNeverNaN(Op.getOperand(3), SNaN, Depth + 1);
     default:
       return false;
     }
   }
   default:
     return false;
   }
 }

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

case AMDGPU::G_INTRINSIC: {
     case Intrinsic::amdgcn_log_clamp:
     case Intrinsic::amdgcn_rcp:
     case Intrinsic::amdgcn_rcp_legacy:
     case Intrinsic::amdgcn_sqrt:
     case Intrinsic::amdgcn_rsq:
     case Intrinsic::amdgcn_rsq_legacy:
     case Intrinsic::amdgcn_rsq_clamp:
     case Intrinsic::amdgcn_fmul_legacy:
+    case Intrinsic::amdgcn_fma_legacy:
     case Intrinsic::amdgcn_ldexp:
     case Intrinsic::amdgcn_frexp_mant:
     case Intrinsic::amdgcn_frexp_exp:
     case Intrinsic::amdgcn_fract:
     case Intrinsic::amdgcn_cvt_pkrtz:
     case Intrinsic::amdgcn_cvt_pknorm_i16:
     case Intrinsic::amdgcn_cvt_pknorm_u16:
     case Intrinsic::amdgcn_cvt_pk_i16:

llvm/lib/Target/AMDGPU/SIInstructions.td

 def : GCNPat <
       (f32 (fadd (AMDGPUfmul_legacy (VOP3NoMods f32:$src0),
                                     (VOP3NoMods f32:$src1)),
                  (VOP3NoMods f32:$src2))),
       (V_MAC_LEGACY_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
                             SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE)
 >;
 
+// Don't allow source modifiers. If there are any source modifiers then it's
+// better to select fma instead of fmac.
+let SubtargetPredicate = HasNoMadMacF32Insts in
+def : GCNPat <
+      (f32 (int_amdgcn_fma_legacy (VOP3NoMods f32:$src0),
+                                  (VOP3NoMods f32:$src1),
+                                  (VOP3NoMods f32:$src2))),
+      (V_FMAC_LEGACY_F32_e64 SRCMODS.NONE, $src0, SRCMODS.NONE, $src1,
+                             SRCMODS.NONE, $src2, DSTCLAMP.NONE, DSTOMOD.NONE)
+>;
+
 let SubtargetPredicate = Has16BitInsts in {
 def : FMADPat <f16, V_MAC_F16_e64, fmad>;
 def : FMADPat <f16, V_MAC_F16_e64, AMDGPUfmad_ftz>;
 } // SubtargetPredicate = Has16BitInsts
 } // AddedComplexity = 9
 
 class FMADModsPat<ValueType Ty, Instruction inst, SDPatternOperator mad_opr>
   : GCNPat<

llvm/lib/Target/AMDGPU/VOP3Instructions.td

 
 let mayRaiseFPException = 0 in {
 let SubtargetPredicate = HasMadMacF32Insts in {
 def V_MAD_LEGACY_F32 : VOP3Inst <"v_mad_legacy_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
 def V_MAD_F32 : VOP3Inst <"v_mad_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, fmad>;
 } // End SubtargetPredicate = HasMadMacInsts
 
 let SubtargetPredicate = HasNoMadMacF32Insts in
-def V_FMA_LEGACY_F32 : VOP3Inst <"v_fma_legacy_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
+def V_FMA_LEGACY_F32 : VOP3Inst <"v_fma_legacy_f32",
+                                 VOP3_Profile<VOP_F32_F32_F32_F32>,
+                                 int_amdgcn_fma_legacy>;
 }
 
 def V_MAD_I32_I24 : VOP3Inst <"v_mad_i32_i24", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;
 def V_MAD_U32_U24 : VOP3Inst <"v_mad_u32_u24", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;
 def V_FMA_F32 : VOP3Inst <"v_fma_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, any_fma>;
 def V_LERP_U8 : VOP3Inst <"v_lerp_u8", VOP3_Profile<VOP_I32_I32_I32_I32>, int_amdgcn_lerp>;
 
 let SchedRW = [WriteDoubleAdd] in {

llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fma.legacy.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GCN %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 < %s | FileCheck -check-prefix=GCN %s
+
+define float @v_fma(float %a, float %b, float %c)  {
+; GCN-LABEL: v_fma:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    v_fmac_legacy_f32_e64 v2, v0, v1
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fma = call float @llvm.amdgcn.fma.legacy(float %a, float %b, float %c)
+  ret float %fma
+}
+
+define float @v_fabs_fma(float %a, float %b, float %c)  {
+; GCN-LABEL: v_fabs_fma:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    v_fma_legacy_f32 v0, |v0|, v1, v2
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.a = call float @llvm.fabs.f32(float %a)
+  %fma = call float @llvm.amdgcn.fma.legacy(float %fabs.a, float %b, float %c)
+  ret float %fma
+}
+
+define float @v_fneg_fabs_fma(float %a, float %b, float %c)  {
+; GCN-LABEL: v_fneg_fabs_fma:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    v_fma_legacy_f32 v0, v0, -|v1|, v2
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %fabs.b = call float @llvm.fabs.f32(float %b)
+  %neg.fabs.b = fneg float %fabs.b
+  %fma = call float @llvm.amdgcn.fma.legacy(float %a, float %neg.fabs.b, float %c)
+  ret float %fma
+}
+
+define float @v_fneg_fma(float %a, float %b, float %c)  {
+; GCN-LABEL: v_fneg_fma:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    v_fma_legacy_f32 v0, v0, v1, -v2
+; GCN-NEXT:    ; implicit-def: $vcc_hi
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+  %neg.c = fneg float %c
+  %fma = call float @llvm.amdgcn.fma.legacy(float %a, float %b, float %neg.c)
+  ret float %fma
+}
+
+declare float @llvm.amdgcn.fma.legacy(float, float, float)
+declare float @llvm.fabs.f32(float)

llvm/test/Transforms/InstSimplify/ConstProp/AMDGPU/fma_legacy.ll

+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -instsimplify -S | FileCheck %s
+
+declare float @llvm.amdgcn.fma.legacy(float, float, float)
+
+define void @test(float* %p) {
+; CHECK-LABEL: @test(
+; CHECK-NEXT:    store volatile float 1.000000e+01, float* [[P:%.*]], align 4
+; CHECK-NEXT:    store volatile float 4.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store volatile float 4.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store volatile float 0.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store volatile float 0.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store volatile float 0.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store volatile float 0.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store volatile float 4.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store volatile float 4.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store volatile float 4.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    store volatile float 4.000000e+00, float* [[P]], align 4
+; CHECK-NEXT:    ret void
+;
+  %a = call float @llvm.amdgcn.fma.legacy(float +2.0, float +3.0, float +4.0)
+  store volatile float %a, float* %p
+  %b = call float @llvm.amdgcn.fma.legacy(float +2.0, float +0.0, float +4.0)
+  store volatile float %b, float* %p
+  %c = call float @llvm.amdgcn.fma.legacy(float +2.0, float -0.0, float +4.0)
+  store volatile float %c, float* %p
+  %d = call float @llvm.amdgcn.fma.legacy(float +0.0, float +0.0, float -0.0)
+  store volatile float %d, float* %p
+  %e = call float @llvm.amdgcn.fma.legacy(float +0.0, float -0.0, float -0.0)
+  store volatile float %e, float* %p
+  %f = call float @llvm.amdgcn.fma.legacy(float -0.0, float +0.0, float -0.0)
+  store volatile float %f, float* %p
+  %g = call float @llvm.amdgcn.fma.legacy(float -0.0, float -0.0, float -0.0)
+  store volatile float %g, float* %p
+  %h = call float @llvm.amdgcn.fma.legacy(float +0.0, float 0x7ff0000000000000, float +4.0) ; +inf
+  store volatile float %h, float* %p
+  %i = call float @llvm.amdgcn.fma.legacy(float 0xfff0000000000000, float +0.0, float +4.0) ; -inf
+  store volatile float %i, float* %p
+  %j = call float @llvm.amdgcn.fma.legacy(float 0x7ff0001000000000, float -0.0, float +4.0) ; +nan
+  store volatile float %j, float* %p
+  %k = call float @llvm.amdgcn.fma.legacy(float -0.0, float 0xfff0000100000000, float +4.0) ; -nan
+  store volatile float %k, float* %p
+  ret void
+}