[VP] fm flag transfer to SDNodes [VE] VVP_FMA fusion

[simoll]

Summary

On the VP side: Fast-math flag transfer from VP intrinsics to VP SDNodes. Tested through the VE parts of this patch.
For VE: Custom fusion code for VVP_FMA. Yes, there could/should be a generic DAGCombiner path for this, however, this is just the first in a series of VE-specific combiner patterns.

[ABataev]

llvm/include/llvm/IR/Operator.h

306

Better to expand auto here to the actual type.

[simoll]

explicit Optional<unsigned>

[craig.topper]

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

7530

Is this correct if the getNode call CSEs with an existing node? Should probably pass the flags to getNode.

[simoll]

Pass NodeFlags in getNode.
Rebased.

[simoll]

Fixing an SelectionDAG bug where Flags wouldn't be set in some cases of SelectionDAG::getNode. This bug fix may have (positive) trickle down effects on other targets.

[craig.topper]

In D121714#3388510, @simoll wrote:

Fixing an SelectionDAG bug where Flags wouldn't be set in some cases of SelectionDAG::getNode. This bug fix may have (positive) trickle down effects on other targets.

Thanks!

The target independent parts look good to me. I didn't look too closely at the VE changes.

[craig.topper]

llvm/lib/Target/VE/CMakeLists.txt

30

Should this be alphabetized before VVPISelLowering?

llvm/lib/Target/VE/VVPCombine.cpp

29

Do you need to check that the FADD is contractable too? I think that's what DAGCombiner.cpp does but the different ways of specifiying fast math make it hard to be sure.

[simoll]

• alphabetic file order in the CMakeLists.txt
• unchanged, just to reiterate: contract only checked on the fmul, not on the fadd.
• formatting

[simoll]

llvm/lib/Target/VE/VVPCombine.cpp

29

The specification isn't clear on this; from the LangRef:

contract

    Allow floating-point contraction (e.g. fusing a multiply followed by an addition into a fused multiply-and-add). This does not enable reassociating to form arbitrary contractions. For example, (a*b) + (c*d) + e can not be transformed into (a*b) + ((c*d) + e) to create two fma operations.

I agree with you. Intuitively, the contract only needs to be on the node that is contracted into another node. Since contraction (in the sense of FMA) means removing the implicit rounding step after the fmul. The rounding after the fadd is unaffacted by this.

llvm/include/llvm/IR/Operator.h

 #define LLVM_IR_OPERATOR_H
 
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/FMF.h"
 #include "llvm/IR/Instruction.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
 #include <cstddef>
 
 namespace llvm {
 
 /// This is a utility class that provides an abstraction for the common
static bool classof(const Value *V) {
     unsigned Opcode;
     if (auto *I = dyn_cast<Instruction>(V))
       Opcode = I->getOpcode();
     else if (auto *CE = dyn_cast<ConstantExpr>(V))
       Opcode = CE->getOpcode();
     else
       return false;
 
+    // Pass this VPIntrinsic as its underlying operation.
+    if (auto *VPI = dyn_cast<VPIntrinsic>(V))
+      if (Optional<unsigned> OpcodeOpt = VPI->getFunctionalOpcode())
+        Opcode = OpcodeOpt.getValue();
+
     switch (Opcode) {
     case Instruction::FNeg:
     case Instruction::FAdd:
     case Instruction::FSub:
     case Instruction::FMul:
     case Instruction::FDiv:
     case Instruction::FRem:
     // FIXME: To clean up and correct the semantics of fast-math-flags, FCmp

llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp

#endif
   SDNode *N;
   SDVTList VTs = getVTList(VT);
 
   if (VT != MVT::Glue) {
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTs, Ops);
     void *IP = nullptr;
 
-    if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
+
+    if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
+      E->intersectFlagsWith(Flags);
       return SDValue(E, 0);
+    }
 
     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
     createOperands(N, Ops);
 
     CSEMap.InsertNode(N, IP);
   } else {
     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
     createOperands(N, Ops);
SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
   if (Inserter)
     Flags = Inserter->getFlags();
   return getNode(Opcode, DL, VTList, Ops, Flags);
 }
 
 SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
                               ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
   if (VTList.NumVTs == 1)
-    return getNode(Opcode, DL, VTList.VTs[0], Ops);
+    return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags);
 
 #ifndef NDEBUG
   for (auto &Op : Ops)
     assert(Op.getOpcode() != ISD::DELETED_NODE &&
            "Operand is DELETED_NODE!");
 #endif
 
   switch (Opcode) {
#endif
   }
 
   // Memoize the node unless it returns a flag.
   SDNode *N;
   if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
     FoldingSetNodeID ID;
     AddNodeIDNode(ID, Opcode, VTList, Ops);
     void *IP = nullptr;
-    if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
+    if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
+      E->intersectFlagsWith(Flags);
       return SDValue(E, 0);
+    }
 
     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
     createOperands(N, Ops);
     CSEMap.InsertNode(N, IP);
   } else {
     N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
     createOperands(N, Ops);
   }

llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) {
     auto Op = getValue(VPIntrin.getArgOperand(I));
     if (I == EVLParamPos)
       Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op);
     OpValues.push_back(Op);
   }
 
   switch (Opcode) {
   default: {
-    SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues);
+    SDNodeFlags NodeFlags;
+    if (auto *FPIntrin = dyn_cast<FPMathOperator>(&VPIntrin)) {
+      NodeFlags.copyFMF(*FPIntrin);
+      NodeFlags.setNoFPExcept(true); // TODO: Constrained VP
+    }
+    SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, NodeFlags);
+    // Transfer FMF and exception flags.
     setValue(&VPIntrin, Result);
     break;
   }
   case ISD::VP_LOAD:
   case ISD::VP_GATHER:
     visitVPLoadGather(VPIntrin, ValueVTs[0], OpValues,
                       Opcode == ISD::VP_GATHER);
     break;

llvm/lib/Target/VE/CMakeLists.txt

add_llvm_target(VECodeGen
   VEISelLowering.cpp
   VEInstrInfo.cpp
   VEMachineFunctionInfo.cpp
   VEMCInstLower.cpp
   VERegisterInfo.cpp
   VESubtarget.cpp
   VETargetMachine.cpp
   VVPISelLowering.cpp
+  VVPCombine.cpp
 
   LINK_COMPONENTS
   Analysis
   AsmPrinter
   CodeGen
   Core
   MC
   SelectionDAG

llvm/lib/Target/VE/VEISelLowering.h

public:
   SDValue splitVectorOp(SDValue Op, VECustomDAG &CDAG) const;
   SDValue splitPackedLoadStore(SDValue Op, VECustomDAG &CDAG) const;
   SDValue legalizePackedAVL(SDValue Op, VECustomDAG &CDAG) const;
   SDValue splitMaskArithmetic(SDValue Op, SelectionDAG &DAG) const;
   /// } VVPLowering
 
   /// Custom DAGCombine {
   SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
+  SDValue combineVVP(SDNode *N, DAGCombinerInfo &DCI) const;
 
   SDValue combineTRUNCATE(SDNode *N, DAGCombinerInfo &DCI) const;
   /// } Custom DAGCombine
 
   SDValue withTargetFlags(SDValue Op, unsigned TF, SelectionDAG &DAG) const;
   SDValue makeHiLoPair(SDValue Op, unsigned HiTF, unsigned LoTF,
                        SelectionDAG &DAG) const;
   SDValue makeAddress(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/VE/VEISelLowering.cpp

return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, VT,
                                     N->getOperand(0), SubI32),
                  0);
 }
 
 SDValue VETargetLowering::PerformDAGCombine(SDNode *N,
                                             DAGCombinerInfo &DCI) const {
   switch (N->getOpcode()) {
   default:
+    if (isVVPOrVEC(N->getOpcode()))
+      return combineVVP(N, DCI);
     break;
   case ISD::TRUNCATE:
     return combineTRUNCATE(N, DCI);
   }
 
   return SDValue();
 }
 

llvm/lib/Target/VE/VVPCombine.cpp

+#include "VECustomDAG.h"
+
+#ifdef DEBUG_TYPE
+#undef DEBUG_TYPE
+#endif
+#define DEBUG_TYPE "vvp-combine"
+
+using namespace llvm;
+
+using Matcher = std::function<bool(SDValue)>;
+
+static Optional<unsigned> match_SomeOperand(SDNode *Op, unsigned VVPOpcode,
+                                            Matcher M) {
+  for (unsigned i = 0; i < 2; ++i) {
+    if ((Op->getOperand(i)->getOpcode() == VVPOpcode) && M(Op->getOperand(i)))
+      return i;
+  }
+  return None;
+}
+
+// vz * vw + vy
+static bool match_FFMA(SDNode *Root, SDValue &VY, SDValue &VZ, SDValue &VW,
+                       SDValue &Mask, SDValue &AVL) {
+  if (Root->getOpcode() != VEISD::VVP_FADD)
+    return false;
+
+  // Detect contractable FMUL leaf.
+  auto MulIdx = match_SomeOperand(Root, VEISD::VVP_FMUL, [](SDValue Op) {
+    return Op->hasOneUse() && Op->getFlags().hasAllowContract();
+  });
+  if (!MulIdx.hasValue())
+    return false;
+  assert(MulIdx.getValue() < 2);
+  const int LeafIdx = 1 - MulIdx.getValue();
+
+  // Take apart.
+  SDValue MulV = Root->getOperand(*MulIdx);
+  VY = Root->getOperand(LeafIdx);
+  VZ = MulV->getOperand(0);
+  VW = MulV->getOperand(1);
+  Mask = Root->getOperand(2);
+  AVL = Root->getOperand(3);
+  return true;
+}
+
+SDValue VETargetLowering::combineVVP(SDNode *N, DAGCombinerInfo &DCI) const {
+  VECustomDAG CDAG(DCI.DAG, N);
+  SDNodeFlags Flags = N->getFlags();
+  MVT ResVT = N->getSimpleValueType(0);
+  switch (N->getOpcode()) {
+    // Fuse FMA, FMSB, FNMA, FNMSB, ..
+  case VEISD::VVP_FADD: {
+    SDValue VY, VZ, VW, Mask, AVL;
+    if (match_FFMA(N, VY, VZ, VW, Mask, AVL))
+      return CDAG.getNode(VEISD::VVP_FFMA, ResVT, {VY, VZ, VW, Mask, AVL},
+                          Flags);
+  } break;
+  }
+  return SDValue();
+}

llvm/lib/Target/VE/VVPISelLowering.cpp

SDValue VETargetLowering::lowerToVVP(SDValue Op, SelectionDAG &DAG) const {
   if (!Mask)
     Mask = CDAG.getConstantMask(Packing, true);
 
   assert(LegalVecVT.isSimple());
   if (isVVPUnaryOp(VVPOpcode))
     return CDAG.getNode(VVPOpcode, LegalVecVT, {Op->getOperand(0), Mask, AVL});
   if (isVVPBinaryOp(VVPOpcode))
     return CDAG.getNode(VVPOpcode, LegalVecVT,
-                        {Op->getOperand(0), Op->getOperand(1), Mask, AVL});
+                        {Op->getOperand(0), Op->getOperand(1), Mask, AVL},
+                        Op->getFlags());
   if (isVVPReductionOp(VVPOpcode)) {
     auto SrcHasStart = hasReductionStartParam(Op->getOpcode());
     SDValue StartV = SrcHasStart ? Op->getOperand(0) : SDValue();
     SDValue VectorV = Op->getOperand(SrcHasStart ? 1 : 0);
     return CDAG.getLegalReductionOpVVP(VVPOpcode, Op.getValueType(), StartV,
                                        VectorV, Mask, AVL, Op->getFlags());
   }
 

llvm/test/CodeGen/VE/Vector/fuse_vp_fma.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -march=ve -mattr=+vpu | FileCheck %s
+
+define fastcc <256 x float> @test_vp_ffma_vvv_256f32(<256 x float> %i0, <256 x float> %i1, <256 x float> %i2, <256 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_ffma_vvv_256f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmad.s %v0, %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %mul = call contract <256 x float> @llvm.vp.fmul.v256f32(<256 x float> %i0, <256 x float> %i1, <256 x i1> %m, i32 %n)
+  %fma = call contract <256 x float> @llvm.vp.fadd.v256f32(<256 x float> %mul, <256 x float> %i2, <256 x i1> %m, i32 %n)
+  ret <256 x float> %fma
+}
+
+define fastcc <256 x float> @test_vp_ffma_rvv_256f32(float %s0, <256 x float> %i1, <256 x float> %i2, <256 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_ffma_rvv_256f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.s %v0, %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %b0 = insertelement <256 x float> undef, float %s0, i32 0
+  %i0 = shufflevector <256 x float> %b0, <256 x float> poison, <256 x i32> zeroinitializer
+  %mul = call contract <256 x float> @llvm.vp.fmul.v256f32(<256 x float> %i0, <256 x float> %i1, <256 x i1> %m, i32 %n)
+  %fma = call contract <256 x float> @llvm.vp.fadd.v256f32(<256 x float> %mul, <256 x float> %i2, <256 x i1> %m, i32 %n)
+  ret <256 x float> %fma
+}
+
+define fastcc <256 x float> @test_vp_ffma_vrv_256f32(<256 x float> %i0, float %s1, <256 x float> %i2, <256 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_ffma_vrv_256f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.s %v0, %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %b1 = insertelement <256 x float> undef, float %s1, i32 0
+  %i1 = shufflevector <256 x float> %b1, <256 x float> poison, <256 x i32> zeroinitializer
+  %mul = call contract <256 x float> @llvm.vp.fmul.v256f32(<256 x float> %i0, <256 x float> %i1, <256 x i1> %m, i32 %n)
+  %fma = call contract <256 x float> @llvm.vp.fadd.v256f32(<256 x float> %mul, <256 x float> %i2, <256 x i1> %m, i32 %n)
+  ret <256 x float> %fma
+}
+
+define fastcc <256 x float> @test_vp_ffma_vvr_256f32(<256 x float> %i0, <256 x float> %i1, float %s2, <256 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_ffma_vvr_256f32:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.s %v0, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %b2 = insertelement <256 x float> undef, float %s2, i32 0
+  %i2 = shufflevector <256 x float> %b2, <256 x float> poison, <256 x i32> zeroinitializer
+  %mul = call contract <256 x float> @llvm.vp.fmul.v256f32(<256 x float> %i0, <256 x float> %i1, <256 x i1> %m, i32 %n)
+  %fma = call contract <256 x float> @llvm.vp.fadd.v256f32(<256 x float> %mul, <256 x float> %i2, <256 x i1> %m, i32 %n)
+  ret <256 x float> %fma
+}
+
+declare <256 x float> @llvm.vp.fadd.v256f32(<256 x float>, <256 x float>, <256 x i1>, i32)
+declare <256 x float> @llvm.vp.fmul.v256f32(<256 x float>, <256 x float>, <256 x i1>, i32)
+
+;;; 256 x double
+
+define fastcc <256 x double> @test_vp_ffma_vvv_256f64(<256 x double> %i0, <256 x double> %i1, <256 x double> %i2, <256 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_ffma_vvv_256f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s0, %s0, (32)0
+; CHECK-NEXT:    lvl %s0
+; CHECK-NEXT:    vfmad.d %v0, %v2, %v0, %v1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %mul = call contract <256 x double> @llvm.vp.fmul.v256f64(<256 x double> %i0, <256 x double> %i1, <256 x i1> %m, i32 %n)
+  %fma = call contract <256 x double> @llvm.vp.fadd.v256f64(<256 x double> %mul, <256 x double> %i2, <256 x i1> %m, i32 %n)
+  ret <256 x double> %fma
+}
+
+define fastcc <256 x double> @test_vp_ffma_rvv_256f64(double %s0, <256 x double> %i1, <256 x double> %i2, <256 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_ffma_rvv_256f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.d %v0, %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %b0 = insertelement <256 x double> undef, double %s0, i32 0
+  %i0 = shufflevector <256 x double> %b0, <256 x double> poison, <256 x i32> zeroinitializer
+  %mul = call contract <256 x double> @llvm.vp.fmul.v256f64(<256 x double> %i0, <256 x double> %i1, <256 x i1> %m, i32 %n)
+  %fma = call contract <256 x double> @llvm.vp.fadd.v256f64(<256 x double> %mul, <256 x double> %i2, <256 x i1> %m, i32 %n)
+  ret <256 x double> %fma
+}
+
+define fastcc <256 x double> @test_vp_ffma_vrv_256f64(<256 x double> %i0, double %s1, <256 x double> %i2, <256 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_ffma_vrv_256f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.d %v0, %v1, %s0, %v0, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %b1 = insertelement <256 x double> undef, double %s1, i32 0
+  %i1 = shufflevector <256 x double> %b1, <256 x double> poison, <256 x i32> zeroinitializer
+  %mul = call contract <256 x double> @llvm.vp.fmul.v256f64(<256 x double> %i0, <256 x double> %i1, <256 x i1> %m, i32 %n)
+  %fma = call contract <256 x double> @llvm.vp.fadd.v256f64(<256 x double> %mul, <256 x double> %i2, <256 x i1> %m, i32 %n)
+  ret <256 x double> %fma
+}
+
+define fastcc <256 x double> @test_vp_ffma_vvr_256f64(<256 x double> %i0, <256 x double> %i1, double %s2, <256 x i1> %m, i32 %n) {
+; CHECK-LABEL: test_vp_ffma_vvr_256f64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    and %s1, %s1, (32)0
+; CHECK-NEXT:    lvl %s1
+; CHECK-NEXT:    vfmad.d %v0, %s0, %v0, %v1, %vm1
+; CHECK-NEXT:    b.l.t (, %s10)
+  %b2 = insertelement <256 x double> undef, double %s2, i32 0
+  %i2 = shufflevector <256 x double> %b2, <256 x double> poison, <256 x i32> zeroinitializer
+  %mul = call contract <256 x double> @llvm.vp.fmul.v256f64(<256 x double> %i0, <256 x double> %i1, <256 x i1> %m, i32 %n)
+  %fma = call contract <256 x double> @llvm.vp.fadd.v256f64(<256 x double> %mul, <256 x double> %i2, <256 x i1> %m, i32 %n)
+  ret <256 x double> %fma
+}
+
+declare <256 x double> @llvm.vp.fadd.v256f64(<256 x double>, <256 x double>, <256 x i1>, i32)
+declare <256 x double> @llvm.vp.fmul.v256f64(<256 x double>, <256 x double>, <256 x i1>, i32)