Index: include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- include/llvm/CodeGen/SelectionDAGNodes.h +++ include/llvm/CodeGen/SelectionDAGNodes.h @@ -347,6 +347,7 @@ /// These are IR-level optimization flags that may be propagated to SDNodes. /// TODO: This data structure should be shared by the IR optimizer and the /// the backend. +/// Propagation of Flags from IR to SDNode is done by SDNodeFlagsAcquirer. struct SDNodeFlags { private: // This bit is used to determine if the flags are in a defined state. @@ -354,6 +355,16 @@ // are defined. bool AnyDefined : 1; + // Following two bit are used for Flags propagation from + // a DAG node to its operands. When Propagate bit is set then + // Flags from DAG node are propagated to only those operands which + // have their Acquire bit set. + // These bits are set by invocation of + // SDNodeFlagsAcquirer::PropagateFlagsToOperands and reset once the + // propagation is through. + bool PropagateFlagsToOperands : 1; + bool AcquireFlagsFromUser : 1; + bool NoUnsignedWrap : 1; bool NoSignedWrap : 1; bool Exact : 1; @@ -368,7 +379,8 @@ public: /// Default constructor turns off all optimization flags. SDNodeFlags() - : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false), + : AnyDefined(false), PropagateFlagsToOperands(false), + AcquireFlagsFromUser(false), NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), UnsafeAlgebra(false), NoNaNs(false), NoInfs(false), NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false), AllowContract(false) {} @@ -419,6 +431,8 @@ setDefined(); AllowContract = b; } + void setAcquireFlagsFromUser(bool b) { AcquireFlagsFromUser = b; } + void setPropagateFlagsToOperands(bool b) { PropagateFlagsToOperands = b; } // These are accessors for each flag. bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } @@ -432,6 +446,9 @@ bool hasVectorReduction() const { return VectorReduction; } bool hasAllowContract() const { return AllowContract; } + bool hasPropagateFlagsToOperands() const { return PropagateFlagsToOperands; } + bool hasAcquireFlagsFromUser() const { return AcquireFlagsFromUser; } + /// Clear any flags in this flag set that aren't also set in Flags. /// If the given Flags are undefined then don't do anything. void intersectWith(const SDNodeFlags Flags) { Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -661,6 +661,10 @@ SDValue getValue(const Value *V); bool findValue(const Value *V) const; + // Returns DAG node of SDValue present in NodeMap for + // a given Value. + SDNode *getDAGNode(const Value *); + SDValue getNonRegisterValue(const Value *V); SDValue getValueImpl(const Value *V); Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -84,6 +84,83 @@ "for some float libcalls"), cl::location(LimitFloatPrecision), cl::init(0)); + +static bool isVectorReductionOp(const User *I); + +/// This class is used for propagating Flags from Instruction to SDNode. +/// These flags are later used by accessing SDNode during different +/// DAG phases. +class SDNodeFlagsAcquirer { +public: + SDNodeFlagsAcquirer(const Instruction *I, SelectionDAGBuilder *SDB) + : Instr(I), SelDB(SDB) {} + + ~SDNodeFlagsAcquirer() { + SDNode *Node = SelDB->getDAGNode(Instr); + if (Node) { + SDNodeFlags InstrFlags; + SDNodeFlags Flags = Node->getFlags(); + bool PropFlagsToOperands = Flags.hasPropagateFlagsToOperands(); + + if (isa(*Instr)) { + InstrFlags.setNoNaNs(Instr->hasNoNaNs()); + InstrFlags.setNoInfs(Instr->hasNoInfs()); + InstrFlags.setUnsafeAlgebra(Instr->hasUnsafeAlgebra()); + InstrFlags.setNoSignedZeros(Instr->hasNoSignedZeros()); + InstrFlags.setAllowContract(Instr->hasAllowContract()); + InstrFlags.setAllowReciprocal(Instr->hasAllowReciprocal()); + } + + if (auto *OFBinOp = dyn_cast(Instr)) { + InstrFlags.setNoSignedWrap(OFBinOp->hasNoSignedWrap()); + InstrFlags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap()); + } + + if (auto *ExactOp = dyn_cast(Instr)) + InstrFlags.setExact(ExactOp->isExact()); + + if (isVectorReductionOp(Instr)) + InstrFlags.setVectorReduction(true); + + Flags.setAcquireFlagsFromUser(false); + Flags.setPropagateFlagsToOperands(false); + + if (!Flags.isDefined()) + Node->setFlags(InstrFlags); + else { + Flags.intersectWith(InstrFlags); + Node->setFlags(Flags); + } + + if (PropFlagsToOperands) + std::for_each(Node->op_begin(), Node->op_end(), + [&](const SDValue &Val) { + if (Val.getNode()->getFlags().hasAcquireFlagsFromUser()) + Val.getNode()->setFlags(Node->getFlags()); + }); + } + } + + // This function sets the Propagation bit over Parent DAG Node + // and Acquire bit over Operand DAG node[s] which inherits the + // flags from its parent. + static void PropagateFlagsToOperands(SDValue &Parent, + ArrayRef Operands) { + SDNodeFlags PFlags = Parent.getNode()->getFlags(); + PFlags.setPropagateFlagsToOperands(true); + Parent.getNode()->setFlags(PFlags); + + SDNodeFlags CFlags; + CFlags.setAcquireFlagsFromUser(true); + for (auto &Val : Operands) + Val.getNode()->setFlags(CFlags); + } + +private: + const Instruction *Instr; + SelectionDAGBuilder *SelDB; +}; + // Limit the width of DAG chains. This is important in general to prevent // DAG-based analysis from blowing up. For example, alias analysis and // load clustering may not complete in reasonable time. It is difficult to @@ -977,6 +1054,8 @@ } void SelectionDAGBuilder::visit(const Instruction &I) { + SDNodeFlagsAcquirer Flags(&I, this); + // Set up outgoing PHI node register values before emitting the terminator. if (isa(&I)) { HandlePHINodesInSuccessorBlocks(I.getParent()); @@ -1058,6 +1137,12 @@ return Result; } +SDNode * SelectionDAGBuilder::getDAGNode(const Value *V) { + if (NodeMap.find(V) == NodeMap.end()) + return nullptr; + return NodeMap[V].getNode(); +} + /// getValue - Return an SDValue for the given Value. SDValue SelectionDAGBuilder::getValue(const Value *V) { // If we already have an SDValue for this value, use it. It's important @@ -2637,42 +2722,11 @@ SDValue Op1 = getValue(I.getOperand(0)); SDValue Op2 = getValue(I.getOperand(1)); - bool nuw = false; - bool nsw = false; - bool exact = false; - bool vec_redux = false; - FastMathFlags FMF; - - if (const OverflowingBinaryOperator *OFBinOp = - dyn_cast(&I)) { - nuw = OFBinOp->hasNoUnsignedWrap(); - nsw = OFBinOp->hasNoSignedWrap(); - } - if (const PossiblyExactOperator *ExactOp = - dyn_cast(&I)) - exact = ExactOp->isExact(); - if (const FPMathOperator *FPOp = dyn_cast(&I)) - FMF = FPOp->getFastMathFlags(); - - if (isVectorReductionOp(&I)) { - vec_redux = true; + if (isVectorReductionOp(&I)) DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n"); - } - - SDNodeFlags Flags; - Flags.setExact(exact); - Flags.setNoSignedWrap(nsw); - Flags.setNoUnsignedWrap(nuw); - Flags.setVectorReduction(vec_redux); - Flags.setAllowReciprocal(FMF.allowReciprocal()); - Flags.setAllowContract(FMF.allowContract()); - Flags.setNoInfs(FMF.noInfs()); - Flags.setNoNaNs(FMF.noNaNs()); - Flags.setNoSignedZeros(FMF.noSignedZeros()); - Flags.setUnsafeAlgebra(FMF.unsafeAlgebra()); SDValue BinNodeValue = DAG.getNode(OpCode, getCurSDLoc(), Op1.getValueType(), - Op1, Op2, Flags); + Op1, Op2); setValue(&I, BinNodeValue); } @@ -5491,6 +5545,8 @@ getValue(I.getArgOperand(0)).getValueType(), Mul, getValue(I.getArgOperand(2))); + + SDNodeFlagsAcquirer::PropagateFlagsToOperands(Add,{Mul}); setValue(&I, Add); } return nullptr; @@ -7910,8 +7966,6 @@ FastMathFlags FMF; if (isa(I)) FMF = I.getFastMathFlags(); - SDNodeFlags SDFlags; - SDFlags.setNoNaNs(FMF.noNaNs()); switch (Intrinsic) { case Intrinsic::experimental_vector_reduce_fadd: @@ -7954,11 +8008,11 @@ Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1); break; case Intrinsic::experimental_vector_reduce_fmax: { - Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags); + Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1); break; } case Intrinsic::experimental_vector_reduce_fmin: { - Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags); + Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1); break; } default: