Index: lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -130,6 +130,7 @@ SDValue ExpandBITREVERSE(SDValue Op); SDValue ExpandCTLZ(SDValue Op); SDValue ExpandCTTZ_ZERO_UNDEF(SDValue Op); + SDValue ExpandStrictFPOp(SDValue Op); /// Implements vector promotion. /// @@ -288,10 +289,30 @@ if (!HasVectorValue) return TranslateLegalizeResults(Op, Result); - EVT QueryType; + TargetLowering::LegalizeAction Action = TargetLowering::Legal; switch (Op.getOpcode()) { default: return TranslateLegalizeResults(Op, Result); + case ISD::STRICT_FSQRT: + case ISD::STRICT_FMA: + case ISD::STRICT_FPOW: + case ISD::STRICT_FPOWI: + case ISD::STRICT_FSIN: + case ISD::STRICT_FCOS: + case ISD::STRICT_FEXP: + case ISD::STRICT_FEXP2: + case ISD::STRICT_FLOG: + case ISD::STRICT_FLOG10: + case ISD::STRICT_FLOG2: + case ISD::STRICT_FRINT: + case ISD::STRICT_FNEARBYINT: + // These pseudo-ops get legalized as if they were their non-strict + // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT + // is also legal, but if ISD::FSQRT requires expansion then so does + // ISD::STRICT_FSQRT. + Action = TLI.getStrictFPOperationAction(Node->getOpcode(), + Node->getValueType(0)); + break; case ISD::ADD: case ISD::SUB: case ISD::MUL: @@ -368,26 +389,30 @@ case ISD::SMUL_LOHI: case ISD::UMUL_LOHI: case ISD::FCANONICALIZE: - QueryType = Node->getValueType(0); + Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); break; case ISD::FP_ROUND_INREG: - QueryType = cast(Node->getOperand(1))->getVT(); + Action = TLI.getOperationAction(Node->getOpcode(), + cast(Node->getOperand(1))->getVT()); break; case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: - QueryType = Node->getOperand(0).getValueType(); + Action = TLI.getOperationAction(Node->getOpcode(), + Node->getOperand(0).getValueType()); break; case ISD::MSCATTER: - QueryType = cast(Node)->getValue().getValueType(); + Action = TLI.getOperationAction(Node->getOpcode(), + cast(Node)->getValue().getValueType()); break; case ISD::MSTORE: - QueryType = cast(Node)->getValue().getValueType(); + Action = TLI.getOperationAction(Node->getOpcode(), + cast(Node)->getValue().getValueType()); break; } LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG)); - switch (TLI.getOperationAction(Node->getOpcode(), QueryType)) { + switch (Action) { default: llvm_unreachable("This action is not supported yet!"); case TargetLowering::Promote: Result = Promote(Op); @@ -700,6 +725,20 @@ return ExpandCTLZ(Op); case ISD::CTTZ_ZERO_UNDEF: return ExpandCTTZ_ZERO_UNDEF(Op); + case ISD::STRICT_FSQRT: + case ISD::STRICT_FMA: + case ISD::STRICT_FPOW: + case ISD::STRICT_FPOWI: + case ISD::STRICT_FSIN: + case ISD::STRICT_FCOS: + case ISD::STRICT_FEXP: + case ISD::STRICT_FEXP2: + case ISD::STRICT_FLOG: + case ISD::STRICT_FLOG10: + case ISD::STRICT_FLOG2: + case ISD::STRICT_FRINT: + case ISD::STRICT_FNEARBYINT: + return ExpandStrictFPOp(Op); default: return DAG.UnrollVectorOp(Op.getNode()); } @@ -1086,6 +1125,51 @@ return DAG.UnrollVectorOp(Op.getNode()); } +SDValue VectorLegalizer::ExpandStrictFPOp(SDValue Op) { + EVT VT = Op.getValueType(); + EVT EltVT = VT.getVectorElementType(); + unsigned NumElems = VT.getVectorNumElements(); + unsigned NumOpers = Op.getNumOperands(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + ArrayRef ValueVTs = {EltVT, MVT::Other}; + SDValue Chain = Op.getOperand(0); + SDValue Result; + SDValue NewChain; + SDLoc dl(Op); + + SmallVector OpValues; + SmallVector OpChains; + for (unsigned i = 0; i < NumElems; ++i) { + SDValue ScalarOp; + SmallVector Opers; + SDValue Idx = DAG.getConstant(i, dl, + TLI.getVectorIdxTy(DAG.getDataLayout())); + + // The Chain is the first operand. + Opers.push_back(Chain); + + // Now process the remaining operands. + for (unsigned j = 1; j < NumOpers; ++j) { + SDValue Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, + EltVT, Op.getOperand(j), Idx); + Opers.push_back(Oper); + } + + ScalarOp = DAG.getNode(Op->getOpcode(), dl, ValueVTs, Opers); + + OpValues.push_back(ScalarOp.getValue(0)); + OpChains.push_back(ScalarOp.getValue(1)); + } + + Result = DAG.getBuildVector(VT, dl, OpValues); + NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains); + + AddLegalizedOperand(Op.getValue(0), Result); + AddLegalizedOperand(Op.getValue(1), NewChain); + + return NewChain; +} + SDValue VectorLegalizer::UnrollVSETCC(SDValue Op) { EVT VT = Op.getValueType(); unsigned NumElems = VT.getVectorNumElements(); Index: test/CodeGen/X86/vector-constrained-fp-intrinsics.ll =================================================================== --- test/CodeGen/X86/vector-constrained-fp-intrinsics.ll +++ test/CodeGen/X86/vector-constrained-fp-intrinsics.ll @@ -0,0 +1,16 @@ +; RUN: llc -O3 -mtriple=x86_64-pc-linux < %s | FileCheck --check-prefix=COMMON --check-prefix=NO-FMA --check-prefix=FMACALL64 --check-prefix=FMACALL32 %s +; RUN: llc -O3 -mtriple=x86_64-pc-linux -mattr=+fma < %s | FileCheck -check-prefix=COMMON --check-prefix=HAS-FMA --check-prefix=FMA64 --check-prefix=FMA32 %s + +; CHECK-LABEL: constrained_vector_pow +; COMMON: pow +define <2 x double> @constrained_vector_pow() { +entry: + %pow = call <2 x double> @llvm.experimental.constrained.pow.v2f64( + <2 x double> , + <2 x double> , + metadata !"round.dynamic", + metadata !"fpexcept.strict") + ret <2 x double> %pow +} + +declare <2 x double> @llvm.experimental.constrained.pow.v2f64(<2 x double>, <2 x double>, metadata, metadata)