Index: include/llvm/CodeGen/ISDOpcodes.h =================================================================== --- include/llvm/CodeGen/ISDOpcodes.h +++ include/llvm/CodeGen/ISDOpcodes.h @@ -290,6 +290,7 @@ STRICT_FEXP, STRICT_FEXP2, STRICT_FLOG, STRICT_FLOG10, STRICT_FLOG2, STRICT_FRINT, STRICT_FNEARBYINT, STRICT_FMAXNUM, STRICT_FMINNUM, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND, STRICT_FTRUNC, + STRICT_FSETCC, /// FMA - Perform a * b + c with no intermediate rounding step. FMA, Index: include/llvm/CodeGen/SelectionDAGNodes.h =================================================================== --- include/llvm/CodeGen/SelectionDAGNodes.h +++ include/llvm/CodeGen/SelectionDAGNodes.h @@ -678,6 +678,7 @@ case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FSETCC: return true; } } Index: include/llvm/CodeGen/TargetLowering.h =================================================================== --- include/llvm/CodeGen/TargetLowering.h +++ include/llvm/CodeGen/TargetLowering.h @@ -833,13 +833,14 @@ case ISD::STRICT_FFLOOR: EqOpc = ISD::FFLOOR; break; case ISD::STRICT_FROUND: EqOpc = ISD::FROUND; break; case ISD::STRICT_FTRUNC: EqOpc = ISD::FTRUNC; break; + case ISD::STRICT_FSETCC: EqOpc = ISD::SETCC; break; } auto Action = getOperationAction(EqOpc, VT); - // We don't currently handle Custom or Promote for strict FP pseudo-ops. + // We don't currently handle Promote for strict FP pseudo-ops. // For now, we just expand for those cases. - if (Action != Legal) + if (Action != Legal && Action != Custom) Action = Expand; return Action; Index: include/llvm/CodeGen/ValueTypes.h =================================================================== --- include/llvm/CodeGen/ValueTypes.h +++ include/llvm/CodeGen/ValueTypes.h @@ -205,7 +205,8 @@ /// Return true if this is an overloaded type for TableGen. bool isOverloaded() const { - return (V==MVT::iAny || V==MVT::fAny || V==MVT::vAny || V==MVT::iPTRAny); + return (V==MVT::bAny || V==MVT::iAny || V==MVT::fAny || + V==MVT::vAny || V==MVT::iPTRAny); } /// Return true if the bit size is a multiple of 8. Index: include/llvm/CodeGen/ValueTypes.td =================================================================== --- include/llvm/CodeGen/ValueTypes.td +++ include/llvm/CodeGen/ValueTypes.td @@ -146,8 +146,11 @@ def isVoid : ValueType<0 , 111>; // Produces no value def untyped: ValueType<8 , 112>; // Produces an untyped value def ExceptRef: ValueType<0, 113>; // WebAssembly's except_ref type -def token : ValueType<0 , 248>; // TokenTy -def MetadataVT: ValueType<0, 249>; // Metadata +def token : ValueType<0 , 247>; // TokenTy +def MetadataVT: ValueType<0, 248>; // Metadata + +// Pseudo valuetype to represent "bool of any format" +def bAny : ValueType<0 , 249>; // Pseudo valuetype mapped to the current pointer size to any address space. // Should only be used in TableGen. Index: include/llvm/IR/IntrinsicInst.h =================================================================== --- include/llvm/IR/IntrinsicInst.h +++ include/llvm/IR/IntrinsicInst.h @@ -257,6 +257,7 @@ case Intrinsic::experimental_constrained_floor: case Intrinsic::experimental_constrained_round: case Intrinsic::experimental_constrained_trunc: + case Intrinsic::experimental_constrained_fcmpeq: return true; default: return false; } Index: include/llvm/IR/Intrinsics.h =================================================================== --- include/llvm/IR/Intrinsics.h +++ include/llvm/IR/Intrinsics.h @@ -100,7 +100,8 @@ Void, VarArg, MMX, Token, Metadata, Half, Float, Double, Quad, Integer, Vector, Pointer, Struct, Argument, ExtendArgument, TruncArgument, HalfVecArgument, - SameVecWidthArgument, PtrToArgument, PtrToElt, VecOfAnyPtrsToElt + SameVecWidthArgument, PtrToArgument, PtrToElt, VecOfAnyPtrsToElt, + SameVecLengthOrScalarArgument } Kind; union { @@ -117,31 +118,33 @@ AK_AnyInteger, AK_AnyFloat, AK_AnyVector, - AK_AnyPointer + AK_AnyPointer, + AK_AnyBoolean }; unsigned getArgumentNumber() const { assert(Kind == Argument || Kind == ExtendArgument || Kind == TruncArgument || Kind == HalfVecArgument || Kind == SameVecWidthArgument || Kind == PtrToArgument || - Kind == PtrToElt); + Kind == PtrToElt || Kind == SameVecLengthOrScalarArgument); return Argument_Info >> 3; } ArgKind getArgumentKind() const { assert(Kind == Argument || Kind == ExtendArgument || Kind == TruncArgument || Kind == HalfVecArgument || - Kind == SameVecWidthArgument || Kind == PtrToArgument); + Kind == SameVecWidthArgument || Kind == PtrToArgument || + Kind == SameVecLengthOrScalarArgument); return (ArgKind)(Argument_Info & 7); } // VecOfAnyPtrsToElt uses both an overloaded argument (for address space) // and a reference argument (for matching vector width and element types) unsigned getOverloadArgNumber() const { - assert(Kind == VecOfAnyPtrsToElt); + assert(Kind == VecOfAnyPtrsToElt || Kind == SameVecLengthOrScalarArgument); return Argument_Info >> 16; } unsigned getRefArgNumber() const { - assert(Kind == VecOfAnyPtrsToElt); + assert(Kind == VecOfAnyPtrsToElt || Kind == SameVecLengthOrScalarArgument); return Argument_Info & 0xFFFF; } Index: include/llvm/IR/Intrinsics.td =================================================================== --- include/llvm/IR/Intrinsics.td +++ include/llvm/IR/Intrinsics.td @@ -161,6 +161,12 @@ : LLVMMatchType { ValueType ElTy = elty.VT; } + +class LLVMVectorSameLengthOrScalar + : LLVMMatchType { + LLVMType ElTy = elty; +} + class LLVMPointerTo : LLVMMatchType; class LLVMPointerToElt : LLVMMatchType; class LLVMVectorOfAnyPointersToElt : LLVMMatchType; @@ -172,6 +178,7 @@ def llvm_void_ty : LLVMType; let isAny = 1 in { def llvm_any_ty : LLVMType; + def llvm_anybool_ty : LLVMType; def llvm_anyint_ty : LLVMType; def llvm_anyfloat_ty : LLVMType; def llvm_anyvector_ty : LLVMType; @@ -596,6 +603,12 @@ [ LLVMMatchType<0>, llvm_metadata_ty, llvm_metadata_ty ]>; + + def int_experimental_constrained_fcmpeq : Intrinsic<[ llvm_anybool_ty ], + [ LLVMVectorSameLengthOrScalar<0, llvm_anyfloat_ty>, + LLVMVectorSameLengthOrScalar<0, llvm_anyfloat_ty>, + llvm_metadata_ty, + llvm_metadata_ty ]>; } // FIXME: Add intrinsics for fcmp, fptrunc, fpext, fptoui and fptosi. // FIXME: Add intrinsics for fabs and copysign? Index: include/llvm/Support/MachineValueType.h =================================================================== --- include/llvm/Support/MachineValueType.h +++ include/llvm/Support/MachineValueType.h @@ -44,6 +44,9 @@ i64 = 6, // This is a 64 bit integer value i128 = 7, // This is a 128 bit integer value + FIRST_BOOLEAN_VALUETYPE = i1, + LAST_BOOLEAN_VALUETYPE = i1, + FIRST_INTEGER_VALUETYPE = i1, LAST_INTEGER_VALUETYPE = i128, @@ -140,6 +143,9 @@ nxv16i64 = 83, // n x 16 x i64 nxv32i64 = 84, // n x 32 x i64 + FIRST_BOOLEAN_VECTOR_VALUETYPE = v1i1, + LAST_BOOLEAN_VECTOR_VALUETYPE = v1024i1, + FIRST_INTEGER_VECTOR_VALUETYPE = v1i1, LAST_INTEGER_VECTOR_VALUETYPE = nxv32i64, @@ -202,10 +208,15 @@ MAX_ALLOWED_VALUETYPE = 128, // A value of type llvm::TokenTy - token = 248, + token = 247, // This is MDNode or MDString. - Metadata = 249, + Metadata = 248, + + // An i1 or vector i1 value. This is used for intrinsics + // that have overloadings based on integer bit widths. + // This is only for tblgen's consumption! + bAny = 249, // An int value the size of the pointer of the current // target to any address space. This must only be used internal to @@ -386,7 +397,7 @@ /// Return true if this is an overloaded type for TableGen. bool isOverloaded() const { - return (SimpleTy==MVT::Any || + return (SimpleTy==MVT::Any || SimpleTy==MVT::bAny || SimpleTy==MVT::iAny || SimpleTy==MVT::fAny || SimpleTy==MVT::vAny || SimpleTy==MVT::iPTRAny); } @@ -630,6 +641,7 @@ case iPTR: llvm_unreachable("Value type size is target-dependent. Ask TLI."); case iPTRAny: + case bAny: case iAny: case fAny: case vAny: @@ -1014,6 +1026,11 @@ return mvt_range(MVT::FIRST_VALUETYPE, MVT::LAST_VALUETYPE); } + static mvt_range boolean_valuetypes() { + return mvt_range(MVT::FIRST_BOOLEAN_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_BOOLEAN_VALUETYPE + 1)); + } + static mvt_range integer_valuetypes() { return mvt_range(MVT::FIRST_INTEGER_VALUETYPE, (MVT::SimpleValueType)(MVT::LAST_INTEGER_VALUETYPE + 1)); @@ -1029,6 +1046,12 @@ (MVT::SimpleValueType)(MVT::LAST_VECTOR_VALUETYPE + 1)); } + static mvt_range boolean_vector_valuetypes() { + return mvt_range( + MVT::FIRST_BOOLEAN_VECTOR_VALUETYPE, + (MVT::SimpleValueType)(MVT::LAST_BOOLEAN_VECTOR_VALUETYPE + 1)); + } + static mvt_range integer_vector_valuetypes() { return mvt_range( MVT::FIRST_INTEGER_VECTOR_VALUETYPE, Index: lib/CodeGen/SelectionDAG/LegalizeDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1114,6 +1114,7 @@ case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FSETCC: // These pseudo-ops get legalized as if they were their non-strict // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT // is also legal, but if ISD::FSQRT requires expansion then so does @@ -1200,30 +1201,36 @@ case TargetLowering::Legal: LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n"); return; - case TargetLowering::Custom: + case TargetLowering::Custom: { LLVM_DEBUG(dbgs() << "Trying custom legalization\n"); + SDNode *N = Node; + + if (N->isStrictFPOpcode()) + N = DAG.mutateStrictFPToFP(Node); + // FIXME: The handling for custom lowering with multiple results is // a complete mess. - if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { - if (!(Res.getNode() != Node || Res.getResNo() != 0)) + if (SDValue Res = TLI.LowerOperation(SDValue(N, 0), DAG)) { + if (!(Res.getNode() != N || Res.getResNo() != 0)) return; - if (Node->getNumValues() == 1) { + if (N->getNumValues() == 1) { LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); // We can just directly replace this node with the lowered value. - ReplaceNode(SDValue(Node, 0), Res); + ReplaceNode(SDValue(N, 0), Res); return; } SmallVector ResultVals; - for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) + for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) ResultVals.push_back(Res.getValue(i)); LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); - ReplaceNode(Node, ResultVals.data()); + ReplaceNode(N, ResultVals.data()); return; } LLVM_DEBUG(dbgs() << "Could not custom legalize node\n"); LLVM_FALLTHROUGH; + } case TargetLowering::Expand: if (ExpandNode(Node)) return; Index: lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -77,6 +77,8 @@ case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break; case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break; case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break; + case ISD::STRICT_FSETCC: + Res = PromoteIntRes_StrictFSETCC(N); break; case ISD::SMIN: case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break; case ISD::UMIN: @@ -678,6 +680,46 @@ N->getOperand(1), LHS, RHS, N->getOperand(4)); } +SDValue DAGTypeLegalizer::PromoteIntRes_StrictFSETCC(SDNode *N) { + // Operand 0 is the Chain. + EVT InVT = N->getOperand(1).getValueType(); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + + EVT SVT = getSetCCResultType(InVT); + + // If we got back a type that needs to be promoted, this likely means the + // the input type also needs to be promoted. So get the promoted type for + // the input and try the query again. + if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) { + if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) { + InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT); + SVT = getSetCCResultType(InVT); + } else { + // Input type isn't promoted, just use the default promoted type. + SVT = NVT; + } + } + + SDLoc dl(N); + assert(SVT.isVector() == N->getOperand(1).getValueType().isVector() && + "Vector compare must return a vector result!"); + + // Get the SETCC result using the canonical SETCC type. + EVT VTs[] = {SVT, MVT::Other}; + SDValue Opers[] = {N->getOperand(0), N->getOperand(1), + N->getOperand(2), N->getOperand(3)}; + SDValue SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers); + + // Convert to the expected type. + SDValue Res = DAG.getSExtOrTrunc(SetCC, dl, NVT); + + // Legalize the chain result - switch anything that used the old chain to + // use the new one. + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + + return Res; +} + SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) { EVT InVT = N->getOperand(0).getValueType(); EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); Index: lib/CodeGen/SelectionDAG/LegalizeTypes.h =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -331,6 +331,7 @@ SDValue PromoteIntRes_VSELECT(SDNode *N); SDValue PromoteIntRes_SELECT_CC(SDNode *N); SDValue PromoteIntRes_SETCC(SDNode *N); + SDValue PromoteIntRes_StrictFSETCC(SDNode *N); SDValue PromoteIntRes_SHL(SDNode *N); SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N); SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N); Index: lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -321,6 +321,7 @@ case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FSETCC: // These pseudo-ops get legalized as if they were their non-strict // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT // is also legal, but if ISD::FSQRT requires expansion then so does @@ -438,7 +439,13 @@ break; case TargetLowering::Custom: { LLVM_DEBUG(dbgs() << "Trying custom legalization\n"); - if (SDValue Tmp1 = TLI.LowerOperation(Op, DAG)) { + SDValue TmpOp = Op; + SDNode *Node = TmpOp.getNode(); + + if (Node->isStrictFPOpcode()) + TmpOp = SDValue(DAG.mutateStrictFPToFP(Node), 0); + + if (SDValue Tmp1 = TLI.LowerOperation(TmpOp, DAG)) { LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n"); Result = Tmp1; break; @@ -780,6 +787,7 @@ case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FSETCC: return ExpandStrictFPOp(Op); default: return DAG.UnrollVectorOp(Op.getNode()); @@ -1194,7 +1202,7 @@ if (OperVT.isVector()) Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, - EltVT, Oper, Idx); + OperVT.getVectorElementType(), Oper, Idx); Opers.push_back(Oper); } Index: lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -170,6 +170,7 @@ case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FSETCC: R = ScalarizeVecRes_StrictFPOp(N); break; } @@ -846,6 +847,7 @@ case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FSETCC: SplitVecRes_StrictFPOp(N, Lo, Hi); break; } @@ -2437,6 +2439,7 @@ case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FSETCC: Res = WidenVecRes_StrictFP(N); break; Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -7456,6 +7456,7 @@ case ISD::STRICT_FFLOOR: NewOpc = ISD::FFLOOR; IsUnary = true; break; case ISD::STRICT_FROUND: NewOpc = ISD::FROUND; IsUnary = true; break; case ISD::STRICT_FTRUNC: NewOpc = ISD::FTRUNC; IsUnary = true; break; + case ISD::STRICT_FSETCC: NewOpc = ISD::SETCC; IsTernary = true; break; } // We're taking this node out of the chain, so we need to re-link things. @@ -7463,14 +7464,14 @@ SDValue OutputChain = SDValue(Node, 1); ReplaceAllUsesOfValueWith(OutputChain, InputChain); - SDVTList VTs = getVTList(Node->getOperand(1).getValueType()); + SDVTList VTs = getVTList(Node->getValueType(0)); SDNode *Res = nullptr; if (IsUnary) Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1) }); else if (IsTernary) Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1), Node->getOperand(2), - Node->getOperand(3)}); + Node->getOperand(3) }); else Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1), Node->getOperand(2) }); Index: lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5657,6 +5657,7 @@ case Intrinsic::experimental_constrained_floor: case Intrinsic::experimental_constrained_round: case Intrinsic::experimental_constrained_trunc: + case Intrinsic::experimental_constrained_fcmpeq: visitConstrainedFPIntrinsic(cast(I)); return nullptr; case Intrinsic::fmuladd: { @@ -6360,6 +6361,26 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic( const ConstrainedFPIntrinsic &FPI) { SDLoc sdl = getCurSDLoc(); + + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SmallVector ValueVTs; + ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs); + ValueVTs.push_back(MVT::Other); // Out chain + + SDValue Chain = getRoot(); + SmallVector Opers; + Opers.push_back(Chain); + if (FPI.isUnaryOp()) { + Opers.push_back(getValue(FPI.getArgOperand(0))); + } else if (FPI.isTernaryOp()) { + Opers.push_back(getValue(FPI.getArgOperand(0))); + Opers.push_back(getValue(FPI.getArgOperand(1))); + Opers.push_back(getValue(FPI.getArgOperand(2))); + } else { + Opers.push_back(getValue(FPI.getArgOperand(0))); + Opers.push_back(getValue(FPI.getArgOperand(1))); + } + unsigned Opcode; switch (FPI.getIntrinsicID()) { default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. @@ -6435,27 +6456,14 @@ case Intrinsic::experimental_constrained_trunc: Opcode = ISD::STRICT_FTRUNC; break; + case Intrinsic::experimental_constrained_fcmpeq: + Opcode = ISD::STRICT_FSETCC; + Opers.push_back(DAG.getCondCode(ISD::SETUEQ)); + break; } - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - SDValue Chain = getRoot(); - SmallVector ValueVTs; - ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs); - ValueVTs.push_back(MVT::Other); // Out chain SDVTList VTs = DAG.getVTList(ValueVTs); - SDValue Result; - if (FPI.isUnaryOp()) - Result = DAG.getNode(Opcode, sdl, VTs, - { Chain, getValue(FPI.getArgOperand(0)) }); - else if (FPI.isTernaryOp()) - Result = DAG.getNode(Opcode, sdl, VTs, - { Chain, getValue(FPI.getArgOperand(0)), - getValue(FPI.getArgOperand(1)), - getValue(FPI.getArgOperand(2)) }); - else - Result = DAG.getNode(Opcode, sdl, VTs, - { Chain, getValue(FPI.getArgOperand(0)), - getValue(FPI.getArgOperand(1)) }); + SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers); assert(Result.getNode()->getNumValues() == 2); SDValue OutChain = Result.getValue(1); Index: lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -266,6 +266,7 @@ case ISD::STRICT_FPOWI: return "strict_fpowi"; case ISD::SETCC: return "setcc"; case ISD::SETCCCARRY: return "setcccarry"; + case ISD::STRICT_FSETCC: return "strict_fsetcc"; case ISD::SELECT: return "select"; case ISD::VSELECT: return "vselect"; case ISD::SELECT_CC: return "select_cc"; Index: lib/IR/Function.cpp =================================================================== --- lib/IR/Function.cpp +++ lib/IR/Function.cpp @@ -689,7 +689,8 @@ IIT_STRUCT6 = 38, IIT_STRUCT7 = 39, IIT_STRUCT8 = 40, - IIT_F128 = 41 + IIT_F128 = 41, + IIT_SAME_VEC_LENGTH_OR_SCALAR_ARG = 42 }; static void DecodeIITType(unsigned &NextElt, ArrayRef Infos, @@ -814,6 +815,13 @@ ArgInfo)); return; } + case IIT_SAME_VEC_LENGTH_OR_SCALAR_ARG: { + unsigned ArgNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); + unsigned short RefNo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); + OutputTable.push_back(IITDescriptor::get(IITDescriptor::SameVecLengthOrScalarArgument, + ArgNo, RefNo)); + return; + } case IIT_SAME_VEC_WIDTH_ARG: { unsigned ArgInfo = (NextElt == Infos.size() ? 0 : Infos[NextElt++]); OutputTable.push_back(IITDescriptor::get(IITDescriptor::SameVecWidthArgument, @@ -947,6 +955,9 @@ case IITDescriptor::HalfVecArgument: return VectorType::getHalfElementsVectorType(cast( Tys[D.getArgumentNumber()])); + case IITDescriptor::SameVecLengthOrScalarArgument: + // Return the overloaded type + return Tys[D.getOverloadArgNumber()]; case IITDescriptor::SameVecWidthArgument: { Type *EltTy = DecodeFixedType(Infos, Tys, Context); Type *Ty = Tys[D.getArgumentNumber()]; @@ -1095,6 +1106,7 @@ case IITDescriptor::AK_AnyFloat: return !Ty->isFPOrFPVectorTy(); case IITDescriptor::AK_AnyVector: return !isa(Ty); case IITDescriptor::AK_AnyPointer: return !isa(Ty); + case IITDescriptor::AK_AnyBoolean: return !Ty->isIntOrIntVectorTy(1); } llvm_unreachable("all argument kinds not covered"); @@ -1134,6 +1146,33 @@ !isa(ArgTys[D.getArgumentNumber()]) || VectorType::getHalfElementsVectorType( cast(ArgTys[D.getArgumentNumber()])) != Ty; + case IITDescriptor::SameVecLengthOrScalarArgument: { + unsigned RefArgNumber = D.getRefArgNumber(); + if (RefArgNumber >= ArgTys.size()) + return true; + + // Record the overloaded type + assert(D.getOverloadArgNumber() == ArgTys.size() && + "Table consistency error"); + ArgTys.push_back(Ty); + + // Handle Vectors + if (Ty->isVectorTy() && ArgTys[RefArgNumber]->isVectorTy()) { + VectorType * ReferenceType = + dyn_cast(ArgTys[RefArgNumber]); + VectorType *ThisArgType = dyn_cast(Ty); + if (!ThisArgType || !ReferenceType || + (ReferenceType->getVectorNumElements() != + ThisArgType->getVectorNumElements())) + return true; + } else { + // Handle Scalars + if (Ty->isVectorTy() || ArgTys[RefArgNumber]->isVectorTy()) + return true; + } + + return false; + } case IITDescriptor::SameVecWidthArgument: { if (D.getArgumentNumber() >= ArgTys.size()) return true; Index: lib/IR/Verifier.cpp =================================================================== --- lib/IR/Verifier.cpp +++ lib/IR/Verifier.cpp @@ -4157,6 +4157,7 @@ case Intrinsic::experimental_constrained_floor: case Intrinsic::experimental_constrained_round: case Intrinsic::experimental_constrained_trunc: + case Intrinsic::experimental_constrained_fcmpeq: visitConstrainedFPIntrinsic( cast(*CS.getInstruction())); break; Index: test/CodeGen/X86/fp-intrinsics.ll =================================================================== --- test/CodeGen/X86/fp-intrinsics.ll +++ test/CodeGen/X86/fp-intrinsics.ll @@ -286,6 +286,18 @@ ret double %rem } +; CHECK-LABEL: f20 +; COMMON: ucomisd +define i1 @f20() { +entry: + %cmp = call i1 @llvm.experimental.constrained.fcmpeq.f64( + double 1.000000e+00, + double 1.000000e+01, + metadata !"round.dynamic", + metadata !"fpexcept.strict") + ret i1 %cmp +} + @llvm.fp.env = thread_local global i8 zeroinitializer, section "llvm.metadata" declare double @llvm.experimental.constrained.fadd.f64(double, double, metadata, metadata) declare double @llvm.experimental.constrained.fsub.f64(double, double, metadata, metadata) @@ -304,5 +316,6 @@ declare double @llvm.experimental.constrained.log2.f64(double, metadata, metadata) declare double @llvm.experimental.constrained.rint.f64(double, metadata, metadata) declare double @llvm.experimental.constrained.nearbyint.f64(double, metadata, metadata) +declare i1 @llvm.experimental.constrained.fcmpeq.f64(double, double, metadata, metadata) declare float @llvm.experimental.constrained.fma.f32(float, float, float, metadata, metadata) declare double @llvm.experimental.constrained.fma.f64(double, double, double, metadata, metadata) Index: test/CodeGen/X86/vector-constrained-fp-intrinsics-cmp.ll =================================================================== --- test/CodeGen/X86/vector-constrained-fp-intrinsics-cmp.ll +++ test/CodeGen/X86/vector-constrained-fp-intrinsics-cmp.ll @@ -0,0 +1,25 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -O3 -mtriple=x86_64-pc-linux < %s | FileCheck %s + +define <2 x i1> @constrained_vector_fcmpeq_v2f64() { +; CHECK-LABEL: constrained_vector_fcmpeq_v2f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: movapd {{.*#+}} xmm1 = [2.0E+0,1.0E+0] +; CHECK-NEXT: movapd {{.*#+}} xmm0 = [1.0E+0,2.0E+0] +; CHECK-NEXT: movapd %xmm0, %xmm2 +; CHECK-NEXT: cmpeqpd %xmm1, %xmm2 +; CHECK-NEXT: cmpunordpd %xmm1, %xmm0 +; CHECK-NEXT: orpd %xmm2, %xmm0 +; CHECK-NEXT: retq +entry: + %fcmp = call <2 x i1> @llvm.experimental.constrained.fcmpeq.v2f64( + <2 x double> , + <2 x double> , + metadata !"round.dynamic", + metadata !"fpexcept.strict") + ret <2 x i1> %fcmp +} + +; Single width declarations +declare <2 x i1> @llvm.experimental.constrained.fcmpeq.v2f64(<2 x double>, <2 x double>, metadata, metadata) + Index: utils/TableGen/CodeGenDAGPatterns.cpp =================================================================== --- utils/TableGen/CodeGenDAGPatterns.cpp +++ utils/TableGen/CodeGenDAGPatterns.cpp @@ -765,6 +765,14 @@ for (MVT Ov : Ovs) { switch (Ov.SimpleTy) { + case MVT::bAny: + for (MVT T : MVT::boolean_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + for (MVT T : MVT::boolean_vector_valuetypes()) + if (Legal.count(T)) + Out.insert(T); + return; case MVT::iPTRAny: Out.insert(MVT::iPTR); return; Index: utils/TableGen/CodeGenTarget.cpp =================================================================== --- utils/TableGen/CodeGenTarget.cpp +++ utils/TableGen/CodeGenTarget.cpp @@ -64,6 +64,7 @@ case MVT::i64: return "MVT::i64"; case MVT::i128: return "MVT::i128"; case MVT::Any: return "MVT::Any"; + case MVT::bAny: return "MVT::bAny"; case MVT::iAny: return "MVT::iAny"; case MVT::fAny: return "MVT::fAny"; case MVT::vAny: return "MVT::vAny"; @@ -634,8 +635,9 @@ // overloaded, all the types can be specified directly. assert(((!TyEl->isSubClassOf("LLVMExtendedType") && !TyEl->isSubClassOf("LLVMTruncatedType") && - !TyEl->isSubClassOf("LLVMVectorSameWidth")) || - VT == MVT::iAny || VT == MVT::vAny) && + !TyEl->isSubClassOf("LLVMVectorSameWidth") && + !TyEl->isSubClassOf("LLVMVectorSameLengthOrScalar")) || + VT == MVT::bAny || VT == MVT::iAny || VT == MVT::vAny) && "Expected iAny or vAny type"); } else VT = getValueType(TyEl->getValueAsDef("VT")); Index: utils/TableGen/IntrinsicEmitter.cpp =================================================================== --- utils/TableGen/IntrinsicEmitter.cpp +++ utils/TableGen/IntrinsicEmitter.cpp @@ -220,7 +220,8 @@ IIT_STRUCT6 = 38, IIT_STRUCT7 = 39, IIT_STRUCT8 = 40, - IIT_F128 = 41 + IIT_F128 = 41, + IIT_SAME_VEC_LENGTH_OR_SCALAR_ARG = 42 }; static void EncodeFixedValueType(MVT::SimpleValueType VT, @@ -270,6 +271,16 @@ Sig.push_back(IIT_TRUNC_ARG); else if (R->isSubClassOf("LLVMHalfElementsVectorType")) Sig.push_back(IIT_HALF_VEC_ARG); + else if (R->isSubClassOf("LLVMVectorSameLengthOrScalar")) { + Sig.push_back(IIT_SAME_VEC_LENGTH_OR_SCALAR_ARG); + unsigned ArgNo = ArgCodes.size(); + ArgCodes.push_back(3 /*vAny*/); + // Encode overloaded ArgNo + Sig.push_back(ArgNo); + // Encode LLVMMatchType ArgNo + Sig.push_back(Number); + return; + } else if (R->isSubClassOf("LLVMVectorSameWidth")) { Sig.push_back(IIT_SAME_VEC_WIDTH_ARG); Sig.push_back((Number << 3) | ArgCodes[Number]); @@ -300,6 +311,7 @@ unsigned Tmp = 0; switch (VT) { default: break; + case MVT::bAny: ++Tmp; LLVM_FALLTHROUGH; case MVT::iPTRAny: ++Tmp; LLVM_FALLTHROUGH; case MVT::vAny: ++Tmp; LLVM_FALLTHROUGH; case MVT::fAny: ++Tmp; LLVM_FALLTHROUGH;