Index: lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -1086,6 +1086,10 @@ SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) { assert(OpNo == 0 && "Only know how to promote the condition!"); + + if (SDValue Res = handleSetccVSelect(N)) + return Res; + SDValue Cond = N->getOperand(0); EVT OpTy = N->getOperand(1).getValueType(); Index: lib/CodeGen/SelectionDAG/LegalizeTypes.h =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -854,6 +854,8 @@ SDValue ExpandOp_INSERT_VECTOR_ELT(SDNode *N); SDValue ExpandOp_SCALAR_TO_VECTOR (SDNode *N); SDValue ExpandOp_NormalStore (SDNode *N, unsigned OpNo); + + SDValue handleSetccVSelect(SDNode *N); }; } // end namespace llvm. Index: lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp =================================================================== --- lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -2826,7 +2826,71 @@ WidenVT, N->getOperand(0)); } +// Handle VSELECT -> SETCC as a unit to avoid scalarization of vector boolean +// result. +SDValue DAGTypeLegalizer::handleSetccVSelect(SDNode *N) { + LLVMContext &Ctx = *DAG.getContext(); + SDValue Cond = N->getOperand(0); + + if (N->getOpcode() != ISD::VSELECT || Cond->getOpcode() != ISD::SETCC) + return SDValue(); + + // Make a new SETCC node, with an updated ValueType. This may need to be + // widened, if so it must be done now. + EVT SetccVT = getSetCCResultType(Cond->getOperand(0).getValueType()); + if (getTypeAction(SetccVT) == TargetLowering::TypeWidenVector) + SetccVT = TLI.getTypeToTransformTo(Ctx, SetccVT); + SDValue SetCC = DAG.getNode(ISD::SETCC, SDLoc(Cond), SetccVT, + Cond->getOperand(0), Cond->getOperand(1), Cond->getOperand(2)); + + // Make a new VSELECT node, with an updated ValueType. This may need to be + // widened, if so it must be done now. WidenVectorOperand does not handle + // ISD::VSELECT, so handle operands here also. + EVT VSelVT = N->getValueType(0); + SDValue VSelOp1 = N->getOperand(1); + SDValue VSelOp2 = N->getOperand(2); + if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector) { + VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT); + VSelOp1 = GetWidenedVector(VSelOp1); + VSelOp2 = GetWidenedVector(VSelOp2); + } + + // We want the mask of VSELECT to have an integer vector type that + // corresponds to its result (and arguments) vector type. + EVT MaskVT = VSelVT; + if (!MaskVT.getScalarType().isInteger()) + MaskVT = MaskVT.changeVectorElementTypeToInteger(); + + // The new SetCCVT and MaskVT types may not have the same element type, but + // we know that the number of defined elements are the same. Handle if + // needed with vector sign extend inreg or truncate. + SDValue Mask; + if (SetccVT == MaskVT) + Mask = SetCC; + else if (SetccVT.getScalarSizeInBits() < MaskVT.getScalarSizeInBits()) + Mask = DAG.getSignExtendVectorInReg(SetCC, SDLoc(N), MaskVT); + else { + EVT SubVT = + EVT::getVectorVT(Ctx, MaskVT.getVectorElementType(), + SetccVT.getVectorNumElements()); + SDValue TruncCC = DAG.getNode(ISD::TRUNCATE, SDLoc(N), SubVT, SetCC); + + unsigned NumSubVecs = VSelVT.getVectorNumElements() / SubVT.getVectorNumElements(); + SmallVector SubConcatOps(NumSubVecs); + SubConcatOps[0] = TruncCC; + for (unsigned i = 1; i < NumSubVecs; ++i) + SubConcatOps[i] = DAG.getUNDEF(SubVT); + + Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), MaskVT, SubConcatOps); + } + + return DAG.getNode(ISD::VSELECT, SDLoc(N), VSelVT, Mask, VSelOp1, VSelOp2); +} + SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) { + if (SDValue Res = handleSetccVSelect(N)) + return Res; + EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); unsigned WidenNumElts = WidenVT.getVectorNumElements();