diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -22252,7 +22252,7 @@ EVT OpVT = N->getOperand(0).getValueType(); // If the operands are legal vectors, leave them alone. - if (TLI.isTypeLegal(OpVT)) + if (TLI.isTypeLegal(OpVT) || OpVT.isScalableVector()) return SDValue(); SDLoc DL(N); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -875,6 +875,8 @@ void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo, SDValue &Hi); + void SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N); + void SplitVecRes_VECTOR_INTERLEAVE(SDNode *N); void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo, SDValue &Hi); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -1000,6 +1000,12 @@ case ISD::VECTOR_SPLICE: SplitVecRes_VECTOR_SPLICE(N, Lo, Hi); break; + case ISD::VECTOR_DEINTERLEAVE: + SplitVecRes_VECTOR_DEINTERLEAVE(N); + return; + case ISD::VECTOR_INTERLEAVE: + SplitVecRes_VECTOR_INTERLEAVE(N); + return; case ISD::VAARG: SplitVecRes_VAARG(N, Lo, Hi); break; @@ -2768,6 +2774,37 @@ DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL)); } +void DAGTypeLegalizer::SplitVecRes_VECTOR_DEINTERLEAVE(SDNode *N) { + + SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi; + GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); + GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); + EVT VT = Op0Lo.getValueType(); + SDLoc DL(N); + SDValue ResLo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, + DAG.getVTList(VT, VT), Op0Lo, Op0Hi); + SDValue ResHi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, + DAG.getVTList(VT, VT), Op1Lo, Op1Hi); + + SetSplitVector(SDValue(N, 0), ResLo.getValue(0), ResHi.getValue(0)); + SetSplitVector(SDValue(N, 1), ResLo.getValue(1), ResHi.getValue(1)); +} + +void DAGTypeLegalizer::SplitVecRes_VECTOR_INTERLEAVE(SDNode *N) { + SDValue Op0Lo, Op0Hi, Op1Lo, Op1Hi; + GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi); + GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi); + EVT VT = Op0Lo.getValueType(); + SDLoc DL(N); + SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, + DAG.getVTList(VT, VT), Op0Lo, Op1Lo), + DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, + DAG.getVTList(VT, VT), Op0Hi, Op1Hi)}; + + SetSplitVector(SDValue(N, 0), Res[0].getValue(0), Res[0].getValue(1)); + SetSplitVector(SDValue(N, 1), Res[1].getValue(0), Res[1].getValue(1)); +} + //===----------------------------------------------------------------------===// // Operand Vector Splitting //===----------------------------------------------------------------------===// diff --git a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll --- a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll +++ b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll @@ -165,6 +165,47 @@ } +; Split illegal types + +define {, } @vector_deinterleave_nxv4i64_nxv8i64( %vec) { +; CHECK-LABEL: vector_deinterleave_nxv4i64_nxv8i64: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z4.d, z2.d, z3.d +; CHECK-NEXT: uzp1 z5.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z6.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z3.d, z2.d, z3.d +; CHECK-NEXT: mov z0.d, z5.d +; CHECK-NEXT: mov z1.d, z4.d +; CHECK-NEXT: mov z2.d, z6.d +; CHECK-NEXT: ret +%retval = call {, } @llvm.experimental.vector.deinterleave2.nxv8i64( %vec) +ret {, } %retval +} + +define {, } @vector_deinterleave_nxv8i64_nxv16i64( %vec) { +; CHECK-LABEL: vector_deinterleave_nxv8i64_nxv16i64: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 z24.d, z4.d, z5.d +; CHECK-NEXT: uzp1 z25.d, z2.d, z3.d +; CHECK-NEXT: uzp1 z26.d, z0.d, z1.d +; CHECK-NEXT: uzp1 z27.d, z6.d, z7.d +; CHECK-NEXT: uzp2 z28.d, z0.d, z1.d +; CHECK-NEXT: uzp2 z29.d, z2.d, z3.d +; CHECK-NEXT: uzp2 z30.d, z4.d, z5.d +; CHECK-NEXT: uzp2 z7.d, z6.d, z7.d +; CHECK-NEXT: mov z0.d, z26.d +; CHECK-NEXT: mov z1.d, z25.d +; CHECK-NEXT: mov z2.d, z24.d +; CHECK-NEXT: mov z3.d, z27.d +; CHECK-NEXT: mov z4.d, z28.d +; CHECK-NEXT: mov z5.d, z29.d +; CHECK-NEXT: mov z6.d, z30.d +; CHECK-NEXT: ret +%retval = call {, } @llvm.experimental.vector.deinterleave2.nxv16i64( %vec) +ret {, } %retval +} + + ; Floating declarations declare {,} @llvm.experimental.vector.deinterleave2.nxv4f16() declare {, } @llvm.experimental.vector.deinterleave2.nxv8f16() @@ -184,3 +225,7 @@ declare {, } @llvm.experimental.vector.deinterleave2.nxv16i1() declare {, } @llvm.experimental.vector.deinterleave2.nxv8i1() declare {, } @llvm.experimental.vector.deinterleave2.nxv4i1() + +; Illegal size type +declare {, } @llvm.experimental.vector.deinterleave2.nxv8i64() +declare {, } @llvm.experimental.vector.deinterleave2.nxv16i64() diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll --- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll +++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll @@ -159,6 +159,35 @@ ret %retval } +; Split illegal type size + +define @interleave2_nxv16i32( %vec0, %vec1) { +; CHECK-LABEL: interleave2_nxv16i32: +; CHECK: // %bb.0: +; CHECK-NEXT: zip1 z4.s, z0.s, z2.s +; CHECK-NEXT: zip2 z5.s, z0.s, z2.s +; CHECK-NEXT: zip1 z2.s, z1.s, z3.s +; CHECK-NEXT: zip2 z3.s, z1.s, z3.s +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %retval = call @llvm.experimental.vector.interleave2.nxv16i32( %vec0, %vec1) + ret %retval +} + +define @interleave2_nxv8i64( %vec0, %vec1) { +; CHECK-LABEL: interleave2_nxv8i64: +; CHECK: // %bb.0: +; CHECK-NEXT: zip1 z4.d, z0.d, z2.d +; CHECK-NEXT: zip2 z5.d, z0.d, z2.d +; CHECK-NEXT: zip1 z2.d, z1.d, z3.d +; CHECK-NEXT: zip2 z3.d, z1.d, z3.d +; CHECK-NEXT: mov z0.d, z4.d +; CHECK-NEXT: mov z1.d, z5.d +; CHECK-NEXT: ret + %retval = call @llvm.experimental.vector.interleave2.nxv8i64( %vec0, %vec1) + ret %retval +} ; Float declarations declare @llvm.experimental.vector.interleave2.nxv4f16(, ) @@ -179,3 +208,7 @@ declare @llvm.experimental.vector.interleave2.nxv16i1(, ) declare @llvm.experimental.vector.interleave2.nxv8i1(, ) declare @llvm.experimental.vector.interleave2.nxv4i1(, ) + +; Illegal type size +declare @llvm.experimental.vector.interleave2.nxv16i32(, ) +declare @llvm.experimental.vector.interleave2.nxv8i64(, )