diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -2445,35 +2445,6 @@ return splatSplitI64WithVL(DL, VT, Scalar, VL, DAG); } -// Is the mask a slidedown that shifts in undefs. -static int matchShuffleAsSlideDown(ArrayRef Mask) { - int Size = Mask.size(); - - // Elements shifted in should be undef. - auto CheckUndefs = [&](int Shift) { - for (int i = Size - Shift; i != Size; ++i) - if (Mask[i] >= 0) - return false; - return true; - }; - - // Elements should be shifted or undef. - auto MatchShift = [&](int Shift) { - for (int i = 0; i != Size - Shift; ++i) - if (Mask[i] >= 0 && Mask[i] != Shift + i) - return false; - return true; - }; - - // Try all possible shifts. - for (int Shift = 1; Shift != Size; ++Shift) - if (CheckUndefs(Shift) && MatchShift(Shift)) - return Shift; - - // No match. - return -1; -} - static bool isInterleaveShuffle(ArrayRef Mask, MVT VT, bool &SwapSources, const RISCVSubtarget &Subtarget) { // We need to be able to widen elements to the next larger integer type. @@ -2516,7 +2487,7 @@ return true; } -static int isElementRotate(SDValue &V1, SDValue &V2, ArrayRef Mask) { +static int isElementRotate(int &Lo, int &Hi, ArrayRef Mask) { int Size = Mask.size(); // We need to detect various ways of spelling a rotation: @@ -2527,7 +2498,8 @@ // [-1, 4, 5, 6, -1, -1, 9, -1] // [-1, 4, 5, 6, -1, -1, -1, -1] int Rotation = 0; - SDValue Lo, Hi; + Lo = -1; + Hi = -1; for (int i = 0; i != Size; ++i) { int M = Mask[i]; if (M < 0) @@ -2551,18 +2523,18 @@ return -1; // Compute which value this mask is pointing at. - SDValue MaskV = M < Size ? V1 : V2; + int MaskSrc = M < Size ? 0 : 1; // Compute which of the two target values this index should be assigned to. // This reflects whether the high elements are remaining or the low elemnts // are remaining. - SDValue &TargetV = StartIdx < 0 ? Hi : Lo; + int &Target = StartIdx < 0 ? Hi : Lo; // Either set up this value if we've not encountered it before, or check // that it remains consistent. - if (!TargetV) - TargetV = MaskV; - else if (TargetV != MaskV) + if (Target < 0) + Target = MaskSrc; + else if (Target != MaskSrc) // This may be a rotation, but it pulls from the inputs in some // unsupported interleaving. return -1; @@ -2570,14 +2542,7 @@ // Check that we successfully analyzed the mask, and normalize the results. assert(Rotation != 0 && "Failed to locate a viable rotation!"); - assert((Lo || Hi) && "Failed to find a rotated input vector!"); - - // Make sure we've found a value for both halves. - if (!Lo || !Hi) - return -1; - - V1 = Lo; - V2 = Hi; + assert((Lo >= 0 || Hi >= 0) && "Failed to find a rotated input vector!"); return Rotation; } @@ -2673,45 +2638,47 @@ ArrayRef Mask = SVN->getMask(); - // Try to match as a slidedown. - int SlideAmt = matchShuffleAsSlideDown(Mask); - if (SlideAmt >= 0) { - // TODO: Should we reduce the VL to account for the upper undef elements? - // Requires additional vsetvlis, but might be faster to execute. - V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); - SDValue SlideDown = - DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT, - DAG.getUNDEF(ContainerVT), V1, - DAG.getConstant(SlideAmt, DL, XLenVT), - TrueMask, VL); - return convertFromScalableVector(VT, SlideDown, DAG, Subtarget); - } - // Match shuffles that concatenate two vectors, rotate the concatenation, // and then extract the original number of elements from the rotated result. // This is equivalent to vector.splice or X86's PALIGNR instruction. Lower // it to a SLIDEDOWN and a SLIDEUP. - // FIXME: We don't really need it to be a concatenation. We just need two - // regions with contiguous elements that need to be shifted down and up. - int Rotation = isElementRotate(V1, V2, Mask); + int Lo, Hi; + int Rotation = isElementRotate(Lo, Hi, Mask); if (Rotation > 0) { + SDValue LoV, HiV; + if (Lo >= 0) + LoV = Lo == 0 ? V1 : V2; + if (Hi >= 0) + HiV = Hi == 0 ? V1 : V2; + // We found a rotation. We need to slide V1 down by Rotation. Using // (NumElts - Rotation) for VL. Then we need to slide V2 up by // (NumElts - Rotation) using NumElts for VL. - V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget); - V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget); + if (LoV) + LoV = convertToScalableVector(ContainerVT, LoV, DAG, Subtarget); + if (HiV) + HiV = convertToScalableVector(ContainerVT, HiV, DAG, Subtarget); unsigned InvRotate = NumElts - Rotation; - SDValue SlideDown = - DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT, - DAG.getUNDEF(ContainerVT), V2, - DAG.getConstant(Rotation, DL, XLenVT), - TrueMask, DAG.getConstant(InvRotate, DL, XLenVT)); - SDValue SlideUp = - DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, SlideDown, V1, - DAG.getConstant(InvRotate, DL, XLenVT), - TrueMask, VL); - return convertFromScalableVector(VT, SlideUp, DAG, Subtarget); + + SDValue Res = DAG.getUNDEF(ContainerVT); + if (HiV) { + // TODO: Should we reduce the VL to account for the upper undef elements? + // Requires additional vsetvlis, but might be faster to execute. It will + // generate many vsetvlis when lowering a reduce.mul. + SDValue DownVL = VL; + // If we are doing a SLIDEDOWN+SLIDEUP, reduce the VL for the SLIDEDOWN. + if (LoV) + DownVL = DAG.getConstant(InvRotate, DL, XLenVT); + Res = + DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT, Res, HiV, + DAG.getConstant(Rotation, DL, XLenVT), TrueMask, DownVL); + } + if (LoV) + Res = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Res, LoV, + DAG.getConstant(InvRotate, DL, XLenVT), TrueMask, VL); + + return convertFromScalableVector(VT, Res, DAG, Subtarget); } // Detect an interleave shuffle and lower to @@ -2933,7 +2900,8 @@ MVT SVT = VT.getSimpleVT(); bool SwapSources; - return (matchShuffleAsSlideDown(M) >= 0) || + int Lo, Hi; + return (isElementRotate(Lo, Hi, M) > 0) || isInterleaveShuffle(M, SVT, SwapSources, Subtarget); } diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-shuffles.ll @@ -256,6 +256,28 @@ ret <8 x float> %s } +define <4 x half> @slideup_v4f16(<4 x half> %x) { +; CHECK-LABEL: slideup_v4f16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, mu +; CHECK-NEXT: vslideup.vi v9, v8, 1 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %s = shufflevector <4 x half> %x, <4 x half> poison, <4 x i32> + ret <4 x half> %s +} + +define <8 x float> @slideup_v8f32(<8 x float> %x) { +; CHECK-LABEL: slideup_v8f32: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, mu +; CHECK-NEXT: vslideup.vi v10, v8, 3 +; CHECK-NEXT: vmv.v.v v8, v10 +; CHECK-NEXT: ret + %s = shufflevector <8 x float> %x, <8 x float> poison, <8 x i32> + ret <8 x float> %s +} + define <8 x float> @splice_unary(<8 x float> %x) { ; CHECK-LABEL: splice_unary: ; CHECK: # %bb.0: diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll @@ -555,6 +555,28 @@ ret <8 x i32> %s } +define <4 x i16> @slideup_v4i16(<4 x i16> %x) { +; CHECK-LABEL: slideup_v4i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e16, mf2, ta, mu +; CHECK-NEXT: vslideup.vi v9, v8, 1 +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %s = shufflevector <4 x i16> %x, <4 x i16> poison, <4 x i32> + ret <4 x i16> %s +} + +define <8 x i32> @slideup_v8i32(<8 x i32> %x) { +; CHECK-LABEL: slideup_v8i32: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 8, e32, m2, ta, mu +; CHECK-NEXT: vslideup.vi v10, v8, 3 +; CHECK-NEXT: vmv.v.v v8, v10 +; CHECK-NEXT: ret + %s = shufflevector <8 x i32> %x, <8 x i32> poison, <8 x i32> + ret <8 x i32> %s +} + define <8 x i16> @splice_unary(<8 x i16> %x) { ; CHECK-LABEL: splice_unary: ; CHECK: # %bb.0: