[AArch64][SVE2p1] Make use of REVD instruction.

[sdesmalen]

Summary

Reversing double-words within a quard-word is possible using the REVD instruction
when SVE2p1 is enabled.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

10650–10652

Please update the string to include the new block size.

10654–10655

Perhaps this block can be removed? There's the BlockSize <= EltSz check just below which seems to handle this?

[sdesmalen]

• Updated asssert string to include 128 block size.
• Removed redundant condition.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

23256–23257

Perhaps worth putting VT.getScalarSizeInBits() == 64 before the called to isREVMask so we bail out earlier.

[sdesmalen]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

23256–23257

good point, I've done that before committing!

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
 
   return true;
 }
 
 /// isREVMask - Check if a vector shuffle corresponds to a REV
 /// instruction with the specified blocksize.  (The order of the elements
 /// within each block of the vector is reversed.)
 static bool isREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
-  assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
-         "Only possible block sizes for REV are: 16, 32, 64");
+  assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64 ||
+          BlockSize == 128) &&
+         "Only possible block sizes for REV are: 16, 32, 64, 128");
 
   unsigned EltSz = VT.getScalarSizeInBits();
-  if (EltSz == 64)
-    return false;
-
   unsigned NumElts = VT.getVectorNumElements();
   unsigned BlockElts = M[0] + 1;
   // If the first shuffle index is UNDEF, be optimistic.
   if (M[0] < 0)
     BlockElts = BlockSize / EltSz;
 
   if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz)
     return false;
 
if (isREVMask(ShuffleMask, VT, LaneSize)) {
 
       Op = DAG.getNode(ISD::BITCAST, DL, NewVT, Op1);
       Op = LowerToPredicatedOp(Op, DAG, RevOp);
       Op = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Op);
       return convertFromScalableVector(DAG, VT, Op);
     }
   }
 
+  if (Subtarget->hasSVE2p1() && VT.getScalarSizeInBits() == 64 &&
+      isREVMask(ShuffleMask, VT, 128)) {
+    if (!VT.isFloatingPoint())
+      return LowerToPredicatedOp(Op, DAG, AArch64ISD::REVD_MERGE_PASSTHRU);
+
+    EVT NewVT = getPackedSVEVectorVT(EVT::getIntegerVT(*DAG.getContext(), 64));
+    Op = DAG.getNode(ISD::BITCAST, DL, NewVT, Op1);
+    Op = LowerToPredicatedOp(Op, DAG, AArch64ISD::REVD_MERGE_PASSTHRU);
+    Op = DAG.getNode(ISD::BITCAST, DL, ContainerVT, Op);
+    return convertFromScalableVector(DAG, VT, Op);
+  }
+
   unsigned WhichResult;
   if (isZIPMask(ShuffleMask, VT, WhichResult) && WhichResult == 0)
     return convertFromScalableVector(
         DAG, VT, DAG.getNode(AArch64ISD::ZIP1, DL, ContainerVT, Op1, Op2));
 
   if (isTRNMask(ShuffleMask, VT, WhichResult)) {
     unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
     return convertFromScalableVector(

llvm/test/CodeGen/AArch64/sve-fixed-length-permute-rev.ll

 ; CHECK-NEXT:    ldp x29, x30, [sp], #16 // 16-byte Folded Reload
 ; CHECK-NEXT:    ret
   %tmp1 = load <4 x i64>, ptr %a
   %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   store <4 x i64> %tmp2, ptr %a
   ret void
 }
 
+; This is the same test as above, but with sve2p1 it can use the REVD instruction to reverse
+; the double-words within quard-words.
+define void @test_revdv4i64_sve2p1(ptr %a) #2 {
+; CHECK-LABEL: test_revdv4i64_sve2p1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i64>, ptr %a
+  %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  store <4 x i64> %tmp2, ptr %a
+  ret void
+}
+
+define void @test_revdv4f64_sve2p1(ptr %a) #2 {
+; CHECK-LABEL: test_revdv4f64_sve2p1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ptrue p0.d, vl4
+; CHECK-NEXT:    ptrue p1.d
+; CHECK-NEXT:    ld1d { z0.d }, p0/z, [x0]
+; CHECK-NEXT:    revd z0.q, p1/m, z0.q
+; CHECK-NEXT:    st1d { z0.d }, p0, [x0]
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x double>, ptr %a
+  %tmp2 = shufflevector <4 x double> %tmp1, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  store <4 x double> %tmp2, ptr %a
+  ret void
+}
+
 ; REV instruction will reverse the order of all elements in the vector.
 ; When the vector length and the target register size are inconsistent,
 ; the correctness of generated REV instruction for shuffle pattern cannot be guaranteed.
 
 ; sve-vector-bits-min=256, sve-vector-bits-max is not set, REV inst can't be generated.
 define void @test_revv8i32(ptr %a) #0 {
 ; CHECK-LABEL: test_revv8i32:
 ; CHECK:       // %bb.0:
; CHECK-NEXT:    ret
   %tmp2 = load <8 x i16>, ptr %b
   %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
   store <16 x i16> %tmp3, ptr %c
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }
 attributes #1 = { "target-features"="+sve" vscale_range(2,2) }
+attributes #2 = { "target-features"="+sve2p1" }

llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-permute-rev.ll

 ; CHECK-NEXT:    add sp, sp, #32
 ; CHECK-NEXT:    ret
   %tmp1 = load <4 x i64>, ptr %a
   %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
   store <4 x i64> %tmp2, ptr %a
   ret void
 }
 
+; This is the same test as above, but with sve2p1 it can use the REVD instruction to reverse
+; the double-words within quard-words.
+define void @test_revdv4i64_sve2p1(ptr %a) #1 {
+; CHECK-LABEL: test_revdv4i64_sve2p1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ptrue p0.d, vl2
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    revd z1.q, p0/m, z1.q
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x i64>, ptr %a
+  %tmp2 = shufflevector <4 x i64> %tmp1, <4 x i64> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  store <4 x i64> %tmp2, ptr %a
+  ret void
+}
+
+define void @test_revdv4f64_sve2p1(ptr %a) #1 {
+; CHECK-LABEL: test_revdv4f64_sve2p1:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    revd z0.q, p0/m, z0.q
+; CHECK-NEXT:    revd z1.q, p0/m, z1.q
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %tmp1 = load <4 x double>, ptr %a
+  %tmp2 = shufflevector <4 x double> %tmp1, <4 x double> undef, <4 x i32> <i32 1, i32 0, i32 3, i32 2>
+  store <4 x double> %tmp2, ptr %a
+  ret void
+}
+
 define void @test_revv8i32(ptr %a) #0 {
 ; CHECK-LABEL: test_revv8i32:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sub sp, sp, #32
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    ldp q0, q1, [x0]
 ; CHECK-NEXT:    mov z2.s, z0.s[1]
 ; CHECK-NEXT:    mov z3.s, z0.s[2]

 ; CHECK-NEXT:    ret
   %tmp1 = load <8 x i32>, ptr %a
   %tmp2 = shufflevector <8 x i32> %tmp1, <8 x i32> undef, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
   store <8 x i32> %tmp2, ptr %a
   ret void
 }
 
 attributes #0 = { "target-features"="+sve" }
+attributes #1 = { "target-features"="+sve2p1" }