[AArch64] define isExtractSubvectorCheap

[sebpop]

Summary

Following the ARM-neon backend, define isExtractSubvectorCheap to return true
when extracting low and high part of a neon register.

The patch disables a test in llvm/test/CodeGen/AArch64/arm64-ext.ll that was
checking that ReconstructShuffle in ISelLowering is working as expected. The
pattern to exercise ReconstructShuffle is a BUILD_VECTOR and the expected
pattern gets transformed earlier by the DAGCombiner into an extract_subvector +
vector_shuffle. As there is no way to disable the combiner to only exercise the
code in ISelLowering, the patch disables the testcase.

[evandro]

It LGTM, but I'd wait a while to give a chance for others to chime up.

llvm/test/CodeGen/AArch64/arm64-ext.ll

100 ↗

(On Diff #136604)

Methinks that this test could stay. Before, it resulted in a mix of EXT, UZP1, ZIP1. With this patch, in a pair of LDR, TBL, which seems to be a good result. Better yet, were the shuffle index changed to <i32 4, i32 8, i32 5, i32 9>, then the result would be just like in the test below.

[sebpop]

llvm/test/CodeGen/AArch64/arm64-ext.ll

100 ↗

(On Diff #136604)

If I replace the pattern with <4,8,5,9> that is trivially matched as a ZIP1 pattern before and after the patch.
The testcase was crafted this way to force the compiler to use VEXT instructions instead of loads and stores on the stack.

As I tried to explain the the commit message, this testcase is fragile in the sense that it requires a BUILD_VECTOR to "survive" all DAG transforms until ISelLowering. The testcase is supposed to check that AArch64TargetLowering::ReconstructShuffle() works, and for that we need a BUILD_VECTOR in ISelLowering. As we now transform the BUILD_VECTOR earlier into an VEXT + vector_shuffle, we don't have the BUILD_VECTOR pattern when we get to ISelLowering.

[javed.absar]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

8176 ↗

(On Diff #136604)

This is similar (identical, in fact, as far as i can see) to whats done in A32. LGTM.

llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.h

public:
   /// \brief Returns false if N is a bit extraction pattern of (X >> C) & Mask.
   bool isDesirableToCommuteWithShift(const SDNode *N) const override;
 
   /// \brief Returns true if it is beneficial to convert a load of a constant
   /// to just the constant itself.
   bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
                                          Type *Ty) const override;
 
+  /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
+  /// with this index.
+  bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
+                               unsigned Index) const override;
+
   Value *emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
                         AtomicOrdering Ord) const override;
   Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val,
                               Value *Addr, AtomicOrdering Ord) const override;
 
   void emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> &Builder) const override;
 
   TargetLoweringBase::AtomicExpansionKind

llvm/trunk/lib/Target/AArch64/AArch64ISelLowering.cpp

if (BitSize == 32)
     Val &= (1LL << 32) - 1;
 
   unsigned LZ = countLeadingZeros((uint64_t)Val);
   unsigned Shift = (63 - LZ) / 16;
   // MOVZ is free so return true for one or fewer MOVK.
   return Shift < 3;
 }
 
+bool AArch64TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
+                                                    unsigned Index) const {
+  if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
+    return false;
+
+  return (Index == 0 || Index == ResVT.getVectorNumElements());
+}
+
 /// Turn vector tests of the signbit in the form of:
 ///   xor (sra X, elt_size(X)-1), -1
 /// into:
 ///   cmge X, X, #0
 static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
                                          const AArch64Subtarget *Subtarget) {
   EVT VT = N->getValueType(0);
   if (!Subtarget->hasNEON() || !VT.isVector())

llvm/trunk/test/CodeGen/AArch64/aarch64-vuzp.ll

 ; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon < %s | FileCheck %s
 
 declare <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8>, <16 x i8>)
 
 ; CHECK-LABEL: fun1:
 ; CHECK: uzp1 {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
-; CHECK-NOT: mov
 define i32 @fun1() {
 entry:
   %vtbl1.i.1 = tail call <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8> <i8 0, i8 16, i8 19, i8 4, i8 -65, i8 -65, i8 -71, i8 -71, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> undef)
   %vuzp.i212.1 = shufflevector <16 x i8> %vtbl1.i.1, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
   %scevgep = getelementptr <8 x i8>, <8 x i8>* undef, i64 1
   store <8 x i8> %vuzp.i212.1, <8 x i8>* %scevgep, align 1
   ret i32 undef
 }
 
 ; CHECK-LABEL: fun2:
 ; CHECK: uzp2 {{v[0-9]+}}.8b, {{v[0-9]+}}.8b, {{v[0-9]+}}.8b
-; CHECK-NOT: mov
 define i32 @fun2() {
 entry:
   %vtbl1.i.1 = tail call <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8> <i8 0, i8 16, i8 19, i8 4, i8 -65, i8 -65, i8 -71, i8 -71, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> undef)
   %vuzp.i212.1 = shufflevector <16 x i8> %vtbl1.i.1, <16 x i8> undef, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %scevgep = getelementptr <8 x i8>, <8 x i8>* undef, i64 1
   store <8 x i8> %vuzp.i212.1, <8 x i8>* %scevgep, align 1
   ret i32 undef
 }
 
 ; CHECK-LABEL: fun3:
 ; CHECK-NOT: uzp1
-; CHECK: mov
 define i32 @fun3() {
 entry:
   %vtbl1.i.1 = tail call <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8> <i8 0, i8 16, i8 19, i8 4, i8 -65, i8 -65, i8 -71, i8 -71, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> undef)
   %vuzp.i212.1 = shufflevector <16 x i8> %vtbl1.i.1, <16 x i8> undef, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 15>
   %scevgep = getelementptr <8 x i8>, <8 x i8>* undef, i64 1
   store <8 x i8> %vuzp.i212.1, <8 x i8>* %scevgep, align 1
   ret i32 undef
 }
 
 ; CHECK-LABEL: fun4:
 ; CHECK-NOT: uzp2
-; CHECK: mov
 define i32 @fun4() {
 entry:
   %vtbl1.i.1 = tail call <16 x i8> @llvm.aarch64.neon.tbl1.v16i8(<16 x i8> <i8 0, i8 16, i8 19, i8 4, i8 -65, i8 -65, i8 -71, i8 -71, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i8> undef)
   %vuzp.i212.1 = shufflevector <16 x i8> %vtbl1.i.1, <16 x i8> undef, <8 x i32> <i32 3, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
   %scevgep = getelementptr <8 x i8>, <8 x i8>* undef, i64 1
   store <8 x i8> %vuzp.i212.1, <8 x i8>* %scevgep, align 1
   ret i32 undef
 }

llvm/trunk/test/CodeGen/AArch64/arm64-ext.ll

;CHECK: {{ext.16b.*#10}}
   %tmp1 = load <8 x i16>, <8 x i16>* %A
   %vext = shufflevector <8 x i16> %tmp1, <8 x i16> undef, <8 x i32> <i32 undef, i32 undef, i32 undef, i32 undef, i32 1, i32 2, i32 3, i32 4>
   ret <8 x i16> %vext;
 }
 
 ; Tests for ReconstructShuffle function. Indices have to be carefully
 ; chosen to reach lowering phase as a BUILD_VECTOR.
 
-; One vector needs vext, the other can be handled by extract_subvector
-; Also checks interleaving of sources is handled correctly.
-; Essence: a vext is used on %A and something saner than stack load/store for final result.
-define <4 x i16> @test_interleaved(<8 x i16>* %A, <8 x i16>* %B) nounwind {
-;CHECK-LABEL: test_interleaved:
-;CHECK: ext.8b
-;CHECK: zip1.4h
-        %tmp1 = load <8 x i16>, <8 x i16>* %A
-        %tmp2 = load <8 x i16>, <8 x i16>* %B
-        %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <4 x i32> <i32 3, i32 8, i32 5, i32 9>
-        ret <4 x i16> %tmp3
-}
-
 ; An undef in the shuffle list should still be optimizable
 define <4 x i16> @test_undef(<8 x i16>* %A, <8 x i16>* %B) nounwind {
 ;CHECK-LABEL: test_undef:
 ;CHECK: zip1.4h
         %tmp1 = load <8 x i16>, <8 x i16>* %A
         %tmp2 = load <8 x i16>, <8 x i16>* %B
         %tmp3 = shufflevector <8 x i16> %tmp1, <8 x i16> %tmp2, <4 x i32> <i32 undef, i32 8, i32 5, i32 9>
         ret <4 x i16> %tmp3
 }

llvm/trunk/test/CodeGen/AArch64/neon-scalar-copy.ll

  ; CHECK-NOT: ins {{[vsdh][0-9]+}}
  ; CHECK-NEXT: ret
  %tmp1 = extractelement <8 x half> %v, i32 0
  ret half  %tmp1
 }
 
 define <1 x i8> @test_vector_dup_bv16B(<16 x i8> %v1) #0 {
  ; CHECK-LABEL: test_vector_dup_bv16B:
- ; CHECK-NEXT: dup v0.16b, v0.b[14]
+ ; CHECK-NEXT: dup v0.8b, v0.b[14]
  ; CHECK-NEXT: ret
  %shuffle.i = shufflevector <16 x i8> %v1, <16 x i8> undef, <1 x i32> <i32 14> 
  ret <1 x i8> %shuffle.i
 }
 
 define <1 x i8> @test_vector_dup_bv8B(<8 x i8> %v1) #0 {
  ; CHECK-LABEL: test_vector_dup_bv8B:
  ; CHECK-NEXT: dup v0.8b, v0.b[7]
  ; CHECK-NEXT: ret
  %shuffle.i = shufflevector <8 x i8> %v1, <8 x i8> undef, <1 x i32> <i32 7> 
  ret <1 x i8> %shuffle.i
 }
 
 define <1 x i16> @test_vector_dup_hv8H(<8 x i16> %v1) #0 {
  ; CHECK-LABEL: test_vector_dup_hv8H:
- ; CHECK-NEXT:  dup v0.8h, v0.h[7]
+ ; CHECK-NEXT:  dup v0.4h, v0.h[7]
  ; CHECK-NEXT: ret
  %shuffle.i = shufflevector <8 x i16> %v1, <8 x i16> undef, <1 x i32> <i32 7> 
  ret <1 x i16> %shuffle.i
 }
 
 define <1 x i16> @test_vector_dup_hv4H(<4 x i16> %v1) #0 {
  ; CHECK-LABEL: test_vector_dup_hv4H:
  ; CHECK-NEXT: dup v0.4h, v0.h[3]
  ; CHECK-NEXT: ret
  %shuffle.i = shufflevector <4 x i16> %v1, <4 x i16> undef, <1 x i32> <i32 3> 
  ret <1 x i16> %shuffle.i
 }
 
 define <1 x i32> @test_vector_dup_sv4S(<4 x i32> %v1) #0 {
  ; CHECK-LABEL: test_vector_dup_sv4S:
- ; CHECK-NEXT: dup v0.4s, v0.s[3]
+ ; CHECK-NEXT: dup v0.2s, v0.s[3]
  ; CHECK-NEXT: ret
  %shuffle = shufflevector <4 x i32> %v1, <4 x i32> undef, <1 x i32> <i32 3> 
  ret <1 x i32> %shuffle
 }
 
 define <1 x i32> @test_vector_dup_sv2S(<2 x i32> %v1) #0 {
  ; CHECK-LABEL: test_vector_dup_sv2S:
  ; CHECK-NEXT: dup v0.2s, v0.s[1]
  ; CHECK-NEXT: ret
  %shuffle = shufflevector <2 x i32> %v1, <2 x i32> undef, <1 x i32> <i32 1> 
  ret <1 x i32> %shuffle
 }
 
 define <1 x i64> @test_vector_dup_dv2D(<2 x i64> %v1) #0 {
  ; CHECK-LABEL: test_vector_dup_dv2D:
  ; CHECK-NEXT: ext {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, #8
  ; CHECK-NEXT: ret
  %shuffle.i = shufflevector <2 x i64> %v1, <2 x i64> undef, <1 x i32> <i32 1> 
  ret <1 x i64> %shuffle.i
 }
 
 define <1 x i64> @test_vector_copy_dup_dv2D(<1 x i64> %a, <2 x i64> %c) #0 {
   ; CHECK-LABEL: test_vector_copy_dup_dv2D:
-  ; CHECK-NEXT: dup v0.2d, v1.d[1]
+  ; CHECK-NEXT: ext {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, {{v[0-9]+}}.16b, #8
   ; CHECK-NEXT: ret
   %vget_lane = extractelement <2 x i64> %c, i32 1
   %vset_lane = insertelement <1 x i64> undef, i64 %vget_lane, i32 0
   ret <1 x i64> %vset_lane
 }
 
 ; Undefined behaviour, so we really don't care what actually gets emitted, just
 ; as long as we don't crash (since it could be dynamically unreachable).