[AArch64] Custom lower concat(v4i8 load, ...)

[dmgreen]

Summary

We already have custom lowering for v4i8 load, which loads as a f32, converts to a vector and bitcasts and extends the result to a v4i16. This adds some custom lowering of concat(v4i8 load, ...) to keep the result as an f32 and create a buildvector of the resulting f32 loads. This helps not create all the extends and bitcasts, which are often difficult to fully clean up.

[SjoerdMeijer]

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

if (Subtarget->hasNEON()) {
     setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
 
     setLoadExtAction(ISD::EXTLOAD,  MVT::v4i16, MVT::v4i8, Custom);
     setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
     setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
     setLoadExtAction(ISD::EXTLOAD,  MVT::v4i32, MVT::v4i8, Custom);
     setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
     setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
+
+    setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i8, Custom);
   }
 
   if (Subtarget->hasSVE()) {
     for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
       setOperationAction(ISD::BITREVERSE, VT, Custom);
       setOperationAction(ISD::BSWAP, VT, Custom);
       setOperationAction(ISD::CTLZ, VT, Custom);
       setOperationAction(ISD::CTPOP, VT, Custom);
SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
   return SDValue();
 }
 
 SDValue AArch64TargetLowering::LowerCONCAT_VECTORS(SDValue Op,
                                                    SelectionDAG &DAG) const {
   if (useSVEForFixedLengthVectorVT(Op.getValueType()))
     return LowerFixedLengthConcatVectorsToSVE(Op, DAG);
 
+  if (Op.getOperand(0).getValueType() == MVT::v4i8) {
+    // If we have a concat of v4i8 loads, convert them to a buildvector of f32
+    // loads to prevent having to go through the v4i8 load legalization that
+    // needs to extend each element into a larger type.
+    if (Op.getNumOperands() % 2 == 0 && all_of(Op->op_values(), [](SDValue V) {
+          return V.getValueType() == MVT::v4i8 &&
+                 (V.getOpcode() == ISD::LOAD || V.isUndef());
+        })) {
+      EVT NVT =
+          EVT::getVectorVT(*DAG.getContext(), MVT::f32, Op.getNumOperands());
+      SmallVector<SDValue> Ops;
+      SDLoc DL(Op);
+
+      for (unsigned i = 0; i < Op.getNumOperands(); i++) {
+        SDValue V = Op.getOperand(i);
+        if (V.isUndef())
+          Ops.push_back(DAG.getUNDEF(MVT::f32));
+        else {
+          LoadSDNode *LD = cast<LoadSDNode>(V);
+          if (!LD->isSimple() || LD->isIndexed() ||
+              LD->getExtensionType() != ISD::NON_EXTLOAD)
+            return SDValue();
+          Ops.push_back(DAG.getLoad(MVT::f32, DL, LD->getChain(),
+                                    LD->getBasePtr(), LD->getMemOperand()));
+        }
+      }
+      return DAG.getBitcast(Op.getValueType(),
+                            DAG.getBuildVector(NVT, DL, Ops));
+    }
+
+    // Let the default expansion happen
+    return SDValue();
+  }
+
   assert(Op.getValueType().isScalableVector() &&
          isTypeLegal(Op.getValueType()) &&
          "Expected legal scalable vector type!");
 
   if (isTypeLegal(Op.getOperand(0).getValueType())) {
     unsigned NumOperands = Op->getNumOperands();
     assert(NumOperands > 1 && isPowerOf2_32(NumOperands) &&
            "Unexpected number of operands in CONCAT_VECTORS");
SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i128,
                                Result.getValue(0), Result.getValue(1));
     Results.append({Pair, Result.getValue(2) /* Chain */});
     return;
   }
   case ISD::EXTRACT_SUBVECTOR:
     ReplaceExtractSubVectorResults(N, Results, DAG);
     return;
   case ISD::INSERT_SUBVECTOR:
-    // Custom lowering has been requested for INSERT_SUBVECTOR -- but delegate
-    // to common code for result type legalisation
+  case ISD::CONCAT_VECTORS:
+    // Custom lowering has been requested for INSERT_SUBVECTOR and
+    // CONCAT_VECTORS -- but delegate to common code for result type
+    // legalisation
     return;
   case ISD::INTRINSIC_WO_CHAIN: {
     EVT VT = N->getValueType(0);
     assert((VT == MVT::i8 || VT == MVT::i16) &&
            "custom lowering for unexpected type");
 
     ConstantSDNode *CN = cast<ConstantSDNode>(N->getOperand(0));
     Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());

llvm/test/CodeGen/AArch64/insert-extend.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64--linux-gnu | FileCheck %s
 
 define <8 x i8> @load4_v4i8_add(float %tmp, <4 x i8> *%a, <4 x i8> *%b) {
 ; CHECK-LABEL: load4_v4i8_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp s0, s1, [x0]
-; CHECK-NEXT:    ldp s2, s3, [x1]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v2.8b
-; CHECK-NEXT:    uzp1 v1.8b, v1.8b, v3.8b
+; CHECK-NEXT:    ld1 { v0.s }[1], [x1], #4
+; CHECK-NEXT:    ld1 { v1.s }[1], [x1]
 ; CHECK-NEXT:    add v0.8b, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
   %la = load <4 x i8>, <4 x i8> *%a
   %lb = load <4 x i8>, <4 x i8> *%b
   %c = getelementptr <4 x i8>, <4 x i8> *%a, i64 1
   %d = getelementptr <4 x i8>, <4 x i8> *%b, i64 1
   %lc = load <4 x i8>, <4 x i8> *%c
   %ld = load <4 x i8>, <4 x i8> *%d
   %s1 = shufflevector <4 x i8> %la, <4 x i8> %lb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s2 = shufflevector <4 x i8> %lc, <4 x i8> %ld, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %add = add <8 x i8> %s1, %s2
   ret <8 x i8> %add
 }
 
 define <8 x i16> @load4_v4i8_zext_add(float %tmp, <4 x i8> *%a, <4 x i8> *%b) {
 ; CHECK-LABEL: load4_v4i8_zext_add:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp s0, s1, [x0]
-; CHECK-NEXT:    ldp s2, s3, [x1]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v2.8b
-; CHECK-NEXT:    uzp1 v1.8b, v1.8b, v3.8b
+; CHECK-NEXT:    ld1 { v0.s }[1], [x1], #4
+; CHECK-NEXT:    ld1 { v1.s }[1], [x1]
 ; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
 ; CHECK-NEXT:    ret
   %la = load <4 x i8>, <4 x i8> *%a
   %lb = load <4 x i8>, <4 x i8> *%b
   %c = getelementptr <4 x i8>, <4 x i8> *%a, i64 1
   %d = getelementptr <4 x i8>, <4 x i8> *%b, i64 1
   %lc = load <4 x i8>, <4 x i8> *%c
   %ld = load <4 x i8>, <4 x i8> *%d
   %s1 = shufflevector <4 x i8> %la, <4 x i8> %lb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s2 = shufflevector <4 x i8> %lc, <4 x i8> %ld, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %z1 = zext <8 x i8> %s1 to <8 x i16>
   %z2 = zext <8 x i8> %s2 to <8 x i16>
   %add = add <8 x i16> %z1, %z2
   ret <8 x i16> %add
 }
 
 define i32 @large(i8* nocapture noundef readonly %p1, i32 noundef %st1, i8* nocapture noundef readonly %p2, i32 noundef %st2) {
 ; CHECK-LABEL: large:
 ; CHECK:       // %bb.0: // %entry
 ; CHECK-NEXT:    // kill: def $w1 killed $w1 def $x1
 ; CHECK-NEXT:    sxtw x8, w1
 ; CHECK-NEXT:    // kill: def $w3 killed $w3 def $x3
-; CHECK-NEXT:    sxtw x9, w3
-; CHECK-NEXT:    ldp s0, s1, [x0]
-; CHECK-NEXT:    ldp s2, s3, [x2]
-; CHECK-NEXT:    add x10, x0, x8
-; CHECK-NEXT:    add x11, x2, x9
-; CHECK-NEXT:    ushll v4.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v0.8h, v3.8b, #0
-; CHECK-NEXT:    ldp s5, s3, [x10]
-; CHECK-NEXT:    add x10, x10, x8
+; CHECK-NEXT:    sxtw x11, w3
+; CHECK-NEXT:    add x9, x0, x8
+; CHECK-NEXT:    add x12, x2, x11
+; CHECK-NEXT:    add x10, x9, x8
+; CHECK-NEXT:    add x13, x12, x11
 ; CHECK-NEXT:    add x8, x10, x8
-; CHECK-NEXT:    ldp s6, s7, [x11]
-; CHECK-NEXT:    ldp s16, s17, [x10]
-; CHECK-NEXT:    ldp s18, s21, [x8]
-; CHECK-NEXT:    add x11, x11, x9
-; CHECK-NEXT:    add x9, x11, x9
-; CHECK-NEXT:    ushll v5.8h, v5.8b, #0
-; CHECK-NEXT:    ushll v16.8h, v16.8b, #0
-; CHECK-NEXT:    ushll v18.8h, v18.8b, #0
-; CHECK-NEXT:    ldp s19, s20, [x11]
-; CHECK-NEXT:    uzp1 v16.8b, v18.8b, v16.8b
-; CHECK-NEXT:    uzp1 v4.8b, v5.8b, v4.8b
-; CHECK-NEXT:    ldp s18, s5, [x9]
-; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEXT:    ushll v6.8h, v6.8b, #0
-; CHECK-NEXT:    ushll v19.8h, v19.8b, #0
-; CHECK-NEXT:    ushll v18.8h, v18.8b, #0
-; CHECK-NEXT:    uzp1 v2.8b, v6.8b, v2.8b
-; CHECK-NEXT:    uzp1 v18.8b, v18.8b, v19.8b
+; CHECK-NEXT:    add x11, x13, x11
+; CHECK-NEXT:    ldp s1, s5, [x9]
+; CHECK-NEXT:    ldp s0, s4, [x8]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x10], #4
+; CHECK-NEXT:    ld1 { v1.s }[1], [x0], #4
+; CHECK-NEXT:    ldp s2, s6, [x11]
+; CHECK-NEXT:    ldp s3, s7, [x12]
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
-; CHECK-NEXT:    ushll v17.8h, v17.8b, #0
-; CHECK-NEXT:    ushll v20.8h, v20.8b, #0
-; CHECK-NEXT:    ushll v6.8h, v16.8b, #0
-; CHECK-NEXT:    ushll v4.8h, v4.8b, #0
-; CHECK-NEXT:    ushll v16.8h, v18.8b, #0
+; CHECK-NEXT:    ld1 { v2.s }[1], [x13], #4
+; CHECK-NEXT:    ld1 { v3.s }[1], [x2], #4
+; CHECK-NEXT:    ld1 { v4.s }[1], [x10]
+; CHECK-NEXT:    ld1 { v5.s }[1], [x0]
+; CHECK-NEXT:    ld1 { v6.s }[1], [x13]
+; CHECK-NEXT:    ld1 { v7.s }[1], [x2]
 ; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEXT:    ushll v19.8h, v21.8b, #0
-; CHECK-NEXT:    ushll v5.8h, v5.8b, #0
-; CHECK-NEXT:    ushll v7.8h, v7.8b, #0
-; CHECK-NEXT:    usubl v18.4s, v6.4h, v16.4h
-; CHECK-NEXT:    usubl2 v6.4s, v6.8h, v16.8h
-; CHECK-NEXT:    usubl v16.4s, v4.4h, v2.4h
-; CHECK-NEXT:    usubl2 v2.4s, v4.8h, v2.8h
-; CHECK-NEXT:    uzp1 v4.8b, v19.8b, v17.8b
-; CHECK-NEXT:    uzp1 v1.8b, v3.8b, v1.8b
-; CHECK-NEXT:    uzp1 v3.8b, v5.8b, v20.8b
-; CHECK-NEXT:    uzp1 v0.8b, v7.8b, v0.8b
-; CHECK-NEXT:    ushll v4.8h, v4.8b, #0
 ; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    usubl2 v5.4s, v4.8h, v3.8h
-; CHECK-NEXT:    usubl v3.4s, v4.4h, v3.4h
-; CHECK-NEXT:    usubl2 v4.4s, v1.8h, v0.8h
-; CHECK-NEXT:    usubl v0.4s, v1.4h, v0.4h
-; CHECK-NEXT:    shl v1.4s, v3.4s, #16
-; CHECK-NEXT:    shl v3.4s, v5.4s, #16
+; CHECK-NEXT:    usubl v16.4s, v0.4h, v2.4h
+; CHECK-NEXT:    usubl2 v0.4s, v0.8h, v2.8h
+; CHECK-NEXT:    usubl v2.4s, v1.4h, v3.4h
+; CHECK-NEXT:    usubl2 v1.4s, v1.8h, v3.8h
+; CHECK-NEXT:    ushll v3.8h, v4.8b, #0
+; CHECK-NEXT:    ushll v4.8h, v5.8b, #0
+; CHECK-NEXT:    ushll v5.8h, v6.8b, #0
+; CHECK-NEXT:    ushll v6.8h, v7.8b, #0
+; CHECK-NEXT:    usubl2 v7.4s, v3.8h, v5.8h
+; CHECK-NEXT:    usubl v3.4s, v3.4h, v5.4h
+; CHECK-NEXT:    usubl2 v5.4s, v4.8h, v6.8h
+; CHECK-NEXT:    usubl v4.4s, v4.4h, v6.4h
+; CHECK-NEXT:    shl v3.4s, v3.4s, #16
+; CHECK-NEXT:    shl v6.4s, v7.4s, #16
+; CHECK-NEXT:    shl v5.4s, v5.4s, #16
 ; CHECK-NEXT:    shl v4.4s, v4.4s, #16
-; CHECK-NEXT:    add v1.4s, v1.4s, v18.4s
-; CHECK-NEXT:    shl v0.4s, v0.4s, #16
-; CHECK-NEXT:    add v3.4s, v3.4s, v6.4s
+; CHECK-NEXT:    add v0.4s, v6.4s, v0.4s
+; CHECK-NEXT:    add v3.4s, v3.4s, v16.4s
+; CHECK-NEXT:    add v1.4s, v5.4s, v1.4s
 ; CHECK-NEXT:    add v2.4s, v4.4s, v2.4s
-; CHECK-NEXT:    rev64 v4.4s, v3.4s
-; CHECK-NEXT:    rev64 v5.4s, v1.4s
-; CHECK-NEXT:    add v0.4s, v0.4s, v16.4s
-; CHECK-NEXT:    rev64 v6.4s, v2.4s
-; CHECK-NEXT:    rev64 v7.4s, v0.4s
-; CHECK-NEXT:    add v16.4s, v3.4s, v4.4s
-; CHECK-NEXT:    add v17.4s, v1.4s, v5.4s
-; CHECK-NEXT:    sub v1.4s, v1.4s, v5.4s
-; CHECK-NEXT:    trn2 v5.4s, v16.4s, v17.4s
-; CHECK-NEXT:    add v18.4s, v2.4s, v6.4s
-; CHECK-NEXT:    add v19.4s, v0.4s, v7.4s
-; CHECK-NEXT:    sub v2.4s, v2.4s, v6.4s
-; CHECK-NEXT:    sub v0.4s, v0.4s, v7.4s
-; CHECK-NEXT:    sub v3.4s, v3.4s, v4.4s
-; CHECK-NEXT:    trn2 v4.4s, v19.4s, v18.4s
-; CHECK-NEXT:    ext v6.16b, v5.16b, v16.16b, #8
-; CHECK-NEXT:    zip1 v7.4s, v0.4s, v2.4s
-; CHECK-NEXT:    trn2 v16.4s, v17.4s, v16.4s
-; CHECK-NEXT:    ext v4.16b, v19.16b, v4.16b, #8
-; CHECK-NEXT:    zip1 v20.4s, v3.4s, v1.4s
-; CHECK-NEXT:    ext v7.16b, v0.16b, v7.16b, #8
-; CHECK-NEXT:    ext v17.16b, v16.16b, v17.16b, #8
-; CHECK-NEXT:    zip2 v18.4s, v19.4s, v18.4s
-; CHECK-NEXT:    zip2 v1.4s, v3.4s, v1.4s
-; CHECK-NEXT:    mov v0.s[3], v2.s[2]
-; CHECK-NEXT:    mov v5.d[1], v4.d[1]
-; CHECK-NEXT:    mov v20.d[1], v7.d[1]
-; CHECK-NEXT:    mov v17.d[1], v18.d[1]
-; CHECK-NEXT:    mov v16.d[1], v4.d[1]
-; CHECK-NEXT:    mov v1.d[1], v0.d[1]
-; CHECK-NEXT:    mov v6.d[1], v18.d[1]
-; CHECK-NEXT:    add v0.4s, v17.4s, v16.4s
-; CHECK-NEXT:    add v2.4s, v1.4s, v20.4s
-; CHECK-NEXT:    sub v3.4s, v5.4s, v6.4s
-; CHECK-NEXT:    sub v1.4s, v20.4s, v1.4s
 ; CHECK-NEXT:    rev64 v4.4s, v0.4s
 ; CHECK-NEXT:    rev64 v5.4s, v3.4s
 ; CHECK-NEXT:    rev64 v6.4s, v1.4s
 ; CHECK-NEXT:    rev64 v7.4s, v2.4s
 ; CHECK-NEXT:    add v16.4s, v0.4s, v4.4s
 ; CHECK-NEXT:    add v17.4s, v3.4s, v5.4s
 ; CHECK-NEXT:    add v18.4s, v1.4s, v6.4s
 ; CHECK-NEXT:    add v19.4s, v2.4s, v7.4s
-; CHECK-NEXT:    sub v2.4s, v2.4s, v7.4s
 ; CHECK-NEXT:    sub v1.4s, v1.4s, v6.4s
-; CHECK-NEXT:    sub v3.4s, v3.4s, v5.4s
+; CHECK-NEXT:    sub v2.4s, v2.4s, v7.4s
 ; CHECK-NEXT:    sub v0.4s, v0.4s, v4.4s
+; CHECK-NEXT:    sub v3.4s, v3.4s, v5.4s
+; CHECK-NEXT:    trn2 v4.4s, v16.4s, v17.4s
+; CHECK-NEXT:    trn2 v5.4s, v19.4s, v18.4s
+; CHECK-NEXT:    zip1 v7.4s, v2.4s, v1.4s
+; CHECK-NEXT:    trn2 v20.4s, v17.4s, v16.4s
+; CHECK-NEXT:    zip1 v6.4s, v0.4s, v3.4s
+; CHECK-NEXT:    zip2 v18.4s, v19.4s, v18.4s
+; CHECK-NEXT:    ext v5.16b, v19.16b, v5.16b, #8
+; CHECK-NEXT:    ext v16.16b, v4.16b, v16.16b, #8
+; CHECK-NEXT:    ext v7.16b, v2.16b, v7.16b, #8
+; CHECK-NEXT:    ext v17.16b, v20.16b, v17.16b, #8
+; CHECK-NEXT:    zip2 v0.4s, v0.4s, v3.4s
+; CHECK-NEXT:    mov v2.s[3], v1.s[2]
+; CHECK-NEXT:    mov v4.d[1], v5.d[1]
+; CHECK-NEXT:    mov v6.d[1], v7.d[1]
+; CHECK-NEXT:    mov v17.d[1], v18.d[1]
+; CHECK-NEXT:    mov v20.d[1], v5.d[1]
+; CHECK-NEXT:    mov v0.d[1], v2.d[1]
+; CHECK-NEXT:    mov v16.d[1], v18.d[1]
+; CHECK-NEXT:    add v1.4s, v17.4s, v20.4s
+; CHECK-NEXT:    add v2.4s, v0.4s, v6.4s
+; CHECK-NEXT:    sub v3.4s, v4.4s, v16.4s
+; CHECK-NEXT:    sub v0.4s, v6.4s, v0.4s
+; CHECK-NEXT:    rev64 v4.4s, v1.4s
+; CHECK-NEXT:    rev64 v5.4s, v3.4s
+; CHECK-NEXT:    rev64 v6.4s, v0.4s
+; CHECK-NEXT:    rev64 v7.4s, v2.4s
+; CHECK-NEXT:    add v16.4s, v1.4s, v4.4s
+; CHECK-NEXT:    add v17.4s, v3.4s, v5.4s
+; CHECK-NEXT:    add v18.4s, v0.4s, v6.4s
+; CHECK-NEXT:    add v19.4s, v2.4s, v7.4s
+; CHECK-NEXT:    sub v2.4s, v2.4s, v7.4s
+; CHECK-NEXT:    sub v0.4s, v0.4s, v6.4s
+; CHECK-NEXT:    sub v3.4s, v3.4s, v5.4s
+; CHECK-NEXT:    sub v1.4s, v1.4s, v4.4s
 ; CHECK-NEXT:    ext v4.16b, v2.16b, v19.16b, #12
-; CHECK-NEXT:    ext v5.16b, v1.16b, v18.16b, #12
+; CHECK-NEXT:    ext v5.16b, v0.16b, v18.16b, #12
 ; CHECK-NEXT:    ext v7.16b, v3.16b, v17.16b, #12
 ; CHECK-NEXT:    rev64 v16.4s, v16.4s
 ; CHECK-NEXT:    ext v6.16b, v4.16b, v2.16b, #4
 ; CHECK-NEXT:    ext v17.16b, v4.16b, v4.16b, #8
-; CHECK-NEXT:    ext v18.16b, v5.16b, v1.16b, #4
+; CHECK-NEXT:    ext v18.16b, v5.16b, v0.16b, #4
 ; CHECK-NEXT:    ext v19.16b, v5.16b, v5.16b, #8
 ; CHECK-NEXT:    ext v20.16b, v7.16b, v3.16b, #4
 ; CHECK-NEXT:    ext v21.16b, v7.16b, v7.16b, #8
 ; CHECK-NEXT:    rev64 v7.4s, v7.4s
-; CHECK-NEXT:    trn2 v0.4s, v16.4s, v0.4s
+; CHECK-NEXT:    trn2 v1.4s, v16.4s, v1.4s
 ; CHECK-NEXT:    rev64 v5.4s, v5.4s
 ; CHECK-NEXT:    rev64 v4.4s, v4.4s
 ; CHECK-NEXT:    ext v6.16b, v6.16b, v17.16b, #12
 ; CHECK-NEXT:    ext v17.16b, v18.16b, v19.16b, #12
 ; CHECK-NEXT:    ext v18.16b, v20.16b, v21.16b, #12
 ; CHECK-NEXT:    ext v3.16b, v7.16b, v3.16b, #4
-; CHECK-NEXT:    ext v7.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT:    ext v1.16b, v5.16b, v1.16b, #4
+; CHECK-NEXT:    ext v7.16b, v1.16b, v1.16b, #8
+; CHECK-NEXT:    ext v0.16b, v5.16b, v0.16b, #4
 ; CHECK-NEXT:    ext v2.16b, v4.16b, v2.16b, #4
 ; CHECK-NEXT:    add v4.4s, v18.4s, v3.4s
-; CHECK-NEXT:    add v5.4s, v0.4s, v7.4s
-; CHECK-NEXT:    add v16.4s, v17.4s, v1.4s
+; CHECK-NEXT:    add v5.4s, v1.4s, v7.4s
+; CHECK-NEXT:    add v16.4s, v17.4s, v0.4s
 ; CHECK-NEXT:    add v19.4s, v6.4s, v2.4s
 ; CHECK-NEXT:    sub v3.4s, v18.4s, v3.4s
-; CHECK-NEXT:    sub v0.4s, v0.4s, v7.4s
+; CHECK-NEXT:    sub v1.4s, v1.4s, v7.4s
 ; CHECK-NEXT:    sub v2.4s, v6.4s, v2.4s
-; CHECK-NEXT:    sub v1.4s, v17.4s, v1.4s
+; CHECK-NEXT:    sub v0.4s, v17.4s, v0.4s
 ; CHECK-NEXT:    mov v19.d[1], v2.d[1]
-; CHECK-NEXT:    mov v16.d[1], v1.d[1]
+; CHECK-NEXT:    mov v16.d[1], v0.d[1]
 ; CHECK-NEXT:    mov v4.d[1], v3.d[1]
-; CHECK-NEXT:    mov v5.d[1], v0.d[1]
+; CHECK-NEXT:    mov v5.d[1], v1.d[1]
 ; CHECK-NEXT:    movi v0.8h, #1
 ; CHECK-NEXT:    movi v7.2d, #0x00ffff0000ffff
 ; CHECK-NEXT:    ushr v1.4s, v4.4s, #15
 ; CHECK-NEXT:    ushr v2.4s, v19.4s, #15
 ; CHECK-NEXT:    ushr v3.4s, v5.4s, #15
 ; CHECK-NEXT:    ushr v6.4s, v16.4s, #15
 ; CHECK-NEXT:    and v2.16b, v2.16b, v0.16b
 ; CHECK-NEXT:    and v6.16b, v6.16b, v0.16b

llvm/test/CodeGen/AArch64/insert-subvector.ll

 
 
 ; i8
 
 define <16 x i8> @load_v16i8_4_1(float %tmp, <16 x i8> %b, <4 x i8> *%a) {
 ; CHECK-LABEL: load_v16i8_4_1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ldr s1, [x0]
-; CHECK-NEXT:    mov v0.s[0], v1.s[0]
+; CHECK-NEXT:    ld1 { v0.s }[0], [x0]
 ; CHECK-NEXT:    ret
   %l = load <4 x i8>, <4 x i8> *%a
   %s1 = shufflevector <4 x i8> %l, <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %s2 = shufflevector <16 x i8> %s1, <16 x i8> %b, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
   ret <16 x i8> %s2
 }
 
 define <16 x i8> @load_v16i8_4_15(float %tmp, <16 x i8> %b, <4 x i8> *%a) {
; CHECK-NEXT:    ret
   %s2 = shufflevector <16 x i8> %s1, <16 x i8> %b, <16 x i32> <i32 16, i32 17, i32 0, i32 1, i32 2, i32 3, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
   ret <16 x i8> %s2
 }
 
 define <16 x i8> @load_v16i8_4_2(float %tmp, <16 x i8> %b, <4 x i8> *%a) {
 ; CHECK-LABEL: load_v16i8_4_2:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ldr s1, [x0]
-; CHECK-NEXT:    mov v0.s[1], v1.s[0]
+; CHECK-NEXT:    ld1 { v0.s }[1], [x0]
 ; CHECK-NEXT:    ret
   %l = load <4 x i8>, <4 x i8> *%a
   %s1 = shufflevector <4 x i8> %l, <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %s2 = shufflevector <16 x i8> %s1, <16 x i8> %b, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 0, i32 1, i32 2, i32 3, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
   ret <16 x i8> %s2
 }
 
 define <16 x i8> @load_v16i8_4_3(float %tmp, <16 x i8> %b, <4 x i8> *%a) {
 ; CHECK-LABEL: load_v16i8_4_3:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ldr s1, [x0]
-; CHECK-NEXT:    mov v0.s[2], v1.s[0]
+; CHECK-NEXT:    ld1 { v0.s }[2], [x0]
 ; CHECK-NEXT:    ret
   %l = load <4 x i8>, <4 x i8> *%a
   %s1 = shufflevector <4 x i8> %l, <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %s2 = shufflevector <16 x i8> %s1, <16 x i8> %b, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 0, i32 1, i32 2, i32 3, i32 28, i32 29, i32 30, i32 31>
   ret <16 x i8> %s2
 }
 
 define <16 x i8> @load_v16i8_4_4(float %tmp, <16 x i8> %b, <4 x i8> *%a) {
 ; CHECK-LABEL: load_v16i8_4_4:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    mov v0.16b, v1.16b
-; CHECK-NEXT:    ldr s1, [x0]
-; CHECK-NEXT:    mov v0.s[3], v1.s[0]
+; CHECK-NEXT:    ld1 { v0.s }[3], [x0]
 ; CHECK-NEXT:    ret
   %l = load <4 x i8>, <4 x i8> *%a
   %s1 = shufflevector <4 x i8> %l, <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
   %s2 = shufflevector <16 x i8> %s1, <16 x i8> %b, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 0, i32 1, i32 2, i32 3>
   ret <16 x i8> %s2
 }
 
 define <8 x i8> @load_v8i8_4_1(float %tmp, <8 x i8> %b, <4 x i8> *%a) {
 ; CHECK-LABEL: load_v8i8_4_1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr s0, [x0]
 ; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
 ; CHECK-NEXT:    mov v0.s[1], v1.s[1]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
   %l = load <4 x i8>, <4 x i8> *%a
   %s1 = shufflevector <4 x i8> %l, <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
   %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
   ret <8 x i8> %s2
 }
 
 define <8 x i8> @load_v8i8_4_2(float %tmp, <8 x i8> %b, <4 x i8> *%a) {
 ; CHECK-LABEL: load_v8i8_4_2:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    uzp1 v2.8b, v0.8b, v0.8b
 ; CHECK-NEXT:    fmov d0, d1
-; CHECK-NEXT:    mov v0.s[1], v2.s[0]
+; CHECK-NEXT:    ldr s1, [x0]
+; CHECK-NEXT:    mov v0.s[1], v1.s[0]
 ; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
   %l = load <4 x i8>, <4 x i8> *%a
   %s1 = shufflevector <4 x i8> %l, <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
   %s2 = shufflevector <8 x i8> %s1, <8 x i8> %b, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 0, i32 1, i32 2, i32 3>
   ret <8 x i8> %s2
 }
 

 }
 
 ; More than a single vector
 
 define <8 x i8> @load2_v4i8(float %tmp, <4 x i8> *%a, <4 x i8> *%b) {
 ; CHECK-LABEL: load2_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    ldr s1, [x1]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    ld1 { v0.s }[1], [x1]
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
   %la = load <4 x i8>, <4 x i8> *%a
   %lb = load <4 x i8>, <4 x i8> *%b
   %s1 = shufflevector <4 x i8> %la, <4 x i8> %lb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   ret <8 x i8> %s1
 }
 
 define <16 x i8> @load3_v4i8(float %tmp, <4 x i8> *%a, <4 x i8> *%b) {
 ; CHECK-LABEL: load3_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp s0, s1, [x0]
-; CHECK-NEXT:    ldr s3, [x1]
-; CHECK-NEXT:    ushll v2.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    umov w8, v2.h[0]
-; CHECK-NEXT:    umov w9, v2.h[1]
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    umov w8, v2.h[2]
-; CHECK-NEXT:    mov v0.b[1], w9
-; CHECK-NEXT:    umov w9, v2.h[3]
-; CHECK-NEXT:    ushll v2.8h, v3.8b, #0
-; CHECK-NEXT:    mov v0.b[2], w8
-; CHECK-NEXT:    umov w8, v2.h[0]
-; CHECK-NEXT:    mov v0.b[3], w9
-; CHECK-NEXT:    umov w9, v2.h[1]
-; CHECK-NEXT:    mov v0.b[4], w8
-; CHECK-NEXT:    umov w8, v2.h[2]
-; CHECK-NEXT:    mov v0.b[5], w9
-; CHECK-NEXT:    umov w9, v2.h[3]
-; CHECK-NEXT:    mov v0.b[6], w8
-; CHECK-NEXT:    umov w8, v1.h[0]
-; CHECK-NEXT:    mov v0.b[7], w9
-; CHECK-NEXT:    umov w9, v1.h[1]
-; CHECK-NEXT:    mov v0.b[8], w8
-; CHECK-NEXT:    umov w8, v1.h[2]
-; CHECK-NEXT:    mov v0.b[9], w9
-; CHECK-NEXT:    umov w9, v1.h[3]
-; CHECK-NEXT:    mov v0.b[10], w8
-; CHECK-NEXT:    mov v0.b[11], w9
+; CHECK-NEXT:    ld1 { v0.s }[1], [x1]
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    ret
   %la = load <4 x i8>, <4 x i8> *%a
   %lb = load <4 x i8>, <4 x i8> *%b
   %c = getelementptr <4 x i8>, <4 x i8> *%a, i64 1
   %d = getelementptr <4 x i8>, <4 x i8> *%b, i64 1
   %lc = load <4 x i8>, <4 x i8> *%c
   %s1 = shufflevector <4 x i8> %la, <4 x i8> %lb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s2 = shufflevector <4 x i8> %lc, <4 x i8> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s3 = shufflevector <8 x i8> %s1, <8 x i8> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i8> %s3
 }
 
 define <16 x i8> @load4_v4i8(float %tmp, <4 x i8> *%a, <4 x i8> *%b) {
 ; CHECK-LABEL: load4_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp s0, s1, [x0]
-; CHECK-NEXT:    ldp s2, s3, [x1]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v3.8h, v3.8b, #0
-; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEXT:    mov v1.d[1], v3.d[0]
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
-; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-NEXT:    ld1 { v0.s }[1], [x1], #4
+; CHECK-NEXT:    ld1 { v1.s }[1], [x1]
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    ret
   %la = load <4 x i8>, <4 x i8> *%a
   %lb = load <4 x i8>, <4 x i8> *%b
   %c = getelementptr <4 x i8>, <4 x i8> *%a, i64 1
   %d = getelementptr <4 x i8>, <4 x i8> *%b, i64 1
   %lc = load <4 x i8>, <4 x i8> *%c
   %ld = load <4 x i8>, <4 x i8> *%d
   %s1 = shufflevector <4 x i8> %la, <4 x i8> %lb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s2 = shufflevector <4 x i8> %lc, <4 x i8> %ld, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s3 = shufflevector <8 x i8> %s1, <8 x i8> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i8> %s3
 }
 
 define <16 x i8> @load2multi1_v4i8(float %tmp, <4 x i8> *%a, <4 x i8> *%b) {
 ; CHECK-LABEL: load2multi1_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldr s0, [x0]
-; CHECK-NEXT:    ldr s1, [x1]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
-; CHECK-NEXT:    uzp1 v0.16b, v0.16b, v0.16b
+; CHECK-NEXT:    ld1 { v0.s }[1], [x1]
+; CHECK-NEXT:    mov v0.d[1], v0.d[0]
 ; CHECK-NEXT:    ret
   %la = load <4 x i8>, <4 x i8> *%a
   %lb = load <4 x i8>, <4 x i8> *%b
   %s1 = shufflevector <4 x i8> %la, <4 x i8> %lb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s3 = shufflevector <8 x i8> %s1, <8 x i8> %s1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i8> %s3
 }
 
 define <16 x i8> @load2multi2_v4i8(float %tmp, <4 x i8> *%a, <4 x i8> *%b) {
 ; CHECK-LABEL: load2multi2_v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr s0, [x1]
-; CHECK-NEXT:    ldr s1, [x0]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    mov v0.d[1], v0.d[0]
-; CHECK-NEXT:    mov v1.d[1], v1.d[0]
-; CHECK-NEXT:    uzp1 v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    ld1r { v0.2s }, [x0]
+; CHECK-NEXT:    ld1r { v1.2s }, [x1]
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
 ; CHECK-NEXT:    ret
   %la = load <4 x i8>, <4 x i8> *%a
   %lb = load <4 x i8>, <4 x i8> *%b
   %s1 = shufflevector <4 x i8> %la, <4 x i8> %la, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s2 = shufflevector <4 x i8> %lb, <4 x i8> %lb, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
   %s3 = shufflevector <8 x i8> %s1, <8 x i8> %s2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   ret <16 x i8> %s3
 }