[AArch64] Extra tablegen patterns for smaller extracted addl/addw/subl/subw

[dmgreen]

Summary

During lowering, especially of smaller vector types, we can end up with add (extract_subvector(zext(x), extract_subvector(zext(y)), which can be turned into extract_subvector(add(zext(y), zext(x))), which can use the addl AArch64 instruction. This adds some tablegen patterns for it, along with addw where only one operand is an extract/extend and subl/subw.

[dmgreen]

Ping, thanks.

[SjoerdMeijer]

Look like some good patterns and improvements to me.

llvm/lib/Target/AArch64/AArch64InstrInfo.td

defm : Neon_mul_acc_widen_patterns<add, AArch64umull,
      UMLALv8i8_v8i16, UMLALv4i16_v4i32, UMLALv2i32_v2i64>;
 defm : Neon_mul_acc_widen_patterns<add, AArch64smull,
      SMLALv8i8_v8i16, SMLALv4i16_v4i32, SMLALv2i32_v2i64>;
 defm : Neon_mul_acc_widen_patterns<sub, AArch64umull,
      UMLSLv8i8_v8i16, UMLSLv4i16_v4i32, UMLSLv2i32_v2i64>;
 defm : Neon_mul_acc_widen_patterns<sub, AArch64smull,
      SMLSLv8i8_v8i16, SMLSLv4i16_v4i32, SMLSLv2i32_v2i64>;
 
+
+multiclass Neon_addl_extract_patterns<SDPatternOperator opnode, SDPatternOperator ext, string Inst> {
+  def : Pat<(v4i16 (opnode (extract_subvector (ext (v8i8 V64:$Rn)), (i64 0)),
+                           (extract_subvector (ext (v8i8 V64:$Rm)), (i64 0)))),
+            (EXTRACT_SUBREG (v8i16 (!cast<Instruction>(Inst#"Lv8i8_v8i16") V64:$Rn, V64:$Rm)), dsub)>;
+  def : Pat<(v2i32 (opnode (extract_subvector (ext (v4i16 V64:$Rn)), (i64 0)),
+                           (extract_subvector (ext (v4i16 V64:$Rm)), (i64 0)))),
+            (EXTRACT_SUBREG (v4i32 (!cast<Instruction>(Inst#"Lv4i16_v4i32") V64:$Rn, V64:$Rm)), dsub)>;
+  def : Pat<(v1i64 (opnode (extract_subvector (ext (v2i32 V64:$Rn)), (i64 0)),
+                           (extract_subvector (ext (v2i32 V64:$Rm)), (i64 0)))),
+            (EXTRACT_SUBREG (v2i64 (!cast<Instruction>(Inst#"Lv2i32_v2i64") V64:$Rn, V64:$Rm)), dsub)>;
+
+  def : Pat<(v4i16 (opnode (v4i16 V64:$Rn),
+                           (extract_subvector (ext (v8i8 V64:$Rm)), (i64 0)))),
+            (EXTRACT_SUBREG (v8i16 (!cast<Instruction>(Inst#"Wv8i8_v8i16") (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)), V64:$Rn, dsub), V64:$Rm)), dsub)>;
+  def : Pat<(v2i32 (opnode (v2i32 V64:$Rn),
+                           (extract_subvector (ext (v4i16 V64:$Rm)), (i64 0)))),
+            (EXTRACT_SUBREG (v4i32 (!cast<Instruction>(Inst#"Wv4i16_v4i32") (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), V64:$Rn, dsub), V64:$Rm)), dsub)>;
+  def : Pat<(v1i64 (opnode (v1i64 V64:$Rn),
+                           (extract_subvector (ext (v2i32 V64:$Rm)), (i64 0)))),
+            (EXTRACT_SUBREG (v2i64 (!cast<Instruction>(Inst#"Wv2i32_v2i64") (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)), V64:$Rn, dsub), V64:$Rm)), dsub)>;
+}
+
+defm : Neon_addl_extract_patterns<add, zanyext, "UADD">;
+defm : Neon_addl_extract_patterns<add, sext, "SADD">;
+defm : Neon_addl_extract_patterns<sub, zanyext, "USUB">;
+defm : Neon_addl_extract_patterns<sub, sext, "SSUB">;
+
 // CodeGen patterns for addhn and subhn instructions, which can actually be
 // written in LLVM IR without too much difficulty.
 
 // Prioritize ADDHN and SUBHN over UZP2.
 let AddedComplexity = 10 in {
 
 // ADDHN
 def : Pat<(v8i8 (trunc (v8i16 (AArch64vlshr (add V128:$Rn, V128:$Rm), (i32 8))))),

llvm/test/CodeGen/AArch64/aarch64-load-ext.ll

; CHECK-BE-NEXT:    ret
   ret <4 x i16> %y
 }
 
 define <4 x i16> @anyext_v4i16(ptr %a, ptr %b) {
 ; CHECK-LE-LABEL: anyext_v4i16:
 ; CHECK-LE:       // %bb.0:
 ; CHECK-LE-NEXT:    ldr s0, [x0]
 ; CHECK-LE-NEXT:    ldr s1, [x1]
-; CHECK-LE-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-LE-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-LE-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-LE-NEXT:    uaddl v0.8h, v0.8b, v1.8b
 ; CHECK-LE-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-LE-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-LE-NEXT:    ret
 ;
 ; CHECK-BE-LABEL: anyext_v4i16:
 ; CHECK-BE:       // %bb.0:
 ; CHECK-BE-NEXT:    ldr s0, [x0]
 ; CHECK-BE-NEXT:    ldr s1, [x1]
 ; CHECK-BE-NEXT:    rev32 v0.8b, v0.8b
 ; CHECK-BE-NEXT:    rev32 v1.8b, v1.8b
-; CHECK-BE-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-BE-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-BE-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-BE-NEXT:    uaddl v0.8h, v0.8b, v1.8b
 ; CHECK-BE-NEXT:    shl v0.4h, v0.4h, #8
 ; CHECK-BE-NEXT:    sshr v0.4h, v0.4h, #8
 ; CHECK-BE-NEXT:    rev64 v0.4h, v0.4h
 ; CHECK-BE-NEXT:    ret
   %x = load <4 x i8>, ptr %a, align 4
   %y = load <4 x i8>, ptr %b, align 4
   %z = add <4 x i8> %x, %y
   %s = sext <4 x i8> %z to <4 x i16>
   ret <4 x i16> %s
 }
 
 define <4 x i32> @anyext_v4i32(ptr %a, ptr %b) {
 ; CHECK-LE-LABEL: anyext_v4i32:
 ; CHECK-LE:       // %bb.0:
 ; CHECK-LE-NEXT:    ldr s0, [x0]
 ; CHECK-LE-NEXT:    ldr s1, [x1]
-; CHECK-LE-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-LE-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-LE-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-LE-NEXT:    uaddl v0.8h, v0.8b, v1.8b
 ; CHECK-LE-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-LE-NEXT:    shl v0.4s, v0.4s, #24
 ; CHECK-LE-NEXT:    sshr v0.4s, v0.4s, #24
 ; CHECK-LE-NEXT:    ret
 ;
 ; CHECK-BE-LABEL: anyext_v4i32:
 ; CHECK-BE:       // %bb.0:
 ; CHECK-BE-NEXT:    ldr s0, [x0]
 ; CHECK-BE-NEXT:    ldr s1, [x1]
 ; CHECK-BE-NEXT:    rev32 v0.8b, v0.8b
 ; CHECK-BE-NEXT:    rev32 v1.8b, v1.8b
-; CHECK-BE-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-BE-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-BE-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-BE-NEXT:    uaddl v0.8h, v0.8b, v1.8b
 ; CHECK-BE-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-BE-NEXT:    shl v0.4s, v0.4s, #24
 ; CHECK-BE-NEXT:    sshr v0.4s, v0.4s, #24
 ; CHECK-BE-NEXT:    rev64 v0.4s, v0.4s
 ; CHECK-BE-NEXT:    ext v0.16b, v0.16b, v0.16b, #8
 ; CHECK-BE-NEXT:    ret
   %x = load <4 x i8>, ptr %a, align 4
   %y = load <4 x i8>, ptr %b, align 4

llvm/test/CodeGen/AArch64/arm64-ld1.ll

; CHECK-NEXT: ret
   ret <2 x float> %tmp2
 }
 
 
 ; Add rdar://13098923 test case: vld1_dup_u32 doesn't generate ld1r.2s
 define void @ld1r_2s_from_dup(ptr nocapture %a, ptr nocapture %b, ptr nocapture %diff) nounwind ssp {
 entry:
 ; CHECK: ld1r_2s_from_dup
-; CHECK: ld1r.2s { [[ARG1:v[0-9]+]] }, [x0]
-; CHECK-NEXT: ld1r.2s { [[ARG2:v[0-9]+]] }, [x1]
-; CHECK-NEXT: ushll.8h [[ARG1]], [[ARG1]], #0
-; CHECK-NEXT: ushll.8h [[ARG2]], [[ARG2]], #0
-; CHECK-NEXT: sub.4h v[[RESREGNUM:[0-9]+]], [[ARG1]], [[ARG2]]
+; CHECK: ld1r.2s { [[ARG2:v[0-9]+]] }, [x1]
+; CHECK-NEXT: ld1r.2s { [[ARG1:v[0-9]+]] }, [x0]
+; CHECK-NEXT: usubl.8h v[[RESREGNUM:[0-9]+]], [[ARG1]], [[ARG2]]
 ; CHECK-NEXT: str d[[RESREGNUM]], [x2]
 ; CHECK-NEXT: ret
   %tmp1 = load i32, ptr %a, align 4
   %tmp2 = insertelement <2 x i32> undef, i32 %tmp1, i32 0
   %lane = shufflevector <2 x i32> %tmp2, <2 x i32> undef, <2 x i32> zeroinitializer
   %tmp3 = bitcast <2 x i32> %lane to <8 x i8>
   %tmp5 = load i32, ptr %b, align 4
   %tmp6 = insertelement <2 x i32> undef, i32 %tmp5, i32 0

llvm/test/CodeGen/AArch64/extbinopload.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s
 
 define <4 x i16> @normal_load_v4i8(ptr %p) {
 ; CHECK-LABEL: normal_load_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp s0, s1, [x0]
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
   %l1 = load <4 x i8>, ptr %p
   %q = getelementptr i8, ptr %p, i32 4
   %l2 = load <4 x i8>, ptr %q
   %e1 = zext <4 x i8> %l1 to <4 x i16>
   %e2 = zext <4 x i8> %l2 to <4 x i16>
   %a = add <4 x i16> %e1, %e2
   ret <4 x i16> %a
; CHECK-NEXT:    ret
   ret <4 x i32> %a
 }
 
 define <4 x i16> @load_v4i8(ptr %p) {
 ; CHECK-LABEL: load_v4i8:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ldp s1, s0, [x0]
 ; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
 ; CHECK-NEXT:    shl v0.4h, v0.4h, #3
-; CHECK-NEXT:    add v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    uaddw v0.8h, v0.8h, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
   %l1 = load <4 x i8>, ptr %p
   %q = getelementptr i8, ptr %p, i32 4
   %l2 = load <4 x i8>, ptr %q
   %e1 = zext <4 x i8> %l1 to <4 x i16>
   %e2 = zext <4 x i8> %l2 to <4 x i16>
   %e3 = shl <4 x i16> %e2, <i16 3, i16 3, i16 3, i16 3>
   %a = add <4 x i16> %e1, %e3

llvm/test/CodeGen/AArch64/neon-extadd-extract.ll

 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
 ; RUN: llc < %s -mtriple aarch64-none-eabi -o - | FileCheck %s
 
 define <4 x i16> @addls_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
 ; CHECK-LABEL: addls_v8i8_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    saddl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = sext <8 x i8> %s0 to <8 x i16>
   %s1s = sext <8 x i8> %s1 to <8 x i16>
   %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %m = add <4 x i16> %t0, %t1
   ret <4 x i16> %m
 }
 
 define <4 x i16> @addws_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
 ; CHECK-LABEL: addws_v8i8_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    saddw v0.8h, v0.8h, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = sext <8 x i8> %s1 to <8 x i16>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %m = add <4 x i16> %s0, %t1
   ret <4 x i16> %m
 }
 
 define <4 x i16> @addlu_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
 ; CHECK-LABEL: addlu_v8i8_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = zext <8 x i8> %s0 to <8 x i16>
   %s1s = zext <8 x i8> %s1 to <8 x i16>
   %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %m = add <4 x i16> %t0, %t1
   ret <4 x i16> %m
 }
 
 define <4 x i16> @addwu_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
 ; CHECK-LABEL: addwu_v8i8_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    add v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    uaddw v0.8h, v0.8h, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = zext <8 x i8> %s1 to <8 x i16>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %m = add <4 x i16> %s0, %t1
   ret <4 x i16> %m
 }
 
 define <4 x i16> @subls_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
 ; CHECK-LABEL: subls_v8i8_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    ssubl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = sext <8 x i8> %s0 to <8 x i16>
   %s1s = sext <8 x i8> %s1 to <8 x i16>
   %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %m = sub <4 x i16> %t0, %t1
   ret <4 x i16> %m
 }
 
 define <4 x i16> @subws_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
 ; CHECK-LABEL: subws_v8i8_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v1.8h, v1.8b, #0
-; CHECK-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ssubw v0.8h, v0.8h, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = sext <8 x i8> %s1 to <8 x i16>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %m = sub <4 x i16> %s0, %t1
   ret <4 x i16> %m
 }
 
 define <4 x i16> @sublu_v8i8_0(<8 x i8> %s0, <8 x i8> %s1) {
 ; CHECK-LABEL: sublu_v8i8_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    usubl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = zext <8 x i8> %s0 to <8 x i16>
   %s1s = zext <8 x i8> %s1 to <8 x i16>
   %t0 = shufflevector <8 x i16> %s0s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %m = sub <4 x i16> %t0, %t1
   ret <4 x i16> %m
 }
 
 define <4 x i16> @subwu_v8i8_0(<4 x i16> %s0, <8 x i8> %s1) {
 ; CHECK-LABEL: subwu_v8i8_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    sub v0.4h, v0.4h, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    usubw v0.8h, v0.8h, v1.8b
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = zext <8 x i8> %s1 to <8 x i16>
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %m = sub <4 x i16> %s0, %t1
   ret <4 x i16> %m
 }
 
entry:
   %t1 = shufflevector <8 x i16> %s1s, <8 x i16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
   %m = mul <4 x i16> %s0, %t1
   ret <4 x i16> %m
 }
 
 define <2 x i32> @addls_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
 ; CHECK-LABEL: addls_v4i16_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    saddl v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = sext <4 x i16> %s0 to <4 x i32>
   %s1s = sext <4 x i16> %s1 to <4 x i32>
   %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %m = add <2 x i32> %t0, %t1
   ret <2 x i32> %m
 }
 
 define <2 x i32> @addws_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
 ; CHECK-LABEL: addws_v4i16_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    saddw v0.4s, v0.4s, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = sext <4 x i16> %s1 to <4 x i32>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %m = add <2 x i32> %s0, %t1
   ret <2 x i32> %m
 }
 
 define <2 x i32> @addlu_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
 ; CHECK-LABEL: addlu_v4i16_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    uaddl v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = zext <4 x i16> %s0 to <4 x i32>
   %s1s = zext <4 x i16> %s1 to <4 x i32>
   %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %m = add <2 x i32> %t0, %t1
   ret <2 x i32> %m
 }
 
 define <2 x i32> @addwu_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
 ; CHECK-LABEL: addwu_v4i16_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-NEXT:    add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    uaddw v0.4s, v0.4s, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = zext <4 x i16> %s1 to <4 x i32>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %m = add <2 x i32> %s0, %t1
   ret <2 x i32> %m
 }
 
 define <2 x i32> @subls_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
 ; CHECK-LABEL: subls_v4i16_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v0.4s, v0.4h, #0
-; CHECK-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    ssubl v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = sext <4 x i16> %s0 to <4 x i32>
   %s1s = sext <4 x i16> %s1 to <4 x i32>
   %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %m = sub <2 x i32> %t0, %t1
   ret <2 x i32> %m
 }
 
 define <2 x i32> @subws_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
 ; CHECK-LABEL: subws_v4i16_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ssubw v0.4s, v0.4s, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = sext <4 x i16> %s1 to <4 x i32>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %m = sub <2 x i32> %s0, %t1
   ret <2 x i32> %m
 }
 
 define <2 x i32> @sublu_v4i16_0(<4 x i16> %s0, <4 x i16> %s1) {
 ; CHECK-LABEL: sublu_v4i16_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
-; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    usubl v0.4s, v0.4h, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = zext <4 x i16> %s0 to <4 x i32>
   %s1s = zext <4 x i16> %s1 to <4 x i32>
   %t0 = shufflevector <4 x i32> %s0s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %m = sub <2 x i32> %t0, %t1
   ret <2 x i32> %m
 }
 
 define <2 x i32> @subwu_v4i16_0(<2 x i32> %s0, <4 x i16> %s1) {
 ; CHECK-LABEL: subwu_v4i16_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-NEXT:    sub v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    usubw v0.4s, v0.4s, v1.4h
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = zext <4 x i16> %s1 to <4 x i32>
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %m = sub <2 x i32> %s0, %t1
   ret <2 x i32> %m
 }
 
entry:
   %t1 = shufflevector <4 x i32> %s1s, <4 x i32> poison, <2 x i32> <i32 0, i32 1>
   %m = mul <2 x i32> %s0, %t1
   ret <2 x i32> %m
 }
 
 define <1 x i64> @addls_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
 ; CHECK-LABEL: addls_v2i32_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-NEXT:    add d0, d0, d1
+; CHECK-NEXT:    saddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = sext <2 x i32> %s0 to <2 x i64>
   %s1s = sext <2 x i32> %s1 to <2 x i64>
   %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
   %m = add <1 x i64> %t0, %t1
   ret <1 x i64> %m
 }
 
 define <1 x i64> @addws_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
 ; CHECK-LABEL: addws_v2i32_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-NEXT:    add d0, d0, d1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    saddw v0.2d, v0.2d, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = sext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
   %m = add <1 x i64> %s0, %t1
   ret <1 x i64> %m
 }
 
 define <1 x i64> @addlu_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
 ; CHECK-LABEL: addlu_v2i32_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-NEXT:    add d0, d0, d1
+; CHECK-NEXT:    uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = zext <2 x i32> %s0 to <2 x i64>
   %s1s = zext <2 x i32> %s1 to <2 x i64>
   %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
   %m = add <1 x i64> %t0, %t1
   ret <1 x i64> %m
 }
 
 define <1 x i64> @addwu_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
 ; CHECK-LABEL: addwu_v2i32_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-NEXT:    add d0, d0, d1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    uaddw v0.2d, v0.2d, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = zext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
   %m = add <1 x i64> %s0, %t1
   ret <1 x i64> %m
 }
 
 define <1 x i64> @subls_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
 ; CHECK-LABEL: subls_v2i32_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v0.2d, v0.2s, #0
-; CHECK-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-NEXT:    sub d0, d0, d1
+; CHECK-NEXT:    ssubl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = sext <2 x i32> %s0 to <2 x i64>
   %s1s = sext <2 x i32> %s1 to <2 x i64>
   %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
   %m = sub <1 x i64> %t0, %t1
   ret <1 x i64> %m
 }
 
 define <1 x i64> @subws_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
 ; CHECK-LABEL: subws_v2i32_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    sshll v1.2d, v1.2s, #0
-; CHECK-NEXT:    sub d0, d0, d1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    ssubw v0.2d, v0.2d, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = sext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
   %m = sub <1 x i64> %s0, %t1
   ret <1 x i64> %m
 }
 
 define <1 x i64> @sublu_v2i32_0(<2 x i32> %s0, <2 x i32> %s1) {
 ; CHECK-LABEL: sublu_v2i32_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v0.2d, v0.2s, #0
-; CHECK-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-NEXT:    sub d0, d0, d1
+; CHECK-NEXT:    usubl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s0s = zext <2 x i32> %s0 to <2 x i64>
   %s1s = zext <2 x i32> %s1 to <2 x i64>
   %t0 = shufflevector <2 x i64> %s0s, <2 x i64> poison, <1 x i32> <i32 0>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
   %m = sub <1 x i64> %t0, %t1
   ret <1 x i64> %m
 }
 
 define <1 x i64> @subwu_v2i32_0(<1 x i64> %s0, <2 x i32> %s1) {
 ; CHECK-LABEL: subwu_v2i32_0:
 ; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    ushll v1.2d, v1.2s, #0
-; CHECK-NEXT:    sub d0, d0, d1
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    usubw v0.2d, v0.2d, v1.2s
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
 ; CHECK-NEXT:    ret
 entry:
   %s1s = zext <2 x i32> %s1 to <2 x i64>
   %t1 = shufflevector <2 x i64> %s1s, <2 x i64> poison, <1 x i32> <i32 0>
   %m = sub <1 x i64> %s0, %t1
   ret <1 x i64> %m
 }
 

llvm/test/CodeGen/AArch64/neon-extadd.ll

 ; CHECK-NEXT:    ld1 { v2.b }[6], [x9]
 ; CHECK-NEXT:    add x9, sp, #48
 ; CHECK-NEXT:    ld1 { v5.b }[2], [x10]
 ; CHECK-NEXT:    add x12, sp, #248
 ; CHECK-NEXT:    add x10, sp, #56
 ; CHECK-NEXT:    ld1 { v3.b }[6], [x9]
 ; CHECK-NEXT:    add x9, sp, #88
 ; CHECK-NEXT:    mov v1.b[6], w6
-; CHECK-NEXT:    ld1 { v4.b }[3], [x12]
 ; CHECK-NEXT:    ld1 { v2.b }[7], [x11]
+; CHECK-NEXT:    ld1 { v4.b }[3], [x12]
 ; CHECK-NEXT:    ld1 { v5.b }[3], [x9]
 ; CHECK-NEXT:    ld1 { v3.b }[7], [x10]
 ; CHECK-NEXT:    mov v1.b[7], w7
-; CHECK-NEXT:    ushll v4.8h, v4.8b, #0
-; CHECK-NEXT:    ushll v5.8h, v5.8b, #0
+; CHECK-NEXT:    uaddl v4.8h, v5.8b, v4.8b
 ; CHECK-NEXT:    uaddl v2.8h, v3.8b, v2.8b
-; CHECK-NEXT:    add v3.4h, v5.4h, v4.4h
 ; CHECK-NEXT:    uaddl v0.8h, v1.8b, v0.8b
-; CHECK-NEXT:    ushll v1.4s, v3.4h, #0
+; CHECK-NEXT:    ushll v1.4s, v4.4h, #0
 ; CHECK-NEXT:    ushll2 v3.4s, v2.8h, #0
 ; CHECK-NEXT:    ushll v2.4s, v2.4h, #0
 ; CHECK-NEXT:    stp q3, q1, [x8, #48]
 ; CHECK-NEXT:    ushll2 v1.4s, v0.8h, #0
 ; CHECK-NEXT:    ushll v0.4s, v0.4h, #0
 ; CHECK-NEXT:    stp q1, q2, [x8, #16]
 ; CHECK-NEXT:    str q0, [x8]
 ; CHECK-NEXT:    ret

llvm/test/CodeGen/AArch64/uadd_sat_vec.ll

; CHECK-NEXT:    ret
   %z = call <8 x i8> @llvm.uadd.sat.v8i8(<8 x i8> %x, <8 x i8> %y)
   store <8 x i8> %z, ptr %pz
   ret void
 }
 
 define void @v4i8(ptr %px, ptr %py, ptr %pz) nounwind {
 ; CHECK-LABEL: v4i8:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr s1, [x0]
-; CHECK-NEXT:    movi d0, #0xff00ff00ff00ff
-; CHECK-NEXT:    ldr s2, [x1]
-; CHECK-NEXT:    ushll v1.8h, v1.8b, #0
-; CHECK-NEXT:    ushll v2.8h, v2.8b, #0
-; CHECK-NEXT:    add v1.4h, v1.4h, v2.4h
-; CHECK-NEXT:    umin v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    movi d2, #0xff00ff00ff00ff
+; CHECK-NEXT:    ldr s1, [x1]
+; CHECK-NEXT:    uaddl v0.8h, v0.8b, v1.8b
+; CHECK-NEXT:    umin v0.4h, v0.4h, v2.4h
 ; CHECK-NEXT:    xtn v0.8b, v0.8h
 ; CHECK-NEXT:    str s0, [x2]
 ; CHECK-NEXT:    ret
   %x = load <4 x i8>, ptr %px
   %y = load <4 x i8>, ptr %py
   %z = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %x, <4 x i8> %y)
   store <4 x i8> %z, ptr %pz
   ret void