[GlobalISel][AArch64] Add generic legalization rule for extends

[paquette]

Summary

This adds a legalization rule for G_ZEXT, G_ANYEXT, and G_SEXT which allows extends whenever the types will fit in registers (or the source is an s1).

Update tests. Add GISel checks throughout all of arm64-vabs.ll, where we now select a good portion of the code. Add GISel checks to arm64-subvector-extend.ll, which has a good number of vector extends in it.

llvm/trunk/lib/Target/AArch64/AArch64LegalizerInfo.cpp

AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
   getActionDefinitionsBuilder(G_FCMP)
       .legalFor({{s32, s32}, {s32, s64}})
       .clampScalar(0, s32, s32)
       .clampScalar(1, s32, s64)
       .widenScalarToNextPow2(1);
 
   // Extensions
   getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
-      .legalForCartesianProduct({s8, s16, s32, s64}, {s1, s8, s16, s32})
-      .legalFor({v8s16, v8s8});
+      .legalIf([=](const LegalityQuery &Query) {
+        unsigned DstSize = Query.Types[0].getSizeInBits();
+
+        // Make sure that we have something that will fit in a register, and
+        // make sure it's a power of 2.
+        if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize))
+          return false;
+
+        const LLT &SrcTy = Query.Types[1];
+
+        // Special case for s1.
+        if (SrcTy == s1)
+          return true;
+
+        // Make sure we fit in a register otherwise. Don't bother checking that
+        // the source type is below 128 bits. We shouldn't be allowing anything
+        // through which is wider than the destination in the first place.
+        unsigned SrcSize = SrcTy.getSizeInBits();
+        if (SrcSize < 8 || !isPowerOf2_32(SrcSize))
+          return false;
+
+        return true;
+      });
 
   getActionDefinitionsBuilder(G_TRUNC).alwaysLegal();
 
   // FP conversions
   getActionDefinitionsBuilder(G_FPTRUNC).legalFor(
       {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}});
   getActionDefinitionsBuilder(G_FPEXT).legalFor(
       {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}});

llvm/trunk/test/CodeGen/AArch64/GlobalISel/legalize-ext.mir

bb.0:
     ; CHECK: [[COPY:%[0-9]+]]:fpr(<8 x s8>) = COPY $d0
     ; CHECK: [[ANYEXT:%[0-9]+]]:_(<8 x s16>) = G_ANYEXT [[COPY]](<8 x s8>)
     ; CHECK: $q0 = COPY [[ANYEXT]](<8 x s16>)
     ; CHECK: RET_ReallyLR implicit $q0
     %1:fpr(<8 x s8>) = COPY $d0
     %2:_(<8 x s16>) = G_ANYEXT %1(<8 x s8>)
     $q0 = COPY %2(<8 x s16>)
     RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_zext_v4s32_from_v4s16
+alignment:       2
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_zext_v4s32_from_v4s16
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(<4 x s32>) = G_ZEXT [[COPY]](<4 x s16>)
+    ; CHECK: $q0 = COPY [[ZEXT]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s32>) = G_ZEXT %0(<4 x s16>)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_sext_v4s32_from_v4s16
+alignment:       2
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_sext_v4s32_from_v4s16
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+    ; CHECK: [[SEXT:%[0-9]+]]:_(<4 x s32>) = G_SEXT [[COPY]](<4 x s16>)
+    ; CHECK: $q0 = COPY [[SEXT]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s32>) = G_SEXT %0(<4 x s16>)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_anyext_v4s32_from_v4s16
+alignment:       2
+tracksRegLiveness: true
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_anyext_v4s32_from_v4s16
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $d0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<4 x s32>) = G_ANYEXT [[COPY]](<4 x s16>)
+    ; CHECK: $q0 = COPY [[ANYEXT]](<4 x s32>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<4 x s16>) = COPY $d0
+    %1:_(<4 x s32>) = G_ANYEXT %0(<4 x s16>)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_zext_v2s64_from_v2s32
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_zext_v2s64_from_v2s32
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(<2 x s64>) = G_ZEXT [[COPY]](<2 x s32>)
+    ; CHECK: $q0 = COPY [[ZEXT]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s64>) = G_ZEXT %0(<2 x s32>)
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_sext_v2s64_from_v2s32
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_sext_v2s64_from_v2s32
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[SEXT:%[0-9]+]]:_(<2 x s64>) = G_SEXT [[COPY]](<2 x s32>)
+    ; CHECK: $q0 = COPY [[SEXT]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s64>) = G_SEXT %0(<2 x s32>)
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...
+---
+name:            test_anyext_v2s64_from_v2s32
+alignment:       2
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: test_anyext_v2s64_from_v2s32
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $d0
+    ; CHECK: [[ANYEXT:%[0-9]+]]:_(<2 x s64>) = G_ANYEXT [[COPY]](<2 x s32>)
+    ; CHECK: $q0 = COPY [[ANYEXT]](<2 x s64>)
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:_(<2 x s32>) = COPY $d0
+    %1:_(<2 x s64>) = G_ANYEXT %0(<2 x s32>)
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+
+...

llvm/trunk/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir

 #
 # DEBUG-NEXT: G_INTRINSIC (opcode {{[0-9]+}}): 0 type indices
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
 # DEBUG-NEXT: G_INTRINSIC_W_SIDE_EFFECTS (opcode {{[0-9]+}}): 0 type indices
 # DEBUG:      .. type index coverage check SKIPPED: no rules defined
 #
 # DEBUG-NEXT: G_ANYEXT (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_TRUNC (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_CONSTANT (opcode {{[0-9]+}}): 1 type index
 # DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_FCONSTANT (opcode {{[0-9]+}}): 1 type index
 # DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_VASTART (opcode {{[0-9]+}}): 1 type index
 # DEBUG:      .. the first uncovered type index: 1, OK
 #
 # DEBUG-NEXT: G_VAARG (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:      .. the first uncovered type index: 2, OK
 #
 # DEBUG-NEXT: G_SEXT (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ZEXT (opcode {{[0-9]+}}): 2 type indices
-# DEBUG:      .. the first uncovered type index: 2, OK
+# DEBUG:      .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_SHL (opcode {{[0-9]+}}): 2 type indices
 # DEBUG:.. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_LSHR (opcode {{[0-9]+}}): 2 type indices
 # DEBUG: .. type index coverage check SKIPPED: user-defined predicate detected
 #
 # DEBUG-NEXT: G_ASHR (opcode {{[0-9]+}}): 2 type indices

llvm/trunk/test/CodeGen/AArch64/GlobalISel/select-int-ext.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=aarch64-- -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s
 
 --- |
   target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
 
   define void @anyext_s64_from_s32() { ret void }
   define void @anyext_s32_from_s8() { ret void }
   define void @anyext_v8s16_from_v8s8() { ret void }
+  define void @anyext_v4s32_from_v4s16() { ret void }
+  define void @anyext_v2s64_from_v2s32() { ret void }
 
   define void @zext_s64_from_s32() { ret void }
   define void @zext_s32_from_s16() { ret void }
   define void @zext_s32_from_s8() { ret void }
   define void @zext_s16_from_s8() { ret void }
   define void @zext_v8s16_from_v8s8() { ret void }
+  define void @zext_v4s32_from_v4s16() { ret void }
+  define void @zext_v2s64_from_v2s32() { ret void }
 
   define void @sext_s64_from_s32() { ret void }
   define void @sext_s32_from_s16() { ret void }
   define void @sext_s32_from_s8() { ret void }
   define void @sext_s16_from_s8() { ret void }
   define void @sext_v8s16_from_v8s8() { ret void }
+  define void @sext_v4s32_from_v4s16() { ret void }
+  define void @sext_v2s64_from_v2s32() { ret void }
 ...
 
 ---
 name:            anyext_s64_from_s32
 legalized:       true
 regBankSelected: true
 
 registers:
bb.0:
     ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
     ; CHECK: [[USHLLv8i8_shift:%[0-9]+]]:fpr128 = USHLLv8i8_shift [[COPY]], 0
     ; CHECK: $q0 = COPY [[USHLLv8i8_shift]]
     ; CHECK: RET_ReallyLR implicit $q0
     %0:fpr(<8 x s8>) = COPY $d0
     %1:fpr(<8 x s16>) = G_ANYEXT %0(<8 x s8>)
     $q0 = COPY %1(<8 x s16>)
     RET_ReallyLR implicit $q0
+...
+
+---
+name:            anyext_v4s32_from_v4s16
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: anyext_v4s32_from_v4s16
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
+    ; CHECK: [[USHLLv4i16_shift:%[0-9]+]]:fpr128 = USHLLv4i16_shift [[COPY]], 0
+    ; CHECK: $q0 = COPY [[USHLLv4i16_shift]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<4 x s16>) = COPY $d0
+    %1:fpr(<4 x s32>) = G_ANYEXT %0(<4 x s16>)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+...
+
+---
+name:            anyext_v2s64_from_v2s32
+alignment:       2
+tracksRegLiveness: true
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: anyext_v2s64_from_v2s32
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
+    ; CHECK: [[USHLLv2i32_shift:%[0-9]+]]:fpr128 = USHLLv2i32_shift [[COPY]], 0
+    ; CHECK: $q0 = COPY [[USHLLv2i32_shift]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<2 x s32>) = COPY $d0
+    %1:fpr(<2 x s64>) = G_ANYEXT %0(<2 x s32>)
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+...
 
 ---
 name:            zext_s64_from_s32
 legalized:       true
 regBankSelected: true
 
 registers:
   - { id: 0, class: gpr }
bb.0:
     %0:fpr(<8 x s8>) = COPY $d0
     %1:fpr(<8 x s16>) = G_ZEXT %0(<8 x s8>)
     $q0 = COPY %1(<8 x s16>)
     RET_ReallyLR implicit $q0
 
 ...
 
 ---
+name:            zext_v4s32_from_v4s16
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: zext_v4s32_from_v4s16
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
+    ; CHECK: [[USHLLv4i16_shift:%[0-9]+]]:fpr128 = USHLLv4i16_shift [[COPY]], 0
+    ; CHECK: $q0 = COPY [[USHLLv4i16_shift]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<4 x s16>) = COPY $d0
+    %1:fpr(<4 x s32>) = G_ZEXT %0(<4 x s16>)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+...
+
+---
+name:            zext_v2s64_from_v2s32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: zext_v2s64_from_v2s32
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
+    ; CHECK: [[USHLLv2i32_shift:%[0-9]+]]:fpr128 = USHLLv2i32_shift [[COPY]], 0
+    ; CHECK: $q0 = COPY [[USHLLv2i32_shift]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<2 x s32>) = COPY $d0
+    %1:fpr(<2 x s64>) = G_ZEXT %0(<2 x s32>)
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0
+...
+
+---
 name:            sext_s64_from_s32
 legalized:       true
 regBankSelected: true
 
 registers:
   - { id: 0, class: gpr }
   - { id: 1, class: gpr }
 
bb.0:
     ; CHECK: $q0 = COPY [[SSHLLv8i8_shift]]
     ; CHECK: RET_ReallyLR implicit $q0
     %0:fpr(<8 x s8>) = COPY $d0
     %1:fpr(<8 x s16>) = G_SEXT %0(<8 x s8>)
     $q0 = COPY %1(<8 x s16>)
     RET_ReallyLR implicit $q0
 
 ...
+
+---
+name:            sext_v4s32_from_v4s16
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: sext_v4s32_from_v4s16
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
+    ; CHECK: [[SSHLLv4i16_shift:%[0-9]+]]:fpr128 = SSHLLv4i16_shift [[COPY]], 0
+    ; CHECK: $q0 = COPY [[SSHLLv4i16_shift]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<4 x s16>) = COPY $d0
+    %1:fpr(<4 x s32>) = G_SEXT %0(<4 x s16>)
+    $q0 = COPY %1(<4 x s32>)
+    RET_ReallyLR implicit $q0
+...
+
+---
+name:            sext_v2s64_from_v2s32
+alignment:       2
+legalized:       true
+regBankSelected: true
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr }
+  - { id: 1, class: fpr }
+machineFunctionInfo: {}
+body:             |
+  bb.0:
+    liveins: $d0
+
+    ; CHECK-LABEL: name: sext_v2s64_from_v2s32
+    ; CHECK: liveins: $d0
+    ; CHECK: [[COPY:%[0-9]+]]:fpr64 = COPY $d0
+    ; CHECK: [[SSHLLv2i32_shift:%[0-9]+]]:fpr128 = SSHLLv2i32_shift [[COPY]], 0
+    ; CHECK: $q0 = COPY [[SSHLLv2i32_shift]]
+    ; CHECK: RET_ReallyLR implicit $q0
+    %0:fpr(<2 x s32>) = COPY $d0
+    %1:fpr(<2 x s64>) = G_SEXT %0(<2 x s32>)
+    $q0 = COPY %1(<2 x s64>)
+    RET_ReallyLR implicit $q0

llvm/trunk/test/CodeGen/AArch64/arm64-subvector-extend.ll

 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* 2>&1 | FileCheck %s --check-prefixes=CHECK,FALLBACK
 
 ; Test efficient codegen of vector extends up from legal type to 128 bit
 ; and 256 bit vector types.
 
 ;-----
 ; Vectors of i16.
 ;-----
+
+; FALLBACK-NOT: remark:{{.*}}(in function: func1)
 define <8 x i16> @func1(<8 x i8> %v0) nounwind {
 ; CHECK-LABEL: func1:
 ; CHECK-NEXT: ushll.8h  v0, v0, #0
 ; CHECK-NEXT: ret
   %r = zext <8 x i8> %v0 to <8 x i16>
   ret <8 x i16> %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}(in function: func2)
 define <8 x i16> @func2(<8 x i8> %v0) nounwind {
 ; CHECK-LABEL: func2:
 ; CHECK-NEXT: sshll.8h  v0, v0, #0
 ; CHECK-NEXT: ret
   %r = sext <8 x i8> %v0 to <8 x i16>
   ret <8 x i16> %r
 }
 
; CHECK-NEXT: ret
   %r = sext <16 x i8> %v0 to <16 x i16>
   ret <16 x i16> %r
 }
 
 ;-----
 ; Vectors of i32.
 ;-----
 
+; FALLBACK-NOT: remark:{{.*}}(in function: afunc1)
 define <4 x i32> @afunc1(<4 x i16> %v0) nounwind {
 ; CHECK-LABEL: afunc1:
 ; CHECK-NEXT: ushll.4s v0, v0, #0
 ; CHECK-NEXT: ret
   %r = zext <4 x i16> %v0 to <4 x i32>
   ret <4 x i32> %r
 }
 
+; FALLBACK-NOT: remark:{{.*}}(in function: afunc2)
 define <4 x i32> @afunc2(<4 x i16> %v0) nounwind {
 ; CHECK-LABEL: afunc2:
 ; CHECK-NEXT: sshll.4s v0, v0, #0
 ; CHECK-NEXT: ret
   %r = sext <4 x i16> %v0 to <4 x i32>
   ret <4 x i32> %r
 }
 

llvm/trunk/test/CodeGen/AArch64/arm64-vabs.ll

 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s
-; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=arm64-eabi -aarch64-neon-syntax=apple 2>&1 | FileCheck %s --check-prefixes=GISEL,FALLBACK
+; RUN: llc < %s -global-isel -global-isel-abort=2 -pass-remarks-missed=gisel* -mtriple=arm64-eabi -aarch64-neon-syntax=apple 2>&1 | FileCheck %s --check-prefixes=FALLBACK,CHECK
 
-; FALLBACK-NOT: remark:{{.*}}(<8 x s16>) = G_ZEXT %4:_(<8 x s8>)
+; FALLBACK-NOT: remark:{{.*}} G_ZEXT
 ; FALLBACK-NOT: remark:{{.*}} sabdl8h
 define <8 x i16> @sabdl8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ;CHECK-LABEL: sabdl8h:
 ;CHECK: sabdl.8h
-;GISEL-LABEL: sabdl8h:
-;GISEL: sabdl.8h
         %tmp1 = load <8 x i8>, <8 x i8>* %A
         %tmp2 = load <8 x i8>, <8 x i8>* %B
         %tmp3 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
         %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
         ret <8 x i16> %tmp4
 }
 
+; FALLBACK-NOT: remark:{{.*}} sabdl4s
 define <4 x i32> @sabdl4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 ;CHECK-LABEL: sabdl4s:
 ;CHECK: sabdl.4s
         %tmp1 = load <4 x i16>, <4 x i16>* %A
         %tmp2 = load <4 x i16>, <4 x i16>* %B
         %tmp3 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
         %tmp4 = zext <4 x i16> %tmp3 to <4 x i32>
         ret <4 x i32> %tmp4
 }
 
+; FALLBACK-NOT: remark:{{.*}} sabdl2d
 define <2 x i64> @sabdl2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 ;CHECK-LABEL: sabdl2d:
 ;CHECK: sabdl.2d
         %tmp1 = load <2 x i32>, <2 x i32>* %A
         %tmp2 = load <2 x i32>, <2 x i32>* %B
         %tmp3 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
         %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
         ret <2 x i64> %tmp4
;CHECK: sabdl.2d
         %load2 = load <4 x i32>, <4 x i32>* %B
         %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
         %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
         %tmp3 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
         %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
         ret <2 x i64> %tmp4
 }
 
+; FALLBACK-NOT: remark:{{.*}} uabdl8h)
 define <8 x i16> @uabdl8h(<8 x i8>* %A, <8 x i8>* %B) nounwind {
 ;CHECK-LABEL: uabdl8h:
 ;CHECK: uabdl.8h
   %tmp1 = load <8 x i8>, <8 x i8>* %A
   %tmp2 = load <8 x i8>, <8 x i8>* %B
   %tmp3 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
   %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
+; FALLBACK-NOT: remark:{{.*}} uabdl4s)
 define <4 x i32> @uabdl4s(<4 x i16>* %A, <4 x i16>* %B) nounwind {
 ;CHECK-LABEL: uabdl4s:
 ;CHECK: uabdl.4s
   %tmp1 = load <4 x i16>, <4 x i16>* %A
   %tmp2 = load <4 x i16>, <4 x i16>* %B
   %tmp3 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
   %tmp4 = zext <4 x i16> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
+; FALLBACK-NOT: remark:{{.*}} uabdl2d)
 define <2 x i64> @uabdl2d(<2 x i32>* %A, <2 x i32>* %B) nounwind {
 ;CHECK-LABEL: uabdl2d:
 ;CHECK: uabdl.2d
   %tmp1 = load <2 x i32>, <2 x i32>* %A
   %tmp2 = load <2 x i32>, <2 x i32>* %B
   %tmp3 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
   %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4

 declare <16 x i8> @llvm.aarch64.neon.abs.v16i8(<16 x i8>) nounwind readnone
 declare <4 x i16> @llvm.aarch64.neon.abs.v4i16(<4 x i16>) nounwind readnone
 declare <8 x i16> @llvm.aarch64.neon.abs.v8i16(<8 x i16>) nounwind readnone
 declare <2 x i32> @llvm.aarch64.neon.abs.v2i32(<2 x i32>) nounwind readnone
 declare <4 x i32> @llvm.aarch64.neon.abs.v4i32(<4 x i32>) nounwind readnone
 declare <1 x i64> @llvm.aarch64.neon.abs.v1i64(<1 x i64>) nounwind readnone
 declare i64 @llvm.aarch64.neon.abs.i64(i64) nounwind readnone
 
+; FALLBACK-NOT: remark:{{.*}} sabal8h
 define <8 x i16> @sabal8h(<8 x i8>* %A, <8 x i8>* %B,  <8 x i16>* %C) nounwind {
 ;CHECK-LABEL: sabal8h:
 ;CHECK: sabal.8h
         %tmp1 = load <8 x i8>, <8 x i8>* %A
         %tmp2 = load <8 x i8>, <8 x i8>* %B
         %tmp3 = load <8 x i16>, <8 x i16>* %C
         %tmp4 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
         %tmp4.1 = zext <8 x i8> %tmp4 to <8 x i16>
         %tmp5 = add <8 x i16> %tmp3, %tmp4.1
         ret <8 x i16> %tmp5
 }
 
+; FALLBACK-NOT: remark:{{.*}} sabal4s
 define <4 x i32> @sabal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
 ;CHECK-LABEL: sabal4s:
 ;CHECK: sabal.4s
         %tmp1 = load <4 x i16>, <4 x i16>* %A
         %tmp2 = load <4 x i16>, <4 x i16>* %B
         %tmp3 = load <4 x i32>, <4 x i32>* %C
         %tmp4 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
         %tmp4.1 = zext <4 x i16> %tmp4 to <4 x i32>
         %tmp5 = add <4 x i32> %tmp3, %tmp4.1
         ret <4 x i32> %tmp5
 }
 
+; FALLBACK-NOT: remark:{{.*}} sabal2d
 define <2 x i64> @sabal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
 ;CHECK-LABEL: sabal2d:
 ;CHECK: sabal.2d
         %tmp1 = load <2 x i32>, <2 x i32>* %A
         %tmp2 = load <2 x i32>, <2 x i32>* %B
         %tmp3 = load <2 x i64>, <2 x i64>* %C
         %tmp4 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
         %tmp4.1 = zext <2 x i32> %tmp4 to <2 x i64>
;CHECK: sabal.2d
         %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
         %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
         %tmp4 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
         %tmp4.1 = zext <2 x i32> %tmp4 to <2 x i64>
         %tmp5 = add <2 x i64> %tmp3, %tmp4.1
         ret <2 x i64> %tmp5
 }
 
+; FALLBACK-NOT: remark:{{.*}} uabal8h
 define <8 x i16> @uabal8h(<8 x i8>* %A, <8 x i8>* %B,  <8 x i16>* %C) nounwind {
 ;CHECK-LABEL: uabal8h:
 ;CHECK: uabal.8h
         %tmp1 = load <8 x i8>, <8 x i8>* %A
         %tmp2 = load <8 x i8>, <8 x i8>* %B
         %tmp3 = load <8 x i16>, <8 x i16>* %C
         %tmp4 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
         %tmp4.1 = zext <8 x i8> %tmp4 to <8 x i16>
         %tmp5 = add <8 x i16> %tmp3, %tmp4.1
         ret <8 x i16> %tmp5
 }
 
+; FALLBACK-NOT: remark:{{.*}} uabal8s
 define <4 x i32> @uabal4s(<4 x i16>* %A, <4 x i16>* %B, <4 x i32>* %C) nounwind {
 ;CHECK-LABEL: uabal4s:
 ;CHECK: uabal.4s
         %tmp1 = load <4 x i16>, <4 x i16>* %A
         %tmp2 = load <4 x i16>, <4 x i16>* %B
         %tmp3 = load <4 x i32>, <4 x i32>* %C
         %tmp4 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
         %tmp4.1 = zext <4 x i16> %tmp4 to <4 x i32>
         %tmp5 = add <4 x i32> %tmp3, %tmp4.1
         ret <4 x i32> %tmp5
 }
 
+; FALLBACK-NOT: remark:{{.*}} uabal2d
 define <2 x i64> @uabal2d(<2 x i32>* %A, <2 x i32>* %B, <2 x i64>* %C) nounwind {
 ;CHECK-LABEL: uabal2d:
 ;CHECK: uabal.2d
         %tmp1 = load <2 x i32>, <2 x i32>* %A
         %tmp2 = load <2 x i32>, <2 x i32>* %B
         %tmp3 = load <2 x i64>, <2 x i64>* %C
         %tmp4 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
         %tmp4.1 = zext <2 x i32> %tmp4 to <2 x i64>