[AArch64][GlobalISel] Legalize integer across-lane intrinsics with actual type

[dzhidzhoev]

Summary

Across-lane intrinsics with integer destination type (uaddv, saddv,
umaxv, smavx, uminv, sminv) were legalized with the destination type
given in the LLVM IR intrinsic’s definition. It was wider than
the actual destination type of the corresponding machine instruction.
InstructionSelect was implicitly supposed to generate underlying
extension instructions for these intrinsics, while the real destination
type was opaque for other GlobalISel passes. Thus,
llvm/test/CodeGen/AArch64/arm64-vaddv.ll failed on GlobalISel since
the generated code was worse in functions that used the value of
an across-lane intrinsic in following FP&SIMD instructions (functions
with _used_by_laneop suffix).

Here intrinsics are legalized and selected with an actual destination
type, making it transparent to other passes. If the destination
value is used in further instructions accepting FPR registers, there
won’t be extra copies across register banks. i16 type is added to
the list of the types of the FPR16 register bank to make it possible,
and a few SelectionDAG patterns are modified to eliminate ambiguity
in TableGen.

[arsenm]

llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h

328–341

This should be a separate patch from the AArch64 changes

[dzhidzhoev]

Addressed @arsenm comment. Moved artifact combiner change to https://reviews.llvm.org/D156832.

[dzhidzhoev]

Rebased.

[aemerson]

LGTM, thanks.

[dmgreen]

Some of this looks a bit odd, with the use of fp registers in integer patterns. Can you change the references of bf16 to f16 in this patch? I think it makes a better default register class for f16 (but might not make any difference in practice), and some of the patterns are already acting upon f16 types.

[dmgreen]

Also, this might also be introducing illegal COPY instructions. The llvm-test-suite has been failing for a while, but after D157202 should be doing better. This looks like it might cause issues again though, if I am running it correctly?

[dzhidzhoev]

Addressed @dmgreen comments.
Changed references of bf16 to f16.
Removed changes modifying RegBankSelect behaviour related to G_ZEXT. Used GIIgnoreCopies flag in patterns to select sext/zext(uaddv/saddv/...) used by
laneop.

llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h

if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
       LLVM_DEBUG(dbgs() << ".. Combine G_TRUNC(G_TRUNC): " << MI);
 
       Builder.buildTrunc(DstReg, TruncSrc);
       UpdatedDefs.push_back(DstReg);
       markInstAndDefDead(MI, *MRI.getVRegDef(TruncSrc), DeadInsts);
       return true;
     }
 
+    // trunc(sext x) -> x
+    Register SextSrc;
+    if (mi_match(SrcReg, MRI, m_GSExt(m_Reg(SextSrc)))) {
+      LLT DstTy = MRI.getType(DstReg);
+      LLT SextSrcTy = MRI.getType(SextSrc);
+      if (DstTy == SextSrcTy) {
+        LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;);
+
+        replaceRegOrBuildCopy(DstReg, SextSrc, MRI, Builder, UpdatedDefs, Observer);
+        UpdatedDefs.push_back(DstReg);
+        markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);
+        return true;
+      }
+    }
+
     return false;
   }
 
   /// Try to fold G_[ASZ]EXT (G_IMPLICIT_DEF).
   bool tryFoldImplicitDef(MachineInstr &MI,
                           SmallVectorImpl<MachineInstr *> &DeadInsts,
                           SmallVectorImpl<Register> &UpdatedDefs) {
     unsigned Opcode = MI.getOpcode();

llvm/lib/Target/AArch64/AArch64InstrFormats.td

multiclass SIMDFPIndexed<bit U, bits<4> opc, string asm,
   }
   } // mayRaiseFPException = 1, Uses = [FPCR]
 
   let Predicates = [HasNEON, HasFullFP16] in {
   def : Pat<(f16 (OpNode
                    (f16 (vector_extract (v8f16 V128:$Rn), (i64 0))),
                    (f16 (vector_extract (v8f16 V128:$Rm), VectorIndexH:$idx)))),
             (!cast<Instruction>(NAME # v1i16_indexed)
-              (EXTRACT_SUBREG V128:$Rn, hsub), V128:$Rm, VectorIndexH:$idx)>;
+              (f16 (EXTRACT_SUBREG V128:$Rn, hsub)), V128:$Rm, VectorIndexH:$idx)>;
   }
 
   let Predicates = [HasNEON] in {
   def : Pat<(f32 (OpNode
                    (f32 (vector_extract (v4f32 V128:$Rn), (i64 0))),
                    (f32 (vector_extract (v4f32 V128:$Rm), VectorIndexS:$idx)))),
             (!cast<Instruction>(NAME # v1i32_indexed)
               (EXTRACT_SUBREG V128:$Rn, ssub), V128:$Rm, VectorIndexS:$idx)>;
multiclass SIMDIndexedLongSQDMLXSDTied<bit U, bits<4> opc, string asm,
   def : Pat<(i32 (Accum (i32 FPR32Op:$Rd),
                         (i32 (vector_extract
                                     (v4i32 (int_aarch64_neon_sqdmull
                                                 (v4i16 V64:$Rn),
                                                 (v4i16 V64:$Rm))),
                                     (i64 0))))),
             (!cast<Instruction>(NAME # v1i32_indexed)
                         FPR32Op:$Rd,
-                        (EXTRACT_SUBREG V64:$Rn, hsub),
+                        (bf16 (EXTRACT_SUBREG V64:$Rn, hsub)),
                         (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rm, dsub),
                         (i64 0))>;
 
   def : Pat<(i32 (Accum (i32 FPR32Op:$Rd),
                         (i32 (vector_extract
                                     (v4i32 (int_aarch64_neon_sqdmull
                                                 (v4i16 V64:$Rn),
                                                 (dup_v8i16 (v8i16 V128_lo:$Rm),
                                                             VectorIndexH:$idx))),
                                     (i64 0))))),
             (!cast<Instruction>(NAME # v1i32_indexed)
                         FPR32Op:$Rd,
-                        (EXTRACT_SUBREG V64:$Rn, hsub),
+                        (bf16 (EXTRACT_SUBREG V64:$Rn, hsub)),
                         V128_lo:$Rm,
                         VectorIndexH:$idx)>;
 
   def v1i64_indexed : BaseSIMDIndexedTied<1, U, 1, 0b10, opc,
                                       FPR64Op, FPR32Op, V128, VectorIndexS,
                                       asm, ".s", "", "", ".s",
     [(set (i64 FPR64Op:$dst),
           (Accum (i64 FPR64Op:$Rd),

llvm/lib/Target/AArch64/AArch64InstrGISel.td

 }
 
 def : Pat<(int_aarch64_stlxp GPR64:$lo, GPR64:$hi, GPR64:$addr),
           (STLXPX GPR64:$lo, GPR64:$hi, GPR64:$addr)>;
 def : Pat<(int_aarch64_stxp GPR64:$lo, GPR64:$hi, GPR64:$addr),
           (STXPX GPR64:$lo, GPR64:$hi, GPR64:$addr)>;
 
 multiclass SIMDAcrossLanesSignedIntrinsicBHS<string baseOpc, Intrinsic intOp> {
-  def : Pat<(i32 (intOp (v8i8 V64:$Rn))),
+  def : Pat<(i32 (sext (i8 (intOp (v8i8 V64:$Rn))))),
         (i32 (SMOVvi8to32
           (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
-            (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn), bsub),
           (i64 0)))>;
-  def : Pat<(i32 (intOp (v16i8 V128:$Rn))),
+  def : Pat<(i8 (intOp (v8i8 V64:$Rn))),
+        (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn)>;
+
+  def : Pat<(i32 (sext (i8 (intOp (v16i8 V128:$Rn))))),
         (i32 (SMOVvi8to32
           (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
            (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn), bsub),
           (i64 0)))>;
+  def : Pat<(i8 (intOp (v16i8 V128:$Rn))),
+        (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn)>;
 
-  def : Pat<(i32 (intOp (v4i16 V64:$Rn))),
+  def : Pat<(i32 (sext (i16 (intOp (v4i16 V64:$Rn))))),
         (i32 (SMOVvi16to32
           (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
            (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn), hsub),
           (i64 0)))>;
-  def : Pat<(i32 (intOp (v8i16 V128:$Rn))),
+  def : Pat<(i16 (intOp (v4i16 V64:$Rn))),
+        (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn)>;
+
+  def : Pat<(i32 (sext (i16 (intOp (v8i16 V128:$Rn))))),
         (i32 (SMOVvi16to32
           (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
            (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn), hsub),
           (i64 0)))>;
+  def : Pat<(i16 (intOp (v8i16 V128:$Rn))),
+        (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn)>;
 
   def : Pat<(i32 (intOp (v4i32 V128:$Rn))),
         (i32 (EXTRACT_SUBREG
           (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
            (!cast<Instruction>(!strconcat(baseOpc, "v4i32v")) V128:$Rn), ssub),
           ssub))>;
 }
 
 multiclass SIMDAcrossLanesUnsignedIntrinsicBHS<string baseOpc,
                                                 Intrinsic intOp> {
-  def : Pat<(i32 (intOp (v8i8 V64:$Rn))),
+  def : Pat<(i32 (zext (i8 (intOp (v8i8 V64:$Rn))))),
+        (COPY_TO_REGCLASS
           (i32 (EXTRACT_SUBREG
             (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
               (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn), bsub),
-          ssub))>;
-  def : Pat<(i32 (intOp (v16i8 V128:$Rn))),
+            ssub)),
+          GPR32)>;
+  def : Pat<(i8 (intOp (v8i8 V64:$Rn))),
+        (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn)>;
+
+  def : Pat<(i32 (zext (i8 (intOp (v16i8 V128:$Rn))))),
+        (COPY_TO_REGCLASS
           (i32 (EXTRACT_SUBREG
             (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
               (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn), bsub),
-          ssub))>;
+            ssub)),
+        GPR32)>;
+  def : Pat<(i8 (intOp (v16i8 V128:$Rn))),
+        (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn)>;
+
 
-  def : Pat<(i32 (intOp (v4i16 V64:$Rn))),
+  def : Pat<(i32 (zext (i16 (intOp (v4i16 V64:$Rn))))),
+        (COPY_TO_REGCLASS
           (i32 (EXTRACT_SUBREG
             (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
               (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn), hsub),
-            ssub))>;
-  def : Pat<(i32 (intOp (v8i16 V128:$Rn))),
+            ssub)),
+          GPR32)>;
+  def : Pat<(i16 (intOp (v4i16 V64:$Rn))),
+        (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn)>;
+
+  def : Pat<(i32 (zext (i16 (intOp (v8i16 V128:$Rn))))),
+        (COPY_TO_REGCLASS
           (i32 (EXTRACT_SUBREG
             (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
               (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn), hsub),
-          ssub))>;
+            ssub)),
+        GPR32)>;
+  def : Pat<(i16 (intOp (v8i16 V128:$Rn))),
+        (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn)>;
 
   def : Pat<(i32 (intOp (v4i32 V128:$Rn))),
         (i32 (EXTRACT_SUBREG
           (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
             (!cast<Instruction>(!strconcat(baseOpc, "v4i32v")) V128:$Rn), ssub),
           ssub))>;
 }
 
-
 defm : SIMDAcrossLanesSignedIntrinsicBHS<"ADDV", int_aarch64_neon_saddv>;
 // vaddv_[su]32 is special; -> ADDP Vd.2S,Vn.2S,Vm.2S; return Vd.s[0];Vn==Vm
 def : Pat<(i32 (int_aarch64_neon_saddv (v2i32 V64:$Rn))),
           (i32 (EXTRACT_SUBREG
             (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
               (ADDPv2i32 V64:$Rn, V64:$Rn), dsub),
             ssub))>;
 
+def : Pat<(i64 (int_aarch64_neon_saddv (v2i64 V128:$Rn))),
+          (i64 (EXTRACT_SUBREG
+          (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)),
+              (ADDPv2i64p V128:$Rn), dsub),
+            dsub))>;
+
 defm : SIMDAcrossLanesUnsignedIntrinsicBHS<"ADDV", int_aarch64_neon_uaddv>;
 def : Pat<(i32 (int_aarch64_neon_uaddv (v2i32 V64:$Rn))),
           (i32 (EXTRACT_SUBREG
             (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
               (ADDPv2i32 V64:$Rn, V64:$Rn), dsub),
             ssub))>;
+def : Pat<(i64 (int_aarch64_neon_uaddv (v2i64 V128:$Rn))),
+          (i64 (EXTRACT_SUBREG
+          (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)),
+              (ADDPv2i64p V128:$Rn), dsub),
+            dsub))>;
 
 defm : SIMDAcrossLanesSignedIntrinsicBHS<"SMAXV", int_aarch64_neon_smaxv>;
 def : Pat<(i32 (int_aarch64_neon_smaxv (v2i32 V64:$Rn))),
           (i32 (EXTRACT_SUBREG
             (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
               (SMAXPv2i32 V64:$Rn, V64:$Rn), dsub),
             ssub))>;
 

llvm/lib/Target/AArch64/AArch64InstrInfo.td

let Predicates = [IsLE, UseSTRQro] in {
   defm : VecROStorePat<ro128, v8f16, FPR128, STRQroW, STRQroX>;
   defm : VecROStorePat<ro128, v8bf16, FPR128, STRQroW, STRQroX>;
 }
 } // AddedComplexity = 10
 
 // Match stores from lane 0 to the appropriate subreg's store.
 multiclass VecROStoreLane0Pat<ROAddrMode ro, SDPatternOperator storeop,
                               ValueType VecTy, ValueType STy,
+                              ValueType SubRegTy,
                               SubRegIndex SubRegIdx,
                               Instruction STRW, Instruction STRX> {
 
   def : Pat<(storeop (STy (vector_extract (VecTy VecListOne128:$Vt), 0)),
                      (ro.Wpat GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)),
-            (STRW (EXTRACT_SUBREG VecListOne128:$Vt, SubRegIdx),
+            (STRW (SubRegTy (EXTRACT_SUBREG VecListOne128:$Vt, SubRegIdx)),
                   GPR64sp:$Rn, GPR32:$Rm, ro.Wext:$extend)>;
 
   def : Pat<(storeop (STy (vector_extract (VecTy VecListOne128:$Vt), 0)),
                      (ro.Xpat GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)),
-            (STRX (EXTRACT_SUBREG VecListOne128:$Vt, SubRegIdx),
+            (STRX (SubRegTy (EXTRACT_SUBREG VecListOne128:$Vt, SubRegIdx)),
                   GPR64sp:$Rn, GPR64:$Rm, ro.Xext:$extend)>;
 }
 
 let AddedComplexity = 19 in {
-  defm : VecROStoreLane0Pat<ro16, truncstorei16, v8i16, i32, hsub, STRHroW, STRHroX>;
-  defm : VecROStoreLane0Pat<ro16,         store, v8f16, f16, hsub, STRHroW, STRHroX>;
-  defm : VecROStoreLane0Pat<ro32,         store, v4i32, i32, ssub, STRSroW, STRSroX>;
-  defm : VecROStoreLane0Pat<ro32,         store, v4f32, f32, ssub, STRSroW, STRSroX>;
-  defm : VecROStoreLane0Pat<ro64,         store, v2i64, i64, dsub, STRDroW, STRDroX>;
-  defm : VecROStoreLane0Pat<ro64,         store, v2f64, f64, dsub, STRDroW, STRDroX>;
+  defm : VecROStoreLane0Pat<ro16, truncstorei16, v8i16, i32, bf16, hsub, STRHroW, STRHroX>;
+  defm : VecROStoreLane0Pat<ro16,         store, v8f16, f16, bf16, hsub, STRHroW, STRHroX>;
+  defm : VecROStoreLane0Pat<ro32,         store, v4i32, i32, i32, ssub, STRSroW, STRSroX>;
+  defm : VecROStoreLane0Pat<ro32,         store, v4f32, f32, i32, ssub, STRSroW, STRSroX>;
+  defm : VecROStoreLane0Pat<ro64,         store, v2i64, i64, i64, dsub, STRDroW, STRDroX>;
+  defm : VecROStoreLane0Pat<ro64,         store, v2f64, f64, i64, dsub, STRDroW, STRDroX>;
 }
 
 //---
 // (unsigned immediate)
 defm STRX : StoreUIz<0b11, 0, 0b00, GPR64z, uimm12s8, "str",
                    [(store GPR64z:$Rt,
                             (am_indexed64 GPR64sp:$Rn, uimm12s8:$offset))]>;
 defm STRW : StoreUIz<0b10, 0, 0b00, GPR32z, uimm12s4, "str",

 def : Pat<(truncstorei8 GPR64:$Rt, (am_indexed8 GPR64sp:$Rn, uimm12s1:$offset)),
   (STRBBui (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, uimm12s1:$offset)>;
 
 } // AddedComplexity = 10
 
 // Match stores from lane 0 to the appropriate subreg's store.
 multiclass VecStoreLane0Pat<ComplexPattern UIAddrMode, SDPatternOperator storeop,
                             ValueType VTy, ValueType STy,
+                            ValueType SubRegTy,
                             SubRegIndex SubRegIdx, Operand IndexType,
                             Instruction STR> {
   def : Pat<(storeop (STy (vector_extract (VTy VecListOne128:$Vt), 0)),
                      (UIAddrMode GPR64sp:$Rn, IndexType:$offset)),
-            (STR (EXTRACT_SUBREG VecListOne128:$Vt, SubRegIdx),
+            (STR (SubRegTy (EXTRACT_SUBREG VecListOne128:$Vt, SubRegIdx)),
                  GPR64sp:$Rn, IndexType:$offset)>;
 }
 
 let AddedComplexity = 19 in {
-  defm : VecStoreLane0Pat<am_indexed16, truncstorei16, v8i16, i32, hsub, uimm12s2, STRHui>;
-  defm : VecStoreLane0Pat<am_indexed16,         store, v8f16, f16, hsub, uimm12s2, STRHui>;
-  defm : VecStoreLane0Pat<am_indexed32,         store, v4i32, i32, ssub, uimm12s4, STRSui>;
-  defm : VecStoreLane0Pat<am_indexed32,         store, v4f32, f32, ssub, uimm12s4, STRSui>;
-  defm : VecStoreLane0Pat<am_indexed64,         store, v2i64, i64, dsub, uimm12s8, STRDui>;
-  defm : VecStoreLane0Pat<am_indexed64,         store, v2f64, f64, dsub, uimm12s8, STRDui>;
+  defm : VecStoreLane0Pat<am_indexed16, truncstorei16, v8i16, i32, bf16, hsub, uimm12s2, STRHui>;
+  defm : VecStoreLane0Pat<am_indexed16,         store, v8f16, f16, bf16, hsub, uimm12s2, STRHui>;
+  defm : VecStoreLane0Pat<am_indexed32,         store, v4i32, i32, i32, ssub, uimm12s4, STRSui>;
+  defm : VecStoreLane0Pat<am_indexed32,         store, v4f32, f32, i32, ssub, uimm12s4, STRSui>;
+  defm : VecStoreLane0Pat<am_indexed64,         store, v2i64, i64, i64, dsub, uimm12s8, STRDui>;
+  defm : VecStoreLane0Pat<am_indexed64,         store, v2f64, f64, i64, dsub, uimm12s8, STRDui>;
 }
 
 //---
 // (unscaled immediate)
 defm STURX : StoreUnscaled<0b11, 0, 0b00, GPR64z, "stur",
                          [(store GPR64z:$Rt,
                                  (am_unscaled64 GPR64sp:$Rn, simm9:$offset))]>;
 defm STURW : StoreUnscaled<0b10, 0, 0b00, GPR32z, "stur",

 def : Pat<(truncstorei16 GPR64:$Rt, (am_unscaled16 GPR64sp:$Rn, simm9:$offset)),
   (STURHHi (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, simm9:$offset)>;
 def : Pat<(truncstorei8 GPR64:$Rt, (am_unscaled8 GPR64sp:$Rn, simm9:$offset)),
   (STURBBi (EXTRACT_SUBREG GPR64:$Rt, sub_32), GPR64sp:$Rn, simm9:$offset)>;
 
 // Match stores from lane 0 to the appropriate subreg's store.
 multiclass VecStoreULane0Pat<SDPatternOperator StoreOp,
                              ValueType VTy, ValueType STy,
+                             ValueType SubRegTy,
                              SubRegIndex SubRegIdx, Instruction STR> {
-  defm : VecStoreLane0Pat<am_unscaled128, StoreOp, VTy, STy, SubRegIdx, simm9, STR>;
+  defm : VecStoreLane0Pat<am_unscaled128, StoreOp, VTy, STy, SubRegTy, SubRegIdx, simm9, STR>;
 }
 
 let AddedComplexity = 19 in {
-  defm : VecStoreULane0Pat<truncstorei16, v8i16, i32, hsub, STURHi>;
-  defm : VecStoreULane0Pat<store,         v8f16, f16, hsub, STURHi>;
-  defm : VecStoreULane0Pat<store,         v4i32, i32, ssub, STURSi>;
-  defm : VecStoreULane0Pat<store,         v4f32, f32, ssub, STURSi>;
-  defm : VecStoreULane0Pat<store,         v2i64, i64, dsub, STURDi>;
-  defm : VecStoreULane0Pat<store,         v2f64, f64, dsub, STURDi>;
+  defm : VecStoreULane0Pat<truncstorei16, v8i16, i32, bf16, hsub, STURHi>;
+  defm : VecStoreULane0Pat<store,         v8f16, f16, bf16, hsub, STURHi>;
+  defm : VecStoreULane0Pat<store,         v4i32, i32, i32, ssub, STURSi>;
+  defm : VecStoreULane0Pat<store,         v4f32, f32, i32, ssub, STURSi>;
+  defm : VecStoreULane0Pat<store,         v2i64, i64, i64, dsub, STURDi>;
+  defm : VecStoreULane0Pat<store,         v2f64, f64, i64, dsub, STURDi>;
 }
 
 //---
 // STR mnemonics fall back to STUR for negative or unaligned offsets.
 def : InstAlias<"str $Rt, [$Rn, $offset]",
                 (STURXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>;
 def : InstAlias<"str $Rt, [$Rn, $offset]",
                 (STURWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
multiclass FMULScalarFromIndexedLane0Patterns<string inst,
                                               string inst_f32_suffix,
                                               string inst_f64_suffix,
                                               SDPatternOperator OpNode,
                                               list<Predicate> preds = []> {
   let Predicates = !listconcat(preds, [HasFullFP16]) in {
   def : Pat<(f16 (OpNode (f16 FPR16:$Rn),
                          (f16 (vector_extract (v8f16 V128:$Rm), (i64 0))))),
             (!cast<Instruction>(inst # inst_f16_suffix)
-              FPR16:$Rn, (EXTRACT_SUBREG V128:$Rm, hsub))>;
+              FPR16:$Rn, (bf16 (EXTRACT_SUBREG V128:$Rm, hsub)))>;
   }
   let Predicates = preds in {
   def : Pat<(f32 (OpNode (f32 FPR32:$Rn),
                          (f32 (vector_extract (v4f32 V128:$Rm), (i64 0))))),
             (!cast<Instruction>(inst # inst_f32_suffix)
               FPR32:$Rn, (EXTRACT_SUBREG V128:$Rm, ssub))>;
   def : Pat<(f64 (OpNode (f64 FPR64:$Rn),
                          (f64 (vector_extract (v2f64 V128:$Rm), (i64 0))))),
def : Pat<(v1f64 (int_aarch64_neon_vcvtfxu2fp (v1i64 FPR64:$Rn),
                                             vecshiftR64:$imm)),
           (UCVTFd FPR64:$Rn, vecshiftR64:$imm)>;
 def : Pat<(int_aarch64_neon_vcvtfxs2fp FPR32:$Rn, vecshiftR32:$imm),
           (SCVTFs FPR32:$Rn, vecshiftR32:$imm)>;
 
 // Patterns for FP16 Intrinsics - requires reg copy to/from as i16s not supported.
 
 def : Pat<(f16 (int_aarch64_neon_vcvtfxs2fp (i32 (sext_inreg FPR32:$Rn, i16)), vecshiftR16:$imm)),
-          (SCVTFh (EXTRACT_SUBREG FPR32:$Rn, hsub), vecshiftR16:$imm)>;
+          (SCVTFh (bf16 (EXTRACT_SUBREG FPR32:$Rn, hsub)), vecshiftR16:$imm)>;
 def : Pat<(f16 (int_aarch64_neon_vcvtfxs2fp (i32 FPR32:$Rn), vecshiftR16:$imm)),
-          (SCVTFh (EXTRACT_SUBREG FPR32:$Rn, hsub), vecshiftR16:$imm)>;
+          (SCVTFh (bf16 (EXTRACT_SUBREG FPR32:$Rn, hsub)), vecshiftR16:$imm)>;
 def : Pat<(f16 (int_aarch64_neon_vcvtfxs2fp (i64 FPR64:$Rn), vecshiftR16:$imm)),
-          (SCVTFh (EXTRACT_SUBREG FPR64:$Rn, hsub), vecshiftR16:$imm)>;
+          (SCVTFh (bf16 (EXTRACT_SUBREG FPR64:$Rn, hsub)), vecshiftR16:$imm)>;
 def : Pat<(f16 (int_aarch64_neon_vcvtfxu2fp
             (and FPR32:$Rn, (i32 65535)),
             vecshiftR16:$imm)),
-          (UCVTFh (EXTRACT_SUBREG FPR32:$Rn, hsub), vecshiftR16:$imm)>;
+          (UCVTFh (bf16 (EXTRACT_SUBREG FPR32:$Rn, hsub)), vecshiftR16:$imm)>;
 def : Pat<(f16 (int_aarch64_neon_vcvtfxu2fp FPR32:$Rn, vecshiftR16:$imm)),
-          (UCVTFh (EXTRACT_SUBREG FPR32:$Rn, hsub), vecshiftR16:$imm)>;
+          (UCVTFh (bf16 (EXTRACT_SUBREG FPR32:$Rn, hsub)), vecshiftR16:$imm)>;
 def : Pat<(f16 (int_aarch64_neon_vcvtfxu2fp (i64 FPR64:$Rn), vecshiftR16:$imm)),
-          (UCVTFh (EXTRACT_SUBREG FPR64:$Rn, hsub), vecshiftR16:$imm)>;
+          (UCVTFh (bf16 (EXTRACT_SUBREG FPR64:$Rn, hsub)), vecshiftR16:$imm)>;
 def : Pat<(i32 (int_aarch64_neon_vcvtfp2fxs (f16 FPR16:$Rn), vecshiftR32:$imm)),
           (i32 (INSERT_SUBREG
             (i32 (IMPLICIT_DEF)),
             (FCVTZSh FPR16:$Rn, vecshiftR32:$imm),
             hsub))>;
 def : Pat<(i64 (int_aarch64_neon_vcvtfp2fxs (f16 FPR16:$Rn), vecshiftR64:$imm)),
           (i64 (INSERT_SUBREG
             (i64 (IMPLICIT_DEF)),

llvm/lib/Target/AArch64/AArch64RegisterInfo.td

 def Q29   : AArch64Reg<29, "q29", [D29], ["v29", ""]>, DwarfRegAlias<B29>;
 def Q30   : AArch64Reg<30, "q30", [D30], ["v30", ""]>, DwarfRegAlias<B30>;
 def Q31   : AArch64Reg<31, "q31", [D31], ["v31", ""]>, DwarfRegAlias<B31>;
 }
 
 def FPR8  : RegisterClass<"AArch64", [i8], 8, (sequence "B%u", 0, 31)> {
   let Size = 8;
 }
-def FPR16 : RegisterClass<"AArch64", [f16, bf16], 16, (sequence "H%u", 0, 31)> {
+def FPR16 : RegisterClass<"AArch64", [f16, bf16, i16], 16, (sequence "H%u", 0, 31)> {
   let Size = 16;
 }
 
 def FPR16_lo : RegisterClass<"AArch64", [f16], 16, (trunc FPR16, 16)> {
   let Size = 16;
 }
 def FPR32 : RegisterClass<"AArch64", [f32, i32], 32,(sequence "S%u", 0, 31)>;
 def FPR64 : RegisterClass<"AArch64", [f64, i64, v2f32, v1f64, v8i8, v4i16, v2i32,

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

MIRBuilder.buildInstr(AArch64::G_ADD_LOW, {DstReg}, {ADRP})
       .addGlobalAddress(GV, Offset,
                         OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
   MI.eraseFromParent();
   return true;
 }
 
 bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
                                              MachineInstr &MI) const {
-  switch (cast<GIntrinsic>(MI).getIntrinsicID()) {
+  Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
+  switch (IntrinsicID) {
   case Intrinsic::vacopy: {
     unsigned PtrSize = ST->isTargetILP32() ? 4 : 8;
     unsigned VaListSize =
       (ST->isTargetDarwin() || ST->isTargetWindows())
           ? PtrSize
           : ST->isTargetILP32() ? 20 : 32;
 
     MachineFunction &MF = *MI.getMF();
unsigned PrfOp = (IsWrite << 4) |    // Load/Store bit
                      (!IsData << 3) |    // IsDataCache bit
                      (Target << 1) |     // Cache level bits
                      (unsigned)IsStream; // Stream bit
 
     MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal);
     MI.eraseFromParent();
     return true;
   }
+  case Intrinsic::aarch64_neon_uaddv:
+  case Intrinsic::aarch64_neon_saddv:
+  case Intrinsic::aarch64_neon_umaxv:
+  case Intrinsic::aarch64_neon_smaxv:
+  case Intrinsic::aarch64_neon_uminv:
+  case Intrinsic::aarch64_neon_sminv: {
+    MachineIRBuilder MIB(MI);
+    MachineRegisterInfo &MRI = *MIB.getMRI();
+    bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
+                    IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
+                    IntrinsicID == Intrinsic::aarch64_neon_sminv;
+
+    auto OldDst = MI.getOperand(0).getReg();
+    auto OldDstTy = MRI.getType(OldDst);
+    LLT NewDstTy = MRI.getType(MI.getOperand(2).getReg()).getElementType();
+    if (OldDstTy == NewDstTy)
+      return true;
+
+    auto NewDst = MRI.createGenericVirtualRegister(NewDstTy);
+
+    Helper.Observer.changingInstr(MI);
+    MI.getOperand(0).setReg(NewDst);
+    Helper.Observer.changedInstr(MI);
+
+    MIB.setInsertPt(MIB.getMBB(), ++MIB.getInsertPt());
+    MIB.buildExtOrTrunc(IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
+                        OldDst, NewDst);
+
+    return true;
+  }
   }
 
   return true;
 }
 
 bool AArch64LegalizerInfo::legalizeShlAshrLshr(
     MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
     GISelChangeObserver &Observer) const {

llvm/lib/Target/AArch64/GISel/AArch64RegisterBankInfo.cpp

 static bool isFPIntrinsic(const MachineRegisterInfo &MRI,
                           const MachineInstr &MI) {
   // TODO: Add more intrinsics.
   switch (cast<GIntrinsic>(MI).getIntrinsicID()) {
   default:
     return false;
   case Intrinsic::aarch64_neon_uaddlv:
   case Intrinsic::aarch64_neon_uaddv:
+  case Intrinsic::aarch64_neon_saddv:
   case Intrinsic::aarch64_neon_umaxv:
+  case Intrinsic::aarch64_neon_smaxv:
   case Intrinsic::aarch64_neon_uminv:
+  case Intrinsic::aarch64_neon_sminv:
+  case Intrinsic::aarch64_neon_faddv:
   case Intrinsic::aarch64_neon_fmaxv:
   case Intrinsic::aarch64_neon_fminv:
   case Intrinsic::aarch64_neon_fmaxnmv:
   case Intrinsic::aarch64_neon_fminnmv:
     return true;
   case Intrinsic::aarch64_neon_saddlv: {
     const LLT SrcTy = MRI.getType(MI.getOperand(2).getReg());
     return SrcTy.getElementType().getSizeInBits() >= 16 &&
            SrcTy.getElementCount().getFixedValue() >= 4;
   }
-  case Intrinsic::aarch64_neon_saddv:
-  case Intrinsic::aarch64_neon_smaxv:
-  case Intrinsic::aarch64_neon_sminv: {
-    const LLT SrcTy = MRI.getType(MI.getOperand(2).getReg());
-    return SrcTy.getElementType().getSizeInBits() >= 32 &&
-           SrcTy.getElementCount().getFixedValue() >= 2;
-  }
   }
 }
 
 bool AArch64RegisterBankInfo::hasFPConstraints(const MachineInstr &MI,
                                                const MachineRegisterInfo &MRI,
                                                const TargetRegisterInfo &TRI,
                                                unsigned Depth) const {
   unsigned Op = MI.getOpcode();
case AArch64::G_DUP: {
     break;
   }
   case TargetOpcode::G_TRUNC: {
     LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
     if (!SrcTy.isVector() && SrcTy.getSizeInBits() == 128)
       OpRegBankIdx = {PMI_FirstFPR, PMI_FirstFPR};
     break;
   }
+  case TargetOpcode::G_ZEXT:
+  case TargetOpcode::G_SEXT: {
+    // Allow G_SEXT/G_ZEXT from small FPR scalars to select across lane
+    // intrinsics.
+    LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+    LLT SrcTy = MRI.getType(MI.getOperand(1).getReg());
+    if (getRegBank(MI.getOperand(1).getReg(), MRI, TRI) ==
+        &AArch64::FPRRegBank) {
+      if ((DstTy.getSizeInBits() == 32 || DstTy.getSizeInBits() == 64) &&
+          (SrcTy.getSizeInBits() == 8 || SrcTy.getSizeInBits() == 16) &&
+          SrcTy.getSizeInBits() < DstTy.getSizeInBits()) {
+        OpRegBankIdx[1] = PMI_FirstFPR;
+      }
+    }
+    break;
+  }
   case TargetOpcode::G_SITOFP:
   case TargetOpcode::G_UITOFP: {
     if (MRI.getType(MI.getOperand(0).getReg()).isVector())
       break;
     // Integer to FP conversions don't necessarily happen between GPR -> FPR
     // regbanks. They can also be done within an FPR register.
     Register SrcReg = MI.getOperand(1).getReg();
     if (getRegBank(SrcReg, MRI, TRI) == &AArch64::FPRRegBank)

llvm/test/CodeGen/AArch64/arm64-smaxv.ll

 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
+; RUN: llc < %s -global-isel=1 -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
 
 define signext i8 @test_vmaxv_s8(<8 x i8> %a1) {
 ; CHECK: test_vmaxv_s8
 ; CHECK: smaxv.8b b[[REGNUM:[0-9]+]], v0
 ; CHECK-NEXT: smov.b w0, v[[REGNUM]][0]
 ; CHECK-NEXT: ret
 entry:
   %vmaxv.i = tail call i32 @llvm.aarch64.neon.smaxv.i32.v8i8(<8 x i8> %a1)

llvm/test/CodeGen/AArch64/arm64-sminv.ll

-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s --check-prefix=CHECK --check-prefix=SDAG
-; RUN: llc < %s -global-isel=1 -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s --check-prefix=CHECK --check-prefix=GISEL
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
+; RUN: llc < %s -global-isel=1 -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
 
 define signext i8 @test_vminv_s8(<8 x i8> %a1) {
 ; CHECK: test_vminv_s8
 ; CHECK: sminv.8b b[[REGNUM:[0-9]+]], v0
-; SDAG-NEXT: smov.b w0, v[[REGNUM]][0]
-; SDAG-NEXT: ret
-; GISEL-NEXT: smov.b w8, v[[REGNUM]][0]
-; GISEL-NEXT: sxtb  w0, w8
-; GISEL-NEXT: ret
+; CHECK-NEXT: smov.b w0, v[[REGNUM]][0]
+; CHECK-NEXT: ret
 entry:
   %vminv.i = tail call i32 @llvm.aarch64.neon.sminv.i32.v8i8(<8 x i8> %a1)
   %0 = trunc i32 %vminv.i to i8
   ret i8 %0
 }
 
 define signext i16 @test_vminv_s16(<4 x i16> %a1) {
 ; CHECK: test_vminv_s16
 ; CHECK: sminv.4h h[[REGNUM:[0-9]+]], v0
-; SDAG-NEXT: smov.h w0, v[[REGNUM]][0]
-; SDAG-NEXT: ret
-; GISEL-NEXT: smov.h w8, v[[REGNUM]][0]
-; GISEL-NEXT: sxth  w0, w8
-; GISEL-NEXT: ret
+; CHECK-NEXT: smov.h w0, v[[REGNUM]][0]
+; CHECK-NEXT: ret
 entry:
   %vminv.i = tail call i32 @llvm.aarch64.neon.sminv.i32.v4i16(<4 x i16> %a1)
   %0 = trunc i32 %vminv.i to i16
   ret i16 %0
 }
 
 define i32 @test_vminv_s32(<2 x i32> %a1) {
 ; CHECK: test_vminv_s32
 ; 2 x i32 is not supported by the ISA, thus, this is a special case
 ; CHECK: sminp.2s v[[REGNUM:[0-9]+]], v0, v0
 ; CHECK-NEXT: fmov w0, s[[REGNUM]]
 ; CHECK-NEXT: ret
 entry:
   %vminv.i = tail call i32 @llvm.aarch64.neon.sminv.i32.v2i32(<2 x i32> %a1)
   ret i32 %vminv.i
 }
 
 define signext i8 @test_vminvq_s8(<16 x i8> %a1) {
 ; CHECK: test_vminvq_s8
 ; CHECK: sminv.16b b[[REGNUM:[0-9]+]], v0
-; SDAG-NEXT: smov.b w0, v[[REGNUM]][0]
-; SDAG-NEXT: ret
-; GISEL-NEXT: smov.b w8, v[[REGNUM]][0]
-; GISEL-NEXT: sxtb  w0, w8
-; GISEL-NEXT: ret
+; CHECK-NEXT: smov.b w0, v[[REGNUM]][0]
+; CHECK-NEXT: ret
 entry:
   %vminv.i = tail call i32 @llvm.aarch64.neon.sminv.i32.v16i8(<16 x i8> %a1)
   %0 = trunc i32 %vminv.i to i8
   ret i8 %0
 }
 
 define signext i16 @test_vminvq_s16(<8 x i16> %a1) {
 ; CHECK: test_vminvq_s16
 ; CHECK: sminv.8h h[[REGNUM:[0-9]+]], v0
-; SDAG-NEXT: smov.h w0, v[[REGNUM]][0]
-; SDAG-NEXT: ret
-; GISEL-NEXT: smov.h w8, v[[REGNUM]][0]
-; GISEL-NEXT: sxth  w0, w8
-; GISEL-NEXT: ret
+; CHECK-NEXT: smov.h w0, v[[REGNUM]][0]
+; CHECK-NEXT: ret
 entry:
   %vminv.i = tail call i32 @llvm.aarch64.neon.sminv.i32.v8i16(<8 x i16> %a1)
   %0 = trunc i32 %vminv.i to i16
   ret i16 %0
 }
 
 define i32 @test_vminvq_s32(<4 x i32> %a1) {
 ; CHECK: test_vminvq_s32
 ; CHECK: sminv.4s [[REGNUM:s[0-9]+]], v0
 ; CHECK-NEXT: fmov w0, [[REGNUM]]
 ; CHECK-NEXT: ret
 entry:
   %vminv.i = tail call i32 @llvm.aarch64.neon.sminv.i32.v4i32(<4 x i32> %a1)
   ret i32 %vminv.i
 }
 
 define <8 x i8> @test_vminv_s8_used_by_laneop(<8 x i8> %a1, <8 x i8> %a2) {
 ; CHECK-LABEL: test_vminv_s8_used_by_laneop:
 ; CHECK: sminv.8b b[[REGNUM:[0-9]+]], v1
-; SDAG-NEXT: mov.b v0[3], v[[REGNUM]][0]
-; SDAG-NEXT: ret
-; GISEL-NEXT: smov.b  w8, v[[REGNUM]][0]
-; GISEL-NEXT: mov.b v0[3], w8
-; GISEL-NEXT: ret
+; CHECK-NEXT: mov.b v0[3], v[[REGNUM]][0]
+; CHECK-NEXT: ret
 entry:
   %0 = tail call i32 @llvm.aarch64.neon.sminv.i32.v8i8(<8 x i8> %a2)
   %1 = trunc i32 %0 to i8
   %2 = insertelement <8 x i8> %a1, i8 %1, i32 3
   ret <8 x i8> %2
 }
 
 define <4 x i16> @test_vminv_s16_used_by_laneop(<4 x i16> %a1, <4 x i16> %a2) {
 ; CHECK-LABEL: test_vminv_s16_used_by_laneop:
 ; CHECK: sminv.4h h[[REGNUM:[0-9]+]], v1
-; SDAG-NEXT: mov.h v0[3], v[[REGNUM]][0]
-; SDAG-NEXT: ret
-; GISEL-NEXT: smov.h  w8, v[[REGNUM]][0]
-; GISEL-NEXT: mov.h v0[3], w8
-; GISEL-NEXT: ret
+; CHECK-NEXT: mov.h v0[3], v[[REGNUM]][0]
+; CHECK-NEXT: ret
 entry:
   %0 = tail call i32 @llvm.aarch64.neon.sminv.i32.v4i16(<4 x i16> %a2)
   %1 = trunc i32 %0 to i16
   %2 = insertelement <4 x i16> %a1, i16 %1, i32 3
   ret <4 x i16> %2
 }
 
 define <2 x i32> @test_vminv_s32_used_by_laneop(<2 x i32> %a1, <2 x i32> %a2) {
 ; CHECK-LABEL: test_vminv_s32_used_by_laneop:
 ; CHECK: sminp.2s v[[REGNUM:[0-9]+]], v1, v1
 ; CHECK-NEXT: mov.s v0[1], v[[REGNUM]][0]
 ; CHECK-NEXT: ret
 entry:
   %0 = tail call i32 @llvm.aarch64.neon.sminv.i32.v2i32(<2 x i32> %a2)
   %1 = insertelement <2 x i32> %a1, i32 %0, i32 1
   ret <2 x i32> %1
 }
 
 define <16 x i8> @test_vminvq_s8_used_by_laneop(<16 x i8> %a1, <16 x i8> %a2) {
 ; CHECK-LABEL: test_vminvq_s8_used_by_laneop:
 ; CHECK: sminv.16b b[[REGNUM:[0-9]+]], v1
-; SDAG-NEXT: mov.b v0[3], v[[REGNUM]][0]
-; SDAG-NEXT: ret
-; GISEL-NEXT: smov.b  w8, v[[REGNUM]][0]
-; GISEL-NEXT: mov.b v0[3], w8
-; GISEL-NEXT: ret
+; CHECK-NEXT: mov.b v0[3], v[[REGNUM]][0]
+; CHECK-NEXT: ret
 entry:
   %0 = tail call i32 @llvm.aarch64.neon.sminv.i32.v16i8(<16 x i8> %a2)
   %1 = trunc i32 %0 to i8
   %2 = insertelement <16 x i8> %a1, i8 %1, i32 3
   ret <16 x i8> %2
 }
 
 define <8 x i16> @test_vminvq_s16_used_by_laneop(<8 x i16> %a1, <8 x i16> %a2) {
 ; CHECK-LABEL: test_vminvq_s16_used_by_laneop:
 ; CHECK: sminv.8h h[[REGNUM:[0-9]+]], v1
-; SDAG-NEXT: mov.h v0[3], v[[REGNUM]][0]
-; SDAG-NEXT: ret
-; GISEL-NEXT: smov.h  w8, v[[REGNUM]][0]
-; GISEL-NEXT: mov.h v0[3], w8
-; GISEL-NEXT: ret
+; CHECK-NEXT: mov.h v0[3], v[[REGNUM]][0]
+; CHECK-NEXT: ret
 entry:
   %0 = tail call i32 @llvm.aarch64.neon.sminv.i32.v8i16(<8 x i16> %a2)
   %1 = trunc i32 %0 to i16
   %2 = insertelement <8 x i16> %a1, i16 %1, i32 3
   ret <8 x i16> %2
 }
 
 define <4 x i32> @test_vminvq_s32_used_by_laneop(<4 x i32> %a1, <4 x i32> %a2) {

llvm/test/CodeGen/AArch64/arm64-umaxv.ll

-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s --check-prefix CHECK --check-prefix SDAG
-; RUN: llc < %s -global-isel=1 -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s --check-prefix CHECK --check-prefix GISEL
+; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
+; RUN: llc < %s -global-isel=1 -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
 
 define i32 @vmax_u8x8(<8 x i8> %a) nounwind ssp {
 ; CHECK-LABEL: vmax_u8x8:
 ; CHECK: umaxv.8b        b[[REG:[0-9]+]], v0
 ; CHECK: fmov    [[REG2:w[0-9]+]], s[[REG]]
 ; CHECK-NOT: and
-; SDAG: cbz     [[REG2]],
-; GISEL: b
+; CHECK: cbz     [[REG2]],
 entry:
   %vmaxv.i = tail call i32 @llvm.aarch64.neon.umaxv.i32.v8i8(<8 x i8> %a) nounwind
   %tmp = trunc i32 %vmaxv.i to i8
   %tobool = icmp eq i8 %tmp, 0
   br i1 %tobool, label %return, label %if.then
 
 if.then:
   %call1 = tail call i32 @bar() nounwind
   br label %return
 
 return:
   %retval.0 = phi i32 [ %call1, %if.then ], [ 0, %entry ]
   ret i32 %retval.0
 }
 
 declare i32 @bar(...)
 
 define i32 @vmax_u4x16(<4 x i16> %a) nounwind ssp {
 ; CHECK-LABEL: vmax_u4x16:
 ; CHECK: umaxv.4h        h[[REG:[0-9]+]], v0
 ; CHECK: fmov    [[REG2:w[0-9]+]], s[[REG]]
 ; CHECK-NOT: and
-; SDAG: cbz     [[REG2]],
-; GISEL: b
+; CHECK: cbz     [[REG2]],
 entry:
   %vmaxv.i = tail call i32 @llvm.aarch64.neon.umaxv.i32.v4i16(<4 x i16> %a) nounwind
   %tmp = trunc i32 %vmaxv.i to i16
   %tobool = icmp eq i16 %tmp, 0
   br i1 %tobool, label %return, label %if.then
 
 if.then:
   %call1 = tail call i32 @bar() nounwind
   br label %return
 
 return:
   %retval.0 = phi i32 [ %call1, %if.then ], [ 0, %entry ]
   ret i32 %retval.0
 }
 
 define i32 @vmax_u8x16(<8 x i16> %a) nounwind ssp {
 ; CHECK-LABEL: vmax_u8x16:
 ; CHECK: umaxv.8h        h[[REG:[0-9]+]], v0
 ; CHECK: fmov    [[REG2:w[0-9]+]], s[[REG]]
 ; CHECK-NOT: and
-; SDAG: cbz     [[REG2]],
-; GISEL: b
+; CHECK: cbz     [[REG2]],
 entry:
   %vmaxv.i = tail call i32 @llvm.aarch64.neon.umaxv.i32.v8i16(<8 x i16> %a) nounwind
   %tmp = trunc i32 %vmaxv.i to i16
   %tobool = icmp eq i16 %tmp, 0
   br i1 %tobool, label %return, label %if.then
 
 if.then:
   %call1 = tail call i32 @bar() nounwind
   br label %return
 
 return:
   %retval.0 = phi i32 [ %call1, %if.then ], [ 0, %entry ]
   ret i32 %retval.0
 }
 
 define i32 @vmax_u16x8(<16 x i8> %a) nounwind ssp {
 ; CHECK-LABEL: vmax_u16x8:
 ; CHECK: umaxv.16b        b[[REG:[0-9]+]], v0
 ; CHECK: fmov     [[REG2:w[0-9]+]], s[[REG]]
 ; CHECK-NOT: and
-; SDAG: cbz     [[REG2]],
-; GISEL: b
+; CHECK: cbz     [[REG2]],
 entry:
   %vmaxv.i = tail call i32 @llvm.aarch64.neon.umaxv.i32.v16i8(<16 x i8> %a) nounwind
   %tmp = trunc i32 %vmaxv.i to i8
   %tobool = icmp eq i8 %tmp, 0
   br i1 %tobool, label %return, label %if.then
 
 if.then:
   %call1 = tail call i32 @bar() nounwind

llvm/test/CodeGen/AArch64/arm64-uminv.ll

 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
+; RUN: llc < %s -global-isel=1 -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false | FileCheck %s
 
 define i32 @vmin_u8x8(<8 x i8> %a) nounwind ssp {
 ; CHECK-LABEL: vmin_u8x8:
 ; CHECK: uminv.8b        b[[REG:[0-9]+]], v0
 ; CHECK: fmov    [[REG2:w[0-9]+]], s[[REG]]
 ; CHECK-NOT: and
 ; CHECK: cbz     [[REG2]],
 entry:

llvm/test/CodeGen/AArch64/arm64-vaddv.ll

 ; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false -mcpu=cyclone | FileCheck %s
+; RUN: llc < %s -global-isel -mtriple=arm64-eabi -aarch64-neon-syntax=apple -asm-verbose=false -mcpu=cyclone | FileCheck %s
 
 define signext i8 @test_vaddv_s8(<8 x i8> %a1) {
 ; CHECK-LABEL: test_vaddv_s8:
 ; CHECK: addv.8b b[[REGNUM:[0-9]+]], v0
 ; CHECK-NEXT: smov.b w0, v[[REGNUM]][0]
 ; CHECK-NEXT: ret
 entry:
   %vaddv.i = tail call i32 @llvm.aarch64.neon.saddv.i32.v8i8(<8 x i8> %a1)

llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-trunc.mir

bb.0:
     ; CHECK: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
     ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[DEF]](s64)
     ; CHECK-NEXT: $vgpr0 = COPY [[TRUNC]](s32)
     %0:_(s64) = G_IMPLICIT_DEF
     %1:_(s48) = G_TRUNC %0
     %2:_(s32) = G_TRUNC %1
     $vgpr0 = COPY %2
 ...
+
+---
+name: trunc_sext
+
+body: |
+  bb.0:
+    ; Test that trunc(sext) is replaced with sext source.
+    ; CHECK-LABEL: name: trunc_sext
+    ; CHECK: [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+    ; CHECK-NEXT: $vgpr0 = COPY [[DEF]](s32)
+    %0:_(s32) = G_IMPLICIT_DEF
+    %1:_(s64) = G_SEXT %0
+    %2:_(s32) = G_TRUNC %1
+    $vgpr0 = COPY %2
+...