[GISel][AArch64] Close some gaps

[tschuett]

Summary

I had to update two failing existing tests.

[dmgreen]

Chuong in our team is currently looking into zext/sext of vector instructions. I think he said it needed something to handle v4i16->v4i64 type extends that need to be split into two step, but it was working for a lot of the other cases.

[tschuett]

Sorry for the churn! I found:
remark: unable to legalize instruction: %942:_(<4 x s64>) = G_ZEXT %941:_(<4 x s32>) [-Rpass-missed=gisel-legalize]
remark: unable to legalize instruction: %190:_(<8 x s32>) = G_ZEXT %188:_(<8 x s16>) [-Rpass-missed=gisel-legalize]
I did the following on a Graviton 3:
https://gist.github.com/tschuett/d448c87ee985af99f90568fddf9ffd82

[dmgreen]

Sounds good. There is a lot of extra complexity in the vector lowering - a lot of cases that can go wrong or be missed. I am hoping that we can clear up most of them and that will put us in a better place to start optimizing what remains.

A lot of the test updates look very similar to what he had I think, which is a good sign. I'm hoping that something like https://reviews.llvm.org/D155871 can help to get USHLL2 working.

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

 
 static cl::opt<bool>
     EnableCSEInIRTranslator("enable-cse-in-irtranslator",
                             cl::desc("Should enable CSE in irtranslator"),
                             cl::Optional, cl::init(false));
 char IRTranslator::ID = 0;
 
 INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
                       false, false)
 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
 INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(StackProtector)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI",
                     false, false)
 
 static void reportTranslationError(MachineFunction &MF,
                                    const TargetPassConfig &TPC,
                                    OptimizationRemarkEmitter &ORE,
                                    OptimizationRemarkMissed &R) {
   MF.getProperties().set(MachineFunctionProperties::Property::FailedISel);
 
   // Print the function name explicitly if we don't have a debug location (which
#endif
     assert((CurrInst->getDebugLoc() == MI.getDebugLoc() ||
             (MI.getParent()->isEntryBlock() && !MI.getDebugLoc())) &&
            "Line info was not transferred to all instructions");
   }
 };
 } // namespace
 #endif // ifndef NDEBUG
 
-
 void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<StackProtector>();
   AU.addRequired<TargetPassConfig>();
   AU.addRequired<GISelCSEAnalysisWrapperPass>();
   AU.addRequired<AssumptionCacheTracker>();
   if (OptLevel != CodeGenOpt::None) {
     AU.addRequired<BranchProbabilityInfoWrapperPass>();
     AU.addRequired<AAResultsWrapperPass>();
ArrayRef<Register> IRTranslator::getOrCreateVRegs(const Value &Val) {
 
   if (Val.getType()->isVoidTy())
     return *VMap.getVRegs(Val);
 
   // Create entry for this type.
   auto *VRegs = VMap.getVRegs(Val);
   auto *Offsets = VMap.getOffsets(Val);
 
-  assert(Val.getType()->isSized() &&
-         "Don't know how to create an empty vreg");
+  assert(Val.getType()->isSized() && "Don't know how to create an empty vreg");
 
   SmallVector<LLT, 4> SplitTys;
   computeValueLLTs(*DL, *Val.getType(), SplitTys,
                    Offsets->empty() ? Offsets : nullptr);
 
   if (!isa<Constant>(Val)) {
     for (auto Ty : SplitTys)
       VRegs->push_back(MRI->createGenericVirtualRegister(Ty));

 }
 
 bool IRTranslator::translateCompare(const User &U,
                                     MachineIRBuilder &MIRBuilder) {
   auto *CI = dyn_cast<CmpInst>(&U);
   Register Op0 = getOrCreateVReg(*U.getOperand(0));
   Register Op1 = getOrCreateVReg(*U.getOperand(1));
   Register Res = getOrCreateVReg(U);
-  CmpInst::Predicate Pred =
-      CI ? CI->getPredicate() : static_cast<CmpInst::Predicate>(
+  CmpInst::Predicate Pred = CI ? CI->getPredicate()
+                               : static_cast<CmpInst::Predicate>(
                                      cast<ConstantExpr>(U).getPredicate());
   if (CmpInst::isIntPredicate(Pred))
     MIRBuilder.buildICmp(Pred, Res, Op0, Op1);
   else if (Pred == CmpInst::FCMP_FALSE)
-    MIRBuilder.buildCopy(
-        Res, getOrCreateVReg(*Constant::getNullValue(U.getType())));
+    MIRBuilder.buildCopy(Res,
+                         getOrCreateVReg(*Constant::getNullValue(U.getType())));
   else if (Pred == CmpInst::FCMP_TRUE)
     MIRBuilder.buildCopy(
         Res, getOrCreateVReg(*Constant::getAllOnesValue(U.getType())));
   else {
     uint32_t Flags = 0;
     if (CI)
       Flags = MachineInstr::copyFlagsFromInstruction(*CI);
     MIRBuilder.buildFCmp(Pred, Res, Op0, Op1, Flags);
if (MRI->getType(CondLHS).getSizeInBits() == 1 && CI && CI->isOne() &&
       else
         Cond =
             MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0);
     }
   } else {
     assert(CB.PredInfo.Pred == CmpInst::ICMP_SLE &&
            "Can only handle SLE ranges");
 
-    const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
-    const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
+    const APInt &Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
+    const APInt &High = cast<ConstantInt>(CB.CmpRHS)->getValue();
 
     Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS);
     if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
       Register CondRHS = getOrCreateVReg(*CB.CmpRHS);
       Cond =
           MIB.buildICmp(CmpInst::ICMP_SLE, i1Ty, CmpOpReg, CondRHS).getReg(0);
     } else {
       const LLT CmpTy = MRI->getType(CmpOpReg);
void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB,
   addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()},
                     CB.ThisBB);
 
   MIB.buildBrCond(Cond, *CB.TrueBB);
   MIB.buildBr(*CB.FalseBB);
   MIB.setDebugLoc(OldDbgLoc);
 }
 
-bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W,
-                                          MachineBasicBlock *SwitchMBB,
-                                          MachineBasicBlock *CurMBB,
-                                          MachineBasicBlock *DefaultMBB,
-                                          MachineIRBuilder &MIB,
-                                          MachineFunction::iterator BBI,
-                                          BranchProbability UnhandledProbs,
-                                          SwitchCG::CaseClusterIt I,
-                                          MachineBasicBlock *Fallthrough,
-                                          bool FallthroughUnreachable) {
+bool IRTranslator::lowerJumpTableWorkItem(
+    SwitchCG::SwitchWorkListItem W, MachineBasicBlock *SwitchMBB,
+    MachineBasicBlock *CurMBB, MachineBasicBlock *DefaultMBB,
+    MachineIRBuilder &MIB, MachineFunction::iterator BBI,
+    BranchProbability UnhandledProbs, SwitchCG::CaseClusterIt I,
+    MachineBasicBlock *Fallthrough, bool FallthroughUnreachable) {
   using namespace SwitchCG;
   MachineFunction *CurMF = SwitchMBB->getParent();
   // FIXME: Optimize away range check based on pivot comparisons.
   JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first;
   SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second;
   BranchProbability DefaultProb = W.DefaultProb;
 
   // The jump block hasn't been inserted yet; insert it here.
bool IRTranslator::lowerJumpTableWorkItem(
   // If we're in the right place, emit the jump table header right now.
   if (CurMBB == SwitchMBB) {
     if (!emitJumpTableHeader(*JT, *JTH, CurMBB))
       return false;
     JTH->Emitted = true;
   }
   return true;
 }
-bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I,
-                                            Value *Cond,
-                                            MachineBasicBlock *Fallthrough,
-                                            bool FallthroughUnreachable,
-                                            BranchProbability UnhandledProbs,
-                                            MachineBasicBlock *CurMBB,
-                                            MachineIRBuilder &MIB,
-                                            MachineBasicBlock *SwitchMBB) {
+bool IRTranslator::lowerSwitchRangeWorkItem(
+    SwitchCG::CaseClusterIt I, Value *Cond, MachineBasicBlock *Fallthrough,
+    bool FallthroughUnreachable, BranchProbability UnhandledProbs,
+    MachineBasicBlock *CurMBB, MachineIRBuilder &MIB,
   using namespace SwitchCG;
   const Value *RHS, *LHS, *MHS;
   CmpInst::Predicate Pred;
   if (I->Low == I->High) {
     // Check Cond == I->Low.
     Pred = CmpInst::ICMP_EQ;
     LHS = Cond;
     RHS = I->Low;
if (StructType *StTy = GTI.getStructTypeOrNull()) {
                       .getReg(0);
         Offset = 0;
       }
 
       Register IdxReg = getOrCreateVReg(*Idx);
       LLT IdxTy = MRI->getType(IdxReg);
       if (IdxTy != OffsetTy) {
         if (!IdxTy.isVector() && WantSplatVector) {
-          IdxReg = MIRBuilder.buildSplatVector(
-            OffsetTy.changeElementType(IdxTy), IdxReg).getReg(0);
+          IdxReg =
+              MIRBuilder
+                  .buildSplatVector(OffsetTy.changeElementType(IdxTy), IdxReg)
+                  .getReg(0);
         }
 
         IdxReg = MIRBuilder.buildSExtOrTrunc(OffsetTy, IdxReg).getReg(0);
       }
 
       // N = N + Idx * ElementSize;
       // Avoid doing it for ElementSize of 1.
       Register GepOffsetReg;
       if (ElementSize != 1) {
         auto ElementSizeMIB = MIRBuilder.buildConstant(
             getLLTForType(*OffsetIRTy, *DL), ElementSize);
         GepOffsetReg =
             MIRBuilder.buildMul(OffsetTy, IdxReg, ElementSizeMIB).getReg(0);
       } else
         GepOffsetReg = IdxReg;
 
       BaseReg = MIRBuilder.buildPtrAdd(PtrTy, BaseReg, GepOffsetReg).getReg(0);
     }
   }
 
   if (Offset != 0) {
-    auto OffsetMIB =
-        MIRBuilder.buildConstant(OffsetTy, Offset);
+    auto OffsetMIB = MIRBuilder.buildConstant(OffsetTy, Offset);
     MIRBuilder.buildPtrAdd(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
     return true;
   }
 
   MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
   return true;
 }
 

 }
 
 bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI,
                                                 MachineIRBuilder &MIRBuilder) {
   Register Dst = getOrCreateVReg(CI);
   Register Src0 = getOrCreateVReg(*CI.getOperand(0));
   Register Src1 = getOrCreateVReg(*CI.getOperand(1));
   uint64_t Scale = cast<ConstantInt>(CI.getOperand(2))->getZExtValue();
-  MIRBuilder.buildInstr(Op, {Dst}, { Src0, Src1, Scale });
+  MIRBuilder.buildInstr(Op, {Dst}, {Src0, Src1, Scale});
   return true;
 }
 
 unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) {
   switch (ID) {
-    default:
-      break;
-    case Intrinsic::bswap:
-      return TargetOpcode::G_BSWAP;
-    case Intrinsic::bitreverse:
-      return TargetOpcode::G_BITREVERSE;
-    case Intrinsic::fshl:
-      return TargetOpcode::G_FSHL;
-    case Intrinsic::fshr:
-      return TargetOpcode::G_FSHR;
-    case Intrinsic::ceil:
-      return TargetOpcode::G_FCEIL;
-    case Intrinsic::cos:
-      return TargetOpcode::G_FCOS;
-    case Intrinsic::ctpop:
-      return TargetOpcode::G_CTPOP;
-    case Intrinsic::exp:
-      return TargetOpcode::G_FEXP;
-    case Intrinsic::exp2:
-      return TargetOpcode::G_FEXP2;
-    case Intrinsic::fabs:
-      return TargetOpcode::G_FABS;
-    case Intrinsic::copysign:
-      return TargetOpcode::G_FCOPYSIGN;
-    case Intrinsic::minnum:
-      return TargetOpcode::G_FMINNUM;
-    case Intrinsic::maxnum:
-      return TargetOpcode::G_FMAXNUM;
-    case Intrinsic::minimum:
-      return TargetOpcode::G_FMINIMUM;
-    case Intrinsic::maximum:
-      return TargetOpcode::G_FMAXIMUM;
-    case Intrinsic::canonicalize:
-      return TargetOpcode::G_FCANONICALIZE;
-    case Intrinsic::floor:
-      return TargetOpcode::G_FFLOOR;
-    case Intrinsic::fma:
-      return TargetOpcode::G_FMA;
-    case Intrinsic::log:
-      return TargetOpcode::G_FLOG;
-    case Intrinsic::log2:
-      return TargetOpcode::G_FLOG2;
-    case Intrinsic::log10:
-      return TargetOpcode::G_FLOG10;
-    case Intrinsic::ldexp:
-      return TargetOpcode::G_FLDEXP;
-    case Intrinsic::nearbyint:
-      return TargetOpcode::G_FNEARBYINT;
-    case Intrinsic::pow:
-      return TargetOpcode::G_FPOW;
-    case Intrinsic::powi:
-      return TargetOpcode::G_FPOWI;
-    case Intrinsic::rint:
-      return TargetOpcode::G_FRINT;
-    case Intrinsic::round:
-      return TargetOpcode::G_INTRINSIC_ROUND;
-    case Intrinsic::roundeven:
-      return TargetOpcode::G_INTRINSIC_ROUNDEVEN;
-    case Intrinsic::sin:
-      return TargetOpcode::G_FSIN;
-    case Intrinsic::sqrt:
-      return TargetOpcode::G_FSQRT;
-    case Intrinsic::trunc:
-      return TargetOpcode::G_INTRINSIC_TRUNC;
-    case Intrinsic::readcyclecounter:
-      return TargetOpcode::G_READCYCLECOUNTER;
-    case Intrinsic::ptrmask:
-      return TargetOpcode::G_PTRMASK;
-    case Intrinsic::lrint:
-      return TargetOpcode::G_INTRINSIC_LRINT;
-    // FADD/FMUL require checking the FMF, so are handled elsewhere.
-    case Intrinsic::vector_reduce_fmin:
-      return TargetOpcode::G_VECREDUCE_FMIN;
-    case Intrinsic::vector_reduce_fmax:
-      return TargetOpcode::G_VECREDUCE_FMAX;
-    case Intrinsic::vector_reduce_add:
-      return TargetOpcode::G_VECREDUCE_ADD;
-    case Intrinsic::vector_reduce_mul:
-      return TargetOpcode::G_VECREDUCE_MUL;
-    case Intrinsic::vector_reduce_and:
-      return TargetOpcode::G_VECREDUCE_AND;
-    case Intrinsic::vector_reduce_or:
-      return TargetOpcode::G_VECREDUCE_OR;
-    case Intrinsic::vector_reduce_xor:
-      return TargetOpcode::G_VECREDUCE_XOR;
-    case Intrinsic::vector_reduce_smax:
-      return TargetOpcode::G_VECREDUCE_SMAX;
-    case Intrinsic::vector_reduce_smin:
-      return TargetOpcode::G_VECREDUCE_SMIN;
-    case Intrinsic::vector_reduce_umax:
-      return TargetOpcode::G_VECREDUCE_UMAX;
-    case Intrinsic::vector_reduce_umin:
-      return TargetOpcode::G_VECREDUCE_UMIN;
-    case Intrinsic::lround:
-      return TargetOpcode::G_LROUND;
-    case Intrinsic::llround:
-      return TargetOpcode::G_LLROUND;
   }
   return Intrinsic::not_intrinsic;
 }
 
 bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI,
                                             Intrinsic::ID ID,
                                             MachineIRBuilder &MIRBuilder) {
 
   unsigned Op = getSimpleIntrinsicOpcode(ID);
 
   // Is this a simple intrinsic?
   if (Op == Intrinsic::not_intrinsic)
     return false;
 
+  if (MF->getTarget().getTargetTriple().isAArch64()) {
+    if (auto vector = dyn_cast<VectorType>(CI.getType())) {
+      if (!vector->getElementCount().isScalable() &&
+          vector->getPrimitiveSizeInBits().getFixedValue() > 128) {
+        const Function &F = MF->getFunction();
+
+        OptimizationRemarkMissed R("gisel-irtranslator", "GISelInconvenience",
+                               F.getSubprogram(), &F.getEntryBlock());
+
+        R << "size of return type (vector) exceeds 128 bit on AArch64: op=" <<
+          std::to_string(Op)
+          << " size=" <<
+          std::to_string(vector->getPrimitiveSizeInBits().getFixedValue())
+          << (" (in function: " + MF->getName() + ")").str();
+        ORE->emit(R);
+      }
+    } else if (auto integ = dyn_cast<IntegerType>(CI.getType())) {
+      if (integ->getBitWidth() > 128) {
+        const Function &F = MF->getFunction();
+
+        OptimizationRemarkMissed R("gisel-irtranslator", "GISelInconvenience",
+                               F.getSubprogram(), &F.getEntryBlock());
+
+        R << "size of return type (integer) exceeds 128 bit on AArch64: op=" <<
+          std::to_string(Op)
+          << " size=" <<
+          std::to_string(integ->getBitWidth())
+          << (" (in function: " + MF->getName() + ")").str();
+        ORE->emit(R);
+      }
+    }
+  }
+
   // Yes. Let's translate it.
   SmallVector<llvm::SrcOp, 4> VRegs;
   for (const auto &Arg : CI.args())
     VRegs.push_back(getOrCreateVReg(*Arg));
 
   MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs,
                         MachineInstr::copyFlagsFromInstruction(CI));
   return true;
static unsigned getConstrainedOpcode(Intrinsic::ID ID) {
   case Intrinsic::experimental_constrained_ldexp:
     return TargetOpcode::G_STRICT_FLDEXP;
   default:
     return 0;
   }
 }
 
 bool IRTranslator::translateConstrainedFPIntrinsic(
     const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder) {
   fp::ExceptionBehavior EB = *FPI.getExceptionBehavior();
 
   unsigned Opcode = getConstrainedOpcode(FPI.getIntrinsicID());
   if (!Opcode)
     return false;
 
   uint32_t Flags = MachineInstr::copyFlagsFromInstruction(FPI);
   if (EB == fp::ExceptionBehavior::ebIgnore)
case Intrinsic::dbg_declare: {
     MIRBuilder.buildIndirectDbgValue(getOrCreateVReg(*Address),
                                      DI.getVariable(), DI.getExpression());
     return true;
   }
   case Intrinsic::dbg_label: {
     const DbgLabelInst &DI = cast<DbgLabelInst>(CI);
     assert(DI.getLabel() && "Missing label");
 
-    assert(DI.getLabel()->isValidLocationForIntrinsic(
-               MIRBuilder.getDebugLoc()) &&
-           "Expected inlined-at fields to agree");
+    assert(
+        DI.getLabel()->isValidLocationForIntrinsic(MIRBuilder.getDebugLoc()) &&
 
     MIRBuilder.buildDbgLabel(DI.getLabel());
     return true;
   }
   case Intrinsic::vaend:
     // No target I know of cares about va_end. Certainly no in-tree target
     // does. Simplest intrinsic ever!
     return true;
bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID,
   case Intrinsic::smin:
     return translateBinaryOp(TargetOpcode::G_SMIN, CI, MIRBuilder);
   case Intrinsic::smax:
     return translateBinaryOp(TargetOpcode::G_SMAX, CI, MIRBuilder);
   case Intrinsic::abs:
     // TODO: Preserve "int min is poison" arg in GMIR?
     return translateUnaryOp(TargetOpcode::G_ABS, CI, MIRBuilder);
   case Intrinsic::smul_fix:
-    return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI, MIRBuilder);
+    return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIX, CI,
+                                        MIRBuilder);
   case Intrinsic::umul_fix:
-    return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI, MIRBuilder);
+    return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIX, CI,
+                                        MIRBuilder);
   case Intrinsic::smul_fix_sat:
-    return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI, MIRBuilder);
+    return translateFixedPointIntrinsic(TargetOpcode::G_SMULFIXSAT, CI,
+                                        MIRBuilder);
   case Intrinsic::umul_fix_sat:
-    return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI, MIRBuilder);
+    return translateFixedPointIntrinsic(TargetOpcode::G_UMULFIXSAT, CI,
+                                        MIRBuilder);
   case Intrinsic::sdiv_fix:
-    return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI, MIRBuilder);
+    return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIX, CI,
+                                        MIRBuilder);
   case Intrinsic::udiv_fix:
-    return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI, MIRBuilder);
+    return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIX, CI,
+                                        MIRBuilder);
   case Intrinsic::sdiv_fix_sat:
-    return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI, MIRBuilder);
+    return translateFixedPointIntrinsic(TargetOpcode::G_SDIVFIXSAT, CI,
+                                        MIRBuilder);
   case Intrinsic::udiv_fix_sat:
-    return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI, MIRBuilder);
+    return translateFixedPointIntrinsic(TargetOpcode::G_UDIVFIXSAT, CI,
+                                        MIRBuilder);
   case Intrinsic::fmuladd: {
     const TargetMachine &TM = MF->getTarget();
     const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering();
     Register Dst = getOrCreateVReg(CI);
     Register Op0 = getOrCreateVReg(*CI.getArgOperand(0));
     Register Op1 = getOrCreateVReg(*CI.getArgOperand(1));
     Register Op2 = getOrCreateVReg(*CI.getArgOperand(2));
     if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
case Intrinsic::stackrestore: {
 
     MIRBuilder.buildCopy(StackPtr, Reg);
     return true;
   }
   case Intrinsic::cttz:
   case Intrinsic::ctlz: {
     ConstantInt *Cst = cast<ConstantInt>(CI.getArgOperand(1));
     bool isTrailing = ID == Intrinsic::cttz;
-    unsigned Opcode = isTrailing
-                          ? Cst->isZero() ? TargetOpcode::G_CTTZ
+    unsigned Opcode = isTrailing      ? Cst->isZero()
+                                            ? TargetOpcode::G_CTTZ
                                             : TargetOpcode::G_CTTZ_ZERO_UNDEF
-                          : Cst->isZero() ? TargetOpcode::G_CTLZ
-                                          : TargetOpcode::G_CTLZ_ZERO_UNDEF;
     MIRBuilder.buildInstr(Opcode, {getOrCreateVReg(CI)},
                           {getOrCreateVReg(*CI.getArgOperand(0))});
     return true;
   }
   case Intrinsic::invariant_start: {
     LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL);
     Register Undef = MRI->createGenericVirtualRegister(PtrTy);
     MIRBuilder.buildUndef(Undef);
case Intrinsic::read_register: {
     MIRBuilder
         .buildInstr(TargetOpcode::G_READ_REGISTER, {getOrCreateVReg(CI)}, {})
         .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()));
     return true;
   }
   case Intrinsic::write_register: {
     Value *Arg = CI.getArgOperand(0);
     MIRBuilder.buildInstr(TargetOpcode::G_WRITE_REGISTER)
         .addMetadata(cast<MDNode>(cast<MetadataAsValue>(Arg)->getMetadata()))
         .addUse(getOrCreateVReg(*CI.getArgOperand(1)));
     return true;
   }
   case Intrinsic::localescape: {
     MachineBasicBlock &EntryMBB = MF->front();
     StringRef EscapedName = GlobalValue::dropLLVMManglingEscape(MF->getName());
 
     // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
     // is the same on all targets.
case Intrinsic::is_fpclass: {
 
     MIRBuilder
         .buildInstr(TargetOpcode::G_IS_FPCLASS, {getOrCreateVReg(CI)},
                     {getOrCreateVReg(*FpValue)})
         .addImm(TestMaskValue->getZExtValue());
 
     return true;
   }
-#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)  \
+#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC)                         \
   case Intrinsic::INTRINSIC:
 #include "llvm/IR/ConstrainedOps.def"
     return translateConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(CI),
                                            MIRBuilder);
-
   }
   return false;
 }
 
 bool IRTranslator::translateInlineAsm(const CallBase &CB,
                                       MachineIRBuilder &MIRBuilder) {
 
   const InlineAsmLowering *ALI = MF->getSubtarget().getInlineAsmLowering();
if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) {
 
     // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
     //       didn't yield anything useful.
     MachinePointerInfo MPI;
     if (Info.ptrVal)
       MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
     else if (Info.fallbackAddressSpace)
       MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
-    MIB.addMemOperand(
-        MF->getMachineMemOperand(MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata()));
+    MIB.addMemOperand(MF->getMachineMemOperand(MPI, Info.flags, MemTy,
+                                               Alignment, CI.getAAMetadata()));
   }
 
   return true;
 }
 
 bool IRTranslator::findUnwindDestinations(
-    const BasicBlock *EHPadBB,
-    BranchProbability Prob,
+    const BasicBlock *EHPadBB, BranchProbability Prob,
     SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
         &UnwindDests) {
   EHPersonality Personality = classifyEHPersonality(
       EHPadBB->getParent()->getFunction().getPersonalityFn());
   bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
   bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
   bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX;
   bool IsSEH = isAsynchronousEHPersonality(Personality);
bool IRTranslator::translateLandingPad(const User &U,
   // for its exception pointer and selector value. The extraction of exception
   // pointer or selector value from token type landingpads is not currently
   // supported.
   if (LP.getType()->isTokenTy())
     return true;
 
   // Add a label to mark the beginning of the landing pad.  Deletion of the
   // landing pad can thus be detected via the MachineModuleInfo.
-  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL)
-    .addSym(MF->addLandingPad(&MBB));
+  MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(MF->addLandingPad(&MBB));
 
   // If the unwinder does not preserve all registers, ensure that the
   // function marks the clobbered registers as used.
   const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
   if (auto *RegMask = TRI.getCustomEHPadPreservedMask(*MF))
     MF->getRegInfo().addPhysRegsUsedFromRegMask(RegMask);
 
   LLT Ty = getLLTForType(*LP.getType(), *DL);
bool IRTranslator::translateVAArg(const User &U, MachineIRBuilder &MIRBuilder) {
   // others). Fortunately the ABIs I know of where that matters don't use va_arg
   // anyway but that's not guaranteed.
   MIRBuilder.buildInstr(TargetOpcode::G_VAARG, {getOrCreateVReg(U)},
                         {getOrCreateVReg(*U.getOperand(0)),
                          DL->getABITypeAlign(U.getType()).value()});
   return true;
 }
 
-bool IRTranslator::translateUnreachable(const User &U, MachineIRBuilder &MIRBuilder) {
-    if (!MF->getTarget().Options.TrapUnreachable)
+bool IRTranslator::translateUnreachable(const User &U,
+                                        MachineIRBuilder &MIRBuilder) {
     return true;
 
   auto &UI = cast<UnreachableInst>(U);
   // We may be able to ignore unreachable behind a noreturn call.
   if (MF->getTarget().Options.NoTrapAfterNoreturn) {
     const BasicBlock &BB = *UI.getParent();
     if (&UI != &BB.front()) {
       BasicBlock::const_iterator PredI =
           std::prev(BasicBlock::const_iterator(UI));
       if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
         if (Call->doesNotReturn())
           return true;
       }
     }
   }
 
   MIRBuilder.buildIntrinsic(Intrinsic::trap, ArrayRef<Register>(), true);
MIRBuilder.buildAtomicRMW(
       Opcode, Res, Addr, Val,
       *MF->getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
                                 Flags, MRI->getType(Val), getMemOpAlign(I),
                                 I.getAAMetadata(), nullptr, I.getSyncScopeID(),
                                 I.getOrdering()));
   return true;
 }
 
-bool IRTranslator::translateFence(const User &U,
-                                  MachineIRBuilder &MIRBuilder) {
+bool IRTranslator::translateFence(const User &U, MachineIRBuilder &MIRBuilder) {
   const FenceInst &Fence = cast<FenceInst>(U);
   MIRBuilder.buildFence(static_cast<unsigned>(Fence.getOrdering()),
                         Fence.getSyncScopeID());
   return true;
 }
 
 bool IRTranslator::translateFreeze(const User &U,
                                    MachineIRBuilder &MIRBuilder) {
if (CV->getNumElements() == 1)
       return translateCopy(C, *CV->getElementAsConstant(0), *EntryBuilder);
     SmallVector<Register, 4> Ops;
     for (unsigned i = 0; i < CV->getNumElements(); ++i) {
       Constant &Elt = *CV->getElementAsConstant(i);
       Ops.push_back(getOrCreateVReg(Elt));
     }
     EntryBuilder->buildBuildVector(Reg, Ops);
   } else if (auto CE = dyn_cast<ConstantExpr>(&C)) {
-    switch(CE->getOpcode()) {
+    switch (CE->getOpcode()) {
 #define HANDLE_INST(NUM, OPCODE, CLASS)                                        \
   case Instruction::OPCODE:                                                    \
     return translate##OPCODE(*CE, *EntryBuilder.get());
 #include "llvm/IR/Instruction.def"
     default:
       return false;
     }
   } else if (auto CV = dyn_cast<ConstantVector>(&C)) {
bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   LibInfo = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
   FuncInfo.CanLowerReturn = CLI->checkReturnTypeForCallConv(*MF);
 
   const auto &TLI = *MF->getSubtarget().getTargetLowering();
 
   SL = std::make_unique<GISelSwitchLowering>(this, FuncInfo);
   SL->init(TLI, TM, *DL);
 
-
-
   assert(PendingPHIs.empty() && "stale PHIs");
 
   // Targets which want to use big endian can enable it using
   // enableBigEndian()
   if (!DL->isLittleEndian() && !CLI->enableBigEndian()) {
     // Currently we don't properly handle big endian code.
     OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                F.getSubprogram(), &F.getEntryBlock());
bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) {
   DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc();
   SwiftError.setFunction(CurMF);
   SwiftError.createEntriesInEntryBlock(DbgLoc);
 
   bool IsVarArg = F.isVarArg();
   bool HasMustTailInVarArgFn = false;
 
   // Create all blocks, in IR order, to preserve the layout.
-  for (const BasicBlock &BB: F) {
+  for (const BasicBlock &BB : F) {
     auto *&MBB = BBToMBB[&BB];
 
     MBB = MF->CreateMachineBasicBlock(&BB);
     MF->push_back(MBB);
 
     if (BB.hasAddressTaken())
       MBB->setAddressTakenIRBlock(const_cast<BasicBlock *>(&BB));
 
OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure",
                                F.getSubprogram(), &F.getEntryBlock());
     R << "unable to lower function: " << ore::NV("Prototype", F.getType());
     reportTranslationError(*MF, *TPC, *ORE, R);
     return false;
   }
 
   // Lower the actual args into this basic block.
   SmallVector<ArrayRef<Register>, 8> VRegArgs;
-  for (const Argument &Arg: F.args()) {
+  for (const Argument &Arg : F.args()) {
     if (DL->getTypeStoreSize(Arg.getType()).isZero())
       continue; // Don't handle zero sized types.
     ArrayRef<Register> VRegs = getOrCreateVRegs(Arg);
     VRegArgs.push_back(VRegs);
 
     if (Arg.hasSwiftErrorAttr()) {
       assert(VRegs.size() == 1 && "Too many vregs for Swift error");
       SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]);

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

case TargetOpcode::G_TRUNC: {
     }
 
     auto Unmerge = MIRBuilder.buildUnmerge(NarrowTy, MI.getOperand(1));
     MIRBuilder.buildCopy(MI.getOperand(0), Unmerge.getReg(0));
     MI.eraseFromParent();
     return Legalized;
   }
 
+  case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
   case TargetOpcode::G_FREEZE: {
     if (TypeIdx != 0)
       return UnableToLegalize;
 
     LLT Ty = MRI.getType(MI.getOperand(0).getReg());
     // Should widen scalar first
     if (Ty.getSizeInBits() % NarrowTy.getSizeInBits() != 0)
       return UnableToLegalize;
case TargetOpcode::G_BITREVERSE: {
     MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
 
     auto ShiftAmt = MIRBuilder.buildConstant(WideTy, DiffBits);
     auto Shift = MIRBuilder.buildLShr(WideTy, DstExt, ShiftAmt);
     MIRBuilder.buildTrunc(DstReg, Shift);
     Observer.changedInstr(MI);
     return Legalized;
   }
+  case TargetOpcode::G_CONSTANT_FOLD_BARRIER:
   case TargetOpcode::G_FREEZE:
     Observer.changingInstr(MI);
     widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
     widenScalarDst(MI, WideTy);
     Observer.changedInstr(MI);
     return Legalized;
 
   case TargetOpcode::G_ABS:

llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

getActionDefinitionsBuilder(G_STORE)
           {{s8, p0, s8, 8},     {s16, p0, s8, 8},  // truncstorei8 from s16
            {s32, p0, s8, 8},                       // truncstorei8 from s32
            {s64, p0, s8, 8},                       // truncstorei8 from s64
            {s16, p0, s16, 8},   {s32, p0, s16, 8}, // truncstorei16 from s32
            {s64, p0, s16, 8},                      // truncstorei16 from s64
            {s32, p0, s8, 8},    {s32, p0, s16, 8},    {s32, p0, s32, 8},
            {s64, p0, s64, 8},   {s64, p0, s32, 8}, // truncstorei32 from s64
            {p0, p0, s64, 8},    {s128, p0, s128, 8},  {v16s8, p0, s128, 8},
-           {v8s8, p0, s64, 8},  {v4s16, p0, s64, 8},  {v8s16, p0, s128, 8},
-           {v2s32, p0, s64, 8}, {v4s32, p0, s128, 8}, {v2s64, p0, s128, 8}})
+           {v8s8, p0, s64, 8},  {v4s8, p0, s32, 8},  {v4s16, p0, s64, 8},
+           {v8s16, p0, s128, 8}, {v2s32, p0, s64, 8}, {v4s32, p0, s128, 8},
+           {v2s64, p0, s128, 8}})
       .clampScalar(0, s8, s64)
       .lowerIf([=](const LegalityQuery &Query) {
         return Query.Types[0].isScalar() &&
                Query.Types[0] != Query.MMODescrs[0].MemoryTy;
       })
       // Maximum: sN * k = 128
       .clampMaxNumElements(0, s8, 16)
       .clampMaxNumElements(0, s16, 8)
auto ExtLegalFunc = [=](const LegalityQuery &Query) {
     unsigned SrcSize = SrcTy.getSizeInBits();
     if (SrcSize < 8 || !isPowerOf2_32(SrcSize))
       return false;
 
     return true;
   };
   getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
       .legalIf(ExtLegalFunc)
-      .clampScalar(0, s64, s64); // Just for s128, others are handled above.
+      .clampScalar(0, s64, s64) // Just for s128, others are handled above.
+      .clampMaxNumElements(0, s32, 4)
+      .clampMaxNumElements(0, s64, 2);
+
 
   getActionDefinitionsBuilder(G_TRUNC)
       .minScalarOrEltIf(
           [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
           0, s8)
       .customIf([=](const LegalityQuery &Query) {
         LLT DstTy = Query.Types[0];
         LLT SrcTy = Query.Types[1];

llvm/test/CodeGen/AArch64/GlobalISel/legalize-constant-fold-barrier.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-linux-gnu -O0 -run-pass=legalizer %s -o - -global-isel-abort=1 | FileCheck %s
+---
+name:            odd
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: odd
+    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 1
+    ; CHECK-NEXT: [[CONSTANT_FOLD_BARRIER:%[0-9]+]]:_(s64) = G_CONSTANT_FOLD_BARRIER [[C]]
+    ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s40) = G_TRUNC [[CONSTANT_FOLD_BARRIER]](s64)
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s40) = COPY [[TRUNC]](s40)
+    %0:_(s40) = G_CONSTANT i40 1
+    %1:_(s40) = G_CONSTANT_FOLD_BARRIER %0(s40)
+    %2:_(s40) = COPY %1(s40)
+...

llvm/test/CodeGen/AArch64/GlobalISel/legalize-sext-zext-128.mir

 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=aarch64 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
 ---
+name:            illegal_zext_4xs64
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: illegal_zext_4xs64
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(<4 x s32>) = IMPLICIT_DEF
+    %2:_(<4 x s64>) = G_ZEXT %0(<4 x s32>)
+    RET_ReallyLR
+
+...
+---
+name:            illegal_zext_8xs32
+body:             |
+  bb.1:
+    liveins: $x0, $x1
+    ; CHECK-LABEL: name: illegal_zext_8xs32
+    ; CHECK: liveins: $x0, $x1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: RET_ReallyLR
+    %0:_(<8 x s16>) = IMPLICIT_DEF
+    %2:_(<8 x s32>) = G_ZEXT %0(<8 x s16>)
+    RET_ReallyLR
+
+...
+---
 name:            narrow_sext_s128
 tracksRegLiveness: true
 body:             |
   bb.1:
     liveins: $x0, $x1
 
     ; CHECK-LABEL: name: narrow_sext_s128
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
-    ; CHECK: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s64)
-    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY]](s64), [[ASHR]](s64)
-    ; CHECK: G_STORE [[MV]](s128), [[COPY1]](p0) :: (store (s128))
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 63
+    ; CHECK-NEXT: [[ASHR:%[0-9]+]]:_(s64) = G_ASHR [[COPY]], [[C]](s64)
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY]](s64), [[ASHR]](s64)
+    ; CHECK-NEXT: G_STORE [[MV]](s128), [[COPY1]](p0) :: (store (s128))
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(s64) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s128) = G_SEXT %0(s64)
     G_STORE %2(s128), %1(p0) :: (store (s128))
     RET_ReallyLR
 
 ...
 ---
 name:            narrow_zext_s128
 tracksRegLiveness: true
 body:             |
   bb.1:
     liveins: $x0, $x1
 
     ; CHECK-LABEL: name: narrow_zext_s128
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY]](s64), [[C]](s64)
-    ; CHECK: G_STORE [[MV]](s128), [[COPY1]](p0) :: (store (s128))
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[COPY]](s64), [[C]](s64)
+    ; CHECK-NEXT: G_STORE [[MV]](s128), [[COPY1]](p0) :: (store (s128))
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(s64) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s128) = G_ZEXT %0(s64)
     G_STORE %2(s128), %1(p0) :: (store (s128))
     RET_ReallyLR
 
 ...
 ---
 name:            narrow_zext_s128_from_s32
 tracksRegLiveness: true
 body:             |
   bb.1:
     liveins: $w0, $x1
 
     ; CHECK-LABEL: name: narrow_zext_s128_from_s32
     ; CHECK: liveins: $w0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
-    ; CHECK: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[C]](s32)
-    ; CHECK: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[MV]](s64), [[C1]](s64)
-    ; CHECK: G_STORE [[MV1]](s128), [[COPY1]](p0) :: (store (s128))
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $w0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY]](s32), [[C]](s32)
+    ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[MV]](s64), [[C1]](s64)
+    ; CHECK-NEXT: G_STORE [[MV1]](s128), [[COPY1]](p0) :: (store (s128))
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(s32) = COPY $w0
     %1:_(p0) = COPY $x1
     %2:_(s128) = G_ZEXT %0(s32)
     G_STORE %2(s128), %1(p0) :: (store (s128))
     RET_ReallyLR
 
 ...
 ---
 name:            narrow_zext_s192
 tracksRegLiveness: true
 body:             |
   bb.1:
     liveins: $x0, $x1
 
     ; CHECK-LABEL: name: narrow_zext_s192
     ; CHECK: liveins: $x0, $x1
-    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
-    ; CHECK: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
-    ; CHECK: G_STORE [[COPY]](s64), [[COPY1]](p0) :: (store (s64))
-    ; CHECK: G_STORE [[C]](s64), [[COPY1]](p0) :: (store (s64))
-    ; CHECK: G_STORE [[C]](s64), [[COPY1]](p0) :: (store (s64))
-    ; CHECK: RET_ReallyLR
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+    ; CHECK-NEXT: G_STORE [[COPY]](s64), [[COPY1]](p0) :: (store (s64))
+    ; CHECK-NEXT: G_STORE [[C]](s64), [[COPY1]](p0) :: (store (s64))
+    ; CHECK-NEXT: G_STORE [[C]](s64), [[COPY1]](p0) :: (store (s64))
+    ; CHECK-NEXT: RET_ReallyLR
     %0:_(s64) = COPY $x0
     %1:_(p0) = COPY $x1
     %2:_(s192) = G_ZEXT %0(s64)
     %3:_(s64), %4:_(s64), %5:_(s64) = G_UNMERGE_VALUES %2(s192)
     G_STORE %3, %1(p0) :: (store (s64))
     G_STORE %4, %1(p0) :: (store (s64))
     G_STORE %5, %1(p0) :: (store (s64))
     RET_ReallyLR
 
 ...

llvm/test/CodeGen/AArch64/GlobalISel/legalize-store.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64-linux-gnu -O0 -run-pass=legalizer %s -o - -global-isel-abort=1 | FileCheck %s
+---
+name:            justAnotherStore
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: justAnotherStore
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s8>) = COPY $w0
+    ; CHECK-NEXT: G_STORE [[COPY1]](<4 x s8>), [[COPY]](p0) :: (store (<4 x s8>))
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(p0) = COPY [[COPY]](p0)
+    %0:_(p0) = COPY $x0
+    %1:_(<4 x s8>) = COPY $w0
+    G_STORE %1:_(<4 x s8>), %0:_(p0)  :: (store (<4 x s8>))
+    %2:_(p0) = COPY %0(p0)
+...

llvm/test/CodeGen/AArch64/arm64-subvector-extend.ll

 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    sshll.4s v0, v0, #0
 ; CHECK-NEXT:    ret
   %r = sext <4 x i16> %v0 to <4 x i32>
   ret <4 x i32> %r
 }
 
 define <8 x i32> @afunc3(<8 x i16> %v0) nounwind {
-; CHECK-LABEL: afunc3:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll2.4s v1, v0, #0
-; CHECK-NEXT:    ushll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECKDAG-LABEL: afunc3:
+; CHECKDAG:       // %bb.0:
+; CHECKDAG-NEXT:    ushll2.4s v1, v0, #0
+; CHECKDAG-NEXT:    ushll.4s v0, v0, #0
+; CHECKDAG-NEXT:    ret
+;
+; FALLBACK-LABEL: afunc3:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    mov d1, v0[1]
+; FALLBACK-NEXT:    ushll.4s v0, v0, #0
+; FALLBACK-NEXT:    ushll.4s v1, v1, #0
+; FALLBACK-NEXT:    ret
   %r = zext <8 x i16> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 define <8 x i32> @afunc4(<8 x i16> %v0) nounwind {
-; CHECK-LABEL: afunc4:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll2.4s v1, v0, #0
-; CHECK-NEXT:    sshll.4s v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECKDAG-LABEL: afunc4:
+; CHECKDAG:       // %bb.0:
+; CHECKDAG-NEXT:    sshll2.4s v1, v0, #0
+; CHECKDAG-NEXT:    sshll.4s v0, v0, #0
+; CHECKDAG-NEXT:    ret
+;
+; FALLBACK-LABEL: afunc4:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    mov d1, v0[1]
+; FALLBACK-NEXT:    sshll.4s v0, v0, #0
+; FALLBACK-NEXT:    sshll.4s v1, v1, #0
+; FALLBACK-NEXT:    ret
   %r = sext <8 x i16> %v0 to <8 x i32>
   ret <8 x i32> %r
 }
 
 define <8 x i32> @bfunc1(<8 x i8> %v0) nounwind {
 ; CHECK-LABEL: bfunc1:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ushll.8h v0, v0, #0
; CHECK-NEXT:    ret
   ret <8 x i32> %r
 }
 
 ;-----
 ; Vectors of i64.
 ;-----
 
 define <4 x i64> @zfunc1(<4 x i32> %v0) nounwind {
-; CHECK-LABEL: zfunc1:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ushll2.2d v1, v0, #0
-; CHECK-NEXT:    ushll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECKDAG-LABEL: zfunc1:
+; CHECKDAG:       // %bb.0:
+; CHECKDAG-NEXT:    ushll2.2d v1, v0, #0
+; CHECKDAG-NEXT:    ushll.2d v0, v0, #0
+; CHECKDAG-NEXT:    ret
+;
+; FALLBACK-LABEL: zfunc1:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    mov d1, v0[1]
+; FALLBACK-NEXT:    ushll.2d v0, v0, #0
+; FALLBACK-NEXT:    ushll.2d v1, v1, #0
+; FALLBACK-NEXT:    ret
   %r = zext <4 x i32> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @zfunc2(<4 x i32> %v0) nounwind {
-; CHECK-LABEL: zfunc2:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sshll2.2d v1, v0, #0
-; CHECK-NEXT:    sshll.2d v0, v0, #0
-; CHECK-NEXT:    ret
+; CHECKDAG-LABEL: zfunc2:
+; CHECKDAG:       // %bb.0:
+; CHECKDAG-NEXT:    sshll2.2d v1, v0, #0
+; CHECKDAG-NEXT:    sshll.2d v0, v0, #0
+; CHECKDAG-NEXT:    ret
+;
+; FALLBACK-LABEL: zfunc2:
+; FALLBACK:       // %bb.0:
+; FALLBACK-NEXT:    mov d1, v0[1]
+; FALLBACK-NEXT:    sshll.2d v0, v0, #0
+; FALLBACK-NEXT:    sshll.2d v1, v1, #0
+; FALLBACK-NEXT:    ret
   %r = sext <4 x i32> %v0 to <4 x i64>
   ret <4 x i64> %r
 }
 
 define <4 x i64> @bfunc3(<4 x i16> %v0) nounwind {
 ; CHECK-LABEL: bfunc3:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    ushll.4s v0, v0, #0

llvm/test/CodeGen/AArch64/arm64-vabs.ll

; CHECK-NEXT:    ret
   %tmp2 = load <2 x i32>, ptr %B
   %tmp3 = call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
   %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; FALLBACK-NOT: remark:{{.*}} sabdl2_8h
 define <8 x i16> @sabdl2_8h(ptr %A, ptr %B) nounwind {
-; DAG-LABEL:   sabdl2_8h:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr d0, [x0, #8]
-; DAG-NEXT:      ldr d1, [x1, #8]
-; DAG-NEXT:      sabdl.8h v0, v0, v1
-; DAG-NEXT:      ret
+; DAG-LABEL: sabdl2_8h:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr d0, [x0, #8]
+; DAG-NEXT:    ldr d1, [x1, #8]
+; DAG-NEXT:    sabdl.8h v0, v0, v1
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: sabdl2_8h:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
-; GISEL-NEXT:    sabdl.8h  v0, v0, v1
+; GISEL-NEXT:    sabdl.8h v0, v0, v1
 ; GISEL-NEXT:    ret
   %load1 = load <16 x i8>, ptr %A
   %load2 = load <16 x i8>, ptr %B
   %tmp1 = shufflevector <16 x i8> %load1, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %tmp2 = shufflevector <16 x i8> %load2, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %tmp3 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
   %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 ; FALLBACK-NOT: remark:{{.*}} sabdl2_4s
 define <4 x i32> @sabdl2_4s(ptr %A, ptr %B) nounwind {
-; DAG-LABEL:   sabdl2_4s:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr d0, [x0, #8]
-; DAG-NEXT:      ldr d1, [x1, #8]
-; DAG-NEXT:      sabdl.4s v0, v0, v1
-; DAG-NEXT:      ret
+; DAG-LABEL: sabdl2_4s:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr d0, [x0, #8]
+; DAG-NEXT:    ldr d1, [x1, #8]
+; DAG-NEXT:    sabdl.4s v0, v0, v1
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: sabdl2_4s:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
 ; GISEL-NEXT:    sabdl.4s v0, v0, v1
 ; GISEL-NEXT:    ret
   %load1 = load <8 x i16>, ptr %A
   %load2 = load <8 x i16>, ptr %B
   %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %tmp3 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
   %tmp4 = zext <4 x i16> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; FALLBACK-NOT: remark:{{.*}} sabdl2_2d
 define <2 x i64> @sabdl2_2d(ptr %A, ptr %B) nounwind {
-; DAG-LABEL:   sabdl2_2d:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr d0, [x0, #8]
-; DAG-NEXT:      ldr d1, [x1, #8]
-; DAG-NEXT:      sabdl.2d v0, v0, v1
-; DAG-NEXT:      ret
+; DAG-LABEL: sabdl2_2d:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr d0, [x0, #8]
+; DAG-NEXT:    ldr d1, [x1, #8]
+; DAG-NEXT:    sabdl.2d v0, v0, v1
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: sabdl2_2d:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
 ; GISEL-NEXT:    sabdl.2d v0, v0, v1
; CHECK-NEXT:    ret
   %tmp2 = load <2 x i32>, ptr %B
   %tmp3 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
   %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
 }
 
 ; FALLBACK-NOT: remark:{{.*}} uabdl2_8h
 define <8 x i16> @uabdl2_8h(ptr %A, ptr %B) nounwind {
-; DAG-LABEL:   uabdl2_8h:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr d0, [x0, #8]
-; DAG-NEXT:      ldr d1, [x1, #8]
-; DAG-NEXT:      uabdl.8h v0, v0, v1
-; DAG-NEXT:      ret
+; DAG-LABEL: uabdl2_8h:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr d0, [x0, #8]
+; DAG-NEXT:    ldr d1, [x1, #8]
+; DAG-NEXT:    uabdl.8h v0, v0, v1
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: uabdl2_8h:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
-; GISEL-NEXT:    uabdl.8h  v0, v0, v1
+; GISEL-NEXT:    uabdl.8h v0, v0, v1
 ; GISEL-NEXT:    ret
   %load1 = load <16 x i8>, ptr %A
   %load2 = load <16 x i8>, ptr %B
   %tmp1 = shufflevector <16 x i8> %load1, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %tmp2 = shufflevector <16 x i8> %load2, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
 
   %tmp3 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
   %tmp4 = zext <8 x i8> %tmp3 to <8 x i16>
   ret <8 x i16> %tmp4
 }
 
 ; FALLBACK-NOT: remark:{{.*}} uabdl2_4s
 define <4 x i32> @uabdl2_4s(ptr %A, ptr %B) nounwind {
-; DAG-LABEL:   uabdl2_4s:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr d0, [x0, #8]
-; DAG-NEXT:      ldr d1, [x1, #8]
-; DAG-NEXT:      uabdl.4s v0, v0, v1
-; DAG-NEXT:      ret
+; DAG-LABEL: uabdl2_4s:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr d0, [x0, #8]
+; DAG-NEXT:    ldr d1, [x1, #8]
+; DAG-NEXT:    uabdl.4s v0, v0, v1
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: uabdl2_4s:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
-; GISEL-NEXT:    uabdl.4s  v0, v0, v1
+; GISEL-NEXT:    uabdl.4s v0, v0, v1
 ; GISEL-NEXT:    ret
   %load1 = load <8 x i16>, ptr %A
   %load2 = load <8 x i16>, ptr %B
   %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %tmp3 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
   %tmp4 = zext <4 x i16> %tmp3 to <4 x i32>
   ret <4 x i32> %tmp4
 }
 
 ; FALLBACK-NOT: remark:{{.*}} uabdl2_2d
 define <2 x i64> @uabdl2_2d(ptr %A, ptr %B) nounwind {
-; DAG-LABEL:   uabdl2_2d:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr d0, [x0, #8]
-; DAG-NEXT:      ldr d1, [x1, #8]
-; DAG-NEXT:      uabdl.2d v0, v0, v1
-; DAG-NEXT:      ret
+; DAG-LABEL: uabdl2_2d:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr d0, [x0, #8]
+; DAG-NEXT:    ldr d1, [x1, #8]
+; DAG-NEXT:    uabdl.2d v0, v0, v1
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: uabdl2_2d:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
-; GISEL-NEXT:    uabdl.2d  v0, v0, v1
+; GISEL-NEXT:    uabdl.2d v0, v0, v1
 ; GISEL-NEXT:    ret
   %load1 = load <4 x i32>, ptr %A
   %load2 = load <4 x i32>, ptr %B
   %tmp1 = shufflevector <4 x i32> %load1, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %tmp2 = shufflevector <4 x i32> %load2, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %tmp3 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
   %tmp4 = zext <2 x i32> %tmp3 to <2 x i64>
   ret <2 x i64> %tmp4
; CHECK-NEXT:    ret
   ret i32 %reduced_v
 }
 
 
 declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
 declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
 
 define i32 @uabd8h_rdx(ptr %a, ptr %b) {
-; CHECK-LABEL: uabd8h_rdx:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    ldr q0, [x0]
-; CHECK-NEXT:    ldr q1, [x1]
-; CHECK-NEXT:    uabd.8h v0, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v0
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; DAG-LABEL: uabd8h_rdx:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr q0, [x0]
+; DAG-NEXT:    ldr q1, [x1]
+; DAG-NEXT:    uabd.8h v0, v0, v1
+; DAG-NEXT:    uaddlv.8h s0, v0
+; DAG-NEXT:    fmov w0, s0
+; DAG-NEXT:    ret
+;
+; GISEL-LABEL: uabd8h_rdx:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    ldr q1, [x0]
+; GISEL-NEXT:    ldr q2, [x1]
+; GISEL-NEXT:    movi.2d v0, #0000000000000000
+; GISEL-NEXT:    mov d3, v1[1]
+; GISEL-NEXT:    mov d4, v2[1]
+; GISEL-NEXT:    usubl.4s v1, v1, v2
+; GISEL-NEXT:    usubl.4s v2, v3, v4
+; GISEL-NEXT:    cmgt.4s v3, v0, v1
+; GISEL-NEXT:    neg.4s v4, v1
+; GISEL-NEXT:    cmgt.4s v0, v0, v2
+; GISEL-NEXT:    shl.4s v3, v3, #31
+; GISEL-NEXT:    shl.4s v0, v0, #31
+; GISEL-NEXT:    neg.4s v5, v2
+; GISEL-NEXT:    sshr.4s v3, v3, #31
+; GISEL-NEXT:    sshr.4s v0, v0, #31
+; GISEL-NEXT:    bit.16b v1, v4, v3
+; GISEL-NEXT:    bsl.16b v0, v5, v2
+; GISEL-NEXT:    add.4s v0, v1, v0
+; GISEL-NEXT:    addv.4s s0, v0
+; GISEL-NEXT:    fmov w0, s0
+; GISEL-NEXT:    ret
   %aload = load <8 x i16>, ptr %a, align 1
   %bload = load <8 x i16>, ptr %b, align 1
   %aext = zext <8 x i16> %aload to <8 x i32>
   %bext = zext <8 x i16> %bload to <8 x i32>
   %abdiff = sub nsw <8 x i32> %aext, %bext
   %abcmp = icmp slt <8 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <8 x i32> zeroinitializer, %abdiff
   %absel = select <8 x i1> %abcmp, <8 x i32> %ababs, <8 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %absel)
   ret i32 %reduced_v
 }
 
 define i32 @sabd8h_rdx(<8 x i16> %a, <8 x i16> %b) {
-; CHECK-LABEL: sabd8h_rdx:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    sabd.8h v0, v0, v1
-; CHECK-NEXT:    uaddlv.8h s0, v0
-; CHECK-NEXT:    fmov w0, s0
-; CHECK-NEXT:    ret
+; DAG-LABEL: sabd8h_rdx:
+; DAG:       // %bb.0:
+; DAG-NEXT:    sabd.8h v0, v0, v1
+; DAG-NEXT:    uaddlv.8h s0, v0
+; DAG-NEXT:    fmov w0, s0
+; DAG-NEXT:    ret
+;
+; GISEL-LABEL: sabd8h_rdx:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    mov d3, v0[1]
+; GISEL-NEXT:    mov d4, v1[1]
+; GISEL-NEXT:    movi.2d v2, #0000000000000000
+; GISEL-NEXT:    ssubl.4s v0, v0, v1
+; GISEL-NEXT:    ssubl.4s v1, v3, v4
+; GISEL-NEXT:    cmgt.4s v3, v2, v0
+; GISEL-NEXT:    neg.4s v4, v0
+; GISEL-NEXT:    cmgt.4s v2, v2, v1
+; GISEL-NEXT:    shl.4s v3, v3, #31
+; GISEL-NEXT:    shl.4s v2, v2, #31
+; GISEL-NEXT:    neg.4s v5, v1
+; GISEL-NEXT:    sshr.4s v3, v3, #31
+; GISEL-NEXT:    sshr.4s v2, v2, #31
+; GISEL-NEXT:    bit.16b v0, v4, v3
+; GISEL-NEXT:    bit.16b v1, v5, v2
+; GISEL-NEXT:    add.4s v0, v0, v1
+; GISEL-NEXT:    addv.4s s0, v0
+; GISEL-NEXT:    fmov w0, s0
+; GISEL-NEXT:    ret
   %aext = sext <8 x i16> %a to <8 x i32>
   %bext = sext <8 x i16> %b to <8 x i32>
   %abdiff = sub nsw <8 x i32> %aext, %bext
   %abcmp = icmp slt <8 x i32> %abdiff, zeroinitializer
   %ababs = sub nsw <8 x i32> zeroinitializer, %abdiff
   %absel = select <8 x i1> %abcmp, <8 x i32> %ababs, <8 x i32> %abdiff
   %reduced_v = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %absel)
   ret i32 %reduced_v
; GISEL-NEXT:    ret
   %tmp4.1 = zext <2 x i32> %tmp4 to <2 x i64>
   %tmp4.1.1 = zext <2 x i32> %tmp4 to <2 x i64>
   %tmp5 = add <2 x i64> %tmp3, %tmp4.1
   ret <2 x i64> %tmp5
 }
 
 ; FALLBACK-NOT: remark:{{.*}} sabal2_8h
 define <8 x i16> @sabal2_8h(ptr %A, ptr %B, ptr %C) nounwind {
-; DAG-LABEL:   sabal2_8h:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr q0, [x2]
-; DAG-NEXT:      ldr d1, [x0, #8]
-; DAG-NEXT:      ldr d2, [x1, #8]
-; DAG-NEXT:      sabal.8h v0, v1, v2
-; DAG-NEXT:      ret
+; DAG-LABEL: sabal2_8h:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr q0, [x2]
+; DAG-NEXT:    ldr d1, [x0, #8]
+; DAG-NEXT:    ldr d2, [x1, #8]
+; DAG-NEXT:    sabal.8h v0, v1, v2
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: sabal2_8h:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v2, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
 ; GISEL-NEXT:    ldr q0, [x2]
 ; GISEL-NEXT:    sabal.8h v0, v2, v1
 ; GISEL-NEXT:    ret
   %load1 = load <16 x i8>, ptr %A
   %load2 = load <16 x i8>, ptr %B
   %tmp3 = load <8 x i16>, ptr %C
   %tmp1 = shufflevector <16 x i8> %load1, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %tmp2 = shufflevector <16 x i8> %load2, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %tmp4 = call <8 x i8> @llvm.aarch64.neon.sabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
   %tmp4.1 = zext <8 x i8> %tmp4 to <8 x i16>
   %tmp5 = add <8 x i16> %tmp3, %tmp4.1
   ret <8 x i16> %tmp5
 }
 
 ; FALLBACK-NOT: remark:{{.*}} sabal2_4s
 define <4 x i32> @sabal2_4s(ptr %A, ptr %B, ptr %C) nounwind {
-; DAG-LABEL:   sabal2_4s:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr q0, [x2]
-; DAG-NEXT:      ldr d1, [x0, #8]
-; DAG-NEXT:      ldr d2, [x1, #8]
-; DAG-NEXT:      sabal.4s v0, v1, v2
-; DAG-NEXT:      ret
+; DAG-LABEL: sabal2_4s:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr q0, [x2]
+; DAG-NEXT:    ldr d1, [x0, #8]
+; DAG-NEXT:    ldr d2, [x1, #8]
+; DAG-NEXT:    sabal.4s v0, v1, v2
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: sabal2_4s:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v2, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
 ; GISEL-NEXT:    ldr q0, [x2]
 ; GISEL-NEXT:    sabal.4s v0, v2, v1
 ; GISEL-NEXT:    ret
   %load1 = load <8 x i16>, ptr %A
   %load2 = load <8 x i16>, ptr %B
   %tmp3 = load <4 x i32>, ptr %C
   %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %tmp4 = call <4 x i16> @llvm.aarch64.neon.sabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
   %tmp4.1 = zext <4 x i16> %tmp4 to <4 x i32>
   %tmp5 = add <4 x i32> %tmp3, %tmp4.1
   ret <4 x i32> %tmp5
 }
 
 ; FALLBACK-NOT: remark:{{.*}} sabal2_2d
 define <2 x i64> @sabal2_2d(ptr %A, ptr %B, ptr %C) nounwind {
-; DAG-LABEL:   sabal2_2d:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr q0, [x2]
-; DAG-NEXT:      ldr d1, [x0, #8]
-; DAG-NEXT:      ldr d2, [x1, #8]
-; DAG-NEXT:      sabal.2d v0, v1, v2
-; DAG-NEXT:      ret
+; DAG-LABEL: sabal2_2d:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr q0, [x2]
+; DAG-NEXT:    ldr d1, [x0, #8]
+; DAG-NEXT:    ldr d2, [x1, #8]
+; DAG-NEXT:    sabal.2d v0, v1, v2
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: sabal2_2d:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v2, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
 ; GISEL-NEXT:    ldr q0, [x2]
; GISEL-NEXT:    ret
   %tmp4 = call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %tmp1, <2 x i32> %tmp2)
   %tmp4.1 = zext <2 x i32> %tmp4 to <2 x i64>
   %tmp5 = add <2 x i64> %tmp3, %tmp4.1
   ret <2 x i64> %tmp5
 }
 
 ; FALLBACK-NOT: remark:{{.*}} uabal2_8h
 define <8 x i16> @uabal2_8h(ptr %A, ptr %B, ptr %C) nounwind {
-; DAG-LABEL:   uabal2_8h:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr q0, [x2]
-; DAG-NEXT:      ldr d1, [x0, #8]
-; DAG-NEXT:      ldr d2, [x1, #8]
-; DAG-NEXT:      uabal.8h v0, v1, v2
-; DAG-NEXT:      ret
+; DAG-LABEL: uabal2_8h:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr q0, [x2]
+; DAG-NEXT:    ldr d1, [x0, #8]
+; DAG-NEXT:    ldr d2, [x1, #8]
+; DAG-NEXT:    uabal.8h v0, v1, v2
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: uabal2_8h:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v2, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
 ; GISEL-NEXT:    ldr q0, [x2]
 ; GISEL-NEXT:    uabal.8h v0, v2, v1
 ; GISEL-NEXT:    ret
   %load1 = load <16 x i8>, ptr %A
   %load2 = load <16 x i8>, ptr %B
   %tmp3 = load <8 x i16>, ptr %C
   %tmp1 = shufflevector <16 x i8> %load1, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %tmp2 = shufflevector <16 x i8> %load2, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %tmp4 = call <8 x i8> @llvm.aarch64.neon.uabd.v8i8(<8 x i8> %tmp1, <8 x i8> %tmp2)
   %tmp4.1 = zext <8 x i8> %tmp4 to <8 x i16>
   %tmp5 = add <8 x i16> %tmp3, %tmp4.1
   ret <8 x i16> %tmp5
 }
 
 ; FALLBACK-NOT: remark:{{.*}} uabal2_4s
 define <4 x i32> @uabal2_4s(ptr %A, ptr %B, ptr %C) nounwind {
-; DAG-LABEL:   uabal2_4s:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr q0, [x2]
-; DAG-NEXT:      ldr d1, [x0, #8]
-; DAG-NEXT:      ldr d2, [x1, #8]
-; DAG-NEXT:      uabal.4s v0, v1, v2
-; DAG-NEXT:      ret
+; DAG-LABEL: uabal2_4s:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr q0, [x2]
+; DAG-NEXT:    ldr d1, [x0, #8]
+; DAG-NEXT:    ldr d2, [x1, #8]
+; DAG-NEXT:    uabal.4s v0, v1, v2
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: uabal2_4s:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v2, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
 ; GISEL-NEXT:    ldr q0, [x2]
 ; GISEL-NEXT:    uabal.4s v0, v2, v1
 ; GISEL-NEXT:    ret
   %load1 = load <8 x i16>, ptr %A
   %load2 = load <8 x i16>, ptr %B
   %tmp3 = load <4 x i32>, ptr %C
   %tmp1 = shufflevector <8 x i16> %load1, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %tmp2 = shufflevector <8 x i16> %load2, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %tmp4 = call <4 x i16> @llvm.aarch64.neon.uabd.v4i16(<4 x i16> %tmp1, <4 x i16> %tmp2)
   %tmp4.1 = zext <4 x i16> %tmp4 to <4 x i32>
   %tmp5 = add <4 x i32> %tmp3, %tmp4.1
   ret <4 x i32> %tmp5
 }
 
 ; FALLBACK-NOT: remark:{{.*}} uabal2_2d
 define <2 x i64> @uabal2_2d(ptr %A, ptr %B, ptr %C) nounwind {
-; DAG-LABEL:   uabal2_2d:
-; DAG:         // %bb.0:
-; DAG-NEXT:      ldr q0, [x2]
-; DAG-NEXT:      ldr d1, [x0, #8]
-; DAG-NEXT:      ldr d2, [x1, #8]
-; DAG-NEXT:      uabal.2d v0, v1, v2
-; DAG-NEXT:      ret
+; DAG-LABEL: uabal2_2d:
+; DAG:       // %bb.0:
+; DAG-NEXT:    ldr q0, [x2]
+; DAG-NEXT:    ldr d1, [x0, #8]
+; DAG-NEXT:    ldr d2, [x1, #8]
+; DAG-NEXT:    uabal.2d v0, v1, v2
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: uabal2_2d:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    ldr q0, [x0]
 ; GISEL-NEXT:    ldr q1, [x1]
 ; GISEL-NEXT:    ext.16b v2, v0, v0, #8
 ; GISEL-NEXT:    ext.16b v1, v1, v0, #8
 ; GISEL-NEXT:    ldr q0, [x2]
; CHECK-NEXT:    ret
   ret double %abs
 }
 
 declare float @llvm.fabs.f32(float) nounwind readnone
 declare double @llvm.fabs.f64(double) nounwind readnone
 
 ; FALLBACK-NOT: remark:{{.*}} uabdl_from_extract_dup
 define <2 x i64> @uabdl_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
-; CHECK-LABEL: uabdl_from_extract_dup:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup.2s v1, w0
+; DAG-LABEL: uabdl_from_extract_dup:
+; DAG:       // %bb.0:
+; DAG-NEXT:    dup.2s v1, w0
+; DAG-NEXT:    uabdl.2d v0, v0, v1
+; DAG-NEXT:    ret
+;
+; GISEL-LABEL: uabdl_from_extract_dup:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    dup.2s v1, w0
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #0
-; CHECK-NEXT:    uabdl.2d v0, v0, v1
-; CHECK-NEXT:    ret
+; GISEL-NEXT:    uabdl.2d v0, v0, v1
+; GISEL-NEXT:    ret
   %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
   %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
 
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
 
   %res = tail call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
   %res1 = zext <2 x i32> %res to <2 x i64>
   ret <2 x i64> %res1
 }
 
 ; FALLBACK-NOT: remark:{{.*}} uabdl2_from_extract_dup
 define <2 x i64> @uabdl2_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
-; DAG-LABEL:   uabdl2_from_extract_dup:
-; DAG:         // %bb.0:
-; DAG-NEXT:      dup.4s v1, w0
-; DAG-NEXT:      uabdl2.2d v0, v0, v1
-; DAG-NEXT:      ret
+; DAG-LABEL: uabdl2_from_extract_dup:
+; DAG:       // %bb.0:
+; DAG-NEXT:    dup.4s v1, w0
+; DAG-NEXT:    uabdl2.2d v0, v0, v1
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: uabdl2_from_extract_dup:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    dup.2s v1, w0
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #8
 ; GISEL-NEXT:    uabdl.2d v0, v0, v1
 ; GISEL-NEXT:    ret
   %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
   %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
 
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
 
   %res = tail call <2 x i32> @llvm.aarch64.neon.uabd.v2i32(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
   %res1 = zext <2 x i32> %res to <2 x i64>
   ret <2 x i64> %res1
 }
 
 ; FALLBACK-NOT: remark:{{.*}} sabdl_from_extract_dup
 define <2 x i64> @sabdl_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
-; CHECK-LABEL: sabdl_from_extract_dup:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    dup.2s v1, w0
+; DAG-LABEL: sabdl_from_extract_dup:
+; DAG:       // %bb.0:
+; DAG-NEXT:    dup.2s v1, w0
+; DAG-NEXT:    sabdl.2d v0, v0, v1
+; DAG-NEXT:    ret
+;
+; GISEL-LABEL: sabdl_from_extract_dup:
+; GISEL:       // %bb.0:
+; GISEL-NEXT:    dup.2s v1, w0
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #0
-; CHECK-NEXT:    sabdl.2d v0, v0, v1
-; CHECK-NEXT:    ret
+; GISEL-NEXT:    sabdl.2d v0, v0, v1
+; GISEL-NEXT:    ret
   %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0
   %rhsvec = insertelement <2 x i32> %rhsvec.tmp, i32 %rhs, i32 1
 
   %lhs.high = shufflevector <4 x i32> %lhs, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
 
   %res = tail call <2 x i32> @llvm.aarch64.neon.sabd.v2i32(<2 x i32> %lhs.high, <2 x i32> %rhsvec) nounwind
   %res1 = zext <2 x i32> %res to <2 x i64>
   ret <2 x i64> %res1
 }
 
 ; FALLBACK-NOT: remark:{{.*}} sabdl2_from_extract_dup
 define <2 x i64> @sabdl2_from_extract_dup(<4 x i32> %lhs, i32 %rhs) {
-; DAG-LABEL:   sabdl2_from_extract_dup:
-; DAG:         // %bb.0:
-; DAG-NEXT:      dup.4s v1, w0
-; DAG-NEXT:      sabdl2.2d v0, v0, v1
-; DAG-NEXT:      ret
+; DAG-LABEL: sabdl2_from_extract_dup:
+; DAG:       // %bb.0:
+; DAG-NEXT:    dup.4s v1, w0
+; DAG-NEXT:    sabdl2.2d v0, v0, v1
+; DAG-NEXT:    ret
 ;
 ; GISEL-LABEL: sabdl2_from_extract_dup:
 ; GISEL:       // %bb.0:
 ; GISEL-NEXT:    dup.2s v1, w0
 ; GISEL-NEXT:    ext.16b v0, v0, v0, #8
 ; GISEL-NEXT:    sabdl.2d v0, v0, v1
 ; GISEL-NEXT:    ret
   %rhsvec.tmp = insertelement <2 x i32> undef, i32 %rhs, i32 0