[GlobalISel] Accept multiple vregs for lowerCall's result

[rovka]

Summary

Change the interface of CallLowering::lowerCall to accept several
virtual registers for the call result, instead of just one. This is a
follow-up to D46018.

CallLowering::lowerReturn was similarly refactored in D49660 and
lowerFormalArguments in D63549.

With this change, we no longer pack the virtual registers generated for
aggregates into one big lump before delegating to the target. Therefore,
the target can decide itself whether it wants to handle them as separate
pieces or use one big register.

ARM and AArch64 have been updated to use the passed in virtual registers
directly, which means we no longer need to generate so many
merge/extract instructions.

NFCI for AMDGPU, Mips and X86.

[rovka]

More context.

[aemerson]

LGTM.

llvm/trunk/include/llvm/CodeGen/GlobalISel/CallLowering.h

virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv,
     return false;
   }
 
   /// Lower the given call instruction, including argument and return value
   /// marshalling.
   ///
   /// \p CI is the call/invoke instruction.
   ///
-  /// \p ResReg is a register where the call's return value should be stored (or
-  /// 0 if there is no return value).
+  /// \p ResRegs are the registers where the call's return value should be
+  /// stored (or 0 if there is no return value). There will be one register for
+  /// each non-aggregate type, as returned by \c computeValueLLTs.
   ///
   /// \p ArgRegs is a list of virtual registers containing each argument that
   /// needs to be passed.
   ///
   /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout
   /// parameter, and contains the vreg that the swifterror should be copied into
   /// after the call.
   ///
   /// \p GetCalleeReg is a callback to materialize a register for the callee if
   /// the target determines it cannot jump to the destination based purely on \p
   /// CI. This might be because \p CI is indirect, or because of the limited
   /// range of an immediate jump.
   ///
   /// \return true if the lowering succeeded, false otherwise.
   bool lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
-                 Register ResReg, ArrayRef<Register> ArgRegs,
+                 ArrayRef<Register> ResRegs, ArrayRef<Register> ArgRegs,
                  Register SwiftErrorVReg,
                  std::function<unsigned()> GetCalleeReg) const;
-
 };
 
 } // end namespace llvm
 
 #endif // LLVM_CODEGEN_GLOBALISEL_CALLLOWERING_H

llvm/trunk/lib/CodeGen/GlobalISel/CallLowering.cpp

 
 #define DEBUG_TYPE "call-lowering"
 
 using namespace llvm;
 
 void CallLowering::anchor() {}
 
 bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
-                             Register ResReg, ArrayRef<Register> ArgRegs,
+                             ArrayRef<Register> ResRegs,
+                             ArrayRef<Register> ArgRegs,
                              Register SwiftErrorVReg,
                              std::function<unsigned()> GetCalleeReg) const {
   auto &DL = CS.getParent()->getParent()->getParent()->getDataLayout();
 
   // First step is to marshall all the function's parameters into the correct
   // physregs and memory locations. Gather the sequence of argument types that
   // we'll pass to the assigner function.
   SmallVector<ArgInfo, 8> OrigArgs;
bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS,
   }
 
   MachineOperand Callee = MachineOperand::CreateImm(0);
   if (const Function *F = CS.getCalledFunction())
     Callee = MachineOperand::CreateGA(F, 0);
   else
     Callee = MachineOperand::CreateReg(GetCalleeReg(), false);
 
-  ArgInfo OrigRet{ResReg, CS.getType(), ISD::ArgFlagsTy{}};
+  ArgInfo OrigRet{ResRegs, CS.getType(), ISD::ArgFlagsTy{}};
   if (!OrigRet.Ty->isVoidTy())
     setArgFlags(OrigRet, AttributeList::ReturnIndex, DL, CS);
 
   return lowerCall(MIRBuilder, CS.getCallingConv(), Callee, OrigRet, OrigArgs,
                    SwiftErrorVReg);
 }
 
 template <typename FuncInfoTy>

llvm/trunk/lib/CodeGen/GlobalISel/IRTranslator.cpp

bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
   Intrinsic::ID ID = Intrinsic::not_intrinsic;
   if (F && F->isIntrinsic()) {
     ID = F->getIntrinsicID();
     if (TII && ID == Intrinsic::not_intrinsic)
       ID = static_cast<Intrinsic::ID>(TII->getIntrinsicID(F));
   }
 
   if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic) {
-    bool IsSplitType = valueIsSplit(CI);
-    Register Res = IsSplitType ? MRI->createGenericVirtualRegister(
-                                     getLLTForType(*CI.getType(), *DL))
-                               : getOrCreateVReg(CI);
+    ArrayRef<Register> Res = getOrCreateVRegs(CI);
 
     SmallVector<Register, 8> Args;
     Register SwiftErrorVReg;
     for (auto &Arg: CI.arg_operands()) {
       if (CLI->supportSwiftError() && isSwiftError(Arg)) {
         LLT Ty = getLLTForType(*Arg->getType(), *DL);
         Register InVReg = MRI->createGenericVirtualRegister(Ty);
         MIRBuilder.buildCopy(InVReg, SwiftError.getOrCreateVRegUseAt(
                                          &CI, &MIRBuilder.getMBB(), Arg));
         Args.push_back(InVReg);
         SwiftErrorVReg =
             SwiftError.getOrCreateVRegDefAt(&CI, &MIRBuilder.getMBB(), Arg);
         continue;
       }
       Args.push_back(packRegs(*Arg, MIRBuilder));
     }
 
     MF->getFrameInfo().setHasCalls(true);
     bool Success =
         CLI->lowerCall(MIRBuilder, &CI, Res, Args, SwiftErrorVReg,
                        [&]() { return getOrCreateVReg(*CI.getCalledValue()); });
 
-    if (IsSplitType)
-      unpackRegs(CI, Res, MIRBuilder);
-
     return Success;
   }
 
   assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
 
   if (translateKnownIntrinsic(CI, ID, MIRBuilder))
     return true;
 
bool IRTranslator::translateInvoke(const User &U,
   if (!isa<LandingPadInst>(EHPadBB->front()))
     return false;
 
   // Emit the actual call, bracketed by EH_LABELs so that the MF knows about
   // the region covered by the try.
   MCSymbol *BeginSymbol = Context.createTempSymbol();
   MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol);
 
-  Register Res;
+  ArrayRef<Register> Res;
   if (!I.getType()->isVoidTy())
-    Res = MRI->createGenericVirtualRegister(getLLTForType(*I.getType(), *DL));
+    Res = getOrCreateVRegs(I);
   SmallVector<Register, 8> Args;
-  Register SwiftErrorVReg;
+  Register SwiftErrorVReg = 0;
   for (auto &Arg : I.arg_operands()) {
     if (CLI->supportSwiftError() && isSwiftError(Arg)) {
       LLT Ty = getLLTForType(*Arg->getType(), *DL);
       Register InVReg = MRI->createGenericVirtualRegister(Ty);
       MIRBuilder.buildCopy(InVReg, SwiftError.getOrCreateVRegUseAt(
                                        &I, &MIRBuilder.getMBB(), Arg));
       Args.push_back(InVReg);
       SwiftErrorVReg =
           SwiftError.getOrCreateVRegDefAt(&I, &MIRBuilder.getMBB(), Arg);
       continue;
     }
 
     Args.push_back(packRegs(*Arg, MIRBuilder));
   }
 
   if (!CLI->lowerCall(MIRBuilder, &I, Res, Args, SwiftErrorVReg,
                       [&]() { return getOrCreateVReg(*I.getCalledValue()); }))
     return false;
 
-  unpackRegs(I, Res, MIRBuilder);
-
   MCSymbol *EndSymbol = Context.createTempSymbol();
   MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol);
 
   // FIXME: track probabilities.
   MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB),
                     &ReturnMBB = getMBB(*ReturnBB);
   MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol);
   MIRBuilder.getMBB().addSuccessor(&ReturnMBB);

llvm/trunk/lib/Target/AArch64/AArch64CallLowering.cpp

if (Callee.isReg())
     MIB->getOperand(0).setReg(constrainOperandRegClass(
         MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
         *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Callee, 0));
 
   // Finally we can copy the returned value back into its virtual-register. In
   // symmetry with the arugments, the physical register must be an
   // implicit-define of the call instruction.
   CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(F.getCallingConv());
-  assert(OrigRet.Regs.size() == 1 && "Can't handle multple regs yet");
-  if (OrigRet.Regs[0]) {
+  if (!OrigRet.Ty->isVoidTy()) {
     SplitArgs.clear();
 
-    SmallVector<uint64_t, 8> RegOffsets;
-    SmallVector<Register, 8> SplitRegs;
     splitToValueTypes(OrigRet, SplitArgs, DL, MRI, F.getCallingConv(),
                       [&](unsigned Reg, uint64_t Offset) {
-                        RegOffsets.push_back(Offset);
-                        SplitRegs.push_back(Reg);
+                        llvm_unreachable(
+                            "Call results should already be split");
                       });
 
     CallReturnHandler Handler(MIRBuilder, MRI, MIB, RetAssignFn);
     if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
       return false;
-
-    if (!RegOffsets.empty())
-      MIRBuilder.buildSequence(OrigRet.Regs[0], SplitRegs, RegOffsets);
   }
 
   if (SwiftErrorVReg) {
     MIB.addDef(AArch64::X21, RegState::Implicit);
     MIRBuilder.buildCopy(SwiftErrorVReg, Register(AArch64::X21));
   }
 
   CallSeqStart.addImm(Handler.StackSize).addImm(0);
   MIRBuilder.buildInstr(AArch64::ADJCALLSTACKUP)
       .addImm(Handler.StackSize)
       .addImm(0);
 
   return true;
 }

llvm/trunk/lib/Target/ARM/ARMCallLowering.cpp

bool ARMCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
   // Now we can add the actual call instruction to the correct basic block.
   MIRBuilder.insertInstr(MIB);
 
   if (!OrigRet.Ty->isVoidTy()) {
     if (!isSupportedType(DL, TLI, OrigRet.Ty))
       return false;
 
     ArgInfos.clear();
-    SmallVector<Register, 8> SplitRegs;
-    splitToValueTypes(OrigRet, ArgInfos, MF,
-                      [&](Register Reg) { SplitRegs.push_back(Reg); });
+    splitToValueTypes(OrigRet, ArgInfos, MF, [&](Register Reg) {
+      llvm_unreachable("Call results should already be split");
+    });
 
     auto RetAssignFn = TLI.CCAssignFnForReturn(CallConv, IsVarArg);
     CallReturnHandler RetHandler(MIRBuilder, MRI, MIB, RetAssignFn);
     if (!handleAssignments(MIRBuilder, ArgInfos, RetHandler))
       return false;
-
-    if (!SplitRegs.empty()) {
-      // We have split the value and allocated each individual piece, now build
-      // it up again.
-      assert(OrigRet.Regs.size() == 1 && "Can't handle multple regs yet");
-      MIRBuilder.buildMerge(OrigRet.Regs[0], SplitRegs);
-    }
   }
 
   // We now know the size of the stack - update the ADJCALLSTACKDOWN
   // accordingly.
   CallSeqStart.addImm(ArgHandler.StackSize).addImm(0).add(predOps(ARMCC::AL));
 
   MIRBuilder.buildInstr(ARM::ADJCALLSTACKUP)
       .addImm(ArgHandler.StackSize)
       .addImm(0)
       .add(predOps(ARMCC::AL));
 
   return true;
 }

llvm/trunk/lib/Target/ARM/ARMLegalizerInfo.cpp

bool ARMLegalizerInfo::legalizeCustom(MachineInstr &MI,
   MIRBuilder.setInstr(MI);
   LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext();
 
   switch (MI.getOpcode()) {
   default:
     return false;
   case G_SREM:
   case G_UREM: {
-    unsigned OriginalResult = MI.getOperand(0).getReg();
+    Register OriginalResult = MI.getOperand(0).getReg();
     auto Size = MRI.getType(OriginalResult).getSizeInBits();
     if (Size != 32)
       return false;
 
     auto Libcall =
         MI.getOpcode() == G_SREM ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32;
 
     // Our divmod libcalls return a struct containing the quotient and the
-    // remainder. We need to create a virtual register for it.
+    // remainder. Create a new, unused register for the quotient and use the
+    // destination of the original instruction for the remainder.
     Type *ArgTy = Type::getInt32Ty(Ctx);
     StructType *RetTy = StructType::get(Ctx, {ArgTy, ArgTy}, /* Packed */ true);
-    auto RetVal = MRI.createGenericVirtualRegister(
-        getLLTForType(*RetTy, MIRBuilder.getMF().getDataLayout()));
-
-    auto Status = createLibcall(MIRBuilder, Libcall, {RetVal, RetTy},
+    Register RetRegs[] = {MRI.createGenericVirtualRegister(LLT::scalar(32)),
+                          OriginalResult};
+    auto Status = createLibcall(MIRBuilder, Libcall, {RetRegs, RetTy},
                                 {{MI.getOperand(1).getReg(), ArgTy},
                                  {MI.getOperand(2).getReg(), ArgTy}});
     if (Status != LegalizerHelper::Legalized)
       return false;
-
-    // The remainder is the second result of divmod. Split the return value into
-    // a new, unused register for the quotient and the destination of the
-    // original instruction for the remainder.
-    MIRBuilder.buildUnmerge(
-        {MRI.createGenericVirtualRegister(LLT::scalar(32)), OriginalResult},
-        RetVal);
     break;
   }
   case G_FCMP: {
     assert(MRI.getType(MI.getOperand(2).getReg()) ==
                MRI.getType(MI.getOperand(3).getReg()) &&
            "Mismatched operands for G_FCMP");
     auto OpSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits();
 

llvm/trunk/lib/Target/Mips/MipsCallLowering.cpp

bool MipsCallLowering::lowerCall(MachineIRBuilder &MIRBuilder,
 
   for (auto &Arg : OrigArgs) {
     if (!isSupportedType(Arg.Ty))
       return false;
     if (Arg.Flags.isByVal() || Arg.Flags.isSRet())
       return false;
   }
 
-  assert(OrigRet.Regs.size() == 1 && "Can't handle multple regs yet");
   if (OrigRet.Regs[0] && !isSupportedType(OrigRet.Ty))
     return false;
 
   MachineFunction &MF = MIRBuilder.getMF();
   const Function &F = MF.getFunction();
   const MipsTargetLowering &TLI = *getTLI<MipsTargetLowering>();
   const MipsTargetMachine &TM =
       static_cast<const MipsTargetMachine &>(MF.getTarget());

llvm/trunk/lib/Target/X86/X86CallLowering.cpp

MIB->getOperand(0).setReg(constrainOperandRegClass(
         MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(),
         *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), Callee, 0));
 
   // Finally we can copy the returned value back into its virtual-register. In
   // symmetry with the arguments, the physical register must be an
   // implicit-define of the call instruction.
 
   if (!OrigRet.Ty->isVoidTy()) {
-    assert(OrigRet.Regs.size() == 1 && "Can't handle multple regs yet");
+    if (OrigRet.Regs.size() > 1)
+      return false;
 
     SplitArgs.clear();
     SmallVector<Register, 8> NewRegs;
 
     if (!splitToValueTypes(OrigRet, SplitArgs, DL, MRI,
                            [&](ArrayRef<Register> Regs) {
                              NewRegs.assign(Regs.begin(), Regs.end());
                            }))

llvm/trunk/test/CodeGen/AArch64/GlobalISel/call-translator.ll

 ; CHECK: $x1 = COPY [[E1]](s64)
 ; CHECK: $x2 = COPY [[E2]](s64)
 ; CHECK: $x3 = COPY [[E3]](s64)
 ; CHECK: BL @arr_callee, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2, implicit $x3, implicit-def $x0, implicit-def $x1, implicit-def $x2, implicit-def $x3
 ; CHECK: [[E0:%[0-9]+]]:_(s64) = COPY $x0
 ; CHECK: [[E1:%[0-9]+]]:_(s64) = COPY $x1
 ; CHECK: [[E2:%[0-9]+]]:_(s64) = COPY $x2
 ; CHECK: [[E3:%[0-9]+]]:_(s64) = COPY $x3
-; CHECK: [[RES:%[0-9]+]]:_(s256) = G_MERGE_VALUES [[E0]](s64), [[E1]](s64), [[E2]](s64), [[E3]](s64)
-; CHECK: G_EXTRACT [[RES]](s256), 64
+; CHECK: $x0 = COPY [[E1]]
 declare [4 x i64] @arr_callee([4 x i64])
 define i64 @test_arr_call([4 x i64]* %addr) {
   %arg = load [4 x i64], [4 x i64]* %addr
   %res = call [4 x i64] @arr_callee([4 x i64] %arg)
   %val = extractvalue [4 x i64] %res, 1
   ret i64 %val
 }
 

llvm/trunk/test/CodeGen/ARM/GlobalISel/arm-param-lowering.ll

 ; CHECK: [[R0:%[0-9]+]]:_(s32), [[R1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[INS2]](s64)
 ; CHECK: $r0 = COPY [[R0]]
 ; CHECK: $r1 = COPY [[R1]]
 ; ARM: BL @tiny_int_arrays_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
 ; THUMB: tBL 14, $noreg, @tiny_int_arrays_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
 ; CHECK: [[R0:%[0-9]+]]:_(s32) = COPY $r0
 ; CHECK: [[R1:%[0-9]+]]:_(s32) = COPY $r1
 ; CHECK: [[R2:%[0-9]+]]:_(s32) = COPY $r2
-; CHECK: [[RES_ARR:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[R0]](s32), [[R1]](s32), [[R2]](s32)
 ; CHECK: ADJCALLSTACKUP 0, 0, 14, $noreg, implicit-def $sp, implicit $sp
-; CHECK: [[EXT3:%[0-9]+]]:_(s32) = G_EXTRACT [[RES_ARR]](s96), 0
-; CHECK: [[EXT4:%[0-9]+]]:_(s32) = G_EXTRACT [[RES_ARR]](s96), 32
-; CHECK: [[EXT5:%[0-9]+]]:_(s32) = G_EXTRACT [[RES_ARR]](s96), 64
 ; FIXME: This doesn't seem correct with regard to the AAPCS docs (which say
 ; that composite types larger than 4 bytes should be passed through memory),
 ; but it's what DAGISel does. We should fix it in the common code for both.
-; CHECK: $r0 = COPY [[EXT3]]
-; CHECK: $r1 = COPY [[EXT4]]
-; CHECK: $r2 = COPY [[EXT5]]
+; CHECK: $r0 = COPY [[R0]]
+; CHECK: $r1 = COPY [[R1]]
+; CHECK: $r2 = COPY [[R2]]
 ; ARM: BX_RET 14, $noreg, implicit $r0, implicit $r1, implicit $r2
 ; THUMB: tBX_RET 14, $noreg, implicit $r0, implicit $r1, implicit $r2
 entry:
   %r = notail call arm_aapcscc [3 x i32] @tiny_int_arrays_target([2 x i32] %arr)
   ret [3 x i32] %r
 }
 
 declare arm_aapcscc void @multiple_int_arrays_target([2 x i32], [2 x i32])

 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[ARR2_OFFSET:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
 ; CHECK: [[ARR2_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[ARR2_OFFSET]](s32)
 ; CHECK: G_STORE [[ARR2]](s64), [[ARR2_ADDR]](p0){{.*}}store 8
 ; ARM: BL @fp_arrays_aapcs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
 ; THUMB: tBL 14, $noreg, @fp_arrays_aapcs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
 ; CHECK: [[R0:%[0-9]+]]:_(s32) = COPY $r0
 ; CHECK: [[R1:%[0-9]+]]:_(s32) = COPY $r1
-; CHECK: [[R_MERGED:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[R0]](s32), [[R1]](s32)
 ; CHECK: ADJCALLSTACKUP 8, 0, 14, $noreg, implicit-def $sp, implicit $sp
-; CHECK: [[EXT4:%[0-9]+]]:_(s32) = G_EXTRACT [[R_MERGED]](s64), 0
-; CHECK: [[EXT5:%[0-9]+]]:_(s32) = G_EXTRACT [[R_MERGED]](s64), 32
-; CHECK: $r0 = COPY [[EXT4]]
-; CHECK: $r1 = COPY [[EXT5]]
+; CHECK: $r0 = COPY [[R0]]
+; CHECK: $r1 = COPY [[R1]]
 ; ARM: BX_RET 14, $noreg, implicit $r0, implicit $r1
 ; THUMB: tBX_RET 14, $noreg, implicit $r0, implicit $r1
 entry:
   %r = notail call arm_aapcscc [2 x float] @fp_arrays_aapcs_target([3 x double] %arr)
   ret [2 x float] %r
 }
 
 declare arm_aapcs_vfpcc [4 x float] @fp_arrays_aapcs_vfp_target([3 x double], [3 x float], [4 x double])

 ; CHECK: [[Z3_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[Z3_OFFSET]](s32)
 ; CHECK: G_STORE [[Z3]](s64), [[Z3_ADDR]](p0){{.*}}store 8
 ; ARM: BL @fp_arrays_aapcs_vfp_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $d0, implicit $d1, implicit $d2, implicit $s6, implicit $s7, implicit $s8, implicit-def $s0, implicit-def $s1, implicit-def $s2, implicit-def $s3
 ; THUMB: tBL 14, $noreg, @fp_arrays_aapcs_vfp_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $d0, implicit $d1, implicit $d2, implicit $s6, implicit $s7, implicit $s8, implicit-def $s0, implicit-def $s1, implicit-def $s2, implicit-def $s3
 ; CHECK: [[R0:%[0-9]+]]:_(s32) = COPY $s0
 ; CHECK: [[R1:%[0-9]+]]:_(s32) = COPY $s1
 ; CHECK: [[R2:%[0-9]+]]:_(s32) = COPY $s2
 ; CHECK: [[R3:%[0-9]+]]:_(s32) = COPY $s3
-; CHECK: [[R_MERGED:%[0-9]+]]:_(s128) = G_MERGE_VALUES [[R0]](s32), [[R1]](s32), [[R2]](s32), [[R3]](s32)
 ; CHECK: ADJCALLSTACKUP 32, 0, 14, $noreg, implicit-def $sp, implicit $sp
-; CHECK: [[EXT11:%[0-9]+]]:_(s32) = G_EXTRACT [[R_MERGED]](s128), 0
-; CHECK: [[EXT12:%[0-9]+]]:_(s32) = G_EXTRACT [[R_MERGED]](s128), 32
-; CHECK: [[EXT13:%[0-9]+]]:_(s32) = G_EXTRACT [[R_MERGED]](s128), 64
-; CHECK: [[EXT14:%[0-9]+]]:_(s32) = G_EXTRACT [[R_MERGED]](s128), 96
-; CHECK: $s0 = COPY [[EXT11]]
-; CHECK: $s1 = COPY [[EXT12]]
-; CHECK: $s2 = COPY [[EXT13]]
-; CHECK: $s3 = COPY [[EXT14]]
+; CHECK: $s0 = COPY [[R0]]
+; CHECK: $s1 = COPY [[R1]]
+; CHECK: $s2 = COPY [[R2]]
+; CHECK: $s3 = COPY [[R3]]
 ; ARM: BX_RET 14, $noreg, implicit $s0, implicit $s1, implicit $s2, implicit $s3
 ; THUMB: tBX_RET 14, $noreg, implicit $s0, implicit $s1, implicit $s2, implicit $s3
 entry:
   %r = notail call arm_aapcs_vfpcc [4 x float] @fp_arrays_aapcs_vfp_target([3 x double] %x, [3 x float] %y, [4 x double] %z)
   ret [4 x float] %r
 }
 
 declare arm_aapcscc [2 x i32*] @tough_arrays_target([6 x [4 x i32]] %arr)

 ; Match the second-to-last offset, so we can get the correct SP for the last element
 ; CHECK: G_CONSTANT i32 72
 ; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp
 ; CHECK: [[OFF_LAST_ELEMENT:%[0-9]+]]:_(s32) = G_CONSTANT i32 76
 ; CHECK: [[LAST_STACK_ARG_ADDR:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[OFF_LAST_ELEMENT]](s32)
 ; CHECK: G_STORE [[LAST_STACK_ELEMENT]](s32), [[LAST_STACK_ARG_ADDR]]{{.*}}store 4
 ; ARM: BL @tough_arrays_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
 ; THUMB: tBL 14, $noreg, @tough_arrays_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit $r2, implicit $r3, implicit-def $r0, implicit-def $r1
-; CHECK: [[R0:%[0-9]+]]:_(s32) = COPY $r0
-; CHECK: [[R1:%[0-9]+]]:_(s32) = COPY $r1
-; CHECK: [[RES_ARR:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[R0]](s32), [[R1]](s32)
+; CHECK: [[R0:%[0-9]+]]:_(p0) = COPY $r0
+; CHECK: [[R1:%[0-9]+]]:_(p0) = COPY $r1
 ; CHECK: ADJCALLSTACKUP 80, 0, 14, $noreg, implicit-def $sp, implicit $sp
-; CHECK: [[EXT1:%[0-9]+]]:_(p0) = G_EXTRACT [[RES_ARR]](s64), 0
-; CHECK: [[EXT2:%[0-9]+]]:_(p0) = G_EXTRACT [[RES_ARR]](s64), 32
-; CHECK: $r0 = COPY [[EXT1]]
-; CHECK: $r1 = COPY [[EXT2]]
+; CHECK: $r0 = COPY [[R0]]
+; CHECK: $r1 = COPY [[R1]]
 ; ARM: BX_RET 14, $noreg, implicit $r0, implicit $r1
 ; THUMB: tBX_RET 14, $noreg, implicit $r0, implicit $r1
 entry:
   %r = notail call arm_aapcscc [2 x i32*] @tough_arrays_target([6 x [4 x i32]] %arr)
   ret [2 x i32*] %r
 }
 
 declare arm_aapcscc {i32, i32} @structs_target({i32, i32})

 ; CHECK: ADJCALLSTACKDOWN 0, 0, 14, $noreg, implicit-def $sp, implicit $sp
 ; CHECK: [[X0:%[0-9]+]]:_(s32), [[X1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[INS2]](s64)
 ; CHECK-DAG: $r0 = COPY [[X0]](s32)
 ; CHECK-DAG: $r1 = COPY [[X1]](s32)
 ; ARM: BL @structs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
 ; THUMB: tBL 14, $noreg, @structs_target, csr_aapcs, implicit-def $lr, implicit $sp, implicit $r0, implicit $r1, implicit-def $r0, implicit-def $r1
 ; CHECK: [[R0:%[0-9]+]]:_(s32) = COPY $r0
 ; CHECK: [[R1:%[0-9]+]]:_(s32) = COPY $r1
-; CHECK: [[R:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[R0]](s32), [[R1]](s32)
 ; CHECK: ADJCALLSTACKUP 0, 0, 14, $noreg, implicit-def $sp, implicit $sp
-; CHECK: [[EXT3:%[0-9]+]]:_(s32) = G_EXTRACT [[R]](s64), 0
-; CHECK: [[EXT4:%[0-9]+]]:_(s32) = G_EXTRACT [[R]](s64), 32
-; CHECK: $r0 = COPY [[EXT3]](s32)
-; CHECK: $r1 = COPY [[EXT4]](s32)
+; CHECK: $r0 = COPY [[R0]](s32)
+; CHECK: $r1 = COPY [[R1]](s32)
 ; ARM: BX_RET 14, $noreg, implicit $r0, implicit $r1
 ; THUMB: tBX_RET 14, $noreg, implicit $r0, implicit $r1
   %r = notail call arm_aapcscc {i32, i32} @structs_target({i32, i32} %x)
   ret {i32, i32} %r
 }