[GlobalISel] Translate calls to memcpy et al to G_INTRINSIC_W_SIDE_EFFECTs and legalize later.

[aemerson]

Summary

I plan on adding memcpy optimizations in the GlobalISel pipeline, but we can't do that unless we delay lowering to actual function calls. This patch changes the translator to generate G_INTRINSIC_W_SIDE_EFFECTS for these functions, and then have each target specify that using the new custom legalizer for intrinsics hook that they want it expanded it a libcall.

[arsenm]

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

1142–1143

I would expect these to be passed through

[aemerson]

Allow non-zero address spaces.

[aemerson]

Factor out the code to get the intrinsic ID from an MI into a helper in Utils.

[arsenm]

llvm/lib/CodeGen/GlobalISel/Utils.cpp

408 ↗

(On Diff #210462)

I've been using MI.getOperand(MI.getNumExplicitDefs()).getIntrinsicID() for this. I've been considering just adding getIntrinsicID() to MachineInstr directly

[aemerson]

llvm/lib/CodeGen/GlobalISel/Utils.cpp

408 ↗

(On Diff #210462)

I'll try that now.

[aemerson]

Add a getIntrinsicID() method to MachineInstr.

[aemerson]

Translate the volatile flag to an immediate instead of a G_CONSTANT.

[aemerson]

Sigh, I think we also need to pass through the alignments of the src and dest parameters. SelectionDAG does this during building using MemCpyInst::{getDestAlignment(),getSrcAlignment()} and I think we currently lose that information during translation.

[arsenm]

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

1164

Should these get 1/2 mem operands?

[aemerson]

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

1164

Maybe for the alignments? I'm not sure about the volatile flag though, it seems to apply to the whole intrinsic not to specific pointer.

[arsenm]

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

1164

Size, alignment, and IR object. I. guess you would have to propagate the volatile to both

[aemerson]

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

1164

Worth a try. However the size would have to be a lie, since the memory accessed would be potentially unknown. Perhaps that doesn't matter as it's an intrinsic and any code dealing with it would need to know not to never try to use the size info.

[arsenm]

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

1164

I guess you could preserve the size in the static case? I'm not sure if MMOs allow unknown size. Preserving the alignment and address space is important at least

[aemerson]

Store alignment, volatile and addrspace info in MMOs.

[aemerson]

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

1164

I decided to leave the size as just 1 in every case, I think it's better than sometimes having correct information.

llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h

private:
 
   /// Translate an LLVM load instruction into generic IR.
   bool translateLoad(const User &U, MachineIRBuilder &MIRBuilder);
 
   /// Translate an LLVM store instruction into generic IR.
   bool translateStore(const User &U, MachineIRBuilder &MIRBuilder);
 
   /// Translate an LLVM string intrinsic (memcpy, memset, ...).
-  bool translateMemfunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
-                        unsigned ID);
+  bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder,
+                        Intrinsic::ID ID);
 
   void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder);
 
   bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op,
                                   MachineIRBuilder &MIRBuilder);
 
   /// Helper function for translateSimpleIntrinsic.
   /// \return The generic opcode for \p IntrinsicID if \p IntrinsicID is a

llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h

 };
 
 /// Helper function that creates the given libcall.
 LegalizerHelper::LegalizeResult
 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall,
               const CallLowering::ArgInfo &Result,
               ArrayRef<CallLowering::ArgInfo> Args);
 
+/// Create a libcall to memcpy et al.
+LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder,
+                                                 MachineRegisterInfo &MRI,
+                                                 MachineInstr &MI);
+
 } // End namespace llvm.
 
 #endif

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

if (Offset != 0) {
     MIRBuilder.buildGEP(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0));
     return true;
   }
 
   MIRBuilder.buildCopy(getOrCreateVReg(U), BaseReg);
   return true;
 }
 
-bool IRTranslator::translateMemfunc(const CallInst &CI,
+bool IRTranslator::translateMemFunc(const CallInst &CI,
                                     MachineIRBuilder &MIRBuilder,
-                                    unsigned ID) {
+                                    Intrinsic::ID ID) {
 
   // If the source is undef, then just emit a nop.
   if (isa<UndefValue>(CI.getArgOperand(1))) {
     switch (ID) {
     case Intrinsic::memmove:
     case Intrinsic::memcpy:
     case Intrinsic::memset:
       return true;
     default:
       break;
     }
   }
 
+  if (ID != Intrinsic::memset) {
     LLT SizeTy = getLLTForType(*CI.getArgOperand(2)->getType(), *DL);
     Type *DstTy = CI.getArgOperand(0)->getType();
     if (cast<PointerType>(DstTy)->getAddressSpace() != 0 ||
-      SizeTy.getSizeInBits() != DL->getPointerSizeInBits(0))
+        SizeTy.getSizeInBits() != DL->getPointerSizeInBits(0)) {
+      LLVM_DEBUG(dbgs() << "Dst pointer has non-0 addrspace\n");
       return false;
-
-  SmallVector<CallLowering::ArgInfo, 8> Args;
-  for (int i = 0; i < 3; ++i) {
-    const auto &Arg = CI.getArgOperand(i);
-    Args.emplace_back(getOrCreateVReg(*Arg), Arg->getType());
     }
 
-  const char *Callee;
-  switch (ID) {
-  case Intrinsic::memmove:
-  case Intrinsic::memcpy: {
     Type *SrcTy = CI.getArgOperand(1)->getType();
-    if(cast<PointerType>(SrcTy)->getAddressSpace() != 0)
+    if (cast<PointerType>(SrcTy)->getAddressSpace() != 0) {
+      LLVM_DEBUG(dbgs() << "Src pointer has non-0 addrspace\n");
       return false;
-    Callee = ID == Intrinsic::memcpy ? "memcpy" : "memmove";
-    break;
     }
-  case Intrinsic::memset:
-    Callee = "memset";
-    break;
-  default:
-    return false;
   }
 
-  return CLI->lowerCall(MIRBuilder, CI.getCallingConv(),
-                        MachineOperand::CreateES(Callee),
-                        CallLowering::ArgInfo({0}, CI.getType()), Args);
+  ArrayRef<Register> Res;
+  auto ICall = MIRBuilder.buildIntrinsic(ID, Res, true);
+  for (const auto &Arg : CI.arg_operands())
+    ICall.addUse(getOrCreateVReg(*Arg));
+  return true;
 }
 
 void IRTranslator::getStackGuard(Register DstReg,
                                  MachineIRBuilder &MIRBuilder) {
   const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
   MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF));
   auto MIB = MIRBuilder.buildInstr(TargetOpcode::LOAD_STACK_GUARD);
   MIB.addDef(DstReg);
 
   auto &TLI = *MF->getSubtarget().getTargetLowering();
   Value *Global = TLI.getSDagStackGuard(*MF->getFunction().getParent());
   if (!Global)
     return;
 
if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
       MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Dst}, {FMul, Op2},
                             MachineInstr::copyFlagsFromInstruction(CI));
     }
     return true;
   }
   case Intrinsic::memcpy:
   case Intrinsic::memmove:
   case Intrinsic::memset:
-    return translateMemfunc(CI, MIRBuilder, ID);
+    return translateMemFunc(CI, MIRBuilder, ID);
   case Intrinsic::eh_typeid_for: {
     GlobalValue *GV = ExtractTypeInfo(CI.getArgOperand(0));
     Register Reg = getOrCreateVReg(CI);
     unsigned TypeID = MF->getTypeIDFor(GV);
     MIRBuilder.buildConstant(Reg, TypeID);
     return true;
   }
   case Intrinsic::objectsize: {

llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size,
 
   SmallVector<CallLowering::ArgInfo, 3> Args;
   for (unsigned i = 1; i < MI.getNumOperands(); i++)
     Args.push_back({MI.getOperand(i).getReg(), OpType});
   return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
                        Args);
 }
 
+LegalizerHelper::LegalizeResult
+llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
+                       MachineInstr &MI) {
+  assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
+  auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
+
+  SmallVector<CallLowering::ArgInfo, 3> Args;
+  // Collect the arguments, but ignore the last one which is the volatile flag.
+  for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) {
+    Register Reg = MI.getOperand(i).getReg();
+
+    // Need derive an IR type for call lowering.
+    LLT OpLLT = MRI.getType(Reg);
+    Type *OpTy = nullptr;
+    if (OpLLT.isPointer())
+      OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace());
+    else
+      OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits());
+    Args.push_back({Reg, OpTy});
+  }
+
+  auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
+  auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
+  Intrinsic::ID ID = MI.getOperand(0).getIntrinsicID();
+  RTLIB::Libcall RTLibcall;
+  switch (ID) {
+  case Intrinsic::memcpy:
+    RTLibcall = RTLIB::MEMCPY;
+    break;
+  case Intrinsic::memset:
+    RTLibcall = RTLIB::MEMSET;
+    break;
+  case Intrinsic::memmove:
+    RTLibcall = RTLIB::MEMMOVE;
+    break;
+  default:
+    return LegalizerHelper::UnableToLegalize;
+  }
+  const char *Name = TLI.getLibcallName(RTLibcall);
+
+  MIRBuilder.setInstr(MI);
+  MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
+  if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(RTLibcall),
+                     MachineOperand::CreateES(Name),
+                     CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx)), Args))
+    return LegalizerHelper::UnableToLegalize;
+
+  return LegalizerHelper::Legalized;
+}
+
 static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
                                        Type *FromType) {
   auto ToMVT = MVT::getVT(ToType);
   auto FromMVT = MVT::getVT(FromType);
 
   switch (Opcode) {
   case TargetOpcode::G_FPEXT:
     return RTLIB::getFPEXT(FromMVT, ToMVT);

llvm/lib/Target/AArch64/AArch64LegalizerInfo.h

 class AArch64LegalizerInfo : public LegalizerInfo {
 public:
   AArch64LegalizerInfo(const AArch64Subtarget &ST);
 
   bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI,
                       MachineIRBuilder &MIRBuilder,
                       GISelChangeObserver &Observer) const override;
 
+  bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
+                         MachineIRBuilder &MIRBuilder) const override;
+
 private:
   bool legalizeVaArg(MachineInstr &MI, MachineRegisterInfo &MRI,
                      MachineIRBuilder &MIRBuilder) const;
   bool legalizeLoadStore(MachineInstr &MI, MachineRegisterInfo &MRI,
                          MachineIRBuilder &MIRBuilder,
                          GISelChangeObserver &Observer) const;
   bool legalizeShlAshrLshr(MachineInstr &MI, MachineRegisterInfo &MRI,
                            MachineIRBuilder &MIRBuilder,
                            GISelChangeObserver &Observer) const;
 };
 } // End llvm namespace.
 #endif

llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp

 //===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
 /// This file implements the targeting of the Machinelegalizer class for
 /// AArch64.
 /// \todo This should be generated by TableGen.
 //===----------------------------------------------------------------------===//
 
 #include "AArch64LegalizerInfo.h"
 #include "AArch64Subtarget.h"
+#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Type.h"
 
bool AArch64LegalizerInfo::legalizeCustom(MachineInstr &MI,
   case TargetOpcode::G_ASHR:
   case TargetOpcode::G_LSHR:
     return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
   }
 
   llvm_unreachable("expected switch to return");
 }
 
+bool AArch64LegalizerInfo::legalizeIntrinsic(
+    MachineInstr &MI, MachineRegisterInfo &MRI,
+    MachineIRBuilder &MIRBuilder) const {
+  // We have to find the intrinsic ID by scanning.
+  unsigned IDIdx = 0;
+  for (; IDIdx < MI.getNumOperands(); ++IDIdx)
+    if (MI.getOperand(IDIdx).isIntrinsicID())
+      break;
+  assert(MI.getOperand(IDIdx).isIntrinsicID() && "Could not find intrinsic ID");
+
+  switch (MI.getOperand(IDIdx).getIntrinsicID()) {
+  case Intrinsic::memcpy:
+  case Intrinsic::memset:
+  case Intrinsic::memmove:
+    if (createMemLibcall(MIRBuilder, MRI, MI) ==
+        LegalizerHelper::UnableToLegalize)
+      return false;
+    MI.eraseFromParent();
+    return true;
+  default:
+    break;
+  }
+  return true;
+}
+
 bool AArch64LegalizerInfo::legalizeShlAshrLshr(
     MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder,
     GISelChangeObserver &Observer) const {
   assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
          MI.getOpcode() == TargetOpcode::G_LSHR ||
          MI.getOpcode() == TargetOpcode::G_SHL);
   // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the
   // imported patterns can select it later. Either way, it will be legal.

llvm/lib/Target/Mips/MipsLegalizerInfo.h

 /// This class provides legalization strategies.
 class MipsLegalizerInfo : public LegalizerInfo {
 public:
   MipsLegalizerInfo(const MipsSubtarget &ST);
 
   bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI,
                       MachineIRBuilder &MIRBuilder,
                       GISelChangeObserver &Observer) const override;
+
+  bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
+                         MachineIRBuilder &MIRBuilder) const override;
 };
 } // end namespace llvm
 #endif

llvm/lib/Target/Mips/MipsLegalizerInfo.cpp

bool MipsLegalizerInfo::legalizeCustom(MachineInstr &MI,
                                        GISelChangeObserver &Observer) const {
 
   using namespace TargetOpcode;
 
   MIRBuilder.setInstr(MI);
 
   return false;
 }
+
+bool MipsLegalizerInfo::legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
+                                          MachineIRBuilder &MIRBuilder) const {
+  // We have to find the intrinsic ID by scanning.
+  unsigned IDIdx = 0;
+  for (; IDIdx < MI.getNumOperands(); ++IDIdx)
+    if (MI.getOperand(IDIdx).isIntrinsicID())
+      break;
+  assert(MI.getOperand(IDIdx).isIntrinsicID() && "Could not find intrinsic ID");
+
+  switch (MI.getOperand(IDIdx).getIntrinsicID()) {
+  case Intrinsic::memcpy:
+  case Intrinsic::memset:
+  case Intrinsic::memmove:
+    if (createMemLibcall(MIRBuilder, MRI, MI) ==
+        LegalizerHelper::UnableToLegalize)
+      return false;
+    MI.eraseFromParent();
+    return true;
+  default:
+    break;
+  }
+  return true;
+}

llvm/lib/Target/X86/X86LegalizerInfo.h

private:
   /// Keep a reference to the X86Subtarget around so that we can
   /// make the right decision when generating code for different targets.
   const X86Subtarget &Subtarget;
   const X86TargetMachine &TM;
 
 public:
   X86LegalizerInfo(const X86Subtarget &STI, const X86TargetMachine &TM);
 
+  bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI,
+                         MachineIRBuilder &MIRBuilder) const override;
+
 private:
   void setLegalizerInfo32bit();
   void setLegalizerInfo64bit();
   void setLegalizerInfoSSE1();
   void setLegalizerInfoSSE2();
   void setLegalizerInfoSSE41();
   void setLegalizerInfoAVX();
   void setLegalizerInfoAVX2();
   void setLegalizerInfoAVX512();
   void setLegalizerInfoAVX512DQ();
   void setLegalizerInfoAVX512BW();
 };
 } // namespace llvm
 #endif

llvm/lib/Target/X86/X86LegalizerInfo.cpp

 //===- X86LegalizerInfo.cpp --------------------------------------*- C++ -*-==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 /// \file
 /// This file implements the targeting of the Machinelegalizer class for X86.
 /// \todo This should be generated by TableGen.
 //===----------------------------------------------------------------------===//
 
 #include "X86LegalizerInfo.h"
 #include "X86Subtarget.h"
 #include "X86TargetMachine.h"
+#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
 #include "llvm/CodeGen/TargetOpcodes.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/Type.h"
 
 using namespace llvm;
 using namespace TargetOpcode;
 using namespace LegalizeActions;
setLegalizeScalarToDifferentSizeStrategy(
       G_GEP, 1, widenToLargerTypesUnsupportedOtherwise);
   setLegalizeScalarToDifferentSizeStrategy(
       G_CONSTANT, 0, widenToLargerTypesAndNarrowToLargest);
 
   computeTables();
   verify(*STI.getInstrInfo());
 }
 
+bool X86LegalizerInfo::legalizeIntrinsic(MachineInstr &MI,
+                                         MachineRegisterInfo &MRI,
+                                         MachineIRBuilder &MIRBuilder) const {
+  // We have to find the intrinsic ID by scanning.
+  unsigned IDIdx = 0;
+  for (; IDIdx < MI.getNumOperands(); ++IDIdx)
+    if (MI.getOperand(IDIdx).isIntrinsicID())
+      break;
+  assert(MI.getOperand(IDIdx).isIntrinsicID() && "Could not find intrinsic ID");
+
+  switch (MI.getOperand(IDIdx).getIntrinsicID()) {
+  case Intrinsic::memcpy:
+  case Intrinsic::memset:
+  case Intrinsic::memmove:
+    if (createMemLibcall(MIRBuilder, MRI, MI) ==
+        LegalizerHelper::UnableToLegalize)
+      return false;
+    MI.eraseFromParent();
+    return true;
+  default:
+    break;
+  }
+  return true;
+}
+
 void X86LegalizerInfo::setLegalizerInfo32bit() {
 
   const LLT p0 = LLT::pointer(0, TM.getPointerSizeInBits(0));
   const LLT s1 = LLT::scalar(1);
   const LLT s8 = LLT::scalar(8);
   const LLT s16 = LLT::scalar(16);
   const LLT s32 = LLT::scalar(32);
   const LLT s64 = LLT::scalar(64);

llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll

 }
 
 declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8*, i64, i1)
 define void @test_memcpy(i8* %dst, i8* %src, i64 %size) {
 ; CHECK-LABEL: name: test_memcpy
 ; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: $x0 = COPY [[DST]]
-; CHECK: $x1 = COPY [[SRC]]
-; CHECK: $x2 = COPY [[SIZE]]
-; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64)
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
   ret void
 }
 
 declare void @llvm.memmove.p0i8.p0i8.i64(i8*, i8*, i64, i1)
 define void @test_memmove(i8* %dst, i8* %src, i64 %size) {
 ; CHECK-LABEL: name: test_memmove
 ; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: $x0 = COPY [[DST]]
-; CHECK: $x1 = COPY [[SRC]]
-; CHECK: $x2 = COPY [[SIZE]]
-; CHECK: BL &memmove, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64)
   call void @llvm.memmove.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0)
   ret void
 }
 
 declare void @llvm.memset.p0i8.i64(i8*, i8, i64, i1)
 define void @test_memset(i8* %dst, i8 %val, i64 %size) {
 ; CHECK-LABEL: name: test_memset
 ; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0
 ; CHECK: [[SRC_C:%[0-9]+]]:_(s32) = COPY $w1
 ; CHECK: [[SRC:%[0-9]+]]:_(s8) = G_TRUNC [[SRC_C]]
 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2
-; CHECK: $x0 = COPY [[DST]]
-; CHECK: [[SRC_TMP:%[0-9]+]]:_(s32) = G_ANYEXT [[SRC]]
-; CHECK: $w1 = COPY [[SRC_TMP]]
-; CHECK: $x2 = COPY [[SIZE]]
-; CHECK: BL &memset, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $w1, implicit $x2
+; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[DST]](p0), [[SRC]](s8), [[SIZE]](s64)
   call void @llvm.memset.p0i8.i64(i8* %dst, i8 %val, i64 %size, i1 0)
   ret void
 }
 
 declare i64 @llvm.objectsize.i64(i8*, i1)
 declare i32 @llvm.objectsize.i32(i8*, i1)
 define void @test_objectsize(i8* %addr0, i8* %addr1) {
 ; CHECK-LABEL: name: test_objectsize

llvm/test/CodeGen/AArch64/GlobalISel/legalize-memcpy-et-al.mir

+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -march=aarch64 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s
+---
+name:            test_memcpy
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w2, $x0, $x1
+
+    ; CHECK-LABEL: name: test_memcpy
+    ; CHECK: liveins: $w2, $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
+    ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: $x0 = COPY [[COPY]](p0)
+    ; CHECK: $x1 = COPY [[COPY1]](p0)
+    ; CHECK: $x2 = COPY [[ZEXT]](s64)
+    ; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
+    ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: RET_ReallyLR
+    %0:_(p0) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(s32) = COPY $w2
+    %4:_(s1) = G_CONSTANT i1 false
+    %3:_(s64) = G_ZEXT %2(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %3(s64), %4(s1)
+    RET_ReallyLR
+
+...
+---
+name:            test_memmove
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w2, $x0, $x1
+
+    ; CHECK-LABEL: name: test_memmove
+    ; CHECK: liveins: $w2, $x0, $x1
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
+    ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: $x0 = COPY [[COPY]](p0)
+    ; CHECK: $x1 = COPY [[COPY1]](p0)
+    ; CHECK: $x2 = COPY [[ZEXT]](s64)
+    ; CHECK: BL &memmove, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2
+    ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: RET_ReallyLR
+    %0:_(p0) = COPY $x0
+    %1:_(p0) = COPY $x1
+    %2:_(s32) = COPY $w2
+    %4:_(s1) = G_CONSTANT i1 false
+    %3:_(s64) = G_ZEXT %2(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %3(s64), %4(s1)
+    RET_ReallyLR
+
+...
+---
+name:            test_memset
+tracksRegLiveness: true
+body:             |
+  bb.1:
+    liveins: $w1, $w2, $x0
+
+    ; CHECK-LABEL: name: test_memset
+    ; CHECK: liveins: $w1, $w2, $x0
+    ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0
+    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1
+    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2
+    ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32)
+    ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: $x0 = COPY [[COPY]](p0)
+    ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
+    ; CHECK: $w1 = COPY [[COPY3]](s32)
+    ; CHECK: $x2 = COPY [[ZEXT]](s64)
+    ; CHECK: BL &memset, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $w1, implicit $x2
+    ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp
+    ; CHECK: RET_ReallyLR
+    %0:_(p0) = COPY $x0
+    %1:_(s32) = COPY $w1
+    %2:_(s32) = COPY $w2
+    %5:_(s1) = G_CONSTANT i1 false
+    %3:_(s8) = G_TRUNC %1(s32)
+    %4:_(s64) = G_ZEXT %2(s32)
+    G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %3(s8), %4(s64), %5(s1)
+    RET_ReallyLR
+
+...

llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll

 
 define void @call_symbol(i8* nocapture readonly %src, i8* nocapture %dest, i32 signext %length) {
   ; MIPS32-LABEL: name: call_symbol
   ; MIPS32: bb.1.entry:
   ; MIPS32:   liveins: $a0, $a1, $a2
   ; MIPS32:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
   ; MIPS32:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
   ; MIPS32:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
-  ; MIPS32:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
-  ; MIPS32:   $a0 = COPY [[COPY1]](p0)
-  ; MIPS32:   $a1 = COPY [[COPY]](p0)
-  ; MIPS32:   $a2 = COPY [[COPY2]](s32)
-  ; MIPS32:   JAL &memcpy, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2
-  ; MIPS32:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; MIPS32:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32), [[C]](s1)
   ; MIPS32:   RetRA
   ; MIPS32_PIC-LABEL: name: call_symbol
   ; MIPS32_PIC: bb.1.entry:
   ; MIPS32_PIC:   liveins: $a0, $a1, $a2
   ; MIPS32_PIC:   [[COPY:%[0-9]+]]:_(p0) = COPY $a0
   ; MIPS32_PIC:   [[COPY1:%[0-9]+]]:_(p0) = COPY $a1
   ; MIPS32_PIC:   [[COPY2:%[0-9]+]]:_(s32) = COPY $a2
-  ; MIPS32_PIC:   ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp
-  ; MIPS32_PIC:   $a0 = COPY [[COPY1]](p0)
-  ; MIPS32_PIC:   $a1 = COPY [[COPY]](p0)
-  ; MIPS32_PIC:   $a2 = COPY [[COPY2]](s32)
-  ; MIPS32_PIC:   JAL &memcpy, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2
-  ; MIPS32_PIC:   ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp
+  ; MIPS32_PIC:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; MIPS32_PIC:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32), [[C]](s1)
   ; MIPS32_PIC:   RetRA
 entry:
   call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 %length, i1 false)
   ret void
 }

llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll

 
 define float @test_return_f1(float %f.coerce) {
   ; ALL-LABEL: name: test_return_f1
   ; ALL: bb.1.entry:
   ; ALL:   liveins: $xmm0
   ; ALL:   [[COPY:%[0-9]+]]:_(s128) = COPY $xmm0
   ; ALL:   [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s128)
   ; ALL:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; ALL:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.f
   ; ALL:   G_STORE [[TRUNC]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %ir.coerce.dive2)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %ir.coerce.dive13)
   ; ALL:   [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s32)
   ; ALL:   $xmm0 = COPY [[ANYEXT]](s128)
   ; ALL:   RET 0, implicit $xmm0
 entry:
   %retval = alloca %struct.f1, align 4
   %f = alloca %struct.f1, align 4
   %coerce.dive = getelementptr inbounds %struct.f1, %struct.f1* %f, i32 0, i32 0

 
 define double @test_return_d1(double %d.coerce) {
   ; ALL-LABEL: name: test_return_d1
   ; ALL: bb.1.entry:
   ; ALL:   liveins: $xmm0
   ; ALL:   [[COPY:%[0-9]+]]:_(s128) = COPY $xmm0
   ; ALL:   [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[COPY]](s128)
   ; ALL:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; ALL:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.d
   ; ALL:   G_STORE [[TRUNC]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.coerce.dive2)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.coerce.dive13)
   ; ALL:   [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s64)
   ; ALL:   $xmm0 = COPY [[ANYEXT]](s128)
   ; ALL:   RET 0, implicit $xmm0
 entry:
   %retval = alloca %struct.d1, align 8
   %d = alloca %struct.d1, align 8
   %coerce.dive = getelementptr inbounds %struct.d1, %struct.d1* %d, i32 0, i32 0
define { double, double } @test_return_d2(double %d.coerce0, double %d.coerce1) {
   ; ALL-LABEL: name: test_return_d2
   ; ALL: bb.1.entry:
   ; ALL:   liveins: $xmm0, $xmm1
   ; ALL:   [[COPY:%[0-9]+]]:_(s128) = COPY $xmm0
   ; ALL:   [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[COPY]](s128)
   ; ALL:   [[COPY1:%[0-9]+]]:_(s128) = COPY $xmm1
   ; ALL:   [[TRUNC1:%[0-9]+]]:_(s64) = G_TRUNC [[COPY1]](s128)
   ; ALL:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; ALL:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.d
   ; ALL:   G_STORE [[TRUNC]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.1)
-  ; ALL:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C1]](s64)
+  ; ALL:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C2]](s64)
   ; ALL:   G_STORE [[TRUNC1]](s64), [[GEP]](p0) :: (store 8 into %ir.2)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.5)
-  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C1]](s64)
+  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C2]](s64)
   ; ALL:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.5 + 8)
   ; ALL:   [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s64)
   ; ALL:   $xmm0 = COPY [[ANYEXT]](s128)
   ; ALL:   [[ANYEXT1:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD1]](s64)
   ; ALL:   $xmm1 = COPY [[ANYEXT1]](s128)
   ; ALL:   RET 0, implicit $xmm0, implicit $xmm1
 entry:
   %retval = alloca %struct.d2, align 8

 }
 
 define i32 @test_return_i1(i32 %i.coerce) {
   ; ALL-LABEL: name: test_return_i1
   ; ALL: bb.1.entry:
   ; ALL:   liveins: $edi
   ; ALL:   [[COPY:%[0-9]+]]:_(s32) = COPY $edi
   ; ALL:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; ALL:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i
   ; ALL:   G_STORE [[COPY]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %ir.coerce.dive2)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %ir.coerce.dive13)
   ; ALL:   $eax = COPY [[LOAD]](s32)
   ; ALL:   RET 0, implicit $eax
 entry:
   %retval = alloca %struct.i1, align 4
   %i = alloca %struct.i1, align 4
   %coerce.dive = getelementptr inbounds %struct.i1, %struct.i1* %i, i32 0, i32 0
   store i32 %i.coerce, i32* %coerce.dive, align 4
   %0 = bitcast %struct.i1* %retval to i8*
   %1 = bitcast %struct.i1* %i to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %0, i8* align 4 %1, i64 4, i1 false)
   %coerce.dive1 = getelementptr inbounds %struct.i1, %struct.i1* %retval, i32 0, i32 0
   %2 = load i32, i32* %coerce.dive1, align 4
   ret i32 %2
 }
 
 define i64 @test_return_i2(i64 %i.coerce) {
   ; ALL-LABEL: name: test_return_i2
   ; ALL: bb.1.entry:
   ; ALL:   liveins: $rdi
   ; ALL:   [[COPY:%[0-9]+]]:_(s64) = COPY $rdi
   ; ALL:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; ALL:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i
   ; ALL:   G_STORE [[COPY]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.0, align 4)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.3, align 4)
   ; ALL:   $rax = COPY [[LOAD]](s64)
   ; ALL:   RET 0, implicit $rax
 entry:
   %retval = alloca %struct.i2, align 4
   %i = alloca %struct.i2, align 4
   %0 = bitcast %struct.i2* %i to i64*
   store i64 %i.coerce, i64* %0, align 4
   %1 = bitcast %struct.i2* %retval to i8*
   %2 = bitcast %struct.i2* %i to i8*
   call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %1, i8* align 4 %2, i64 8, i1 false)
   %3 = bitcast %struct.i2* %retval to i64*
   %4 = load i64, i64* %3, align 4
   ret i64 %4
 }
 
 define { i64, i32 } @test_return_i3(i64 %i.coerce0, i32 %i.coerce1) {
   ; ALL-LABEL: name: test_return_i3
   ; ALL: bb.1.entry:
   ; ALL:   liveins: $esi, $rdi
   ; ALL:   [[COPY:%[0-9]+]]:_(s64) = COPY $rdi
   ; ALL:   [[COPY1:%[0-9]+]]:_(s32) = COPY $esi
   ; ALL:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 12
+  ; ALL:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i
   ; ALL:   [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.2.coerce
   ; ALL:   [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.3.tmp
   ; ALL:   G_STORE [[COPY]](s64), [[FRAME_INDEX2]](p0) :: (store 8 into %ir.0, align 4)
-  ; ALL:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX2]], [[C1]](s64)
+  ; ALL:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX2]], [[C2]](s64)
   ; ALL:   G_STORE [[COPY1]](s32), [[GEP]](p0) :: (store 4 into %ir.1)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX2]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX3]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX1]](p0), [[FRAME_INDEX2]](p0), [[C]](s64), [[C1]](s1)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1)
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX3]](p0), [[FRAME_INDEX]](p0), [[C]](s64), [[C1]](s1)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX3]](p0) :: (load 8 from %ir.tmp)
-  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX3]], [[C1]](s64)
+  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX3]], [[C2]](s64)
   ; ALL:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p0) :: (load 4 from %ir.tmp + 8, align 8)
   ; ALL:   $rax = COPY [[LOAD]](s64)
   ; ALL:   $edx = COPY [[LOAD1]](s32)
   ; ALL:   RET 0, implicit $rax, implicit $edx
 entry:
   %retval = alloca %struct.i3, align 4
   %i = alloca %struct.i3, align 4
   %coerce = alloca { i64, i32 }, align 4

 
 define { i64, i64 } @test_return_i4(i64 %i.coerce0, i64 %i.coerce1) {
   ; ALL-LABEL: name: test_return_i4
   ; ALL: bb.1.entry:
   ; ALL:   liveins: $rdi, $rsi
   ; ALL:   [[COPY:%[0-9]+]]:_(s64) = COPY $rdi
   ; ALL:   [[COPY1:%[0-9]+]]:_(s64) = COPY $rsi
   ; ALL:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; ALL:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
   ; ALL:   [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval
   ; ALL:   [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i
   ; ALL:   G_STORE [[COPY]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.1, align 4)
-  ; ALL:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
-  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C1]](s64)
+  ; ALL:   [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
+  ; ALL:   [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C2]](s64)
   ; ALL:   G_STORE [[COPY1]](s64), [[GEP]](p0) :: (store 8 into %ir.2, align 4)
-  ; ALL:   ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
-  ; ALL:   $rdi = COPY [[FRAME_INDEX]](p0)
-  ; ALL:   $rsi = COPY [[FRAME_INDEX1]](p0)
-  ; ALL:   $rdx = COPY [[C]](s64)
-  ; ALL:   CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx
-  ; ALL:   ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp
+  ; ALL:   G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1)
   ; ALL:   [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.5, align 4)
-  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C1]](s64)
+  ; ALL:   [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C2]](s64)
   ; ALL:   [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.5 + 8, align 4)
   ; ALL:   $rax = COPY [[LOAD]](s64)
   ; ALL:   $rdx = COPY [[LOAD1]](s64)
   ; ALL:   RET 0, implicit $rax, implicit $rdx
 entry:
   %retval = alloca %struct.i4, align 4
   %i = alloca %struct.i4, align 4
   %0 = bitcast %struct.i4* %i to { i64, i64 }*