diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -213,8 +213,8 @@ bool translateStore(const User &U, MachineIRBuilder &MIRBuilder); /// Translate an LLVM string intrinsic (memcpy, memset, ...). - bool translateMemfunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, - unsigned ID); + bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, + Intrinsic::ID ID); void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -236,6 +236,11 @@ const CallLowering::ArgInfo &Result, ArrayRef Args); +/// Create a libcall to memcpy et al. +LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder, + MachineRegisterInfo &MRI, + MachineInstr &MI); + } // End namespace llvm. #endif diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1119,9 +1119,9 @@ return true; } -bool IRTranslator::translateMemfunc(const CallInst &CI, +bool IRTranslator::translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, - unsigned ID) { + Intrinsic::ID ID) { // If the source is undef, then just emit a nop. if (isa(CI.getArgOperand(1))) { @@ -1135,38 +1135,27 @@ } } - LLT SizeTy = getLLTForType(*CI.getArgOperand(2)->getType(), *DL); - Type *DstTy = CI.getArgOperand(0)->getType(); - if (cast(DstTy)->getAddressSpace() != 0 || - SizeTy.getSizeInBits() != DL->getPointerSizeInBits(0)) - return false; - - SmallVector Args; - for (int i = 0; i < 3; ++i) { - const auto &Arg = CI.getArgOperand(i); - Args.emplace_back(getOrCreateVReg(*Arg), Arg->getType()); - } + if (ID != Intrinsic::memset) { + LLT SizeTy = getLLTForType(*CI.getArgOperand(2)->getType(), *DL); + Type *DstTy = CI.getArgOperand(0)->getType(); + if (cast(DstTy)->getAddressSpace() != 0 || + SizeTy.getSizeInBits() != DL->getPointerSizeInBits(0)) { + LLVM_DEBUG(dbgs() << "Dst pointer has non-0 addrspace\n"); + return false; + } - const char *Callee; - switch (ID) { - case Intrinsic::memmove: - case Intrinsic::memcpy: { Type *SrcTy = CI.getArgOperand(1)->getType(); - if(cast(SrcTy)->getAddressSpace() != 0) + if (cast(SrcTy)->getAddressSpace() != 0) { + LLVM_DEBUG(dbgs() << "Src pointer has non-0 addrspace\n"); return false; - Callee = ID == Intrinsic::memcpy ? "memcpy" : "memmove"; - break; - } - case Intrinsic::memset: - Callee = "memset"; - break; - default: - return false; + } } - return CLI->lowerCall(MIRBuilder, CI.getCallingConv(), - MachineOperand::CreateES(Callee), - CallLowering::ArgInfo({0}, CI.getType()), Args); + ArrayRef Res; + auto ICall = MIRBuilder.buildIntrinsic(ID, Res, true); + for (const auto &Arg : CI.arg_operands()) + ICall.addUse(getOrCreateVReg(*Arg)); + return true; } void IRTranslator::getStackGuard(Register DstReg, @@ -1433,7 +1422,7 @@ case Intrinsic::memcpy: case Intrinsic::memmove: case Intrinsic::memset: - return translateMemfunc(CI, MIRBuilder, ID); + return translateMemFunc(CI, MIRBuilder, ID); case Intrinsic::eh_typeid_for: { GlobalValue *GV = ExtractTypeInfo(CI.getArgOperand(0)); Register Reg = getOrCreateVReg(CI); diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -317,6 +317,56 @@ Args); } +LegalizerHelper::LegalizeResult +llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, + MachineInstr &MI) { + assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS); + auto &Ctx = MIRBuilder.getMF().getFunction().getContext(); + + SmallVector Args; + // Collect the arguments, but ignore the last one which is the volatile flag. + for (unsigned i = 1; i < MI.getNumOperands() - 1; i++) { + Register Reg = MI.getOperand(i).getReg(); + + // Need derive an IR type for call lowering. + LLT OpLLT = MRI.getType(Reg); + Type *OpTy = nullptr; + if (OpLLT.isPointer()) + OpTy = Type::getInt8PtrTy(Ctx, OpLLT.getAddressSpace()); + else + OpTy = IntegerType::get(Ctx, OpLLT.getSizeInBits()); + Args.push_back({Reg, OpTy}); + } + + auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering(); + auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering(); + Intrinsic::ID ID = MI.getOperand(0).getIntrinsicID(); + RTLIB::Libcall RTLibcall; + switch (ID) { + case Intrinsic::memcpy: + RTLibcall = RTLIB::MEMCPY; + break; + case Intrinsic::memset: + RTLibcall = RTLIB::MEMSET; + break; + case Intrinsic::memmove: + RTLibcall = RTLIB::MEMMOVE; + break; + default: + return LegalizerHelper::UnableToLegalize; + } + const char *Name = TLI.getLibcallName(RTLibcall); + + MIRBuilder.setInstr(MI); + MIRBuilder.getMF().getFrameInfo().setHasCalls(true); + if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(RTLibcall), + MachineOperand::CreateES(Name), + CallLowering::ArgInfo({0}, Type::getVoidTy(Ctx)), Args)) + return LegalizerHelper::UnableToLegalize; + + return LegalizerHelper::Legalized; +} + static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, Type *FromType) { auto ToMVT = MVT::getVT(ToType); diff --git a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.h --- a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.h +++ b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.h @@ -31,6 +31,9 @@ MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const override; + bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder) const override; + private: bool legalizeVaArg(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder) const; diff --git a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp --- a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp @@ -13,6 +13,7 @@ #include "AArch64LegalizerInfo.h" #include "AArch64Subtarget.h" +#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -617,6 +618,31 @@ llvm_unreachable("expected switch to return"); } +bool AArch64LegalizerInfo::legalizeIntrinsic( + MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder) const { + // We have to find the intrinsic ID by scanning. + unsigned IDIdx = 0; + for (; IDIdx < MI.getNumOperands(); ++IDIdx) + if (MI.getOperand(IDIdx).isIntrinsicID()) + break; + assert(MI.getOperand(IDIdx).isIntrinsicID() && "Could not find intrinsic ID"); + + switch (MI.getOperand(IDIdx).getIntrinsicID()) { + case Intrinsic::memcpy: + case Intrinsic::memset: + case Intrinsic::memmove: + if (createMemLibcall(MIRBuilder, MRI, MI) == + LegalizerHelper::UnableToLegalize) + return false; + MI.eraseFromParent(); + return true; + default: + break; + } + return true; +} + bool AArch64LegalizerInfo::legalizeShlAshrLshr( MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const { diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.h b/llvm/lib/Target/Mips/MipsLegalizerInfo.h --- a/llvm/lib/Target/Mips/MipsLegalizerInfo.h +++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.h @@ -28,6 +28,9 @@ bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const override; + + bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder) const override; }; } // end namespace llvm #endif diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp --- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp +++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp @@ -152,3 +152,27 @@ return false; } + +bool MipsLegalizerInfo::legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder) const { + // We have to find the intrinsic ID by scanning. + unsigned IDIdx = 0; + for (; IDIdx < MI.getNumOperands(); ++IDIdx) + if (MI.getOperand(IDIdx).isIntrinsicID()) + break; + assert(MI.getOperand(IDIdx).isIntrinsicID() && "Could not find intrinsic ID"); + + switch (MI.getOperand(IDIdx).getIntrinsicID()) { + case Intrinsic::memcpy: + case Intrinsic::memset: + case Intrinsic::memmove: + if (createMemLibcall(MIRBuilder, MRI, MI) == + LegalizerHelper::UnableToLegalize) + return false; + MI.eraseFromParent(); + return true; + default: + break; + } + return true; +} diff --git a/llvm/lib/Target/X86/X86LegalizerInfo.h b/llvm/lib/Target/X86/X86LegalizerInfo.h --- a/llvm/lib/Target/X86/X86LegalizerInfo.h +++ b/llvm/lib/Target/X86/X86LegalizerInfo.h @@ -32,6 +32,9 @@ public: X86LegalizerInfo(const X86Subtarget &STI, const X86TargetMachine &TM); + bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder) const override; + private: void setLegalizerInfo32bit(); void setLegalizerInfo64bit(); diff --git a/llvm/lib/Target/X86/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/X86LegalizerInfo.cpp --- a/llvm/lib/Target/X86/X86LegalizerInfo.cpp +++ b/llvm/lib/Target/X86/X86LegalizerInfo.cpp @@ -13,6 +13,7 @@ #include "X86LegalizerInfo.h" #include "X86Subtarget.h" #include "X86TargetMachine.h" +#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/DerivedTypes.h" @@ -84,6 +85,31 @@ verify(*STI.getInstrInfo()); } +bool X86LegalizerInfo::legalizeIntrinsic(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder) const { + // We have to find the intrinsic ID by scanning. + unsigned IDIdx = 0; + for (; IDIdx < MI.getNumOperands(); ++IDIdx) + if (MI.getOperand(IDIdx).isIntrinsicID()) + break; + assert(MI.getOperand(IDIdx).isIntrinsicID() && "Could not find intrinsic ID"); + + switch (MI.getOperand(IDIdx).getIntrinsicID()) { + case Intrinsic::memcpy: + case Intrinsic::memset: + case Intrinsic::memmove: + if (createMemLibcall(MIRBuilder, MRI, MI) == + LegalizerHelper::UnableToLegalize) + return false; + MI.eraseFromParent(); + return true; + default: + break; + } + return true; +} + void X86LegalizerInfo::setLegalizerInfo32bit() { const LLT p0 = LLT::pointer(0, TM.getPointerSizeInBits(0)); diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll @@ -1130,10 +1130,7 @@ ; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0 ; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2 -; CHECK: $x0 = COPY [[DST]] -; CHECK: $x1 = COPY [[SRC]] -; CHECK: $x2 = COPY [[SIZE]] -; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2 +; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64) call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0) ret void } @@ -1144,10 +1141,7 @@ ; CHECK: [[DST:%[0-9]+]]:_(p0) = COPY $x0 ; CHECK: [[SRC:%[0-9]+]]:_(p0) = COPY $x1 ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2 -; CHECK: $x0 = COPY [[DST]] -; CHECK: $x1 = COPY [[SRC]] -; CHECK: $x2 = COPY [[SIZE]] -; CHECK: BL &memmove, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2 +; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), [[DST]](p0), [[SRC]](p0), [[SIZE]](s64) call void @llvm.memmove.p0i8.p0i8.i64(i8* %dst, i8* %src, i64 %size, i1 0) ret void } @@ -1159,11 +1153,7 @@ ; CHECK: [[SRC_C:%[0-9]+]]:_(s32) = COPY $w1 ; CHECK: [[SRC:%[0-9]+]]:_(s8) = G_TRUNC [[SRC_C]] ; CHECK: [[SIZE:%[0-9]+]]:_(s64) = COPY $x2 -; CHECK: $x0 = COPY [[DST]] -; CHECK: [[SRC_TMP:%[0-9]+]]:_(s32) = G_ANYEXT [[SRC]] -; CHECK: $w1 = COPY [[SRC_TMP]] -; CHECK: $x2 = COPY [[SIZE]] -; CHECK: BL &memset, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $w1, implicit $x2 +; CHECK: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), [[DST]](p0), [[SRC]](s8), [[SIZE]](s64) call void @llvm.memset.p0i8.i64(i8* %dst, i8 %val, i64 %size, i1 0) ret void } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memcpy-et-al.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memcpy-et-al.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-memcpy-et-al.mir @@ -0,0 +1,91 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -march=aarch64 -run-pass=legalizer -verify-machineinstrs %s -o - | FileCheck %s +--- +name: test_memcpy +tracksRegLiveness: true +body: | + bb.1: + liveins: $w2, $x0, $x1 + + ; CHECK-LABEL: name: test_memcpy + ; CHECK: liveins: $w2, $x0, $x1 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1 + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2 + ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32) + ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp + ; CHECK: $x0 = COPY [[COPY]](p0) + ; CHECK: $x1 = COPY [[COPY1]](p0) + ; CHECK: $x2 = COPY [[ZEXT]](s64) + ; CHECK: BL &memcpy, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2 + ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp + ; CHECK: RET_ReallyLR + %0:_(p0) = COPY $x0 + %1:_(p0) = COPY $x1 + %2:_(s32) = COPY $w2 + %4:_(s1) = G_CONSTANT i1 false + %3:_(s64) = G_ZEXT %2(s32) + G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), %0(p0), %1(p0), %3(s64), %4(s1) + RET_ReallyLR + +... +--- +name: test_memmove +tracksRegLiveness: true +body: | + bb.1: + liveins: $w2, $x0, $x1 + + ; CHECK-LABEL: name: test_memmove + ; CHECK: liveins: $w2, $x0, $x1 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[COPY1:%[0-9]+]]:_(p0) = COPY $x1 + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2 + ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32) + ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp + ; CHECK: $x0 = COPY [[COPY]](p0) + ; CHECK: $x1 = COPY [[COPY1]](p0) + ; CHECK: $x2 = COPY [[ZEXT]](s64) + ; CHECK: BL &memmove, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $x1, implicit $x2 + ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp + ; CHECK: RET_ReallyLR + %0:_(p0) = COPY $x0 + %1:_(p0) = COPY $x1 + %2:_(s32) = COPY $w2 + %4:_(s1) = G_CONSTANT i1 false + %3:_(s64) = G_ZEXT %2(s32) + G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memmove), %0(p0), %1(p0), %3(s64), %4(s1) + RET_ReallyLR + +... +--- +name: test_memset +tracksRegLiveness: true +body: | + bb.1: + liveins: $w1, $w2, $x0 + + ; CHECK-LABEL: name: test_memset + ; CHECK: liveins: $w1, $w2, $x0 + ; CHECK: [[COPY:%[0-9]+]]:_(p0) = COPY $x0 + ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY $w1 + ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY $w2 + ; CHECK: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY2]](s32) + ; CHECK: ADJCALLSTACKDOWN 0, 0, implicit-def $sp, implicit $sp + ; CHECK: $x0 = COPY [[COPY]](p0) + ; CHECK: [[COPY3:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32) + ; CHECK: $w1 = COPY [[COPY3]](s32) + ; CHECK: $x2 = COPY [[ZEXT]](s64) + ; CHECK: BL &memset, csr_aarch64_aapcs, implicit-def $lr, implicit $sp, implicit $x0, implicit $w1, implicit $x2 + ; CHECK: ADJCALLSTACKUP 0, 0, implicit-def $sp, implicit $sp + ; CHECK: RET_ReallyLR + %0:_(p0) = COPY $x0 + %1:_(s32) = COPY $w1 + %2:_(s32) = COPY $w2 + %5:_(s1) = G_CONSTANT i1 false + %3:_(s8) = G_TRUNC %1(s32) + %4:_(s64) = G_ZEXT %2(s32) + G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memset), %0(p0), %3(s8), %4(s64), %5(s1) + RET_ReallyLR + +... diff --git a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll --- a/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll +++ b/llvm/test/CodeGen/Mips/GlobalISel/irtranslator/call.ll @@ -153,12 +153,8 @@ ; MIPS32: [[COPY:%[0-9]+]]:_(p0) = COPY $a0 ; MIPS32: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1 ; MIPS32: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2 - ; MIPS32: ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp - ; MIPS32: $a0 = COPY [[COPY1]](p0) - ; MIPS32: $a1 = COPY [[COPY]](p0) - ; MIPS32: $a2 = COPY [[COPY2]](s32) - ; MIPS32: JAL &memcpy, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2 - ; MIPS32: ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp + ; MIPS32: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false + ; MIPS32: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32), [[C]](s1) ; MIPS32: RetRA ; MIPS32_PIC-LABEL: name: call_symbol ; MIPS32_PIC: bb.1.entry: @@ -166,12 +162,8 @@ ; MIPS32_PIC: [[COPY:%[0-9]+]]:_(p0) = COPY $a0 ; MIPS32_PIC: [[COPY1:%[0-9]+]]:_(p0) = COPY $a1 ; MIPS32_PIC: [[COPY2:%[0-9]+]]:_(s32) = COPY $a2 - ; MIPS32_PIC: ADJCALLSTACKDOWN 16, 0, implicit-def $sp, implicit $sp - ; MIPS32_PIC: $a0 = COPY [[COPY1]](p0) - ; MIPS32_PIC: $a1 = COPY [[COPY]](p0) - ; MIPS32_PIC: $a2 = COPY [[COPY2]](s32) - ; MIPS32_PIC: JAL &memcpy, csr_o32, implicit-def $ra, implicit-def $sp, implicit $a0, implicit $a1, implicit $a2 - ; MIPS32_PIC: ADJCALLSTACKUP 16, 0, implicit-def $sp, implicit $sp + ; MIPS32_PIC: [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 false + ; MIPS32_PIC: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[COPY1]](p0), [[COPY]](p0), [[COPY2]](s32), [[C]](s1) ; MIPS32_PIC: RetRA entry: call void @llvm.memcpy.p0i8.p0i8.i32(i8* align 1 %dest, i8* align 1 %src, i32 %length, i1 false) diff --git a/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll b/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll --- a/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll +++ b/llvm/test/CodeGen/X86/GlobalISel/x86_64-irtranslator-struct-return.ll @@ -16,15 +16,11 @@ ; ALL: [[COPY:%[0-9]+]]:_(s128) = COPY $xmm0 ; ALL: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s128) ; ALL: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4 + ; ALL: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false ; ALL: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval ; ALL: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.f ; ALL: G_STORE [[TRUNC]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %ir.coerce.dive2) - ; ALL: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: $rdi = COPY [[FRAME_INDEX]](p0) - ; ALL: $rsi = COPY [[FRAME_INDEX1]](p0) - ; ALL: $rdx = COPY [[C]](s64) - ; ALL: CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx - ; ALL: ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp + ; ALL: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1) ; ALL: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %ir.coerce.dive13) ; ALL: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s32) ; ALL: $xmm0 = COPY [[ANYEXT]](s128) @@ -51,15 +47,11 @@ ; ALL: [[COPY:%[0-9]+]]:_(s128) = COPY $xmm0 ; ALL: [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[COPY]](s128) ; ALL: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; ALL: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false ; ALL: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval ; ALL: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.d ; ALL: G_STORE [[TRUNC]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.coerce.dive2) - ; ALL: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: $rdi = COPY [[FRAME_INDEX]](p0) - ; ALL: $rsi = COPY [[FRAME_INDEX1]](p0) - ; ALL: $rdx = COPY [[C]](s64) - ; ALL: CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx - ; ALL: ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp + ; ALL: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1) ; ALL: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.coerce.dive13) ; ALL: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s64) ; ALL: $xmm0 = COPY [[ANYEXT]](s128) @@ -86,20 +78,16 @@ ; ALL: [[COPY1:%[0-9]+]]:_(s128) = COPY $xmm1 ; ALL: [[TRUNC1:%[0-9]+]]:_(s64) = G_TRUNC [[COPY1]](s128) ; ALL: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16 + ; ALL: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false ; ALL: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval ; ALL: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.d ; ALL: G_STORE [[TRUNC]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.1) - ; ALL: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 - ; ALL: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C1]](s64) + ; ALL: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; ALL: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C2]](s64) ; ALL: G_STORE [[TRUNC1]](s64), [[GEP]](p0) :: (store 8 into %ir.2) - ; ALL: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: $rdi = COPY [[FRAME_INDEX]](p0) - ; ALL: $rsi = COPY [[FRAME_INDEX1]](p0) - ; ALL: $rdx = COPY [[C]](s64) - ; ALL: CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx - ; ALL: ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp + ; ALL: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1) ; ALL: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.5) - ; ALL: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C1]](s64) + ; ALL: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C2]](s64) ; ALL: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.5 + 8) ; ALL: [[ANYEXT:%[0-9]+]]:_(s128) = G_ANYEXT [[LOAD]](s64) ; ALL: $xmm0 = COPY [[ANYEXT]](s128) @@ -128,15 +116,11 @@ ; ALL: liveins: $edi ; ALL: [[COPY:%[0-9]+]]:_(s32) = COPY $edi ; ALL: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 4 + ; ALL: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false ; ALL: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval ; ALL: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i ; ALL: G_STORE [[COPY]](s32), [[FRAME_INDEX1]](p0) :: (store 4 into %ir.coerce.dive2) - ; ALL: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: $rdi = COPY [[FRAME_INDEX]](p0) - ; ALL: $rsi = COPY [[FRAME_INDEX1]](p0) - ; ALL: $rdx = COPY [[C]](s64) - ; ALL: CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx - ; ALL: ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp + ; ALL: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1) ; ALL: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX]](p0) :: (load 4 from %ir.coerce.dive13) ; ALL: $eax = COPY [[LOAD]](s32) ; ALL: RET 0, implicit $eax @@ -159,15 +143,11 @@ ; ALL: liveins: $rdi ; ALL: [[COPY:%[0-9]+]]:_(s64) = COPY $rdi ; ALL: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; ALL: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false ; ALL: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval ; ALL: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i ; ALL: G_STORE [[COPY]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.0, align 4) - ; ALL: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: $rdi = COPY [[FRAME_INDEX]](p0) - ; ALL: $rsi = COPY [[FRAME_INDEX1]](p0) - ; ALL: $rdx = COPY [[C]](s64) - ; ALL: CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx - ; ALL: ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp + ; ALL: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1) ; ALL: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.3, align 4) ; ALL: $rax = COPY [[LOAD]](s64) ; ALL: RET 0, implicit $rax @@ -191,34 +171,20 @@ ; ALL: [[COPY:%[0-9]+]]:_(s64) = COPY $rdi ; ALL: [[COPY1:%[0-9]+]]:_(s32) = COPY $esi ; ALL: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 12 + ; ALL: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false ; ALL: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval ; ALL: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i ; ALL: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.2.coerce ; ALL: [[FRAME_INDEX3:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.3.tmp ; ALL: G_STORE [[COPY]](s64), [[FRAME_INDEX2]](p0) :: (store 8 into %ir.0, align 4) - ; ALL: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 - ; ALL: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX2]], [[C1]](s64) + ; ALL: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; ALL: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX2]], [[C2]](s64) ; ALL: G_STORE [[COPY1]](s32), [[GEP]](p0) :: (store 4 into %ir.1) - ; ALL: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: $rdi = COPY [[FRAME_INDEX1]](p0) - ; ALL: $rsi = COPY [[FRAME_INDEX2]](p0) - ; ALL: $rdx = COPY [[C]](s64) - ; ALL: CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx - ; ALL: ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: $rdi = COPY [[FRAME_INDEX]](p0) - ; ALL: $rsi = COPY [[FRAME_INDEX1]](p0) - ; ALL: $rdx = COPY [[C]](s64) - ; ALL: CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx - ; ALL: ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: $rdi = COPY [[FRAME_INDEX3]](p0) - ; ALL: $rsi = COPY [[FRAME_INDEX]](p0) - ; ALL: $rdx = COPY [[C]](s64) - ; ALL: CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx - ; ALL: ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp + ; ALL: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX1]](p0), [[FRAME_INDEX2]](p0), [[C]](s64), [[C1]](s1) + ; ALL: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1) + ; ALL: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX3]](p0), [[FRAME_INDEX]](p0), [[C]](s64), [[C1]](s1) ; ALL: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX3]](p0) :: (load 8 from %ir.tmp) - ; ALL: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX3]], [[C1]](s64) + ; ALL: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX3]], [[C2]](s64) ; ALL: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[GEP1]](p0) :: (load 4 from %ir.tmp + 8, align 8) ; ALL: $rax = COPY [[LOAD]](s64) ; ALL: $edx = COPY [[LOAD1]](s32) @@ -252,20 +218,16 @@ ; ALL: [[COPY:%[0-9]+]]:_(s64) = COPY $rdi ; ALL: [[COPY1:%[0-9]+]]:_(s64) = COPY $rsi ; ALL: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 16 + ; ALL: [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 false ; ALL: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.retval ; ALL: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.i ; ALL: G_STORE [[COPY]](s64), [[FRAME_INDEX1]](p0) :: (store 8 into %ir.1, align 4) - ; ALL: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 - ; ALL: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C1]](s64) + ; ALL: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; ALL: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX1]], [[C2]](s64) ; ALL: G_STORE [[COPY1]](s64), [[GEP]](p0) :: (store 8 into %ir.2, align 4) - ; ALL: ADJCALLSTACKDOWN64 0, 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp - ; ALL: $rdi = COPY [[FRAME_INDEX]](p0) - ; ALL: $rsi = COPY [[FRAME_INDEX1]](p0) - ; ALL: $rdx = COPY [[C]](s64) - ; ALL: CALL64pcrel32 &memcpy, csr_64, implicit $rsp, implicit $ssp, implicit $rdi, implicit $rsi, implicit $rdx - ; ALL: ADJCALLSTACKUP64 0, 0, implicit-def $rsp, implicit-def $eflags, implicit-def $ssp, implicit $rsp, implicit $ssp + ; ALL: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.memcpy), [[FRAME_INDEX]](p0), [[FRAME_INDEX1]](p0), [[C]](s64), [[C1]](s1) ; ALL: [[LOAD:%[0-9]+]]:_(s64) = G_LOAD [[FRAME_INDEX]](p0) :: (load 8 from %ir.5, align 4) - ; ALL: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C1]](s64) + ; ALL: [[GEP1:%[0-9]+]]:_(p0) = G_GEP [[FRAME_INDEX]], [[C2]](s64) ; ALL: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[GEP1]](p0) :: (load 8 from %ir.5 + 8, align 4) ; ALL: $rax = COPY [[LOAD]](s64) ; ALL: $rdx = COPY [[LOAD1]](s64)