diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -105,6 +105,9 @@ // known to be exact. NoFPExcept = 1 << 14, // Instruction does not raise // floatint-point exceptions. + NoMerge = 1 << 15, // Passes that drop source location info + // (e.g. branch folding) should skip + // this instruction. }; private: diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -278,6 +278,7 @@ struct CallSiteDbgInfo { CallSiteInfo CSInfo; MDNode *HeapAllocSite = nullptr; + bool NoMerge = false; }; DenseMap SDCallSiteDbgInfo; @@ -1916,6 +1917,18 @@ return It->second.HeapAllocSite; } + void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) { + if (NoMerge) + SDCallSiteDbgInfo[Node].NoMerge = NoMerge; + } + + bool getNoMergeSiteInfo(const SDNode *Node) { + auto I = SDCallSiteDbgInfo.find(Node); + if (I == SDCallSiteDbgInfo.end()) + return false; + return I->second.NoMerge; + } + /// Return the current function's default denormal handling kind for the given /// floating point type. DenormalMode getDenormalMode(EVT VT) const { diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -3613,6 +3613,7 @@ bool IsConvergent : 1; bool IsPatchPoint : 1; bool IsPreallocated : 1; + bool NoMerge : 1; // IsTailCall should be modified by implementations of // TargetLowering::LowerCall that perform tail call conversions. @@ -3636,7 +3637,8 @@ CallLoweringInfo(SelectionDAG &DAG) : RetSExt(false), RetZExt(false), IsVarArg(false), IsInReg(false), DoesNotReturn(false), IsReturnValueUsed(true), IsConvergent(false), - IsPatchPoint(false), IsPreallocated(false), DAG(DAG) {} + IsPatchPoint(false), IsPreallocated(false), NoMerge(false), + DAG(DAG) {} CallLoweringInfo &setDebugLoc(const SDLoc &dl) { DL = dl; @@ -3685,7 +3687,8 @@ IsReturnValueUsed = !Call.use_empty(); RetSExt = Call.hasRetAttr(Attribute::SExt); RetZExt = Call.hasRetAttr(Attribute::ZExt); - + NoMerge = Call.hasFnAttr(Attribute::NoMerge); + Callee = Target; CallConv = Call.getCallingConv(); diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp --- a/llvm/lib/CodeGen/BranchFolding.cpp +++ b/llvm/lib/CodeGen/BranchFolding.cpp @@ -348,6 +348,9 @@ MBBI1->isInlineAsm()) { break; } + if (MBBI1->getFlag(MachineInstr::NoMerge) || + MBBI2->getFlag(MachineInstr::NoMerge)) + break; ++TailLen; I1 = MBBI1; I2 = MBBI2; diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -778,6 +778,8 @@ OS << "exact "; if (MI.getFlag(MachineInstr::NoFPExcept)) OS << "nofpexcept "; + if (MI.getFlag(MachineInstr::NoMerge)) + OS << "nomerge "; OS << TII->getName(MI.getOpcode()); if (I < E) diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -1595,6 +1595,8 @@ OS << "exact "; if (getFlag(MachineInstr::NoFPExcept)) OS << "nofpexcept "; + if (getFlag(MachineInstr::NoMerge)) + OS << "nomerge "; // Print the opcode name. if (TII) diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -872,6 +872,10 @@ DAG->getTarget().Options.EmitCallSiteInfo) MF.addCallArgsForwardingRegs(MI, DAG->getSDCallSiteInfo(Node)); + if (DAG->getNoMergeSiteInfo(Node)) { + MI->setFlag(MachineInstr::MIFlag::NoMerge); + } + return MI; }; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -4541,6 +4541,7 @@ // Returns a chain and a flag for retval copy to use. Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops); + DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); InFlag = Chain.getValue(1); DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -2542,6 +2542,7 @@ // Returns a chain and a flag for retval copy to use. Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops); + DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); InFlag = Chain.getValue(1); DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h --- a/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -979,12 +979,13 @@ const bool IsPatchPoint : 1; const bool IsIndirect : 1; const bool HasNest : 1; + const bool NoMerge : 1; CallFlags(CallingConv::ID CC, bool IsTailCall, bool IsVarArg, - bool IsPatchPoint, bool IsIndirect, bool HasNest) + bool IsPatchPoint, bool IsIndirect, bool HasNest, bool NoMerge) : CallConv(CC), IsTailCall(IsTailCall), IsVarArg(IsVarArg), IsPatchPoint(IsPatchPoint), IsIndirect(IsIndirect), - HasNest(HasNest) {} + HasNest(HasNest), NoMerge(NoMerge) {} }; private: diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -5586,6 +5586,7 @@ std::array ReturnTypes = {{MVT::Other, MVT::Glue}}; Chain = DAG.getNode(CallOpc, dl, ReturnTypes, Ops); + DAG.addNoMergeSiteInfo(Chain.getNode(), CFlags.NoMerge); Glue = Chain.getValue(1); // When performing tail call optimization the callee pops its arguments off @@ -5667,7 +5668,8 @@ isIndirectCall(Callee, DAG, Subtarget, isPatchPoint), // hasNest Subtarget.is64BitELFABI() && - any_of(Outs, [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); })); + any_of(Outs, [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }), + CLI.NoMerge); if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -2353,6 +2353,7 @@ } Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops); + DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); Glue = Chain.getValue(1); // Mark the end of the call, which is glued to the call itself. diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -4336,6 +4336,7 @@ Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); } InFlag = Chain.getValue(1); + DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); // Save heapallocsite metadata. diff --git a/llvm/test/CodeGen/AArch64/nomerge.ll b/llvm/test/CodeGen/AArch64/nomerge.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/nomerge.ll @@ -0,0 +1,36 @@ +; RUN: llc < %s -mtriple=aarch64 -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: // %bb.0: // %entry +; CHECK: // %bb.1: // %entry +; CHECK: // %bb.2: // %if.then +; CHECK-NEXT: bl bar +; CHECK: b bar +; CHECK: .LBB0_3: // %if.then2 +; CHECK-NEXT: bl bar +; CHECK: .LBB0_4: // %if.end3 +; CHECK: b bar diff --git a/llvm/test/CodeGen/ARM/nomerge.ll b/llvm/test/CodeGen/ARM/nomerge.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/ARM/nomerge.ll @@ -0,0 +1,36 @@ +; RUN: llc < %s -mtriple=arm -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: @ %bb.0: @ %entry +; CHECK: @ %bb.1: @ %entry +; CHECK: @ %bb.2: @ %if.then +; CHECK-NEXT: bl bar +; CHECK: b bar +; CHECK: .LBB0_3: @ %if.then2 +; CHECK-NEXT: bl bar +; CHECK: .LBB0_4: @ %if.end3 +; CHECK: b bar diff --git a/llvm/test/CodeGen/PowerPC/nomerge.ll b/llvm/test/CodeGen/PowerPC/nomerge.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/nomerge.ll @@ -0,0 +1,35 @@ +; RUN: llc < %s -mtriple=powerpc -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %entry +; CHECK: # %bb.1: # %entry +; CHECK: # %bb.2: # %if.then +; CHECK-NEXT: bl bar +; CHECK: .LBB0_3: # %if.then2 +; CHECK-NEXT: bl bar +; CHECK: .LBB0_4: # %if.end3 +; CHECK-NEXT: bl bar diff --git a/llvm/test/CodeGen/RISCV/nomerge.ll b/llvm/test/CodeGen/RISCV/nomerge.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/nomerge.ll @@ -0,0 +1,35 @@ +; RUN: llc < %s -mtriple=riscv64 -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %entry +; CHECK: # %bb.1: # %entry +; CHECK: # %bb.2: # %if.then +; CHECK-NEXT: call bar +; CHECK: .LBB0_3: # %if.then2 +; CHECK-NEXT: call bar +; CHECK: .LBB0_4: # %if.end3 +; CHECK: tail bar diff --git a/llvm/test/CodeGen/X86/nomerge.ll b/llvm/test/CodeGen/X86/nomerge.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/X86/nomerge.ll @@ -0,0 +1,36 @@ +; RUN: llc < %s -mtriple=x86_64 -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %entry +; CHECK: # %bb.1: # %entry +; CHECK: # %bb.2: # %if.then +; CHECK-NEXT: callq bar +; CHECK: jmp bar # TAILCALL +; CHECK: .LBB0_3: # %if.then2 +; CHECK: callq bar +; CHECK: .LBB0_4: # %if.end3 +; CHECK: jmp bar # TAILCALL