diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -731,6 +731,7 @@ void CodeGenPassBuilder::addISelPrepare(AddIRPass &addPass) const { derived().addPreISel(addPass); + addPass(CallBrPrepare()); // Add both the safe stack and the stack protection passes: each of them will // only protect functions that have corresponding attributes. addPass(SafeStackPass()); diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -123,6 +123,7 @@ DUMMY_FUNCTION_PASS("cfguard-check", CFGuardCheckPass, ()) DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ()) DUMMY_FUNCTION_PASS("select-optimize", SelectOptimizePass, ()) +DUMMY_FUNCTION_PASS("callbrprepare", CallBrPrepare, ()) #undef DUMMY_FUNCTION_PASS #ifndef DUMMY_MODULE_PASS diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -583,6 +583,8 @@ /// This pass converts conditional moves to conditional jumps when profitable. FunctionPass *createSelectOptimizePass(); + + FunctionPass *createCallBrPass(); } // End llvm namespace #endif diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -86,6 +86,7 @@ void initializeCFIInstrInserterPass(PassRegistry&); void initializeCFLAndersAAWrapperPassPass(PassRegistry&); void initializeCFLSteensAAWrapperPassPass(PassRegistry&); +void initializeCallBrPreparePass(PassRegistry&); void initializeCallGraphDOTPrinterPass(PassRegistry&); void initializeCallGraphPrinterLegacyPassPass(PassRegistry&); void initializeCallGraphViewerPass(PassRegistry&); diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -35,6 +35,7 @@ BasicBlockSections.cpp BasicBlockSectionsProfileReader.cpp CalcSpillWeights.cpp + CallBrPrepare.cpp CallingConvLower.cpp CFGuardLongjmp.cpp CFIFixup.cpp diff --git a/llvm/lib/CodeGen/CallBrPrepare.cpp b/llvm/lib/CodeGen/CallBrPrepare.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/CodeGen/CallBrPrepare.cpp @@ -0,0 +1,66 @@ +//===-- CallBrPrepare - Prepare callbr for code generation ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass lowers callbrs in LLVM IR to something closer that the backend can +// perform correct instruction selection for. In particular, callbr may have +// inline asm with output constraints that require specific physical registers. +// These physical register will need to be copied back to virtual registers +// somewhere. If we have critical edges, we may need to split them. Regardless +// of critical edges, we create distinct new SSA values for each indirect +// destination of the callbr, then remap users of the direct destinations value +// which aren't dominated by the direct destination, but are by the indirect +// destination. +// +// This enables us to keep the use of the callbr's Value simple in IR, and +// expand control flow when necessary at the last minute before instruction +// selection. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" + +using namespace llvm; + +#define DEBUG_TYPE "callbrprepare" + +namespace { + +class CallBrPrepare : public FunctionPass { +public: + CallBrPrepare() : FunctionPass(ID) {} + static char ID; + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnFunction(Function &Fn) override; +}; + +} // end anonymous namespace + +char CallBrPrepare::ID = 0; +INITIALIZE_PASS(CallBrPrepare, DEBUG_TYPE, "Prepare callbr", false, false) + +FunctionPass *llvm::createCallBrPass() { return new CallBrPrepare(); } + +void CallBrPrepare::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesAll(); +} + +bool CallBrPrepare::runOnFunction(Function &Fn) { + for (BasicBlock &BB : Fn) { + auto *CBR = dyn_cast(BB.getTerminator()); + if (!CBR) + continue; + // TODO: something interesting. + // https://discourse.llvm.org/t/rfc-syncing-asm-goto-with-outputs-with-gcc/65453/8 + } + return false; +} diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -24,6 +24,7 @@ initializeBasicBlockSectionsPass(Registry); initializeBranchFolderPassPass(Registry); initializeBranchRelaxationPass(Registry); + initializeCallBrPreparePass(Registry); initializeCFGuardLongjmpPass(Registry); initializeCFIFixupPass(Registry); initializeCFIInstrInserterPass(Registry); diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -1007,6 +1007,8 @@ if (requiresCodeGenSCCOrder()) addPass(new DummyCGSCCPass); + addPass(createCallBrPass()); + // Add both the safe stack and the stack protection passes: each of them will // only protect functions that have corresponding attributes. addPass(createSafeStackPass()); diff --git a/llvm/test/CodeGen/AArch64/O0-pipeline.ll b/llvm/test/CodeGen/AArch64/O0-pipeline.ll --- a/llvm/test/CodeGen/AArch64/O0-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O0-pipeline.ll @@ -29,6 +29,7 @@ ; CHECK-NEXT: AArch64 Stack Tagging ; CHECK-NEXT: SME ABI Pass ; CHECK-NEXT: Exception handling preparation +; CHECK-NEXT: Prepare callbr ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier diff --git a/llvm/test/CodeGen/AArch64/O3-pipeline.ll b/llvm/test/CodeGen/AArch64/O3-pipeline.ll --- a/llvm/test/CodeGen/AArch64/O3-pipeline.ll +++ b/llvm/test/CodeGen/AArch64/O3-pipeline.ll @@ -99,6 +99,7 @@ ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: FunctionPass Manager ; CHECK-NEXT: Merge internal globals +; CHECK-NEXT: Prepare callbr ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier diff --git a/llvm/test/CodeGen/AArch64/callbr-prepare.ll b/llvm/test/CodeGen/AArch64/callbr-prepare.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/callbr-prepare.ll @@ -0,0 +1,30 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt %s -callbrprepare -S -o - | FileCheck %s + +; TODO: update this test to split critical edges. +define i32 @test0() { +; CHECK-LABEL: @test0( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"() +; CHECK-NEXT: to label [[DIRECT:%.*]] [label %indirect] +; CHECK: direct: +; CHECK-NEXT: [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"() +; CHECK-NEXT: to label [[DIRECT2:%.*]] [label %indirect] +; CHECK: direct2: +; CHECK-NEXT: ret i32 0 +; CHECK: indirect: +; CHECK-NEXT: [[OUT3:%.*]] = phi i32 [ [[OUT]], [[ENTRY:%.*]] ], [ [[OUT2]], [[DIRECT]] ] +; CHECK-NEXT: ret i32 [[OUT3]] +; +entry: + %out = callbr i32 asm "# $0", "=r,!i"() + to label %direct [label %indirect] +direct: + %out2 = callbr i32 asm "# $0", "=r,!i"() + to label %direct2 [label %indirect] +direct2: + ret i32 0 +indirect: + %out3 = phi i32 [%out, %entry], [%out2, %direct] + ret i32 %out3 +} diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll --- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll +++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll @@ -91,6 +91,7 @@ ; GCN-O0-NEXT: Loop-Closed SSA Form Pass ; GCN-O0-NEXT: DummyCGSCCPass ; GCN-O0-NEXT: FunctionPass Manager +; GCN-O0-NEXT: Prepare callbr ; GCN-O0-NEXT: Safe Stack instrumentation pass ; GCN-O0-NEXT: Insert stack protectors ; GCN-O0-NEXT: Dominator Tree Construction @@ -277,6 +278,7 @@ ; GCN-O1-NEXT: Loop-Closed SSA Form Pass ; GCN-O1-NEXT: DummyCGSCCPass ; GCN-O1-NEXT: FunctionPass Manager +; GCN-O1-NEXT: Prepare callbr ; GCN-O1-NEXT: Safe Stack instrumentation pass ; GCN-O1-NEXT: Insert stack protectors ; GCN-O1-NEXT: Dominator Tree Construction @@ -567,6 +569,7 @@ ; GCN-O1-OPTS-NEXT: Loop-Closed SSA Form Pass ; GCN-O1-OPTS-NEXT: DummyCGSCCPass ; GCN-O1-OPTS-NEXT: FunctionPass Manager +; GCN-O1-OPTS-NEXT: Prepare callbr ; GCN-O1-OPTS-NEXT: Safe Stack instrumentation pass ; GCN-O1-OPTS-NEXT: Insert stack protectors ; GCN-O1-OPTS-NEXT: Dominator Tree Construction @@ -866,6 +869,7 @@ ; GCN-O2-NEXT: Analysis if a function is memory bound ; GCN-O2-NEXT: DummyCGSCCPass ; GCN-O2-NEXT: FunctionPass Manager +; GCN-O2-NEXT: Prepare callbr ; GCN-O2-NEXT: Safe Stack instrumentation pass ; GCN-O2-NEXT: Insert stack protectors ; GCN-O2-NEXT: Dominator Tree Construction @@ -1179,6 +1183,7 @@ ; GCN-O3-NEXT: Analysis if a function is memory bound ; GCN-O3-NEXT: DummyCGSCCPass ; GCN-O3-NEXT: FunctionPass Manager +; GCN-O3-NEXT: Prepare callbr ; GCN-O3-NEXT: Safe Stack instrumentation pass ; GCN-O3-NEXT: Insert stack protectors ; GCN-O3-NEXT: Dominator Tree Construction diff --git a/llvm/test/CodeGen/ARM/O3-pipeline.ll b/llvm/test/CodeGen/ARM/O3-pipeline.ll --- a/llvm/test/CodeGen/ARM/O3-pipeline.ll +++ b/llvm/test/CodeGen/ARM/O3-pipeline.ll @@ -66,6 +66,7 @@ ; CHECK-NEXT: Transform predicated vector loops to use MVE tail predication ; CHECK-NEXT: A No-Op Barrier Pass ; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: Prepare callbr ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier diff --git a/llvm/test/CodeGen/PowerPC/O0-pipeline.ll b/llvm/test/CodeGen/PowerPC/O0-pipeline.ll --- a/llvm/test/CodeGen/PowerPC/O0-pipeline.ll +++ b/llvm/test/CodeGen/PowerPC/O0-pipeline.ll @@ -30,6 +30,7 @@ ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics ; CHECK-NEXT: Exception handling preparation +; CHECK-NEXT: Prepare callbr ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier diff --git a/llvm/test/CodeGen/PowerPC/O3-pipeline.ll b/llvm/test/CodeGen/PowerPC/O3-pipeline.ll --- a/llvm/test/CodeGen/PowerPC/O3-pipeline.ll +++ b/llvm/test/CodeGen/PowerPC/O3-pipeline.ll @@ -80,6 +80,7 @@ ; CHECK-NEXT: Lazy Block Frequency Analysis ; CHECK-NEXT: Optimization Remark Emitter ; CHECK-NEXT: Hardware Loop Insertion +; CHECK-NEXT: Prepare callbr ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier diff --git a/llvm/test/CodeGen/RISCV/O0-pipeline.ll b/llvm/test/CodeGen/RISCV/O0-pipeline.ll --- a/llvm/test/CodeGen/RISCV/O0-pipeline.ll +++ b/llvm/test/CodeGen/RISCV/O0-pipeline.ll @@ -31,6 +31,7 @@ ; CHECK-NEXT: Scalarize Masked Memory Intrinsics ; CHECK-NEXT: Expand reduction intrinsics ; CHECK-NEXT: Exception handling preparation +; CHECK-NEXT: Prepare callbr ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier diff --git a/llvm/test/CodeGen/RISCV/O3-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-pipeline.ll --- a/llvm/test/CodeGen/RISCV/O3-pipeline.ll +++ b/llvm/test/CodeGen/RISCV/O3-pipeline.ll @@ -66,6 +66,7 @@ ; CHECK-NEXT: Exception handling preparation ; CHECK-NEXT: A No-Op Barrier Pass ; CHECK-NEXT: FunctionPass Manager +; CHECK-NEXT: Prepare callbr ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier diff --git a/llvm/test/CodeGen/X86/O0-pipeline.ll b/llvm/test/CodeGen/X86/O0-pipeline.ll --- a/llvm/test/CodeGen/X86/O0-pipeline.ll +++ b/llvm/test/CodeGen/X86/O0-pipeline.ll @@ -32,6 +32,7 @@ ; CHECK-NEXT: Expand reduction intrinsics ; CHECK-NEXT: Expand indirectbr instructions ; CHECK-NEXT: Exception handling preparation +; CHECK-NEXT: Prepare callbr ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier diff --git a/llvm/test/CodeGen/X86/opt-pipeline.ll b/llvm/test/CodeGen/X86/opt-pipeline.ll --- a/llvm/test/CodeGen/X86/opt-pipeline.ll +++ b/llvm/test/CodeGen/X86/opt-pipeline.ll @@ -72,6 +72,7 @@ ; CHECK-NEXT: CodeGen Prepare ; CHECK-NEXT: Dominator Tree Construction ; CHECK-NEXT: Exception handling preparation +; CHECK-NEXT: Prepare callbr ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp --- a/llvm/tools/opt/opt.cpp +++ b/llvm/tools/opt/opt.cpp @@ -387,7 +387,8 @@ "view-regions", "view-regions-only", "select-optimize", "expand-large-div-rem", "structurizecfg", "fix-irreducible", - "expand-large-fp-convert"}; + "expand-large-fp-convert", "callbrprepare", + }; for (const auto &P : PassNamePrefix) if (Pass.startswith(P)) return true; @@ -438,6 +439,7 @@ initializeExpandMemCmpPassPass(Registry); initializeScalarizeMaskedMemIntrinLegacyPassPass(Registry); initializeSelectOptimizePass(Registry); + initializeCallBrPreparePass(Registry); initializeCodeGenPreparePass(Registry); initializeAtomicExpandPass(Registry); initializeRewriteSymbolsLegacyPassPass(Registry); diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn --- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn @@ -33,6 +33,7 @@ "CFIFixup.cpp", "CFIInstrInserter.cpp", "CalcSpillWeights.cpp", + "CallBrPrepare.cpp", "CallingConvLower.cpp", "CodeGen.cpp", "CodeGenCommonISel.cpp",