diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -714,8 +714,9 @@ void CodeGenPassBuilder::addISelPrepare(AddIRPass &addPass) const { derived().addPreISel(addPass); - // Add both the safe stack and the stack protection passes: each of them will - // only protect functions that have corresponding attributes. + // addPass(CallBrPrepare()); // TODO(ndesaulniers): impl pass + // Add both the safe stack and the stack protection passes: each of them will + // only protect functions that have corresponding attributes. addPass(SafeStackPass()); addPass(StackProtectorPass()); diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -123,6 +123,7 @@ DUMMY_FUNCTION_PASS("cfguard-check", CFGuardCheckPass, ()) DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ()) DUMMY_FUNCTION_PASS("select-optimize", SelectOptimizePass, ()) +DUMMY_FUNCTION_PASS("callbrprepare", CallBrPrepare, ()) #undef DUMMY_FUNCTION_PASS #ifndef DUMMY_MODULE_PASS diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h --- a/llvm/include/llvm/CodeGen/Passes.h +++ b/llvm/include/llvm/CodeGen/Passes.h @@ -583,6 +583,8 @@ /// This pass converts conditional moves to conditional jumps when profitable. FunctionPass *createSelectOptimizePass(); + + FunctionPass *createCallBrPass(); } // End llvm namespace #endif diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -83,6 +83,9 @@ void initializeCFGViewerLegacyPassPass(PassRegistry&); void initializeCFIFixupPass(PassRegistry&); void initializeCFIInstrInserterPass(PassRegistry&); +void initializeCFLAndersAAWrapperPassPass(PassRegistry &); +void initializeCFLSteensAAWrapperPassPass(PassRegistry &); +void initializeCallBrPreparePass(PassRegistry &); void initializeCallGraphDOTPrinterPass(PassRegistry&); void initializeCallGraphPrinterLegacyPassPass(PassRegistry&); void initializeCallGraphViewerPass(PassRegistry&); diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -35,6 +35,7 @@ BasicBlockSections.cpp BasicBlockSectionsProfileReader.cpp CalcSpillWeights.cpp + CallBrPrepare.cpp CallingConvLower.cpp CFGuardLongjmp.cpp CFIFixup.cpp diff --git a/llvm/lib/CodeGen/CallBrPrepare.cpp b/llvm/lib/CodeGen/CallBrPrepare.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/CodeGen/CallBrPrepare.cpp @@ -0,0 +1,66 @@ +//===-- CallBrPrepare - Prepare callbr for code generation ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass lowers callbrs in LLVM IR to something closer that the backend can +// perform correct instruction selection for. In particular, callbr may have +// inline asm with output constraints that require specific physical registers. +// These physical register will need to be copied back to virtual registers +// somewhere. If we have critical edges, we may need to split them. Regardless +// of critical edges, we create distinct new SSA values for each indirect +// destination of the callbr, then remap users of the direct destinations value +// which aren't dominated by the direct destination, but are by the indirect +// destination. +// +// This enables us to keep the use of the callbr's Value simple in IR, and +// expand control flow when necessary at the last minute before instruction +// selection. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/Passes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Instructions.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" + +using namespace llvm; + +#define DEBUG_TYPE "callbrprepare" + +namespace { + +class CallBrPrepare : public FunctionPass { +public: + CallBrPrepare() : FunctionPass(ID) {} + static char ID; + void getAnalysisUsage(AnalysisUsage &AU) const override; + bool runOnFunction(Function &Fn) override; +}; + +} // end anonymous namespace + +char CallBrPrepare::ID = 0; +INITIALIZE_PASS(CallBrPrepare, DEBUG_TYPE, "Prepare callbr", false, false) + +FunctionPass *llvm::createCallBrPass() { return new CallBrPrepare(); } + +void CallBrPrepare::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesAll(); +} + +bool CallBrPrepare::runOnFunction(Function &Fn) { + for (BasicBlock &BB : Fn) { + auto *CBR = dyn_cast(BB.getTerminator()); + if (!CBR) + continue; + // TODO: something interesting. + // https://discourse.llvm.org/t/rfc-syncing-asm-goto-with-outputs-with-gcc/65453/8 + } + return false; +} diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -24,6 +24,7 @@ initializeBasicBlockSectionsPass(Registry); initializeBranchFolderPassPass(Registry); initializeBranchRelaxationPass(Registry); + initializeCallBrPreparePass(Registry); initializeCFGuardLongjmpPass(Registry); initializeCFIFixupPass(Registry); initializeCFIInstrInserterPass(Registry); diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -977,6 +977,8 @@ if (requiresCodeGenSCCOrder()) addPass(new DummyCGSCCPass); + // addPass(createCallBrPass()); // TODO(ndesaulniers): impl pass + // Add both the safe stack and the stack protection passes: each of them will // only protect functions that have corresponding attributes. addPass(createSafeStackPass()); diff --git a/llvm/test/CodeGen/AArch64/callbr-prepare.ll b/llvm/test/CodeGen/AArch64/callbr-prepare.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/callbr-prepare.ll @@ -0,0 +1,30 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt %s -callbrprepare -S -o - | FileCheck %s + +; TODO: update this test to split critical edges. +define i32 @test0() { +; CHECK-LABEL: @test0( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[OUT:%.*]] = callbr i32 asm "# $0", "=r,!i"() +; CHECK-NEXT: to label [[DIRECT:%.*]] [label %indirect] +; CHECK: direct: +; CHECK-NEXT: [[OUT2:%.*]] = callbr i32 asm "# $0", "=r,!i"() +; CHECK-NEXT: to label [[DIRECT2:%.*]] [label %indirect] +; CHECK: direct2: +; CHECK-NEXT: ret i32 0 +; CHECK: indirect: +; CHECK-NEXT: [[OUT3:%.*]] = phi i32 [ [[OUT]], [[ENTRY:%.*]] ], [ [[OUT2]], [[DIRECT]] ] +; CHECK-NEXT: ret i32 [[OUT3]] +; +entry: + %out = callbr i32 asm "# $0", "=r,!i"() + to label %direct [label %indirect] +direct: + %out2 = callbr i32 asm "# $0", "=r,!i"() + to label %direct2 [label %indirect] +direct2: + ret i32 0 +indirect: + %out3 = phi i32 [%out, %entry], [%out2, %direct] + ret i32 %out3 +} diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp --- a/llvm/tools/opt/opt.cpp +++ b/llvm/tools/opt/opt.cpp @@ -393,7 +393,8 @@ "expand-large-div-rem", "structurizecfg", "fix-irreducible", - "expand-large-fp-convert" + "expand-large-fp-convert", + "callbrprepare", }; for (const auto &P : PassNamePrefix) if (Pass.startswith(P)) @@ -445,6 +446,7 @@ initializeExpandMemCmpPassPass(Registry); initializeScalarizeMaskedMemIntrinLegacyPassPass(Registry); initializeSelectOptimizePass(Registry); + initializeCallBrPreparePass(Registry); initializeCodeGenPreparePass(Registry); initializeAtomicExpandPass(Registry); initializeRewriteSymbolsLegacyPassPass(Registry); diff --git a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn --- a/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/CodeGen/BUILD.gn @@ -34,6 +34,7 @@ "CFIFixup.cpp", "CFIInstrInserter.cpp", "CalcSpillWeights.cpp", + "CallBrPrepare.cpp", "CallingConvLower.cpp", "CodeGen.cpp", "CodeGenCommonISel.cpp",