diff --git a/llvm/lib/Target/X86/LLVMBuild.txt b/llvm/lib/Target/X86/LLVMBuild.txt --- a/llvm/lib/Target/X86/LLVMBuild.txt +++ b/llvm/lib/Target/X86/LLVMBuild.txt @@ -30,5 +30,5 @@ type = Library name = X86CodeGen parent = X86 -required_libraries = Analysis AsmPrinter CodeGen Core MC SelectionDAG Support Target X86Desc X86Info GlobalISel ProfileData CFGuard +required_libraries = Analysis AsmPrinter CodeGen Core MC Passes SelectionDAG Support Target X86Desc X86Info GlobalISel ProfileData CFGuard add_to_library_groups = X86 diff --git a/llvm/lib/Target/X86/X86PassRegistry.def b/llvm/lib/Target/X86/X86PassRegistry.def new file mode 100644 --- /dev/null +++ b/llvm/lib/Target/X86/X86PassRegistry.def @@ -0,0 +1,66 @@ +//===- X86PassRegistry.def - Registry of passes for X86 ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// X86 pass registry +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif + +#undef FUNCTION_PASS + +#ifndef DUMMY_FUNCTION_PASS +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif +DUMMY_FUNCTION_PASS("x86-win-eh-state", X86WinEHStatePass, ()) +#undef DUMMY_FUNCTION_PASS + +#ifndef MACHINE_FUNCTION_PASS +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif + +#undef MACHINE_FUNCTION_PASS + +// PASS_NAME is for mocking machine passes, remove it after all machine passes +// are added new pass manager interface. +#ifndef DUMMY_MACHINE_FUNCTION_PASS +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif +DUMMY_MACHINE_FUNCTION_PASS("x86-isel-dag", X86ISelDagPass, (getTM(), getOptLevel())) +DUMMY_MACHINE_FUNCTION_PASS("x86-global-basereg", X86GlobalBaseRegPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-condbr-folding", X86CondBrFoldingDummyPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-cmov-converter", X86CmovConverterDummyPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-setcc", X86FixupSetCCPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-opt-leas", X86OptimizeLEAsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-callframe-opt", X86CallFrameOptimizationPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-avoid-store-foword-block", X86AvoidStoreForwardingBlocksPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-speculative-load-hardening", X86SpeculativeLoadHardeningPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-flags-copy-lowering", X86FlagsCopyLoweringDummyPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-win-alloc-expander", X86WinAllocaExpanderPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-domain-reassign", X86DomainReassignmentPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fp-stackifier", X86FloatingPointStackifierPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-expand-pseudo", X86ExpandPseudoPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-exec-domain-fix", X86ExecutionDomainFixPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-indirectbr-tracking", X86IndirectBranchTrackingPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-issue-vzero-upper", X86IssueVZeroUpperPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-bwinsts", X86FixupBWInstsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-pad-short-funcs", X86PadShortFunctionsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-leas", X86FixupLEAsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-evex-to-vex-insts", X86EvexToVexInstsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-discriminate-memops", X86DiscriminateMemOpsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-insert-prefetch", X86InsertPrefetchPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-insert-x87-wait", X86InsertX87waitPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-retpoline-thunks", X86RetpolineThunksPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-avoid-trailing-call", X86AvoidTrailingCallPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-asm-printer", X86AsmPrinterPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("cleanup-local-dyn-tls", CleanupLocalDynamicTLSPass, ()) +#undef DUMMY_MACHINE_FUNCTION_PASS \ No newline at end of file diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h --- a/llvm/lib/Target/X86/X86TargetMachine.h +++ b/llvm/lib/Target/X86/X86TargetMachine.h @@ -49,6 +49,26 @@ // Set up the pass pipeline. TargetPassConfig *createPassConfig(PassManagerBase &PM) override; + Expected> + buildCodeGenPipeline(raw_pwrite_stream &, raw_pwrite_stream *, + CodeGenFileType, CGPassBuilderOption, + MachineFunctionAnalysisManager &, + PassInstrumentationCallbacks *) override; + + void registerModuleAnalyses(ModuleAnalysisManager &) override; + void registerFunctionAnalyses(FunctionAnalysisManager &) override; + + bool parseModulePass(StringRef, ModulePassManager &) override; + bool parseFunctionPass(StringRef, FunctionPassManager &) override; + bool parseLoopPass(StringRef, LoopPassManager &) override; + + std::pair getPassNameFromLegacyName(StringRef) override; + + Expected + parseMIRPipeline(StringRef, CGPassBuilderOption, + MachineFunctionAnalysisManager &, + PassInstrumentationCallbacks *) override; + TargetLoweringObjectFile *getObjFileLowering() const override { return TLOF.get(); } diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -26,6 +26,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/CodeGenPassBuilder.h" #include "llvm/CodeGen/ExecutionDomainFix.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" #include "llvm/CodeGen/GlobalISel/IRTranslator.h" @@ -561,3 +562,338 @@ std::unique_ptr X86PassConfig::getCSEConfig() const { return getStandardCSEConfigForOpt(TM->getOptLevel()); } + +namespace { + +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + struct PASS_NAME : public PassInfoMixin { \ + template PASS_NAME(Ts &&...) {} \ + PreservedAnalyses run(Function &, FunctionAnalysisManager &) { \ + return PreservedAnalyses::all(); \ + } \ + }; +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + struct PASS_NAME : public PassInfoMixin { \ + template PASS_NAME(Ts &&...) {} \ + PreservedAnalyses run(MachineFunction &, \ + MachineFunctionAnalysisManager &) { \ + return PreservedAnalyses::all(); \ + } \ + static AnalysisKey Key; \ + }; \ + AnalysisKey PASS_NAME::Key; +#include "X86PassRegistry.def" + +/// X86 Code Generator Pass Configuration Options. +struct X86CodeGenPassBuilder + : public CodeGenPassBuilder { + X86CodeGenPassBuilder(X86TargetMachine &TM, + CGPassBuilderOption Opt = CGPassBuilderOption(), + PassInstrumentationCallbacks *PIC = nullptr) + : CodeGenPassBuilder(TM, Opt, PIC) { + // Target-specific `CGPassBuilderOption` could be overridden here. + } + + bool parseTargetModulePass(StringRef, ModulePassManager &) const; + bool parseTargetFunctionPass(StringRef, FunctionPassManager &) const; + Error parseTargetMachinePass(StringRef, AddMachinePass &) const; + std::pair getTargetPassNameFromLegacyName(StringRef) const; + + void addIRPasses(AddIRPass &) const; + void addPreISel(AddIRPass &) const; + Error addInstSelector(AddMachinePass &) const; + Error addIRTranslator(AddMachinePass &) const; + Error addLegalizeMachineIR(AddMachinePass &) const; + Error addRegBankSelect(AddMachinePass &) const; + Error addGlobalInstructionSelect(AddMachinePass &) const; + void addILPOpts(AddMachinePass &) const; + void addMachineSSAOptimization(AddMachinePass &) const; + void addPreRegAlloc(AddMachinePass &) const; + void addPostRegAlloc(AddMachinePass &) const; + void addPreEmitPass(AddMachinePass &) const; + void addPreEmitPass2(AddMachinePass &) const; + void addPreSched2(AddMachinePass &) const; + void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const; +}; + +} // namespace + +void X86CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { + addPass(AtomicExpandPass()); + + CodeGenPassBuilder::addIRPasses(addPass); + + if (TM.getOptLevel() != CodeGenOpt::None) + addPass(InterleavedAccessPass()); + + // Add passes that handle indirect branch removal and insertion of a retpoline + // thunk. These will be a no-op unless a function subtarget has the retpoline + // feature enabled. + addPass(IndirectBrExpandPass()); + + // Add Control Flow Guard checks. + const Triple &TT = TM.getTargetTriple(); + if (TT.isOSWindows()) { + if (TT.getArch() == Triple::x86_64) { + addPass(CFGuardDispatchPass()); + } else { + addPass(CFGuardCheckPass()); + } + } +} + +Error X86CodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const { + // Install an instruction selector. + addPass(X86ISelDagPass(getTM(), getOptLevel())); + + // For ELF, cleanup any local-dynamic TLS accesses. + if (TM.getTargetTriple().isOSBinFormatELF() && + TM.getOptLevel() != CodeGenOpt::None) + addPass(CleanupLocalDynamicTLSPass()); + + addPass(X86GlobalBaseRegPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addIRTranslator(AddMachinePass &addPass) const { + addPass(IRTranslatorPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addLegalizeMachineIR( + AddMachinePass &addPass) const { + addPass(LegalizerPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addRegBankSelect(AddMachinePass &addPass) const { + addPass(RegBankSelectPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addGlobalInstructionSelect( + AddMachinePass &addPass) const { + addPass(InstructionSelectPass()); + return Error::success(); +} + +void X86CodeGenPassBuilder::addILPOpts(AddMachinePass &addPass) const { + addPass(EarlyIfConverterPass()); + if (EnableMachineCombinerPass) + addPass(MachineCombinerPass()); + addPass(X86CmovConverterDummyPass()); // TODO +} + +void X86CodeGenPassBuilder::addPreISel(AddIRPass &addPass) const { + // Only add this pass for 32-bit x86 Windows. + const Triple &TT = TM.getTargetTriple(); + if (TT.isOSWindows() && TT.getArch() == Triple::x86) + addPass(X86WinEHStatePass()); +} + +void X86CodeGenPassBuilder::addPreRegAlloc(AddMachinePass &addPass) const { + if (TM.getOptLevel() != CodeGenOpt::None) { + addPass(LiveRangeShrinkPass()); + addPass(X86FixupSetCCPass()); + addPass(X86OptimizeLEAsPass()); + addPass(X86CallFrameOptimizationPass()); + addPass(X86AvoidStoreForwardingBlocksPass()); + } + + addPass(X86SpeculativeLoadHardeningPass()); + addPass(X86FlagsCopyLoweringDummyPass()); // TODO: port to NPM and rename + addPass(X86WinAllocaExpanderPass()); +} + +void X86CodeGenPassBuilder::addMachineSSAOptimization( + AddMachinePass &addPass) const { + addPass(X86DomainReassignmentPass()); + CodeGenPassBuilder::addMachineSSAOptimization(addPass); +} + +void X86CodeGenPassBuilder::addPostRegAlloc(AddMachinePass &addPass) const { + addPass(X86FloatingPointStackifierPass()); +} + +void X86CodeGenPassBuilder::addPreSched2(AddMachinePass &addPass) const { + addPass(X86ExpandPseudoPass()); +} + +void X86CodeGenPassBuilder::addPreEmitPass(AddMachinePass &addPass) const { + if (getOptLevel() != CodeGenOpt::None) { + addPass(X86ExecutionDomainFixPass()); + addPass(BreakFalseDepsPass()); + } + + addPass(X86IndirectBranchTrackingPass()); + + addPass(X86IssueVZeroUpperPass()); + + if (getOptLevel() != CodeGenOpt::None) { + addPass(X86FixupBWInstsPass()); + addPass(X86PadShortFunctionsPass()); + addPass(X86FixupLEAsPass()); + } + addPass(X86EvexToVexInstsPass()); + addPass(X86DiscriminateMemOpsPass()); + addPass(X86InsertPrefetchPass()); + addPass(X86InsertX87waitPass()); +} + +void X86CodeGenPassBuilder::addPreEmitPass2(AddMachinePass &addPass) const { + const Triple &TT = TM.getTargetTriple(); + const MCAsmInfo *MAI = TM.getMCAsmInfo(); + + addPass(X86RetpolineThunksPass()); + + // Insert extra int3 instructions after trailing call instructions to avoid + // issues in the unwinder. + if (TT.isOSWindows() && TT.getArch() == Triple::x86_64) + addPass(X86AvoidTrailingCallPass()); + + // Verify basic block incoming and outgoing cfa offset and register values and + // correct CFA calculation rule where needed by inserting appropriate CFI + // instructions. + if (!TT.isOSDarwin() && + (!TT.isOSWindows() || + MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) + addPass(CFIInstrInserterPass()); + // Identify valid longjmp targets for Windows Control Flow Guard. + if (TT.isOSWindows()) + addPass(CFGuardLongjmpPass()); +} + +void X86CodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass, + CreateMCStreamer callback) const { + addPass(X86AsmPrinterPass(callback)); +} + +bool X86CodeGenPassBuilder::parseTargetModulePass( + StringRef Name, ModulePassManager &MPM) const { +#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) { \ + MPM.addPass(createModuleToFunctionPassAdaptor(PASS_NAME CONSTRUCTOR)); \ + return true; \ + } +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) { \ + MPM.addPass(createModuleToFunctionPassAdaptor(PASS_NAME CONSTRUCTOR)); \ + return true; \ + } +#include "X86PassRegistry.def" + + return false; +} + +bool X86CodeGenPassBuilder::parseTargetFunctionPass( + StringRef Name, FunctionPassManager &FPM) const { +#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) { \ + FPM.addPass(PASS_NAME CONSTRUCTOR); \ + return true; \ + } +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) { \ + FPM.addPass(PASS_NAME CONSTRUCTOR); \ + return true; \ + } +#include "X86PassRegistry.def" + + return false; +} + +Error X86CodeGenPassBuilder::parseTargetMachinePass( + StringRef Name, AddMachinePass &addPass) const { +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) { \ + addPass(PASS_NAME CONSTRUCTOR); \ + return Error::success(); \ + } +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) { \ + addPass(PASS_NAME CONSTRUCTOR); \ + return Error::success(); \ + } +#include "X86PassRegistry.def" + + return Error::success(); +} + +std::pair +X86CodeGenPassBuilder::getTargetPassNameFromLegacyName(StringRef Name) const { + std::pair Ret; + +#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, true}; +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, true}; +#include "X86PassRegistry.def" + + return Ret; +} + +Expected> +X86TargetMachine::buildCodeGenPipeline(raw_pwrite_stream &Out, + raw_pwrite_stream *DwoOut, + CodeGenFileType FileType, + CGPassBuilderOption Opts, + MachineFunctionAnalysisManager &MFAM, + PassInstrumentationCallbacks *PIC) { + X86CodeGenPassBuilder X86CGPB{*this, Opts, PIC}; + X86CGPB.registerAnalyses(MFAM); + return X86CGPB.buildPipeline(Out, DwoOut, FileType); +} + +void X86TargetMachine::registerModuleAnalyses(ModuleAnalysisManager &MAM) { + X86CodeGenPassBuilder X86CGPB{*this}; + X86CGPB.registerModuleAnalyses(MAM); +} + +void X86TargetMachine::registerFunctionAnalyses(FunctionAnalysisManager &FAM) { + X86CodeGenPassBuilder X86CGPB{*this}; + X86CGPB.registerFunctionAnalyses(FAM); +} + +bool X86TargetMachine::parseModulePass(StringRef Name, ModulePassManager &MPM) { + X86CodeGenPassBuilder X86CGPB{*this}; + return X86CGPB.parseModulePass(Name, MPM); +} + +bool X86TargetMachine::parseFunctionPass(StringRef Name, + FunctionPassManager &FPM) { + X86CodeGenPassBuilder X86CGPB{*this}; + return X86CGPB.parseFunctionPass(Name, FPM); +} + +bool X86TargetMachine::parseLoopPass(StringRef Name, LoopPassManager &LPM) { + X86CodeGenPassBuilder X86CGPB{*this}; + return X86CGPB.parseLoopPass(Name, LPM); +} + +std::pair +X86TargetMachine::getPassNameFromLegacyName(StringRef Name) { + X86CodeGenPassBuilder X86CGPB{*this}; + return X86CGPB.getPassNameFromLegacyName(Name); +} + +Expected X86TargetMachine::parseMIRPipeline( + StringRef PipelineText, CGPassBuilderOption Opts, + MachineFunctionAnalysisManager &MFAM, PassInstrumentationCallbacks *PIC) { + X86CodeGenPassBuilder X86CGPB{*this, Opts, PIC}; + X86CGPB.registerAnalyses(MFAM); + return X86CGPB.parseMIRPipeline(PipelineText); +}