Add support for using pass plugins from lld

[daniel_a]

Summary

This commit adds support for using new-style pass manager pass plugins from lld.

The commit touches some different parts of the source tree, mainly a configuration option is added to libLTO where an optional list of new-style pass plugins are added. If that list (vector) isn't empty and the user has requested to use a custom pass pipeline, then the pass plugins will be loaded into the passbuilder, using the same code as opt uses when loading custom passes.

Then lld is modified to take a new command line parameter --lto-newpm-load-pass-plugin which can be given multiple times to make lld load a pass plugin.

I also had to modify the CMakeLists.txt of lld to export its symbols for dynamic linking.

To be able to add tests for this functionality I had to add a sample new-style pass to llvm/lib/Transforms/HelloNewPM. The code for that plugin is taken from the article Writing an LLVM Pass in 2018 written by Minh-Yi Hsu,

I also added a test for opts loading of new pass plugins, since it didn't have one.

Of course I'm open to splitting the changes into different ones, but since I didn't want to send a patch without adding tests I made a larger patch.

[ruiu]

Adding people who knows more about LTO than me as reviewers...

[daniel_a]

lld/tools/lld/CMakeLists.txt

4

I think we need to add set(LLVM_NO_DEAD_STRIP 1) here as well, otherwise plugins won’t necessarily find the symbols they need.

[daniel_a]

I’m closing this since there’s been no activity on the patch at all since I opened it. It would be a change that would be helpful for any researcher trying to improve LTO in LLVM, but if the project isn’t interested then there’s nothing much to do, and after more than one and a half years of silence it’s hard to believe that the project would be interested.

lld/ELF/Config.h

struct Configuration {
   llvm::StringRef dwoDir;
   llvm::StringRef entry;
   llvm::StringRef emulation;
   llvm::StringRef fini;
   llvm::StringRef init;
   llvm::StringRef ltoAAPipeline;
   llvm::StringRef ltoCSProfileFile;
   llvm::StringRef ltoNewPmPasses;
+  std::vector<llvm::StringRef> ltoNewPmPlugins;
   llvm::StringRef ltoObjPath;
   llvm::StringRef ltoSampleProfile;
   llvm::StringRef mapFile;
   llvm::StringRef outputFile;
   llvm::StringRef optRemarksFilename;
   llvm::StringRef optRemarksPasses;
   llvm::StringRef optRemarksFormat;
   llvm::StringRef progName;

lld/ELF/Driver.cpp

config->ignoreFunctionAddressEquality =
       args.hasArg(OPT_ignore_function_address_equality);
   config->init = args.getLastArgValue(OPT_init, "_init");
   config->ltoAAPipeline = args.getLastArgValue(OPT_lto_aa_pipeline);
   config->ltoCSProfileGenerate = args.hasArg(OPT_lto_cs_profile_generate);
   config->ltoCSProfileFile = args.getLastArgValue(OPT_lto_cs_profile_file);
   config->ltoDebugPassManager = args.hasArg(OPT_lto_debug_pass_manager);
   config->ltoNewPassManager = args.hasArg(OPT_lto_new_pass_manager);
   config->ltoNewPmPasses = args.getLastArgValue(OPT_lto_newpm_passes);
+  config->ltoNewPmPlugins = args::getStrings(args, OPT_lto_newpm_load_pass_plugins);
   config->ltoo = args::getInteger(args, OPT_lto_O, 2);
   config->ltoObjPath = args.getLastArgValue(OPT_plugin_opt_obj_path_eq);
   config->ltoPartitions = args::getInteger(args, OPT_lto_partitions, 1);
   config->ltoSampleProfile = args.getLastArgValue(OPT_lto_sample_profile);
   config->mapFile = args.getLastArgValue(OPT_Map);
   config->mipsGotSize = args::getInteger(args, OPT_mips_got_size, 0xfff0);
   config->mergeArmExidx =
       args.hasFlag(OPT_merge_exidx_entries, OPT_no_merge_exidx_entries, true);

lld/ELF/LTO.cpp

static lto::Config createConfig() {
   c.CodeModel = getCodeModelFromCMModel();
   c.DisableVerify = config->disableVerify;
   c.DiagHandler = diagnosticHandler;
   c.OptLevel = config->ltoo;
   c.CPU = getCPUStr();
   c.MAttrs = getMAttrs();
   c.CGOptLevel = args::getCGOptLevel(config->ltoo);
 
+  for (StringRef s : config->ltoNewPmPlugins) {
+      c.OptPlugins.push_back(s);
+  }
+
   // Set up a custom pipeline if we've been asked to.
   c.OptPipeline = config->ltoNewPmPasses;
   c.AAPipeline = config->ltoAAPipeline;
 
   // Set up optimization remarks if we've been asked to.
   c.RemarksFilename = config->optRemarksFilename;
   c.RemarksPasses = config->optRemarksPasses;
   c.RemarksWithHotness = config->optRemarksWithHotness;

lld/ELF/Options.td

 def lto_aa_pipeline: J<"lto-aa-pipeline=">,
   HelpText<"AA pipeline to run during LTO. Used in conjunction with -lto-newpm-passes">;
 def lto_debug_pass_manager: F<"lto-debug-pass-manager">,
   HelpText<"Debug new pass manager">;
 def lto_new_pass_manager: F<"lto-new-pass-manager">,
   HelpText<"Use new pass manager">;
 def lto_newpm_passes: J<"lto-newpm-passes=">,
   HelpText<"Passes to run during LTO">;
+def lto_newpm_load_pass_plugins: J<"lto-newpm-load-pass-plugin=">,
+  HelpText<"New style pass plugins to load">;
 def lto_O: J<"lto-O">, MetaVarName<"<opt-level>">,
   HelpText<"Optimization level for LTO">;
 def lto_partitions: J<"lto-partitions=">,
   HelpText<"Number of LTO codegen partitions">;
 def lto_cs_profile_generate: F<"lto-cs-profile-generate">,
   HelpText<"Perform context senstive PGO instrumentation">;
 def lto_cs_profile_file: J<"lto-cs-profile-file=">,
   HelpText<"Context sensitive profile file path">;

lld/test/ELF/lto/ltopasses-load-plugin.ll

+; REQUIRES: x86
+; RUN: llvm-as %s -o %t.o
+; RUN: ld.lld %t.o -o %t%shlibext -save-temps --lto-newpm-passes=hello-new-pm \
+; RUN:    --lto-newpm-load-pass-plugin=%llvmshlibdir/LLVMHelloNewPM%shlibext 2>&1 | grep Hello
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@junk = global i32 0
+
+define i32* @somefunk() {
+  ret i32* @junk
+}

lld/test/lit.cfg.py

 
 tool_patterns = [
     'llc', 'llvm-as', 'llvm-mc', 'llvm-nm', 'llvm-objdump', 'llvm-pdbutil',
     'llvm-dwarfdump', 'llvm-readelf', 'llvm-readobj', 'obj2yaml', 'yaml2obj',
     'opt', 'llvm-dis']
 
 llvm_config.add_tool_substitutions(tool_patterns)
 
+# Add site-specific substitutions
+config.substitutions.append(('%llvmshlibdir', config.llvm_shlib_dir))
+config.substitutions.append(('%shlibext', config.llvm_shlib_ext))
+config.substitutions.append(('%exeext', config.llvm_exe_ext))
+
 # LLD tests tend to be flaky on NetBSD, so add some retries.
 # We don't do this on other platforms because it's slower.
 if platform.system() in ['NetBSD']:
     config.test_retry_attempts = 2
 
 # When running under valgrind, we mangle '-vg' onto the end of the triple so we
 # can check it with XFAIL and XTARGET.
 if lit_config.useValgrind:

lld/test/lit.site.cfg.py.in

 @LIT_SITE_CFG_IN_HEADER@
 
 import lit.util
 
 config.have_dia_sdk = lit.util.pythonize_bool("@LLVM_ENABLE_DIA_SDK@")
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_obj_root = "@LLVM_BINARY_DIR@"
 config.llvm_tools_dir = "@LLVM_TOOLS_DIR@"
 config.llvm_libs_dir = "@LLVM_LIBS_DIR@"
 config.llvm_libxml2_enabled = @LLVM_LIBXML2_ENABLED@
+config.llvm_shlib_dir = "@SHLIBDIR@"
+config.llvm_shlib_ext = "@SHLIBEXT@"
+config.llvm_exe_ext = "@EXEEXT@"
 config.lit_tools_dir = "@LLVM_LIT_TOOLS_DIR@"
 config.lld_obj_root = "@LLD_BINARY_DIR@"
 config.lld_libs_dir = "@LLVM_LIBRARY_OUTPUT_INTDIR@"
 config.lld_tools_dir = "@LLVM_RUNTIME_OUTPUT_INTDIR@"
 config.target_triple = "@TARGET_TRIPLE@"
 config.python_executable = "@PYTHON_EXECUTABLE@"
 config.have_zlib = @HAVE_LIBZ@
 config.sizeof_void_p = @CMAKE_SIZEOF_VOID_P@

lld/tools/lld/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   Support
   )
 
 add_lld_tool(lld
   lld.cpp
   )
+export_executable_symbols(lld)
 
 target_link_libraries(lld
   PRIVATE
   lldCommon
   lldCOFF
   lldDriver
   lldELF
   lldMinGW

llvm/include/llvm/LTO/Config.h

struct Config {
   bool Freestanding = false;
 
   /// Disable entirely the optimizer, including importing for ThinLTO
   bool CodeGenOnly = false;
 
   /// Run PGO context sensitive IR instrumentation.
   bool RunCSIRInstr = false;
 
+  /// A list of optional pass plugins to load into the pass manager.
+  /// These will only be used if a custom OptPipeline has been set,
+  /// since otherwise no newly defined passes are going to be used anyway.
+  std::vector<std::string> OptPlugins;
+
   /// If this field is set, the set of passes run in the middle-end optimizer
   /// will be the one specified by the string. Only works with the new pass
   /// manager as the old one doesn't have this ability.
   std::string OptPipeline;
 
   // If this field is set, it has the same effect of specifying an AA pipeline
   // identified by the string. Only works with the new pass manager, in
   // conjunction OptPipeline.
LTOLLVMContext(const Config &C) : DiagHandler(C.DiagHandler) {
     setDiscardValueNames(C.ShouldDiscardValueNames);
     enableDebugTypeODRUniquing();
     setDiagnosticHandler(
         std::make_unique<LTOLLVMDiagnosticHandler>(&DiagHandler), true);
   }
   DiagnosticHandlerFunction DiagHandler;
 };
 
-}
-}
+} // namespace lto
+} // namespace llvm
 
 #endif

llvm/lib/LTO/LTOBackend.cpp

 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/RemarkStreamer.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Object/ModuleSymbolTable.h"
 #include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/PassPlugin.h"
 #include "llvm/Passes/StandardInstrumentations.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/ThreadPool.h"
static void runNewPMPasses(Config &Conf, Module &Mod, TargetMachine *TM,
   } else if (!Conf.CSIRProfile.empty()) {
     PGOOpt = PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping,
                         PGOOptions::IRUse, PGOOptions::CSIRUse);
   }
 
   PassInstrumentationCallbacks PIC;
   StandardInstrumentations SI;
   SI.registerCallbacks(PIC);
-  PassBuilder PB(TM, PipelineTuningOptions(),PGOOpt, &PIC);
+  PassBuilder PB(TM, PipelineTuningOptions(), PGOOpt, &PIC);
   AAManager AA;
 
   // Parse a custom AA pipeline if asked to.
   if (auto Err = PB.parseAAPipeline(AA, "default"))
     report_fatal_error("Error parsing default AA pipeline");
 
   LoopAnalysisManager LAM(Conf.DebugPassManager);
   FunctionAnalysisManager FAM(Conf.DebugPassManager);
static void runNewPMPasses(Config &Conf, Module &Mod, TargetMachine *TM,
   else
     MPM = PB.buildLTODefaultPipeline(OL, Conf.DebugPassManager, ExportSummary);
   MPM.run(Mod, MAM);
 
   // FIXME (davide): verify the output.
 }
 
 static void runNewPMCustomPasses(Module &Mod, TargetMachine *TM,
+                                 std::vector<std::string> &PassPlugins,
                                  std::string PipelineDesc,
                                  std::string AAPipelineDesc,
                                  bool DisableVerify) {
   PassBuilder PB(TM);
   AAManager AA;
 
   // Parse a custom AA pipeline if asked to.
   if (!AAPipelineDesc.empty())
static void runNewPMCustomPasses(Module &Mod, TargetMachine *TM,
   PB.registerLoopAnalyses(LAM);
   PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
 
   ModulePassManager MPM;
 
   // Always verify the input.
   MPM.addPass(VerifierPass());
 
+  // Load requested pass plugins and let them register pass builder callbacks
+  for (auto &PluginFN : PassPlugins) {
+    auto PassPlugin = PassPlugin::Load(PluginFN);
+    if (!PassPlugin) {
+      errs() << "Failed to load passes from '" << PluginFN
+             << "'. Request ignored.\n";
+      errs() << PassPlugin.takeError() << '\n';
+      continue;
+    }
+
+    PassPlugin->registerPassBuilderCallbacks(PB);
+  }
+
   // Now, add all the passes we've been requested to.
   if (auto Err = PB.parsePassPipeline(MPM, PipelineDesc))
     report_fatal_error("unable to parse pass pipeline description '" +
                        PipelineDesc + "': " + toString(std::move(Err)));
 
   if (!DisableVerify)
     MPM.addPass(VerifierPass());
   MPM.run(Mod, MAM);
static void runOldPMPasses(Config &Conf, Module &Mod, TargetMachine *TM,
   passes.run(Mod);
 }
 
 bool opt(Config &Conf, TargetMachine *TM, unsigned Task, Module &Mod,
          bool IsThinLTO, ModuleSummaryIndex *ExportSummary,
          const ModuleSummaryIndex *ImportSummary) {
   // FIXME: Plumb the combined index into the new pass manager.
   if (!Conf.OptPipeline.empty())
-    runNewPMCustomPasses(Mod, TM, Conf.OptPipeline, Conf.AAPipeline,
-                         Conf.DisableVerify);
+    runNewPMCustomPasses(Mod, TM, Conf.OptPlugins, Conf.OptPipeline,
+                         Conf.AAPipeline, Conf.DisableVerify);
   else if (Conf.UseNewPM)
     runNewPMPasses(Conf, Mod, TM, Conf.OptLevel, IsThinLTO, ExportSummary,
                    ImportSummary);
   else
     runOldPMPasses(Conf, Mod, TM, IsThinLTO, ExportSummary, ImportSummary);
   return !Conf.PostOptModuleHook || Conf.PostOptModuleHook(Task, Mod);
 }
 
Expected<const Target *> initAndLookupTarget(Config &C, Module &Mod) {
 
   std::string Msg;
   const Target *T = TargetRegistry::lookupTarget(Mod.getTargetTriple(), Msg);
   if (!T)
     return make_error<StringError>(Msg, inconvertibleErrorCode());
   return T;
 }
 
-}
+} // namespace
 
 static Error
 finalizeOptimizationRemarks(std::unique_ptr<ToolOutputFile> DiagOutputFile) {
   // Make sure we flush the diagnostic remarks file in case the linker doesn't
   // call the global destructors before exiting.
   if (!DiagOutputFile)
     return Error::success();
   DiagOutputFile->keep();
if (ParallelCodeGenParallelismLevel == 1) {
     splitCodeGen(C, TM.get(), AddStream, ParallelCodeGenParallelismLevel,
                  std::move(Mod));
   }
   return finalizeOptimizationRemarks(std::move(DiagnosticOutputFile));
 }
 
 static void dropDeadSymbols(Module &Mod, const GVSummaryMapTy &DefinedGlobals,
                             const ModuleSummaryIndex &Index) {
-  std::vector<GlobalValue*> DeadGVs;
+  std::vector<GlobalValue *> DeadGVs;
   for (auto &GV : Mod.global_values())
     if (GlobalValueSummary *GVS = DefinedGlobals.lookup(GV.getGUID()))
       if (!Index.isGlobalValueLive(GVS)) {
         DeadGVs.push_back(&GV);
         convertToDeclaration(GV);
       }
 
   // Now that all dead bodies have been dropped, delete the actual objects

llvm/lib/Transforms/CMakeLists.txt

 add_subdirectory(Utils)
 add_subdirectory(Instrumentation)
 add_subdirectory(AggressiveInstCombine)
 add_subdirectory(InstCombine)
 add_subdirectory(Scalar)
 add_subdirectory(IPO)
 add_subdirectory(Vectorize)
 add_subdirectory(Hello)
+add_subdirectory(HelloNewPM)
 add_subdirectory(ObjCARC)
 add_subdirectory(Coroutines)

llvm/lib/Transforms/HelloNewPM/CMakeLists.txt

+if(WIN32 OR CYGWIN)
+  set(LLVM_LINK_COMPONENTS Core Support)
+endif()
+
+add_llvm_library( LLVMHelloNewPM MODULE BUILDTREE_ONLY
+  HelloNewPM.cpp
+
+  DEPENDS
+  intrinsics_gen
+  PLUGIN_TOOL
+  opt
+  )

llvm/lib/Transforms/HelloNewPM/HelloNewPM.cpp

+#include "llvm/IR/PassManager.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Passes/PassPlugin.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+
+namespace {
+
+struct HelloNewPMPass : public PassInfoMixin<HelloNewPMPass> {
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) {
+    if (F.hasName()) {
+      errs() << "Hello " << F.getName() << '\n';
+    }
+    return PreservedAnalyses::all();
+  }
+};
+
+} // namespace
+
+extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK
+llvmGetPassPluginInfo() {
+  return {LLVM_PLUGIN_API_VERSION, "HelloNewPMPass", "v0.1",
+          [](PassBuilder &PB) {
+            PB.registerPipelineParsingCallback(
+                [](StringRef Name, FunctionPassManager &FPM,
+                   ArrayRef<PassBuilder::PipelineElement>) {
+                  if (Name == "hello-new-pm") {
+                    FPM.addPass(HelloNewPMPass());
+                    return true;
+                  }
+                  return false;
+                });
+          }};
+}

llvm/test/Feature/load_module_newpm.ll

+; RUN: opt < %s -load-pass-plugin=%llvmshlibdir/LLVMHelloNewPM%shlibext --passes=hello-new-pm \
+; RUN:   -disable-output 2>&1 | grep Hello
+; REQUIRES: plugins
+; FIXME: On Cygming, it might fail without building LLVMHello manually.
+
+@junk = global i32 0
+
+define i32* @somefunk() {
+  ret i32* @junk
+}
+