[LTO][Legacy] Add new C inferface to query libcall functions

[steven_wu]

Summary

This is needed to implemented the same approach as lld (implemented in r338434)
for how to handling symbols that can be generated by LTO code generator
but not present in the symbol table for linker that uses legacy C APIs.

libLTO is in charge of providing the list of symbols. Linker is in
charge of implementing the eager loading from static libraries using
the list of symbols.

rdar://problem/52853974

[tejohnson]

How are these interfaces usually tested?

llvm/include/llvm-c/lto.h

806

s/visile/visible

[steven_wu]

lto.cpp is not really tested. Maybe we can test this by move this into lib/LTO then integrate the new version into lld?

[kledzik]

How many symbols are there today? Do we want this overhead of a function call for each? Could you instead return a count and array pointer (like argc and argv) or have a "foreach" function that takes a block and calls it back for each symbol?

[steven_wu]

In D67568#1670049, @kledzik wrote:

How many symbols are there today? Do we want this overhead of a function call for each? Could you instead return a count and array pointer (like argc and argv) or have a "foreach" function that takes a block and calls it back for each symbol?

I am just following the old convention. The number of symbols are around 500. foreach is not that cheaper because it is one function call per iteration. I am fine with either solution.

[dexonsmith]

In D67568#1670131, @steven_wu wrote:

In D67568#1670049, @kledzik wrote:

How many symbols are there today? Do we want this overhead of a function call for each? Could you instead return a count and array pointer (like argc and argv) or have a "foreach" function that takes a block and calls it back for each symbol?

I am just following the old convention. The number of symbols are around 500. foreach is not that cheaper because it is one function call per iteration. I am fine with either solution.

Adding an argv equivalent seems reasonable to me:

unsigned int lto_get_num_runtime_lib_symbols(void);
const char **lto_get_runtime_lib_symbol_names(void);

That locks down the implementation as an array of const char * but that's probably fine.

llvm/include/llvm-c/lto.h

812

This is C, so you need (void) here to have a prototype.

[steven_wu]

• [LTO][Legacy] Add new C inferface to query libcall functions
• [lld] Update lld to use the new LTO API to add libcall symbols

[tejohnson]

LGTM

[steven_wu]

I break this into two commits. r372021 and r372022.

lld/COFF/Driver.cpp

 #include "lld/Common/Filesystem.h"
 #include "lld/Common/Memory.h"
 #include "lld/Common/Threads.h"
 #include "lld/Common/Timer.h"
 #include "lld/Common/Version.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/BinaryFormat/Magic.h"
+#include "llvm/LTO/LTO.h"
 #include "llvm/Object/ArchiveWriter.h"
 #include "llvm/Object/COFFImportFile.h"
 #include "llvm/Object/COFFModuleDefinition.h"
 #include "llvm/Object/WindowsMachineFlag.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/Debug.h"
symtab->forEachSymbol([&](Symbol *s) {
     e.sym = def;
     if (Chunk *c = def->getChunk())
       if (!(c->getOutputCharacteristics() & IMAGE_SCN_MEM_EXECUTE))
         e.data = true;
     config->exports.push_back(e);
   });
 }
 
-static const char *libcallRoutineNames[] = {
-#define HANDLE_LIBCALL(code, name) name,
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-};
-
 void LinkerDriver::link(ArrayRef<const char *> argsArr) {
   // Needed for LTO.
   InitializeAllTargetInfos();
   InitializeAllTargets();
   InitializeAllTargetMCs();
   InitializeAllAsmParsers();
   InitializeAllAsmPrinters();
 
do {
     }
 
     // If any inputs are bitcode files, the LTO code generator may create
     // references to library functions that are not explicit in the bitcode
     // file's symbol table. If any of those library functions are defined in a
     // bitcode file in an archive member, we need to arrange to use LTO to
     // compile those archive members by adding them to the link beforehand.
     if (!BitcodeFile::instances.empty())
-      for (const char *s : libcallRoutineNames)
+      for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
         symtab->addLibcall(s);
 
     // Windows specific -- if __load_config_used can be resolved, resolve it.
     if (symtab->findUnderscore("_load_config_used"))
       addUndefined(mangle("_load_config_used"));
   } while (run());
 
   if (args.hasArg(OPT_include_optional)) {

lld/ELF/Driver.cpp

 #include "lld/Common/Memory.h"
 #include "lld/Common/Strings.h"
 #include "lld/Common/TargetOptionsCommandFlags.h"
 #include "lld/Common/Threads.h"
 #include "lld/Common/Version.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/LTO/LTO.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Compression.h"
 #include "llvm/Support/GlobPattern.h"
 #include "llvm/Support/LEB128.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/TarWriter.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/raw_ostream.h"
template <class ELFT> static uint32_t getAndFeatures() {
   // Force enable pointer authentication Plt, we don't warn in this case as
   // this does not require support in the object for correctness.
   if (config->pacPlt)
     ret |= GNU_PROPERTY_AARCH64_FEATURE_1_PAC;
 
   return ret;
 }
 
-static const char *libcallRoutineNames[] = {
-#define HANDLE_LIBCALL(code, name) name,
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-};
-
 // Do actual linking. Note that when this function is called,
 // all linker scripts have already been parsed.
 template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // If a -hash-style option was not given, set to a default value,
   // which varies depending on the target.
   if (!args.hasArg(OPT_hash_style)) {
     if (config->emachine == EM_MIPS)
       config->sysvHash = true;
template <class ELFT> void LinkerDriver::link(opt::InputArgList &args) {
   // atomics, which would prevent the program from running even if it does not
   // use 64-bit atomics.
   //
   // Therefore, we only add libcall symbols to the link before LTO if we have
   // to, i.e. if the symbol's definition is in bitcode. Any other required
   // libcall symbols will be added to the link after LTO when we add the LTO
   // object file to the link.
   if (!bitcodeFiles.empty())
-    for (const char *s : libcallRoutineNames)
+    for (auto *s : lto::LTO::getRuntimeLibcallSymbols())
       handleLibcall(s);
 
   // Return if there were name resolution errors.
   if (errorCount())
     return;
 
   // Now when we read all script files, we want to finalize order of linker
   // script commands, which can be not yet final because of INSERT commands.

llvm/include/llvm-c/lto.h

 
 /**
  * @defgroup LLVMCLTO LTO
  * @ingroup LLVMC
  *
  * @{
  */
 
-#define LTO_API_VERSION 24
+#define LTO_API_VERSION 25
 
 /**
  * \since prior to LTO_API_VERSION=3
  */
 typedef enum {
     LTO_SYMBOL_ALIGNMENT_MASK              = 0x0000001F, /* log2 of alignment */
     LTO_SYMBOL_PERMISSIONS_MASK            = 0x000000E0,
     LTO_SYMBOL_PERMISSIONS_CODE            = 0x000000A0,
/**
   *
   * \since LTO_API_VERSION=24
   */
 extern const char * lto_input_get_dependent_library(lto_input_t input,
                                                     size_t index,
                                                     size_t *size);
 
 /**
+ * Returns the list of libcall symbols that can be generated by LTO
+ * that might not be visible from the symbol table of bitcode files.
+ *
+ * \since prior to LTO_API_VERSION=25
+ */
+extern const char *const *lto_runtime_lib_symbols_list(size_t *size);
+
+/**
  * @} // endgoup LLVMCLTO
  * @defgroup LLVMCTLTO ThinLTO
  * @ingroup LLVMC
  *
  * @{
  */
 
 /**

  * \since LTO_API_VERSION=18
  */
 extern void thinlto_codegen_add_cross_referenced_symbol(thinlto_code_gen_t cg,
                                                         const char *name,
                                                         int length);
 
 /**
  * @} // endgoup LLVMCTLTO
  * @defgroup LLVMCTLTO_CACHING ThinLTO Cache Control
  * @ingroup LLVMCTLTO
  *
  * These entry points control the ThinLTO cache. The cache is intended to
  * support incremental builds, and thus needs to be persistent across builds.
  * The client enables the cache by supplying a path to an existing directory.
  * The code generator will use this to store objects files that may be reused
  * during a subsequent build.
  * To avoid filling the disk space, a few knobs are provided:
  *  - The pruning interval limits the frequency at which the garbage collector
  *    will try to scan the cache directory to prune expired entries.
  *    Setting to a negative number disables the pruning.
  *  - The pruning expiration time indicates to the garbage collector how old an
  *    entry needs to be to be removed.
  *  - Finally, the garbage collector can be instructed to prune the cache until

llvm/include/llvm/LTO/LTO.h

public:
   ///
   /// The Cache parameter is optional. If supplied, it will be used to cache
   /// native object files and add them to the link.
   ///
   /// The client will receive at most one callback (via either AddStream or
   /// Cache) for each task identifier.
   Error run(AddStreamFn AddStream, NativeObjectCache Cache = nullptr);
 
+  /// Static method that returns a list of libcall symbols that can be generated
+  /// by LTO but might not be visible from bitcode symbol table.
+  static ArrayRef<const char*> getRuntimeLibcallSymbols();
+
 private:
   Config Conf;
 
   struct RegularLTOState {
     RegularLTOState(unsigned ParallelCodeGenParallelismLevel, Config &Conf);
     struct CommonResolution {
       uint64_t Size = 0;
       unsigned Align = 0;

llvm/lib/LTO/LTO.cpp

if (!Conf.CodeGenOnly) {
     if (Conf.PostInternalizeModuleHook &&
         !Conf.PostInternalizeModuleHook(0, *RegularLTO.CombinedModule))
       return Error::success();
   }
   return backend(Conf, AddStream, RegularLTO.ParallelCodeGenParallelismLevel,
                  std::move(RegularLTO.CombinedModule), ThinLTO.CombinedIndex);
 }
 
+static const char *libcallRoutineNames[] = {
+#define HANDLE_LIBCALL(code, name) name,
+#include "llvm/IR/RuntimeLibcalls.def"
+#undef HANDLE_LIBCALL
+};
+
+ArrayRef<const char*> LTO::getRuntimeLibcallSymbols() {
+  return makeArrayRef(libcallRoutineNames);
+}
+
 /// This class defines the interface to the ThinLTO backend.
 class lto::ThinBackendProc {
 protected:
   Config &Conf;
   ModuleSummaryIndex &CombinedIndex;
   const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries;
 
 public:

llvm/tools/lto/lto.cpp

extern unsigned lto_input_get_num_dependent_libraries(lto_input_t input) {
   return LTOModule::getDependentLibraryCount(unwrap(input));
 }
 
 extern const char *lto_input_get_dependent_library(lto_input_t input,
                                                    size_t index,
                                                    size_t *size) {
   return LTOModule::getDependentLibrary(unwrap(input), index, size);
 }
+
+extern const char *const *lto_runtime_lib_symbols_list(size_t *size) {
+  auto symbols = lto::LTO::getRuntimeLibcallSymbols();
+  *size = symbols.size();
+  return symbols.data();
+}

llvm/tools/lto/lto.exports

 thinlto_codegen_disable_codegen
 thinlto_module_get_num_object_files
 thinlto_module_get_object_file
 thinlto_set_generated_objects_dir
 lto_input_create
 lto_input_dispose
 lto_input_get_num_dependent_libraries
 lto_input_get_dependent_library
+lto_runtime_lib_symbols_list