Teach llvm-jitlink to support archives in inputs files and -load_hidden

[benlangmuir]

Summary

Similar to the ld64 command-line options. These use the same underlying mechanisms as -l and -hidden-l, but allow specifying an absolute path to the archive. This is often more convenient for a one-off, or when adding a new search path could change how existing -l options are resolved.

[lhames]

LGTM. Thanks Ben!

llvm/test/ExecutionEngine/JITLink/X86/MachO_archive_load_hidden_expect_success.s

 # RUN: rm -rf %t && mkdir -p %t
 # RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
 # RUN:   -o %t/MachO_extra_def_strong.o %S/Inputs/MachO_extra_def_strong.s
 # RUN: llvm-ar crs %t/libExtraDef.a %t/MachO_extra_def_strong.o
 # RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
 # RUN:   -o %t/MachO_archive_load_hidden_support.o %s
 # RUN: llvm-jitlink -noexec %t/MachO_archive_load_hidden_support.o \
 # RUN:   -L%t -hidden-lExtraDef
+# RUN: llvm-jitlink -noexec %t/MachO_archive_load_hidden_support.o \
+# RUN:   -load_hidden %t/libExtraDef.a
 #
 # Expect this test to succeed -- ExtraDef should be hidden, but visible to
 # ExtraDefRef as they're linked in the same JITDylib. This tests that we're
 # correctly handling the change in object interface when linking ExtraDef.
 
         .section  __TEXT,__text,regular,pure_instructions
 
         .globl  _main

llvm/test/ExecutionEngine/JITLink/X86/MachO_archive_support.s

 # RUN: rm -rf %t && mkdir -p %t
 # RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
 # RUN:   -o %t/MachO_extra_def_strong.o %S/Inputs/MachO_extra_def_strong.s
 # RUN: llvm-ar crs %t/libExtraDef.a %t/MachO_extra_def_strong.o
 # RUN: llvm-mc -triple x86_64-apple-macosx10.9 -filetype=obj \
 # RUN:   -o %t/MachO_archive_support.o %s
 # RUN: llvm-jitlink -noexec %t/MachO_archive_support.o -lFoo -jd Foo -L%t -lExtraDef
+# RUN: llvm-jitlink -noexec %t/MachO_archive_support.o -lFoo -jd Foo %t/libExtraDef.a
 #
 # Test that archives can be loaded and referenced from other JITDylibs.
 
         .section  __TEXT,__text,regular,pure_instructions
 
         .globl  _main
         .p2align  4, 0x90
 _main:
         retq
 
   .section  __DATA,__data
   .globl  ExtraDefRef
   .p2align  3
 ExtraDefRef:
   .quad ExtraDef

llvm/tools/llvm-jitlink/llvm-jitlink.cpp

LibrarySearchPaths("L",
                        cl::desc("Add dir to the list of library search paths"),
                        cl::Prefix, cl::cat(JITLinkCategory));
 
 static cl::list<std::string>
     Libraries("l",
               cl::desc("Link against library X in the library search paths"),
               cl::Prefix, cl::cat(JITLinkCategory));
 
-static cl::list<std::string> LibrariesHidden(
-    "hidden-l", cl::desc("Link against library X in the library search paths"),
+static cl::list<std::string>
+    LibrariesHidden("hidden-l",
+                    cl::desc("Link against library X in the library search "
+                             "paths with hidden visibility"),
                     cl::Prefix, cl::cat(JITLinkCategory));
 
+static cl::list<std::string>
+    LoadHidden("load_hidden",
+               cl::desc("Link against library X with hidden visibility"),
+               cl::cat(JITLinkCategory));
+
 static cl::opt<bool> NoExec("noexec", cl::desc("Do not execute loaded code"),
                             cl::init(false), cl::cat(JITLinkCategory));
 
 static cl::list<std::string>
     CheckFiles("check", cl::desc("File containing verifier checks"),
                cl::ZeroOrMore, cl::cat(JITLinkCategory));
 
 static cl::opt<std::string>
static Error addObjects(Session &S,
 
   // Load each object into the corresponding JITDylib..
   LLVM_DEBUG(dbgs() << "Adding objects...\n");
   for (auto InputFileItr = InputFiles.begin(), InputFileEnd = InputFiles.end();
        InputFileItr != InputFileEnd; ++InputFileItr) {
     unsigned InputFileArgIdx =
         InputFiles.getPosition(InputFileItr - InputFiles.begin());
     const std::string &InputFile = *InputFileItr;
+    if (StringRef(InputFile).endswith(".a"))
+      continue;
     auto &JD = *std::prev(IdxToJD.lower_bound(InputFileArgIdx))->second;
     LLVM_DEBUG(dbgs() << "  " << InputFileArgIdx << ": \"" << InputFile
                       << "\" to " << JD.getName() << "\n";);
     auto ObjBuffer = errorOrToExpected(MemoryBuffer::getFile(InputFile));
     if (!ObjBuffer)
       return ObjBuffer.takeError();
 
     if (S.HarnessFiles.empty()) {
LLVM_DEBUG({
     for (auto &KV : JDSearchPaths) {
       dbgs() << "  " << KV.first->getName() << ": [";
       for (auto &LibSearchPath : KV.second)
         dbgs() << " \"" << LibSearchPath << "\"";
       dbgs() << " ]\n";
     }
   });
 
-  // 2. Collect library loads from -lx, -hidden-lx.
+  // 2. Collect library loads
   struct LibraryLoad {
     StringRef LibName;
+    bool IsPath = false;
     unsigned Position;
     StringRef *CandidateExtensions;
     enum { Standard, Hidden } Modifier;
   };
   std::vector<LibraryLoad> LibraryLoads;
+  // Add archive files from the inputs to LibraryLoads.
+  for (auto InputFileItr = InputFiles.begin(), InputFileEnd = InputFiles.end();
+       InputFileItr != InputFileEnd; ++InputFileItr) {
+    StringRef InputFile = *InputFileItr;
+    if (!InputFile.endswith(".a"))
+      continue;
+    LibraryLoad LL;
+    LL.LibName = InputFile;
+    LL.IsPath = true;
+    LL.Position = InputFiles.getPosition(InputFileItr - InputFiles.begin());
+    LL.CandidateExtensions = nullptr;
+    LL.Modifier = LibraryLoad::Standard;
+    LibraryLoads.push_back(std::move(LL));
+  }
+
+  // Add -load_hidden arguments to LibraryLoads.
+  for (auto LibItr = LoadHidden.begin(), LibEnd = LoadHidden.end();
+       LibItr != LibEnd; ++LibItr) {
+    LibraryLoad LL;
+    LL.LibName = *LibItr;
+    LL.IsPath = true;
+    LL.Position = LoadHidden.getPosition(LibItr - LoadHidden.begin());
+    LL.CandidateExtensions = nullptr;
+    LL.Modifier = LibraryLoad::Hidden;
+    LibraryLoads.push_back(std::move(LL));
+  }
   StringRef StandardExtensions[] = {".so", ".dylib", ".a"};
   StringRef ArchiveExtensionsOnly[] = {".a"};
 
   // Add -lx arguments to LibraryLoads.
   for (auto LibItr = Libraries.begin(), LibEnd = Libraries.end();
        LibItr != LibEnd; ++LibItr) {
     LibraryLoad LL;
     LL.LibName = *LibItr;
LL.Position =
         LibrariesHidden.getPosition(LibHiddenItr - LibrariesHidden.begin());
     LL.CandidateExtensions = ArchiveExtensionsOnly;
     LL.Modifier = LibraryLoad::Hidden;
     LibraryLoads.push_back(std::move(LL));
   }
 
   // If there are any load-<modified> options then turn on flag overrides
   // to avoid flag mismatch errors.
-  if (!LibrariesHidden.empty())
+  if (!LibrariesHidden.empty() || !LoadHidden.empty())
     S.ObjLayer.setOverrideObjectFlagsWithResponsibilityFlags(true);
 
   // Sort library loads by position in the argument list.
   llvm::sort(LibraryLoads, [](const LibraryLoad &LHS, const LibraryLoad &RHS) {
     return LHS.Position < RHS.Position;
   });
 
   // 3. Process library loads.
+  auto AddArchive = [&](const char *Path, const LibraryLoad &LL)
+      -> Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> {
+    unique_function<Expected<MaterializationUnit::Interface>(
+        ExecutionSession & ES, MemoryBufferRef ObjBuffer)>
+        GetObjFileInterface;
+    switch (LL.Modifier) {
+    case LibraryLoad::Standard:
+      GetObjFileInterface = getObjectFileInterface;
+      break;
+    case LibraryLoad::Hidden:
+      GetObjFileInterface = getObjectFileInterfaceHidden;
+      break;
+    }
+    return StaticLibraryDefinitionGenerator::Load(
+        S.ObjLayer, Path, S.ES.getExecutorProcessControl().getTargetTriple(),
+        std::move(GetObjFileInterface));
+  };
+
   for (auto &LL : LibraryLoads) {
     bool LibFound = false;
     auto &JD = *std::prev(IdxToJD.lower_bound(LL.Position))->second;
 
     // If this is the name of a JITDylib then link against that.
     if (auto *LJD = S.ES.getJITDylibByName(LL.LibName)) {
       JD.addToLinkOrder(*LJD);
       continue;
     }
 
+    if (LL.IsPath) {
+      auto G = AddArchive(LL.LibName.str().c_str(), LL);
+      if (!G)
+        return createFileError(LL.LibName, G.takeError());
+      JD.addGenerator(std::move(*G));
+      LLVM_DEBUG({
+        dbgs() << "Adding generator for static library " << LL.LibName << " to "
+               << JD.getName() << "\n";
+      });
+      continue;
+    }
+
     // Otherwise look through the search paths.
     auto JDSearchPathsItr = JDSearchPaths.find(&JD);
     if (JDSearchPathsItr != JDSearchPaths.end()) {
       for (StringRef SearchPath : JDSearchPathsItr->second) {
         for (const char *LibExt : {".dylib", ".so", ".a"}) {
           SmallVector<char, 256> LibPath;
           LibPath.reserve(SearchPath.size() + strlen("lib") +
                           LL.LibName.size() + strlen(LibExt) +
if (JDSearchPathsItr != JDSearchPaths.end()) {
               dbgs() << "Adding generator for dynamic library "
                      << LibPath.data() << " to " << JD.getName() << "\n";
             });
             JD.addGenerator(std::move(*G));
             break;
           }
           case file_magic::archive:
           case file_magic::macho_universal_binary: {
-            unique_function<Expected<MaterializationUnit::Interface>(
-                ExecutionSession & ES, MemoryBufferRef ObjBuffer)>
-                GetObjFileInterface;
-            switch (LL.Modifier) {
-            case LibraryLoad::Standard:
-              GetObjFileInterface = getObjectFileInterface;
-              break;
-            case LibraryLoad::Hidden:
-              GetObjFileInterface = getObjectFileInterfaceHidden;
-              break;
-            }
-            auto G = StaticLibraryDefinitionGenerator::Load(
-                S.ObjLayer, LibPath.data(),
-                S.ES.getExecutorProcessControl().getTargetTriple(),
-                std::move(GetObjFileInterface));
+            auto G = AddArchive(LibPath.data(), LL);
             if (!G)
               return G.takeError();
             JD.addGenerator(std::move(*G));
             LLVM_DEBUG({
               dbgs() << "Adding generator for static library " << LibPath.data()
                      << " to " << JD.getName() << "\n";
             });
             break;