Implement --start-lib and --end-lib.

[ruiu]

Summary

Implement --start-lib and --end-lib.

[• rafael]

General idea sounds reasonable, but we have to make sure it works with bitcode too.

ELF/InputFiles.cpp

285

Why do you need this?

ELF/InputFiles.h

44

So, --start-lib foo.bc --end-lib should work. This suggests that it should be a flag in the class saying that it is lazy, not a new type.

164

It should be possible to use bitcode files in --start-lib/--end-lib, so this should inherit from InputFile, like ArchiveFile does

It can then have two subclasses or just know that it can be wrapping a .o or a .bc.

[silvas]

Have you tried adding a debug mode that internally translates regular archives into --start-lib/--end-lib and see which tests fail?

[ruiu]

That's a good idea. I'll try.

[ruiu]

Updated as per review comments.

ELF/InputFiles.cpp

285

Removed.

ELF/InputFiles.h

44

Reusing the existing File classes for lazy files was not easy as I expected. For example, the existing parse function takes a list of COMDAT group names. When creating lazy files, we don't want to read or write the list because parsing a lazy file should have any semantic meaning. Therefore, it seems to be easier to keep LazyObjectFile as a separate class than making a flag.

164

Done.

[ruiu]

This patch worked when I made a change to handle each file in an archive file using LazyObjectFile instead of ArchiveFile (with a few exception related to error messages containing an archive file name), so it looks like it's working fine for regular object files and bitcode files.

[• rafael]

This is looking really nice. If needed we can try optimizing not parsing symbols twice in the future.

LGTM with some nits and a bitcode test added.

ELF/InputFiles.cpp

585

Can you use Syms.slice(FirstNonLocal)?

599

You probably want to use BitcodeFile::shouldSkip.

ELF/SymbolTable.h

90

Why change this?

[ruiu]

Submitted as r265710 with an additional test for bitcode input files.

ELF/InputFiles.cpp

585

Done.

599

Done.

ELF/SymbolTable.h

90

That was a mistake. Reverted.

ELF/Driver.h

public:
   void addLibrary(StringRef Name);
 
 private:
   std::vector<MemoryBufferRef> getArchiveMembers(MemoryBufferRef MB);
   void readConfigs(llvm::opt::InputArgList &Args);
   void createFiles(llvm::opt::InputArgList &Args);
   template <class ELFT> void link(llvm::opt::InputArgList &Args);
 
-  llvm::BumpPtrAllocator Alloc;
+  // True if we are in --whole-archive and --no-whole-archive.
   bool WholeArchive = false;
+
+  // True if we are in --start-lib and --end-lib.
+  bool InLib = false;
+
+  llvm::BumpPtrAllocator Alloc;
   std::vector<std::unique_ptr<InputFile>> Files;
   std::vector<std::unique_ptr<MemoryBuffer>> OwningMBs;
 };
 
 // Parses command line options.
 class ELFOptTable : public llvm::opt::OptTable {
 public:
   ELFOptTable();

ELF/Driver.cpp

void LinkerDriver::addFile(StringRef Path) {
   case file_magic::elf_shared_object:
     if (Config->Relocatable) {
       error("attempted static link of dynamic object " + Path);
       return;
     }
     Files.push_back(createSharedFile(MBRef));
     return;
   default:
+    if (InLib)
+      Files.push_back(make_unique<LazyObjectFile>(MBRef));
+    else
       Files.push_back(createObjectFile(MBRef));
   }
 }
 
 // Add a given library by searching it from input search paths.
 void LinkerDriver::addLibrary(StringRef Name) {
   std::string Path = searchLibrary(Name);
   if (Path.empty())
     error("unable to find library -l" + Name);
case OPT_Bdynamic:
       Config->Static = false;
       break;
     case OPT_whole_archive:
       WholeArchive = true;
       break;
     case OPT_no_whole_archive:
       WholeArchive = false;
       break;
+    case OPT_start_lib:
+      InLib = true;
+      break;
+    case OPT_end_lib:
+      InLib = false;
+      break;
     }
   }
 
   if (Files.empty() && !HasError)
     error("no input files.");
 }
 
 template <class ELFT> void LinkerDriver::link(opt::InputArgList &Args) {

ELF/InputFiles.h

 
 class InputFile;
 class Lazy;
 class SymbolBody;
 
 // The root class of input files.
 class InputFile {
 public:
-  enum Kind { ObjectKind, SharedKind, ArchiveKind, BitcodeKind };
+  enum Kind {
+    ObjectKind,
+    SharedKind,
+    LazyObjectKind,
+    ArchiveKind,
+    BitcodeKind,
+  };
+
   Kind kind() const { return FileKind; }
 
   StringRef getName() const { return MB.getBufferIdentifier(); }
   MemoryBufferRef MB;
 
   // Filename of .a which contained this file. If this file was
   // not in an archive file, it is the empty string. We use this
   // string for creating error messages.
private:
   std::unique_ptr<MipsReginfoInputSection<ELFT>> MipsReginfo;
 
   llvm::BumpPtrAllocator Alloc;
   llvm::SpecificBumpPtrAllocator<InputSection<ELFT>> IAlloc;
   llvm::SpecificBumpPtrAllocator<MergeInputSection<ELFT>> MAlloc;
   llvm::SpecificBumpPtrAllocator<EHInputSection<ELFT>> EHAlloc;
 };
 
+// LazyObjectFile is analogous to ArchiveFile in the sense that
+// the file contains lazy symbols. The difference is that
+// LazyObjectFile wraps a single file instead of multiple files.
+//
+// This class is used for --start-lib and --end-lib options which
+// instruct the linker to link object files between them with the
+// archive-file-like semantics.
+class LazyObjectFile : public InputFile {
+public:
+  explicit LazyObjectFile(MemoryBufferRef M) : InputFile(LazyObjectKind, M) {}
+
+  static bool classof(const InputFile *F) {
+    return F->kind() == LazyObjectKind;
+  }
+
+  void parse();
+
+  llvm::MutableArrayRef<LazyObject> getLazySymbols() { return LazySymbols; }
+
+private:
+  std::vector<StringRef> getSymbols();
+  template <class ELFT> std::vector<StringRef> getElfSymbols();
+  std::vector<StringRef> getBitcodeSymbols();
+
+  llvm::BumpPtrAllocator Alloc;
+  llvm::StringSaver Saver{Alloc};
+  std::vector<LazyObject> LazySymbols;
+};
+
+// An ArchiveFile object represents a .a file.
 class ArchiveFile : public InputFile {
 public:
   explicit ArchiveFile(MemoryBufferRef M) : InputFile(ArchiveKind, M) {}
   static bool classof(const InputFile *F) { return F->kind() == ArchiveKind; }
   void parse();
 
   // Returns a memory buffer for a given symbol. An empty memory buffer
   // is returned if we have already returned the same memory buffer.
   // (So that we don't instantiate same members more than once.)
   MemoryBufferRef getMember(const Archive::Symbol *Sym);
 
-  llvm::MutableArrayRef<Lazy> getLazySymbols() { return LazySymbols; }
+  llvm::MutableArrayRef<LazyArchive> getLazySymbols() { return LazySymbols; }
 
 private:
   std::unique_ptr<Archive> File;
-  std::vector<Lazy> LazySymbols;
+  std::vector<LazyArchive> LazySymbols;
   llvm::DenseSet<uint64_t> Seen;
 };
 
 class BitcodeFile : public InputFile {
 public:
   explicit BitcodeFile(MemoryBufferRef M);
   static bool classof(const InputFile *F);
   void parse(llvm::DenseSet<StringRef> &ComdatGroups);

ELF/InputFiles.cpp

elf::ObjectFile<ELFT>::createInputSection(const Elf_Shdr &Sec) {
   if (!Config->Relocatable && Name == ".eh_frame")
     return new (EHAlloc.Allocate()) EHInputSection<ELFT>(this, &Sec);
   if (shouldMerge<ELFT>(Sec))
     return new (MAlloc.Allocate()) MergeInputSection<ELFT>(this, &Sec);
   return new (IAlloc.Allocate()) InputSection<ELFT>(this, &Sec);
 }
 
 template <class ELFT> void elf::ObjectFile<ELFT>::initializeSymbols() {
   this->initStringTable();
   Elf_Sym_Range Syms = this->getElfSymbols(false);
   uint32_t NumSymbols = std::distance(Syms.begin(), Syms.end());
   SymbolBodies.reserve(NumSymbols);
   for (const Elf_Sym &Sym : Syms)
     SymbolBodies.push_back(createSymbolBody(&Sym));
 }
 
 template <class ELFT>
std::unique_ptr<InputFile> elf::createObjectFile(MemoryBufferRef MB,
   F->ArchiveName = ArchiveName;
   return F;
 }
 
 std::unique_ptr<InputFile> elf::createSharedFile(MemoryBufferRef MB) {
   return createELFFile<SharedFile>(MB);
 }
 
+void LazyObjectFile::parse() {
+  for (StringRef Sym : getSymbols())
+    LazySymbols.emplace_back(Sym, this->MB);
+}
+
+template <class ELFT> std::vector<StringRef> LazyObjectFile::getElfSymbols() {
+  typedef typename ELFT::Shdr Elf_Shdr;
+  typedef typename ELFT::Sym Elf_Sym;
+  typedef typename ELFT::SymRange Elf_Sym_Range;
+
+  const ELFFile<ELFT> Obj = createELFObj<ELFT>(this->MB);
+  for (const Elf_Shdr &Sec : Obj.sections()) {
+    if (Sec.sh_type != SHT_SYMTAB)
+      continue;
+    Elf_Sym_Range Syms = Obj.symbols(&Sec);
+    uint32_t FirstNonLocal = Sec.sh_info;
+    std::vector<StringRef> V;
+    StringRef StringTable = check(Obj.getStringTableForSymtab(Sec));
+    for (const Elf_Sym *I = Syms.begin() + FirstNonLocal, *E = Syms.end();
+         I != E; ++I)
+      V.push_back(check(I->getName(StringTable)));
+    return V;
+  }
+  return {};
+}
+
+std::vector<StringRef> LazyObjectFile::getBitcodeSymbols() {
+  LLVMContext Context;
+  std::unique_ptr<IRObjectFile> Obj =
+      check(IRObjectFile::create(this->MB, Context));
+  std::vector<StringRef> V;
+  for (const BasicSymbolRef &Sym : Obj->symbols()) {
+    if (!(Sym.getFlags() & BasicSymbolRef::SF_Global))
+      continue;
+    SmallString<64> Name;
+    raw_svector_ostream OS(Name);
+    Sym.printName(OS);
+    V.push_back(Saver.save(StringRef(Name)));
+  }
+  return V;
+}
+
+// Returns a vector of globally-visible symbol names.
+std::vector<StringRef> LazyObjectFile::getSymbols() {
+  using namespace sys::fs;
+
+  StringRef Buf = this->MB.getBuffer();
+  if (identify_magic(Buf) == file_magic::bitcode)
+    return getBitcodeSymbols();
+
+  std::pair<unsigned char, unsigned char> Type = getElfArchType(Buf);
+  if (Type.first == ELF::ELFCLASS32) {
+    if (Type.second == ELF::ELFDATA2LSB)
+      return getElfSymbols<ELF32LE>();
+    return getElfSymbols<ELF32BE>();
+  }
+  if (Type.second == ELF::ELFDATA2LSB)
+    return getElfSymbols<ELF64LE>();
+  return getElfSymbols<ELF64BE>();
+}
+
 template class elf::ELFFileBase<ELF32LE>;
 template class elf::ELFFileBase<ELF32BE>;
 template class elf::ELFFileBase<ELF64LE>;
 template class elf::ELFFileBase<ELF64BE>;
 
 template class elf::ObjectFile<ELF32LE>;
 template class elf::ObjectFile<ELF32BE>;
 template class elf::ObjectFile<ELF64LE>;
 template class elf::ObjectFile<ELF64BE>;
 
 template class elf::SharedFile<ELF32LE>;
 template class elf::SharedFile<ELF32BE>;
 template class elf::SharedFile<ELF64LE>;
 template class elf::SharedFile<ELF64BE>;

ELF/Options.td

def dynamic_linker : Separate<["--", "-"], "dynamic-linker">,
   HelpText<"Which dynamic linker to use">;
 
 def eh_frame_hdr : Flag<["--"], "eh-frame-hdr">,
   HelpText<"Request creation of .eh_frame_hdr section and PT_GNU_EH_FRAME segment header">;
 
 def enable_new_dtags : Flag<["--"], "enable-new-dtags">,
   HelpText<"Enable new dynamic tags">;
 
+def end_lib : Flag<["--"], "end-lib">,
+  HelpText<"End a library">;
+
 def entry : Separate<["--", "-"], "entry">, MetaVarName<"<entry>">,
   HelpText<"Name of entry point symbol">;
 
 def export_dynamic : Flag<["--", "-"], "export-dynamic">,
   HelpText<"Put symbols in the dynamic symbol table">;
 
 def fini : Separate<["-"], "fini">, MetaVarName<"<symbol>">,
   HelpText<"Specify a finalizer function">;
def script : Separate<["--"], "script">,
   HelpText<"Read linker script">;
 
 def shared : Flag<["-"], "shared">,
   HelpText<"Build a shared object">;
 
 def soname : Joined<["-"], "soname=">,
   HelpText<"Set DT_SONAME">;
 
+def start_lib : Flag<["--"], "start-lib">,
+  HelpText<"Start a library">;
+
 def strip_all : Flag<["--"], "strip-all">,
   HelpText<"Strip all symbols">;
 
 def sysroot : Joined<["--"], "sysroot=">,
   HelpText<"Set the system root">;
 
 def threads : Joined<["--"], "threads">;
 

ELF/OutputSections.cpp

SymbolTableSection<ELFT>::getOutputSection(SymbolBody *Sym) {
   case SymbolBody::DefinedCommonKind:
     return Out<ELFT>::Bss;
   case SymbolBody::SharedKind:
     if (cast<SharedSymbol<ELFT>>(Sym)->needsCopy())
       return Out<ELFT>::Bss;
     break;
   case SymbolBody::UndefinedElfKind:
   case SymbolBody::UndefinedBitcodeKind:
-  case SymbolBody::LazyKind:
+  case SymbolBody::LazyArchiveKind:
+  case SymbolBody::LazyObjectKind:
     break;
   case SymbolBody::DefinedBitcodeKind:
     llvm_unreachable("should have been replaced");
   }
   return nullptr;
 }
 
 template <class ELFT>

ELF/SymbolTable.h

private:
   llvm::BumpPtrAllocator Alloc;
 
   // Comdat groups define "link once" sections. If two comdat groups have the
   // same name, only one of them is linked, and the other is ignored. This set
   // is used to uniquify them.
   llvm::DenseSet<StringRef> ComdatGroups;
 
   // The symbol table owns all file objects.
-  std::vector<std::unique_ptr<ArchiveFile>> ArchiveFiles;
+  std::vector<std::unique_ptr<InputFile>> ArchiveFiles;
   std::vector<std::unique_ptr<ObjectFile<ELFT>>> ObjectFiles;
+  std::vector<std::unique_ptr<LazyObjectFile>> LazyObjectFiles;
   std::vector<std::unique_ptr<SharedFile<ELFT>>> SharedFiles;
   std::vector<std::unique_ptr<BitcodeFile>> BitcodeFiles;
 
   // Set of .so files to not link the same shared object file more than once.
   llvm::DenseSet<StringRef> SoNames;
 
   std::unique_ptr<BitcodeCompiler> Lto;
 };
 
 } // namespace elf
 } // namespace lld
 
 #endif

ELF/SymbolTable.cpp

if (auto *F = dyn_cast<SharedFile<ELFT>>(FileP)) {
 
     SharedFiles.emplace_back(cast<SharedFile<ELFT>>(File.release()));
     F->parseRest();
     for (SharedSymbol<ELFT> &B : F->getSharedSymbols())
       resolve(&B);
     return;
   }
 
-  // LLVM bitcode file.
+  // LLVM bitcode file
   if (auto *F = dyn_cast<BitcodeFile>(FileP)) {
     BitcodeFiles.emplace_back(cast<BitcodeFile>(File.release()));
     F->parse(ComdatGroups);
     for (SymbolBody *B : F->getSymbols())
       if (B)
         resolve(B);
     return;
   }
 
-  // .o file
+  // Lazy object file
+  if (auto *F = dyn_cast<LazyObjectFile>(FileP)) {
+    LazyObjectFiles.emplace_back(cast<LazyObjectFile>(File.release()));
+    F->parse();
+    for (Lazy &Sym : F->getLazySymbols())
+      addLazy(&Sym);
+    return;
+  }
+
+  // Regular object file
   auto *F = cast<ObjectFile<ELFT>>(FileP);
   ObjectFiles.emplace_back(cast<ObjectFile<ELFT>>(File.release()));
   F->parse(ComdatGroups);
   for (SymbolBody *B : F->getNonLocalSymbols())
     resolve(B);
 }
 
 template <class ELFT> void SymbolTable<ELFT>::addCombinedLtoObject() {
if (Undef->isWeak()) {
     L->Type = Undef->Type;
 
     // FIXME: Do we need to copy more?
     return;
   }
 
   // Fetch a member file that has the definition for L.
   // getMember returns nullptr if the member was already read from the library.
-  if (std::unique_ptr<InputFile> File = L->getMember())
+  if (std::unique_ptr<InputFile> File = L->getFile())
     addFile(std::move(File));
 }
 
 // This function takes care of the case in which shared libraries depend on
 // the user program (not the other way, which is usual). Shared libraries
 // may have undefined symbols, expecting that the user program provides
 // the definitions for them. An example is BSD's __progname symbol.
 // We need to put such symbols to the main program's .dynsym so that

ELF/Symbols.h

enum Kind {
     DefinedRegularKind = DefinedFirst,
     SharedKind,
     DefinedCommonKind,
     DefinedBitcodeKind,
     DefinedSyntheticKind,
     DefinedLast = DefinedSyntheticKind,
     UndefinedElfKind,
     UndefinedBitcodeKind,
-    LazyKind
+    LazyArchiveKind,
+    LazyObjectKind,
   };
 
   Kind kind() const { return static_cast<Kind>(SymbolKind); }
 
   bool isWeak() const { return Binding == llvm::ELF::STB_WEAK; }
   bool isUndefined() const {
     return SymbolKind == UndefinedBitcodeKind || SymbolKind == UndefinedElfKind;
   }
   bool isDefined() const { return SymbolKind <= DefinedLast; }
   bool isCommon() const { return SymbolKind == DefinedCommonKind; }
-  bool isLazy() const { return SymbolKind == LazyKind; }
+  bool isLazy() const {
+    return SymbolKind == LazyArchiveKind || SymbolKind == LazyObjectKind;
+  }
   bool isShared() const { return SymbolKind == SharedKind; }
   bool isLocal() const { return Binding == llvm::ELF::STB_LOCAL; }
   bool isUsedInRegularObj() const { return IsUsedInRegularObj; }
   bool isPreemptible() const;
 
   // Returns the symbol name.
   StringRef getName() const {
     assert(!isLocal());

 
 // This class represents a symbol defined in an archive file. It is
 // created from an archive file header, and it knows how to load an
 // object file from an archive to replace itself with a defined
 // symbol. If the resolver finds both Undefined and Lazy for
 // the same name, it will ask the Lazy to load a file.
 class Lazy : public SymbolBody {
 public:
-  Lazy(ArchiveFile *F, const llvm::object::Archive::Symbol S)
-      : SymbolBody(LazyKind, S.getName(), llvm::ELF::STB_GLOBAL,
-                   llvm::ELF::STV_DEFAULT, /* Type */ 0),
-        File(F), Sym(S) {}
+  Lazy(SymbolBody::Kind K, StringRef Name)
+      : SymbolBody(K, Name, llvm::ELF::STB_GLOBAL, llvm::ELF::STV_DEFAULT,
+                   /* Type */ 0) {}
 
-  static bool classof(const SymbolBody *S) { return S->kind() == LazyKind; }
+  static bool classof(const SymbolBody *S) { return S->isLazy(); }
 
   // Returns an object file for this symbol, or a nullptr if the file
   // was already returned.
-  std::unique_ptr<InputFile> getMember();
+  std::unique_ptr<InputFile> getFile();
+};
+
+// LazyArchive symbols represents symbols in archive files.
+class LazyArchive : public Lazy {
+public:
+  LazyArchive(ArchiveFile *F, const llvm::object::Archive::Symbol S)
+      : Lazy(LazyArchiveKind, S.getName()), File(F), Sym(S) {}
+
+  static bool classof(const SymbolBody *S) {
+    return S->kind() == LazyArchiveKind;
+  }
+
+  std::unique_ptr<InputFile> getFile();
 
 private:
   ArchiveFile *File;
   const llvm::object::Archive::Symbol Sym;
 };
 
+// LazyObject symbols represents symbols in object files between
+// --start-lib and --end-lib options.
+class LazyObject : public Lazy {
+public:
+  LazyObject(StringRef Name, MemoryBufferRef M)
+      : Lazy(LazyObjectKind, Name), MBRef(M) {}
+
+  static bool classof(const SymbolBody *S) {
+    return S->kind() == LazyObjectKind;
+  }
+
+  std::unique_ptr<InputFile> getFile();
+
+private:
+  MemoryBufferRef MBRef;
+};
+
 // Some linker-generated symbols need to be created as
 // DefinedRegular symbols.
 template <class ELFT> struct ElfSym {
   // The content for _etext and etext symbols.
   static DefinedRegular<ELFT> *Etext;
   static DefinedRegular<ELFT> *Etext2;
 
   // The content for _edata and edata symbols.

ELF/Symbols.cpp

if (!SS.NeedsCopyOrPltAddr)
       return 0;
     if (SS.isFunc())
       return Body.getPltVA<ELFT>();
     return Out<ELFT>::Bss->getVA() + SS.OffsetInBss;
   }
   case SymbolBody::UndefinedElfKind:
   case SymbolBody::UndefinedBitcodeKind:
     return 0;
-  case SymbolBody::LazyKind:
+  case SymbolBody::LazyArchiveKind:
+  case SymbolBody::LazyObjectKind:
     assert(Body.isUsedInRegularObj() && "lazy symbol reached writer");
     return 0;
   case SymbolBody::DefinedBitcodeKind:
     llvm_unreachable("should have been replaced");
   }
   llvm_unreachable("invalid symbol kind");
 }
 
: Defined(SymbolBody::DefinedSyntheticKind, N, STB_GLOBAL, StOther,
               0 /* Type */),
       Value(Value), Section(Section) {}
 
 DefinedCommon::DefinedCommon(StringRef N, uint64_t Size, uint64_t Alignment,
                              uint8_t Binding, uint8_t StOther, uint8_t Type)
     : Defined(SymbolBody::DefinedCommonKind, N, Binding, StOther, Type),
       Alignment(Alignment), Size(Size) {}
 
-std::unique_ptr<InputFile> Lazy::getMember() {
+std::unique_ptr<InputFile> Lazy::getFile() {
+  if (auto *S = dyn_cast<LazyArchive>(this))
+    return S->getFile();
+  return cast<LazyObject>(this)->getFile();
+}
+
+std::unique_ptr<InputFile> LazyArchive::getFile() {
   MemoryBufferRef MBRef = File->getMember(&Sym);
 
   // getMember returns an empty buffer if the member was already
   // read from the library.
   if (MBRef.getBuffer().empty())
     return std::unique_ptr<InputFile>(nullptr);
   return createObjectFile(MBRef, File->getName());
 }
 
+std::unique_ptr<InputFile> LazyObject::getFile() {
+  return createObjectFile(MBRef);
+}
+
 // Returns the demangled C++ symbol name for Name.
 std::string elf::demangle(StringRef Name) {
 #if !defined(HAVE_CXXABI_H)
   return Name;
 #else
   if (!Config->Demangle)
     return Name;
 

test/ELF/start-lib.s

+// REQUIRES: x86
+
+// RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t1.o
+// RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux \
+// RUN:   %p/Inputs/whole-archive.s -o %t2.o
+
+// RUN: ld.lld -o %t3 %t1.o %t2.o
+// RUN: llvm-readobj --symbols %t3 | FileCheck --check-prefix=ADDED %s
+// ADDED: Name: _bar
+
+// RUN: ld.lld -o %t3 %t1.o --start-lib %t2.o
+// RUN: llvm-readobj --symbols %t3 | FileCheck --check-prefix=LIB %s
+// LIB-NOT: Name: _bar
+
+.globl _start
+_start: