[WebAssembly] Add support for --whole-archive.

[ruiu]

wasm/Driver.cpp

189

Maybe you should move this function to Common.

[sbc100]

• revert

[sbc100]

I did consider that, but the COFF version is slightly different. Perhaps as a followup to keep this change strictly wasm-only?

[ruiu]

LGTM

Yeah, actually the ELF lld's versions of the function is also slightly different, so it might be a bit hard to merge them to a single function.

test/wasm/whole-archive.test

+RUN: llc -filetype=obj %p/Inputs/start.ll -o %t.o
+RUN: llc -filetype=obj %p/Inputs/ret32.ll -o %t.ret32.o
+RUN: rm -f %t.a
+RUN: llvm-ar rcs %t.a %t.ret32.o
+
+Should not add symbols from the archive by default as they are not required
+RUN: wasm-ld --no-gc-sections -o %t.wasm %t.o %t.a
+RUN: obj2yaml %t.wasm | FileCheck --check-prefix=NOTADDED %s
+NOTADDED: FunctionNames:
+NOTADDED-NOT: Name: ret32
+NOTADDED: ...
+
+Should add symbols from the archive if --whole-archive is used
+RUN: wasm-ld --no-gc-sections -o %t.wasm %t.o --whole-archive %t.a
+RUN: obj2yaml %t.wasm | FileCheck --check-prefix=ADDED %s
+ADDED: FunctionNames:
+ADDED:   Name: ret32
+ADDED: ...
+
+--no-whole-archive should restore default behaviour
+RUN: wasm-ld --no-gc-sections -o %t.wasm %t.o --whole-archive --no-whole-archive %t.a
+RUN: obj2yaml %t.wasm | FileCheck --check-prefix=NOTADDED %s
+
+--whole-archive and --no-whole-archive should affect only archives which follow them
+RUN: wasm-ld --no-gc-sections -o %t.wasm %t.o %t.a --whole-archive --no-whole-archive
+RUN: obj2yaml %t.wasm | FileCheck --check-prefix=NOTADDED %s
+RUN: wasm-ld --no-gc-sections -o %t.wasm %t.o --whole-archive %t.a --no-whole-archive
+RUN: obj2yaml %t.wasm | FileCheck --check-prefix=ADDED %s
+
+--whole-archive should also work with thin archives
+RUN: rm -f %tthin.a
+RUN: llvm-ar --format=gnu rcsT %tthin.a %t.ret32.o
+RUN: wasm-ld --no-gc-sections -o %t.wasm %t.o --whole-archive %tthin.a
+RUN: obj2yaml %t.wasm | FileCheck --check-prefix=ADDED %s

wasm/Driver.cpp

 class LinkerDriver {
 public:
   void link(ArrayRef<const char *> ArgsArr);
 
 private:
   void createFiles(opt::InputArgList &Args);
   void addFile(StringRef Path);
   void addLibrary(StringRef Name);
+
+  // True if we are in --whole-archive and --no-whole-archive.
+  bool InWholeArchive = false;
+
   std::vector<InputFile *> Files;
 };
 } // anonymous namespace
 
 bool lld::wasm::link(ArrayRef<const char *> Args, bool CanExitEarly,
                      raw_ostream &Error) {
   errorHandler().LogName = sys::path::filename(Args[0]);
   errorHandler().ErrorOS = &Error;

 // attribute/flag in the object file itself.
 // See: https://github.com/WebAssembly/tool-conventions/issues/35
 static void readImportFile(StringRef Filename) {
   if (Optional<MemoryBufferRef> Buf = readFile(Filename))
     for (StringRef Sym : args::getLines(*Buf))
       Config->AllowUndefinedSymbols.insert(Sym);
 }
 
+// Returns slices of MB by parsing MB as an archive file.
+// Each slice consists of a member file in the archive.
+std::vector<MemoryBufferRef> static getArchiveMembers(
+    MemoryBufferRef MB) {
+  std::unique_ptr<Archive> File =
+      CHECK(Archive::create(MB),
+            MB.getBufferIdentifier() + ": failed to parse archive");
+
+  std::vector<MemoryBufferRef> V;
+  Error Err = Error::success();
+  for (const ErrorOr<Archive::Child> &COrErr : File->children(Err)) {
+    Archive::Child C =
+        CHECK(COrErr, MB.getBufferIdentifier() +
+                          ": could not get the child of the archive");
+    MemoryBufferRef MBRef =
+        CHECK(C.getMemoryBufferRef(),
+              MB.getBufferIdentifier() +
+                  ": could not get the buffer for a child of the archive");
+    V.push_back(MBRef);
+  }
+  if (Err)
+    fatal(MB.getBufferIdentifier() + ": Archive::children failed: " +
+          toString(std::move(Err)));
+
+  // Take ownership of memory buffers created for members of thin archives.
+  for (std::unique_ptr<MemoryBuffer> &MB : File->takeThinBuffers())
+    make<std::unique_ptr<MemoryBuffer>>(std::move(MB));
+
+  return V;
+}
+
 void LinkerDriver::addFile(StringRef Path) {
   Optional<MemoryBufferRef> Buffer = readFile(Path);
   if (!Buffer.hasValue())
     return;
   MemoryBufferRef MBRef = *Buffer;
 
   switch (identify_magic(MBRef.getBuffer())) {
   case file_magic::archive: {
+    // Handle -whole-archive.
+    if (InWholeArchive) {
+      for (MemoryBufferRef &M : getArchiveMembers(MBRef))
+        Files.push_back(createObjectFile(M));
+      return;
+    }
+
     SmallString<128> ImportFile = Path;
     path::replace_extension(ImportFile, ".imports");
     if (fs::exists(ImportFile))
       readImportFile(ImportFile.str());
 
     Files.push_back(make<ArchiveFile>(MBRef));
     return;
   }
   case file_magic::bitcode:
-    Files.push_back(make<BitcodeFile>(MBRef));
+  case file_magic::wasm_object:
+    Files.push_back(createObjectFile(MBRef));
     break;
   default:
-    Files.push_back(make<ObjFile>(MBRef));
+    error("unknown file type: " + MBRef.getBufferIdentifier());
   }
 }
 
 // Add a given library by searching it from input search paths.
 void LinkerDriver::addLibrary(StringRef Name) {
   for (StringRef Dir : Config->SearchPaths) {
     if (Optional<std::string> S = findFile(Dir, "lib" + Name + ".a")) {
       addFile(*S);
       return;
     }
   }
 
   error("unable to find library -l" + Name);
 }
 
 void LinkerDriver::createFiles(opt::InputArgList &Args) {
   for (auto *Arg : Args) {
     switch (Arg->getOption().getUnaliasedOption().getID()) {
     case OPT_l:
       addLibrary(Arg->getValue());
       break;
     case OPT_INPUT:
       addFile(Arg->getValue());
       break;
+    case OPT_whole_archive:
+      InWholeArchive = true;
+      break;
+    case OPT_no_whole_archive:
+      InWholeArchive = false;
+      break;
     }
   }
 }
 
 static StringRef getEntry(opt::InputArgList &Args, StringRef Default) {
   auto *Arg = Args.getLastArg(OPT_entry, OPT_no_entry);
   if (!Arg)
     return Default;

wasm/InputFiles.h

 public:
   explicit BitcodeFile(MemoryBufferRef M) : InputFile(BitcodeKind, M) {}
   static bool classof(const InputFile *F) { return F->kind() == BitcodeKind; }
 
   void parse() override;
   std::unique_ptr<llvm::lto::InputFile> Obj;
 };
 
+// Will report a fatal() error if the input buffer is not a valid bitcode
+// or was object file.
+InputFile *createObjectFile(MemoryBufferRef MB);
+
 // Opens a given file.
 llvm::Optional<MemoryBufferRef> readFile(StringRef Path);
 
 } // namespace wasm
 
 std::string toString(const wasm::InputFile *File);
 
 } // namespace lld
 
 #endif

wasm/InputFiles.cpp

Optional<MemoryBufferRef> lld::wasm::readFile(StringRef Path) {
   }
   std::unique_ptr<MemoryBuffer> &MB = *MBOrErr;
   MemoryBufferRef MBRef = MB->getMemBufferRef();
   make<std::unique_ptr<MemoryBuffer>>(std::move(MB)); // take MB ownership
 
   return MBRef;
 }
 
+InputFile *lld::wasm::createObjectFile(MemoryBufferRef MB) {
+  file_magic Magic = identify_magic(MB.getBuffer());
+  if (Magic == file_magic::wasm_object)
+    return make<ObjFile>(MB);
+
+  if (Magic == file_magic::bitcode)
+    return make<BitcodeFile>(MB);
+
+  fatal("unknown file type: " + MB.getBufferIdentifier());
+}
+
 void ObjFile::dumpInfo() const {
   log("info for: " + getName() +
       "\n              Symbols : " + Twine(Symbols.size()) +
       "\n     Function Imports : " + Twine(WasmObj->getNumImportedFunctions()) +
       "\n       Global Imports : " + Twine(WasmObj->getNumImportedGlobals()));
 }
 
 // Relocations contain either symbol or type indices.  This function takes a
void ArchiveFile::addMember(const Archive::Symbol *Sym) {
   LLVM_DEBUG(dbgs() << "loading lazy: " << Sym->getName() << "\n");
   LLVM_DEBUG(dbgs() << "from archive: " << toString(this) << "\n");
 
   MemoryBufferRef MB =
       CHECK(C.getMemoryBufferRef(),
             "could not get the buffer for the member defining symbol " +
                 Sym->getName());
 
-  InputFile *Obj;
-
-  file_magic Magic = identify_magic(MB.getBuffer());
-  if (Magic == file_magic::wasm_object) {
-    Obj = make<ObjFile>(MB);
-  } else if (Magic == file_magic::bitcode) {
-    Obj = make<BitcodeFile>(MB);
-  } else {
-    error("unknown file type: " + MB.getBufferIdentifier());
-    return;
-  }
-
+  InputFile *Obj = createObjectFile(MB);
   Obj->ArchiveName = getName();
   Symtab->addFile(Obj);
 }
 
 static uint8_t mapVisibility(GlobalValue::VisibilityTypes GvVisibility) {
   switch (GvVisibility) {
   case GlobalValue::DefaultVisibility:
     return WASM_SYMBOL_VISIBILITY_DEFAULT;

wasm/Options.td

def max_memory: J<"max-memory=">,
   HelpText<"Maximum size of the linear memory">;
 
 def no_entry: F<"no-entry">,
   HelpText<"Do not output any entry point">;
 
 def stack_first: F<"stack-first">,
   HelpText<"Place stack at start of linear memory rather than after data">;
 
+defm whole_archive: B<"whole-archive",
+    "Force load of all members in a static library",
+    "Do not force load of all members in a static library (default)">;
+
 // Aliases
 def alias_entry_e: JoinedOrSeparate<["-"], "e">, Alias<entry>;
 def alias_entry_entry: J<"entry=">, Alias<entry>;
 def alias_initial_memory_i: Flag<["-"], "i">, Alias<initial_memory>;
 def alias_max_memory_m: Flag<["-"], "m">, Alias<max_memory>;
 def alias_relocatable_r: Flag<["-"], "r">, Alias<relocatable>;
 def alias_undefined_u: JoinedOrSeparate<["-"], "u">, Alias<undefined>;
 

wasm/SymbolTable.cpp

for (Symbol *Sym : SymVector) {
     if (!Sym->IsUsedInRegularObj)
       continue;
     Undefs.insert(Sym);
   }
 
   if (Undefs.empty())
     return;
 
-  for (ObjFile *File : ObjectFiles)
+  for (InputFile *File : ObjectFiles)
     for (Symbol *Sym : File->getSymbols())
       if (Undefs.count(Sym))
         error(toString(File) + ": undefined symbol: " + toString(*Sym));
 
   for (Symbol *Sym : Undefs)
     if (!Sym->getFile())
       error("undefined symbol: " + toString(*Sym));
 }