[lld][WebAssembly] Handle duplicate archive member names in ThinLTO

[sbc100]

Summary

This entire change, including the test case, comes almost verbatim
from the ELF driver.

Fixes: https://github.com/emscripten-core/emscripten/issues/12763

[dschuff]

it would be really cool if we could get ThinLTO working. It has most of the benefits of LTO at a small fraction of the cost.

[sbc100]

In D112723#3094317, @dschuff wrote:

it would be really cool if we could get ThinLTO working. It has most of the benefits of LTO at a small fraction of the cost.

I think it does work... its not not well tested.

lld/test/wasm/lto/Inputs/thin1.ll

+; Copied from lld/test/ELF/lto/Inputs/thin1.ll
+
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+define i32 @foo(i32 %goo) {
+entry:
+  %goo.addr = alloca i32, align 4
+  store i32 %goo, i32* %goo.addr, align 4
+  %0 = load i32, i32* %goo.addr, align 4
+  %1 = load i32, i32* %goo.addr, align 4
+  %mul = mul nsw i32 %0, %1
+  ret i32 %mul
+}

lld/test/wasm/lto/Inputs/thin2.ll

+; Copied from lld/test/ELF/lto/Inputs/thin2.ll
+
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+define i32 @blah(i32 %meh) #0 {
+entry:
+  %meh.addr = alloca i32, align 4
+  store i32 %meh, i32* %meh.addr, align 4
+  %0 = load i32, i32* %meh.addr, align 4
+  %sub = sub nsw i32 %0, 48
+  ret i32 %sub
+}

lld/test/wasm/lto/thinlto-thin-archive-collision.ll

+; Copied from lld/test/ELF/lto/thinlto-thin-archive-collision.ll
+
+; RUN: rm -fr %t && mkdir %t && cd %t
+; RUN: mkdir thinlto-archives thinlto-archives/a thinlto-archives/b
+; RUN: opt -thinlto-bc -o thinlto-archives/main.o %s
+; RUN: opt -thinlto-bc -o thinlto-archives/a/thin.o %S/Inputs/thin1.ll
+; RUN: opt -thinlto-bc -o thinlto-archives/b/thin.o %S/Inputs/thin2.ll
+; RUN: llvm-ar qcT thinlto-archives/thin.a thinlto-archives/a/thin.o thinlto-archives/b/thin.o
+; RUN: wasm-ld thinlto-archives/main.o thinlto-archives/thin.a -o thinlto-archives/main.exe --save-temps
+; RUN: FileCheck %s < thinlto-archives/main.exe.resolution.txt
+
+; CHECK: thinlto-archives/main.o
+; CHECK: thinlto-archives/thin.a(thin.o at {{[1-9][0-9]+}})
+; CHECK-NEXT: -r=thinlto-archives/thin.a(thin.o at {{[1-9][0-9]+}}),foo,p
+; CHECK: thinlto-archives/thin.a(thin.o at {{[1-9][0-9]+}})
+; CHECK-NEXT: -r=thinlto-archives/thin.a(thin.o at {{[1-9][0-9]+}}),blah,p
+
+target datalayout = "e-m:e-p:32:32-p10:8:8-p20:8:8-i64:64-n32:64-S128"
+target triple = "wasm32-unknown-unknown"
+
+declare i32 @blah(i32 %meh)
+declare i32 @foo(i32 %goo)
+
+define i32 @_start() {
+  call i32 @foo(i32 0)
+  call i32 @blah(i32 0)
+  ret i32 0
+}

lld/wasm/InputFiles.h

 public:
   explicit SharedFile(MemoryBufferRef m) : InputFile(SharedKind, m) {}
   static bool classof(const InputFile *f) { return f->kind() == SharedKind; }
 };
 
 // .bc file
 class BitcodeFile : public InputFile {
 public:
-  explicit BitcodeFile(MemoryBufferRef m, StringRef archiveName)
-      : InputFile(BitcodeKind, m) {
-    this->archiveName = std::string(archiveName);
-
-    // If this isn't part of an archive, it's eagerly linked, so mark it live.
-    if (archiveName.empty())
-      markLive();
-  }
+  BitcodeFile(MemoryBufferRef m, StringRef archiveName,
+              uint64_t offsetInArchive);
   static bool classof(const InputFile *f) { return f->kind() == BitcodeKind; }
 
   void parse();
   std::unique_ptr<llvm::lto::InputFile> obj;
 
   // Set to true once LTO is complete in order prevent further bitcode objects
   // being added.
   static bool doneLTO;
 };
 
 inline bool isBitcode(MemoryBufferRef mb) {
   return identify_magic(mb.getBuffer()) == llvm::file_magic::bitcode;
 }
 
 // Will report a fatal() error if the input buffer is not a valid bitcode
 // or wasm object file.
-InputFile *createObjectFile(MemoryBufferRef mb, StringRef archiveName = "");
+InputFile *createObjectFile(MemoryBufferRef mb, StringRef archiveName = "",
+                            uint64_t offsetInArchive = 0);
 
 // Opens a given file.
 llvm::Optional<MemoryBufferRef> readFile(StringRef path);
 
 } // namespace wasm
 
 std::string toString(const wasm::InputFile *file);
 
 } // namespace lld
 
 #endif

lld/wasm/InputFiles.cpp

 #include "InputElement.h"
 #include "OutputSegment.h"
 #include "SymbolTable.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
 #include "lld/Common/Reproduce.h"
 #include "llvm/Object/Binary.h"
 #include "llvm/Object/Wasm.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/TarWriter.h"
 #include "llvm/Support/raw_ostream.h"
 
 #define DEBUG_TYPE "lld"
 
 using namespace llvm;
 using namespace llvm::object;
 using namespace llvm::wasm;
+using namespace llvm::sys;
 
 namespace lld {
 
 // Returns a string in the format of "foo.o" or "foo.a(bar.o)".
 std::string toString(const wasm::InputFile *file) {
   if (!file)
     return "<internal>";
 
Optional<MemoryBufferRef> readFile(StringRef path) {
   MemoryBufferRef mbref = mb->getMemBufferRef();
   make<std::unique_ptr<MemoryBuffer>>(std::move(mb)); // take MB ownership
 
   if (tar)
     tar->append(relativeToRoot(path), mbref.getBuffer());
   return mbref;
 }
 
-InputFile *createObjectFile(MemoryBufferRef mb, StringRef archiveName) {
+InputFile *createObjectFile(MemoryBufferRef mb, StringRef archiveName,
+                            uint64_t offsetInArchive) {
   file_magic magic = identify_magic(mb.getBuffer());
   if (magic == file_magic::wasm_object) {
     std::unique_ptr<Binary> bin =
         CHECK(createBinary(mb), mb.getBufferIdentifier());
     auto *obj = cast<WasmObjectFile>(bin.get());
     if (obj->isSharedObject())
       return make<SharedFile>(mb);
     return make<ObjFile>(mb, archiveName);
   }
 
   if (magic == file_magic::bitcode)
-    return make<BitcodeFile>(mb, archiveName);
+    return make<BitcodeFile>(mb, archiveName, offsetInArchive);
 
   fatal("unknown file type: " + mb.getBufferIdentifier());
 }
 
 // Relocations contain either symbol or type indices.  This function takes a
 // relocation and returns relocated index (i.e. translates from the input
 // symbol/type space to the output symbol/type space).
 uint32_t ObjFile::calcNewIndex(const WasmRelocation &reloc) const {
void ArchiveFile::addMember(const Archive::Symbol *sym) {
   LLVM_DEBUG(dbgs() << "loading lazy: " << sym->getName() << "\n");
   LLVM_DEBUG(dbgs() << "from archive: " << toString(this) << "\n");
 
   MemoryBufferRef mb =
       CHECK(c.getMemoryBufferRef(),
             "could not get the buffer for the member defining symbol " +
                 sym->getName());
 
-  InputFile *obj = createObjectFile(mb, getName());
+  InputFile *obj = createObjectFile(mb, getName(), c.getChildOffset());
   symtab->addFile(obj);
 }
 
 static uint8_t mapVisibility(GlobalValue::VisibilityTypes gvVisibility) {
   switch (gvVisibility) {
   case GlobalValue::DefaultVisibility:
     return WASM_SYMBOL_VISIBILITY_DEFAULT;
   case GlobalValue::HiddenVisibility:
if (objSym.isUndefined() || excludedByComdat) {
     return symtab->addUndefinedData(name, flags, &f);
   }
 
   if (objSym.isExecutable())
     return symtab->addDefinedFunction(name, flags, &f, nullptr);
   return symtab->addDefinedData(name, flags, &f, nullptr, 0, 0);
 }
 
+BitcodeFile::BitcodeFile(MemoryBufferRef m, StringRef archiveName,
+                         uint64_t offsetInArchive)
+    : InputFile(BitcodeKind, m) {
+  this->archiveName = std::string(archiveName);
+
+  std::string path = mb.getBufferIdentifier().str();
+
+  // ThinLTO assumes that all MemoryBufferRefs given to it have a unique
+  // name. If two archives define two members with the same name, this
+  // causes a collision which result in only one of the objects being taken
+  // into consideration at LTO time (which very likely causes undefined
+  // symbols later in the link stage). So we append file offset to make
+  // filename unique.
+  StringRef name = archiveName.empty()
+                       ? saver.save(path)
+                       : saver.save(archiveName + "(" + path::filename(path) +
+                                    " at " + utostr(offsetInArchive) + ")");
+  MemoryBufferRef mbref(mb.getBuffer(), name);
+
+  obj = check(lto::InputFile::create(mbref));
+
+  // If this isn't part of an archive, it's eagerly linked, so mark it live.
+  if (archiveName.empty())
+    markLive();
+}
+
 bool BitcodeFile::doneLTO = false;
 
 void BitcodeFile::parse() {
   if (doneLTO) {
     error(toString(this) + ": attempt to add bitcode file after LTO.");
     return;
   }
 
-  obj = check(lto::InputFile::create(MemoryBufferRef(
-      mb.getBuffer(), saver.save(archiveName + mb.getBufferIdentifier()))));
   Triple t(obj->getTargetTriple());
   if (!t.isWasm()) {
     error(toString(this) + ": machine type must be wasm32 or wasm64");
     return;
   }
   checkArch(t.getArch());
   std::vector<bool> keptComdats;
   // TODO Support nodeduplicate https://bugs.llvm.org/show_bug.cgi?id=50531