[ELF] --plugin-opt=thinlto-index-only: create empty index even if all bitcode files are lazy

[MaskRay]

Summary

The gold plugin behavior (creating empty index files for lazy bitcode
files) was added in D46034, but it missed the case when there is no
non-lazy bitcode files, e.g.

ld.lld -shared crti.o crtbeginS.o --start-lib bitcode.o --end-lib ...

crti.o crtbeginS.o are not bitcode, but our distributed build system
wants bitcode.o.thinlto.bc to confirm all expected outputs are created
based on all of the modules provided to the linker.

[MaskRay]

Update descriptions

[tejohnson]

Thanks! LGTM with one suggestion below.

ELF/LTO.cpp

216

Suggest renaming to thinLTOCreateEmptyIndexFiles for clarity.

[MaskRay]

Rename to thinLTOCreateEmptyIndexFiles

[MaskRay]

Fix comments

[ruiu]

Is there any way to call thinLTOCreateEmptyIndexFiles from only one place?

ELF/LTO.h

 private:
   std::unique_ptr<llvm::lto::LTO> LTOObj;
   std::vector<SmallString<0>> Buf;
   std::vector<std::unique_ptr<MemoryBuffer>> Files;
   llvm::DenseSet<StringRef> UsedStartStop;
   std::unique_ptr<llvm::raw_fd_ostream> IndexFile;
   llvm::DenseSet<StringRef> ThinIndices;
 };
+
+void thinLTOCreateEmptyIndexFiles();
 } // namespace elf
 } // namespace lld
 
 #endif

ELF/LTO.cpp

for (size_t I = 0, E = Syms.size(); I != E; ++I) {
     // We tell LTO to not apply interprocedural optimization for wrapped
     // (with --wrap) symbols because otherwise LTO would inline them while
     // their values are still not final.
     R.LinkerRedefined = !Sym->CanInline;
   }
   checkError(LTOObj->add(std::move(F.Obj), Resols));
 }
 
-static void createEmptyIndex(StringRef ModulePath) {
-  std::string Path = replaceThinLTOSuffix(getThinLTOOutputFile(ModulePath));
+// If LazyObjFile has not been added to link, emit empty index files.
+// This is needed because this is what GNU gold plugin does and we have a
+// distributed build system that depends on that behavior.
+void elf::thinLTOCreateEmptyIndexFiles() {
+  for (LazyObjFile *F : LazyObjFiles) {
+    if (F->AddedToLink || !isBitcode(F->MB))
+      continue;
+    std::string Path = replaceThinLTOSuffix(getThinLTOOutputFile(F->getName()));
     std::unique_ptr<raw_fd_ostream> OS = openFile(Path + ".thinlto.bc");
     if (!OS)
-    return;
+      continue;
 
     ModuleSummaryIndex M(/*HaveGVs*/ false);
     M.setSkipModuleByDistributedBackend();
     WriteIndexToFile(M, *OS);
-
     if (Config->ThinLTOEmitImportsFiles)
       openFile(Path + ".imports");
   }
+}
 
 // Merge all the bitcode files we have seen, codegen the result
 // and return the resulting ObjectFile(s).
 std::vector<InputFile *> BitcodeCompiler::compile() {
   unsigned MaxTasks = LTOObj->getMaxTasks();
   Buf.resize(MaxTasks);
   Files.resize(MaxTasks);
 
std::vector<InputFile *> BitcodeCompiler::compile() {
   // Emit empty index files for non-indexed files
   for (StringRef S : ThinIndices) {
     std::string Path = getThinLTOOutputFile(S);
     openFile(Path + ".thinlto.bc");
     if (Config->ThinLTOEmitImportsFiles)
       openFile(Path + ".imports");
   }
 
-  // If LazyObjFile has not been added to link, emit empty index files.
-  // This is needed because this is what GNU gold plugin does and we have a
-  // distributed build system that depends on that behavior.
   if (Config->ThinLTOIndexOnly) {
-    for (LazyObjFile *F : LazyObjFiles)
-      if (!F->AddedToLink && isBitcode(F->MB))
-        createEmptyIndex(F->getName());
+    thinLTOCreateEmptyIndexFiles();
 
     if (!Config->LTOObjPath.empty())
       saveBuffer(Buf[0], Config->LTOObjPath);
 
     // ThinLTO with index only option is required to generate only the index
     // files. After that, we exit from linker and ThinLTO backend runs in a
     // distributed environment.
     if (IndexFile)

ELF/SymbolTable.cpp

 // This function is where all the optimizations of link-time
 // optimization happens. When LTO is in use, some input files are
 // not in native object file format but in the LLVM bitcode format.
 // This function compiles bitcode files into a few big native files
 // using LLVM functions and replaces bitcode symbols with the results.
 // Because all bitcode files that the program consists of are passed
 // to the compiler at once, it can do whole-program optimization.
 template <class ELFT> void SymbolTable::addCombinedLTOObject() {
-  if (BitcodeFiles.empty())
+  if (BitcodeFiles.empty()) {
+    if (Config->ThinLTOIndexOnly)
+      thinLTOCreateEmptyIndexFiles();
     return;
+  }
 
   // Compile bitcode files and replace bitcode symbols.
   LTO.reset(new BitcodeCompiler);
   for (BitcodeFile *F : BitcodeFiles)
     LTO->add(*F);
 
   for (InputFile *File : LTO->compile()) {
     DenseSet<CachedHashStringRef> DummyGroups;

test/ELF/lto/thinlto-index-only.ll

 ; Test LLD generates empty imports file either because of thinlto-emit-imports-files option.
 ; RUN: rm -f %t1.o.thinlto.bc
 ; RUN: rm -f %t1.o.imports
 ; RUN: ld.lld --plugin-opt=thinlto-index-only -shared %t2.o --start-lib %t1.o --end-lib \
 ; RUN: --plugin-opt=thinlto-emit-imports-files -o %t3
 ; RUN: ls %t1.o.thinlto.bc
 ; RUN: ls %t1.o.imports
 
+; Ensure LLD generates an empty index for each bitcode file even if all bitcode files are lazy.
+; RUN: rm -f %t1.o.thinlto.bc
+; RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux-gnu /dev/null -o %tdummy.o
+; RUN: ld.lld --plugin-opt=thinlto-index-only -shared %tdummy.o --start-lib %t1.o --end-lib
+; RUN: ls %t1.o.thinlto.bc
+
 ; NM: T f
 
 ; The backend index for this module contains summaries from itself and
 ; Inputs/thinlto.ll, as it imports from the latter.
 ; BACKEND1: <MODULE_STRTAB_BLOCK
 ; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}thinlto-index-only.ll.tmp{{.*}}.o'
 ; BACKEND1-NEXT: <ENTRY {{.*}} record string = '{{.*}}thinlto-index-only.ll.tmp{{.*}}.o'
 ; BACKEND1-NEXT: </MODULE_STRTAB_BLOCK