[ELF] --plugin-opt=thinlto-index-only: create empty index even if all bitcode files are lazy

[MaskRay]

Summary

The gold plugin behavior (creating empty index files for lazy bitcode
files) was added in D46034, but it missed the case when there is no
non-lazy bitcode files, e.g.

ld.lld -shared crti.o crtbeginS.o --start-lib bitcode.o --end-lib ...

crti.o crtbeginS.o are not bitcode, but our distributed build system
wants bitcode.o.thinlto.bc to confirm all expected outputs are created
based on all of the modules provided to the linker.

[MaskRay]

Update descriptions

[tejohnson]

Thanks! LGTM with one suggestion below.

ELF/LTO.cpp

217

Suggest renaming to thinLTOCreateEmptyIndexFiles for clarity.

[MaskRay]

Rename to thinLTOCreateEmptyIndexFiles

[MaskRay]

Fix comments

[ruiu]

Is there any way to call thinLTOCreateEmptyIndexFiles from only one place?

ELF/LTO.h

 private:
   std::unique_ptr<llvm::lto::LTO> LTOObj;
   std::vector<SmallString<0>> Buf;
   std::vector<std::unique_ptr<MemoryBuffer>> Files;
   llvm::DenseSet<StringRef> UsedStartStop;
   std::unique_ptr<llvm::raw_fd_ostream> IndexFile;
   llvm::DenseSet<StringRef> ThinIndices;
 };
+
+void thinLTOCreateEmptyIndex();
 } // namespace elf
 } // namespace lld
 
 #endif

ELF/LTO.cpp

for (size_t I = 0, E = Syms.size(); I != E; ++I) {
     // We tell LTO to not apply interprocedural optimization for wrapped
     // (with --wrap) symbols because otherwise LTO would inline them while
     // their values are still not final.
     R.LinkerRedefined = !Sym->CanInline;
   }
   checkError(LTOObj->add(std::move(F.Obj), Resols));
 }
 
-static void createEmptyIndex(StringRef ModulePath) {
-  std::string Path = replaceThinLTOSuffix(getThinLTOOutputFile(ModulePath));
+// If LazyObjFile has not been added to link, emit empty index files.
+// This is needed because this is what GNU gold plugin does and we have a
+// distributed build system that depends on that behavior.
+void elf::thinLTOCreateEmptyIndex() {
+  for (LazyObjFile *F : LazyObjFiles) {
+    if (F->AddedToLink || !isBitcode(F->MB))
+      continue;
+    std::string Path = replaceThinLTOSuffix(getThinLTOOutputFile(F->getName()));
     std::unique_ptr<raw_fd_ostream> OS = openFile(Path + ".thinlto.bc");
     if (!OS)
-    return;
+      continue;
 
     ModuleSummaryIndex M(/*HaveGVs*/ false);
     M.setSkipModuleByDistributedBackend();
     WriteIndexToFile(M, *OS);
-
     if (Config->ThinLTOEmitImportsFiles)
       openFile(Path + ".imports");
   }
+}
 
 // Merge all the bitcode files we have seen, codegen the result
 // and return the resulting ObjectFile(s).
 std::vector<InputFile *> BitcodeCompiler::compile() {
   unsigned MaxTasks = LTOObj->getMaxTasks();
   Buf.resize(MaxTasks);
   Files.resize(MaxTasks);
 
std::vector<InputFile *> BitcodeCompiler::compile() {
   // Emit empty index files for non-indexed files
   for (StringRef S : ThinIndices) {
     std::string Path = getThinLTOOutputFile(S);
     openFile(Path + ".thinlto.bc");
     if (Config->ThinLTOEmitImportsFiles)
       openFile(Path + ".imports");
   }
 
-  // If LazyObjFile has not been added to link, emit empty index files.
-  // This is needed because this is what GNU gold plugin does and we have a
-  // distributed build system that depends on that behavior.
   if (Config->ThinLTOIndexOnly) {
-    for (LazyObjFile *F : LazyObjFiles)
-      if (!F->AddedToLink && isBitcode(F->MB))
-        createEmptyIndex(F->getName());
+    thinLTOCreateEmptyIndex();
 
     if (!Config->LTOObjPath.empty())
       saveBuffer(Buf[0], Config->LTOObjPath);
 
     // ThinLTO with index only option is required to generate only the index
     // files. After that, we exit from linker and ThinLTO backend runs in a
     // distributed environment.
     if (IndexFile)

ELF/SymbolTable.cpp

 // This function is where all the optimizations of link-time
 // optimization happens. When LTO is in use, some input files are
 // not in native object file format but in the LLVM bitcode format.
 // This function compiles bitcode files into a few big native files
 // using LLVM functions and replaces bitcode symbols with the results.
 // Because all bitcode files that the program consists of are passed
 // to the compiler at once, it can do whole-program optimization.
 template <class ELFT> void SymbolTable::addCombinedLTOObject() {
-  if (BitcodeFiles.empty())
+  if (BitcodeFiles.empty()) {
+    if (Config->ThinLTOIndexOnly)
+      thinLTOCreateEmptyIndex();
     return;
+  }
 
   // Compile bitcode files and replace bitcode symbols.
   LTO.reset(new BitcodeCompiler);
   for (BitcodeFile *F : BitcodeFiles)
     LTO->add(*F);
 
   for (InputFile *File : LTO->compile()) {
     DenseSet<CachedHashStringRef> DummyGroups;

test/ELF/lto/thinlto-index-only.ll

 ; Test LLD generates empty imports file either because of thinlto-emit-imports-files option.
 ; RUN: rm -f %t1.o.thinlto.bc
 ; RUN: rm -f %t1.o.imports
 ; RUN: ld.lld --plugin-opt=thinlto-index-only -shared %t2.o --start-lib %t1.o --end-lib \
 ; RUN: --plugin-opt=thinlto-emit-imports-files -o %t3
 ; RUN: ls %t1.o.thinlto.bc
 ; RUN: ls %t1.o.imports
 
+; Ensure LLD generates an empty index for a lazy object file even if there is no non-lazy bitcode file.
+; RUN: rm -f %t1.o.thinlto.bc
+; RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux-gnu /dev/null -o %tdummy.o
+; RUN: ld.lld --plugin-opt=thinlto-index-only -shared %tdummy.o --start-lib %t1.o --end-lib
+; RUN: ls %t1.o.thinlto.bc
+
 ; Ensure lld generates an error if unable to write an empty index file
 ; for lazy object file that is not added to link.
 ; RUN: rm -f %t1.o.thinlto.bc
 ; RUN: touch %t1.o.thinlto.bc
 ; RUN: chmod 400 %t1.o.thinlto.bc
 ; RUN: not ld.lld --plugin-opt=thinlto-index-only -shared %t2.o --start-lib %t1.o --end-lib \
 ; RUN:   -o %t3 2>&1 | FileCheck %s
 ; CHECK: cannot open {{.*}}1.o.thinlto.bc: {{P|p}}ermission denied