llvm-lto support for generating combined function indexes

[tejohnson]

Summary

This patch adds support to llvm-lto that mirrors the support added by
r249270 to the gold plugin. This enables better testing of combined
index generation for ThinLTO.

Added a new test, and this support will be used in the test in D13515.

Note that the support directly uses lib/Object interfaces, since there
are no lib/LTO interfaces for function indexes (yet). This is similar to
gold which no longer uses lib/LTO interfaces. In order to add lto.h/.cpp
interfaces for function indexes (for linkers other than gold), is it
necessary to add lib/LTO support for function indexes, or can the lto
interfaces use lib/Object directly as gold does? Depending on the answer
to that I may want to add lib/LTO support as part of this patch and
use that in llvm-lto.

[mehdi_amini]

LGTM (modulo a few minor comments).

test/tools/llvm-lto/thinlto.ll

1

Can you put a comment a the top of the file saying what's the point of this test?

tools/llvm-lto/llvm-lto.cpp

183

I believe we have autobrief and you can leave the \brief tag out.

189

?

190

Could a local variable be enough?

[tejohnson]

Thanks for the review. Responses below, will commit fixed patch.

tools/llvm-lto/llvm-lto.cpp

189

Cleaned up.

190

Changed CombinedIndex to a local. This was copied out of my gold-plugin code which I will cleanup similarly in a follow-on commit.

test/tools/llvm-lto/Inputs/thinlto.ll

+define void @g() {
+entry:
+  ret void
+}

test/tools/llvm-lto/thinlto.ll

+; RUN: llvm-as -function-summary %s -o %t.o
+; RUN: llvm-as -function-summary %p/Inputs/thinlto.ll -o %t2.o
+; RUN: llvm-lto -thinlto -o %t3 %t.o %t2.o
+; RUN: llvm-bcanalyzer -dump %t3.thinlto.bc | FileCheck %s --check-prefix=COMBINED
+; RUN: not test -e %t3
+
+; COMBINED: <MODULE_STRTAB_BLOCK
+; COMBINED-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/llvm-lto/Output/thinlto.ll.tmp{{.*}}.o'
+; COMBINED-NEXT: <ENTRY {{.*}} record string = '{{.*}}/test/tools/llvm-lto/Output/thinlto.ll.tmp{{.*}}.o'
+; COMBINED-NEXT: </MODULE_STRTAB_BLOCK
+; COMBINED-NEXT: <FUNCTION_SUMMARY_BLOCK
+; COMBINED-NEXT: <COMBINED_ENTRY
+; COMBINED-NEXT: <COMBINED_ENTRY
+; COMBINED-NEXT: </FUNCTION_SUMMARY_BLOCK
+; COMBINED-NEXT: <VALUE_SYMTAB
+; COMBINED-NEXT: <COMBINED_FNENTRY {{.*}} record string = '{{f|g}}'
+; COMBINED-NEXT: <COMBINED_FNENTRY {{.*}} record string = '{{f|g}}'
+; COMBINED-NEXT: </VALUE_SYMTAB
+
+define void @f() {
+entry:
+  ret void
+}

tools/llvm-lto/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
+  BitReader
+  Core
   LTO
   MC
+  Object
   Support
   Target
   )
 
 add_llvm_tool(llvm-lto
   llvm-lto.cpp
   )
 

tools/llvm-lto/LLVMBuild.txt

 ;   http://llvm.org/docs/LLVMBuild.html
 ;
 ;===------------------------------------------------------------------------===;
 
 [component_0]
 type = Tool
 name = llvm-lto
 parent = Tools
-required_libraries = LTO Support all-targets
+required_libraries = BitReader Core LTO Object Support all-targets

tools/llvm-lto/llvm-lto.cpp

 //===-- llvm-lto: a simple command-line program to link modules with LTO --===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // This program takes in a list of bitcode files, links them, performs link-time
 // optimization, and outputs an object file.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/StringSet.h"
+#include "llvm/Bitcode/ReaderWriter.h"
 #include "llvm/CodeGen/CommandFlags.h"
+#include "llvm/IR/LLVMContext.h"
 #include "llvm/LTO/LTOCodeGenerator.h"
 #include "llvm/LTO/LTOModule.h"
+#include "llvm/Object/FunctionIndexObjectFile.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/PrettyStackTrace.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Support/raw_ostream.h"

 static cl::opt<bool>
 DisableLTOVectorization("disable-lto-vectorization", cl::init(false),
   cl::desc("Do not run loop or slp vectorization during LTO"));
 
 static cl::opt<bool>
 UseDiagnosticHandler("use-diagnostic-handler", cl::init(false),
   cl::desc("Use a diagnostic handler to test the handler interface"));
 
+static cl::opt<bool> ThinLTO(
+    "thinlto", cl::init(false),
+    cl::desc("Only write combined global index for ThinLTO backends"));
+
 static cl::list<std::string>
 InputFilenames(cl::Positional, cl::OneOrMore,
   cl::desc("<input bitcode files>"));
 
 static cl::opt<std::string>
 OutputFilename("o", cl::init(""),
   cl::desc("Override output filename"),
   cl::value_desc("filename"));
for (auto &Filename : InputFilenames) {
     // List the symbols.
     outs() << Filename << ":\n";
     for (int I = 0, E = Module->getSymbolCount(); I != E; ++I)
       outs() << Module->getSymbolName(I) << "\n";
   }
   return 0;
 }
 
+/// Parse the function index out of an IR file and return the function
+/// index object if found, or nullptr if not.
+static std::unique_ptr<FunctionInfoIndex> getFunctionIndexForFile(
+    StringRef Path, std::string &Error, LLVMContext &Context) {
+  std::unique_ptr<MemoryBuffer> Buffer;
+  ErrorOr<std::unique_ptr<MemoryBuffer>> BufferOrErr =
+      MemoryBuffer::getFile(Path);
+  if (std::error_code EC = BufferOrErr.getError()) {
+    Error = EC.message();
+    return nullptr;
+  }
+  Buffer = std::move(BufferOrErr.get());
+  ErrorOr<std::unique_ptr<object::FunctionIndexObjectFile>> ObjOrErr =
+      object::FunctionIndexObjectFile::create(Buffer->getMemBufferRef(),
+                                              Context);
+  if (std::error_code EC = ObjOrErr.getError()) {
+    Error = EC.message();
+    return nullptr;
+  }
+  return (*ObjOrErr)->takeIndex();
+}
+
+/// \brief Create a combined index file from the input IR files and write it.
+///
+/// This is meant to enable testing of ThinLTO combined index generation,
+/// currently available via the gold plugin via -thinlto.
+static int createCombinedFunctionIndex(StringRef Command) {
+  LLVMContext Context;
+  // Context.setDiagnosticHandler(handleDiagnostics, nullptr, true);
+  std::unique_ptr<FunctionInfoIndex> CombinedIndex(new FunctionInfoIndex());
+  uint64_t NextModuleId = 0;
+  for (auto &Filename : InputFilenames) {
+    std::string Error;
+    std::unique_ptr<FunctionInfoIndex> Index =
+        getFunctionIndexForFile(Filename, Error, Context);
+    if (!Index) {
+      errs() << Command << ": error loading file '" << Filename
+             << "': " << Error << "\n";
+      return 1;
+    }
+    CombinedIndex->mergeFrom(std::move(Index), ++NextModuleId);
+  }
+  std::error_code EC;
+  assert(!OutputFilename.empty());
+  raw_fd_ostream OS(OutputFilename + ".thinlto.bc", EC,
+                    sys::fs::OpenFlags::F_None);
+  if (EC) {
+    errs() << Command << ": error opening the file '" << OutputFilename
+           << ".thinlto.bc': " << EC.message() << "\n";
+    return 1;
+  }
+  WriteFunctionSummaryToFile(CombinedIndex.get(), OS);
+  OS.close();
+  return 0;
+}
+
 int main(int argc, char **argv) {
   // Print a stack trace if we signal out.
   sys::PrintStackTraceOnErrorSignal();
   PrettyStackTraceProgram X(argc, argv);
 
   llvm_shutdown_obj Y; // Call llvm_shutdown() on exit.
   cl::ParseCommandLineOptions(argc, argv, "llvm LTO linker\n");
 
int main(int argc, char **argv) {
   InitializeAllAsmParsers();
 
   // set up the TargetOptions for the machine
   TargetOptions Options = InitTargetOptionsFromCodeGenFlags();
 
   if (ListSymbolsOnly)
     return listSymbols(argv[0], Options);
 
+  if (ThinLTO) return createCombinedFunctionIndex(argv[0]);
+
   unsigned BaseArg = 0;
 
   LTOCodeGenerator CodeGen;
 
   if (UseDiagnosticHandler)
     CodeGen.setDiagnosticHandler(handleDiagnostics, nullptr);
 
   CodeGen.setCodePICModel(RelocModel);