[ThinLTO] ThinLTO object file interfaces

[tejohnson]

Summary

This patch adds the ThinLTOObjectFile class which has interfaces for reading and writing ThinLTO blocks in bitcode files. As discussed in the File API and Data Structures RFC (see below for link), the ThinLTO information is read in different compiler/linker invocations than the rest of the Module IR, which is why the ThinLTO information uses a different ObjectFile subclass and a data structure outside of the Module class. See the RFC for more details.

The ThinLTO function index/summary data structures are defined in http://reviews.llvm.org/D11721, and the bitcode reading/writing support is defined in http://reviews.llvm.org/D11722.

Related RFCs:
RFC: ThinLTO File Format (http://lists.cs.uiuc.edu/pipermail/llvmdev/2015-August/088947.html)
RFC: ThinLTO File API and Data Structures (http://lists.cs.uiuc.edu/pipermail/llvmdev/2015-August/088951.html)

[tejohnson]

Pinging since the first one didn't get sent to llvm-commits (I added the subscriber after uploading, which apparently doesn't work well).

[tejohnson]

Updated to remove native wrapper support (as I did yesterday with the bitcode reading/writing patch http://reviews.llvm.org/D11722), as per discussion in RFC thread on the ThinLTO File Format.

[tejohnson]

I added changes to the gold-plugin.cpp to use the interfaces here and in related patches D11721 and D11722 to implement ThinLTO phase-2 compilation which checks if the linked files contain ThinLTO information, and if so simply builds and writes the combined index file and exits.

Also removed an interface that wasn't necessary for writing the combined index..

[tejohnson]

Update function name per new D11722 patch.

[tejohnson]

Obsolete. New patch is http://reviews.llvm.org/D13107

include/llvm/Object/Binary.h

protected:
   MemoryBufferRef Data;
 
   Binary(unsigned int Type, MemoryBufferRef Source);
 
   enum {
     ID_Archive,
     ID_MachOUniversalBinary,
     ID_IR, // LLVM IR
+    ID_THINLTO, // ThinLTO info
 
     // Object and children.
     ID_StartObjects,
     ID_COFF,
 
     ID_ELF32L, // ELF 32-bit, little endian
     ID_ELF32B, // ELF 32-bit, big endian
     ID_ELF64L, // ELF 64-bit, little endian
public:
   bool isCOFF() const {
     return TypeID == ID_COFF;
   }
 
   bool isIR() const {
     return TypeID == ID_IR;
   }
 
+  bool isThinLTO() const {
+    return TypeID == ID_THINLTO;
+  }
+
   bool isLittleEndian() const {
     return !(TypeID == ID_ELF32B || TypeID == ID_ELF64B ||
              TypeID == ID_MachO32B || TypeID == ID_MachO64B);
   }
 };
 
 /// @brief Create a Binary from Source, autodetecting the file type.
 ///

include/llvm/Object/ThinLTOObjectFile.h

+//===- ThinLTOObjectFile.h - ThinLTO object file implementation -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the ThinLTOObjectFile template class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECT_THINLTOOBJECTFILE_H
+#define LLVM_OBJECT_THINLTOOBJECTFILE_H
+
+#include "llvm/Object/SymbolicFile.h"
+
+namespace llvm {
+class ThinLTOFunctionSummaryIndex;
+
+namespace object {
+class ObjectFile;
+
+class ThinLTOObjectFile : public SymbolicFile {
+  std::unique_ptr<ThinLTOFunctionSummaryIndex> Index;
+
+public:
+  ThinLTOObjectFile(MemoryBufferRef Object,
+                    std::unique_ptr<ThinLTOFunctionSummaryIndex> I);
+  ~ThinLTOObjectFile() override;
+
+  // TODO: Walk through ThinLTOFunctionMap entries for function symbols.
+  void moveSymbolNext(DataRefImpl &Symb) const override { }
+  std::error_code printSymbolName(raw_ostream &OS, DataRefImpl Symb) const override {
+    return std::error_code();
+  }
+  uint32_t getSymbolFlags(DataRefImpl Symb) const override { return 0; }
+  basic_symbol_iterator symbol_begin_impl() const override { }
+  basic_symbol_iterator symbol_end_impl() const override { }
+
+  const ThinLTOFunctionSummaryIndex &getIndex() const {
+    return const_cast<ThinLTOObjectFile*>(this)->getIndex();
+  }
+  ThinLTOFunctionSummaryIndex &getIndex() {
+    return *Index;
+  }
+  std::unique_ptr<ThinLTOFunctionSummaryIndex> takeIndex();
+
+  static inline bool classof(const Binary *v) {
+    return v->isThinLTO();
+  }
+
+  /// \brief Looks for ThinLTO information in the given memory buffer,
+  /// returns true if found, else false.
+  static bool hasThinLTOInMemBuffer(MemoryBufferRef Object,
+                                    LLVMContext &Context);
+
+  /// \brief Parse ThinLTO information in the given memory buffer.
+  /// Return new ThinLTOObjectFile instance containing parsed ThinLTO
+  /// summary/index.
+  static ErrorOr<std::unique_ptr<ThinLTOObjectFile>>
+      create(MemoryBufferRef Object, LLVMContext &Context,
+             bool ReadFuncSummaryData = true);
+
+  /// \brief Parse the ThinLTO summary information for function with the
+  /// given name out of the given buffer. Parsed information is
+  /// stored on the index object saved in this object.
+  std::error_code findThinLTOFunctionInfoInMemBuffer(
+          MemoryBufferRef Object, LLVMContext &Context, StringRef FunctionName);
+};
+}
+}
+
+#endif

lib/Object/CMakeLists.txt

add_llvm_library(LLVMObject
   IRObjectFile.cpp
   MachOObjectFile.cpp
   MachOUniversal.cpp
   Object.cpp
   ObjectFile.cpp
   RecordStreamer.cpp
   SymbolicFile.cpp
   SymbolSize.cpp
+  ThinLTOObjectFile.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Object
 
   DEPENDS
   intrinsics_gen
   )

lib/Object/ThinLTOObjectFile.cpp

+//===- ThinLTOObjectFile.cpp - ThinLTO object file implementation ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Part of the ThinLTOObjectFile class implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Object/ThinLTOObjectFile.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Bitcode/ReaderWriter.h"
+#include "llvm/IR/ThinLTOInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+using namespace object;
+
+ThinLTOObjectFile::ThinLTOObjectFile(
+    MemoryBufferRef Object, std::unique_ptr<ThinLTOFunctionSummaryIndex> I)
+    : SymbolicFile(Binary::ID_THINLTO, Object), Index(std::move(I)) {
+}
+
+ThinLTOObjectFile::~ThinLTOObjectFile() {
+}
+
+std::unique_ptr<ThinLTOFunctionSummaryIndex> ThinLTOObjectFile::takeIndex() {
+  return std::move(Index);
+}
+
+// Looks for ThinLTO information in the given memory buffer
+// returns true if found, else false.
+bool ThinLTOObjectFile::hasThinLTOInMemBuffer(MemoryBufferRef Object,
+                                              LLVMContext &Context) {
+  sys::fs::file_magic Type = sys::fs::identify_magic(Object.getBuffer());
+  switch (Type) {
+    case sys::fs::file_magic::bitcode:
+      return hasThinLTOIndex(Object, Context, nullptr);
+    default:
+      assert(false);
+  }
+  return false;
+}
+
+// Parse ThinLTO information in the given memory buffer.
+// Return new ThinLTOObjectFile instance containing parsed ThinLTO
+// summary/index.
+ErrorOr<std::unique_ptr<ThinLTOObjectFile>>
+llvm::object::ThinLTOObjectFile::create(MemoryBufferRef Object,
+                                        LLVMContext &Context,
+                                        bool ReadFuncSummaryData) {
+  std::unique_ptr<ThinLTOFunctionSummaryIndex> Index;
+  sys::fs::file_magic Type = sys::fs::identify_magic(Object.getBuffer());
+  switch (Type) {
+    case sys::fs::file_magic::bitcode: {
+      ErrorOr<std::unique_ptr<ThinLTOFunctionSummaryIndex>> IOrErr =
+          getThinLTOIndex(Object, Context, nullptr, ReadFuncSummaryData);
+      if (std::error_code EC = IOrErr.getError())
+        return EC;
+
+      Index = std::move(IOrErr.get());
+
+      break;
+    }
+    default:
+      return object_error::invalid_file_type;
+  }
+
+  return llvm::make_unique<ThinLTOObjectFile>(Object, std::move(Index));
+}
+
+// Parse the ThinLTO summary information for function with the
+// given name out of the given buffer. Parsed information is
+// stored on the index object saved in this object.
+std::error_code ThinLTOObjectFile::findThinLTOFunctionInfoInMemBuffer(
+    MemoryBufferRef Object, LLVMContext &Context, StringRef FunctionName) {
+  sys::fs::file_magic Type = sys::fs::identify_magic(Object.getBuffer());
+  switch (Type) {
+    case sys::fs::file_magic::bitcode: {
+      return readThinLTOFunctionInfo(Object, Context, nullptr,
+                                     FunctionName, std::move(Index));
+    }
+    default:
+      return object_error::invalid_file_type;
+  }
+}

tools/gold/gold-plugin.cpp

 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Verifier.h"
 #include "llvm/Linker/Linker.h"
 #include "llvm/MC/SubtargetFeature.h"
 #include "llvm/Object/IRObjectFile.h"
+#include "llvm/Object/ThinLTOObjectFile.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/TargetSelect.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"

 static ld_plugin_get_symbols get_symbols = nullptr;
 static ld_plugin_add_input_file add_input_file = nullptr;
 static ld_plugin_set_extra_library_path set_extra_library_path = nullptr;
 static ld_plugin_get_view get_view = nullptr;
 static ld_plugin_message message = discard_message;
 static Reloc::Model RelocationModel = Reloc::Default;
 static std::string output_name = "";
 static std::list<claimed_file> Modules;
+static bool IsThinLTO = false;
 static std::vector<std::string> Cleanup;
 static llvm::TargetOptions TargetOpts;
 
 namespace options {
   enum OutputType {
     OT_NORMAL,
     OT_DISABLE,
     OT_BC_ONLY,
static ld_plugin_status claim_file_hook(const ld_plugin_input_file *file,
   }
   std::unique_ptr<object::IRObjectFile> Obj = std::move(*ObjOrErr);
 
   Modules.resize(Modules.size() + 1);
   claimed_file &cf = Modules.back();
 
   cf.handle = file->handle;
 
+  // If the file contains a ThinLTO index, then we are doing ThinLTO
+  // compilation. Don't need to process the symbols in this case, and
+  // we remember that this is ThinLTO so that we simply build a combined
+  // index file after all files are claimed.
+  if (hasThinLTOIndex(BufferRef, Context, nullptr)) {
+    IsThinLTO = true;
+    return LDPS_OK;
+  }
+
   for (auto &Sym : Obj->symbols()) {
     uint32_t Symflags = Sym.getFlags();
     if (shouldSkip(Symflags))
       continue;
 
     cf.syms.push_back(ld_plugin_symbol());
     ld_plugin_symbol &sym = cf.syms.back();
     sym.version = nullptr;

 
 static void freeSymName(ld_plugin_symbol &Sym) {
   free(Sym.name);
   free(Sym.comdat_key);
   Sym.name = nullptr;
   Sym.comdat_key = nullptr;
 }
 
+static std::unique_ptr<ThinLTOFunctionSummaryIndex>
+getThinLTOIndexForFile(LLVMContext &Context, claimed_file &F,
+                       ld_plugin_input_file &Info) {
+
+  if (get_symbols(F.handle, F.syms.size(), &F.syms[0]) != LDPS_OK)
+    message(LDPL_FATAL, "Failed to get symbol information");
+
+  const void *View;
+  if (get_view(F.handle, &View) != LDPS_OK)
+    message(LDPL_FATAL, "Failed to get a view of file");
+
+  MemoryBufferRef BufferRef(StringRef((const char *)View, Info.filesize),
+                            Info.name);
+  ErrorOr<std::unique_ptr<object::ThinLTOObjectFile>> ObjOrErr =
+      object::ThinLTOObjectFile::create(BufferRef, Context);
+
+  if (std::error_code EC = ObjOrErr.getError())
+    message(LDPL_FATAL, "Could not read ThinLTO bitcode from file : %s",
+            EC.message().c_str());
+
+  object::ThinLTOObjectFile &Obj = **ObjOrErr;
+
+  return Obj.takeIndex();
+}
+
 static std::unique_ptr<Module>
 getModuleForFile(LLVMContext &Context, claimed_file &F,
                  ld_plugin_input_file &Info, raw_fd_ostream *ApiFile,
                  StringSet<> &Internalize, StringSet<> &Maybe) {
 
   if (get_symbols(F.handle, F.syms.size(), F.syms.data()) != LDPS_OK)
     message(LDPL_FATAL, "Failed to get symbol information");
 

 /// native object file. Then, perform optimization and codegen.
 static ld_plugin_status allSymbolsReadHook(raw_fd_ostream *ApiFile) {
   if (Modules.empty())
     return LDPS_OK;
 
   LLVMContext Context;
   Context.setDiagnosticHandler(diagnosticHandler, nullptr, true);
 
+  // If we are doing ThinLTO compilation, simply build the combined
+  // function index/summary and emit it. We don't need to parse the modules
+  // and link them in this case.
+  if (IsThinLTO) {
+    std::unique_ptr<ThinLTOFunctionSummaryIndex> CombinedIndex;
+    for (claimed_file &F : Modules) {
+      ld_plugin_input_file File;
+      if (get_input_file(F.handle, &File) != LDPS_OK)
+        message(LDPL_FATAL, "Failed to get file information");
+
+      std::unique_ptr<ThinLTOFunctionSummaryIndex> Index =
+          getThinLTOIndexForFile(Context, F, File);
+      CombinedIndex->mergeFrom(Index.get());
+    }
+
+    std::error_code EC;
+    raw_fd_ostream OS(output_name + ".thinlto.bc", EC,
+                      sys::fs::OpenFlags::F_None);
+    if (EC)
+      message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s",
+              output_name.data(), EC.message().c_str());
+    WriteThinLTOToFile(CombinedIndex.get(), OS);
+    OS.close();
+
+    cleanup_hook();
+    exit(0);
+  }
+
   std::unique_ptr<Module> Combined(new Module("ld-temp.o", Context));
   Linker L(Combined.get());
 
   std::string DefaultTriple = sys::getDefaultTargetTriple();
 
   StringSet<> Internalize;
   StringSet<> Maybe;
   for (claimed_file &F : Modules) {