Add an internalization step to the ThinLTOCodeGenerator

[mehdi_amini]

Summary

keeping as much as possible internal/private is
known to help the optimizer. Let's try to benefit from
this in ThinLTO.
Note: this is early work, but is enough to build clang (and
all the LLVM tools). I still need to write some lit-tests...

[mehdi_amini]

Rebase

[mehdi_amini]

Rebase

[mehdi_amini]

Add a test

[tejohnson]

lib/LTO/ThinLTOCodeGenerator.cpp

318

It seems there is already some support in libLTO for detecting these, see the call to LTOModule::addAsmGlobalSymbolUndef(). Can that be leveraged here? What is the difference between what is being detected by these two cases?

If it needs to use the IRObject method, can that functionality be refactored into a method in libObject that is callable from here? It seems like it could return the AsmSymbols vector and you could just iterate that here looking for a Key that is BasicSymbolRef::SF_Undefined in the returned pair, and is !TheModule.getNamedValue(Key). Using a refactored common routine would make it easier to do similar optimizations in the gold and lld cases as well.

387

Missing comment

395

I guess the assumption is that there is nothing to export if there is nothing to preserve? Should there be an assertion here that ExportList is empty?

415

doxygen style /// comment (here and with other added functions)

429

Why not add these to the PreservedGV set?

616

Should this be a FIXME?

[mehdi_amini]

Address comments

lib/LTO/ThinLTOCodeGenerator.cpp

395

No this is completely independent: if the client didn't supply anything to preserve, we don't internalize at all. Changed the comment.

415

I think it is relevant only for public API. These are static function so they won't show up in doxygen: http://llvm.org/doxygen/ThinLTOCodeGenerator_8cpp.html

429

PreservedGV drives the internalizer, this is possibly driving other places in LLVM I think, like GlobalDCE for example.

616

added

[mehdi_amini]

r267317

include/llvm/LTO/ThinLTOCodeGenerator.h

public:
 
   /**
    * Perform cross-module importing for the module identified by
    * ModuleIdentifier.
    */
   void crossModuleImport(Module &Module, ModuleSummaryIndex &Index);
 
   /**
+   * Perform internalization.
+   */
+  void internalize(Module &Module, ModuleSummaryIndex &Index);
+
+  /**
    * Perform post-importing ThinLTO optimizations.
    */
   void optimize(Module &Module);
 
   /**
    * Perform ThinLTO CodeGen.
    */
   std::unique_ptr<MemoryBuffer> codegen(Module &Module);

lib/LTO/ThinLTOCodeGenerator.cpp

 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/LTO/ThinLTOCodeGenerator.h"
 
 #ifdef HAVE_LLVM_REVISION
 #include "LLVMLTORevision.h"
 #endif
+
+#include "UpdateCompilerUsed.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Bitcode/BitcodeWriterPass.h"
 #include "llvm/Bitcode/ReaderWriter.h"
 #include "llvm/ExecutionEngine/ObjectMemoryBuffer.h"
 #include "llvm/IR/DiagnosticPrinter.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/LegacyPassManager.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/IRReader/IRReader.h"
 #include "llvm/Linker/Linker.h"
 #include "llvm/MC/SubtargetFeature.h"
+#include "llvm/Object/IRObjectFile.h"
 #include "llvm/Object/ModuleSummaryIndexObjectFile.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/CachePruning.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/SHA1.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/ThreadPool.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/FunctionImport.h"
+#include "llvm/Transforms/IPO/Internalize.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #include "llvm/Transforms/ObjCARC.h"
 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
 
 using namespace llvm;
 
 #define DEBUG_TYPE "thinlto"
 
static void optimizeModule(Module &TheModule, TargetMachine &TM) {
   PM.add(createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
 
   // Add optimizations
   PMB.populateThinLTOPassManager(PM);
 
   PM.run(TheModule);
 }
 
+// Create a DenseSet of GlobalValue to be used with the Internalizer.
+static DenseSet<const GlobalValue *> computePreservedSymbolsForModule(
+    Module &TheModule, const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
+    const FunctionImporter::ExportSetTy &ExportList) {
+  DenseSet<const GlobalValue *> PreservedGV;
+  if (GUIDPreservedSymbols.empty())
+    // Early exit: internalize is disabled when there is nothing to preserve.
+    return PreservedGV;
+
+  auto AddPreserveGV = [&](const GlobalValue &GV) {
+    auto GUID = GV.getGUID();
+    if (GUIDPreservedSymbols.count(GUID) || ExportList.count(GUID))
+      PreservedGV.insert(&GV);
+  };
+
+  for (auto &GV : TheModule)
+    AddPreserveGV(GV);
+  for (auto &GV : TheModule.globals())
+    AddPreserveGV(GV);
+  for (auto &GV : TheModule.aliases())
+    AddPreserveGV(GV);
+
+  return PreservedGV;
+}
+
+// Run internalization on \p TheModule
+static void
+doInternalizeModule(Module &TheModule, const TargetMachine &TM,
+                    const DenseSet<const GlobalValue *> &PreservedGV) {
+  if (PreservedGV.empty()) {
+    // Be friendly and don't nuke totally the module when the client didn't
+    // supply anything to preserve.
+    return;
+  }
+
+  // Parse inline ASM and collect the list of symbols that are not defined in
+  // the current module.
+  StringSet<> AsmUndefinedRefs;
+  object::IRObjectFile::CollectAsmUndefinedRefs(
+      Triple(TheModule.getTargetTriple()), TheModule.getModuleInlineAsm(),
+      [&AsmUndefinedRefs](StringRef Name, object::BasicSymbolRef::Flags Flags) {
+        if (Flags & object::BasicSymbolRef::SF_Undefined)
+          AsmUndefinedRefs.insert(Name);
+      });
+
+  // Update the llvm.compiler_used globals to force preserving libcalls and
+  // symbols referenced from asm
+  UpdateCompilerUsed(TheModule, TM, AsmUndefinedRefs);
+
+  // Declare a callback for the internalize pass that will ask for every
+  // candidate GlobalValue if it can be internalized or not.
+  auto MustPreserveGV =
+      [&](const GlobalValue &GV) -> bool { return PreservedGV.count(&GV); };
+
+  llvm::internalizeModule(TheModule, MustPreserveGV);
+}
+
+// Convert the PreservedSymbols map from "Name" based to "GUID" based.
+static DenseSet<GlobalValue::GUID>
+computeGUIDPreservedSymbols(const StringSet<> &PreservedSymbols,
+                            const Triple &TheTriple) {
+  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
+  for (auto &Entry : PreservedSymbols) {
+    StringRef Name = Entry.first();
+    if (TheTriple.isOSBinFormatMachO() && Name.size() > 0 && Name[0] == '_')
+      Name = Name.drop_front();
+    GUIDPreservedSymbols.insert(GlobalValue::getGUID(Name));
+  }
+  return GUIDPreservedSymbols;
+}
+
 std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
                                             TargetMachine &TM) {
   SmallVector<char, 128> OutputBuffer;
 
   // CodeGen
   {
     raw_svector_ostream OS(OutputBuffer);
     legacy::PassManager PM;
 
     // If the bitcode files contain ARC code and were compiled with optimization,
     // the ObjCARCContractPass must be run, so do it unconditionally here.
     PM.add(createObjCARCContractPass());
 
     // Setup the codegen now.
     if (TM.addPassesToEmitFile(PM, OS, TargetMachine::CGFT_ObjectFile,
                                /* DisableVerify */ true))
       report_fatal_error("Failed to setup codegen");
 
     // Run codegen now. resulting binary is in OutputBuffer.
     PM.run(TheModule);
   }
   return make_unique<ObjectMemoryBuffer>(std::move(OutputBuffer));
 }
 
 /// Manage caching for a single Module.
 class ModuleCacheEntry {
   SmallString<128> EntryPath;
 
 public:
   // Create a cache entry. This compute a unique hash for the Module considering
   // the current list of export/import, and offer an interface to query to
   // access the content in the cache.
   ModuleCacheEntry(
       StringRef CachePath, const ModuleSummaryIndex &Index, StringRef ModuleID,
       const FunctionImporter::ImportMapTy &ImportList,
       const FunctionImporter::ExportSetTy &ExportList,
       const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
       const std::map<GlobalValue::GUID, GlobalValueSummary *> &DefinedFunctions,
       const DenseSet<GlobalValue::GUID> &PreservedSymbols) {
     if (CachePath.empty())
       return;
 
     // Compute the unique hash for this entry
     // This is based on the current compiler version, the module itself, the
     // export list, the hash for every single module in the import list, the
     // list of ResolvedODR for the module, and the list of preserved symbols.
 
     SHA1 Hasher;
 
     // Start with the compiler revision
     Hasher.update(LLVM_VERSION_STRING);
for (auto &Entry : PreservedSymbols) {
       if (DefinedFunctions.count(Entry))
         Hasher.update(
             ArrayRef<uint8_t>((uint8_t *)&Entry, sizeof(GlobalValue::GUID)));
     }
 
     sys::path::append(EntryPath, CachePath, toHex(Hasher.result()));
   }
 
+  // Access the path to this entry in the cache.
+  StringRef getEntryPath() { return EntryPath; }
+
   // Try loading the buffer for this cache entry.
   ErrorOr<std::unique_ptr<MemoryBuffer>> tryLoadingBuffer() {
     if (EntryPath.empty())
       return std::error_code();
     return MemoryBuffer::getFile(EntryPath);
   }
 
   // Cache the Produced object file
void write(MemoryBufferRef OutputBuffer) {
     sys::fs::rename(TempFilename, EntryPath);
   }
 };
 
 static std::unique_ptr<MemoryBuffer> ProcessThinLTOModule(
     Module &TheModule, const ModuleSummaryIndex &Index,
     StringMap<MemoryBufferRef> &ModuleMap, TargetMachine &TM,
     const FunctionImporter::ImportMapTy &ImportList,
+    const FunctionImporter::ExportSetTy &ExportList,
+    const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
     std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
     ThinLTOCodeGenerator::CachingOptions CacheOptions, bool DisableCodeGen,
     StringRef SaveTempsDir, unsigned count) {
 
   // Save temps: after IPO.
   saveTempBitcode(TheModule, SaveTempsDir, count, ".1.IPO.bc");
 
+  // Prepare for internalization by computing the set of symbols to preserve.
+  // We need to compute the list of symbols to preserve during internalization
+  // before doing any promotion because after renaming we won't (easily) match
+  // to the original name.
+  auto PreservedGV = computePreservedSymbolsForModule(
+      TheModule, GUIDPreservedSymbols, ExportList);
+
   // "Benchmark"-like optimization: single-source case
   bool SingleModule = (ModuleMap.size() == 1);
 
   if (!SingleModule) {
     promoteModule(TheModule, Index);
 
     // Resolve the LinkOnce/Weak ODR, trying to turn them into
     // "available_externally" when possible.
     // This is a compile-time optimization.
     fixupODR(TheModule, ResolvedODR);
 
     // Save temps: after promotion.
     saveTempBitcode(TheModule, SaveTempsDir, count, ".2.promoted.bc");
+  }
 
+  // Internalization
+  doInternalizeModule(TheModule, TM, PreservedGV);
+
+  // Save internalized bitcode
+  saveTempBitcode(TheModule, SaveTempsDir, count, ".3.internalized.bc");
+
+  if (!SingleModule) {
     crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
 
     // Save temps: after cross-module import.
-    saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
+    saveTempBitcode(TheModule, SaveTempsDir, count, ".4.imported.bc");
   }
 
   optimizeModule(TheModule, TM);
 
-  saveTempBitcode(TheModule, SaveTempsDir, count, ".3.opt.bc");
+  saveTempBitcode(TheModule, SaveTempsDir, count, ".5.opt.bc");
 
   if (DisableCodeGen) {
     // Configured to stop before CodeGen, serialize the bitcode and return.
     SmallVector<char, 128> OutputBuffer;
     {
       raw_svector_ostream OS(OutputBuffer);
       ModuleSummaryIndexBuilder IndexBuilder(&TheModule);
       WriteBitcodeToFile(&TheModule, OS, true, &IndexBuilder.getIndex());
#endif
   Modules.push_back(Buffer);
 }
 
 void ThinLTOCodeGenerator::preserveSymbol(StringRef Name) {
   PreservedSymbols.insert(Name);
 }
 
 void ThinLTOCodeGenerator::crossReferenceSymbol(StringRef Name) {
-  CrossReferencedSymbols.insert(Name);
+  // FIXME: At the moment, we don't take advantage of this extra information,
+  // we're conservatively considering cross-references as preserved.
+  //  CrossReferencedSymbols.insert(Name);
+  PreservedSymbols.insert(Name);
 }
 
 // TargetMachine factory
 std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
   std::string ErrMsg;
   const Target *TheTarget =
       TargetRegistry::lookupTarget(TheTriple.str(), ErrMsg);
   if (!TheTarget) {
void ThinLTOCodeGenerator::crossModuleImport(Module &TheModule,
   ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
                            ExportLists);
   auto &ImportList = ImportLists[TheModule.getModuleIdentifier()];
 
   crossImportIntoModule(TheModule, Index, ModuleMap, ImportList);
 }
 
 /**
+ * Perform internalization.
+ */
+void ThinLTOCodeGenerator::internalize(Module &TheModule,
+                                       ModuleSummaryIndex &Index) {
+  initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
+  auto ModuleCount = Index.modulePaths().size();
+  auto ModuleIdentifier = TheModule.getModuleIdentifier();
+
+  // Convert the preserved symbols set from string to GUID
+  auto GUIDPreservedSymbols =
+      computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
+
+  // Collect for each module the list of function it defines (GUID -> Summary).
+  StringMap<std::map<GlobalValue::GUID, GlobalValueSummary *>>
+      ModuleToDefinedGVSummaries(ModuleCount);
+  Index.collectDefinedGVSummariesPerModule(ModuleToDefinedGVSummaries);
+
+  // Generate import/export list
+  StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
+  StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
+  ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
+                           ExportLists);
+  auto &ExportList = ExportLists[ModuleIdentifier];
+
+  // Internalization
+  auto PreservedGV = computePreservedSymbolsForModule(
+      TheModule, GUIDPreservedSymbols, ExportList);
+  doInternalizeModule(TheModule, *TMBuilder.create(), PreservedGV);
+}
+
+/**
  * Perform post-importing ThinLTO optimizations.
  */
 void ThinLTOCodeGenerator::optimize(Module &TheModule) {
   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
+
+  // Optimize now
   optimizeModule(TheModule, *TMBuilder.create());
 }
 
 /**
  * Perform ThinLTO CodeGen.
  */
 std::unique_ptr<MemoryBuffer> ThinLTOCodeGenerator::codegen(Module &TheModule) {
   initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
void ThinLTOCodeGenerator::run() {
   // Collect the import/export lists for all modules from the call-graph in the
   // combined index.
   StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
   StringMap<FunctionImporter::ExportSetTy> ExportLists(ModuleCount);
   ComputeCrossModuleImport(*Index, ModuleToDefinedGVSummaries, ImportLists,
                            ExportLists);
 
   // Convert the preserved symbols set from string to GUID, this is needed for
-  // computing the caching.
-  DenseSet<GlobalValue::GUID> GUIDPreservedSymbols(PreservedSymbols.size());
-  for (auto &Entry : PreservedSymbols)
-    GUIDPreservedSymbols.insert(GlobalValue::getGUID(Entry.first()));
+  // computing the caching hash and the internalization.
+  auto GUIDPreservedSymbols =
+      computeGUIDPreservedSymbols(PreservedSymbols, TMBuilder.TheTriple);
 
   // Parallel optimizer + codegen
   {
     ThreadPool Pool(ThreadCount);
     int count = 0;
     for (auto &ModuleBuffer : Modules) {
       Pool.async([&](int count) {
         auto ModuleIdentifier = ModuleBuffer.getBufferIdentifier();
         auto &ExportList = ExportLists[ModuleIdentifier];
 
         auto &DefinedFunctions = ModuleToDefinedGVSummaries[ModuleIdentifier];
 
         // Resolve ODR, this has to be done early because it impacts the caching
         // We use a std::map here to be able to have a defined ordering when
         // producing a hash for the cache entry.
         std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> ResolvedODR;
-        ResolveODR(*Index, ExportList, DefinedFunctions,
-                   ModuleIdentifier, ResolvedODR);
+        ResolveODR(*Index, ExportList, DefinedFunctions, ModuleIdentifier,
+                   ResolvedODR);
 
         // The module may be cached, this helps handling it.
-        ModuleCacheEntry CacheEntry(
-            CacheOptions.Path, *Index, ModuleBuffer.getBufferIdentifier(),
-            ImportLists[ModuleBuffer.getBufferIdentifier()],
-            ExportLists[ModuleBuffer.getBufferIdentifier()], ResolvedODR,
-            DefinedFunctions, GUIDPreservedSymbols);
+        ModuleCacheEntry CacheEntry(CacheOptions.Path, *Index, ModuleIdentifier,
+                                    ImportLists[ModuleIdentifier], ExportList,
+                                    ResolvedODR, DefinedFunctions,
+                                    GUIDPreservedSymbols);
 
         {
           auto ErrOrBuffer = CacheEntry.tryLoadingBuffer();
+          DEBUG(dbgs() << "Cache " << (ErrOrBuffer ? "hit" : "miss") << " '"
+                       << CacheEntry.getEntryPath() << "' for buffer " << count
+                       << " " << ModuleIdentifier << "\n");
+
           if (ErrOrBuffer) {
             // Cache Hit!
             ProducedBinaries[count] = std::move(ErrOrBuffer.get());
             return;
           }
         }
 
         LLVMContext Context;
         Context.setDiscardValueNames(LTODiscardValueNames);
         Context.enableDebugTypeODRUniquing();
 
         // Parse module now
         auto TheModule = loadModuleFromBuffer(ModuleBuffer, Context, false);
 
         // Save temps: original file.
-        if (!SaveTempsDir.empty()) {
-          saveTempBitcode(*TheModule, SaveTempsDir, count, ".0.original.bc");
-        }
 
         auto &ImportList = ImportLists[ModuleIdentifier];
+        // Run the main process now, and generates a binary
         auto OutputBuffer = ProcessThinLTOModule(
             *TheModule, *Index, ModuleMap, *TMBuilder.create(), ImportList,
-            ResolvedODR, CacheOptions, DisableCodeGen, SaveTempsDir, count);
+            ExportList, GUIDPreservedSymbols, ResolvedODR, CacheOptions,
+            DisableCodeGen, SaveTempsDir, count);
 
         CacheEntry.write(*OutputBuffer);
         ProducedBinaries[count] = std::move(OutputBuffer);
       }, count);
       count++;
     }
   }
 

test/ThinLTO/X86/internalize.ll

+;; RUN: opt -module-summary %s -o %t1.bc
+; RUN: llvm-lto -thinlto-action=thinlink -o %t.index.bc %t1.bc
+; RUN: llvm-lto -thinlto-action=internalize -thinlto-index %t.index.bc %t1.bc -o - | llvm-dis -o - | FileCheck %s --check-prefix=REGULAR
+; RUN: llvm-lto -thinlto-action=internalize -thinlto-index %t.index.bc %t1.bc -o -  --exported-symbol=foo | llvm-dis -o - | FileCheck %s --check-prefix=INTERNALIZE
+
+; REGULAR: define void @foo
+; REGULAR: define void @bar
+; INTERNALIZE: define void @foo
+; INTERNALIZE: define internal void @bar
+
+target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-apple-macosx10.11.0"
+
+define void @foo() {
+    ret void
+}
+define void @bar() {
+    ret void
+}
 No newline at end of file

tools/llvm-lto/llvm-lto.cpp

 static cl::opt<bool>
     ThinLTO("thinlto", cl::init(false),
             cl::desc("Only write combined global index for ThinLTO backends"));
 
 enum ThinLTOModes {
   THINLINK,
   THINPROMOTE,
   THINIMPORT,
+  THININTERNALIZE,
   THINOPT,
   THINCODEGEN,
   THINALL
 };
 
 cl::opt<ThinLTOModes> ThinLTOMode(
     "thinlto-action", cl::desc("Perform a single ThinLTO stage:"),
     cl::values(
         clEnumValN(
             THINLINK, "thinlink",
             "ThinLink: produces the index by linking only the summaries."),
         clEnumValN(THINPROMOTE, "promote",
                    "Perform pre-import promotion (requires -thinlto-index)."),
         clEnumValN(THINIMPORT, "import", "Perform both promotion and "
                                          "cross-module importing (requires "
                                          "-thinlto-index)."),
+        clEnumValN(THININTERNALIZE, "internalize",
+                   "Perform internalization driven by -exported-symbol "
+                   "(requires -thinlto-index)."),
         clEnumValN(THINOPT, "optimize", "Perform ThinLTO optimizations."),
         clEnumValN(THINCODEGEN, "codegen", "CodeGen (expected to match llc)"),
         clEnumValN(THINALL, "run", "Perform ThinLTO end-to-end"),
         clEnumValEnd));
 
 static cl::opt<std::string>
     ThinLTOIndex("thinlto-index",
                  cl::desc("Provide the index produced by a ThinLink, required "
                           "to perform the promotion and/or importing."));
 
 static cl::opt<bool>
     SaveModuleFile("save-merged-module", cl::init(false),
                    cl::desc("Write merged LTO module to file before CodeGen"));
 
 static cl::list<std::string> InputFilenames(cl::Positional, cl::OneOrMore,
                                             cl::desc("<input bitcode files>"));
 
 static cl::opt<std::string> OutputFilename("o", cl::init(""),
                                            cl::desc("Override output filename"),
                                            cl::value_desc("filename"));
 
-static cl::list<std::string>
-    ExportedSymbols("exported-symbol",
-                    cl::desc("Symbol to export from the resulting object file"),
-                    cl::ZeroOrMore);
+static cl::list<std::string> ExportedSymbols(
+    "exported-symbol",
+    cl::desc("List of symbols to export from the resulting object file"),
 
 static cl::list<std::string>
     DSOSymbols("dso-symbol",
                cl::desc("Symbol to put in the symtab in the resulting dso"),
                cl::ZeroOrMore);
 
 static cl::opt<bool> ListSymbolsOnly(
     "list-symbols-only", cl::init(false),

 
 class ThinLTOProcessing {
 public:
   ThinLTOCodeGenerator ThinGenerator;
 
   ThinLTOProcessing(const TargetOptions &Options) {
     ThinGenerator.setCodePICModel(RelocModel);
     ThinGenerator.setTargetOptions(Options);
+
+    // Add all the exported symbols to the table of symbols to preserve.
+    for (unsigned i = 0; i < ExportedSymbols.size(); ++i)
+      ThinGenerator.preserveSymbol(ExportedSymbols[i]);
   }
 
   void run() {
     switch (ThinLTOMode) {
     case THINLINK:
       return thinLink();
     case THINPROMOTE:
       return promote();
     case THINIMPORT:
       return import();
+    case THININTERNALIZE:
+      return internalize();
     case THINOPT:
       return optimize();
     case THINCODEGEN:
       return codegen();
     case THINALL:
       return runAll();
     }
   }
for (auto &Filename : InputFilenames) {
       std::string OutputName = OutputFilename;
       if (OutputName.empty()) {
         OutputName = Filename + ".thinlto.imported.bc";
       }
       writeModuleToFile(*TheModule, OutputName);
     }
   }
 
+  void internalize() {
+    if (InputFilenames.size() != 1 && !OutputFilename.empty())
+      report_fatal_error("Can't handle a single output filename and multiple "
+                         "input files, do not provide an output filename and "
+                         "the output files will be suffixed from the input "
+                         "ones.");
+
+    if (ExportedSymbols.empty())
+      errs() << "Warning: -internalize will not perform without "
+                "-exported-symbol\n";
+
+    auto Index = loadCombinedIndex();
+    auto InputBuffers = loadAllFilesForIndex(*Index);
+    for (auto &MemBuffer : InputBuffers)
+      ThinGenerator.addModule(MemBuffer->getBufferIdentifier(),
+                              MemBuffer->getBuffer());
+
+    for (auto &Filename : InputFilenames) {
+      LLVMContext Ctx;
+      auto TheModule = loadModule(Filename, Ctx);
+
+      ThinGenerator.internalize(*TheModule, *Index);
+
+      std::string OutputName = OutputFilename;
+      if (OutputName.empty()) {
+        OutputName = Filename + ".thinlto.internalized.bc";
+      }
+      writeModuleToFile(*TheModule, OutputName);
+    }
+  }
+
   void optimize() {
     if (InputFilenames.size() != 1 && !OutputFilename.empty())
       report_fatal_error("Can't handle a single output filename and multiple "
                          "input files, do not provide an output filename and "
                          "the output files will be suffixed from the input "
                          "ones.");
     if (!ThinLTOIndex.empty())
       errs() << "Warning: -thinlto-index ignored for optimize stage";