[ThinLTO][test] Add tests for emitting files in-process

[Northbadge]

Summary

Completes D127777 by adding llvm-side tests for emitting index and imports files from in-process ThinLTO

[mtrofin]

lgtm, but please wait for tejohnson's feedback

[tejohnson]

Thanks! Couple minor comments.

llvm/test/ThinLTO/X86/emit-inprocess-files.ll

8

Also, this comment isn't quite right - the minimized bitcode is not just minus the debug metadata. It is without any of the IR other than the minimal symbol info needed for the summary. Better to just say "as well as minimized bitcode containing just the summary"

11

Feel free to remove this if we are already testing the minimized bitcode option elsewhere.

llvm/tools/llvm-lto2/llvm-lto2.cpp

85

As suggested below, we should check this for the distributed indexes case too. Also, either change the wording to something like "Has no effect unless specified with ...", or do some error checking.

313–314

For completeness, this should also use the new ThinLTOEmitImports. But then you will unfortunately need to update a number of existing tests (hopefully not too many as we don't tend to look at the contents or for the existence of the imports files though - a quick search suggests maybe only llvm/test/ThinLTO/X86/emit_imports.ll).

[Northbadge]

Thanks for the review! Added support for -thinlto-emit-imports with -thinlto-distributed-indexes

[tejohnson]

lgtm

llvm/test/ThinLTO/X86/distributed_import.ll

 ; Test distributed build thin link output from llvm-lto2
 
-; Generate bitcode files with summary, as well as minimized bitcode without
-; the debug metadata for the thin link.
+; Generate bitcode files with summary, as well as minimized bitcode containing just the summary
 ; RUN: opt -thinlto-bc %s -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.bc
 ; RUN: opt -thinlto-bc %p/Inputs/distributed_import.ll -thin-link-bitcode-file=%t2.thinlink.bc -o %t2.bc
 ; RUN: llvm-bcanalyzer -dump %t1.thinlink.bc | FileCheck --check-prefix=THINLINKBITCODE %s
 
 ; First perform the thin link on the normal bitcode file.
 ; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.o -save-temps \
 ; RUN:     -thinlto-distributed-indexes \
 ; RUN:     -r=%t1.bc,g, \

llvm/test/ThinLTO/X86/emit-inprocess-files.ll

+; Test InProcessThinLTO thin link output from llvm-lto2
+; Partially copied from distributed_import.ll, and added checks for {dis|en}abling imports
+
+; RUN: rm -f %t1.bc.thinlto.bc %t2.bc.thinlto.bc %t.out.1 %t.out.2 %t1.bc.imports %t2.bc.imports
+
+; Generate bitcode files with summary, as well as minimized bitcode containing just the summary
+; RUN: opt -thinlto-bc %s -thin-link-bitcode-file=%t1.thinlink.bc -o %t1.bc
+; RUN: opt -thinlto-bc %p/Inputs/distributed_import.ll -thin-link-bitcode-file=%t2.thinlink.bc -o %t2.bc
+
+; First perform the thin link on the normal bitcode file using
+; -thinlto-distributed-indexes, collecting outputs to be compared with later.
+; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.o -save-temps \
+; RUN:     -thinlto-distributed-indexes \
+; RUN:     -thinlto-emit-imports \
+; RUN:     -r=%t1.bc,g, \
+; RUN:     -r=%t1.bc,analias, \
+; RUN:     -r=%t1.bc,f,px \
+; RUN:     -r=%t2.bc,g,px \
+; RUN:     -r=%t2.bc,analias,px \
+; RUN:     -r=%t2.bc,aliasee,px
+; RUN: mv %t1.bc.thinlto.bc %t1.bc.thinlto.bc.orig
+; RUN: mv %t2.bc.thinlto.bc %t2.bc.thinlto.bc.orig
+; RUN: mv %t1.bc.imports %t1.bc.imports.orig
+; RUN: mv %t2.bc.imports %t2.bc.imports.orig
+
+; Now use -thinlto-emit-indexes instead.
+; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.o -save-temps \
+; RUN:     -thinlto-emit-indexes \
+; RUN:     -r=%t1.bc,g, \
+; RUN:     -r=%t1.bc,analias, \
+; RUN:     -r=%t1.bc,f,px \
+; RUN:     -r=%t2.bc,g,px \
+; RUN:     -r=%t2.bc,analias,px \
+; RUN:     -r=%t2.bc,aliasee,px \
+; RUN:     -o=%t.out
+
+; Since InProcessThinLTO ran, there should be output
+; RUN: ls %t.out.1
+; RUN: ls %t.out.2
+
+; Ensure imports weren't generated since -thinlto-emit-imports wasn't specified
+; RUN: not ls %t1.bc.imports
+; RUN: not ls %t2.bc.imports
+
+; Compare the generated index files.
+; RUN: diff %t1.bc.thinlto.bc %t1.bc.thinlto.bc.orig
+; RUN: diff %t2.bc.thinlto.bc %t2.bc.thinlto.bc.orig
+
+; RUN: rm -f %t1.bc.thinlto.bc %t2.bc.thinlto.bc %t.out.1 %t.out.2
+
+; Do the thin link again but also emit imports files now
+; RUN: llvm-lto2 run %t1.bc %t2.bc -o %t.o -save-temps \
+; RUN:     -thinlto-emit-indexes \
+; RUN:     -thinlto-emit-imports \
+; RUN:     -r=%t1.bc,g, \
+; RUN:     -r=%t1.bc,analias, \
+; RUN:     -r=%t1.bc,f,px \
+; RUN:     -r=%t2.bc,g,px \
+; RUN:     -r=%t2.bc,analias,px \
+; RUN:     -r=%t2.bc,aliasee,px \
+; RUN:     -o=%t.out
+
+; Check the output
+; RUN: ls %t.out.1
+; RUN: ls %t.out.2
+; RUN: diff %t1.bc.thinlto.bc %t1.bc.thinlto.bc.orig
+; RUN: diff %t2.bc.thinlto.bc %t2.bc.thinlto.bc.orig
+; RUN: diff %t1.bc.imports %t1.bc.imports.orig
+; RUN: diff %t2.bc.imports %t2.bc.imports.orig
+
+target triple = "x86_64-unknown-linux-gnu"
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+
+declare i32 @g(...)
+declare void @analias(...)
+
+define void @f() {
+entry:
+  call i32 (...) @g()
+  call void (...) @analias()
+  ret void
+}
+
+!llvm.dbg.cu = !{}
+
+!1 = !{i32 2, !"Debug Info Version", i32 3}
+!llvm.module.flags = !{!1}

llvm/test/ThinLTO/X86/emit_imports.ll

 ; RUN: llvm-lto2 run %t1.bc %t2.bc %t2b.bc %t3.bc -o %t.o -save-temps \
 ; RUN:     -thinlto-distributed-indexes \
 ; RUN:     -r=%t1.bc,g, \
 ; RUN:     -r=%t1.bc,h, \
 ; RUN:     -r=%t1.bc,f,px \
 ; RUN:     -r=%t2.bc,g,px \
 ; RUN:     -r=%t2b.bc,h,px
 
+; Ensure imports weren't generated since -thinlto-emit-imports wasn't specified
+; RUN: not ls %t1.bc.imports
+; RUN: not ls %t2.bc.imports
+; RUN: not ls %t3.bc.imports
+
+; RUN: llvm-lto2 run %t1.bc %t2.bc %t2b.bc %t3.bc -o %t.o -save-temps \
+; RUN:     -thinlto-distributed-indexes \
+; RUN:     -thinlto-emit-imports \
+; RUN:     -r=%t1.bc,g, \
+; RUN:     -r=%t1.bc,h, \
+; RUN:     -r=%t1.bc,f,px \
+; RUN:     -r=%t2.bc,g,px \
+; RUN:     -r=%t2b.bc,h,px
+
 ; RUN: cat %t1.bc.imports | count 2
 ; RUN: cat %t1.bc.imports | FileCheck %s --check-prefix=IMPORTS1
 
 ; The imports file for Input/emit_imports.ll is empty as it does not import anything.
 ; RUN: cat %t2.bc.imports | count 0
 
 ; The imports file for Input/empty.ll is empty but should exist.
 ; RUN: cat %t3.bc.imports | count 0

llvm/tools/llvm-lto2/llvm-lto2.cpp

 static cl::opt<bool> SaveTemps("save-temps", cl::desc("Save temporary files"));
 
 static cl::opt<bool>
     ThinLTODistributedIndexes("thinlto-distributed-indexes", cl::init(false),
                               cl::desc("Write out individual index and "
                                        "import files for the "
                                        "distributed backend case"));
 
+static cl::opt<bool>
+    ThinLTOEmitIndexes("thinlto-emit-indexes", cl::init(false),
+                       cl::desc("Write out individual index files via "
+                                "InProcessThinLTO"));
+
+static cl::opt<bool>
+    ThinLTOEmitImports("thinlto-emit-imports", cl::init(false),
+                       cl::desc("Write out individual imports files via "
+                                "InProcessThinLTO. Has no effect unless "
+                                "specified with -thinlto-emit-indexes or "
+                                "-thinlto-distributed-indexes"));
+
 // Default to using all available threads in the system, but using only one
 // thread per core (no SMT).
 // Use -thinlto-threads=all to use hardware_concurrency() instead, which means
 // to use all hardware threads or cores in the system.
 static cl::opt<std::string> Threads("thinlto-threads");
 
 static cl::list<std::string> SymbolResolutions(
     "r",
static int run(int argc, char **argv) {
   Conf.OverrideTriple = OverrideTriple;
   Conf.DefaultTriple = DefaultTriple;
   Conf.StatsFile = StatsFile;
   Conf.PTO.LoopVectorization = Conf.OptLevel > 1;
   Conf.PTO.SLPVectorization = Conf.OptLevel > 1;
   Conf.OpaquePointers = LtoOpaquePointers;
 
   ThinBackend Backend;
   if (ThinLTODistributedIndexes)
-    Backend = createWriteIndexesThinBackend(/* OldPrefix */ "",
-                                            /* NewPrefix */ "",
-                                            /* ShouldEmitImportsFiles */ true,
-                                            /* LinkedObjectsFile */ nullptr,
-                                            /* OnWrite */ {});
+    Backend =
+        createWriteIndexesThinBackend(/* OldPrefix */ "",
+                                      /* NewPrefix */ "", ThinLTOEmitImports,
   else
     Backend = createInProcessThinBackend(
-        llvm::heavyweight_hardware_concurrency(Threads));
+        llvm::heavyweight_hardware_concurrency(Threads),
+        /* OnWrite */ {}, ThinLTOEmitIndexes, ThinLTOEmitImports);
+
   // Track whether we hit an error; in particular, in the multi-threaded case,
   // we can't exit() early because the rest of the threads wouldn't have had a
   // change to be join-ed, and that would result in a "terminate called without
   // an active exception". Altogether, this results in nondeterministic
   // behavior. Instead, we don't exit in the multi-threaded case, but we make
   // sure to report the error and then at the end (after joining cleanly)
   // exit(1).
   std::atomic<bool> HasErrors;