[LinkerWrapper] Emit assembly files from LTO with `save-temps`

[jhuber6]

Summary

Currently in LTO mode we don't emit any .s files for non-NVPTX targets
during LTO. This makes it diffcult to investigate any failures in the
assembly. This patch makes the save-temps mode output an assembly file
and then assembles it separately. I decided to simply invoke clang for
this as it would be a lot of effort to invoke the MCStramer interface
directly.

[arsenm]

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

531–537

can use initializer list?

[arsenm]

Missing test?

[jhuber6]

Add test.

[jhuber6]

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

531–537

Probably could, this is the same style we use in Clang.cpp so I usually end up using it.

[jhuber6]

Fix test, should be good to go now.

[arsenm]

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

531–537

well Clang.cpp is old, should move towards doing not old things

[jhuber6]

Changing to initializer list.

[arsenm]

Thanks, maybe I'll actually be able to comprehend openmp now

[thakis]

Looks like this breaks tests on Windows: http://45.33.8.238/win/73276/step_7.txt

Please take a look and revert for now if it takes a while to fix.

[jhuber6]

In D141543#4050080, @thakis wrote:

Looks like this breaks tests on Windows: http://45.33.8.238/win/73276/step_7.txt

Please take a look and revert for now if it takes a while to fix.

Yes, seems I forgot that clang is clang.exe on Windows. I'll fix it real quick.

[thakis]

Thanks! Still failing with that though: http://45.33.8.238/win/73278/step_7.txt

[jhuber6]

In D141543#4050174, @thakis wrote:

Thanks! Still failing with that though: http://45.33.8.238/win/73278/step_7.txt

Hm, I'm unsure what would be causing that. I could always just disable the test on Windows. This program isn't really used on Windows anyway.

[thakis]

cb2f23a9b0de9c6747afb1efdc6e05b4691e0c50 did the trick. Thanks for the fix!

clang/test/Driver/linker-wrapper.c

 // RUN:   --image=file=%t.elf.o,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908
 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
 // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
 // RUN:   --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=AMDGPU-LINK
 
 // AMDGPU-LINK: lld{{.*}}-flavor gnu --no-undefined -shared -plugin-opt=-amdgpu-internalize-symbols -plugin-opt=mcpu=gfx908 -o {{.*}}.out {{.*}}.o {{.*}}.o
 
 // RUN: clang-offload-packager -o %t.out \
+// RUN:   --image=file=%t.amdgpu.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030 \
+// RUN:   --image=file=%t.amdgpu.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx1030
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
+// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run --save-temps -O2 \
+// RUN:   --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=AMDGPU-LTO-TEMPS
+
+// AMDGPU-LTO-TEMPS: clang" -o [[OBJ:.+]] -fPIC -c --target=amdgcn-amd-amdhsa -O2 -mcpu=gfx1030 {{.*}}.s
+// AMDGPU-LTO-TEMPS: lld{{.*}}-flavor gnu --no-undefined -shared -plugin-opt=-amdgpu-internalize-symbols -plugin-opt=mcpu=gfx1030 -o {{.*}}.out {{.*}}.o
+
+// RUN: clang-offload-packager -o %t.out \
 // RUN:   --image=file=%t.amdgpu.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908 \
 // RUN:   --image=file=%t.amdgpu.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908
 // RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
 // RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run \
 // RUN:   --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=AMDGPU-LINK-LTO
 
 // AMDGPU-LINK-LTO: lld{{.*}}-flavor gnu --no-undefined -shared -plugin-opt=-amdgpu-internalize-symbols -plugin-opt=mcpu=gfx908 -o {{.*}}.out {{.*}}.o
 

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

case llvm::Triple::x86_64:
     return "elf_x86_64";
   case llvm::Triple::ve:
     return "elf64ve";
   default:
     return nullptr;
   }
 }
 
+Expected<StringRef> assemble(StringRef InputFile, const ArgList &Args) {
+  llvm::TimeTraceScope TimeScope("Clang Assembler");
+  // Use `clang` to invoke the generic assembler.
+  Expected<std::string> ClangPath =
+      findProgram("clang", {getMainExecutable("clang")});
+  if (!ClangPath)
+    return ClangPath.takeError();
+
+  const llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
+  StringRef Arch = Args.getLastArgValue(OPT_arch_EQ);
+  // Create a new file to write the linked device image to. Assume that the
+  // input filename already has the device and architecture.
+  auto TempFileOrErr = createOutputFile(sys::path::stem(InputFile), "o");
+  if (!TempFileOrErr)
+    return TempFileOrErr.takeError();
+
+  StringRef OptLevel = Args.getLastArgValue(OPT_opt_level, "O2");
+  SmallVector<StringRef, 16> CmdArgs{
+      *ClangPath,
+      "-o",
+      *TempFileOrErr,
+      "-fPIC",
+      "-c",
+      Args.MakeArgString("--target=" + Triple.getTriple()),
+      Args.MakeArgString("-" + OptLevel),
+      Triple.isAMDGPU() ? Args.MakeArgString("-mcpu=" + Arch)
+                        : Args.MakeArgString("-march=" + Arch),
+      InputFile,
+  };
+
+  if (Error Err = executeCommands(*ClangPath, CmdArgs))
+    return std::move(Err);
+
+  return *TempFileOrErr;
+}
+
 Expected<StringRef> link(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
   llvm::TimeTraceScope TimeScope("Generic linker");
   const llvm::Triple Triple(Args.getLastArgValue(OPT_triple_EQ));
   StringRef Arch = Args.getLastArgValue(OPT_arch_EQ);
 
   // Create a new file to write the linked device image to.
   auto TempFileOrErr =
       createOutputFile(sys::path::filename(ExecutableName) + "-" +
Conf.PostInternalizeModuleHook = [=](size_t Task, const Module &M) {
       raw_fd_ostream LinkedBitcode(File, EC, sys::fs::OF_None);
       if (EC)
         reportError(errorCodeToError(EC));
       WriteBitcodeToFile(M, LinkedBitcode);
       return true;
     };
   }
   Conf.PostOptModuleHook = Hook;
-  Conf.CGFileType = Triple.isNVPTX() ? CGFT_AssemblyFile : CGFT_ObjectFile;
+  Conf.CGFileType =
+      (Triple.isNVPTX() || SaveTemps) ? CGFT_AssemblyFile : CGFT_ObjectFile;
 
   // TODO: Handle remark files
   Conf.HasWholeProgramVisibility = Args.hasArg(OPT_whole_program);
 
   return std::make_unique<lto::LTO>(std::move(Conf), Backend);
 }
 
 // Returns true if \p S is valid as a C language identifier and will be given
Error linkBitcodeFiles(SmallVectorImpl<OffloadFile> &InputFiles,
   // Run the LTO job to compile the bitcode.
   size_t MaxTasks = LTOBackend->getMaxTasks();
   SmallVector<StringRef> Files(MaxTasks);
   auto AddStream =
       [&](size_t Task,
           const Twine &ModuleName) -> std::unique_ptr<CachedFileStream> {
     int FD = -1;
     auto &TempFile = Files[Task];
-    StringRef Extension = (Triple.isNVPTX()) ? "s" : "o";
+    StringRef Extension = (Triple.isNVPTX() || SaveTemps) ? "s" : "o";
     std::string TaskStr = Task ? "." + std::to_string(Task) : "";
     auto TempFileOrErr =
         createOutputFile(sys::path::filename(ExecutableName) + "-device-" +
                              Triple.getTriple() + TaskStr,
                          Extension);
     if (!TempFileOrErr)
       reportError(TempFileOrErr.takeError());
     TempFile = *TempFileOrErr;
if (Args.hasArg(OPT_embed_bitcode)) {
     if (BitcodeOutput.size() != 1 || !SingleOutput)
       return createStringError(inconvertibleErrorCode(),
                                "Cannot embed bitcode with multiple files.");
     OutputFiles.push_back(Args.MakeArgString(BitcodeOutput.front()));
     return Error::success();
   }
 
   // Is we are compiling for NVPTX we need to run the assembler first.
-  if (Triple.isNVPTX()) {
+  if (Triple.isNVPTX() || SaveTemps) {
     for (StringRef &File : Files) {
-      auto FileOrErr = nvptx::assemble(File, Args, !SingleOutput);
+
+      auto FileOrErr = Triple.isNVPTX()
+                           ? nvptx::assemble(File, Args, !SingleOutput)
+                           : generic::assemble(File, Args);
       if (!FileOrErr)
         return FileOrErr.takeError();
       File = *FileOrErr;
     }
   }
 
   // Append the new inputs to the device linker input.
   for (StringRef File : Files)