[OpenMP] Use offload bundler when outputting textual formats with new driver

[jhuber6]

Summary

The new OpenMP offloading currently driver embeds offloading IR directly
into the host object. This is easier to handle in the tooling, however
makes it difficult to easily inspect the output generated by the
compiler. The old method used the clang-offload-bundler tool put the IR
side-by-side. We should do something similar with the new driver to
prevent the need to use save-temps for simple cases. An example of this
is shown when using a tool like godbolt to share code:
https://godbolt.org/z/eYEcYsbs5.

This doesn't address bundling preprocessor output yet.

[MaskRay]

clang/test/Driver/openmp-offload-gpu.c

350

Delete excessive indentation

Generally indenting the continuation lines by 2 column is sufficient.

[jhuber6]

clang/test/Driver/openmp-offload-gpu.c

350

All the other lines are formatted like this in this file so I was just keeping it consistent. I could make a separate patch to clean it up.

clang/include/clang/Driver/Driver.h

public:
 
   /// BuildOffloadingActions - Construct the list of actions to perform for the
   /// offloading toolchain that will be embedded in the host.
   ///
   /// \param C - The compilation that is being built.
   /// \param Args - The input arguments.
   /// \param Input - The input type and arguments
   /// \param HostAction - The host action used in the offloading toolchain.
+  /// \param DeviceAction - The leftover offload action if not merged.
   Action *BuildOffloadingActions(Compilation &C,
                                  llvm::opt::DerivedArgList &Args,
-                                 const InputTy &Input,
-                                 Action *HostAction) const;
+                                 const InputTy &Input, Action *HostAction,
+                                 Action *&DeviceAction) const;
 
   /// Check that the file referenced by Value exists. If it doesn't,
   /// issue a diagnostic and return false.
   /// If TypoCorrect is true and the file does not exist, see if it looks
   /// like a likely typo for a flag and if so print a "did you mean" blurb.
   bool DiagnoseInputExistence(const llvm::opt::DerivedArgList &Args,
                               StringRef Value, types::ID Ty,
                               bool TypoCorrect) const;

clang/lib/Driver/Driver.cpp

for (auto &I : Inputs) {
     if (PL.empty())
       continue;
 
     auto FullPL = types::getCompilationPhases(InputType);
 
     // Build the pipeline for this file.
     Action *Current = C.MakeAction<InputAction>(*InputArg, InputType);
 
+    // The leftover offloading action not merged into the host, if any.
+    Action *CurrentOffload = nullptr;
+
     // Use the current host action in any of the offloading actions, if
     // required.
     if (!Args.hasArg(options::OPT_fopenmp_new_driver))
       if (OffloadBuilder.addHostDependenceToDeviceActions(Current, InputArg))
         break;
 
     for (phases::ID Phase : PL) {
 
for (phases::ID Phase : PL) {
         ExtractAPIAction->addHeaderInput(Current);
         Current = nullptr;
         break;
       }
 
       // Try to build the offloading actions and add the result as a dependency
       // to the host.
       if (Args.hasArg(options::OPT_fopenmp_new_driver))
-        Current = BuildOffloadingActions(C, Args, I, Current);
+        Current = BuildOffloadingActions(C, Args, I, Current, CurrentOffload);
 
       // FIXME: Should we include any prior module file outputs as inputs of
       // later actions in the same command line?
 
       // Otherwise construct the appropriate action.
       Action *NewCurrent = ConstructPhaseAction(C, Args, Phase, Current);
 
       // We didn't create a new action, so we will just move to the next phase.
for (phases::ID Phase : PL) {
       if (!Args.hasArg(options::OPT_fopenmp_new_driver))
         if (OffloadBuilder.addHostDependenceToDeviceActions(Current, InputArg))
           break;
 
       if (Current->getType() == types::TY_Nothing)
         break;
     }
 
+    // Bundle any leftover device output with the host action.
+    if (Current && CurrentOffload) {
+      ActionList AL{CurrentOffload, Current};
+      Current = C.MakeAction<OffloadBundlingJobAction>(AL);
+    }
+
     // If we ended with something, add to the output list.
     if (Current)
       Actions.push_back(Current);
 
     // Add any top level actions generated for offloading.
     if (!Args.hasArg(options::OPT_fopenmp_new_driver))
       OffloadBuilder.appendTopLevelActions(Actions, Current, InputArg);
     else if (Current)
void Driver::BuildActions(Compilation &C, DerivedArgList &Args,
   // Claim --cuda-host-only and --cuda-compile-host-device, which may be passed
   // to non-CUDA compilations and should not trigger warnings there.
   Args.ClaimAllArgs(options::OPT_cuda_host_only);
   Args.ClaimAllArgs(options::OPT_cuda_compile_host_device);
 }
 
 Action *Driver::BuildOffloadingActions(Compilation &C,
                                        llvm::opt::DerivedArgList &Args,
-                                       const InputTy &Input,
-                                       Action *HostAction) const {
+                                       const InputTy &Input, Action *HostAction,
+                                       Action *&DeviceAction) const {
   if (!isa<CompileJobAction>(HostAction))
     return HostAction;
 
   OffloadAction::DeviceDependences DDeps;
 
   types::ID InputType = Input.first;
   const Arg *InputArg = Input.second;
 
   const Action::OffloadKind OffloadKinds[] = {Action::OFK_OpenMP};
 
+  auto PL = types::getCompilationPhases(*this, Args, InputType);
+
   for (Action::OffloadKind Kind : OffloadKinds) {
     SmallVector<const ToolChain *, 2> ToolChains;
     ActionList DeviceActions;
 
     auto TCRange = C.getOffloadToolChains(Kind);
     for (auto TI = TCRange.first, TE = TCRange.second; TI != TE; ++TI)
       ToolChains.push_back(TI->second);
 
     if (ToolChains.empty())
       continue;
 
     for (unsigned I = 0; I < ToolChains.size(); ++I)
       DeviceActions.push_back(C.MakeAction<InputAction>(*InputArg, InputType));
 
     if (DeviceActions.empty())
       return HostAction;
 
-    auto PL = types::getCompilationPhases(*this, Args, InputType);
-
     for (phases::ID Phase : PL) {
       if (Phase == phases::Link) {
         assert(Phase == PL.back() && "linking must be final compilation step.");
         break;
       }
 
       auto TC = ToolChains.begin();
       for (Action *&A : DeviceActions) {
for (Action::OffloadKind Kind : OffloadKinds) {
 
     auto TC = ToolChains.begin();
     for (Action *A : DeviceActions) {
       DDeps.add(*A, **TC, /*BoundArch=*/nullptr, Kind);
       TC++;
     }
   }
 
+  // We shouldn't embed the device action in the host if we are targeting a
+  // textual output format.
+  if (PL.back() != phases::Assemble && PL.back() != phases::Link) {
+    DeviceAction = C.MakeAction<OffloadAction>(
+        DDeps, DDeps.getActions().back()->getType());
+    return HostAction;
+  }
+
   OffloadAction::HostDependence HDep(
       *HostAction, *C.getSingleOffloadToolChain<Action::OFK_Host>(),
       /*BoundArch=*/nullptr, DDeps);
   return C.MakeAction<OffloadAction>(HDep, DDeps);
 }
 
 Action *Driver::ConstructPhaseAction(
     Compilation &C, const ArgList &Args, phases::ID Phase, Action *Input,

clang/test/Driver/openmp-offload-gpu.c

 // NEW_DRIVER: "nvptx64-nvidia-cuda" - "NVPTX::Assembler", inputs: ["[[DEVICE_ASM]]"], output: "[[DEVICE_OBJ:.+]]" 
 
 // RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target=nvptx64-nvida-cuda -march=sm_70 \
 // RUN:          --libomptarget-nvptx-bc-path=%S/Inputs/libomptarget/libomptarget-new-nvptx-test.bc \
 // RUN:          -fopenmp-new-driver -no-canonical-prefixes -nogpulib %s -o openmp-offload-gpu 2>&1 \
 // RUN:   | FileCheck -check-prefix=NEW_DRIVER_EMBEDDING %s
 
 // NEW_DRIVER_EMBEDDING: -fembed-offload-object=[[CUBIN:.*\.cubin]],openmp.nvptx64-nvidia-cuda.sm_70
+
+// RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=nvptx64-nvidia-cuda -Xopenmp-target=nvptx64-nvida-cuda -march=sm_70 \
+// RUN:          --libomptarget-nvptx-bc-path=%S/Inputs/libomptarget/libomptarget-new-nvptx-test.bc \
+// RUN:          -fopenmp-new-driver -no-canonical-prefixes -S -emit-llvm -nogpulib %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=NEW_DRIVER_BUNDLING %s
+
+// NEW_DRIVER_BUNDLING: clang-offload-bundler" "-type=ll"