[Driver][OpenMP] Update actions builder to create bundling action when necessary.

[sfantao]

Summary

In order to save the user from dealing with multiple output files (for host and device) while using separate compilation, a new action OffloadBundlingAction is used when the last phase is not linking. This action will then result in a job that uses the proposed bundling tool to create a single preprocessed/IR/ASM/Object file from multiple ones.

The job creation for the new action will be proposed in a separate patch.

[sfantao]

• Rebase.
• Fix format.

[sfantao]

• Rebase

[sfantao]

• Rebase.

[sfantao]

• Rebase.

[ABataev]

LG

[hfinkel]

LGTM

[lijiansong]

lib/Driver/Driver.cpp

2113–2124

hello,
if I want to compile device-only device code, e.g. clang foo.mlu a.cpp -o bar,
I want to get the following action graph:
for foo.mlu:
input -> preprocess->compile->backend->assemble ,
Then i will get foo.o,
for a.cpp:
input -> preprocess->compile->backend->assemble
then I will get a.o
i want to link foo.o with a.o to get bar, a.cpp is the main, while foo.mlu has the device computation logic.

How can I manage it with offload?

[lijiansong]

This comment was removed by lijiansong.

include/clang/Driver/Action.h

enum ActionClass {
     CompileJobClass,
     BackendJobClass,
     AssembleJobClass,
     LinkJobClass,
     LipoJobClass,
     DsymutilJobClass,
     VerifyDebugInfoJobClass,
     VerifyPCHJobClass,
+    OffloadBundlingJobClass,
 
     JobClassFirst = PreprocessJobClass,
-    JobClassLast = VerifyPCHJobClass
+    JobClassLast = OffloadBundlingJobClass
   };
 
   // The offloading kind determines if this action is binded to a particular
   // programming model. Each entry reserves one bit. We also have a special kind
   // to designate the host offloading tool chain.
   enum OffloadKind {
     OFK_None = 0x00,
     // The host offloading tool chain.
class VerifyPCHJobAction : public VerifyJobAction {
   void anchor() override;
 public:
   VerifyPCHJobAction(Action *Input, types::ID Type);
   static bool classof(const Action *A) {
     return A->getKind() == VerifyPCHJobClass;
   }
 };
 
+class OffloadBundlingJobAction : public JobAction {
+  void anchor() override;
+
+public:
+  // Offloading bundling doesn't change the type of output.
+  OffloadBundlingJobAction(ActionList &Inputs);
+
+  static bool classof(const Action *A) {
+    return A->getKind() == OffloadBundlingJobClass;
+  }
+};
+
 } // end namespace driver
 } // end namespace clang
 
 #endif

lib/Driver/Action.cpp

const char *Action::getClassName(ActionClass AC) {
   case CompileJobClass: return "compiler";
   case BackendJobClass: return "backend";
   case AssembleJobClass: return "assembler";
   case LinkJobClass: return "linker";
   case LipoJobClass: return "lipo";
   case DsymutilJobClass: return "dsymutil";
   case VerifyDebugInfoJobClass: return "verify-debug-info";
   case VerifyPCHJobClass: return "verify-pch";
+  case OffloadBundlingJobClass:
+    return "clang-offload-bundler";
   }
 
   llvm_unreachable("invalid class");
 }
 
 void Action::propagateDeviceOffloadInfo(OffloadKind OKind, const char *OArch) {
   // Offload action set its own kinds on their dependences.
   if (Kind == OffloadClass)

 VerifyDebugInfoJobAction::VerifyDebugInfoJobAction(Action *Input,
                                                    types::ID Type)
     : VerifyJobAction(VerifyDebugInfoJobClass, Input, Type) {}
 
 void VerifyPCHJobAction::anchor() {}
 
 VerifyPCHJobAction::VerifyPCHJobAction(Action *Input, types::ID Type)
     : VerifyJobAction(VerifyPCHJobClass, Input, Type) {}
+
+void OffloadBundlingJobAction::anchor() {}
+
+OffloadBundlingJobAction::OffloadBundlingJobAction(ActionList &Inputs)
+    : JobAction(OffloadBundlingJobClass, Inputs, Inputs.front()->getType()) {}

lib/Driver/Driver.cpp

public:
     /// Append linker actions generated by the builder. Return true if errors
     /// were found.
     virtual void appendLinkDependences(OffloadAction::DeviceDependences &DA) {}
 
     /// Initialize the builder. Return true if any initialization errors are
     /// found.
     virtual bool initialize() { return false; }
 
+    /// Return true if the builder can use bundling/unbundling.
+    virtual bool canUseBundlerUnbundler() const { return false; }
+
     /// Return true if this builder is valid. We have a valid builder if we have
     /// associated device tool chains.
     bool isValid() { return !ToolChains.empty(); }
 
     /// Return the associated offload kind.
     Action::OffloadKind getAssociatedOffloadKind() {
       return AssociatedOffloadKind;
     }
ActionBuilderReturnCode addDeviceDepences(Action *HostAction) override {
           DDep.add(*A, **TC, /*BoundArch=*/nullptr, Action::OFK_OpenMP);
           A = C.MakeAction<OffloadAction>(HDep, DDep);
           ++TC;
         }
       }
       return ABRT_Success;
     }
 
+    void appendTopLevelActions(ActionList &AL) override {
+      if (OpenMPDeviceActions.empty())
+        return;
+
+      // We should always have an action for each input.
+      assert(OpenMPDeviceActions.size() == ToolChains.size() &&
+             "Number of OpenMP actions and toolchains do not match.");
+
+      // Append all device actions followed by the proper offload action.
+      auto TI = ToolChains.begin();
+      for (auto *A : OpenMPDeviceActions) {
+        OffloadAction::DeviceDependences Dep;
+        Dep.add(*A, **TI, /*BoundArch=*/nullptr, Action::OFK_OpenMP);
+        AL.push_back(C.MakeAction<OffloadAction>(Dep, A->getType()));
+        ++TI;
+      }
+      // We no longer need the action stored in this builder.
+      OpenMPDeviceActions.clear();
+    }
+
     void appendLinkDependences(OffloadAction::DeviceDependences &DA) override {
       assert(ToolChains.size() == DeviceLinkerInputs.size() &&
              "Toolchains and linker inputs sizes do not match.");
 
       // Append a new link action for each device.
       auto TC = ToolChains.begin();
       for (auto &LI : DeviceLinkerInputs) {
         auto *DeviceLinkAction =
bool initialize() override {
       auto OpenMPTCRange = C.getOffloadToolChains<Action::OFK_OpenMP>();
       for (auto TI = OpenMPTCRange.first, TE = OpenMPTCRange.second; TI != TE;
            ++TI)
         ToolChains.push_back(TI->second);
 
       DeviceLinkerInputs.resize(ToolChains.size());
       return false;
     }
+
+    bool canUseBundlerUnbundler() const override {
+      // OpenMP should use bundled files whenever possible.
+      return true;
+    }
   };
 
   ///
   /// TODO: Add the implementation for other specialized builders here.
   ///
 
   /// Specialized builders being used by this offloading action builder.
   SmallVector<DeviceActionBuilder *, 4> SpecializedBuilders;
 
+  /// Flag set to true if all valid builders allow file bundling/unbundling.
+  bool CanUseBundler;
+
 public:
   OffloadingActionBuilder(Compilation &C, DerivedArgList &Args,
                           const Driver::InputList &Inputs)
       : C(C), Args(Args) {
     // Create a specialized builder for each device toolchain.
 
     IsValid = true;
 
     // Create a specialized builder for CUDA.
     SpecializedBuilders.push_back(new CudaActionBuilder(C, Args, Inputs));
 
     // Create a specialized builder for OpenMP.
     SpecializedBuilders.push_back(new OpenMPActionBuilder(C, Args, Inputs));
 
     //
     // TODO: Build other specialized builders here.
     //
 
-    // Initialize all the builders, keeping track of errors.
-    for (auto *SB : SpecializedBuilders)
+    // Initialize all the builders, keeping track of errors. If all valid
+    // builders agree that we can use bundling, set the flag to true.
+    unsigned ValidBuilders = 0u;
+    unsigned ValidBuildersSupportingBundling = 0u;
+    for (auto *SB : SpecializedBuilders) {
       IsValid = IsValid && !SB->initialize();
+
+      // Update the counters if the builder is valid.
+      if (SB->isValid()) {
+        ++ValidBuilders;
+        if (SB->canUseBundlerUnbundler())
+          ++ValidBuildersSupportingBundling;
+      }
+    }
+    CanUseBundler =
+        ValidBuilders && ValidBuilders == ValidBuildersSupportingBundling;
   }
 
   ~OffloadingActionBuilder() {
     for (auto *SB : SpecializedBuilders)
       delete SB;
   }
 
   /// Generate an action that adds device dependences (if any) to a host action.
for (auto *SB : SpecializedBuilders) {
       // offload kind because the host will have to use it.
       if (RetCode != DeviceActionBuilder::ABRT_Inactive)
         OffloadKind |= SB->getAssociatedOffloadKind();
     }
 
     return false;
   }
 
-  /// Add the offloading top level actions to the provided action list.
+  /// Add the offloading top level actions to the provided action list. This
+  /// function can replace the host action by a bundling action if the
+  /// programming models allow it.
   bool appendTopLevelActions(ActionList &AL, Action *HostAction,
                              const Arg *InputArg) {
-    auto NumActions = AL.size();
-
+    // Get the device actions to be appended.
+    ActionList OffloadAL;
     for (auto *SB : SpecializedBuilders) {
       if (!SB->isValid())
         continue;
-      SB->appendTopLevelActions(AL);
+      SB->appendTopLevelActions(OffloadAL);
     }
 
-    assert(NumActions <= AL.size() && "Expecting more actions, not less!");
+    // If we can use the bundler, replace the host action by the bundling one in
+    // the resulting list. Otherwise, just append the device actions.
+    if (CanUseBundler && !OffloadAL.empty()) {
+      // Add the host action to the list in order to create the bundling action.
+      OffloadAL.push_back(HostAction);
+
+      // We expect that the host action was just appended to the action list
+      // before this method was called.
+      assert(HostAction == AL.back() && "Host action not in the list??");
+      HostAction = C.MakeAction<OffloadBundlingJobAction>(OffloadAL);
+      AL.back() = HostAction;
+    } else
+      AL.append(OffloadAL.begin(), OffloadAL.end());
 
     // Propagate to the current host action (if any) the offload information
     // associated with the current input.
     if (HostAction)
       HostAction->propagateHostOffloadInfo(InputArgToOffloadKindMap[InputArg],
                                            /*BoundArch=*/nullptr);
 
     // If any action is added by the builders, -o is ambiguous if we have more
-    // than one top-level action.
-    if (NumActions < AL.size() && Args.hasArg(options::OPT_o) &&
+    // than one top-level action. If a bundler is used, there is no ambiguity.
+    if (!CanUseBundler && !OffloadAL.empty() && Args.hasArg(options::OPT_o) &&
         AL.size() > 1) {
       C.getDriver().Diag(
           clang::diag::err_drv_output_argument_with_multiple_files);
       return true;
     }
 
     return false;
   }

lib/Driver/ToolChain.cpp

Tool *ToolChain::getTool(Action::ActionClass AC) const {
   case Action::CompileJobClass:
   case Action::PrecompileJobClass:
   case Action::PreprocessJobClass:
   case Action::AnalyzeJobClass:
   case Action::MigrateJobClass:
   case Action::VerifyPCHJobClass:
   case Action::BackendJobClass:
     return getClang();
+
+  case Action::OffloadBundlingJobClass:
+    // FIXME: Add a tool for the bundling actions.
+    return nullptr;
   }
 
   llvm_unreachable("Invalid tool kind.");
 }
 
 static StringRef getArchNameForCompilerRTLib(const ToolChain &TC,
                                              const ArgList &Args) {
   const llvm::Triple &Triple = TC.getTriple();

test/Driver/openmp-offload.c

 //
 // Generate host object from the BC file and link using the linker script.
 //
 // CHK-COMMANDS: clang{{.*}}" "-cc1" "-triple" "powerpc64le--linux" "-emit-obj" {{.*}}"-fopenmp" {{.*}}"-o" "[[HOSTOBJ:.+\.o]]" "-x" "ir" "[[HOSTBC]]"
 // CHK-COMMANDS: ld" {{.*}}"-o" "[[HOSTBIN:.+\.out]]"  {{.*}}"-lomptarget" {{.*}}"-T" "[[HOSTLK:.+\.lk]]"
 // CHK-COMMANDS-ST: clang{{.*}}" "-cc1" "-triple" "powerpc64le--linux" "-S" {{.*}}"-fopenmp" {{.*}}"-o" "[[HOSTASM:.+\.s]]" "-x" "ir" "[[HOSTBC]]"
 // CHK-COMMANDS-ST: clang{{.*}}" "-cc1as" "-triple" "powerpc64le--linux" "-filetype" "obj" {{.*}}"-o" [[HOSTOBJ:.+\.o]]" [[HOSTASM:.+\.s]]
 // CHK-COMMANDS-ST: ld" {{.*}}"-o" "[[HOSTBIN:.+\.out]]"  {{.*}}"-lomptarget" {{.*}}"-T" "[[HOSTLK:.+\.lk]]"
+
+
+/// ###########################################################################
+
+/// Check separate compilation with offloading - bundling actions
+// RUN:   %clang -### -ccc-print-phases -fopenmp -c -o %t.o -lsomelib -target powerpc64le-linux -fopenmp-targets=powerpc64le-ibm-linux-gnu,x86_64-pc-linux-gnu %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=CHK-BUACTIONS %s
+
+// CHK-BUACTIONS: 0: input, "[[INPUT:.+\.c]]", c, (host-openmp)
+// CHK-BUACTIONS: 1: preprocessor, {0}, cpp-output, (host-openmp)
+// CHK-BUACTIONS: 2: compiler, {1}, ir, (host-openmp)
+// CHK-BUACTIONS: 3: input, "[[INPUT]]", c, (device-openmp)
+// CHK-BUACTIONS: 4: preprocessor, {3}, cpp-output, (device-openmp)
+// CHK-BUACTIONS: 5: compiler, {4}, ir, (device-openmp)
+// CHK-BUACTIONS: 6: offload, "host-openmp (powerpc64le--linux)" {2}, "device-openmp (powerpc64le-ibm-linux-gnu)" {5}, ir
+// CHK-BUACTIONS: 7: backend, {6}, assembler, (device-openmp)
+// CHK-BUACTIONS: 8: assembler, {7}, object, (device-openmp)
+// CHK-BUACTIONS: 9: offload, "device-openmp (powerpc64le-ibm-linux-gnu)" {8}, object
+// CHK-BUACTIONS: 10: input, "[[INPUT]]", c, (device-openmp)
+// CHK-BUACTIONS: 11: preprocessor, {10}, cpp-output, (device-openmp)
+// CHK-BUACTIONS: 12: compiler, {11}, ir, (device-openmp)
+// CHK-BUACTIONS: 13: offload, "host-openmp (powerpc64le--linux)" {2}, "device-openmp (x86_64-pc-linux-gnu)" {12}, ir
+// CHK-BUACTIONS: 14: backend, {13}, assembler, (device-openmp)
+// CHK-BUACTIONS: 15: assembler, {14}, object, (device-openmp)
+// CHK-BUACTIONS: 16: offload, "device-openmp (x86_64-pc-linux-gnu)" {15}, object
+// CHK-BUACTIONS: 17: backend, {2}, assembler, (host-openmp)
+// CHK-BUACTIONS: 18: assembler, {17}, object, (host-openmp)
+// CHK-BUACTIONS: 19: clang-offload-bundler, {9, 16, 18}, object, (host-openmp)