[Clang][Bundler] Add 'exclude' flag to target objects sections

[sdmitriev]

Summary

This flag tells link editor to exclude section from linker inputs when linking executable or shared library.

[ABataev]

clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp

467–475

1. Extra spaces in the comment.
2. TODO better to have on a separate line.
3. Maybe it is better to postpone it until llvm-objcopy can do this in one run?

508

Better to make it a function

527

from

[sdmitriev]

clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp

467–475

1. Will fix.
2. Ok.
3. I do not think that anybody is working on that problem now, so it is not clear how much time that will take. Also merging two llvm-objcopy invocations into one will be a simple NFC change, so I think this should not block this patch.

527

This was supposed to be 'for'.

[sdmitriev]

Addressed review comments.

[sdmitriev]

Any additional comments?

[ABataev]

LG

clang/test/Driver/clang-offload-bundler.c

 // section (we are using a PowerPC little-endian host which uses ELF). We
 // have an already bundled file to check the unbundle and do a dry run on the
 // bundling as it cannot be tested in all host platforms that will run these
 // tests.
 //
 
 // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o -### 2>&1 \
 // RUN: | FileCheck %s -DHOST=%itanium_abi_triple -DINOBJ1=%t.o -DINOBJ2=%t.tgt1 -DINOBJ3=%t.tgt2 -DOUTOBJ=%t.bundle3.o --check-prefix CK-OBJ-CMD
-// CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--add-section=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=[[INOBJ1]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=[[INOBJ2]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=[[INOBJ3]]" "[[INOBJ1]]" "[[OUTOBJ]]"
+// CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--add-section=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=[[INOBJ1]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=[[INOBJ2]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=[[INOBJ3]]" "[[INOBJ1]]" "[[TEMPOBJ:.*]]"
+// CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=readonly,exclude" "[[TEMPOBJ]]" "[[OUTOBJ]]"
 
 // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o
 // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle
 // RUN: diff %t.o %t.res.o
 // RUN: diff %t.tgt1 %t.res.tgt1
 // RUN: diff %t.tgt2 %t.res.tgt2
 // RUN: clang-offload-bundler -type=o -targets=openmp-powerpc64le-ibm-linux-gnu,host-%itanium_abi_triple,openmp-x86_64-pc-linux-gnu -outputs=%t.res.tgt1,%t.res.o,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle
 // RUN: diff %t.o %t.res.o

clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp

 #include "llvm/Object/Binary.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/StringSaver.h"
 #include "llvm/Support/WithColor.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
     assert(NumberOfProcessedInputs <= NumberOfInputs &&
            "Processing more inputs that actually exist!");
     assert(HostInputIndex != ~0u && "Host input index not defined.");
 
     // If this is not the last output, we don't have to do anything.
     if (NumberOfProcessedInputs != NumberOfInputs)
       return Error::success();
 
+    // We will use llvm-objcopy to add target objects sections to the output
+    // fat object. These sections should have 'exclude' flag set which tells
+    // link editor to remove them from linker inputs when linking executable or
+    // shared library. llvm-objcopy currently does not support adding new
+    // section and changing flags for the added section in one invocation, and
+    // because of that we have to run it two times. First run adds sections and
+    // the second changes flags.
+    // TODO: change it to one run once llvm-objcopy starts supporting that.
+
     // Find llvm-objcopy in order to create the bundle binary.
     ErrorOr<std::string> Objcopy = sys::findProgramByName(
         "llvm-objcopy", sys::path::parent_path(BundlerExecutable));
     if (!Objcopy)
       Objcopy = sys::findProgramByName("llvm-objcopy");
     if (!Objcopy)
       return createStringError(Objcopy.getError(),
                                "unable to find 'llvm-objcopy' in path");
 
     // We write to the output file directly. So, we close it and use the name
     // to pass down to llvm-objcopy.
     OS.close();
 
-    // Compose command line for the objcopy tool.
+    // Create an intermediate temporary file to save object after the first
+    // llvm-objcopy run.
+    SmallString<128u> IntermediateObj;
+    if (std::error_code EC = sys::fs::createTemporaryFile(
+            "clang-offload-bundler", "tmp", IntermediateObj))
+      return createFileError(IntermediateObj, EC);
+    FileRemover IntermediateObjRemover(IntermediateObj);
+
+    // Compose llvm-objcopy command line for add target objects' sections.
     BumpPtrAllocator Alloc;
     StringSaver SS{Alloc};
     SmallVector<StringRef, 8u> ObjcopyArgs{"llvm-objcopy"};
     for (unsigned I = 0; I < NumberOfInputs; ++I)
       ObjcopyArgs.push_back(SS.save(Twine("--add-section=") +
                                     OFFLOAD_BUNDLER_MAGIC_STR + TargetNames[I] +
                                     "=" + InputFileNames[I]));
     ObjcopyArgs.push_back(InputFileNames[HostInputIndex]);
+    ObjcopyArgs.push_back(IntermediateObj);
+
+    if (Error Err = executeObjcopy(*Objcopy, ObjcopyArgs))
+      return Err;
+
+    // And run llvm-objcopy for the second time to update section flags.
+    ObjcopyArgs.resize(1);
+    for (unsigned I = 0; I < NumberOfInputs; ++I)
+      ObjcopyArgs.push_back(SS.save(Twine("--set-section-flags=") +
+                                    OFFLOAD_BUNDLER_MAGIC_STR + TargetNames[I] +
+                                    "=readonly,exclude"));
+    ObjcopyArgs.push_back(IntermediateObj);
     ObjcopyArgs.push_back(OutputFileNames.front());
 
-    // If the user asked for the commands to be printed out, we do that instead
-    // of executing it.
+    if (Error Err = executeObjcopy(*Objcopy, ObjcopyArgs))
+      return Err;
+
+    return Error::success();
+  }
+
+  Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
+    return Error::success();
+  }
+
+private:
+  static Error executeObjcopy(StringRef Objcopy, ArrayRef<StringRef> Args) {
+    // If the user asked for the commands to be printed out, we do that
+    // instead of executing it.
     if (PrintExternalCommands) {
-      errs() << "\"" << *Objcopy << "\"";
-      for (StringRef Arg : drop_begin(ObjcopyArgs, 1))
+      errs() << "\"" << Objcopy << "\"";
+      for (StringRef Arg : drop_begin(Args, 1))
         errs() << " \"" << Arg << "\"";
       errs() << "\n";
     } else {
-      if (sys::ExecuteAndWait(*Objcopy, ObjcopyArgs))
+      if (sys::ExecuteAndWait(Objcopy, Args))
         return createStringError(inconvertibleErrorCode(),
                                  "'llvm-objcopy' tool failed");
     }
-
-    return Error::success();
-  }
-
-  Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
     return Error::success();
   }
 };
 
 /// Handler for text files. The bundled file will have the following format.
 ///
 /// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
 /// Bundle 1