[WIP][RFC] Sample code for containerizing offload images into one ELF.

[vzakhari]

Summary

This code cannot currently work within llorg, because it is mostly based on https://github.com/intel/llvm state of clang-offload-wrapper. This patch is uploaded to facilitate discussion at OpenMP at LLVM weekly meeting.

The code shows how to containerize several offload images passed to clang-offload-wrapper into a single ELF file. It also re-implements the LLVMOMPOFFLOAD notes embedding that currently uses llvm-objcopy in llorg.

The implementation uses yaml::yaml2elf utility.

[vzakhari]

Not working on the project any more.

clang/tools/clang-offload-wrapper/ClangOffloadWrapper.cpp

 #include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Object/ELFObjectFile.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/ObjectYAML/ELFYAML.h"
+#include "llvm/ObjectYAML/yaml2obj.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/EndianStream.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
public:
   const Module &wrapBinaries(ArrayRef<ArrayRef<char>> Binaries) {
     GlobalVariable *Desc = createBinDesc(Binaries);
     assert(Desc && "no binary descriptor");
     createRegisterFunction(Desc);
     createUnregisterFunction(Desc);
     return M;
   }
 
+  Expected<StringRef> getTmpFile(Twine NamePattern) {
+    Expected<sys::fs::TempFile> TempFile =
+        sys::fs::TempFile::create(NamePattern);
+    if (!TempFile)
+      return TempFile.takeError();
+
+    std::string FileName = TempFile->TmpName;
+
+    if (Error E = TempFile->keep(FileName))
+      return std::move(E);
+
+    TempFiles.push_back(std::move(FileName));
+    return TempFiles.back();
+  }
+
+  void containerizeOpenMPImages() {
+    // If there are no OpenMP images, there is nothing to do.
+    auto OpenMPPackIt = Packs.find(OffloadKind::OpenMP);
+    if (OpenMPPackIt == Packs.end())
+      return;
+    SameKindPack *OpenMPPack = OpenMPPackIt->second.get();
+
+    // Start creating notes for the ELF container.
+    std::vector<ELFYAML::NoteEntry> Notes;
+    std::string ImgVersion = toHex(OPENMP_OFFLOAD_IMAGE_VERSION);
+    Notes.emplace_back(ELFYAML::NoteEntry{"LLVMOMPOFFLOAD",
+        yaml::BinaryRef(ImgVersion),
+        ELF::NT_LLVM_OPENMP_OFFLOAD_VERSION});
+    std::string Producer = toHex("LLVM");
+    Notes.emplace_back(ELFYAML::NoteEntry{"LLVMOMPOFFLOAD",
+        yaml::BinaryRef(Producer),
+        ELF::NT_LLVM_OPENMP_OFFLOAD_PRODUCER});
+    std::string ProdVersion = toHex(LLVM_VERSION_STRING
+#ifdef LLVM_REVISION
+                                    " " LLVM_REVISION
+#endif
+                                    );
+    Notes.emplace_back(ELFYAML::NoteEntry{"LLVMOMPOFFLOAD",
+        yaml::BinaryRef(ProdVersion),
+        ELF::NT_LLVM_OPENMP_OFFLOAD_PRODUCER_VERSION});
+
+    SmallVector<StringRef, 2> ImageFiles;
+
+    // Find SPIR-V images and create notes with auxiliary information
+    // for each of them.
+    unsigned ImageIdx = 0;
+    for (auto I = OpenMPPack->begin(); I != OpenMPPack->end(); ++I) {
+      const BinaryWrapper::Image &Img = *(I->get());
+      if (Img.Tgt.find("spir") != 0)
+        continue;
+
+      ImageFiles.push_back(Img.File);
+      ++ImageIdx;
+    }
+
+    // If there are no SPIR-V images, there is nothing to do.
+    if (ImageIdx == 0)
+      return;
+
+    StringRef ToolNameRef(ToolName);
+
+    // Helper to emit warnings.
+    auto warningOS = [ToolNameRef]() -> raw_ostream & {
+      return WithColor::warning(errs(), ToolNameRef);
+    };
+    auto handleErrorAsWarning = [&warningOS](Error E) {
+      logAllUnhandledErrors(std::move(E), warningOS());
+    };
+
+    // Reserve  a temporary file for the ELF image.
+    Expected<StringRef> ImageFileName =
+        getTmpFile(Output + Twine(".spirv.elfimage.%%%%%%%.tmp"));
+    if (!ImageFileName) {
+      logAllUnhandledErrors(ImageFileName.takeError(), warningOS());
+      return;
+    }
+
+    std::error_code EC;
+    raw_fd_ostream ELFOS(ImageFileName->str(), EC);
+    if (EC) {
+      warningOS() << "cannot open ELF file " << ImageFileName->str() << ": "
+                  << EC.message() << "\n";
+      return;
+    }
+
+    // Write ELF file using yaml2elf instead of writing ELF by ourselves.
+    // We use 64-bit little-endian ELF currently.
+    ELFYAML::FileHeader Header{};
+    Header.Class = ELF::ELFCLASS64;
+    Header.Data = ELF::ELFDATA2LSB;
+    Header.Type = ELF::ET_NONE;
+    // Do not use any existing machine, otherwise, other plugins
+    // may claim this image.
+    Header.Machine = ELF::EM_NONE;
+
+    // Create a section with notes.
+    ELFYAML::NoteSection Section{};
+    Section.Type = ELF::SHT_NOTE;
+    Section.Name = ".note.openmp";
+    Section.Notes.emplace(std::move(Notes));
+
+    // The main ELFYAML structure describing our ELF container.
+    ELFYAML::Object Object{};
+    Object.Header = Header;
+    Object.Chunks.push_back(
+        std::make_unique<ELFYAML::NoteSection>(std::move(Section)));
+
+    SmallVector<std::unique_ptr<MemoryBuffer>, 4> Buffers;
+    SmallVector<std::string, 4> SectionNames;
+
+    ImageIdx = 0;
+    // Put each image into its own __openmp_offload_spirv_# section.
+    for (auto Name : ImageFiles) {
+      // Read the image into a memory buffer.
+      ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
+          MemoryBuffer::getFile(Name);
+      if (!BufOrErr) {
+        handleErrorAsWarning(
+            createFileError(Name, BufOrErr.getError()));
+        return;
+      }
+      Buffers.push_back(std::move(*BufOrErr));
+
+      // Create a new section for the image.
+      ELFYAML::RawContentSection Section{};
+      Section.Info = yaml::Hex64(0);
+      Section.Type = ELF::SHT_PROGBITS;
+      SectionNames.emplace_back(
+          ("__openmp_offload_spirv_" + Twine(ImageIdx)).str());
+      Section.Name = SectionNames.back();
+      Section.Content = yaml::BinaryRef(makeArrayRef(
+          reinterpret_cast<const uint8_t *>(Buffers.back()->getBufferStart()),
+          Buffers.back()->getBufferSize()));
+
+      // Put the new section into the ELFYAML::Object.
+      Object.Chunks.push_back(
+          std::make_unique<ELFYAML::RawContentSection>(std::move(Section)));
+      ++ImageIdx;
+    }
+
+    yaml::ErrorHandler ErrHandler =
+        [&warningOS](const Twine &Msg) {
+          warningOS() << "cannot convert YAML to ELF: " << Msg << "\n";
+        };
+
+    if (!yaml::yaml2elf(Object, ELFOS, ErrHandler, UINT64_MAX)) {
+      warningOS() << "YAML to ELF conversion failed.\n";
+      return;
+    }
+    ELFOS.close();
+
+    if (ELFOS.has_error()) {
+      warningOS() << "cannot write ELF file " << ImageFileName->str() << ": "
+                  << ELFOS.error().message() << "\n";
+      ELFOS.clear_error();
+      return;
+    }
+
+    // Delete the original Images from the list and replace them
+    // with a single combined container ELF.
+    for (auto I = OpenMPPack->begin(); I != OpenMPPack->end();) {
+      const BinaryWrapper::Image &Img = *(I->get());
+      if (Img.Tgt.find("spir") != 0)
+        ++I;
+      else
+        I = OpenMPPack->erase(I);
+    }
+
+    OpenMPPack->emplace_back(std::make_unique<Image>(
+        *ImageFileName, "",
+        "spirv",
+        BinaryImageFormat::none, "", "",
+        "", ""));
+  }
+
   std::unique_ptr<MemoryBuffer> addELFNotes(std::unique_ptr<MemoryBuffer> Buf,
                                             StringRef OriginalFileName) {
     // Cannot add notes, if llvm-objcopy is not available.
     //
     // I did not find a clean way to add a new notes section into an existing
     // ELF file. llvm-objcopy seems to recreate a new ELF from scratch,
     // and we just try to use llvm-objcopy here.
     if (ObjcopyPath.empty())