Index: test/Driver/clang-offload-bundler.c =================================================================== --- /dev/null +++ test/Driver/clang-offload-bundler.c @@ -0,0 +1,188 @@ +// +// Generate all the types of files we can bundle. +// +// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -E -o %t.i +// RUN: %clangxx -O0 -target powerpc64le-ibm-linux-gnu -x c++ %s -E -o %t.ii +// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -S -emit-llvm -o %t.ll +// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -c -emit-llvm -o %t.bc +// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -S -o %t.s +// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -emit-ast -o %t.ast + +// +// Generate an empty file to help with the checks of empty files. +// +// RUN: touch %t.empty + +// +// Generate a couple of files to bundle with. +// +// RUN: echo 'Content of device file 1' > %t.tgt1 +// RUN: echo 'Content of device file 2' > %t.tgt2 + +// +// Check help message. +// +// RUN: clang-offload-bundler --help | FileCheck %s --check-prefix CK-HELP +// CK-HELP: {{.*}}OVERVIEW: A tool to bundle several input files of the specified type +// CK-HELP: {{.*}}referring to the same source file but different targets into a single +// CK-HELP: {{.*}}one. The resulting file can also be unbundled into different files by +// CK-HELP: {{.*}}this tool if -unbundle is provided. +// CK-HELP: {{.*}}USAGE: clang-offload-bundler [options] +// CK-HELP: {{.*}}-inputs= - [,...] +// CK-HELP: {{.*}}-outputs= - [,...] +// CK-HELP: {{.*}}-targets= - [-,...] +// CK-HELP: {{.*}}-type= - Type of the files to be bundled/unbundled. +// CK-HELP: {{.*}}Current supported types are: +// CK-HELP: {{.*}}i {{.*}}- cpp-output +// CK-HELP: {{.*}}ii {{.*}}- c++-cpp-output +// CK-HELP: {{.*}}ll {{.*}}- llvm +// CK-HELP: {{.*}}bc {{.*}}- llvm-bc +// CK-HELP: {{.*}}s {{.*}}- assembler +// CK-HELP: {{.*}}o {{.*}}- object +// CK-HELP: {{.*}}gch {{.*}}- precompiled-header +// CK-HELP: {{.*}}ast {{.*}}- clang AST file +// CK-HELP: {{.*}}-unbundle {{.*}}- Unbundle bundled file into several output files. + +// +// Check errors. +// +// RUN: not clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i -unbundle 2>&1 | FileCheck %s --check-prefix CK-ERR1 +// CK-ERR1: error: only one input file supported in unbundling mode. +// CK-ERR1: error: number of output files and targets should match in unbundling mode. + +// RUN: not clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR2 +// RUN: not clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR2 +// CK-ERR2: error: number of input files and targets should match in bundling mode. + +// RUN: not clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.i,%t.tgt1,%t.tgt2 -inputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR3 +// CK-ERR3: error: only one output file supported in bundling mode. +// CK-ERR3: error: number of input files and targets should match in bundling mode. + +// RUN: not clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu -outputs=%t.i,%t.tgt1,%t.tgt2 -inputs=%t.bundle.i -unbundle 2>&1 | FileCheck %s --check-prefix CK-ERR4 +// RUN: not clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.i,%t.tgt1 -inputs=%t.bundle.i -unbundle 2>&1 | FileCheck %s --check-prefix CK-ERR4 +// CK-ERR4: error: number of output files and targets should match in unbundling mode. + +// RUN: not clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2.notexist -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR5 +// RUN: not clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.i,%t.tgt1,%t.tgt2 -inputs=%t.bundle.i.notexist -unbundle 2>&1 | FileCheck %s --check-prefix CK-ERR5 +// CK-ERR5: error: Can't open file {{.+}}.notexist: No such file or directory + +// RUN: not clang-offload-bundler -type=invalid -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR6 +// CK-ERR6: error: invalid file type specified. + +// RUN: not clang-offload-bundler 2>&1 | FileCheck %s --check-prefix CK-ERR7 +// CK-ERR7-DAG: clang-offload-bundler: for the -type option: must be specified at least once! +// CK-ERR7-DAG: clang-offload-bundler: for the -inputs option: must be specified at least once! +// CK-ERR7-DAG: clang-offload-bundler: for the -outputs option: must be specified at least once! +// CK-ERR7-DAG: clang-offload-bundler: for the -targets option: must be specified at least once! + +// RUN: not clang-offload-bundler -type=i -targets=hxst-powerpcxxle-ibm-linux-gnu,openxp-pxxerpc64le-ibm-linux-gnu,xpenmp-x86_xx-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR8 +// CK-ERR8: error: invalid target 'hxst-powerpcxxle-ibm-linux-gnu', unknown offloading kind 'hxst', unknown target triple 'powerpcxxle-ibm-linux-gnu'. +// CK-ERR8: error: invalid target 'openxp-pxxerpc64le-ibm-linux-gnu', unknown offloading kind 'openxp', unknown target triple 'pxxerpc64le-ibm-linux-gnu'. +// CK-ERR8: error: invalid target 'xpenmp-x86_xx-pc-linux-gnu', unknown offloading kind 'xpenmp', unknown target triple 'x86_xx-pc-linux-gnu'. + +// RUN: not clang-offload-bundler -type=i -targets=openmp-powerpc64le-linux,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR9A +// RUN: not clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,host-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR9B +// CK-ERR9A: error: expecting exactly one host target but got 0. +// CK-ERR9B: error: expecting exactly one host target but got 2. + +// +// Check text bundle. This is a readable format, so we check for the format we expect to find. +// +// RUN: clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.i +// RUN: clang-offload-bundler -type=ii -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.ii,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.ii +// RUN: clang-offload-bundler -type=ll -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.ll,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.ll +// RUN: clang-offload-bundler -type=s -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.s,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.s +// RUN: FileCheck %s --input-file %t.bundle3.i --check-prefix CK-TEXTI +// RUN: FileCheck %s --input-file %t.bundle3.ii --check-prefix CK-TEXTI +// RUN: FileCheck %s --input-file %t.bundle3.ll --check-prefix CK-TEXTLL +// RUN: FileCheck %s --input-file %t.bundle3.s --check-prefix CK-TEXTS + +// CK-TEXTI: // __CLANG_OFFLOAD_BUNDLE____START__ host-powerpc64le-ibm-linux-gnu +// CK-TEXTI: int A = 0; +// CK-TEXTI: test_func(void) +// CK-TEXTI: // __CLANG_OFFLOAD_BUNDLE____END__ host-powerpc64le-ibm-linux-gnu +// CK-TEXTI: // __CLANG_OFFLOAD_BUNDLE____START__ openmp-powerpc64le-ibm-linux-gnu +// CK-TEXTI: Content of device file 1 +// CK-TEXTI: // __CLANG_OFFLOAD_BUNDLE____END__ openmp-powerpc64le-ibm-linux-gnu +// CK-TEXTI: // __CLANG_OFFLOAD_BUNDLE____START__ openmp-x86_64-pc-linux-gnu +// CK-TEXTI: Content of device file 2 +// CK-TEXTI: // __CLANG_OFFLOAD_BUNDLE____END__ openmp-x86_64-pc-linux-gnu + +// CK-TEXTLL: ; __CLANG_OFFLOAD_BUNDLE____START__ host-powerpc64le-ibm-linux-gnu +// CK-TEXTLL: @A = global i32 0 +// CK-TEXTLL: define {{.*}}@test_func() +// CK-TEXTLL: ; __CLANG_OFFLOAD_BUNDLE____END__ host-powerpc64le-ibm-linux-gnu +// CK-TEXTLL: ; __CLANG_OFFLOAD_BUNDLE____START__ openmp-powerpc64le-ibm-linux-gnu +// CK-TEXTLL: Content of device file 1 +// CK-TEXTLL: ; __CLANG_OFFLOAD_BUNDLE____END__ openmp-powerpc64le-ibm-linux-gnu +// CK-TEXTLL: ; __CLANG_OFFLOAD_BUNDLE____START__ openmp-x86_64-pc-linux-gnu +// CK-TEXTLL: Content of device file 2 +// CK-TEXTLL: ; __CLANG_OFFLOAD_BUNDLE____END__ openmp-x86_64-pc-linux-gnu + +// CK-TEXTS: # __CLANG_OFFLOAD_BUNDLE____START__ host-powerpc64le-ibm-linux-gnu +// CK-TEXTS: .globl {{.*}}test_func +// CK-TEXTS: .globl {{.*}}A +// CK-TEXTS: # __CLANG_OFFLOAD_BUNDLE____END__ host-powerpc64le-ibm-linux-gnu +// CK-TEXTS: # __CLANG_OFFLOAD_BUNDLE____START__ openmp-powerpc64le-ibm-linux-gnu +// CK-TEXTS: Content of device file 1 +// CK-TEXTS: # __CLANG_OFFLOAD_BUNDLE____END__ openmp-powerpc64le-ibm-linux-gnu +// CK-TEXTS: # __CLANG_OFFLOAD_BUNDLE____START__ openmp-x86_64-pc-linux-gnu +// CK-TEXTS: Content of device file 2 +// CK-TEXTS: # __CLANG_OFFLOAD_BUNDLE____END__ openmp-x86_64-pc-linux-gnu + +// +// Check text unbundle. Check if we get the exact same content that we bundled before for each file. +// +// RUN: clang-offload-bundler -type=i -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.i,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.i -unbundle +// RUN: diff %t.i %t.res.i +// RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: diff %t.tgt2 %t.res.tgt2 +// RUN: clang-offload-bundler -type=ii -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.ii,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.ii -unbundle +// RUN: diff %t.ii %t.res.ii +// RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: diff %t.tgt2 %t.res.tgt2 +// RUN: clang-offload-bundler -type=ll -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.ll,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.ll -unbundle +// RUN: diff %t.ll %t.res.ll +// RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: diff %t.tgt2 %t.res.tgt2 +// RUN: clang-offload-bundler -type=s -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.s,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.s -unbundle +// RUN: diff %t.s %t.res.s +// RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: diff %t.tgt2 %t.res.tgt2 + +// Check if we can unbundle a file with no magic strings. +// RUN: clang-offload-bundler -type=s -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.s,%t.res.tgt1,%t.res.tgt2 -inputs=%t.s -unbundle +// RUN: diff %t.s %t.res.s +// RUN: diff %t.empty %t.res.tgt1 +// RUN: diff %t.empty %t.res.tgt2 + +// +// Check binary bundle/unbundle. The content that we have before bundling must be the same we have after unbundling. +// +// RUN: clang-offload-bundler -type=bc -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.bc,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.bc +// RUN: clang-offload-bundler -type=gch -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.ast,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.gch +// RUN: clang-offload-bundler -type=ast -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.ast,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.ast +// RUN: clang-offload-bundler -type=bc -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.bc,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.bc -unbundle +// RUN: diff %t.bc %t.res.bc +// RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: diff %t.tgt2 %t.res.tgt2 +// RUN: clang-offload-bundler -type=gch -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.gch,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.gch -unbundle +// RUN: diff %t.ast %t.res.gch +// RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: diff %t.tgt2 %t.res.tgt2 +// RUN: clang-offload-bundler -type=ast -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.ast,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.ast -unbundle +// RUN: diff %t.ast %t.res.ast +// RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: diff %t.tgt2 %t.res.tgt2 + +// Check if we can unbundle a file with no magic strings. +// RUN: clang-offload-bundler -type=bc -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.bc,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bc -unbundle +// RUN: diff %t.bc %t.res.bc +// RUN: diff %t.empty %t.res.tgt1 +// RUN: diff %t.empty %t.res.tgt2 + +// Some code so that we can create a binary out of this file. +int A = 0; +void test_func(void) { + ++A; +} Index: tools/CMakeLists.txt =================================================================== --- tools/CMakeLists.txt +++ tools/CMakeLists.txt @@ -5,6 +5,7 @@ add_clang_subdirectory(clang-format) add_clang_subdirectory(clang-format-vs) add_clang_subdirectory(clang-fuzzer) +add_clang_subdirectory(clang-offload-bundler) add_clang_subdirectory(c-index-test) Index: tools/clang-offload-bundler/CMakeLists.txt =================================================================== --- /dev/null +++ tools/clang-offload-bundler/CMakeLists.txt @@ -0,0 +1,17 @@ +set(LLVM_LINK_COMPONENTS support) + +add_clang_executable(clang-offload-bundler + ClangOffloadBundler.cpp + ) + +set(CLANG_OFFLOAD_BUNDLER_LIB_DEPS + clangBasic + LLVMBitWriter + LLVMObject + ) + +target_link_libraries(clang-offload-bundler + ${CLANG_OFFLOAD_BUNDLER_LIB_DEPS} + ) + +install(TARGETS clang-offload-bundler RUNTIME DESTINATION bin) Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp =================================================================== --- /dev/null +++ tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,683 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp - Clang format tool -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// \brief This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/FileManager.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/Bitcode/ReaderWriter.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" + +using namespace llvm; +using namespace llvm::object; + +static cl::opt Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory + ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list + InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list + OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list + TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[-,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt + FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " gch - precompiled-header\n" + " ast - clang AST file"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt + Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +/// \brief Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// \brief Obtain the offload kind and real machine triple out of the target +/// information specified by the user. +static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, + StringRef &Triple) { + auto KindTriplePair = Target.split('-'); + OffloadKind = KindTriplePair.first; + Triple = KindTriplePair.second; +} +static bool hasHostKind(StringRef Target) { + StringRef OffloadKind; + StringRef Triple; + getOffloadKindAndTriple(Target, OffloadKind, Triple); + return OffloadKind == "host"; +} + +/// \brief Generic file handler interface. +class FileHandler { +public: + /// \brief Update the file handler with information from the header of the + /// bundled file + virtual void ReadHeader(MemoryBuffer &Input) = 0; + /// \brief Read the marker of the next bundled to be read in the file. The + /// triple of the target associated with that bundled is returned. An empty + /// string is returned if there are no more bundles to be read. + virtual StringRef ReadBundleStart(MemoryBuffer &Input) = 0; + /// \brief Read the marker that closes the current bundle. + virtual void ReadBundleEnd(MemoryBuffer &Input) = 0; + /// \brief Read the current bundle and write the result into the stream \a OS. + virtual void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0; + + /// \brief Write the header of the bundled file to \a OS based on the + /// information gathered from \a Inputs. + virtual void WriteHeader(raw_fd_ostream &OS, + ArrayRef> Inputs) = 0; + /// \brief Write the marker that initiates a bundle for the triple \a + /// TargetTriple to \a OS. + virtual void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) = 0; + /// \brief Write the marker that closes a bundle for the triple \a + /// TargetTriple to \a OS. + virtual void WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) = 0; + /// \brief Write the bundle from \a Input into \a OS. + virtual void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0; + + FileHandler() {} + virtual ~FileHandler() {} +}; + +// Handler for binary files. The bundled file will have the following format +// (all integers are stored in little-endian format): +// +// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string) +// +// NumberOfOffloadBundles (8-byte integer) +// +// OffsetOfBundle1 (8-byte integer) +// SizeOfBundle1 (8-byte integer) +// NumberOfBytesInTripleOfBundle1 (8-byte integer) +// TripleOfBundle1 (byte length defined before) +// +// ... +// +// OffsetOfBundleN (8-byte integer) +// SizeOfBundleN (8-byte integer) +// NumberOfBytesInTripleOfBundleN (8-byte integer) +// TripleOfBundleN (byte length defined before) +// +// Bundle1 +// ... +// BundleN + +/// \brief Read 8-byte integers to/from a buffer in little-endian format. +static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) { + uint64_t Res = 0; + const char *Data = Buffer.data(); + + for (unsigned i = 0; i < 8; ++i) { + Res <<= 8; + uint64_t Char = (uint64_t)Data[pos + 7 - i]; + Res |= 0xffu & Char; + } + return Res; +} + +/// \brief Write and write 8-byte integers to/from a buffer in little-endian +/// format. +static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) { + + for (unsigned i = 0; i < 8; ++i) { + char Char = (char)(Val & 0xffu); + OS.write(&Char, 1); + Val >>= 8; + } +} + +class BinaryFileHandler final : public FileHandler { + /// \brief Information about the bundles extracted from the header. + struct BundleInfo { + /// \brief Size of the bundle. + uint64_t Size; + /// \brief Offset at which the bundle starts in the bundled file. + uint64_t Offset; + BundleInfo() : Size(0), Offset(0) {} + BundleInfo(uint64_t Size, uint64_t Offset) : Size(Size), Offset(Offset) {} + }; + /// Map between a triple and the corresponding bundle information. + StringMap BundlesInfo; + + /// Iterator for the bundle information that is being read. + StringMap::iterator CurBundleInfo; + +public: + void ReadHeader(MemoryBuffer &Input) { + StringRef FC = Input.getBuffer(); + + // Initialize the current bundle with the end of the container. + CurBundleInfo = BundlesInfo.end(); + + // Check if buffer is smaller than magic string. + size_t ReadChars = sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1; + if (ReadChars > FC.size()) + return; + + // Check if no magic was found. + StringRef Magic(FC.data(), sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1); + if (!Magic.equals(OFFLOAD_BUNDLER_MAGIC_STR)) + return; + + // Read number of bundles. + if (ReadChars + 8 > FC.size()) + return; + + uint64_t NumberOfBundles = Read8byteIntegerFromBuffer(FC, ReadChars); + ReadChars += 8; + + // Read bundle offsets, sizes and triples. + for (uint64_t i = 0; i < NumberOfBundles; ++i) { + + // Read offset. + if (ReadChars + 8 > FC.size()) + return; + + uint64_t Offset = Read8byteIntegerFromBuffer(FC, ReadChars); + ReadChars += 8; + + // Read size. + if (ReadChars + 8 > FC.size()) + return; + + uint64_t Size = Read8byteIntegerFromBuffer(FC, ReadChars); + ReadChars += 8; + + // Read triple size. + if (ReadChars + 8 > FC.size()) + return; + + uint64_t TripleSize = Read8byteIntegerFromBuffer(FC, ReadChars); + ReadChars += 8; + + // Read triple. + if (ReadChars + TripleSize > FC.size()) + return; + + StringRef Triple(&FC.data()[ReadChars], TripleSize); + ReadChars += TripleSize; + + // Check if the offset and size make sense. + if (!Size || !Offset || Offset + Size > FC.size()) + return; + + assert(BundlesInfo.find(Triple) == BundlesInfo.end() && + "Triple is duplicated??"); + BundlesInfo[Triple] = BundleInfo(Size, Offset); + } + // Set the iterator to where we will start to read. + CurBundleInfo = BundlesInfo.begin(); + } + StringRef ReadBundleStart(MemoryBuffer &Input) { + if (CurBundleInfo == BundlesInfo.end()) + return StringRef(); + + return CurBundleInfo->first(); + } + void ReadBundleEnd(MemoryBuffer &Input) { + assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!"); + ++CurBundleInfo; + return; + } + void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) { + assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!"); + StringRef FC = Input.getBuffer(); + OS.write(FC.data() + CurBundleInfo->second.Offset, + CurBundleInfo->second.Size); + } + + void WriteHeader(raw_fd_ostream &OS, + ArrayRef> Inputs) { + // Compute size of the header. + uint64_t HeaderSize = 0; + + HeaderSize += sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1; + HeaderSize += 8; // Number of Bundles + + for (auto &T : TargetNames) { + HeaderSize += 3 * 8; // Bundle offset, Size of bundle and size of triple. + HeaderSize += T.size(); // The triple. + } + + // Write to the buffer the header. + OS << OFFLOAD_BUNDLER_MAGIC_STR; + + Write8byteIntegerToBuffer(OS, TargetNames.size()); + + unsigned Idx = 0; + for (auto &T : TargetNames) { + MemoryBuffer &MB = *Inputs[Idx++].get(); + // Bundle offset. + Write8byteIntegerToBuffer(OS, HeaderSize); + // Size of the bundle (adds to the next bundle's offset) + Write8byteIntegerToBuffer(OS, MB.getBufferSize()); + HeaderSize += MB.getBufferSize(); + // Size of the triple + Write8byteIntegerToBuffer(OS, T.size()); + // Triple + OS << T; + } + } + void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) { return; } + void WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) { return; } + void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) { + OS.write(Input.getBufferStart(), Input.getBufferSize()); + return; + } + + BinaryFileHandler() : FileHandler() {} + ~BinaryFileHandler() {} +}; + +// Handler for text files. The bundled file will have the following format. +// +// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple" +// Bundle 1 +// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple" +// ... +// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple" +// Bundle N +// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple" +class TextFileHandler final : public FileHandler { + /// \brief String that begins a line comment. + StringRef Comment; + + /// \brief String that initiates a bundle. + std::string BundleStartString; + + /// \brief String that closes a bundle. + std::string BundleEndString; + + /// \brief Number of chars read from input. + size_t ReadChars; + +protected: + void ReadHeader(MemoryBuffer &Input) {} + StringRef ReadBundleStart(MemoryBuffer &Input) { + StringRef FC = Input.getBuffer(); + + // Find start of the bundle. + ReadChars = FC.find(BundleStartString, ReadChars); + if (ReadChars == FC.npos) + return StringRef(); + + // Get position of the triple. + size_t TripleStart = ReadChars = ReadChars + BundleStartString.size(); + + // Get position that closes the triple. + size_t TripleEnd = ReadChars = FC.find("\n", ReadChars); + if (TripleEnd == FC.npos) + return StringRef(); + + // Next time we read after the new line. + ++ReadChars; + + return StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart); + } + void ReadBundleEnd(MemoryBuffer &Input) { + StringRef FC = Input.getBuffer(); + + // Read up to the next new line. + assert(FC[ReadChars] == '\n' && "The bundle should end with a new line."); + + size_t TripleEnd = ReadChars = FC.find("\n", ReadChars + 1); + if (TripleEnd == FC.npos) + return; + + // Next time we read after the new line. + ++ReadChars; + + return; + } + void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) { + StringRef FC = Input.getBuffer(); + size_t BundleStart = ReadChars; + + // Find end of the bundle. + size_t BundleEnd = ReadChars = FC.find(BundleEndString, ReadChars); + + StringRef Bundle(&FC.data()[BundleStart], BundleEnd - BundleStart); + OS << Bundle; + } + + void WriteHeader(raw_fd_ostream &OS, + ArrayRef> Inputs) {} + void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) { + OS << BundleStartString << TargetTriple << "\n"; + return; + } + void WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) { + OS << BundleEndString << TargetTriple << "\n"; + return; + } + void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) { + ; + OS << Input.getBuffer(); + return; + } + +public: + TextFileHandler(StringRef Comment) + : FileHandler(), Comment(Comment), ReadChars(0) { + BundleStartString = + "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__START__ "; + BundleEndString = + "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ "; + } +}; + +/// \brief Return an appropriate handler given the input files and options. +static FileHandler *CreateFileHandler(MemoryBuffer &FirstInput) { + if (FilesType == "i") + return new TextFileHandler(/*Comment=*/"//"); + if (FilesType == "ii") + return new TextFileHandler(/*Comment=*/"//"); + if (FilesType == "ll") + return new TextFileHandler(/*Comment=*/";"); + if (FilesType == "bc") + return new BinaryFileHandler(); + if (FilesType == "s") + return new TextFileHandler(/*Comment=*/"#"); + if (FilesType == "o") + return new BinaryFileHandler(); + if (FilesType == "gch") + return new BinaryFileHandler(); + if (FilesType == "ast") + return new BinaryFileHandler(); + + llvm::errs() << "error: invalid file type specified.\n"; + return nullptr; +} + +/// \brief Bundle the files. Return true if an error was found. +static bool BundleFiles() { + std::error_code EC; + + // Create output file. + raw_fd_ostream OutputFile(OutputFileNames.front(), EC, sys::fs::F_None); + + if (EC) { + llvm::errs() << "error: Can't open file " << OutputFileNames.front() + << ".\n"; + return true; + } + + // Open input files. + std::vector> InputBuffers( + InputFileNames.size()); + + unsigned Idx = 0; + for (auto I : InputFileNames) { + ErrorOr> CodeOrErr = + MemoryBuffer::getFileOrSTDIN(I); + if (std::error_code EC = CodeOrErr.getError()) { + llvm::errs() << "error: Can't open file " << I << ": " << EC.message() + << "\n"; + return true; + } + InputBuffers[Idx++] = std::move(CodeOrErr.get()); + } + + // Get the file handler. + std::unique_ptr FH; + FH.reset(CreateFileHandler(*InputBuffers.front().get())); + + // Quit if we don't have a handler. + if (!FH.get()) + return true; + + // Write header. + FH.get()->WriteHeader(OutputFile, InputBuffers); + + // Write all bundles along with the start/end markers. + auto Input = InputBuffers.begin(); + for (auto Triple = TargetNames.begin(); Triple < TargetNames.end(); + ++Triple, ++Input) { + FH.get()->WriteBundleStart(OutputFile, *Triple); + FH.get()->WriteBundle(OutputFile, *Input->get()); + FH.get()->WriteBundleEnd(OutputFile, *Triple); + } + return false; +} + +// Unbundle the files. Return true if an error was found. +static bool UnbundleFiles() { + // Open Input file. + ErrorOr> CodeOrErr = + MemoryBuffer::getFileOrSTDIN(InputFileNames.front()); + if (std::error_code EC = CodeOrErr.getError()) { + llvm::errs() << "error: Can't open file " << InputFileNames.front() << ": " + << EC.message() << "\n"; + return true; + } + + MemoryBuffer &Input = *CodeOrErr.get(); + + // Select the right files handler. + std::unique_ptr FH; + FH.reset(CreateFileHandler(Input)); + + // Quit if we don't have a handler. + if (!FH.get()) + return true; + + // Read the header of the bundled file. + FH.get()->ReadHeader(Input); + + // Create a work list that consist of the map triple/output file. + StringMap Worklist; + auto Output = OutputFileNames.begin(); + for (auto Triple = TargetNames.begin(); Triple < TargetNames.end(); + ++Triple, ++Output) + Worklist[*Triple] = *Output; + + // Read all the bundles that are in the work list. If we find no bundles we + // assume the file is meant for the host target. + bool FoundHostBundle = false; + while (!Worklist.empty()) { + StringRef CurTriple = FH.get()->ReadBundleStart(Input); + + // We don't have more bundles. + if (CurTriple.empty()) + break; + + auto Output = Worklist.find(CurTriple); + // The file may have more bundles for other targets, that we don't care + // about. Therefore, move on to the next triple + if (Output == Worklist.end()) { + continue; + } + + // Check if the output file can be opened and copy the bundle to it. + std::error_code EC; + raw_fd_ostream OutputFile(Output->second, EC, sys::fs::F_None); + if (EC) { + llvm::errs() << "error: Can't open file " << Output->second << ": " + << EC.message() << "\n"; + return true; + } + FH.get()->ReadBundle(OutputFile, Input); + FH.get()->ReadBundleEnd(Input); + Worklist.remove(&*Output); + + // Record if we found the host bundle. + if (hasHostKind(CurTriple)) + FoundHostBundle = true; + } + + // If no bundles were found, assume the input file is the host bundle and + // create empty files for the remaining targets. + if (Worklist.size() == TargetNames.size()) { + for (auto &E : Worklist) { + std::error_code EC; + raw_fd_ostream OutputFile(E.second, EC, sys::fs::F_None); + if (EC) { + llvm::errs() << "error: Can't open file " << E.second << ": " + << EC.message() << "\n"; + return true; + } + + // If this entry has a host kind, copy the input file to the output file. + if (hasHostKind(E.first())) + OutputFile.write(Input.getBufferStart(), Input.getBufferSize()); + } + return false; + } + + // If we found elements, we emit an error if none of those were for the host. + if (!FoundHostBundle) { + llvm::errs() << "error: Can't find bundles for all requested targets\n"; + return true; + } + + // If we still have any elements in the worklist, create empty files for them. + for (auto &E : Worklist) { + std::error_code EC; + raw_fd_ostream OutputFile(E.second, EC, sys::fs::F_None); + if (EC) { + llvm::errs() << "error: Can't open file " << E.second << ": " + << EC.message() << "\n"; + return true; + } + } + + return false; +} + +static void PrintVersion() { + raw_ostream &OS = outs(); + OS << clang::getClangToolFullVersion("clang-offload-bundler") << '\n'; +} + +int main(int argc, const char **argv) { + llvm::sys::PrintStackTraceOnErrorSignal(argv[0]); + + cl::HideUnrelatedOptions(ClangOffloadBundlerCategory); + cl::SetVersionPrinter(PrintVersion); + cl::ParseCommandLineOptions( + argc, argv, + "A tool to bundle several input files of the specified type \n" + "referring to the same source file but different targets into a single \n" + "one. The resulting file can also be unbundled into different files by \n" + "this tool if -unbundle is provided.\n"); + + if (Help) + cl::PrintHelpMessage(); + + bool Error = false; + if (Unbundle) { + if (InputFileNames.size() != 1) { + Error = true; + llvm::errs() + << "error: only one input file supported in unbundling mode.\n"; + } + if (OutputFileNames.size() != TargetNames.size()) { + Error = true; + llvm::errs() << "error: number of output files and targets should match " + "in unbundling mode.\n"; + } + } else { + if (OutputFileNames.size() != 1) { + Error = true; + llvm::errs() + << "error: only one output file supported in bundling mode.\n"; + } + if (InputFileNames.size() != TargetNames.size()) { + Error = true; + llvm::errs() << "error: number of input files and targets should match " + "in bundling mode.\n"; + } + } + + // Verify that the offload kinds and triples are known. We also check that we + // have exactly one host target. + unsigned HostTargetNum = 0u; + for (StringRef Target : TargetNames) { + StringRef Kind; + StringRef Triple; + getOffloadKindAndTriple(Target, Kind, Triple); + + bool KindIsValid = !Kind.empty(); + KindIsValid &= StringSwitch(Kind) + .Case("host", true) + .Case("openmp", true) + .Default(false); + + bool TripleIsValid = !Triple.empty(); + llvm::Triple T(Triple); + TripleIsValid &= T.getArch() != Triple::UnknownArch; + + if (!KindIsValid || !TripleIsValid) { + Error = true; + llvm::errs() << "error: invalid target '" << Target << "'"; + + if (!KindIsValid) + llvm::errs() << ", unknown offloading kind '" << Kind << "'"; + if (!TripleIsValid) + llvm::errs() << ", unknown target triple '" << Triple << "'"; + llvm::errs() << ".\n"; + } + + if (KindIsValid && Kind == "host") + ++HostTargetNum; + } + + if (HostTargetNum != 1) { + Error = true; + llvm::errs() << "error: expecting exactly one host target but got " + << HostTargetNum << ".\n"; + } + + if (Error) + return 1; + + if (Unbundle) + return UnbundleFiles(); + else + return BundleFiles(); + + return 0; +}