Index: clang/test/Driver/clang-offload-bundler.c =================================================================== --- clang/test/Driver/clang-offload-bundler.c +++ clang/test/Driver/clang-offload-bundler.c @@ -34,6 +34,7 @@ // CK-HELP: {{.*}}this tool if -unbundle is provided. // CK-HELP: {{.*}}USAGE: clang-offload-bundler [options] // CK-HELP: {{.*}}-inputs= - [,...] +// CK-HELP: {{.*}}-list {{.*}}- List bundle IDs in the bundled file. // CK-HELP: {{.*}}-outputs= - [,...] // CK-HELP: {{.*}}-targets= - [-,...] // CK-HELP: {{.*}}-type= - Type of the files to be bundled/unbundled. @@ -77,8 +78,10 @@ // RUN: not clang-offload-bundler 2>&1 | FileCheck %s --check-prefix CK-ERR7 // CK-ERR7-DAG: clang-offload-bundler: for the --type option: must be specified at least once! // CK-ERR7-DAG: clang-offload-bundler: for the --inputs option: must be specified at least once! -// CK-ERR7-DAG: clang-offload-bundler: for the --outputs option: must be specified at least once! -// CK-ERR7-DAG: clang-offload-bundler: for the --targets option: must be specified at least once! + +// RUN: not clang-offload-bundler -type=i -inputs=%t.i,%t.tgt1,%t.tgt2 2>&1 | FileCheck %s -check-prefix=CK-ERR7A +// CK-ERR7A-DAG: error: for the --targets option: must be specified at least once! +// CK-ERR7A-DAG: error: for the --outputs option: must be specified at least once! // RUN: not clang-offload-bundler -type=i -targets=hxst-powerpcxxle-ibm-linux-gnu,openxp-pxxerpc64le-ibm-linux-gnu,xpenmp-x86_xx-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR8 // CK-ERR8: error: invalid target 'hxst-powerpcxxle-ibm-linux-gnu', unknown offloading kind 'hxst', unknown target triple 'powerpcxxle-ibm-linux-gnu' @@ -151,24 +154,28 @@ // // Check text unbundle. Check if we get the exact same content that we bundled before for each file. // +// RUN: clang-offload-bundler -type=i -inputs=%t.bundle3.i -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=i -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.i,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.i -unbundle // RUN: diff %t.i %t.res.i // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=i -targets=openmp-powerpc64le-ibm-linux-gnu -outputs=%t.res.tgt1 -inputs=%t.bundle3.i -unbundle // RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: clang-offload-bundler -type=ii -inputs=%t.bundle3.ii -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=ii -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.ii,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.ii -unbundle // RUN: diff %t.ii %t.res.ii // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=ii -targets=openmp-x86_64-pc-linux-gnu -outputs=%t.res.tgt2 -inputs=%t.bundle3.ii -unbundle // RUN: diff %t.tgt2 %t.res.tgt2 +// RUN: clang-offload-bundler -type=ll -inputs=%t.bundle3.ll -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=ll -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.ll,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.ll -unbundle // RUN: diff %t.ll %t.res.ll // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=ll -targets=openmp-powerpc64le-ibm-linux-gnu -outputs=%t.res.tgt1 -inputs=%t.bundle3.ll -unbundle // RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: clang-offload-bundler -type=s -inputs=%t.bundle3.s -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=s -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.s,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.s -unbundle // RUN: diff %t.s %t.res.s // RUN: diff %t.tgt1 %t.res.tgt1 @@ -181,6 +188,7 @@ // RUN: diff %t.tgt2 %t.res.tgt2 // Check if we can unbundle a file with no magic strings. +// RUN: clang-offload-bundler -type=s -inputs=%t.s -list | FileCheck -check-prefix=CKLST2 --allow-empty %s // RUN: clang-offload-bundler -type=s -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.s,%t.res.tgt1,%t.res.tgt2 -inputs=%t.s -unbundle // RUN: diff %t.s %t.res.s // RUN: diff %t.empty %t.res.tgt1 @@ -201,18 +209,21 @@ // RUN: clang-offload-bundler -type=gch -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.ast,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.gch // RUN: clang-offload-bundler -type=ast -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.ast,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.ast // RUN: clang-offload-bundler -type=ast -targets=openmp-powerpc64le-ibm-linux-gnu,host-%itanium_abi_triple,openmp-x86_64-pc-linux-gnu -inputs=%t.tgt1,%t.ast,%t.tgt2 -outputs=%t.bundle3.unordered.ast +// RUN: clang-offload-bundler -type=bc -inputs=%t.bundle3.bc -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=bc -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.bc,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.bc -unbundle // RUN: diff %t.bc %t.res.bc // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=bc -targets=openmp-powerpc64le-ibm-linux-gnu -outputs=%t.res.tgt1 -inputs=%t.bundle3.bc -unbundle // RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: clang-offload-bundler -type=gch -inputs=%t.bundle3.gch -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=gch -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.gch,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.gch -unbundle // RUN: diff %t.ast %t.res.gch // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=gch -targets=openmp-x86_64-pc-linux-gnu -outputs=%t.res.tgt2 -inputs=%t.bundle3.gch -unbundle // RUN: diff %t.tgt2 %t.res.tgt2 +// RUN: clang-offload-bundler -type=ast -inputs=%t.bundle3.ast -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=ast -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.ast,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.ast -unbundle // RUN: diff %t.ast %t.res.ast // RUN: diff %t.tgt1 %t.res.tgt1 @@ -257,6 +268,7 @@ // CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=readonly,exclude" "[[TEMPOBJ]]" "[[OUTOBJ]]" // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o +// RUN: clang-offload-bundler -type=o -inputs=%t.bundle3.o -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle // RUN: diff %t.bundle3.o %t.res.o // RUN: diff %t.tgt1 %t.res.tgt1 @@ -269,6 +281,7 @@ // RUN: diff %t.tgt1 %t.res.tgt1 // Check if we can unbundle a file with no magic strings. +// RUN: clang-offload-bundler -type=o -inputs=%t.o -list | FileCheck -check-prefix=CKLST2 --allow-empty %s // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.o -unbundle // RUN: diff %t.o %t.res.o // RUN: diff %t.empty %t.res.tgt1 @@ -288,6 +301,28 @@ // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 +// +// Check -list option +// + +// RUN: clang-offload-bundler -bundle-align=4096 -type=bc -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.bc,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.bc +// RUN: not clang-offload-bundler -type=bc -inputs=%t.bundle3.bc -unbundle -list 2>&1 | FileCheck -check-prefix=CKLST-ERR %s +// CKLST-ERR: error: -unbundle and -list cannot be used together +// RUN: not clang-offload-bundler -type=bc -inputs=%t.bundle3.bc -targets=host-%itanium_abi_triple -list 2>&1 | FileCheck -check-prefix=CKLST-ERR2 %s +// CKLST-ERR2: error: -targets option is invalid for -list +// RUN: not clang-offload-bundler -type=bc -inputs=%t.bundle3.bc -outputs=out.txt -list 2>&1 | FileCheck -check-prefix=CKLST-ERR3 %s +// CKLST-ERR3: error: -outputs option is invalid for -list +// RUN: not clang-offload-bundler -type=bc -inputs=%t.bundle3.bc,%t.bc -list 2>&1 | FileCheck -check-prefix=CKLST-ERR4 %s +// CKLST-ERR4: error: only one input file supported for -list + +// CKLST-DAG: host- +// CKLST-DAG: openmp-powerpc64le-ibm-linux-gnu +// CKLST-DAG: openmp-x86_64-pc-linux-gnu + +// CKLST2-NOT: host- +// CKLST2-NOT: openmp-powerpc64le-ibm-linux-gnu +// CKLST2-NOT: openmp-x86_64-pc-linux-gnu + // Some code so that we can create a binary out of this file. int A = 0; void test_func(void) { Index: clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp =================================================================== --- clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp +++ clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -62,11 +62,11 @@ cl::desc("[,...]"), cl::cat(ClangOffloadBundlerCategory)); static cl::list - OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, + OutputFileNames("outputs", cl::CommaSeparated, cl::desc("[,...]"), cl::cat(ClangOffloadBundlerCategory)); static cl::list - TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, + TargetNames("targets", cl::CommaSeparated, cl::desc("[-,...]"), cl::cat(ClangOffloadBundlerCategory)); static cl::opt @@ -89,6 +89,10 @@ cl::desc("Unbundle bundled file into several output files.\n"), cl::init(false), cl::cat(ClangOffloadBundlerCategory)); +static cl::opt + ListBundleIDs("list", cl::desc("List bundle IDs in the bundled file.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + static cl::opt PrintExternalCommands( "###", cl::desc("Print any external commands that are to be executed " @@ -127,6 +131,10 @@ /// Generic file handler interface. class FileHandler { public: + struct BundleInfo { + StringRef BundleID; + }; + FileHandler() {} virtual ~FileHandler() {} @@ -163,6 +171,48 @@ /// Write the bundle from \a Input into \a OS. virtual Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0; + + /// List bundle IDs in \a Input. + virtual Error listBundleIDs(MemoryBuffer &Input) { + if (Error Err = ReadHeader(Input)) + return Err; + + return forEachBundle(Input, [&](const BundleInfo &Info) -> Error { + llvm::outs() << Info.BundleID << '\n'; + Error Err = listBundleIDsCallback(Input, Info); + if (Err) + return Err; + return Error::success(); + }); + } + + /// For each bundle in \a Input, do \a Func. + Error forEachBundle(MemoryBuffer &Input, + std::function Func) { + while (true) { + Expected> CurTripleOrErr = ReadBundleStart(Input); + if (!CurTripleOrErr) + return CurTripleOrErr.takeError(); + + // No more bundles. + if (!*CurTripleOrErr) + break; + + StringRef CurTriple = **CurTripleOrErr; + assert(!CurTriple.empty()); + + BundleInfo Info{CurTriple}; + if (Error Err = Func(Info)) + return Err; + } + return Error::success(); + } + +protected: + virtual Error listBundleIDsCallback(MemoryBuffer &Input, + const BundleInfo &Info) { + return Error::success(); + } }; /// Handler for binary files. The bundled file will have the following format @@ -212,22 +262,23 @@ class BinaryFileHandler final : public FileHandler { /// Information about the bundles extracted from the header. - struct BundleInfo final { + struct BinaryBundleInfo final : public BundleInfo { /// Size of the bundle. uint64_t Size = 0u; /// Offset at which the bundle starts in the bundled file. uint64_t Offset = 0u; - BundleInfo() {} - BundleInfo(uint64_t Size, uint64_t Offset) : Size(Size), Offset(Offset) {} + BinaryBundleInfo() {} + BinaryBundleInfo(uint64_t Size, uint64_t Offset) + : Size(Size), Offset(Offset) {} }; /// Map between a triple and the corresponding bundle information. - StringMap BundlesInfo; + StringMap BundlesInfo; /// Iterator for the bundle information that is being read. - StringMap::iterator CurBundleInfo; - StringMap::iterator NextBundleInfo; + StringMap::iterator CurBundleInfo; + StringMap::iterator NextBundleInfo; /// Current bundle target to be written. std::string CurWriteBundleTarget; @@ -297,7 +348,7 @@ assert(BundlesInfo.find(Triple) == BundlesInfo.end() && "Triple is duplicated??"); - BundlesInfo[Triple] = BundleInfo(Size, Offset); + BundlesInfo[Triple] = BinaryBundleInfo(Size, Offset); } // Set the iterator to where we will start to read. CurBundleInfo = BundlesInfo.end(); @@ -351,7 +402,7 @@ Write8byteIntegerToBuffer(OS, HeaderSize); // Size of the bundle (adds to the next bundle's offset) Write8byteIntegerToBuffer(OS, MB.getBufferSize()); - BundlesInfo[T] = BundleInfo(MB.getBufferSize(), HeaderSize); + BundlesInfo[T] = BinaryBundleInfo(MB.getBufferSize(), HeaderSize); HeaderSize += MB.getBufferSize(); // Size of the triple Write8byteIntegerToBuffer(OS, T.size()); @@ -715,6 +766,18 @@ BundleEndString = "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ "; } + + Error listBundleIDsCallback(MemoryBuffer &Input, + const BundleInfo &Info) final { + // TODO: To list bundle IDs in a bundled text file we need to go through + // all bundles. The format of bundled text file may need to include a + // header if the performance of listing bundle IDs of bundled text file is + // important. + ReadChars = Input.getBuffer().find(BundleEndString, ReadChars); + if (Error Err = ReadBundleEnd(Input)) + return Err; + return Error::success(); + } }; /// Return an appropriate object file handler. We use the specific object @@ -815,6 +878,27 @@ return Error::success(); } +// List bundle IDs. Return true if an error was found. +static Error ListBundleIDsInFile() { + // Open Input file. + ErrorOr> CodeOrErr = + MemoryBuffer::getFileOrSTDIN(InputFileNames.front()); + if (std::error_code EC = CodeOrErr.getError()) + return createFileError(InputFileNames.front(), EC); + + MemoryBuffer &Input = **CodeOrErr; + + // Select the right files handler. + Expected> FileHandlerOrErr = + CreateFileHandler(Input); + if (!FileHandlerOrErr) + return FileHandlerOrErr.takeError(); + + std::unique_ptr &FH = *FileHandlerOrErr; + assert(FH); + return FH->listBundleIDs(Input); +} + // Unbundle the files. Return true if an error was found. static Error UnbundleFiles() { // Open Input file. @@ -942,6 +1026,41 @@ }; bool Error = false; + if (!ListBundleIDs) { + if (OutputFileNames.getNumOccurrences() == 0) { + Error = true; + reportError(createStringError( + errc::invalid_argument, + "for the --outputs option: must be specified at least once!")); + } + if (TargetNames.getNumOccurrences() == 0) { + Error = true; + reportError(createStringError( + errc::invalid_argument, + "for the --targets option: must be specified at least once!")); + } + } + if (Unbundle && ListBundleIDs) { + Error = true; + reportError(createStringError( + errc::invalid_argument, "-unbundle and -list cannot be used together")); + } else if (ListBundleIDs) { + if (InputFileNames.size() != 1) { + Error = true; + reportError(createStringError(errc::invalid_argument, + "only one input file supported for -list")); + } + if (OutputFileNames.size()) { + Error = true; + reportError(createStringError(errc::invalid_argument, + "-outputs option is invalid for -list")); + } + if (TargetNames.size()) { + Error = true; + reportError(createStringError(errc::invalid_argument, + "-targets option is invalid for -list")); + } + } if (Unbundle) { if (InputFileNames.size() != 1) { Error = true; @@ -955,7 +1074,7 @@ "number of output files and targets should " "match in unbundling mode")); } - } else { + } else if (!ListBundleIDs) { if (OutputFileNames.size() != 1) { Error = true; reportError(createStringError( @@ -1014,7 +1133,8 @@ // Host triple is not really needed for unbundling operation, so do not // treat missing host triple as error if we do unbundling. - if ((Unbundle && HostTargetNum > 1) || (!Unbundle && HostTargetNum != 1)) { + if ((Unbundle && HostTargetNum > 1) || + (!Unbundle && !ListBundleIDs && HostTargetNum != 1)) { Error = true; reportError(createStringError(errc::invalid_argument, "expecting exactly one host target but got " + @@ -1030,7 +1150,10 @@ if (!llvm::sys::fs::exists(BundlerExecutable)) BundlerExecutable = sys::fs::getMainExecutable(argv[0], &BundlerExecutable); - if (llvm::Error Err = Unbundle ? UnbundleFiles() : BundleFiles()) { + llvm::Error Err = ListBundleIDs + ? ListBundleIDsInFile() + : (Unbundle ? UnbundleFiles() : BundleFiles()); + if (Err) { reportError(std::move(Err)); return 1; }