Index: clang/test/Driver/clang-offload-bundler.c =================================================================== --- clang/test/Driver/clang-offload-bundler.c +++ clang/test/Driver/clang-offload-bundler.c @@ -34,6 +34,7 @@ // CK-HELP: {{.*}}this tool if -unbundle is provided. // CK-HELP: {{.*}}USAGE: clang-offload-bundler [options] // CK-HELP: {{.*}}-inputs= - [,...] +// CK-HELP: {{.*}}-list {{.*}}- List bundle IDs in the bundled file. // CK-HELP: {{.*}}-outputs= - [,...] // CK-HELP: {{.*}}-targets= - [-,...] // CK-HELP: {{.*}}-type= - Type of the files to be bundled/unbundled. @@ -77,8 +78,10 @@ // RUN: not clang-offload-bundler 2>&1 | FileCheck %s --check-prefix CK-ERR7 // CK-ERR7-DAG: clang-offload-bundler: for the --type option: must be specified at least once! // CK-ERR7-DAG: clang-offload-bundler: for the --inputs option: must be specified at least once! -// CK-ERR7-DAG: clang-offload-bundler: for the --outputs option: must be specified at least once! -// CK-ERR7-DAG: clang-offload-bundler: for the --targets option: must be specified at least once! + +// RUN: not clang-offload-bundler -type=i -inputs=%t.i,%t.tgt1,%t.tgt2 2>&1 | FileCheck %s -check-prefix=CK-ERR7A +// CK-ERR7A-DAG: error: for the --targets option: must be specified at least once! +// CK-ERR7A-DAG: error: for the --outputs option: must be specified at least once! // RUN: not clang-offload-bundler -type=i -targets=hxst-powerpcxxle-ibm-linux-gnu,openxp-pxxerpc64le-ibm-linux-gnu,xpenmp-x86_xx-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR8 // CK-ERR8: error: invalid target 'hxst-powerpcxxle-ibm-linux-gnu', unknown offloading kind 'hxst', unknown target triple 'powerpcxxle-ibm-linux-gnu' @@ -151,24 +154,28 @@ // // Check text unbundle. Check if we get the exact same content that we bundled before for each file. // +// RUN: clang-offload-bundler -type=i -inputs=%t.bundle3.i -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=i -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.i,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.i -unbundle // RUN: diff %t.i %t.res.i // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=i -targets=openmp-powerpc64le-ibm-linux-gnu -outputs=%t.res.tgt1 -inputs=%t.bundle3.i -unbundle // RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: clang-offload-bundler -type=ii -inputs=%t.bundle3.ii -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=ii -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.ii,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.ii -unbundle // RUN: diff %t.ii %t.res.ii // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=ii -targets=openmp-x86_64-pc-linux-gnu -outputs=%t.res.tgt2 -inputs=%t.bundle3.ii -unbundle // RUN: diff %t.tgt2 %t.res.tgt2 +// RUN: clang-offload-bundler -type=ll -inputs=%t.bundle3.ll -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=ll -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.ll,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.ll -unbundle // RUN: diff %t.ll %t.res.ll // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=ll -targets=openmp-powerpc64le-ibm-linux-gnu -outputs=%t.res.tgt1 -inputs=%t.bundle3.ll -unbundle // RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: clang-offload-bundler -type=s -inputs=%t.bundle3.s -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=s -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.s,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.s -unbundle // RUN: diff %t.s %t.res.s // RUN: diff %t.tgt1 %t.res.tgt1 @@ -181,6 +188,7 @@ // RUN: diff %t.tgt2 %t.res.tgt2 // Check if we can unbundle a file with no magic strings. +// RUN: clang-offload-bundler -type=s -inputs=%t.s -list | FileCheck -check-prefix=CKLST2 --allow-empty %s // RUN: clang-offload-bundler -type=s -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.s,%t.res.tgt1,%t.res.tgt2 -inputs=%t.s -unbundle // RUN: diff %t.s %t.res.s // RUN: diff %t.empty %t.res.tgt1 @@ -201,18 +209,21 @@ // RUN: clang-offload-bundler -type=gch -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.ast,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.gch // RUN: clang-offload-bundler -type=ast -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.ast,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.ast // RUN: clang-offload-bundler -type=ast -targets=openmp-powerpc64le-ibm-linux-gnu,host-%itanium_abi_triple,openmp-x86_64-pc-linux-gnu -inputs=%t.tgt1,%t.ast,%t.tgt2 -outputs=%t.bundle3.unordered.ast +// RUN: clang-offload-bundler -type=bc -inputs=%t.bundle3.bc -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=bc -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.bc,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.bc -unbundle // RUN: diff %t.bc %t.res.bc // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=bc -targets=openmp-powerpc64le-ibm-linux-gnu -outputs=%t.res.tgt1 -inputs=%t.bundle3.bc -unbundle // RUN: diff %t.tgt1 %t.res.tgt1 +// RUN: clang-offload-bundler -type=gch -inputs=%t.bundle3.gch -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=gch -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.gch,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.gch -unbundle // RUN: diff %t.ast %t.res.gch // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 // RUN: clang-offload-bundler -type=gch -targets=openmp-x86_64-pc-linux-gnu -outputs=%t.res.tgt2 -inputs=%t.bundle3.gch -unbundle // RUN: diff %t.tgt2 %t.res.tgt2 +// RUN: clang-offload-bundler -type=ast -inputs=%t.bundle3.ast -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=ast -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.ast,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.ast -unbundle // RUN: diff %t.ast %t.res.ast // RUN: diff %t.tgt1 %t.res.tgt1 @@ -257,6 +268,7 @@ // CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=readonly,exclude" "[[TEMPOBJ]]" "[[OUTOBJ]]" // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o +// RUN: clang-offload-bundler -type=o -inputs=%t.bundle3.o -list | FileCheck -check-prefix=CKLST %s // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle // RUN: diff %t.bundle3.o %t.res.o // RUN: diff %t.tgt1 %t.res.tgt1 @@ -269,6 +281,7 @@ // RUN: diff %t.tgt1 %t.res.tgt1 // Check if we can unbundle a file with no magic strings. +// RUN: clang-offload-bundler -type=o -inputs=%t.o -list | FileCheck -check-prefix=CKLST2 --allow-empty %s // RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.o -unbundle // RUN: diff %t.o %t.res.o // RUN: diff %t.empty %t.res.tgt1 @@ -288,6 +301,28 @@ // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 +// +// Check -list option +// + +// RUN: clang-offload-bundler -bundle-align=4096 -type=bc -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.bc,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.bc +// RUN: not clang-offload-bundler -type=bc -inputs=%t.bundle3.bc -unbundle -list 2>&1 | FileCheck -check-prefix=CKLST-ERR %s +// CKLST-ERR: error: -unbundle and -list cannot be used together +// RUN: not clang-offload-bundler -type=bc -inputs=%t.bundle3.bc -targets=host-%itanium_abi_triple -list 2>&1 | FileCheck -check-prefix=CKLST-ERR2 %s +// CKLST-ERR2: error: -targets option is invalid for -list +// RUN: not clang-offload-bundler -type=bc -inputs=%t.bundle3.bc -outputs=out.txt -list 2>&1 | FileCheck -check-prefix=CKLST-ERR3 %s +// CKLST-ERR3: error: -outputs option is invalid for -list +// RUN: not clang-offload-bundler -type=bc -inputs=%t.bundle3.bc,%t.bc -list 2>&1 | FileCheck -check-prefix=CKLST-ERR4 %s +// CKLST-ERR4: error: only one input file supported for -list + +// CKLST-DAG: host- +// CKLST-DAG: openmp-powerpc64le-ibm-linux-gnu +// CKLST-DAG: openmp-x86_64-pc-linux-gnu + +// CKLST2-NOT: host- +// CKLST2-NOT: openmp-powerpc64le-ibm-linux-gnu +// CKLST2-NOT: openmp-x86_64-pc-linux-gnu + // Some code so that we can create a binary out of this file. int A = 0; void test_func(void) { Index: clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp =================================================================== --- clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp +++ clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -62,11 +62,11 @@ cl::desc("[,...]"), cl::cat(ClangOffloadBundlerCategory)); static cl::list - OutputFileNames("outputs", cl::CommaSeparated, cl::OneOrMore, + OutputFileNames("outputs", cl::CommaSeparated, cl::desc("[,...]"), cl::cat(ClangOffloadBundlerCategory)); static cl::list - TargetNames("targets", cl::CommaSeparated, cl::OneOrMore, + TargetNames("targets", cl::CommaSeparated, cl::desc("[-,...]"), cl::cat(ClangOffloadBundlerCategory)); static cl::opt @@ -89,6 +89,10 @@ cl::desc("Unbundle bundled file into several output files.\n"), cl::init(false), cl::cat(ClangOffloadBundlerCategory)); +static cl::opt + ListBundleIDs("list", cl::desc("List bundle IDs in the bundled file.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + static cl::opt PrintExternalCommands( "###", cl::desc("Print any external commands that are to be executed " @@ -109,6 +113,41 @@ /// Path to the current binary. static std::string BundlerExecutable; +namespace { + +// This class implements a list of temporary files that are removed upon +// object destruction. +class TempFileHandlerRAII { +public: + ~TempFileHandlerRAII() { + for (const auto &File : Files) + sys::fs::remove(File); + } + + // Creates temporary file with given contents. + Expected Create(Optional> Contents) { + SmallString<128u> File; + if (std::error_code EC = + sys::fs::createTemporaryFile("clang-offload-bundler", "tmp", File)) + return createFileError(File, EC); + Files.push_front(File); + + if (Contents) { + std::error_code EC; + raw_fd_ostream OS(File, EC); + if (EC) + return createFileError(File, EC); + OS.write(Contents->data(), Contents->size()); + } + return Files.front(); + } + +private: + std::forward_list> Files; +}; + +} // end anonymous namespace + /// Obtain the offload kind and real machine triple out of the target /// information specified by the user. static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind, @@ -163,6 +202,9 @@ /// Write the bundle from \a Input into \a OS. virtual Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0; + + /// List bundle IDs in \a Input. + virtual Error listBundleIDs(MemoryBuffer &Input) = 0; }; /// Handler for binary files. The bundled file will have the following format @@ -290,6 +332,8 @@ StringRef Triple(&FC.data()[ReadChars], TripleSize); ReadChars += TripleSize; + if (ListBundleIDs) + llvm::outs() << Triple << '\n'; // Check if the offset and size make sense. if (!Offset || Offset + Size > FC.size()) @@ -376,42 +420,13 @@ OS.write(Input.getBufferStart(), Input.getBufferSize()); return Error::success(); } -}; - -namespace { - -// This class implements a list of temporary files that are removed upon -// object destruction. -class TempFileHandlerRAII { -public: - ~TempFileHandlerRAII() { - for (const auto &File : Files) - sys::fs::remove(File); - } - // Creates temporary file with given contents. - Expected Create(Optional> Contents) { - SmallString<128u> File; - if (std::error_code EC = - sys::fs::createTemporaryFile("clang-offload-bundler", "tmp", File)) - return createFileError(File, EC); - Files.push_front(File); - - if (Contents) { - std::error_code EC; - raw_fd_ostream OS(File, EC); - if (EC) - return createFileError(File, EC); - OS.write(Contents->data(), Contents->size()); - } - return Files.front(); + Error listBundleIDs(MemoryBuffer &Input) final { + // List bundle IDs in a binary file only needs go through the header. + return ReadHeader(Input); } - -private: - std::forward_list> Files; }; -} // end anonymous namespace /// Handler for object files. The bundles are organized by sections with a /// designated name. @@ -472,8 +487,11 @@ IsOffloadSection(*CurrentSection); if (!TripleOrErr) return TripleOrErr.takeError(); - if (*TripleOrErr) + if (*TripleOrErr) { + if (ListBundleIDs) + llvm::outs() << **TripleOrErr << '\n'; return **TripleOrErr; + } } return None; } @@ -595,6 +613,25 @@ return Error::success(); } + Error listBundleIDs(MemoryBuffer &Input) final { + if (Error Err = ReadHeader(Input)) + return Err; + + while (true) { + // To list bundle IDs in bundled object files we need to read the start of + // each bundle but do not need read the whole bundle. + Expected> CurTripleOrErr = ReadBundleStart(Input); + if (!CurTripleOrErr) + return CurTripleOrErr.takeError(); + + // No more bundles. + if (!*CurTripleOrErr) + break; + } + + return Error::success(); + } + private: static Error executeObjcopy(StringRef Objcopy, ArrayRef Args) { // If the user asked for the commands to be printed out, we do that @@ -657,7 +694,10 @@ // Next time we read after the new line. ++ReadChars; - return StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart); + auto Triple = StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart); + if (ListBundleIDs) + llvm::outs() << Triple << '\n'; + return Triple; } Error ReadBundleEnd(MemoryBuffer &Input) final { @@ -715,6 +755,46 @@ BundleEndString = "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ "; } + + Error listBundleIDs(MemoryBuffer &Input) final { + if (Error Err = ReadHeader(Input)) + return Err; + + // Create an intermediate temporary file for reading the bundles. + TempFileHandlerRAII TempFiles; + Expected IntermediateFileOrErr = TempFiles.Create(None); + if (!IntermediateFileOrErr) + return IntermediateFileOrErr.takeError(); + StringRef IntermediateFile = *IntermediateFileOrErr; + + while (true) { + Expected> CurTripleOrErr = ReadBundleStart(Input); + if (!CurTripleOrErr) + return CurTripleOrErr.takeError(); + + // No more bundles. + if (!*CurTripleOrErr) + break; + + StringRef CurTriple = **CurTripleOrErr; + assert(!CurTriple.empty()); + + std::error_code EC; + raw_fd_ostream OutputFile(IntermediateFile, EC, sys::fs::OF_None); + if (EC) + return createFileError(IntermediateFile, EC); + // TODO: To list bundle IDs in a bundled text file we need to go through + // all bundles. The format of bundled text file may need to include a + // header if the performance of listing bundle IDs of bundled text file is + // important. + if (Error Err = ReadBundle(OutputFile, Input)) + return Err; + if (Error Err = ReadBundleEnd(Input)) + return Err; + } + + return Error::success(); + } }; /// Return an appropriate object file handler. We use the specific object @@ -916,6 +996,27 @@ return Error::success(); } +// List bundle IDs. Return true if an error was found. +static Error ListBundleIDsInFile() { + // Open Input file. + ErrorOr> CodeOrErr = + MemoryBuffer::getFileOrSTDIN(InputFileNames.front()); + if (std::error_code EC = CodeOrErr.getError()) + return createFileError(InputFileNames.front(), EC); + + MemoryBuffer &Input = **CodeOrErr; + + // Select the right files handler. + Expected> FileHandlerOrErr = + CreateFileHandler(Input); + if (!FileHandlerOrErr) + return FileHandlerOrErr.takeError(); + + std::unique_ptr &FH = *FileHandlerOrErr; + assert(FH); + return FH->listBundleIDs(Input); +} + static void PrintVersion(raw_ostream &OS) { OS << clang::getClangToolFullVersion("clang-offload-bundler") << '\n'; } @@ -942,6 +1043,41 @@ }; bool Error = false; + if (!ListBundleIDs) { + if (OutputFileNames.getNumOccurrences() == 0) { + Error = true; + reportError(createStringError( + errc::invalid_argument, + "for the --outputs option: must be specified at least once!")); + } + if (TargetNames.getNumOccurrences() == 0) { + Error = true; + reportError(createStringError( + errc::invalid_argument, + "for the --targets option: must be specified at least once!")); + } + } + if (Unbundle && ListBundleIDs) { + Error = true; + reportError(createStringError( + errc::invalid_argument, "-unbundle and -list cannot be used together")); + } else if (ListBundleIDs) { + if (InputFileNames.size() != 1) { + Error = true; + reportError(createStringError(errc::invalid_argument, + "only one input file supported for -list")); + } + if (OutputFileNames.size()) { + Error = true; + reportError(createStringError(errc::invalid_argument, + "-outputs option is invalid for -list")); + } + if (TargetNames.size()) { + Error = true; + reportError(createStringError(errc::invalid_argument, + "-targets option is invalid for -list")); + } + } if (Unbundle) { if (InputFileNames.size() != 1) { Error = true; @@ -955,7 +1091,7 @@ "number of output files and targets should " "match in unbundling mode")); } - } else { + } else if (!ListBundleIDs) { if (OutputFileNames.size() != 1) { Error = true; reportError(createStringError( @@ -1014,7 +1150,8 @@ // Host triple is not really needed for unbundling operation, so do not // treat missing host triple as error if we do unbundling. - if ((Unbundle && HostTargetNum > 1) || (!Unbundle && HostTargetNum != 1)) { + if ((Unbundle && HostTargetNum > 1) || + (!Unbundle && !ListBundleIDs && HostTargetNum != 1)) { Error = true; reportError(createStringError(errc::invalid_argument, "expecting exactly one host target but got " + @@ -1030,7 +1167,10 @@ if (!llvm::sys::fs::exists(BundlerExecutable)) BundlerExecutable = sys::fs::getMainExecutable(argv[0], &BundlerExecutable); - if (llvm::Error Err = Unbundle ? UnbundleFiles() : BundleFiles()) { + llvm::Error Err = ListBundleIDs + ? ListBundleIDsInFile() + : (Unbundle ? UnbundleFiles() : BundleFiles()); + if (Err) { reportError(std::move(Err)); return 1; }