diff --git a/clang/test/Driver/clang-offload-bundler.c b/clang/test/Driver/clang-offload-bundler.c --- a/clang/test/Driver/clang-offload-bundler.c +++ b/clang/test/Driver/clang-offload-bundler.c @@ -47,6 +47,7 @@ // CK-HELP: {{.*}}o {{.*}}- object // CK-HELP: {{.*}}gch {{.*}}- precompiled-header // CK-HELP: {{.*}}ast {{.*}}- clang AST file +// CK-HELP: {{.*}}a {{.*}}- archive of objects // CK-HELP: {{.*}}-unbundle {{.*}}- Unbundle bundled file into several output files. // @@ -91,6 +92,9 @@ // CK-ERR9A: error: expecting exactly one host target but got 0 // CK-ERR9B: error: Duplicate targets are not allowed +// RUN: not clang-offload-bundler -type=a -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.a 2>&1 | FileCheck %s --check-prefix CK-ERR10 +// CK-ERR10: error: bundling is not supported for archives + // // Check text bundle. This is a readable format, so we check for the format we expect to find. // @@ -319,6 +323,19 @@ // RUN: 2>&1 | FileCheck -check-prefix=DUP %s // DUP: error: Duplicate targets are not allowed +// +// Check archive unbundling. +// +// RUN: rm -f %t.a +// RUN: llvm-ar crv %t.a %t.bundle3.o +// RUN: clang-offload-bundler -type=a -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.host.a,%t.tgt1.a,%t.tgt2.a -inputs=%t.a -unbundle +// RUN: cmp %t.host.a %t.a +// RUN: llvm-ar t %t.tgt1.a | FileCheck %s --check-prefix=CHECK-AR-LIST1 +// RUN: llvm-ar t %t.tgt2.a | FileCheck %s --check-prefix=CHECK-AR-LIST2 +// +// CHECK-AR-LIST1: openmp-powerpc64le-ibm-linux-gnu.{{.+}}.bundle3.o +// CHECK-AR-LIST2: openmp-x86_64-pc-linux-gnu.{{.+}}.bundle3.o + // Some code so that we can create a binary out of this file. int A = 0; void test_func(void) { diff --git a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp --- a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp +++ b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -20,8 +20,11 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" +#include "llvm/Object/Archive.h" +#include "llvm/Object/ArchiveWriter.h" #include "llvm/Object/Binary.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Casting.h" @@ -83,7 +86,8 @@ " s - assembler\n" " o - object\n" " gch - precompiled-header\n" - " ast - clang AST file"), + " ast - clang AST file\n" + " a - archive of objects"), cl::cat(ClangOffloadBundlerCategory)); static cl::opt Unbundle("unbundle", @@ -152,7 +156,7 @@ virtual Error ReadBundleEnd(MemoryBuffer &Input) = 0; /// Read the current bundle and write the result into the stream \a OS. - virtual Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0; + virtual Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) = 0; /// Write the header of the bundled file to \a OS based on the information /// gathered from \a Inputs. @@ -324,7 +328,7 @@ return Error::success(); } - Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final { + Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final { assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!"); StringRef FC = Input.getBuffer(); OS.write(FC.data() + CurBundleInfo->second.Offset, @@ -487,7 +491,7 @@ Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); } - Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final { + Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final { Expected ContentOrErr = CurrentSection->getContents(); if (!ContentOrErr) return ContentOrErr.takeError(); @@ -681,7 +685,7 @@ return Error::success(); } - Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final { + Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final { StringRef FC = Input.getBuffer(); size_t BundleStart = ReadChars; @@ -724,6 +728,186 @@ } }; +/// Archive file handler. Only unbundling is supported so far. +class ArchiveFileHandler final : public FileHandler { + /// Archive we are dealing with. + std::unique_ptr Ar; + + /// Set of bundle names from all archive objects. + StringSet<> Bundles; + + /// Iterators over the bundle names. + StringSet<>::iterator CurrBundle = Bundles.end(); + StringSet<>::iterator NextBundle = Bundles.end(); + +public: + ArchiveFileHandler() = default; + ~ArchiveFileHandler() = default; + + Error ReadHeader(MemoryBuffer &Input) override { + // Create archive instance from the given input. + auto ArOrErr = Archive::create(Input); + if (!ArOrErr) + return ArOrErr.takeError(); + Ar = std::move(*ArOrErr); + + // Read all archive children. + Error Err = Error::success(); + for (const auto &C : Ar->children(Err)) { + auto BinOrErr = C.getAsBinary(); + if (!BinOrErr) { + if (auto Err = isNotObjectErrorInvalidFileType(BinOrErr.takeError())) + return Err; + continue; + } + + // We are interested in object files only, so ignore the child if it is + // not an object. + auto &Bin = BinOrErr.get(); + if (!Bin->isObject()) + continue; + + auto Obj = std::unique_ptr(cast(Bin.release())); + auto Buf = MemoryBuffer::getMemBuffer(Obj->getMemoryBufferRef(), false); + + // Create object file handler for the child object and collect the list of + // bundles from it. + ObjectFileHandler OFH(std::move(Obj)); + if (Error Err = OFH.ReadHeader(*Buf)) + return Err; + Expected> NameOrErr = OFH.ReadBundleStart(*Buf); + if (!NameOrErr) + return NameOrErr.takeError(); + while (*NameOrErr) { + Bundles.insert(**NameOrErr); + NameOrErr = OFH.ReadBundleStart(*Buf); + if (!NameOrErr) + return NameOrErr.takeError(); + } + } + if (Err) + return Err; + + CurrBundle = Bundles.end(); + NextBundle = Bundles.begin(); + return Error::success(); + } + + Expected> ReadBundleStart(MemoryBuffer &Input) override { + if (NextBundle == Bundles.end()) + return None; + CurrBundle = NextBundle++; + return CurrBundle->first(); + } + + Error ReadBundleEnd(MemoryBuffer &Input) override { return Error::success(); } + + Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) override { + // For 'host' archive bundle just copy input data to the output stream. + if (hasHostKind(CurrBundle->first())) { + OS << Input.getBuffer(); + return Error::success(); + } + + // Extracted objects data for archive mode. + SmallVector ArNames; + SmallVector, 8u> ArData; + SmallVector ArMembers; + + // Read all archive children. + Error Err = Error::success(); + for (const auto &C : Ar->children(Err)) { + auto BinOrErr = C.getAsBinary(); + if (!BinOrErr) { + if (auto Err = isNotObjectErrorInvalidFileType(BinOrErr.takeError())) + return Err; + continue; + } + + // We are interested in object files only, so ignore the child if it is + // not an object. + auto &Bin = BinOrErr.get(); + if (!Bin->isObject()) + continue; + + auto Obj = std::unique_ptr(cast(Bin.release())); + auto Buf = MemoryBuffer::getMemBuffer(Obj->getMemoryBufferRef(), false); + + // Get the child name. We will use it for the target archive member. + auto ChildNameOrErr = C.getName(); + if (!ChildNameOrErr) + return ChildNameOrErr.takeError(); + + ObjectFileHandler OFH(std::move(Obj)); + if (Error Err = OFH.ReadHeader(*Buf)) + return Err; + Expected> NameOrErr = OFH.ReadBundleStart(*Buf); + if (!NameOrErr) + return NameOrErr.takeError(); + while (*NameOrErr) { + auto TT = **NameOrErr; + if (TT == CurrBundle->first()) { + // This is the bundle we are looking for. Extract target bundle from + // the child and add it to the list of target archive members. Use + // bundle name as a prefix for the child name is the target archive. + auto &Name = ArNames.emplace_back((TT + "." + *ChildNameOrErr).str()); + auto &Data = ArData.emplace_back(); + raw_svector_ostream ChildOS{Data}; + + // Extract the bundle. + if (Error Err = OFH.ReadBundle(ChildOS, *Buf)) + return Err; + + ArMembers.emplace_back( + MemoryBufferRef{StringRef(Data.data(), Data.size()), Name}); + + if (Error Err = OFH.ReadBundleEnd(*Buf)) + return Err; + } + + NameOrErr = OFH.ReadBundleStart(*Buf); + if (!NameOrErr) + return NameOrErr.takeError(); + } + } + if (Err) + return Err; + + // Determine archive kind to use for this offload target. + StringRef TargetKind; + StringRef TargetTriple; + getOffloadKindAndTriple(CurrBundle->first(), TargetKind, TargetTriple); + auto ArKind = + Triple(TargetTriple).isOSDarwin() ? Archive::K_DARWIN : Archive::K_GNU; + + // And write archive to the output. + Expected> NewAr = + writeArchiveToBuffer(ArMembers, true, ArKind, true, false); + if (!NewAr) + return NewAr.takeError(); + OS << NewAr.get()->getBuffer(); + + return Error::success(); + } + + Error WriteHeader(raw_fd_ostream &OS, + ArrayRef> Inputs) override { + llvm_unreachable("unsupported for the ArchiveFileHandler"); + } + + Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) override { + llvm_unreachable("unsupported for the ArchiveFileHandler"); + } + + Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) override { + llvm_unreachable("unsupported for the ArchiveFileHandler"); + } + + Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) override { + llvm_unreachable("unsupported for the ArchiveFileHandler"); + } +}; + /// Return an appropriate object file handler. We use the specific object /// handler if we know how to deal with that format, otherwise we use a default /// binary file handler. @@ -768,6 +952,8 @@ return std::make_unique(); if (FilesType == "ast") return std::make_unique(); + if (FilesType == "a") + return std::make_unique(); return createStringError(errc::invalid_argument, "'" + FilesType + "': invalid file type specified"); @@ -777,6 +963,10 @@ static Error BundleFiles() { std::error_code EC; + if (FilesType == "a") + return createStringError(errc::invalid_argument, + "bundling is not supported for archives"); + // Create output file. raw_fd_ostream OutputFile(OutputFileNames.front(), EC, sys::fs::OF_None); if (EC)