Index: llvm/trunk/test/tools/llvm-lipo/Inputs/arm64-slice.yaml =================================================================== --- llvm/trunk/test/tools/llvm-lipo/Inputs/arm64-slice.yaml +++ llvm/trunk/test/tools/llvm-lipo/Inputs/arm64-slice.yaml @@ -0,0 +1,101 @@ +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x0100000C + cpusubtype: 0x00000000 + filetype: 0x00000001 + ncmds: 4 + sizeofcmds: 352 + flags: 0x00002000 + reserved: 0x00000000 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: '' + vmaddr: 0 + vmsize: 56 + fileoff: 384 + filesize: 56 + maxprot: 7 + initprot: 7 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0x0000000000000000 + size: 20 + offset: 0x00000180 + align: 2 + reloff: 0x00000000 + nreloc: 0 + flags: 0x80000400 + reserved1: 0x00000000 + reserved2: 0x00000000 + reserved3: 0x00000000 + - sectname: __compact_unwind + segname: __LD + addr: 0x0000000000000018 + size: 32 + offset: 0x00000198 + align: 3 + reloff: 0x000001B8 + nreloc: 1 + flags: 0x02000000 + reserved1: 0x00000000 + reserved2: 0x00000000 + reserved3: 0x00000000 + - cmd: LC_VERSION_MIN_IPHONEOS + cmdsize: 16 + version: 327680 + sdk: 0 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 448 + nsyms: 3 + stroff: 496 + strsize: 20 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 2 + iextdefsym: 2 + nextdefsym: 1 + iundefsym: 3 + nundefsym: 0 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 +LinkEditData: + NameList: + - n_strx: 13 + n_type: 0x0E + n_sect: 1 + n_desc: 0 + n_value: 0 + - n_strx: 7 + n_type: 0x0E + n_sect: 2 + n_desc: 0 + n_value: 24 + - n_strx: 1 + n_type: 0x0F + n_sect: 1 + n_desc: 0 + n_value: 0 + StringTable: + - '' + - _main + - ltmp1 + - ltmp0 + - '' +... Index: llvm/trunk/test/tools/llvm-lipo/Inputs/armv7-slice.yaml =================================================================== --- llvm/trunk/test/tools/llvm-lipo/Inputs/armv7-slice.yaml +++ llvm/trunk/test/tools/llvm-lipo/Inputs/armv7-slice.yaml @@ -0,0 +1,76 @@ +--- !mach-o +FileHeader: + magic: 0xFEEDFACE + cputype: 0x0000000C + cpusubtype: 0x00000009 + filetype: 0x00000001 + ncmds: 4 + sizeofcmds: 244 + flags: 0x00002000 +LoadCommands: + - cmd: LC_SEGMENT + cmdsize: 124 + segname: '' + vmaddr: 0 + vmsize: 10 + fileoff: 272 + filesize: 10 + maxprot: 7 + initprot: 7 + nsects: 1 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0x0000000000000000 + size: 10 + offset: 0x00000110 + align: 1 + reloff: 0x00000000 + nreloc: 0 + flags: 0x80000400 + reserved1: 0x00000000 + reserved2: 0x00000000 + reserved3: 0x00000000 + - cmd: LC_VERSION_MIN_IPHONEOS + cmdsize: 16 + version: 327680 + sdk: 0 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 284 + nsyms: 1 + stroff: 296 + strsize: 8 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 1 + iundefsym: 1 + nundefsym: 0 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 +LinkEditData: + NameList: + - n_strx: 1 + n_type: 0x0F + n_sect: 1 + n_desc: 8 + n_value: 0 + StringTable: + - '' + - _main + - '' +... Index: llvm/trunk/test/tools/llvm-lipo/Inputs/x86_64-slice.yaml =================================================================== --- llvm/trunk/test/tools/llvm-lipo/Inputs/x86_64-slice.yaml +++ llvm/trunk/test/tools/llvm-lipo/Inputs/x86_64-slice.yaml @@ -0,0 +1,89 @@ +--- !mach-o +FileHeader: + magic: 0xFEEDFACF + cputype: 0x01000007 + cpusubtype: 0x00000003 + filetype: 0x00000001 + ncmds: 4 + sizeofcmds: 352 + flags: 0x00002000 + reserved: 0x00000000 +LoadCommands: + - cmd: LC_SEGMENT_64 + cmdsize: 232 + segname: '' + vmaddr: 0 + vmsize: 80 + fileoff: 384 + filesize: 80 + maxprot: 7 + initprot: 7 + nsects: 2 + flags: 0 + Sections: + - sectname: __text + segname: __TEXT + addr: 0x0000000000000000 + size: 15 + offset: 0x00000180 + align: 4 + reloff: 0x00000000 + nreloc: 0 + flags: 0x80000400 + reserved1: 0x00000000 + reserved2: 0x00000000 + reserved3: 0x00000000 + - sectname: __eh_frame + segname: __TEXT + addr: 0x0000000000000010 + size: 64 + offset: 0x00000190 + align: 3 + reloff: 0x00000000 + nreloc: 0 + flags: 0x6800000B + reserved1: 0x00000000 + reserved2: 0x00000000 + reserved3: 0x00000000 + - cmd: LC_VERSION_MIN_MACOSX + cmdsize: 16 + version: 656384 + sdk: 0 + - cmd: LC_SYMTAB + cmdsize: 24 + symoff: 464 + nsyms: 1 + stroff: 480 + strsize: 8 + - cmd: LC_DYSYMTAB + cmdsize: 80 + ilocalsym: 0 + nlocalsym: 0 + iextdefsym: 0 + nextdefsym: 1 + iundefsym: 1 + nundefsym: 0 + tocoff: 0 + ntoc: 0 + modtaboff: 0 + nmodtab: 0 + extrefsymoff: 0 + nextrefsyms: 0 + indirectsymoff: 0 + nindirectsyms: 0 + extreloff: 0 + nextrel: 0 + locreloff: 0 + nlocrel: 0 +LinkEditData: + NameList: + - n_strx: 1 + n_type: 0x0F + n_sect: 1 + n_desc: 0 + n_value: 0 + StringTable: + - '' + - _main + - '' +... Index: llvm/trunk/test/tools/llvm-lipo/create-executable.test =================================================================== --- llvm/trunk/test/tools/llvm-lipo/create-executable.test +++ llvm/trunk/test/tools/llvm-lipo/create-executable.test @@ -0,0 +1,11 @@ +# RUN: yaml2obj %p/Inputs/i386-slice.yaml > %t-i386.o +# RUN: yaml2obj %p/Inputs/x86_64-slice.yaml > %t-x86_64.o + +# RUN: chmod -x %t-i386.o +# RUN: chmod -x %t-x86_64.o +# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t-universal.o +# RUN: ! test -x %t-universal.o + +# RUN: chmod +x %t-i386.o +# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t-universal.o +# RUN: test -x %t-universal.o Index: llvm/trunk/test/tools/llvm-lipo/create-invalid-input.test =================================================================== --- llvm/trunk/test/tools/llvm-lipo/create-invalid-input.test +++ llvm/trunk/test/tools/llvm-lipo/create-invalid-input.test @@ -0,0 +1,8 @@ +# RUN: yaml2obj %p/Inputs/i386-slice.yaml > %t-32.o +# RUN: yaml2obj %p/Inputs/i386-x86_64-universal.yaml > %t-universal.o + +# RUN: not llvm-lipo %t-32.o -create 2>&1 | FileCheck --check-prefix=NO_OUTPUT %s +# NO_OUTPUT: error: create expects a single output file to be specified + +# RUN: not llvm-lipo %t-universal.o %t-32.o -create -output %t.o 2>&1 | FileCheck --check-prefix=DUPLICATE_ARCHS %s +# DUPLICATE_ARCHS: have the same architecture i386 and therefore cannot be in the same universal binary Index: llvm/trunk/test/tools/llvm-lipo/create-without-alignment.test =================================================================== --- llvm/trunk/test/tools/llvm-lipo/create-without-alignment.test +++ llvm/trunk/test/tools/llvm-lipo/create-without-alignment.test @@ -0,0 +1,32 @@ +# RUN: yaml2obj %p/Inputs/i386-slice.yaml > %t-i386.o +# RUN: yaml2obj %p/Inputs/x86_64-slice.yaml > %t-x86_64.o + +# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t-universal-llvm.o + +# RUN: yaml2obj %p/Inputs/i386-x86_64-universal.yaml > %t-universal.o +# RUN: cmp %t-universal-llvm.o %t-universal.o + +# RUN: yaml2obj %p/Inputs/armv7-slice.yaml > %t-armv7.o +# RUN: yaml2obj %p/Inputs/arm64-slice.yaml > %t-arm64.o + +# RUN: llvm-lipo %t-arm64.o %t-armv7.o %t-universal.o -create -output %t-universal-2.o +# RUN: llvm-lipo %t-universal-2.o -thin x86_64 -output %t-x86_64_extracted.o +# RUN: cmp %t-x86_64_extracted.o %t-x86_64.o +# RUN: llvm-lipo %t-universal-2.o -thin armv7 -output %t-armv7-extracted.o +# RUN: cmp %t-armv7-extracted.o %t-armv7.o + +# RUN: llvm-objdump %t-universal-2.o -m --universal-headers | FileCheck %s +# CHECK: fat_magic FAT_MAGIC +# CHECK: nfat_arch 4 +# CHECK: architecture i386 +# CHECK: offset 4096 +# CHECK: align 2^12 (4096) +# CHECK: architecture x86_64 +# CHECK: offset 8192 +# CHECK: align 2^12 (4096) +# CHECK: architecture armv7 +# CHECK: offset 16384 +# CHECK: align 2^14 (16384) +# CHECK: architecture arm64 +# CHECK: offset 32768 +# CHECK: align 2^14 (16384) Index: llvm/trunk/test/tools/llvm-lipo/thin-executable-universal-binary.test =================================================================== --- llvm/trunk/test/tools/llvm-lipo/thin-executable-universal-binary.test +++ llvm/trunk/test/tools/llvm-lipo/thin-executable-universal-binary.test @@ -4,7 +4,7 @@ # RUN: chmod -x %t-universal.o # RUN: llvm-lipo %t-universal.o -thin i386 -output %t32.o -# RUN: test ! -x %t32.o +# RUN: ! test -x %t32.o # RUN: chmod +x %t-universal.o # RUN: llvm-lipo %t-universal.o -thin i386 -output %t32-ex.o Index: llvm/trunk/tools/llvm-lipo/LipoOpts.td =================================================================== --- llvm/trunk/tools/llvm-lipo/LipoOpts.td +++ llvm/trunk/tools/llvm-lipo/LipoOpts.td @@ -23,6 +23,11 @@ HelpText<"Create a thin output file of specified arch_type from the " "fat input file. Requires -output option">; +def create : Option<["-", "--"], "create", KIND_FLAG>, + Group, + HelpText<"Create a universal binary output file from the input " + "files. Requires -output option">; + def output : Option<["-", "--"], "output", KIND_SEPARATE>, HelpText<"Create output file with specified name">; def o : JoinedOrSeparate<["-"], "o">, Alias; Index: llvm/trunk/tools/llvm-lipo/llvm-lipo.cpp =================================================================== --- llvm/trunk/tools/llvm-lipo/llvm-lipo.cpp +++ llvm/trunk/tools/llvm-lipo/llvm-lipo.cpp @@ -80,6 +80,7 @@ PrintArchs, VerifyArch, ThinArch, + CreateUniversal, }; struct Config { @@ -90,6 +91,14 @@ LipoAction ActionToPerform; }; +struct Slice { + const MachOObjectFile *ObjectFile; + // Requires Alignment field to store slice alignment values from universal + // binaries. Also needed to order the slices using compareSlices, so the total + // file size can be calculated before creating the output buffer. + uint32_t Alignment; +}; + } // end namespace static void validateArchitectureName(StringRef ArchitectureName) { @@ -108,7 +117,7 @@ Config C; LipoOptTable T; unsigned MissingArgumentIndex, MissingArgumentCount; - llvm::opt::InputArgList InputArgs = + opt::InputArgList InputArgs = T.ParseArgs(ArgsArr, MissingArgumentIndex, MissingArgumentCount); if (MissingArgumentCount) @@ -186,6 +195,12 @@ C.ActionToPerform = LipoAction::ThinArch; return C; + case LIPO_create: + if (C.OutputFile.empty()) + reportError("create expects a single output file to be specified"); + C.ActionToPerform = LipoAction::CreateUniversal; + return C; + default: reportError("llvm-lipo action unspecified"); } @@ -195,8 +210,7 @@ readInputBinaries(ArrayRef InputFiles) { SmallVector, 1> InputBinaries; for (StringRef InputFile : InputFiles) { - Expected> BinaryOrErr = - createBinary(InputFile); + Expected> BinaryOrErr = createBinary(InputFile); if (!BinaryOrErr) reportError(InputFile, BinaryOrErr.takeError()); // TODO: Add compatibility for archive files @@ -241,33 +255,35 @@ exit(EXIT_SUCCESS); } -static void printArchOrUnknown(const MachOObjectFile *ObjectFile) { - // Prints trailing space and unknown in this format for compatibility with - // cctools lipo. - const std::string ObjectArch = ObjectFile->getArchTriple().getArchName(); - if (ObjectArch.empty()) - outs() << "unknown(" << ObjectFile->getHeader().cputype << "," - << ObjectFile->getHeader().cpusubtype << ") "; - else - outs() << ObjectArch + " "; +// Returns a string of the given Object file's architecture type +// Unknown architectures formatted unknown(CPUType,CPUSubType) for compatibility +// with cctools lipo +static std::string getArchString(const MachOObjectFile &ObjectFile) { + const Triple T = ObjectFile.getArchTriple(); + const StringRef ObjectArch = T.getArchName(); + if (!ObjectArch.empty()) + return ObjectArch; + return ("unknown(" + Twine(ObjectFile.getHeader().cputype) + "," + + Twine(ObjectFile.getHeader().cpusubtype & ~MachO::CPU_SUBTYPE_MASK) + + ")") + .str(); } LLVM_ATTRIBUTE_NORETURN static void printArchs(ArrayRef> InputBinaries) { + // Prints trailing space for compatibility with cctools lipo. assert(InputBinaries.size() == 1 && "Incorrect number of input binaries"); const Binary *InputBinary = InputBinaries.front().getBinary(); if (auto UO = dyn_cast(InputBinary)) { - for (MachOUniversalBinary::object_iterator I = UO->begin_objects(), - E = UO->end_objects(); - I != E; ++I) { + for (const auto &O : UO->objects()) { Expected> BinaryOrError = - I->getAsObjectFile(); + O.getAsObjectFile(); if (!BinaryOrError) reportError(InputBinary->getFileName(), BinaryOrError.takeError()); - printArchOrUnknown(BinaryOrError.get().get()); + outs() << getArchString(*BinaryOrError.get().get()) << " "; } } else if (auto O = dyn_cast(InputBinary)) { - printArchOrUnknown(O); + outs() << getArchString(*O) << " "; } else { llvm_unreachable("Unexpected binary format"); } @@ -314,6 +330,173 @@ exit(EXIT_SUCCESS); } +static void checkArchDuplicates(const ArrayRef &Slices) { + DenseMap CPUIds; + auto CPUIDForSlice = [](const Slice &S) { + return static_cast(S.ObjectFile->getHeader().cputype) << 32 | + S.ObjectFile->getHeader().cpusubtype; + }; + for (const auto &S : Slices) { + auto Entry = CPUIds.try_emplace(CPUIDForSlice(S), S.ObjectFile); + if (!Entry.second) + reportError(Entry.first->second->getFileName() + " and " + + S.ObjectFile->getFileName() + " have the same architecture " + + getArchString(*S.ObjectFile) + + " and therefore cannot be in the same universal binary"); + } +} + +static uint32_t calculateAlignment(const MachOObjectFile *ObjectFile) { + // TODO: Implement getAlign() and remove hard coding + // Will be implemented in a follow-up. + + switch (ObjectFile->getHeader().cputype) { + case MachO::CPU_TYPE_I386: + case MachO::CPU_TYPE_X86_64: + case MachO::CPU_TYPE_POWERPC: + case MachO::CPU_TYPE_POWERPC64: + return 12; // log2 value of page size(4k) for x86 and PPC + case MachO::CPU_TYPE_ARM: + case MachO::CPU_TYPE_ARM64: + case MachO::CPU_TYPE_ARM64_32: + return 14; // log2 value of page size(16k) for Darwin ARM + default: + return 12; + } +} + +// This function replicates ordering from cctools lipo for consistency +static bool compareSlices(const Slice &Lhs, const Slice &Rhs) { + if (Lhs.ObjectFile->getHeader().cputype == + Rhs.ObjectFile->getHeader().cputype) + return Lhs.ObjectFile->getHeader().cpusubtype < + Rhs.ObjectFile->getHeader().cpusubtype; + + // force arm64-family to follow after all other slices for compatibility + // with cctools lipo + if (Lhs.ObjectFile->getHeader().cputype == MachO::CPU_TYPE_ARM64) + return false; + if (Rhs.ObjectFile->getHeader().cputype == MachO::CPU_TYPE_ARM64) + return true; + + // Sort by alignment to minimize file size + return Lhs.Alignment < Rhs.Alignment; +} + +// Updates vector ExtractedObjects with the MachOObjectFiles extracted from +// Universal Binary files to transfer ownership. +static SmallVector buildSlices( + ArrayRef> InputBinaries, + SmallVectorImpl> &ExtractedObjects) { + SmallVector Slices; + for (auto &IB : InputBinaries) { + const Binary *InputBinary = IB.getBinary(); + if (auto UO = dyn_cast(InputBinary)) { + for (const auto &O : UO->objects()) { + Expected> BinaryOrError = + O.getAsObjectFile(); + if (!BinaryOrError) + reportError(InputBinary->getFileName(), BinaryOrError.takeError()); + ExtractedObjects.push_back(std::move(BinaryOrError.get())); + Slices.push_back(Slice{ExtractedObjects.back().get(), O.getAlign()}); + } + } else if (auto O = dyn_cast(InputBinary)) { + Slices.push_back(Slice{O, calculateAlignment(O)}); + } else { + llvm_unreachable("Unexpected binary format"); + } + } + return Slices; +} + +static SmallVector +buildFatArchList(ArrayRef Slices) { + SmallVector FatArchList; + uint64_t Offset = + sizeof(MachO::fat_header) + Slices.size() * sizeof(MachO::fat_arch); + + for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) { + Offset = alignTo(Offset, 1 << Slices[Index].Alignment); + const MachOObjectFile *ObjectFile = Slices[Index].ObjectFile; + if (Offset > UINT32_MAX) + reportError("fat file too large to be created because the offset " + "field in struct fat_arch is only 32-bits and the offset " + + Twine(Offset) + " for " + ObjectFile->getFileName() + + " for architecture " + getArchString(*ObjectFile) + + "exceeds that."); + + MachO::fat_arch FatArch; + FatArch.cputype = ObjectFile->getHeader().cputype; + FatArch.cpusubtype = ObjectFile->getHeader().cpusubtype; + FatArch.offset = Offset; + FatArch.size = ObjectFile->getMemoryBufferRef().getBufferSize(); + FatArch.align = Slices[Index].Alignment; + Offset += FatArch.size; + FatArchList.push_back(FatArch); + } + return FatArchList; +} + +static void createUniversalBinary(SmallVectorImpl &Slices, + StringRef OutputFileName) { + MachO::fat_header FatHeader; + FatHeader.magic = MachO::FAT_MAGIC; + FatHeader.nfat_arch = Slices.size(); + + stable_sort(Slices, compareSlices); + SmallVector FatArchList = buildFatArchList(Slices); + + const bool IsExecutable = any_of(Slices, [](Slice S) { + return sys::fs::can_execute(S.ObjectFile->getFileName()); + }); + const uint64_t OutputFileSize = + FatArchList.back().offset + FatArchList.back().size; + Expected> OutFileOrError = + FileOutputBuffer::create(OutputFileName, OutputFileSize, + IsExecutable ? FileOutputBuffer::F_executable + : 0); + if (!OutFileOrError) + reportError(OutputFileName, OutFileOrError.takeError()); + std::unique_ptr OutFile = std::move(OutFileOrError.get()); + std::memset(OutFile->getBufferStart(), 0, OutputFileSize); + + if (sys::IsLittleEndianHost) + MachO::swapStruct(FatHeader); + std::memcpy(OutFile->getBufferStart(), &FatHeader, sizeof(MachO::fat_header)); + + for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) { + MemoryBufferRef BufferRef = Slices[Index].ObjectFile->getMemoryBufferRef(); + std::copy(BufferRef.getBufferStart(), BufferRef.getBufferEnd(), + OutFile->getBufferStart() + FatArchList[Index].offset); + } + + // FatArchs written after Slices in order reduce the number of swaps for the + // LittleEndian case + if (sys::IsLittleEndianHost) + for (MachO::fat_arch &FA : FatArchList) + MachO::swapStruct(FA); + std::memcpy(OutFile->getBufferStart() + sizeof(MachO::fat_header), + FatArchList.begin(), + sizeof(MachO::fat_arch) * FatArchList.size()); + + if (Error E = OutFile->commit()) + reportError(OutputFileName, std::move(E)); +} + +LLVM_ATTRIBUTE_NORETURN +static void createUniversalBinary(ArrayRef> InputBinaries, + StringRef OutputFileName) { + assert(InputBinaries.size() >= 1 && "Incorrect number of input binaries"); + assert(!OutputFileName.empty() && "Create expects a single output file"); + + SmallVector, 1> ExtractedObjects; + SmallVector Slices = buildSlices(InputBinaries, ExtractedObjects); + checkArchDuplicates(Slices); + createUniversalBinary(Slices, OutputFileName); + + exit(EXIT_SUCCESS); +} + int main(int argc, char **argv) { InitLLVM X(argc, argv); Config C = parseLipoOptions(makeArrayRef(argv + 1, argc)); @@ -330,6 +513,9 @@ case LipoAction::ThinArch: extractSlice(InputBinaries, C.ThinArchType, C.OutputFile); break; + case LipoAction::CreateUniversal: + createUniversalBinary(InputBinaries, C.OutputFile); + break; } return EXIT_SUCCESS; }