[dsymutil] Add the ability to generate universal binaries with a fat64 header

[JDevlieghere]

Summary

Add the ability to generate universal binaries with a fat64 header.

rdar://107223939

[JDevlieghere]

Limit test to Darwin as it relies on lipo

[pete]

LGTM

[aprantl]

I understand that auto-detection can be a double-edged sword, but — should this be autodetected based on the input size? Or this impossible because we only know the sizer after it's too late?

llvm/tools/dsymutil/dsymutil.cpp

787

typo: create

[JDevlieghere]

In D146879#4224622, @aprantl wrote:

I understand that auto-detection can be a double-edged sword, but — should this be autodetected based on the input size? Or this impossible because we only know the sizer after it's too late?

Given that the tooling support isn't quite there yet, I think this should be opt-in, but I don't mind making this the default if the output size will exceed 4GB in the future.

llvm/docs/CommandGuide/dsymutil.rst

  architectures will be linked by default and any architectures that can't be
  properly linked will cause :program:`dsymutil` to return an error.
 
 .. option:: --dump-debug-map
 
  Dump the *executable*'s debug-map (the list of the object files containing the
  debug information) in YAML format and exit. No DWARF link will take place.
 
+.. option:: --fat64
+
+ Use a 64-bit header when emitting universal binaries.
+
 .. option:: --flat, -f
 
  Produce a flat dSYM file. A ``.dwarf`` extension will be appended to the
  executable name unless the output file is specified using the ``-o`` option.
 
 .. option:: --gen-reproducer
 
  Generate a reproducer consisting of the input object files. Alias for

llvm/test/tools/dsymutil/cmdline.test

 RUN: dsymutil -help 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=HELP %s
 RUN: cat %S/../../../docs/CommandGuide/dsymutil.rst  | FileCheck %s
 
 HELP: OVERVIEW: manipulate archived DWARF debug symbol files.
 HELP: USAGE: {{.*}}dsymutil{{[^ ]*}} [options] <input files>
 HELP-NOT: -reverse-iterate
 HELP: Dsymutil Options:
 CHECK: -accelerator
 CHECK: -arch <arch>
 CHECK: -dump-debug-map
+CHECK: -fat64
 CHECK: -flat
 CHECK: -gen-reproducer
 CHECK: -help
 CHECK: -keep-function-for-static
 CHECK: -no-odr
 CHECK: -no-output
 CHECK: -no-swiftmodule-timestamp
 CHECK: -num-threads  <threads>

llvm/test/tools/dsymutil/fat-header.test

+REQUIRES: system-darwin
+
+RUN: dsymutil -oso-prepend-path %p %p/Inputs/fat-test.arm.dylib -o %t.fat32.dSYM
+RUN: llvm-objdump -m --universal-headers %t.fat32.dSYM/Contents/Resources/DWARF/fat-test.arm.dylib | FileCheck %s -check-prefixes=FAT32
+
+RUN: dsymutil -oso-prepend-path %p %p/Inputs/fat-test.arm.dylib -o %t.fat64.dSYM -fat64
+RUN: llvm-objdump -m --universal-headers %t.fat64.dSYM/Contents/Resources/DWARF/fat-test.arm.dylib | FileCheck %s -check-prefixes=FAT64
+
+FAT32: fat_magic FAT_MAGIC
+FAT64: fat_magic FAT_MAGIC_64

llvm/tools/dsymutil/LinkUtils.h

struct LinkOptions {
 
   /// Do not check swiftmodule timestamp
   bool NoTimestamp = false;
 
   /// Whether we want a static variable to force us to keep its enclosing
   /// function.
   bool KeepFunctionForStatic = false;
 
+  /// Use a 64-bit header when emitting universal binaries.
+  bool Fat64 = false;
+
   /// Number of threads.
   unsigned Threads = 1;
 
   // Output file type.
   OutputFileType FileType = OutputFileType::Object;
 
   /// The accelerator table kind
   DsymutilAccelTableKind TheAccelTableKind;

llvm/tools/dsymutil/MachOUtils.h

struct DwarfRelocationApplicationInfo {
   DwarfRelocationApplicationInfo(uint64_t AddressFromDwarfVM, uint32_t Value,
                                  bool ShouldSubtractDwarfVM)
       : AddressFromDwarfStart(AddressFromDwarfVM), Value(Value),
         ShouldSubtractDwarfVM(ShouldSubtractDwarfVM) {}
 };
 
 bool generateUniversalBinary(SmallVectorImpl<ArchAndFile> &ArchFiles,
                              StringRef OutputFileName, const LinkOptions &,
-                             StringRef SDKPath);
+                             StringRef SDKPath, bool Fat64 = false);
 bool generateDsymCompanion(
     llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS, const DebugMap &DM,
     SymbolMapTranslator &Translator, MCStreamer &MS, raw_fd_ostream &OutFile,
     const std::vector<MachOUtils::DwarfRelocationApplicationInfo>
         &RelocationsToApply);
 
 std::string getArchName(StringRef Arch);
 } // namespace MachOUtils
 } // namespace dsymutil
 } // namespace llvm
 #endif // LLVM_TOOLS_DSYMUTIL_MACHOUTILS_H

llvm/tools/dsymutil/MachOUtils.cpp

if (result) {
     return false;
   }
 
   return true;
 }
 
 bool generateUniversalBinary(SmallVectorImpl<ArchAndFile> &ArchFiles,
                              StringRef OutputFileName,
-                             const LinkOptions &Options, StringRef SDKPath) {
+                             const LinkOptions &Options, StringRef SDKPath,
+                             bool Fat64) {
   // No need to merge one file into a universal fat binary.
   if (ArchFiles.size() == 1) {
     if (auto E = ArchFiles.front().File->keep(OutputFileName)) {
       WithColor::error() << "while keeping " << ArchFiles.front().path()
                          << " as " << OutputFileName << ": "
                          << toString(std::move(E)) << "\n";
       return false;
     }
     return true;
   }
 
   SmallVector<StringRef, 8> Args;
   Args.push_back("lipo");
   Args.push_back("-create");
 
   for (auto &Thin : ArchFiles)
     Args.push_back(Thin.path());
 
-  // Align segments to match dsymutil-classic alignment
+  // Align segments to match dsymutil-classic alignment.
   for (auto &Thin : ArchFiles) {
     Thin.Arch = getArchName(Thin.Arch);
     Args.push_back("-segalign");
     Args.push_back(Thin.Arch);
     Args.push_back("20");
   }
 
+  // Use a 64-bit fat header if requested.
+  if (Fat64)
+    Args.push_back("-fat64");
+
   Args.push_back("-output");
   Args.push_back(OutputFileName.data());
 
   if (Options.Verbose) {
     outs() << "Running lipo\n";
     for (auto Arg : Args)
       outs() << ' ' << Arg;
     outs() << "\n";

llvm/tools/dsymutil/Options.td

 def flat: F<"flat">,
   HelpText<"Produce a flat dSYM file (not a bundle).">,
   Group<grp_general>;
 def: Flag<["-"], "f">,
   Alias<flat>,
   HelpText<"Alias for --flat">,
   Group<grp_general>;
 
+def fat64: F<"fat64">,
+  HelpText<"Use a 64-bit header when emitting universal binaries.">,
+  Group<grp_general>;
+
 def update: F<"update">,
   HelpText<"Updates existing dSYM files to contain the latest accelerator tables and other DWARF optimizations.">,
   Group<grp_general>;
 def: Flag<["-"], "u">,
   Alias<update>,
   HelpText<"Alias for --update">,
   Group<grp_general>;
 

llvm/tools/dsymutil/dsymutil.cpp

static Expected<DsymutilOptions> getOptions(opt::InputArgList &Args) {
   Options.LinkOpts.NoODR = Args.hasArg(OPT_no_odr);
   Options.LinkOpts.VerifyInputDWARF =
       flagIsSet(Options.Verify, DWARFVerify::Input);
   Options.LinkOpts.NoOutput = Args.hasArg(OPT_no_output);
   Options.LinkOpts.NoTimestamp = Args.hasArg(OPT_no_swiftmodule_timestamp);
   Options.LinkOpts.Update = Args.hasArg(OPT_update);
   Options.LinkOpts.Verbose = Args.hasArg(OPT_verbose);
   Options.LinkOpts.Statistics = Args.hasArg(OPT_statistics);
+  Options.LinkOpts.Fat64 = Args.hasArg(OPT_fat64);
   Options.LinkOpts.KeepFunctionForStatic =
       Args.hasArg(OPT_keep_func_for_static);
 
   if (opt::Arg *ReproducerPath = Args.getLastArg(OPT_use_reproducer)) {
     Options.ReproMode = ReproducerMode::Use;
     Options.ReproducerPath = ReproducerPath->getValue();
   } else {
     if (Expected<ReproducerMode> ReproMode = getReproducerMode(Args)) {
for (auto &InputFile : Options.InputFiles) {
 
     if (Crashed)
       (*Repro)->generate();
 
     if (!AllOK)
       return EXIT_FAILURE;
 
     if (NeedsTempFiles) {
+      const bool Fat64 = Options.LinkOpts.Fat64;
+      if (!Fat64) {
         // Universal Mach-O files can't have an archicture slice that starts
-      // beyond the 4GB boundary. "lipo" can creeate a 64 bit universal header,
-      // but not all tools can parse these files so we want to return an error
-      // if the file can't be encoded as a file with a 32 bit universal header.
-      // To detect this, we check the size of each architecture's skinny Mach-O
-      // file and add up the offsets. If they exceed 4GB, then we return an
-      // error.
+        // beyond the 4GB boundary. "lipo" can create a 64 bit universal
+        // header, but not all tools can parse these files so we want to return
+        // an error if the file can't be encoded as a file with a 32 bit
+        // universal header. To detect this, we check the size of each
+        // architecture's skinny Mach-O file and add up the offsets. If they
+        // exceed 4GB, then we return an error.
 
-      // First we compute the right offset where the first architecture will fit
-      // followin the 32 bit universal header. The 32 bit universal header
+        // First we compute the right offset where the first architecture will
+        // fit followin the 32 bit universal header. The 32 bit universal header
         // starts with a uint32_t magic and a uint32_t number of architecture
-      // infos. Then it is followed by 5 uint32_t values for each architecture.
-      // So we set the start offset to the right value so we can calculate the
-      // exact offset that the first architecture slice can start at.
+        // infos. Then it is followed by 5 uint32_t values for each
+        // architecture. So we set the start offset to the right value so we can
+        // calculate the exact offset that the first architecture slice can
+        // start at.
         constexpr uint64_t MagicAndCountSize = 2 * 4;
         constexpr uint64_t UniversalArchInfoSize = 5 * 4;
-      uint64_t FileOffset = MagicAndCountSize +
-          UniversalArchInfoSize * TempFiles.size();
-      for (const auto &File: TempFiles) {
+        uint64_t FileOffset =
+            MagicAndCountSize + UniversalArchInfoSize * TempFiles.size();
+        for (const auto &File : TempFiles) {
           ErrorOr<vfs::Status> stat = Options.LinkOpts.VFS->status(File.path());
           if (!stat)
             break;
           if (FileOffset > UINT32_MAX) {
-          WithColor::error() << formatv(
-              "the universal binary has a slice with a starting offset ({0:x}) "
-              "that exceeds 4GB and will produce an invalid Mach-O file.",
+            WithColor::error()
+                << formatv("the universal binary has a slice with a starting "
+                           "offset ({0:x}) that exceeds 4GB and will produce "
+                           "an invalid Mach-O file. Use the -fat64 flag to "
+                           "generate a universal binary with a 64-bit header "
+                           "but note that not all tools support this format.",
                            FileOffset);
             return EXIT_FAILURE;
           }
           FileOffset += stat->getSize();
         }
-      if (!MachOUtils::generateUniversalBinary(TempFiles,
-                                               OutputLocationOrErr->DWARFFile,
-                                               Options.LinkOpts, SDKPath))
+      }
+      if (!MachOUtils::generateUniversalBinary(
+              TempFiles, OutputLocationOrErr->DWARFFile, Options.LinkOpts,
+              SDKPath, Fat64))
         return EXIT_FAILURE;
     }
   }
 
   return EXIT_SUCCESS;
 }