[AIX] Support of Big archive (write)

[EGuesnet]

Summary

This patch add write support for Big Archive. Patch D100651 is needed. Global symbol table is not supported.

[EGuesnet]

Updating D104367: [AIX] Support of Big archive (write)

Comment and whitespace.

[DiggerLin]

please use clang-format to format the patch.

[DiggerLin]

would you like to rebase the patch based on the https://reviews.llvm.org/D111889([AIX] Support of Big archive (read)) ?, since the patch https://reviews.llvm.org/D111889 landed.

llvm/lib/Object/ArchiveWriter.cpp

228

the terminator is "0x60 0x0a".

362

the value of argument of PrevOffset is not zero, it point to the "Member Table"

[EGuesnet]

Hi @DiggerLin , I do not work on LLVM anymore. Please contact Sanket for more information.

[jhenderson]

@DiggerLin, I think given @EGuesnet is no longer working on this, you should feel free to commandeer this patch, rebase it and amend it as you see fit. Alternatively, start a new patch and @EGuesnet should then abandon this one.

[DiggerLin]

I took over the patch and create a patch https://reviews.llvm.org/D123949. thanks for your work. @EGuesnet .

llvm/lib/Object/ArchiveWriter.cpp

static void printWithSpacePadding(raw_ostream &OS, T Data, unsigned Size) {
   OS.indent(Size - SizeSoFar);
 }
 
 static bool isDarwin(object::Archive::Kind Kind) {
   return Kind == object::Archive::K_DARWIN ||
          Kind == object::Archive::K_DARWIN64;
 }
 
+static bool isAIX(object::Archive::Kind Kind) {
+  return Kind == object::Archive::K_XCOFF;
+}
+
 static bool isBSDLike(object::Archive::Kind Kind) {
   switch (Kind) {
   case object::Archive::K_GNU:
   case object::Archive::K_GNU64:
+  case object::Archive::K_XCOFF:
     return false;
   case object::Archive::K_BSD:
   case object::Archive::K_DARWIN:
   case object::Archive::K_DARWIN64:
     return true;
   case object::Archive::K_COFF:
     break;
   }
   llvm_unreachable("not supported for writting");
 }
 
 template <class T>
 static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val) {
   support::endian::write(Out, Val,
printBSDMemberHeader(raw_ostream &Out, uint64_t Pos, StringRef Name,
   printWithSpacePadding(Out, Twine("#1/") + Twine(NameWithPadding), 16);
   printRestOfMemberHeader(Out, ModTime, UID, GID, Perms,
                           NameWithPadding + Size);
   Out << Name;
   while (Pad--)
     Out.write(uint8_t(0));
 }
 
+static void
+printBigMemberHeader(raw_ostream &Out, uint64_t Pos, StringRef Name,
+                     const sys::TimePoint<std::chrono::seconds> &ModTime,
+                     unsigned UID, unsigned GID, unsigned Perms, uint64_t Size,
+         unsigned PrevOffset) {
+  // Big Archive has a different size for element of Member Header
+  // We cannot use printRestOfMemberHeader.
+  // Name is written at the end of the archive,
+  // followed by the cosmetic terminator "`\n".
+  unsigned NameLen = Name.size();
+  
+  // Position after header is: Offset + ObjectHeaderSize(114) + 
+  // + name length with padding + object size with padding 
+  // If no name, special case: Member table; next member is Global,
+  // currently unsupported, so print 0. TODO in future.
+  unsigned PosAfterHeader = 0;
+  if (NameLen)
+    PosAfterHeader = Pos + 114 + NameLen + NameLen % 2 + Size + Size % 2;
+  
+  printWithSpacePadding(Out, Size, 20); // File member size
+  printWithSpacePadding(Out, PosAfterHeader, 20); // Next member offset
+  printWithSpacePadding(Out, PrevOffset, 20); // Previous member header
+  printWithSpacePadding(Out, sys::toTimeT(ModTime), 12); // File member date
+  printWithSpacePadding(Out, UID % 1000000000000, 12); // UID
+  printWithSpacePadding(Out, GID % 1000000000000, 12); // GID
+  printWithSpacePadding(Out, format("%o", Perms), 12); // Permision
+  printWithSpacePadding(Out, NameLen, 4); // Name length
+  if (NameLen)
+    printWithSpacePadding(Out, Name, NameLen); // Name
+  if (NameLen % 2)
+    Out << (char)0; // Padding with null caracter
+  printWithSpacePadding(Out, "`\n", 2); // Terminator
+}
+
 static bool useStringTable(bool Thin, StringRef Name) {
   return Thin || Name.size() >= 16 || Name.contains('/');
 }
 
 static bool is64BitKind(object::Archive::Kind Kind) {
   switch (Kind) {
   case object::Archive::K_GNU:
   case object::Archive::K_BSD:
   case object::Archive::K_DARWIN:
   case object::Archive::K_COFF:
+  case object::Archive::K_XCOFF:
     return false;
   case object::Archive::K_DARWIN64:
   case object::Archive::K_GNU64:
     return true;
   }
   llvm_unreachable("not supported for writting");
 }
 

 }
 
 static void writeSymbolTableHeader(raw_ostream &Out, object::Archive::Kind Kind,
                                    bool Deterministic, uint64_t Size) {
   if (isBSDLike(Kind)) {
     const char *Name = is64BitKind(Kind) ? "__.SYMDEF_64" : "__.SYMDEF";
     printBSDMemberHeader(Out, Out.tell(), Name, now(Deterministic), 0, 0, 0,
                          Size);
+  } else if (isAIX(Kind)) {
+    const char *Name = "";
+    printBigMemberHeader(Out, Out.tell(), Name, now(Deterministic),
+                     0, 0, 0, Size, 0);
   } else {
     const char *Name = is64BitKind(Kind) ? "/SYM64" : "";
     printGNUSmallMemberHeader(Out, Name, now(Deterministic), 0, 0, 0, Size);
   }
 }
 
 static void writeSymbolTable(raw_ostream &Out, object::Archive::Kind Kind,
                              bool Deterministic, ArrayRef<MemberData> Members,
computeMemberData(raw_ostream &StringTable, raw_ostream &SymNames,
   std::map<StringRef, unsigned> FilenameCount;
   if (UniqueTimestamps) {
     for (const NewArchiveMember &M : NewMembers)
       FilenameCount[M.MemberName]++;
     for (auto &Entry : FilenameCount)
       Entry.second = Entry.second > 1 ? 1 : 0;
   }
 
+  // Big Archive needs to know offset and offset of the previous symbol
+  unsigned PrevOffset = 0;
   for (const NewArchiveMember &M : NewMembers) {
     std::string Header;
     raw_string_ostream Out(Header);
 
     MemoryBufferRef Buf = M.Buf->getMemBufferRef();
     StringRef Data = Thin ? "" : Buf.getBuffer();
 
     // ld64 expects the members to be 8-byte aligned for 64-bit content and at
for (const NewArchiveMember &M : NewMembers) {
     uint64_t Size = Buf.getBufferSize() + MemberPadding;
     if (Size > object::Archive::MaxMemberSize) {
       std::string StringMsg =
           "File " + M.MemberName.str() + " exceeds size limit";
       return make_error<object::GenericBinaryError>(
           std::move(StringMsg), object::object_error::parse_failed);
     }
 
+    // Big Archive needs more offset, we cannot use printMemberHeader.
+    if (isAIX(Kind)) {
+      if (Pos == 0) {
+        // For the first object, Fixed Length Header size
+  // must be added to offset.
+        Pos += 128;
+      }
+      printBigMemberHeader(Out, Pos, M.MemberName, ModTime, M.UID, M.GID,
+                                M.Perms, Size, PrevOffset);
+      PrevOffset = Pos;
+    } else
       printMemberHeader(Out, Pos, StringTable, MemberNames, Kind, Thin, M,
                       ModTime, Size);
     Out.flush();
 
     std::vector<unsigned> Symbols;
     if (NeedSymbols) {
       Expected<std::vector<unsigned>> SymbolsOrErr =
           getSymbols(Buf, SymNames, HasObject);
       if (auto E = SymbolsOrErr.takeError())
         return std::move(E);
Expected<std::vector<MemberData>> DataOrErr =
                         WriteSymtab, NewMembers);
   if (Error E = DataOrErr.takeError())
     return E;
   std::vector<MemberData> &Data = *DataOrErr;
 
   if (!StringTableBuf.empty())
     Data.insert(Data.begin(), computeStringTable(StringTableBuf));
 
+  // For BigArchive (AIX), compute a table of names and offset, 
+  // used in Member Table.
+  uint64_t NamesSize = 0; // Total Length of names + NULL (without padding).
+  std::vector<int> OffsetArray;
+  std::vector<StringRef> NameArray;
+  if (isAIX(Kind)) {
+    // Loop across object to find offset and names.
+    for (size_t it = 0; it != NewMembers.size(); ++it) {
+      NamesSize += NewMembers[it].MemberName.size() + 1;
+      // First member, offset is 128
+      if (it == 0)
+        OffsetArray.push_back(128);
+      // If current member is not last, fulfill offset of next member
+      // Curent offset + Header size (114) + Object size + Name length + padding for name
+      if (it < NewMembers.size() - 1)
+  OffsetArray.push_back(OffsetArray[it] + 114 + Data[it].Data.size() + NewMembers[it].MemberName.size() + NewMembers[it].MemberName.size() % 2);
+      // Get the game
+      NameArray.push_back(NewMembers[it].MemberName);
+    }
+  }
+
   // We would like to detect if we need to switch to a 64-bit symbol table.
-  if (WriteSymtab) {
-    uint64_t MaxOffset = 8; // For the file signature.
-    uint64_t LastOffset = MaxOffset;
+
+  if (WriteSymtab) {
     uint64_t NumSyms = 0;
     for (const auto &M : Data) {
       // Record the start of the member's offset
       LastOffset = MaxOffset;
       // Account for the size of each part associated with the member.
       MaxOffset += M.Header.size() + M.Data.size() + M.Padding.size();
       NumSyms += M.Symbols.size();
     }
 
     // We assume 32-bit offsets to see if 32-bit symbols are possible or not.
     uint64_t SymtabSize = computeSymbolTableSize(Kind, NumSyms, 4, SymNamesBuf);
     auto computeSymbolTableHeaderSize =
         [=] {
           SmallString<0> TmpBuf;
           raw_svector_ostream Tmp(TmpBuf);
           writeSymbolTableHeader(Tmp, Kind, Deterministic, SymtabSize);
           return TmpBuf.size();
         };
+    if (!(isAIX(Kind)))
       LastOffset += computeSymbolTableHeaderSize() + SymtabSize;
 
     // The SYM64 format is used when an archive's member offsets are larger than
     // 32-bits can hold. The need for this shift in format is detected by
     // writeArchive. To test this we need to generate a file with a member that
     // has an offset larger than 32-bits but this demands a very slow test. To
     // speed the test up we use this environment variable to pretend like the
     // cutoff happens before 32-bits and instead happens at some much smaller
     // value.
     uint64_t Sym64Threshold = 1ULL << 32;
     const char *Sym64Env = std::getenv("SYM64_THRESHOLD");
     if (Sym64Env)
       StringRef(Sym64Env).getAsInteger(10, Sym64Threshold);
 
     // If LastOffset isn't going to fit in a 32-bit varible we need to switch
     // to 64-bit. Note that the file can be larger than 4GB as long as the last
     // member starts before the 4GB offset.
+    // Big Archive not supported yet.
     if (LastOffset >= Sym64Threshold) {
       if (Kind == object::Archive::K_DARWIN)
         Kind = object::Archive::K_DARWIN64;
       else
         Kind = object::Archive::K_GNU64;
     }
   }
 
   if (Thin)
     Out << "!<thin>\n";
+  else if (isAIX(Kind))
+    Out << "<bigaf>\n";
   else
     Out << "!<arch>\n";
 
+  if (!isAIX(Kind)) {
     if (WriteSymtab)
       writeSymbolTable(Out, Kind, Deterministic, Data, SymNamesBuf);
-
     for (const MemberData &M : Data)
       Out << M.Header << M.Data << M.Padding;
+  } else {
+    // Fix Sized Header
+    unsigned FixedHeaderSize = 120;
+    unsigned MemberTableOffset = MaxOffset + FixedHeaderSize;
+    // If there is a global symbol table, offset will be:
+    // unsigned GlobalSymbolOffset = MemberTableOffset + 114 + NamesSize + (NewMembers.size()+1)*20 + 2;
+    // It is curently unsupported. Without global symbol table, offset is put to 0
+    unsigned GlobalSymbolOffset = 0;
+    unsigned Global64SymbolOffset = 0;
+
+    printWithSpacePadding(Out, MemberTableOffset, 20); // Offset to member table
+    printWithSpacePadding(Out, GlobalSymbolOffset, 20); // Offset to global symbol table - Not supported yet 
+    printWithSpacePadding(Out, Global64SymbolOffset, 20); // Offset to 64 bits global symbol table - Not supported yet
+    printWithSpacePadding(Out, "128", 20); // Offset to first archive member - always 128
+    printWithSpacePadding(Out, LastOffset + FixedHeaderSize, 20); // Offset to last Archive member
+    printWithSpacePadding(Out, "0", 20); // Offset to first member of free list - Not supported yet
+
+    for (const MemberData &M : Data) {
+      Out << M.Header << M.Data;
+      if (M.Data.size() % 2)
+        Out << (char)0;
+    }
+
+    // Member Table Header
+    unsigned MemberTableSize = 20 * (OffsetArray.size() + 1) + NamesSize; // Size of Global symbol table
+    printBigMemberHeader(Out, MemberTableOffset, "", sys::toTimePoint(0), 0, 0, 0, MemberTableSize, LastOffset + FixedHeaderSize);
+    
+    // Member table
+    printWithSpacePadding(Out, OffsetArray.size(), 20); // Number of member
+    for (const int MemberOffset: OffsetArray)
+      printWithSpacePadding(Out, MemberOffset, 20); // All offset
+    for (const auto MemberName: NameArray) {
+      Out << MemberName; // All names
+      Out << (char)0; // And a null charcter
+    }
+    if (NamesSize % 2)
+      Out << (char)0; // Padding in names; 
+                      // final name has a null character included, so it ends by one or two null caracters
+
+    // Global Symbol Table (32 bits and 64 bits)
+    // Not supported yet
+  }
 
   Out.flush();
   return Error::success();
 }
 
 Error writeArchive(StringRef ArcName, ArrayRef<NewArchiveMember> NewMembers,
                    bool WriteSymtab, object::Archive::Kind Kind,
                    bool Deterministic, bool Thin,

llvm/test/Object/archive-big-create.test

+RUN: rm -rf %t
+RUN: mkdir -p %t
+RUN: cd %t
+RUN: mkdir -p foo
+
+# Test with empty files
+RUN: touch test1.o
+RUN: touch test2.o
+RUN: llvm-ar qc --format=aix libtest.a test1.o
+RUN: llvm-ar qc --format=aix libtest.a test1.o
+RUN: llvm-ar qc --format=aix libtest.a test2.o
+RUN: llvm-ar t libtest.a | FileCheck --match-full-lines %s
+
+RUN: llvm-ar t libtest.a | FileCheck --match-full-lines %s
+
+CHECK: test1.o
+CHECK: test1.o
+CHECK: test2.o
+
+# Extract object from reference
+RUN: rm -f empty.o
+RUN: llvm-ar x %p/Inputs/Big.a empty.o
+
+# And recreate reference
+RUN: rm -f Big_written.a
+RUN: llvm-ar qc --format=aix Big_written.a %p/Inputs/evenlen %p/Inputs/oddlen empty.o
+
+RUN: llvm-ar tO Big_written.a | FileCheck %s -strict-whitespace -check-prefix=CHECKTO
+CHECKTO: evenlen 0xfa
+CHECKTO: oddlen 0x17a
+CHECKTO: empty.o 0x1fc
+
+RUN: llvm-ar p Big_written.a evenlen | FileCheck %s -strict-whitespace -check-prefix=CHECKEVENLEN 
+CHECKEVENLEN: evenlen

llvm/tools/llvm-ar/llvm-ar.cpp

USAGE: llvm-ar [options] [-]<operation>[modifiers] [relpos] [count] <archive> [files]
        llvm-ar -M [<mri-script]
 
 OPTIONS:
   --format              - archive format to create
     =default            -   default
     =gnu                -   gnu
     =darwin             -   darwin
     =bsd                -   bsd
+    =aix                -   aix (big archive)
   --plugin=<string>     - ignored for compatibility
   -h --help             - display this help and exit
   --rsp-quoting         - quoting style for response files
     =posix              -   posix
     =windows            -   windows
   --version             - print the version and exit
   @<file>               - read options from <file>
 
static void failIfError(Error E, Twine Context = "") {
   });
 }
 
 static SmallVector<const char *, 256> PositionalArgs;
 
 static bool MRI;
 
 namespace {
-enum Format { Default, GNU, BSD, DARWIN, Unknown };
+enum Format { Default, GNU, BSD, DARWIN, AIX, Unknown };
 }
 
 static Format FormatType = Default;
 
 static std::string Options;
 
 // This enumeration delineates the kinds of operations on an archive
 // that are permitted.
if (Thin)
       fail("only the gnu format has a thin mode");
     Kind = object::Archive::K_BSD;
     break;
   case DARWIN:
     if (Thin)
       fail("only the gnu format has a thin mode");
     Kind = object::Archive::K_DARWIN;
     break;
+  case AIX:
+    Kind = object::Archive::K_XCOFF;
+    break;
   case Unknown:
     llvm_unreachable("");
   }
 
   Error E =
       writeArchive(ArchiveName, NewMembersP ? *NewMembersP : NewMembers, Symtab,
                    Kind, Deterministic, Thin, std::move(OldArchiveBuf));
   failIfError(std::move(E), ArchiveName);
for (ArrayRef<const char *>::iterator ArgIt = Argv.begin();
 
     Match = matchFlagWithArg("format", ArgIt, Argv);
     if (Match) {
       FormatType = StringSwitch<Format>(Match)
                        .Case("default", Default)
                        .Case("gnu", GNU)
                        .Case("darwin", DARWIN)
                        .Case("bsd", BSD)
+                       .Case("aix", AIX)
                        .Default(Unknown);
       if (FormatType == Unknown)
         fail(std::string("Invalid format ") + Match);
       continue;
     }
 
     if (matchFlagWithArg("plugin", ArgIt, Argv) ||
         matchFlagWithArg("rsp-quoting", ArgIt, Argv))