[ELF] - Make Target::TrapInstr to be uint8_t

[grimar]

Summary

Currently, it is 4 bytes and we do not really need it.
All our targets use 1-byte trap atm.

And sometimes it is inconvenient to have 4 bytes, for example
when we fill gaps with traps in writeTrapInstr
and want to fill memory area. (see D44775/D44943).

[• espindola]

ELF/Arch/ARM.cpp

63

This is at least odd, as 0xd4 is not a trap on ARM.

Should we just directly represent the fact that the trap instruction has different sizes in different architectures by using an StringRef or ArrayRef?

[• espindola]

ELF/Arch/ARM.cpp

63

Also, nop on powerpc64 (which we are getting more support for) is 60 00 00 00.

[ruiu]

This is towards a wrong direction. uint32_t is a deliberate choice because as Rafael pointed out uint8_t doesn't work for most processors. You cannot expect that trap instruction is 1 byte long.

[grimar]

Given the comments, abandoning.

ELF/Arch/ARM.cpp

ARM::ARM() {
   TlsGotRel = R_ARM_TLS_TPOFF32;
   TlsModuleIndexRel = R_ARM_TLS_DTPMOD32;
   TlsOffsetRel = R_ARM_TLS_DTPOFF32;
   GotBaseSymInGotPlt = false;
   GotEntrySize = 4;
   GotPltEntrySize = 4;
   PltEntrySize = 16;
   PltHeaderSize = 32;
-  TrapInstr = 0xd4d4d4d4;
+  TrapInstr1 = 0xd4;
   // ARM uses Variant 1 TLS
   TcbSize = 8;
   NeedsThunks = true;
 
   // The placing of pre-created ThunkSections is controlled by the
   // ThunkSectionSpacing parameter. The aim is to place the
   // ThunkSection such that all branches from the InputSections prior to the
   // ThunkSection can reach a Thunk placed at the end of the ThunkSection.
if (!llvm::isUInt<27>(Offset)) {
     // We cannot encode the Offset, use the long form.
     writePltHeaderLong(Buf);
     return;
   }
   write32le(Buf + 0, PltData[0]);
   write32le(Buf + 4, PltData[1] | ((Offset >> 20) & 0xff));
   write32le(Buf + 8, PltData[2] | ((Offset >> 12) & 0xff));
   write32le(Buf + 12, PltData[3] | (Offset & 0xfff));
-  write32le(Buf + 16, TrapInstr); // Pad to 32-byte boundary
-  write32le(Buf + 20, TrapInstr);
-  write32le(Buf + 24, TrapInstr);
-  write32le(Buf + 28, TrapInstr);
+  write32le(Buf + 16, 0xd4d4d4d4); // Pad to 32-byte boundary
+  write32le(Buf + 20, 0xd4d4d4d4);
+  write32le(Buf + 24, 0xd4d4d4d4);
+  write32le(Buf + 28, 0xd4d4d4d4);
 }
 
 void ARM::addPltHeaderSymbols(InputSection &IS) const {
   addSyntheticLocal("$a", STT_NOTYPE, 0, 0, IS);
   addSyntheticLocal("$d", STT_NOTYPE, 16, 0, IS);
 }
 
 // Long form PLT entries that do not have any restrictions on the displacement
void ARM::writePlt(uint8_t *Buf, uint64_t GotPltEntryAddr,
   if (!llvm::isUInt<27>(Offset)) {
     // We cannot encode the Offset, use the long form.
     writePltLong(Buf, GotPltEntryAddr, PltEntryAddr, Index, RelOff);
     return;
   }
   write32le(Buf + 0, PltData[0] | ((Offset >> 20) & 0xff));
   write32le(Buf + 4, PltData[1] | ((Offset >> 12) & 0xff));
   write32le(Buf + 8, PltData[2] | (Offset & 0xfff));
-  write32le(Buf + 12, TrapInstr); // Pad to 16-byte boundary
+  write32le(Buf + 12, 0xd4d4d4d4); // Pad to 16-byte boundary
 }
 
 void ARM::addPltSymbols(InputSection &IS, uint64_t Off) const {
   addSyntheticLocal("$a", STT_NOTYPE, Off, 0, IS);
   addSyntheticLocal("$d", STT_NOTYPE, Off + 12, 0, IS);
 }
 
 bool ARM::needsThunk(RelExpr Expr, RelType Type, const InputFile *File,

ELF/Arch/Mips.cpp

template <class ELFT> MIPS<ELFT>::MIPS() {
   GotEntrySize = sizeof(typename ELFT::uint);
   GotPltEntrySize = sizeof(typename ELFT::uint);
   GotBaseSymInGotPlt = false;
   PltEntrySize = 16;
   PltHeaderSize = 32;
   CopyRel = R_MIPS_COPY;
   PltRel = R_MIPS_JUMP_SLOT;
   NeedsThunks = true;
-  TrapInstr = 0xefefefef;
+  TrapInstr1 = 0xef;
 
   if (ELFT::Is64Bits) {
     RelativeRel = (R_MIPS_64 << 8) | R_MIPS_REL32;
     TlsGotRel = R_MIPS_TLS_TPREL64;
     TlsModuleIndexRel = R_MIPS_TLS_DTPMOD64;
     TlsOffsetRel = R_MIPS_TLS_DTPREL64;
   } else {
     RelativeRel = R_MIPS_REL32;

ELF/Arch/X86.cpp

X86::X86() {
   TlsGotRel = R_386_TLS_TPOFF;
   TlsModuleIndexRel = R_386_TLS_DTPMOD32;
   TlsOffsetRel = R_386_TLS_DTPOFF32;
   GotEntrySize = 4;
   GotPltEntrySize = 4;
   PltEntrySize = 16;
   PltHeaderSize = 16;
   TlsGdRelaxSkip = 2;
-  TrapInstr = 0xcccccccc; // 0xcc = INT3
+  TrapInstr1 = 0xcc; // 0xcc = INT3
 }
 
 static bool hasBaseReg(uint8_t ModRM) { return (ModRM & 0xc7) != 0x5; }
 
 RelExpr X86::getRelExpr(RelType Type, const Symbol &S,
                         const uint8_t *Loc) const {
   switch (Type) {
   case R_386_8:

ELF/Arch/X86_64.cpp

template <class ELFT> X86_64<ELFT>::X86_64() {
   TlsGotRel = R_X86_64_TPOFF64;
   TlsModuleIndexRel = R_X86_64_DTPMOD64;
   TlsOffsetRel = R_X86_64_DTPOFF64;
   GotEntrySize = 8;
   GotPltEntrySize = 8;
   PltEntrySize = 16;
   PltHeaderSize = 16;
   TlsGdRelaxSkip = 2;
-  TrapInstr = 0xcccccccc; // 0xcc = INT3
+  TrapInstr1 = 0xcc; // 0xcc = INT3
 
   // Align to the large page size (known as a superpage or huge page).
   // FreeBSD automatically promotes large, superpage-aligned allocations.
   DefaultImageBase = 0x200000;
 }
 
 template <class ELFT>
 RelExpr X86_64<ELFT>::getRelExpr(RelType Type, const Symbol &S,

ELF/OutputSections.cpp

 void OutputSection::sortInitFini() {
   // Sort sections by priority.
   sort([](InputSectionBase *S) { return getPriority(S->Name); });
 }
 
 uint32_t OutputSection::getFiller() {
   if (Filler)
     return *Filler;
-  if (Flags & SHF_EXECINSTR)
-    return Target->TrapInstr;
+
+  if (!(Flags & SHF_EXECINSTR))
     return 0;
+  return Target->TrapInstr1 << 24 | Target->TrapInstr1 << 16 |
+         Target->TrapInstr1 << 8 | Target->TrapInstr1;
 }
 
 template void OutputSection::writeHeaderTo<ELF32LE>(ELF32LE::Shdr *Shdr);
 template void OutputSection::writeHeaderTo<ELF32BE>(ELF32BE::Shdr *Shdr);
 template void OutputSection::writeHeaderTo<ELF64LE>(ELF64LE::Shdr *Shdr);
 template void OutputSection::writeHeaderTo<ELF64BE>(ELF64BE::Shdr *Shdr);
 
 template void OutputSection::writeTo<ELF32LE>(uint8_t *Buf);

ELF/Target.h

public:
   // On PPC ELF V2 abi, the first entry in the .got is the .TOC.
   unsigned GotHeaderEntriesNum = 0;
 
   // Set to 0 for variant 2
   unsigned TcbSize = 0;
 
   bool NeedsThunks = false;
 
-  // A 4-byte field corresponding to one or more trap instructions, used to pad
-  // executable OutputSections.
-  uint32_t TrapInstr = 0;
+  // Trap instruction, used to pad executable OutputSections.
+  uint8_t TrapInstr1 = 0;
 
   virtual RelExpr adjustRelaxExpr(RelType Type, const uint8_t *Data,
                                   RelExpr Expr) const;
   virtual void relaxGot(uint8_t *Loc, uint64_t Val) const;
   virtual void relaxTlsGdToIe(uint8_t *Loc, RelType Type, uint64_t Val) const;
   virtual void relaxTlsGdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const;
   virtual void relaxTlsIeToLe(uint8_t *Loc, RelType Type, uint64_t Val) const;
   virtual void relaxTlsLdToLe(uint8_t *Loc, RelType Type, uint64_t Val) const;

ELF/Writer.cpp

 template <class ELFT> void Writer<ELFT>::writeSectionsBinary() {
   uint8_t *Buf = Buffer->getBufferStart();
   for (OutputSection *Sec : OutputSections)
     if (Sec->Flags & SHF_ALLOC)
       Sec->writeTo<ELFT>(Buf + Sec->Offset);
 }
 
 static void fillTrap(uint8_t *I, uint8_t *End) {
-  for (; I + 4 <= End; I += 4)
-    memcpy(I, &Target->TrapInstr, 4);
+  memset(I, Target->TrapInstr1, End - I);
 }
 
 // Fill the last page of executable segments with trap instructions
 // instead of leaving them as zero. Even though it is not required by any
 // standard, it is in general a good thing to do for security reasons.
 //
 // We'll leave other pages in segments as-is because the rest will be
 // overwritten by output sections.