[BOLT] Add writable segment for allocatable sections

[yota9]

Summary

The data reorder pass needs to create new data section in writable segment.
Also the golang support create 2 new data sections, one of them contains
relocations in PIC binaries so the section must have writable rights.
Currently BOLT creates only one new segment that contains new sections
with RX rights, also create RW segment if there any new writable
sections were allocated during BOLT binary processing.

Vladislav Khmelevsky,
Advanced Software Technology Lab, Huawei

[yota9]

Add test. The test is using reorder data pass and .data section which must appear in the writable segment.
@rafauler @maksfb gentle ping. Also please check D144095 D143748 . Thank you!

[yota9]

clang-format

[yota9]

nit test

[yota9]

Fix last segment size

[yota9]

Add static flag to test

[rafauler]

bolt/lib/Rewrite/RewriteInstance.cpp

4044

Is .eh_frame_hdr part of this new writable segment? Shouldn't it go to the read-only segment? Why is eh_frame_hdr being mapped to the last segment?

[yota9]

bolt/lib/Rewrite/RewriteInstance.cpp

4044

Because currently eh_frame_hdr is appended to the binary after all bianary sections were already maped. I think the patch with rewrite option would fix this behaviour, but for now we can go as-is since currently it only affects instrumentation (not really due to RWX), reorder data pass and golang binaries. As rewrite option review would probably take some time I propose to submit this patch and continue with other golang staff in parallel. Rewrite patch would remove most of this patch except for a tests..

[yota9]

@rafauler Gentle ping

[rafauler]

nit: spellcheck summary
other than that, LGTM.

bolt/include/bolt/Rewrite/RewriteInstance.h

private:
   uint64_t PHDRTableOffset{0};
   unsigned Phnum{0};
 
   /// New code segment info.
   uint64_t NewTextSegmentAddress{0};
   uint64_t NewTextSegmentOffset{0};
   uint64_t NewTextSegmentSize{0};
 
+  /// New writable segment info.
+  uint64_t NewWritableSegmentAddress{0};
+  uint64_t NewWritableSegmentSize{0};
+
   /// Track next available address for new allocatable sections.
   uint64_t NextAvailableAddress{0};
 
   /// Location and size of dynamic relocations.
   std::optional<uint64_t> DynamicRelocationsAddress;
   uint64_t DynamicRelocationsSize{0};
   uint64_t DynamicRelativeRelocationsCount{0};
 

bolt/lib/Rewrite/RewriteInstance.cpp

Section.setOutputFileOffset(
         getFileOffsetForAddress(NewTextSectionStartAddress));
   }
 }
 
 void RewriteInstance::mapAllocatableSections(RuntimeDyld &RTDyld) {
   // Allocate read-only sections first, then writable sections.
   enum : uint8_t { ST_READONLY, ST_READWRITE };
   for (uint8_t SType = ST_READONLY; SType <= ST_READWRITE; ++SType) {
+    const uint64_t LastNextAvailableAddress = NextAvailableAddress;
+    if (SType == ST_READWRITE) {
+      // Align R+W segment to regular page size
+      NextAvailableAddress = alignTo(NextAvailableAddress, BC->RegularPageSize);
+      NewWritableSegmentAddress = NextAvailableAddress;
+    }
+
     for (BinarySection &Section : BC->allocatableSections()) {
       if (!Section.hasValidSectionID())
         continue;
 
       if (Section.isWritable() == (SType == ST_READONLY))
         continue;
 
       if (Section.getOutputAddress()) {
for (BinarySection &Section : BC->allocatableSections()) {
                                       NextAvailableAddress);
         Section.setOutputAddress(NextAvailableAddress);
         Section.setOutputFileOffset(
             getFileOffsetForAddress(NextAvailableAddress));
 
         NextAvailableAddress += Section.getOutputSize();
       }
     }
+
+    if (SType == ST_READONLY) {
+      if (PHDRTableAddress) {
+        // Segment size includes the size of the PHDR area.
+        NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
+      } else {
+        // Existing PHDR table would be updated.
+        NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
+      }
+    } else if (SType == ST_READWRITE) {
+      NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
+      // Restore NextAvailableAddress if no new writable sections
+      if (!NewWritableSegmentSize)
+        NextAvailableAddress = LastNextAvailableAddress;
+    }
   }
 }
 
 void RewriteInstance::updateOutputValues(const MCAsmLayout &Layout) {
   for (BinaryFunction *Function : BC->getAllBinaryFunctions())
     Function->updateOutputValues(Layout);
 }
 
 void RewriteInstance::patchELFPHDRTable() {
   auto ELF64LEFile = dyn_cast<ELF64LEObjectFile>(InputFile);
   if (!ELF64LEFile) {
     errs() << "BOLT-ERROR: only 64-bit LE ELF binaries are supported\n";
     exit(1);
   }
   const ELFFile<ELF64LE> &Obj = ELF64LEFile->getELFFile();
   raw_fd_ostream &OS = Out->os();
 
   // Write/re-write program headers.
   Phnum = Obj.getHeader().e_phnum;
   if (PHDRTableOffset) {
-    // Writing new pheader table.
-    Phnum += 1; // only adding one new segment
-    // Segment size includes the size of the PHDR area.
-    NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
+    // Writing new pheader table and adding one new entry for R+X segment.
+    Phnum += 1;
+    if (NewWritableSegmentSize) {
+      // Adding one more entry for R+W segment.
+      Phnum += 1;
+    }
   } else {
     assert(!PHDRTableAddress && "unexpected address for program header table");
-    // Update existing table.
     PHDRTableOffset = Obj.getHeader().e_phoff;
+    if (NewWritableSegmentSize) {
+      errs() << "Unable to add writable segment with UseGnuStack option\n";
+      exit(1);
+    }
+  }
+
+  // NOTE Currently .eh_frame_hdr appends to the last segment, recalculate
+  // last segments size based on the NextAvailableAddress variable.
+  if (!NewWritableSegmentSize) {
+    if (PHDRTableAddress)
+      NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress;
+    else
       NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress;
+  } else {
+    NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress;
   }
+
   OS.seek(PHDRTableOffset);
 
   bool ModdedGnuStack = false;
   (void)ModdedGnuStack;
   bool AddedSegment = false;
   (void)AddedSegment;
 
   auto createNewTextPhdr = [&]() {
auto createNewTextPhdr = [&]() {
     // is emitted with code, thus everything needs to be writable
     if (opts::Instrument)
       NewPhdr.p_flags |= ELF::PF_W;
     NewPhdr.p_align = BC->PageAlign;
 
     return NewPhdr;
   };
 
+  auto createNewWritableSectionsPhdr = [&]() {
+    ELF64LEPhdrTy NewPhdr;
+    NewPhdr.p_type = ELF::PT_LOAD;
+    NewPhdr.p_offset = getFileOffsetForAddress(NewWritableSegmentAddress);
+    NewPhdr.p_vaddr = NewWritableSegmentAddress;
+    NewPhdr.p_paddr = NewWritableSegmentAddress;
+    NewPhdr.p_filesz = NewWritableSegmentSize;
+    NewPhdr.p_memsz = NewWritableSegmentSize;
+    NewPhdr.p_align = BC->RegularPageSize;
+    NewPhdr.p_flags = ELF::PF_R | ELF::PF_W;
+    return NewPhdr;
+  };
+
   // Copy existing program headers with modifications.
   for (const ELF64LE::Phdr &Phdr : cantFail(Obj.program_headers())) {
     ELF64LE::Phdr NewPhdr = Phdr;
     if (PHDRTableAddress && Phdr.p_type == ELF::PT_PHDR) {
       NewPhdr.p_offset = PHDRTableOffset;
       NewPhdr.p_vaddr = PHDRTableAddress;
       NewPhdr.p_paddr = PHDRTableAddress;
       NewPhdr.p_filesz = sizeof(NewPhdr) * Phnum;
for (const ELF64LE::Phdr &Phdr : cantFail(Obj.program_headers())) {
     } else if (opts::UseGnuStack && Phdr.p_type == ELF::PT_GNU_STACK) {
       NewPhdr = createNewTextPhdr();
       ModdedGnuStack = true;
     } else if (!opts::UseGnuStack && Phdr.p_type == ELF::PT_DYNAMIC) {
       // Insert the new header before DYNAMIC.
       ELF64LE::Phdr NewTextPhdr = createNewTextPhdr();
       OS.write(reinterpret_cast<const char *>(&NewTextPhdr),
                sizeof(NewTextPhdr));
+      if (NewWritableSegmentSize) {
+        ELF64LEPhdrTy NewWritablePhdr = createNewWritableSectionsPhdr();
+        OS.write(reinterpret_cast<const char *>(&NewWritablePhdr),
+                 sizeof(NewWritablePhdr));
+      }
       AddedSegment = true;
     }
     OS.write(reinterpret_cast<const char *>(&NewPhdr), sizeof(NewPhdr));
   }
 
   if (!opts::UseGnuStack && !AddedSegment) {
     // Append the new header to the end of the table.
     ELF64LE::Phdr NewTextPhdr = createNewTextPhdr();
     OS.write(reinterpret_cast<const char *>(&NewTextPhdr), sizeof(NewTextPhdr));
+    if (NewWritableSegmentSize) {
+      ELF64LEPhdrTy NewWritablePhdr = createNewWritableSectionsPhdr();
+      OS.write(reinterpret_cast<const char *>(&NewWritablePhdr),
+               sizeof(NewWritablePhdr));
+    }
   }
 
   assert((!opts::UseGnuStack || ModdedGnuStack) &&
          "could not find GNU_STACK program header to modify");
 }
 
 namespace {
 

bolt/test/Inputs/reorder-data-writable-ptload.fdata

+4 _start 0 4 hot1 0 100

bolt/test/X86/instrumentation-eh_frame_hdr.cpp

+// This test checks that .eh_frame_hdr address is in bounds of the last LOAD
+// end address i.e. the section address is smaller then the LOAD end address.
+
+// REQUIRES: system-linux,bolt-runtime
+
+// RUN: %clangxx %cxxflags -static -Wl,-q %s -o %t.exe -Wl,--entry=_start
+// RUN: llvm-bolt %t.exe -o %t.instr -instrument \
+// RUN:   --instrumentation-file=%t.fdata -instrumentation-sleep-time=1
+// RUN: (llvm-readelf -SW %t.instr | grep -v bolt; llvm-readelf -lW %t.instr | \
+// RUN:   grep LOAD | tail -n 1) | FileCheck %s
+
+// CHECK: {{.*}} .eh_frame_hdr PROGBITS [[#%x, EH_ADDR:]]
+// CHECK: LOAD 0x[[#%x, LD_OFFSET:]] 0x[[#%x, LD_VADDR:]] 0x[[#%x, LD_FSIZE:]]
+// CHECK-SAME: 0x[[#%x, LD_MEMSIZE:]]
+//
+// If .eh_frame_hdr address bigger then last LOAD segment end address test would
+// fail with overflow error, otherwise the result of the expression is 0 that
+// could be found on this line e.g. in LOAD align field.
+// CHECK-SAME: [[#LD_VADDR + LD_MEMSIZE - max(LD_VADDR + LD_MEMSIZE,EH_ADDR)]]
+
+#include <cstdio>
+#include <stdexcept>
+
+void foo() { throw std::runtime_error("Exception from foo()"); }
+
+void bar() { foo(); }
+
+int main() {
+  try {
+    bar();
+  } catch (const std::exception &e) {
+    printf("Exception caught: %s\n", e.what());
+  }
+}
+
+extern "C" {
+void _start();
+}
+
+void _start() { main(); }

bolt/test/reorder-data-writable-ptload.c

+// This test checks that reorder-data pass puts new hot .data section
+// to the writable segment.
+
+// RUN: %clang %cflags -O3 -nostdlib -Wl,-q %s -o %t.exe
+// RUN: llvm-bolt %t.exe -o %t.bolt --reorder-data=".data" \
+// RUN:   -data %S/Inputs/reorder-data-writable-ptload.fdata
+// RUN: llvm-readelf -SlW %t.bolt | FileCheck %s
+
+// CHECK: .bolt.org.data
+// CHECK: {{.*}} .data PROGBITS [[#%x,ADDR:]] [[#%x,OFF:]]
+// CHECK: LOAD 0x{{.*}}[[#OFF]] 0x{{.*}}[[#ADDR]] {{.*}} RW
+
+volatile int cold1 = 42;
+volatile int hot1 = 42;
+volatile int cold2 = 42;
+volatile int cold3 = 42;
+
+void _start() {
+  hot1++;
+  _start();
+}