diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -471,6 +471,10 @@ uint64_t NewTextSegmentOffset{0}; uint64_t NewTextSegmentSize{0}; + /// New writable segment info. + uint64_t NewWritableSegmentAddress{0}; + uint64_t NewWritableSegmentSize{0}; + /// Track next available address for new allocatable sections. uint64_t NextAvailableAddress{0}; diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -3938,6 +3938,13 @@ // Allocate read-only sections first, then writable sections. enum : uint8_t { ST_READONLY, ST_READWRITE }; for (uint8_t SType = ST_READONLY; SType <= ST_READWRITE; ++SType) { + const uint64_t LastNextAvailableAddress = NextAvailableAddress; + if (SType == ST_READWRITE) { + // Align R+W segment to regular page size + NextAvailableAddress = alignTo(NextAvailableAddress, BC->RegularPageSize); + NewWritableSegmentAddress = NextAvailableAddress; + } + for (BinarySection &Section : BC->allocatableSections()) { if (!Section.hasValidSectionID()) continue; @@ -3984,6 +3991,21 @@ NextAvailableAddress += Section.getOutputSize(); } } + + if (SType == ST_READONLY) { + if (PHDRTableAddress) { + // Segment size includes the size of the PHDR area. + NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress; + } else { + // Existing PHDR table would be updated. + NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress; + } + } else if (SType == ST_READWRITE) { + NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress; + // Restore NextAvailableAddress if no new writable sections + if (!NewWritableSegmentSize) + NextAvailableAddress = LastNextAvailableAddress; + } } } @@ -4004,16 +4026,32 @@ // Write/re-write program headers. Phnum = Obj.getHeader().e_phnum; if (PHDRTableOffset) { - // Writing new pheader table. - Phnum += 1; // only adding one new segment - // Segment size includes the size of the PHDR area. - NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress; + // Writing new pheader table and adding one new entry for R+X segment. + Phnum += 1; + if (NewWritableSegmentSize) { + // Adding one more entry for R+W segment. + Phnum += 1; + } } else { assert(!PHDRTableAddress && "unexpected address for program header table"); - // Update existing table. PHDRTableOffset = Obj.getHeader().e_phoff; - NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress; + if (NewWritableSegmentSize) { + errs() << "Unable to add writable segment with UseGnuStack option\n"; + exit(1); + } } + + // NOTE Currently .eh_frame_hdr appends to the last segment, recalculate + // last segments size based on the NextAvailableAddress variable. + if (!NewWritableSegmentSize) { + if (PHDRTableAddress) + NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress; + else + NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress; + } else { + NewWritableSegmentSize = NextAvailableAddress - NewWritableSegmentAddress; + } + OS.seek(PHDRTableOffset); bool ModdedGnuStack = false; @@ -4045,6 +4083,19 @@ return NewPhdr; }; + auto createNewWritableSectionsPhdr = [&]() { + ELF64LEPhdrTy NewPhdr; + NewPhdr.p_type = ELF::PT_LOAD; + NewPhdr.p_offset = getFileOffsetForAddress(NewWritableSegmentAddress); + NewPhdr.p_vaddr = NewWritableSegmentAddress; + NewPhdr.p_paddr = NewWritableSegmentAddress; + NewPhdr.p_filesz = NewWritableSegmentSize; + NewPhdr.p_memsz = NewWritableSegmentSize; + NewPhdr.p_align = BC->RegularPageSize; + NewPhdr.p_flags = ELF::PF_R | ELF::PF_W; + return NewPhdr; + }; + // Copy existing program headers with modifications. for (const ELF64LE::Phdr &Phdr : cantFail(Obj.program_headers())) { ELF64LE::Phdr NewPhdr = Phdr; @@ -4073,6 +4124,11 @@ ELF64LE::Phdr NewTextPhdr = createNewTextPhdr(); OS.write(reinterpret_cast(&NewTextPhdr), sizeof(NewTextPhdr)); + if (NewWritableSegmentSize) { + ELF64LEPhdrTy NewWritablePhdr = createNewWritableSectionsPhdr(); + OS.write(reinterpret_cast(&NewWritablePhdr), + sizeof(NewWritablePhdr)); + } AddedSegment = true; } OS.write(reinterpret_cast(&NewPhdr), sizeof(NewPhdr)); @@ -4082,6 +4138,11 @@ // Append the new header to the end of the table. ELF64LE::Phdr NewTextPhdr = createNewTextPhdr(); OS.write(reinterpret_cast(&NewTextPhdr), sizeof(NewTextPhdr)); + if (NewWritableSegmentSize) { + ELF64LEPhdrTy NewWritablePhdr = createNewWritableSectionsPhdr(); + OS.write(reinterpret_cast(&NewWritablePhdr), + sizeof(NewWritablePhdr)); + } } assert((!opts::UseGnuStack || ModdedGnuStack) && diff --git a/bolt/test/Inputs/reorder-data-writable-ptload.fdata b/bolt/test/Inputs/reorder-data-writable-ptload.fdata new file mode 100644 --- /dev/null +++ b/bolt/test/Inputs/reorder-data-writable-ptload.fdata @@ -0,0 +1 @@ +4 _start 0 4 hot1 0 100 diff --git a/bolt/test/X86/instrumentation-eh_frame_hdr.cpp b/bolt/test/X86/instrumentation-eh_frame_hdr.cpp new file mode 100644 --- /dev/null +++ b/bolt/test/X86/instrumentation-eh_frame_hdr.cpp @@ -0,0 +1,40 @@ +// This test checks that .eh_frame_hdr address is in bounds of the last LOAD +// end address i.e. the section address is smaller then the LOAD end address. + +// REQUIRES: system-linux,bolt-runtime + +// RUN: %clangxx %cxxflags -static -Wl,-q %s -o %t.exe -Wl,--entry=_start +// RUN: llvm-bolt %t.exe -o %t.instr -instrument \ +// RUN: --instrumentation-file=%t.fdata -instrumentation-sleep-time=1 +// RUN: (llvm-readelf -SW %t.instr | grep -v bolt; llvm-readelf -lW %t.instr | \ +// RUN: grep LOAD | tail -n 1) | FileCheck %s + +// CHECK: {{.*}} .eh_frame_hdr PROGBITS [[#%x, EH_ADDR:]] +// CHECK: LOAD 0x[[#%x, LD_OFFSET:]] 0x[[#%x, LD_VADDR:]] 0x[[#%x, LD_FSIZE:]] +// CHECK-SAME: 0x[[#%x, LD_MEMSIZE:]] +// +// If .eh_frame_hdr address bigger then last LOAD segment end address test would +// fail with overflow error, otherwise the result of the expression is 0 that +// could be found on this line e.g. in LOAD align field. +// CHECK-SAME: [[#LD_VADDR + LD_MEMSIZE - max(LD_VADDR + LD_MEMSIZE,EH_ADDR)]] + +#include +#include + +void foo() { throw std::runtime_error("Exception from foo()"); } + +void bar() { foo(); } + +int main() { + try { + bar(); + } catch (const std::exception &e) { + printf("Exception caught: %s\n", e.what()); + } +} + +extern "C" { +void _start(); +} + +void _start() { main(); } diff --git a/bolt/test/reorder-data-writable-ptload.c b/bolt/test/reorder-data-writable-ptload.c new file mode 100644 --- /dev/null +++ b/bolt/test/reorder-data-writable-ptload.c @@ -0,0 +1,21 @@ +// This test checks that reorder-data pass puts new hot .data section +// to the writable segment. + +// RUN: %clang %cflags -O3 -nostdlib -Wl,-q %s -o %t.exe +// RUN: llvm-bolt %t.exe -o %t.bolt --reorder-data=".data" \ +// RUN: -data %S/Inputs/reorder-data-writable-ptload.fdata +// RUN: llvm-readelf -SlW %t.bolt | FileCheck %s + +// CHECK: .bolt.org.data +// CHECK: {{.*}} .data PROGBITS [[#%x,ADDR:]] [[#%x,OFF:]] +// CHECK: LOAD 0x{{.*}}[[#OFF]] 0x{{.*}}[[#ADDR]] {{.*}} RW + +volatile int cold1 = 42; +volatile int hot1 = 42; +volatile int cold2 = 42; +volatile int cold3 = 42; + +void _start() { + hot1++; + _start(); +}