diff --git a/bolt/include/bolt/Core/BinaryBasicBlock.h b/bolt/include/bolt/Core/BinaryBasicBlock.h --- a/bolt/include/bolt/Core/BinaryBasicBlock.h +++ b/bolt/include/bolt/Core/BinaryBasicBlock.h @@ -707,9 +707,16 @@ return Instructions.erase(II); } + /// Store meta-data of instruction to NOP instruction + /// and insert it in place of erased one + bool storeInstructionMetadata(iterator II); + /// Erase non-pseudo instruction at a given iterator \p II. /// Return iterator following the removed instruction. iterator eraseInstruction(iterator II) { + if (storeInstructionMetadata(II)) + return std::next(II); + adjustNumPseudos(*II, -1); return Instructions.erase(II); } @@ -748,12 +755,9 @@ /// Return iterator pointing to the first inserted instruction. template iterator replaceInstruction(iterator II, Itr Begin, Itr End) { - adjustNumPseudos(*II, -1); + II = eraseInstruction(II); adjustNumPseudos(Begin, End, 1); - - auto I = II - Instructions.begin(); - Instructions.insert(Instructions.erase(II), Begin, End); - return I + Instructions.begin(); + return Instructions.insert(II, Begin, End); } iterator replaceInstruction(iterator II, @@ -773,6 +777,14 @@ return Instructions.insert(At, NewInst); } + iterator insertInstructions(iterator At, InstructionListType &Instrs) { + for (MCInst &NewInst : Instrs) { + At = insertInstruction(At, NewInst); + ++At; + } + return At; + } + /// Helper to retrieve any terminators in \p BB before \p Pos. This is used /// to skip CFI instructions and to retrieve the first terminator instruction /// in basic blocks with two terminators (conditional jump and unconditional diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h --- a/bolt/include/bolt/Core/BinaryContext.h +++ b/bolt/include/bolt/Core/BinaryContext.h @@ -19,6 +19,7 @@ #include "bolt/Core/JumpTable.h" #include "bolt/Core/MCPlusBuilder.h" #include "bolt/RuntimeLibs/RuntimeLibrary.h" +#include "bolt/Utils/NameResolver.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/Triple.h" @@ -233,6 +234,8 @@ void deregisterSectionName(const BinarySection &Section); public: + std::vector ExtraSectionsNames; + static Expected> createBinaryContext(const ObjectFile *File, bool IsPIC, std::unique_ptr DwCtx); @@ -472,6 +475,20 @@ return BinaryFunctions; } + /// Return BF by name. If the global function was not found return the first + /// found local + BinaryFunction *getBinaryFunctionByName(StringRef Name) { + BinaryData *Data = getFirstBinaryDataByName(Name); + if (!Data) + return nullptr; + + return getBinaryFunctionAtAddress(Data->getAddress()); + } + + const BinaryFunction *getBinaryFunctionByName(StringRef Name) const { + return const_cast(this)->getBinaryFunctionByName(Name); + } + /// Create BOLT-injected function BinaryFunction *createInjectedBinaryFunction(const std::string &Name, bool IsSimple = true); @@ -858,6 +875,19 @@ return nullptr; } + /// Return BinaryData for the given \p Name + /// If the data was not found return first local BinaryData or nullptr + BinaryData *getFirstBinaryDataByName(StringRef Name) { + BinaryData *Data = getBinaryDataByName(Name); + if (!Data) + return getBinaryDataByName(NameResolver::uniquifyID(Name, 1)); + return Data; + } + + const BinaryData *getFirstBinaryDataByName(StringRef Name) const { + return const_cast(this)->getFirstBinaryDataByName(Name); + } + /// Return true if \p SymbolName was generated internally and was not present /// in the input binary. bool isInternalSymbolName(const StringRef Name) { @@ -971,6 +1001,15 @@ uint64_t Size = 0, unsigned Alignment = 1); + /// wrapper function over registerOrUpdateSection + /// with adding section name to extraSection container. + /// For sections are created from scratch (infra sections) + BinarySection ®isterExtraSection(StringRef Name, unsigned ELFType, + unsigned ELFFlags, + uint8_t *Data = nullptr, + uint64_t Size = 0, + unsigned Alignment = 1); + /// Register the information for the note (non-allocatable) section /// with the given /p Name. If the section already exists, the /// information in the section will be updated with the new data. @@ -1208,6 +1247,9 @@ uint64_t computeInstructionSize(const MCInst &Inst, const MCCodeEmitter *Emitter = nullptr) const { + if (isAArch64()) + return 4; + if (auto Size = MIB->getAnnotationWithDefault(Inst, "Size")) return Size; @@ -1335,6 +1377,10 @@ /* DWARFMustBeAtTheEnd */ false)); return Streamer; } + + /// Returns std pair of binary functons desired and maximum alignment bytes + std::pair getBFAlignment(BinaryFunction &Function, + bool EmitColdPart) const; }; template > diff --git a/bolt/include/bolt/Core/BinaryData.h b/bolt/include/bolt/Core/BinaryData.h --- a/bolt/include/bolt/Core/BinaryData.h +++ b/bolt/include/bolt/Core/BinaryData.h @@ -53,6 +53,8 @@ uint64_t Address{0}; /// Size of this data (can be 0). uint64_t Size{0}; + /// Output size of this data (if changed). + uint64_t OutputSize{0}; /// Alignment of this data. uint16_t Alignment{1}; @@ -133,7 +135,8 @@ StringRef getOutputSectionName() const; uint64_t getOutputAddress() const; uint64_t getOutputOffset() const { return OutputOffset; } - uint64_t getOutputSize() const { return Size; } + uint64_t getOutputSize() const { return OutputSize ? OutputSize : Size; } + void setOutputSize(uint64_t NewSize) { OutputSize = NewSize; } bool isMoved() const; bool containsAddress(uint64_t Address) const { diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -75,6 +75,14 @@ ICP_ALL /// Perform ICP on calls and jump tables. }; +enum IndexOrder { + GO_FIRST_BF_INDEX = 0, + FIRST_BF_INDEX = 1, + GO_UNUSED_BF_INDEX = -3U, + GO_LAST_BF_INDEX = -2U, + INVALID_BF_INDEX = -1U, +}; + /// Information on a single indirect call to a particular callee. struct IndirectCallProfile { MCSymbol *Symbol; @@ -243,7 +251,7 @@ uint64_t MaxSize{std::numeric_limits::max()}; /// Alignment requirements for the function. - uint16_t Alignment{2}; + uint16_t Alignment{MinAlign}; /// Maximum number of bytes used for alignment of hot part of the function. uint16_t MaxAlignmentBytes{0}; @@ -251,6 +259,9 @@ /// Maximum number of bytes used for alignment of cold part of the function. uint16_t MaxColdAlignmentBytes{0}; + // If function is golang stores the offset to functab entry + uint32_t GolangFunctabOffset{0}; + const MCSymbol *PersonalityFunction{nullptr}; uint8_t PersonalityEncoding{dwarf::DW_EH_PE_sdata4 | dwarf::DW_EH_PE_pcrel}; @@ -349,6 +360,9 @@ /// This attribute is only valid when hasCFG() == true. bool HasCanonicalCFG{true}; + /// True if we have a suspicion that the function was written in ASM + bool IsAsm{false}; + /// Name for the section this function code should reside in. std::string CodeSectionName; @@ -404,7 +418,7 @@ const MCSymbol *PLTSymbol{nullptr}; /// Function order for streaming into the destination binary. - uint32_t Index{-1U}; + uint32_t Index{INVALID_BF_INDEX}; /// Get basic block index assuming it belongs to this function. unsigned getIndex(const BinaryBasicBlock *BB) const { @@ -1046,15 +1060,14 @@ uint64_t getOutputSize() const { return OutputSize; } - /// Does this function have a valid streaming order index? - bool hasValidIndex() const { return Index != -1U; } + /// Returns true if this function has a valid streaming order index + bool hasValidIndex() const { return Index != INVALID_BF_INDEX; } /// Get the streaming order index for this function. uint32_t getIndex() const { return Index; } /// Set the streaming order index for this function. void setIndex(uint32_t Idx) { - assert(!hasValidIndex()); Index = Idx; } @@ -1377,6 +1390,9 @@ /// Return true if all CFG edges have local successors. bool hasCanonicalCFG() const { return HasCanonicalCFG; } + /// Return true if we have a suspicion that the function was written in ASM + bool isAsm() const { return IsAsm; } + /// Return true if the original function code has all necessary relocations /// to track addresses of functions emitted to new locations. bool hasExternalRefRelocations() const { return HasExternalRefRelocations; } @@ -1407,6 +1423,15 @@ /// Return true if the body of the function was merged into another function. bool isFolded() const { return FoldedIntoFunction != nullptr; } + /// Sets golang funtab offset for this function + void setGolangFunctabOffset(uint32_t Offset) { GolangFunctabOffset = Offset; } + + /// Return true if the function is presented in original go functab + bool isGolang() const { return GolangFunctabOffset != 0; } + + /// Get golang funtab offset for this function + uint32_t getGolangFunctabOffset() const { return GolangFunctabOffset; } + /// If this function was folded, return the function it was folded into. BinaryFunction *getFoldedIntoFunction() const { return FoldedIntoFunction; } @@ -1768,6 +1793,8 @@ void setHasCanonicalCFG(bool V) { HasCanonicalCFG = V; } + void setIsAsm(bool V) { IsAsm = V; } + void setFolded(BinaryFunction *BF) { FoldedIntoFunction = BF; } BinaryFunction &setPersonalityFunction(uint64_t Addr) { @@ -2293,14 +2320,14 @@ size_t Estimate = 0; for (const BinaryBasicBlock &BB : blocks()) if (BB.isCold()) - Estimate += BC.computeCodeSize(BB.begin(), BB.end()); + Estimate += BB.estimateSize(); return Estimate; } size_t estimateSize() const { size_t Estimate = 0; for (const BinaryBasicBlock &BB : blocks()) - Estimate += BC.computeCodeSize(BB.begin(), BB.end()); + Estimate += BB.estimateSize(); return Estimate; } diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h --- a/bolt/include/bolt/Core/BinarySection.h +++ b/bolt/include/bolt/Core/BinarySection.h @@ -94,6 +94,8 @@ // Set by ExecutableFileMemoryManager. uint32_t Index{0}; // Section index in the output file. mutable bool IsReordered{false}; // Have the contents been reordered? + mutable bool IsChanged{false}; // The section is changed and OutputContents + // should be used for emission. bool IsAnonymous{false}; // True if the name should not be included // in the output file. @@ -285,6 +287,7 @@ } } bool isReordered() const { return IsReordered; } + bool isChanged() const { return IsChanged; } bool isAnonymous() const { return IsAnonymous; } unsigned getELFType() const { return ELFType; } unsigned getELFFlags() const { return ELFFlags; } @@ -340,6 +343,9 @@ /// Does this section have any pending relocations? bool hasPendingRelocations() const { return !PendingRelocations.empty(); } + /// Does this section have any dynamic relocations? + bool hasDynamicRelocations() const { return !DynamicRelocations.empty(); } + /// Remove non-pending relocation with the given /p Offset. bool removeRelocationAt(uint64_t Offset) { auto Itr = Relocations.find(Offset); @@ -347,6 +353,19 @@ Relocations.erase(Itr); return true; } + + return false; + } + + /// Remove the dynamic relocation (if any) at the given /p Offset. + bool removeDynamicRelocationAt(uint64_t Offset) { + Relocation Key{Offset, 0, 0, 0, 0}; + auto Itr = DynamicRelocations.find(Key); + if (Itr != DynamicRelocations.end()) { + DynamicRelocations.erase(Itr); + return true; + } + return false; } @@ -354,7 +373,7 @@ /// Add a new relocation at the given /p Offset. void addRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, - uint64_t Addend, uint64_t Value = 0, + uint64_t Addend = 0, uint64_t Value = 0, bool Pending = false) { assert(Offset < getSize() && "offset not within section bounds"); if (!Pending) { @@ -367,7 +386,7 @@ /// Add a dynamic relocation at the given /p Offset. void addDynamicRelocation(uint64_t Offset, MCSymbol *Symbol, uint64_t Type, - uint64_t Addend, uint64_t Value = 0) { + uint64_t Addend = 0, uint64_t Value = 0) { assert(Offset < getSize() && "offset not within section bounds"); DynamicRelocations.emplace(Relocation{Offset, Symbol, Type, Addend, Value}); } @@ -396,7 +415,7 @@ return Itr != Relocations.end() ? &*Itr : nullptr; } - /// Lookup the relocation (if any) at the given /p Offset. + /// Lookup the dynamic relocation (if any) at the given /p Offset. const Relocation *getDynamicRelocationAt(uint64_t Offset) const { Relocation Key{Offset, 0, 0, 0, 0}; auto Itr = DynamicRelocations.find(Key); @@ -423,6 +442,11 @@ const uint8_t *getOutputData() const { return reinterpret_cast(getOutputContents().data()); } + void setOutputContents(uint8_t *Data, uint64_t Size) { + IsChanged = true; + OutputSize = Size; + OutputContents = StringRef(reinterpret_cast(Data), Size); + } StringRef getOutputContents() const { return OutputContents; } uint64_t getAllocAddress() const { return reinterpret_cast(getOutputData()); diff --git a/bolt/include/bolt/Core/MCPlusBuilder.h b/bolt/include/bolt/Core/MCPlusBuilder.h --- a/bolt/include/bolt/Core/MCPlusBuilder.h +++ b/bolt/include/bolt/Core/MCPlusBuilder.h @@ -1694,6 +1694,21 @@ return Index; } + /// Copy annotations from one instruction to other + void copyAnnotationInst(const MCInst &From, MCInst &To, + AllocatorIdTy AllocatorId = 0) { + const MCInst *AnnotationInst = getAnnotationInst(From); + if (!AnnotationInst) + return; + + for (unsigned I = 0; I < AnnotationInst->getNumOperands(); ++I) { + const int64_t Imm = AnnotationInst->getOperand(I).getImm(); + const unsigned Index = extractAnnotationIndex(Imm); + const int64_t Value = extractAnnotationValue(Imm); + setAnnotationOpValue(To, Index, Value, AllocatorId); + } + } + /// Store an annotation value on an MCInst. This assumes the annotation /// is not already present. template diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h --- a/bolt/include/bolt/Core/Relocation.h +++ b/bolt/include/bolt/Core/Relocation.h @@ -106,6 +106,18 @@ /// Return code for a PC-relative 8-byte relocation static uint64_t getPC64(); + /// Return code for a ABS 4-byte relocation + static uint64_t getAbs32(); + + /// Return code for a ABS 8-byte relocation + static uint64_t getAbs64(); + + /// Return code for ABS relocation based on size + static uint64_t getAbs(uint8_t Size); + + /// Return code for a RELATIVE relocation + static uint64_t getRelative(); + /// Return true if this relocation is PC-relative. Return false otherwise. bool isPCRelative() const { return isPCRelative(Type); } @@ -113,6 +125,9 @@ /// otherwise. bool isRelative() const { return isRelative(Type); } + /// Return true if this relocation is IRELATIVE type. Return false otherwise. + bool isIRelative() const { return isIRelative(Type); } + /// Emit relocation at a current \p Streamer' position. The caller is /// responsible for setting the position correctly. size_t emit(MCStreamer *Streamer) const; diff --git a/bolt/include/bolt/Rewrite/RewriteInstance.h b/bolt/include/bolt/Rewrite/RewriteInstance.h --- a/bolt/include/bolt/Rewrite/RewriteInstance.h +++ b/bolt/include/bolt/Rewrite/RewriteInstance.h @@ -471,6 +471,9 @@ uint64_t NewTextSegmentOffset{0}; uint64_t NewTextSegmentSize{0}; + uint64_t NewExtraSegmentAddress{0}; + uint64_t NewExtraSegmentSize{0}; + /// Track next available address for new allocatable sections. uint64_t NextAvailableAddress{0}; diff --git a/bolt/include/bolt/Utils/NameResolver.h b/bolt/include/bolt/Utils/NameResolver.h --- a/bolt/include/bolt/Utils/NameResolver.h +++ b/bolt/include/bolt/Utils/NameResolver.h @@ -29,9 +29,13 @@ public: /// Return unique version of the \p Name in the form "Name". + static std::string uniquifyID(StringRef Name, uint64_t ID) { + return (Name + Twine(Sep) + Twine(ID)).str(); + } + std::string uniquify(StringRef Name) { const uint64_t ID = ++Counters[Name]; - return (Name + Twine(Sep) + Twine(ID)).str(); + return uniquifyID(Name, ID); } /// For uniquified \p Name, return the original form (that may no longer be diff --git a/bolt/lib/Core/BinaryBasicBlock.cpp b/bolt/lib/Core/BinaryBasicBlock.cpp --- a/bolt/lib/Core/BinaryBasicBlock.cpp +++ b/bolt/lib/Core/BinaryBasicBlock.cpp @@ -648,5 +648,21 @@ LocSyms.reset(nullptr); } +bool BinaryBasicBlock::storeInstructionMetadata(BinaryBasicBlock::iterator II) { + // In case the instruction has locked annotation we will save all + // its annotations to noop instruction and store it in place of + // erased instruction + BinaryContext &BC = Function->getBinaryContext(); + if (!BC.MIB->hasAnnotation(*II, "Locked")) + return false; + + MCInst Noop; + BC.MIB->createNoop(Noop); + std::unique_lock Lock(BC.CtxMutex); + BC.MIB->copyAnnotationInst(*II, Noop); + Instructions[std::distance(Instructions.begin(), II)] = Noop; + return true; +} + } // namespace bolt } // namespace llvm diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp --- a/bolt/lib/Core/BinaryContext.cpp +++ b/bolt/lib/Core/BinaryContext.cpp @@ -2006,6 +2006,17 @@ } } +BinarySection &BinaryContext::registerExtraSection(StringRef Name, + unsigned ELFType, + unsigned ELFFlags, + uint8_t *Data, uint64_t Size, + unsigned Alignment) { + BinarySection &Section = + registerOrUpdateSection(Name, ELFType, ELFFlags, Data, Size, Alignment); + ExtraSectionsNames.emplace_back(Section.getName()); + return Section; +} + bool BinaryContext::deregisterSection(BinarySection &Section) { BinarySection *SectionPtr = &Section; auto Itr = Sections.find(SectionPtr); @@ -2380,5 +2391,18 @@ return OutputRanges; } +std::pair +BinaryContext::getBFAlignment(BinaryFunction &Function, + bool EmitColdPart) const { + unsigned Alignment = Function.getAlignment(); + if (HasRelocations) { + unsigned MaxAlignBytes = EmitColdPart ? Function.getMaxColdAlignmentBytes() + : Function.getMaxAlignmentBytes(); + return std::make_pair(Alignment, MaxAlignBytes); + } + + return std::make_pair(Alignment, Alignment); +} + } // namespace bolt } // namespace llvm diff --git a/bolt/lib/Core/BinaryEmitter.cpp b/bolt/lib/Core/BinaryEmitter.cpp --- a/bolt/lib/Core/BinaryEmitter.cpp +++ b/bolt/lib/Core/BinaryEmitter.cpp @@ -435,8 +435,8 @@ // Check if special alignment for macro-fusion is needed. bool MayNeedMacroFusionAlignment = - (opts::AlignMacroOpFusion == MFT_ALL) || - (opts::AlignMacroOpFusion == MFT_HOT && BB->getKnownExecutionCount()); + ((opts::AlignMacroOpFusion == MFT_ALL) || + (opts::AlignMacroOpFusion == MFT_HOT && BB->getKnownExecutionCount())); BinaryBasicBlock::const_iterator MacroFusionPair; if (MayNeedMacroFusionAlignment) { MacroFusionPair = BB->getMacroOpFusionPair(); @@ -1164,6 +1164,14 @@ Section.emitAsData(Streamer, Prefix + Section.getName()); Section.clearRelocations(); } + + for (auto &SectionName : BC.ExtraSectionsNames) { + ErrorOr Section = BC.getUniqueSectionByName(SectionName); + if (!Section) + continue; + Section->emitAsData(Streamer, SectionName); + Section->clearRelocations(); + } } namespace llvm { diff --git a/bolt/lib/Core/BinarySection.cpp b/bolt/lib/Core/BinarySection.cpp --- a/bolt/lib/Core/BinarySection.cpp +++ b/bolt/lib/Core/BinarySection.cpp @@ -16,6 +16,8 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/Support/CommandLine.h" +#include + #define DEBUG_TYPE "bolt" using namespace llvm; @@ -69,7 +71,7 @@ void BinarySection::emitAsData(MCStreamer &Streamer, const Twine &SectionName) const { - StringRef SectionContents = getContents(); + StringRef SectionContents = isChanged() ? getOutputContents() : getContents(); MCSectionELF *ELFSection = BC.Ctx->getELFSection(SectionName, getELFType(), getELFFlags()); @@ -164,6 +166,11 @@ } BinarySection::~BinarySection() { + if (isChanged()) { + munmap(getOutputData(), getOutputSize()); + return; + } + if (isReordered()) { delete[] getData(); return; diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp --- a/bolt/lib/Core/Relocation.cpp +++ b/bolt/lib/Core/Relocation.cpp @@ -637,6 +637,34 @@ return isPCRelativeX86(Type); } +uint64_t Relocation::getAbs32() { + if (Arch == Triple::aarch64) + return ELF::R_AARCH64_ABS32; + return ELF::R_X86_64_32; +} + +uint64_t Relocation::getAbs64() { + if (Arch == Triple::aarch64) + return ELF::R_AARCH64_ABS64; + return ELF::R_X86_64_64; +} + +uint64_t Relocation::getAbs(uint8_t Size) { + if (Size == sizeof(uint64_t)) + return Relocation::getAbs64(); + else if (Size == sizeof(uint32_t)) + return Relocation::getAbs32(); + + llvm_unreachable("Wrong relocation size"); + return 0; +} + +uint64_t Relocation::getRelative() { + if (Arch == Triple::aarch64) + return ELF::R_AARCH64_RELATIVE; + return ELF::R_X86_64_RELATIVE; +} + size_t Relocation::emit(MCStreamer *Streamer) const { const size_t Size = getSizeForType(Type); MCContext &Ctx = Streamer->getContext(); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -16,8 +16,10 @@ #include "bolt/Core/MCPlusBuilder.h" #include "bolt/Core/ParallelUtilities.h" #include "bolt/Core/Relocation.h" +#include "bolt/Passes/BinaryPasses.h" #include "bolt/Passes/CacheMetrics.h" #include "bolt/Passes/ReorderFunctions.h" +#include "bolt/Passes/SplitFunctions.h" #include "bolt/Profile/BoltAddressTranslation.h" #include "bolt/Profile/DataAggregator.h" #include "bolt/Profile/DataReader.h" @@ -81,6 +83,15 @@ extern cl::list ReorderData; extern cl::opt ReorderFunctions; extern cl::opt TimeBuild; +extern cl::opt PreserveBlocksAlignment; +extern cl::opt AlignBlocks; +extern cl::opt NoInline; +extern cl::opt StringOps; +extern cl::opt FrameOptimization; +extern cl::opt SplitFunctions; +extern llvm::cl::opt InsertRetpolines; +extern cl::opt InstructionsLowering; +extern cl::list Peepholes; static cl::opt ForceToDataRelocations( "force-data-relocations", @@ -1768,6 +1779,13 @@ opts::HotTextMoveSections.addValue(".never_hugify"); } + if ((BC->isAArch64()) && + (opts::AlignBlocks || opts::PreserveBlocksAlignment)) { + errs() << "BOLT-WARNING: Disabling block alignment\n"; + opts::AlignBlocks = false; + opts::PreserveBlocksAlignment = false; + } + if (opts::UseOldText && !BC->OldTextSectionAddress) { errs() << "BOLT-WARNING: cannot use old .text as the section was not found" "\n"; @@ -2915,6 +2933,10 @@ void RewriteInstance::disassembleFunctions() { NamedRegionTimer T("disassembleFunctions", "disassemble functions", TimerGroupName, TimerGroupDesc, opts::TimeRewrite); + // Create annotation indices to allow lock-free execution + BC->MIB->getOrCreateAnnotationIndex("Size"); + BC->MIB->getOrCreateAnnotationIndex("Locked"); + for (auto &BFI : BC->getBinaryFunctions()) { BinaryFunction &Function = BFI.second; @@ -3915,6 +3937,28 @@ void RewriteInstance::mapAllocatableSections(RuntimeDyld &RTDyld) { // Allocate read-only sections first, then writable sections. + // TODO Fix + for (auto Name : BC->ExtraSectionsNames) { + ErrorOr Section = BC->getUniqueSectionByName(Name); + NextAvailableAddress = + alignTo(NextAvailableAddress, Section->getAlignment()); + LLVM_DEBUG({ + dbgs() << "BOLT: mapping section " << Section->getName() << " (0x" + << Twine::utohexstr(Section->getAllocAddress()) << ") to 0x" + << Twine::utohexstr(NextAvailableAddress) << ":0x" + << Twine::utohexstr(NextAvailableAddress + + Section->getOutputSize()) + << '\n'; + }); + + RTDyld.reassignSectionAddress(Section->getSectionID(), + NextAvailableAddress); + Section->setOutputAddress(NextAvailableAddress); + Section->setOutputFileOffset(getFileOffsetForAddress(NextAvailableAddress)); + + NextAvailableAddress += Section->getOutputSize(); + } + enum : uint8_t { ST_READONLY, ST_READWRITE }; for (uint8_t SType = ST_READONLY; SType <= ST_READWRITE; ++SType) { for (BinarySection &Section : BC->allocatableSections()) { @@ -3986,12 +4030,22 @@ // Writing new pheader table. Phnum += 1; // only adding one new segment // Segment size includes the size of the PHDR area. - NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress; + if (BC->ExtraSectionsNames.size()) { + Phnum += 1; + NewTextSegmentSize = NewExtraSegmentAddress - PHDRTableAddress; + NewExtraSegmentSize = NextAvailableAddress - NewExtraSegmentAddress; + } else { + NewTextSegmentSize = NextAvailableAddress - PHDRTableAddress; + } } else { assert(!PHDRTableAddress && "unexpected address for program header table"); // Update existing table. PHDRTableOffset = Obj.getHeader().e_phoff; NewTextSegmentSize = NextAvailableAddress - NewTextSegmentAddress; + if (BC->ExtraSectionsNames.size()) { + errs() << "Could not use additional sections with UseGnuStack option\n"; + exit(1); + } } OS.seek(PHDRTableOffset); @@ -4024,6 +4078,28 @@ return NewPhdr; }; + auto createNewExtraSectionsPhdr = [&](unsigned ELFFlags) { + ELF64LEPhdrTy NewPhdr; + NewPhdr.p_type = ELF::PT_LOAD; + NewPhdr.p_offset = getFileOffsetForAddress(NewExtraSegmentAddress); + NewPhdr.p_vaddr = NewExtraSegmentAddress; + NewPhdr.p_paddr = NewExtraSegmentAddress; + NewPhdr.p_filesz = NewExtraSegmentSize; + NewPhdr.p_memsz = NewExtraSegmentSize; + NewPhdr.p_align = BC->RegularPageSize; + + NewPhdr.p_flags = ELF::PF_R; + if (ELFFlags & ELF::SHF_WRITE) + NewPhdr.p_flags |= ELF::PF_W; + + // FIXME: Currently instrumentation is experimental and the runtime data + // is emitted with code, thus everything needs to be writable + if (ELFFlags & ELF::SHF_EXECINSTR || opts::Instrument) + NewPhdr.p_flags |= ELF::PF_X; + + return NewPhdr; + }; + // Copy existing program headers with modifications. for (const ELF64LE::Phdr &Phdr : cantFail(Obj.program_headers())) { ELF64LE::Phdr NewPhdr = Phdr; @@ -4047,16 +4123,30 @@ } else if (opts::UseGnuStack && Phdr.p_type == ELF::PT_GNU_STACK) { NewPhdr = createNewTextPhdr(); ModdedGnuStack = true; - } else if (!opts::UseGnuStack && Phdr.p_type == ELF::PT_DYNAMIC) { - // Insert the new header before DYNAMIC. - ELF64LE::Phdr NewTextPhdr = createNewTextPhdr(); - OS.write(reinterpret_cast(&NewTextPhdr), - sizeof(NewTextPhdr)); - AddedSegment = true; } + OS.write(reinterpret_cast(&NewPhdr), sizeof(NewPhdr)); } + if (!opts::UseGnuStack) { + ELF64LEPhdrTy NewTextPhdr = createNewTextPhdr(); + OS.write(reinterpret_cast(&NewTextPhdr), sizeof(NewTextPhdr)); + if (BC->ExtraSectionsNames.size()) { + unsigned ELFFlags = 0; + for (const auto &SectionName : BC->ExtraSectionsNames) { + ErrorOr Section = + BC->getUniqueSectionByName(SectionName); + ELFFlags |= Section->getELFFlags(); + } + + ELF64LEPhdrTy NewExtraPhdr = createNewExtraSectionsPhdr(ELFFlags); + OS.write(reinterpret_cast(&NewExtraPhdr), + sizeof(NewExtraPhdr)); + } + + AddedSegment = true; + } + if (!opts::UseGnuStack && !AddedSegment) { // Append the new header to the end of the table. ELF64LE::Phdr NewTextPhdr = createNewTextPhdr(); @@ -4742,9 +4832,10 @@ Function->getCodeSection(FF->getFragmentNum())->getIndex(); } else { // Check if the symbol belongs to moved data object and update it. - BinaryData *BD = opts::ReorderData.empty() - ? nullptr - : BC->getBinaryDataAtAddress(Symbol.st_value); + BinaryData *BD = + opts::ForceToDataRelocations || !opts::ReorderData.empty() + ? BC->getBinaryDataAtAddress(Symbol.st_value) + : nullptr; if (BD && BD->isMoved() && !BD->isJumpTable()) { assert((!BD->getSize() || !Symbol.st_size || Symbol.st_size == BD->getSize()) && @@ -4759,6 +4850,7 @@ << " (" << OutputSection.getIndex() << ")\n"); NewSymbol.st_shndx = OutputSection.getIndex(); NewSymbol.st_value = BD->getOutputAddress(); + NewSymbol.st_size = BD->getOutputSize(); } else { // Otherwise just update the section for the symbol. if (Symbol.st_shndx < ELF::SHN_LORESERVE) @@ -4885,6 +4977,23 @@ addSymbol("__hot_data_end"); } + auto addSectionSymbol = [&](uint64_t Address, unsigned index) { + ELFSymTy Symbol; + Symbol.st_value = Address; + Symbol.st_shndx = index; + Symbol.st_name = AddToStrTab(""); + Symbol.st_size = 0; + Symbol.st_other = 0; + Symbol.setBindingAndType(ELF::STB_LOCAL, ELF::STT_SECTION); + + Symbols.insert(Symbols.begin() + index, Symbol); + }; + + for (const auto &SectionName : BC->ExtraSectionsNames) { + ErrorOr Section = BC->getUniqueSectionByName(SectionName); + addSectionSymbol(Section->getOutputAddress(), Section->getIndex()); + } + // Put local symbols at the beginning. llvm::stable_sort(Symbols, [](const ELFSymTy &A, const ELFSymTy &B) { if (A.getBinding() == ELF::STB_LOCAL && B.getBinding() != ELF::STB_LOCAL) @@ -5045,12 +5154,18 @@ uint64_t SectionAddress = Section.getOutputAddress(); SectionAddress = SectionAddress == 0 ? Section.getAddress() : SectionAddress; - MCSymbol *Symbol = Rel.Symbol; uint32_t SymbolIdx = 0; uint64_t Addend = Rel.Addend; if (Rel.Symbol) { - SymbolIdx = getOutputDynamicSymbolIndex(Symbol); + if (!IsRelative && !Rel.isIRelative()) { + SymbolIdx = getOutputDynamicSymbolIndex(Rel.Symbol); + } else { + // The R_*_(I)RELATIVE relocation inserted by BOLT + ErrorOr Address = BC->getSymbolValue(*Rel.Symbol); + if (Address) + Addend += getNewFunctionOrDataAddress(*Address); + } } else { // Usually this case is used for R_*_(I)RELATIVE relocations const uint64_t Address = getNewFunctionOrDataAddress(Addend);