diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -232,9 +232,6 @@ /// Size of the function in the output file. uint64_t OutputSize{0}; - /// Offset in the file. - uint64_t FileOffset{0}; - /// Maximum size this function is allowed to have. uint64_t MaxSize{std::numeric_limits::max()}; @@ -345,14 +342,6 @@ /// This attribute is only valid when hasCFG() == true. bool HasCanonicalCFG{true}; - /// The address for the code for this function in codegen memory. - /// Used for functions that are emitted in a dedicated section with a fixed - /// address. E.g. for functions that are overwritten in-place. - uint64_t ImageAddress{0}; - - /// The size of the code in memory. - uint64_t ImageSize{0}; - /// Name for the section this function code should reside in. std::string CodeSectionName; @@ -1060,7 +1049,9 @@ } /// Return offset of the function body in the binary file. - uint64_t getFileOffset() const { return FileOffset; } + uint64_t getFileOffset() const { + return getLayout().getMainFragment().getFileOffset(); + } /// Return (original) byte size of the function. uint64_t getSize() const { return Size; } @@ -1698,7 +1689,7 @@ int64_t NewOffset); BinaryFunction &setFileOffset(uint64_t Offset) { - FileOffset = Offset; + getLayout().getMainFragment().setFileOffset(Offset); return *this; } @@ -1785,20 +1776,24 @@ uint16_t getMaxColdAlignmentBytes() const { return MaxColdAlignmentBytes; } BinaryFunction &setImageAddress(uint64_t Address) { - ImageAddress = Address; + getLayout().getMainFragment().setImageAddress(Address); return *this; } /// Return the address of this function' image in memory. - uint64_t getImageAddress() const { return ImageAddress; } + uint64_t getImageAddress() const { + return getLayout().getMainFragment().getImageAddress(); + } BinaryFunction &setImageSize(uint64_t Size) { - ImageSize = Size; + getLayout().getMainFragment().setImageSize(Size); return *this; } /// Return the size of this function' image in memory. - uint64_t getImageSize() const { return ImageSize; } + uint64_t getImageSize() const { + return getLayout().getMainFragment().getImageSize(); + } /// Return true if the function is a secondary fragment of another function. bool isFragment() const { return IsFragment; } @@ -2287,33 +2282,6 @@ bool isAArch64Veneer() const; virtual ~BinaryFunction(); - - /// Info for fragmented functions. - class FragmentInfo { - private: - uint64_t Address{0}; - uint64_t ImageAddress{0}; - uint64_t ImageSize{0}; - uint64_t FileOffset{0}; - - public: - uint64_t getAddress() const { return Address; } - uint64_t getImageAddress() const { return ImageAddress; } - uint64_t getImageSize() const { return ImageSize; } - uint64_t getFileOffset() const { return FileOffset; } - - void setAddress(uint64_t VAddress) { Address = VAddress; } - void setImageAddress(uint64_t Address) { ImageAddress = Address; } - void setImageSize(uint64_t Size) { ImageSize = Size; } - void setFileOffset(uint64_t Offset) { FileOffset = Offset; } - }; - - /// Cold fragment of the function. - FragmentInfo ColdFragment; - - FragmentInfo &cold() { return ColdFragment; } - - const FragmentInfo &cold() const { return ColdFragment; } }; inline raw_ostream &operator<<(raw_ostream &OS, diff --git a/bolt/include/bolt/Core/FunctionLayout.h b/bolt/include/bolt/Core/FunctionLayout.h --- a/bolt/include/bolt/Core/FunctionLayout.h +++ b/bolt/include/bolt/Core/FunctionLayout.h @@ -83,6 +83,20 @@ unsigned StartIndex; unsigned Size = 0; + /// Output address for the fragment. + uint64_t Address = 0; + + /// The address for the code for this fragment in codegen memory. Used for + /// functions that are emitted in a dedicated section with a fixed address, + /// e.g. for functions that are overwritten in-place. + uint64_t ImageAddress = 0; + + /// The size of the code in memory. + uint64_t ImageSize = 0; + + /// Offset in the file. + uint64_t FileOffset = 0; + FunctionFragment(FunctionLayout &Layout, FragmentNum Num); FunctionFragment(const FunctionFragment &) = default; FunctionFragment(FunctionFragment &&) = default; @@ -97,6 +111,15 @@ } bool isSplitFragment() const { return !isMainFragment(); } + uint64_t getAddress() const { return Address; } + void setAddress(uint64_t Value) { Address = Value; } + uint64_t getImageAddress() const { return ImageAddress; } + void setImageAddress(uint64_t Address) { ImageAddress = Address; } + uint64_t getImageSize() const { return ImageSize; } + void setImageSize(uint64_t Size) { ImageSize = Size; } + uint64_t getFileOffset() const { return FileOffset; } + void setFileOffset(uint64_t Offset) { FileOffset = Offset; } + unsigned size() const { return Size; }; bool empty() const { return size() == 0; }; iterator begin(); diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -424,19 +424,19 @@ Sep = "\n "; } } - OS << "\n Number : " << FunctionNumber - << "\n State : " << CurrentState - << "\n Address : 0x" << Twine::utohexstr(Address) - << "\n Size : 0x" << Twine::utohexstr(Size) - << "\n MaxSize : 0x" << Twine::utohexstr(MaxSize) - << "\n Offset : 0x" << Twine::utohexstr(FileOffset) - << "\n Section : " << SectionName - << "\n Orc Section : " << getCodeSectionName() - << "\n LSDA : 0x" << Twine::utohexstr(getLSDAAddress()) - << "\n IsSimple : " << IsSimple - << "\n IsMultiEntry: " << isMultiEntry() - << "\n IsSplit : " << isSplit() - << "\n BB Count : " << size(); + OS << "\n Number : " << FunctionNumber; + OS << "\n State : " << CurrentState; + OS << "\n Address : 0x" << Twine::utohexstr(Address); + OS << "\n Size : 0x" << Twine::utohexstr(Size); + OS << "\n MaxSize : 0x" << Twine::utohexstr(MaxSize); + OS << "\n Offset : 0x" << Twine::utohexstr(getFileOffset()); + OS << "\n Section : " << SectionName; + OS << "\n Orc Section : " << getCodeSectionName(); + OS << "\n LSDA : 0x" << Twine::utohexstr(getLSDAAddress()); + OS << "\n IsSimple : " << IsSimple; + OS << "\n IsMultiEntry: " << isMultiEntry(); + OS << "\n IsSplit : " << isSplit(); + OS << "\n BB Count : " << size(); if (HasFixedIndirectBranch) OS << "\n HasFixedIndirectBranch : true"; @@ -472,8 +472,8 @@ for (const BinaryBasicBlock *BB : Layout.blocks()) OS << LS << BB->getName(); } - if (ImageAddress) - OS << "\n Image : 0x" << Twine::utohexstr(ImageAddress); + if (getImageAddress()) + OS << "\n Image : 0x" << Twine::utohexstr(getImageAddress()); if (ExecutionCount != COUNT_NO_PROFILE) { OS << "\n Exec Count : " << ExecutionCount; OS << "\n Profile Acc : " << format("%.1f%%", ProfileMatchRatio * 100.0f); @@ -4059,7 +4059,7 @@ setOutputDataAddress(BaseAddress + DataOffset); } if (isSplit()) { - for (const FunctionFragment &FF : getLayout().getSplitFragments()) { + for (FunctionFragment &FF : getLayout().getSplitFragments()) { ErrorOr ColdSection = getCodeSection(FF.getFragmentNum()); // If fragment is empty, cold section might not exist @@ -4081,8 +4081,8 @@ const uint64_t ColdStartOffset = Layout.getSymbolOffset(*ColdStartSymbol); const uint64_t ColdEndOffset = Layout.getSymbolOffset(*ColdEndSymbol); - cold().setAddress(ColdBaseAddress + ColdStartOffset); - cold().setImageSize(ColdEndOffset - ColdStartOffset); + FF.setAddress(ColdBaseAddress + ColdStartOffset); + FF.setImageSize(ColdEndOffset - ColdStartOffset); if (hasConstantIsland()) { const uint64_t DataOffset = Layout.getSymbolOffset(*getFunctionColdConstantIslandLabel()); @@ -4109,44 +4109,39 @@ if (getLayout().block_empty()) return; - assert((getLayout().isHotColdSplit() || - (cold().getAddress() == 0 && cold().getImageSize() == 0 && - BC.HasRelocations)) && - "Function must be split two ways or cold fragment must have no " - "address (only in relocation mode)"); - - BinaryBasicBlock *PrevBB = nullptr; for (FunctionFragment &FF : getLayout().fragments()) { + if (FF.empty()) + continue; + const uint64_t FragmentBaseAddress = getCodeSection(isSimple() ? FF.getFragmentNum() : FragmentNum::main()) ->getOutputAddress(); + + BinaryBasicBlock *PrevBB = nullptr; for (BinaryBasicBlock *const BB : FF) { assert(BB->getLabel()->isDefined() && "symbol should be defined"); if (!BC.HasRelocations) { - if (BB->isSplit()) { - assert(FragmentBaseAddress == cold().getAddress()); - } else { + if (BB->isSplit()) + assert(FragmentBaseAddress == FF.getAddress()); + else assert(FragmentBaseAddress == getOutputAddress()); - } } + const uint64_t BBOffset = Layout.getSymbolOffset(*BB->getLabel()); const uint64_t BBAddress = FragmentBaseAddress + BBOffset; BB->setOutputStartAddress(BBAddress); - if (PrevBB) { - uint64_t PrevBBEndAddress = BBAddress; - if (BB->isSplit() != PrevBB->isSplit()) - PrevBBEndAddress = getOutputAddress() + getOutputSize(); - PrevBB->setOutputEndAddress(PrevBBEndAddress); - } + if (PrevBB) + PrevBB->setOutputEndAddress(BBAddress); PrevBB = BB; BB->updateOutputValues(Layout); } + + PrevBB->setOutputEndAddress(PrevBB->isSplit() + ? FF.getAddress() + FF.getImageSize() + : getOutputAddress() + getOutputSize()); } - PrevBB->setOutputEndAddress(PrevBB->isSplit() - ? cold().getAddress() + cold().getImageSize() - : getOutputAddress() + getOutputSize()); } DebugAddressRangesVector BinaryFunction::getOutputAddressRanges() const { @@ -4162,8 +4157,9 @@ getOutputAddress() + getOutputSize()); if (isSplit()) { assert(isEmitted() && "split function should be emitted"); - OutputRanges.emplace_back(cold().getAddress(), - cold().getAddress() + cold().getImageSize()); + for (const FunctionFragment &FF : getLayout().getSplitFragments()) + OutputRanges.emplace_back(FF.getAddress(), + FF.getAddress() + FF.getImageSize()); } if (isSimple()) diff --git a/bolt/lib/Core/FunctionLayout.cpp b/bolt/lib/Core/FunctionLayout.cpp --- a/bolt/lib/Core/FunctionLayout.cpp +++ b/bolt/lib/Core/FunctionLayout.cpp @@ -204,10 +204,12 @@ void FunctionLayout::clear() { Blocks = BasicBlockListType(); - for (FunctionFragment *const FF : Fragments) - delete FF; - Fragments = FragmentListType(); - addFragment(); + // Keep the main fragment around, because it may contain emission information + // even if the function is ignored. + std::for_each(Fragments.begin() + 1, Fragments.end(), + [](FunctionFragment *const FF) { delete FF; }); + Fragments = FragmentListType{Fragments.front()}; + getMainFragment().Size = 0; } const BinaryBasicBlock * diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp --- a/bolt/lib/Profile/BoltAddressTranslation.cpp +++ b/bolt/lib/Profile/BoltAddressTranslation.cpp @@ -73,29 +73,27 @@ LLVM_DEBUG(dbgs() << "Function name: " << Function.getPrintName() << "\n"); LLVM_DEBUG(dbgs() << " Address reference: 0x" << Twine::utohexstr(Function.getOutputAddress()) << "\n"); + MapTy Map; - const bool IsSplit = Function.isSplit(); - for (const BinaryBasicBlock *const BB : Function.getLayout().blocks()) { - if (IsSplit && BB->isCold()) - break; + for (const BinaryBasicBlock *const BB : + Function.getLayout().getMainFragment()) writeEntriesForBB(Map, *BB, Function.getOutputAddress()); - } - Maps.insert(std::pair(Function.getOutputAddress(), Map)); + Maps.emplace(Function.getOutputAddress(), std::move(Map)); - if (!IsSplit) + if (!Function.isSplit()) continue; - // Cold map - Map.clear(); + // Split maps LLVM_DEBUG(dbgs() << " Cold part\n"); - for (const BinaryBasicBlock *const BB : Function.getLayout().blocks()) { - if (!BB->isCold()) - continue; - writeEntriesForBB(Map, *BB, Function.cold().getAddress()); + for (const FunctionFragment &FF : + Function.getLayout().getSplitFragments()) { + Map.clear(); + for (const BinaryBasicBlock *const BB : FF) + writeEntriesForBB(Map, *BB, FF.getAddress()); + + Maps.emplace(FF.getAddress(), std::move(Map)); + ColdPartSource.emplace(FF.getAddress(), Function.getOutputAddress()); } - Maps.insert(std::pair(Function.cold().getAddress(), Map)); - ColdPartSource.insert(std::pair( - Function.cold().getAddress(), Function.getOutputAddress())); } const uint32_t NumFuncs = Maps.size(); diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -3682,37 +3682,34 @@ if (!Function.isSplit()) continue; - for (const FunctionFragment &FF : - Function.getLayout().getSplitFragments()) { + for (FunctionFragment &FF : Function.getLayout().getSplitFragments()) { ErrorOr ColdSection = Function.getCodeSection(FF.getFragmentNum()); assert(ColdSection && "cannot find section for cold part"); // Cold fragments are aligned at 16 bytes. NextAvailableAddress = alignTo(NextAvailableAddress, 16); - BinaryFunction::FragmentInfo &ColdPart = Function.cold(); if (TooLarge) { // The corresponding FDE will refer to address 0. - ColdPart.setAddress(0); - ColdPart.setImageAddress(0); - ColdPart.setImageSize(0); - ColdPart.setFileOffset(0); + FF.setAddress(0); + FF.setImageAddress(0); + FF.setImageSize(0); + FF.setFileOffset(0); } else { - ColdPart.setAddress(NextAvailableAddress); - ColdPart.setImageAddress(ColdSection->getAllocAddress()); - ColdPart.setImageSize(ColdSection->getOutputSize()); - ColdPart.setFileOffset(getFileOffsetForAddress(NextAvailableAddress)); - ColdSection->setOutputAddress(ColdPart.getAddress()); + FF.setAddress(NextAvailableAddress); + FF.setImageAddress(ColdSection->getAllocAddress()); + FF.setImageSize(ColdSection->getOutputSize()); + FF.setFileOffset(getFileOffsetForAddress(NextAvailableAddress)); + ColdSection->setOutputAddress(FF.getAddress()); } - LLVM_DEBUG(dbgs() << "BOLT: mapping cold fragment 0x" - << Twine::utohexstr(ColdPart.getImageAddress()) - << " to 0x" << Twine::utohexstr(ColdPart.getAddress()) - << " with size " - << Twine::utohexstr(ColdPart.getImageSize()) << '\n'); + LLVM_DEBUG( + dbgs() << formatv( + "BOLT: mapping cold fragment {0:x+} to {1:x+} with size {2:x+}\n", + FF.getImageAddress(), FF.getAddress(), FF.getImageSize())); RTDyld.reassignSectionAddress(ColdSection->getSectionID(), - ColdPart.getAddress()); + FF.getAddress()); - NextAvailableAddress += ColdPart.getImageSize(); + NextAvailableAddress += FF.getImageSize(); } } @@ -4474,20 +4471,22 @@ ICFSymbol.st_shndx = ICFParent->getCodeSection()->getIndex(); Symbols.emplace_back(ICFSymbol); } - if (Function.isSplit() && Function.cold().getAddress()) { + if (Function.isSplit()) { for (const FunctionFragment &FF : Function.getLayout().getSplitFragments()) { - ELFSymTy NewColdSym = FunctionSymbol; - const SmallString<256> SymbolName = formatv( - "{0}.cold.{1}", cantFail(FunctionSymbol.getName(StringSection)), - FF.getFragmentNum().get() - 1); - NewColdSym.st_name = AddToStrTab(SymbolName); - NewColdSym.st_shndx = - Function.getCodeSection(FF.getFragmentNum())->getIndex(); - NewColdSym.st_value = Function.cold().getAddress(); - NewColdSym.st_size = Function.cold().getImageSize(); - NewColdSym.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC); - Symbols.emplace_back(NewColdSym); + if (FF.getAddress()) { + ELFSymTy NewColdSym = FunctionSymbol; + const SmallString<256> SymbolName = formatv( + "{0}.cold.{1}", cantFail(FunctionSymbol.getName(StringSection)), + FF.getFragmentNum().get() - 1); + NewColdSym.st_name = AddToStrTab(SymbolName); + NewColdSym.st_shndx = + Function.getCodeSection(FF.getFragmentNum())->getIndex(); + NewColdSym.st_value = FF.getAddress(); + NewColdSym.st_size = FF.getImageSize(); + NewColdSym.setBindingAndType(ELF::STB_LOCAL, ELF::STT_FUNC); + Symbols.emplace_back(NewColdSym); + } } } if (Function.hasConstantIsland()) { @@ -4612,11 +4611,24 @@ NewSymbol.st_value = OutputAddress; // Force secondary entry points to have zero size. NewSymbol.st_size = 0; + + // Find fragment containing entrypoint + FunctionLayout::fragment_const_iterator FF = llvm::find_if( + Function->getLayout().fragments(), [&](const FunctionFragment &FF) { + uint64_t Lo = FF.getAddress(); + uint64_t Hi = Lo + FF.getImageSize(); + return Lo <= OutputAddress && OutputAddress < Hi; + }); + + if (FF == Function->getLayout().fragment_end()) { + errs() << formatv("BOLT-WARNING: unable to find fragment containg " + "secondary entrypoint of {0}\n", + *Function); + FF = Function->getLayout().fragment_begin(); + } + NewSymbol.st_shndx = - OutputAddress >= Function->cold().getAddress() && - OutputAddress < Function->cold().getImageSize() - ? Function->getCodeSection(FragmentNum::cold())->getIndex() - : Function->getCodeSection()->getIndex(); + Function->getCodeSection(FF->getFragmentNum())->getIndex(); } else { // Check if the symbol belongs to moved data object and update it. BinaryData *BD = opts::ReorderData.empty() @@ -4721,8 +4733,10 @@ SmallVector Buf; NewColdSym.st_name = AddToStrTab( Twine(Function->getPrintName()).concat(".cold.0").toStringRef(Buf)); - NewColdSym.st_value = Function->cold().getAddress(); - NewColdSym.st_size = Function->cold().getImageSize(); + const FunctionFragment &ColdFF = + Function->getLayout().getFragment(FragmentNum::cold()); + NewColdSym.st_value = ColdFF.getAddress(); + NewColdSym.st_size = ColdFF.getImageSize(); Symbols.emplace_back(NewColdSym); } } @@ -5147,9 +5161,21 @@ continue; } - if (Function->isSplit() && (Function->cold().getImageAddress() == 0 || - Function->cold().getImageSize() == 0)) + const auto HasAddress = [](const FunctionFragment &FF) { + return FF.empty() || + (FF.getImageAddress() != 0 && FF.getImageSize() != 0); + }; + const bool SplitFragmentsHaveAddress = + llvm::all_of(Function->getLayout().getSplitFragments(), HasAddress); + if (Function->isSplit() && !SplitFragmentsHaveAddress) { + const auto HasNoAddress = [](const FunctionFragment &FF) { + return FF.getImageAddress() == 0 && FF.getImageSize() == 0; + }; + assert(llvm::all_of(Function->getLayout().getSplitFragments(), + HasNoAddress) && + "Some split fragments have an address while others do not"); continue; + } OverwrittenScore += Function->getFunctionScore(); // Overwrite function in the output file. @@ -5181,12 +5207,14 @@ // Write cold part if (opts::Verbosity >= 2) - outs() << "BOLT: rewriting function \"" << *Function - << "\" (cold part)\n"; + outs() << formatv("BOLT: rewriting function \"{0}\" (split parts)\n", + *Function); - OS.pwrite(reinterpret_cast(Function->cold().getImageAddress()), - Function->cold().getImageSize(), - Function->cold().getFileOffset()); + for (const FunctionFragment &FF : + Function->getLayout().getSplitFragments()) { + OS.pwrite(reinterpret_cast(FF.getImageAddress()), + FF.getImageSize(), FF.getFileOffset()); + } ++CountOverwrittenFunctions; if (opts::MaxFunctions && CountOverwrittenFunctions == opts::MaxFunctions) {