diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -162,6 +162,8 @@ /// Indicate that this basic block is the entry block of a cleanup funclet. bool IsCleanupFuncletEntry = false; + unsigned PropellerID{0}; + /// With basic block sections, this stores the Section ID of the basic block. MBBSectionID SectionID{0}; @@ -479,6 +481,8 @@ void setIsEndSection(bool V = true) { IsEndSection = V; } + unsigned getPropellerID() const { return PropellerID; } + /// Returns the section ID of this basic block. MBBSectionID getSectionID() const { return SectionID; } @@ -488,6 +492,8 @@ ((unsigned)SectionID.Type) + SectionID.Number; } + void setPropellerID(unsigned V) { PropellerID = V; } + /// Sets the section ID for this basic block. void setSectionID(MBBSectionID V) { SectionID = V; } diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -349,6 +349,8 @@ bool HasEHScopes = false; bool HasEHFunclets = false; + unsigned NextPropellerID = 0; + /// Section Type for basic blocks, only relevant with basic block sections. BasicBlockSection BBSectionsType = BasicBlockSection::None; diff --git a/llvm/include/llvm/Object/ELFTypes.h b/llvm/include/llvm/Object/ELFTypes.h --- a/llvm/include/llvm/Object/ELFTypes.h +++ b/llvm/include/llvm/Object/ELFTypes.h @@ -803,6 +803,7 @@ uintX_t Addr; // Function address // Struct representing the BBAddrMap information for one basic block. struct BBEntry { + uint32_t ID; // Propeller ID. uint32_t Offset; // Offset of basic block relative to function start. uint32_t Size; // Size of the basic block. @@ -813,8 +814,8 @@ bool IsEHPad; // If this is an exception handling block. bool CanFallThrough; // If this block can fall through to its next. - BBEntry(uint32_t Offset, uint32_t Size, uint32_t Metadata) - : Offset(Offset), Size(Size), HasReturn(Metadata & 1), + BBEntry(uint32_t ID, uint32_t Offset, uint32_t Size, uint32_t Metadata) + : ID(ID), Offset(Offset), Size(Size), HasReturn(Metadata & 1), HasTailCall(Metadata & (1 << 1)), IsEHPad(Metadata & (1 << 2)), CanFallThrough(Metadata & (1 << 3)){}; }; diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1130,11 +1130,14 @@ for (const MachineBasicBlock &MBB : MF) { const MCSymbol *MBBSymbol = MBB.isEntryBlock() ? FunctionSymbol : MBB.getSymbol(); + // Emit the Propeller ID for this basic block. + OutStreamer->emitULEB128IntValue(MBB.getPropellerID()); // Emit the basic block offset. emitLabelDifferenceAsULEB128(MBBSymbol, FunctionSymbol); // Emit the basic block size. When BBs have alignments, their size cannot // always be computed from their offsets. emitLabelDifferenceAsULEB128(MBB.getEndSymbol(), MBBSymbol); + // Emit the Metadata. OutStreamer->emitULEB128IntValue(getBBAddrMapMetadata(MBB)); } OutStreamer->PopSection(); diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp --- a/llvm/lib/CodeGen/BasicBlockSections.cpp +++ b/llvm/lib/CodeGen/BasicBlockSections.cpp @@ -98,8 +98,8 @@ // This struct represents the cluster information for a machine basic block. struct BBClusterInfo { - // MachineBasicBlock ID. - unsigned MBBNumber; + // MachineBasicBlock Propeller ID. + unsigned PropellerID; // Cluster ID this basic block belongs to. unsigned ClusterID; // Position of basic block within the cluster. @@ -160,14 +160,15 @@ // This function updates and optimizes the branching instructions of every basic // block in a given function to account for changes in the layout. -static void updateBranches( - MachineFunction &MF, - const SmallVector &PreLayoutFallThroughs) { +static void +updateBranches(MachineFunction &MF, + DenseMap + &PreLayoutFallThroughs) { const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); SmallVector Cond; for (auto &MBB : MF) { auto NextMBBI = std::next(MBB.getIterator()); - auto *FTMBB = PreLayoutFallThroughs[MBB.getNumber()]; + auto *FTMBB = PreLayoutFallThroughs[&MBB]; // If this block had a fallthrough before we need an explicit unconditional // branch to that block if either // 1- the block ends a section, which means its next block may be @@ -215,13 +216,13 @@ return true; } - V.resize(MF.getNumBlockIDs()); - for (auto bbClusterInfo : P->second) { - // Bail out if the cluster information contains invalid MBB numbers. - if (bbClusterInfo.MBBNumber >= MF.getNumBlockIDs()) - return false; - V[bbClusterInfo.MBBNumber] = bbClusterInfo; - } + unsigned MaxPropellerID = 0; + for (auto bbClusterInfo : P->second) + if (bbClusterInfo.PropellerID > MaxPropellerID) + MaxPropellerID = bbClusterInfo.PropellerID; + V.resize(MaxPropellerID + 1); + for (auto bbClusterInfo : P->second) + V[bbClusterInfo.PropellerID] = bbClusterInfo; return true; } @@ -234,9 +235,10 @@ // and "Cold" succeeding all other clusters. // FuncBBClusterInfo represent the cluster information for basic blocks. If this // is empty, it means unique sections for all basic blocks in the function. -static void -assignSections(MachineFunction &MF, - const std::vector> &FuncBBClusterInfo) { +static void assignSections( + MachineFunction &MF, + const std::vector> &FuncBBClusterInfo, + const DenseMap &PreLayoutPosition) { assert(MF.hasBBSections() && "BB Sections is not set for function."); // This variable stores the section ID of the cluster containing eh_pads (if // all eh_pads are one cluster). If more than one cluster contain eh_pads, we @@ -253,9 +255,10 @@ // If unique sections are desired for all basic blocks of the function, we // set every basic block's section ID equal to its number (basic block // id). This further ensures that basic blocks are ordered canonically. - MBB.setSectionID({static_cast(MBB.getNumber())}); - } else if (FuncBBClusterInfo[MBB.getNumber()].hasValue()) - MBB.setSectionID(FuncBBClusterInfo[MBB.getNumber()]->ClusterID); + MBB.setSectionID(PreLayoutPosition.lookup(&MBB)); + } else if (MBB.getPropellerID() < FuncBBClusterInfo.size() && + FuncBBClusterInfo[MBB.getPropellerID()].hasValue()) + MBB.setSectionID(FuncBBClusterInfo[MBB.getPropellerID()]->ClusterID); else { // BB goes into the special cold section if it is not specified in the // cluster info map. @@ -283,12 +286,15 @@ void llvm::sortBasicBlocksAndUpdateBranches( MachineFunction &MF, MachineBasicBlockComparator MBBCmp) { - SmallVector PreLayoutFallThroughs( - MF.getNumBlockIDs()); + const MachineBasicBlock *EntryBlock = &MF.front(); + DenseMap + PreLayoutFallThroughs; for (auto &MBB : MF) - PreLayoutFallThroughs[MBB.getNumber()] = MBB.getFallThrough(); + PreLayoutFallThroughs[&MBB] = MBB.getFallThrough(); MF.sort(MBBCmp); + // Check that the entry block is not displaced. + assert(&MF.front() == EntryBlock); // Set IsBeginSection and IsEndSection according to the assigned section IDs. MF.assignBeginEndSections(); @@ -354,11 +360,10 @@ hasInstrProfHashMismatch(MF)) return true; - // Renumber blocks before sorting them for basic block sections. This is - // useful during sorting, basic blocks in the same section will retain the - // default order. This renumbering should also be done for basic block - // labels to match the profiles with the correct blocks. - MF.RenumberBlocks(); + DenseMap PreLayoutPosition; + unsigned Position = 0; + for (const auto &MBB : MF) + PreLayoutPosition[&MBB] = Position++; if (BBSectionsType == BasicBlockSection::Labels) { MF.setBBSectionsType(BBSectionsType); @@ -371,7 +376,7 @@ FuncBBClusterInfo)) return true; MF.setBBSectionsType(BBSectionsType); - assignSections(MF, FuncBBClusterInfo); + assignSections(MF, FuncBBClusterInfo, PreLayoutPosition); // We make sure that the cluster including the entry basic block precedes all // other clusters. @@ -405,9 +410,9 @@ // If the two basic block are in the same section, the order is decided by // their position within the section. if (XSectionID.Type == MBBSectionID::SectionType::Default) - return FuncBBClusterInfo[X.getNumber()]->PositionInCluster < - FuncBBClusterInfo[Y.getNumber()]->PositionInCluster; - return X.getNumber() < Y.getNumber(); + return FuncBBClusterInfo[X.getPropellerID()]->PositionInCluster < + FuncBBClusterInfo[Y.getPropellerID()]->PositionInCluster; + return PreLayoutPosition[&X] < PreLayoutPosition[&Y]; }; sortBasicBlocksAndUpdateBranches(MF, Comparator); @@ -464,23 +469,23 @@ if (FI == ProgramBBClusterInfo.end()) return invalidProfileError( "Cluster list does not follow a function name specifier."); - SmallVector BBIndexes; - S.split(BBIndexes, ' '); + SmallVector BBIDs; + S.split(BBIDs, ' '); // Reset current cluster position. CurrentPosition = 0; - for (auto BBIndexStr : BBIndexes) { - unsigned long long BBIndex; - if (getAsUnsignedInteger(BBIndexStr, 10, BBIndex)) + for (auto BBIDStr : BBIDs) { + unsigned long long BBID; + if (getAsUnsignedInteger(BBIDStr, 10, BBID)) return invalidProfileError(Twine("Unsigned integer expected: '") + - BBIndexStr + "'."); - if (!FuncBBIDs.insert(BBIndex).second) + BBIDStr + "'."); + if (!FuncBBIDs.insert(BBID).second) return invalidProfileError(Twine("Duplicate basic block id found '") + - BBIndexStr + "'."); - if (!BBIndex && CurrentPosition) + BBIDStr + "'."); + if (BBID == 0 && CurrentPosition) return invalidProfileError("Entry BB (0) does not begin a cluster."); - FI->second.emplace_back(BBClusterInfo{ - ((unsigned)BBIndex), CurrentCluster, CurrentPosition++}); + FI->second.emplace_back( + BBClusterInfo{((unsigned)BBID), CurrentCluster, CurrentPosition++}); } CurrentCluster++; } else { // This is a function name specifier. diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -412,8 +412,13 @@ /// `new MachineBasicBlock'. MachineBasicBlock * MachineFunction::CreateMachineBasicBlock(const BasicBlock *bb) { - return new (BasicBlockRecycler.Allocate(Allocator)) - MachineBasicBlock(*this, bb); + MachineBasicBlock *MBB = + new (BasicBlockRecycler.Allocate(Allocator)) + MachineBasicBlock(*this, bb); + if (Target.getBBSectionsType() == BasicBlockSection::Labels || + Target.getBBSectionsType() == BasicBlockSection::List) + MBB->setPropellerID(NextPropellerID++); + return MBB; } /// Delete the given MachineBasicBlock. diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp --- a/llvm/lib/Object/ELF.cpp +++ b/llvm/lib/Object/ELF.cpp @@ -650,10 +650,11 @@ std::vector BBEntries; for (uint32_t BlockID = 0; !ULEBSizeErr && Cur && (BlockID < NumBlocks); ++BlockID) { + uint32_t ID = ReadULEB128AsUInt32(); uint32_t Offset = ReadULEB128AsUInt32(); uint32_t Size = ReadULEB128AsUInt32(); uint32_t Metadata = ReadULEB128AsUInt32(); - BBEntries.push_back({Offset, Size, Metadata}); + BBEntries.push_back({ID, Offset, Size, Metadata}); } FunctionEntries.push_back({Address, BBEntries}); } diff --git a/llvm/test/CodeGen/X86/basic-block-sections-labels.ll b/llvm/test/CodeGen/X86/basic-block-sections-labels.ll --- a/llvm/test/CodeGen/X86/basic-block-sections-labels.ll +++ b/llvm/test/CodeGen/X86/basic-block-sections-labels.ll @@ -39,8 +39,8 @@ ; CHECK-LABEL: .LBB_END0_1: ; CHECK-LABEL: .LBB0_2: ; CHECK-LABEL: .LBB_END0_2: -; CHECK-LABEL: .LBB0_3: -; CHECK-LABEL: .LBB_END0_3: +; CHECK-LABEL: .LBB0_4: +; CHECK-LABEL: .LBB_END0_4: ; CHECK-LABEL: .Lfunc_end0: ; UNIQ: .section .llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text._Z3bazb{{$}} @@ -48,15 +48,19 @@ ; NOUNIQ: .section .llvm_bb_addr_map,"o",@llvm_bb_addr_map,.text,unique,1 ; CHECK-NEXT: .quad .Lfunc_begin0 ; CHECK-NEXT: .byte 4 +; CHECK-NEXT: .byte 0 ; CHECK-NEXT: .uleb128 .Lfunc_begin0-.Lfunc_begin0 ; CHECK-NEXT: .uleb128 .LBB_END0_0-.Lfunc_begin0 ; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .byte 1 ; CHECK-NEXT: .uleb128 .LBB0_1-.Lfunc_begin0 ; CHECK-NEXT: .uleb128 .LBB_END0_1-.LBB0_1 ; CHECK-NEXT: .byte 8 +; CHECK-NEXT: .byte 3 ; CHECK-NEXT: .uleb128 .LBB0_2-.Lfunc_begin0 ; CHECK-NEXT: .uleb128 .LBB_END0_2-.LBB0_2 ; CHECK-NEXT: .byte 1 -; CHECK-NEXT: .uleb128 .LBB0_3-.Lfunc_begin0 -; CHECK-NEXT: .uleb128 .LBB_END0_3-.LBB0_3 +; CHECK-NEXT: .byte 2 +; CHECK-NEXT: .uleb128 .LBB0_4-.Lfunc_begin0 +; CHECK-NEXT: .uleb128 .LBB_END0_4-.LBB0_4 ; CHECK-NEXT: .byte 5 diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -6707,6 +6707,7 @@ ListScope L(W, "BB entries"); for (const typename Elf_BBAddrMap::BBEntry &BBE : AM.BBEntries) { DictScope L(W); + W.printNumber("ID", BBE.ID); W.printHex("Offset", BBE.Offset); W.printHex("Size", BBE.Size); W.printBoolean("HasReturn", BBE.HasReturn);