Index: include/llvm/MC/MCAssembler.h =================================================================== --- include/llvm/MC/MCAssembler.h +++ include/llvm/MC/MCAssembler.h @@ -125,11 +125,6 @@ // refactoring too. mutable SmallPtrSet ThumbFuncs; - /// \brief The bundle alignment size currently set in the assembler. - /// - /// By default it's 0, which means bundling is disabled. - unsigned BundleAlignSize; - unsigned RelaxAll : 1; unsigned SubsectionsViaSymbols : 1; unsigned IncrementalLinkerCompatible : 1; @@ -273,6 +268,9 @@ // Layout all section and prepare them for emission. void layout(MCAsmLayout &Layout); + void writeNopData(MCAsmLayout const &Layout, uint64_t Offset, uint64_t Count, + MCObjectWriter *OW, uint64_t BundleSize = 0) const; + // FIXME: This does not belong here. bool getSubsectionsViaSymbols() const { return SubsectionsViaSymbols; } void setSubsectionsViaSymbols(bool Value) { SubsectionsViaSymbols = Value; } @@ -287,16 +285,6 @@ bool getRelaxAll() const { return RelaxAll; } void setRelaxAll(bool Value) { RelaxAll = Value; } - bool isBundlingEnabled() const { return BundleAlignSize != 0; } - - unsigned getBundleAlignSize() const { return BundleAlignSize; } - - void setBundleAlignSize(unsigned Size) { - assert((Size == 0 || !(Size & (Size - 1))) && - "Expect a power-of-two bundle align size"); - BundleAlignSize = Size; - } - /// \name Section List Access /// @{ @@ -407,22 +395,11 @@ FileNames.push_back(FileName); } - /// \brief Write the necessary bundle padding to the given object writer. - /// Expects a fragment \p F containing instructions and its size \p FSize. - void writeFragmentPadding(const MCFragment &F, uint64_t FSize, - MCObjectWriter *OW) const; - /// @} void dump(); }; -/// \brief Compute the amount of padding required before the fragment \p F to -/// obey bundling restrictions, where \p FOffset is the fragment's offset in -/// its section and \p FSize is the fragment's size. -uint64_t computeBundlePadding(const MCAssembler &Assembler, const MCFragment *F, - uint64_t FOffset, uint64_t FSize); - } // end namespace llvm #endif Index: include/llvm/MC/MCELFStreamer.h =================================================================== --- include/llvm/MC/MCELFStreamer.h +++ include/llvm/MC/MCELFStreamer.h @@ -35,7 +35,6 @@ /// state management void reset() override { SeenIdent = false; - BundleGroups.clear(); MCObjectStreamer::reset(); } @@ -77,12 +76,7 @@ void FinishImpl() override; - void EmitBundleAlignMode(unsigned AlignPow2) override; - void EmitBundleLock(bool AlignToEnd) override; - void EmitBundleUnlock() override; - private: - bool isBundleLocked() const; void EmitInstToFragment(const MCInst &Inst, const MCSubtargetInfo &) override; void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override; @@ -92,10 +86,6 @@ void mergeFragment(MCDataFragment *, MCDataFragment *); bool SeenIdent; - - /// BundleGroups - The stack of fragments holding the bundle-locked - /// instructions. - llvm::SmallVector BundleGroups; }; MCELFStreamer *createARMELFStreamer(MCContext &Context, MCAsmBackend &TAB, Index: include/llvm/MC/MCFragment.h =================================================================== --- include/llvm/MC/MCFragment.h +++ include/llvm/MC/MCFragment.h @@ -42,6 +42,8 @@ FT_SafeSEH, FT_CVInlineLines, FT_CVDefRange, + FT_BundleLock, + FT_BundleUnlock, FT_Dummy }; @@ -52,11 +54,7 @@ bool HasInstructions; private: - /// \brief Should this fragment be aligned to the end of a bundle? - bool AlignToBundleEnd; - uint8_t BundlePadding; - /// LayoutOrder - The layout order of this fragment. unsigned LayoutOrder; @@ -80,7 +78,7 @@ protected: MCFragment(FragmentType Kind, bool HasInstructions, - uint8_t BundlePadding, MCSection *Parent = nullptr); + MCSection *Parent = nullptr); ~MCFragment(); private: @@ -111,21 +109,6 @@ /// this is false, but specific fragment types may set it to true. bool hasInstructions() const { return HasInstructions; } - /// \brief Should this fragment be placed at the end of an aligned bundle? - bool alignToBundleEnd() const { return AlignToBundleEnd; } - void setAlignToBundleEnd(bool V) { AlignToBundleEnd = V; } - - /// \brief Get the padding size that must be inserted before this fragment. - /// Used for bundling. By default, no padding is inserted. - /// Note that padding size is restricted to 8 bits. This is an optimization - /// to reduce the amount of space used for each fragment. In practice, larger - /// padding should never be required. - uint8_t getBundlePadding() const { return BundlePadding; } - - /// \brief Set the padding size for this fragment. By default it's a no-op, - /// and only some fragments have a meaningful implementation. - void setBundlePadding(uint8_t N) { BundlePadding = N; } - /// \brief Return true if given frgment has FT_Dummy type. bool isDummy() const { return Kind == FT_Dummy; } @@ -135,7 +118,7 @@ class MCDummyFragment : public MCFragment { public: explicit MCDummyFragment(MCSection *Sec) - : MCFragment(FT_Dummy, false, 0, Sec){}; + : MCFragment(FT_Dummy, false, Sec){}; static bool classof(const MCFragment *F) { return F->getKind() == FT_Dummy; } }; @@ -146,7 +129,7 @@ protected: MCEncodedFragment(MCFragment::FragmentType FType, bool HasInstructions, MCSection *Sec) - : MCFragment(FType, HasInstructions, 0, Sec) {} + : MCFragment(FType, HasInstructions, Sec) {} public: static bool classof(const MCFragment *F) { @@ -297,7 +280,7 @@ public: MCAlignFragment(unsigned Alignment, int64_t Value, unsigned ValueSize, unsigned MaxBytesToEmit, MCSection *Sec = nullptr) - : MCFragment(FT_Align, false, 0, Sec), Alignment(Alignment), + : MCFragment(FT_Align, false, Sec), Alignment(Alignment), EmitNops(false), Value(Value), ValueSize(ValueSize), MaxBytesToEmit(MaxBytesToEmit) {} @@ -332,7 +315,7 @@ public: MCFillFragment(uint8_t Value, uint64_t Size, MCSection *Sec = nullptr) - : MCFragment(FT_Fill, false, 0, Sec), Value(Value), Size(Size) {} + : MCFragment(FT_Fill, false, Sec), Value(Value), Size(Size) {} uint8_t getValue() const { return Value; } uint64_t getSize() const { return Size; } @@ -352,7 +335,7 @@ public: MCOrgFragment(const MCExpr &Offset, int8_t Value, MCSection *Sec = nullptr) - : MCFragment(FT_Org, false, 0, Sec), Offset(&Offset), Value(Value) {} + : MCFragment(FT_Org, false, Sec), Offset(&Offset), Value(Value) {} /// \name Accessors /// @{ @@ -380,7 +363,7 @@ public: MCLEBFragment(const MCExpr &Value_, bool IsSigned_, MCSection *Sec = nullptr) - : MCFragment(FT_LEB, false, 0, Sec), Value(&Value_), IsSigned(IsSigned_) { + : MCFragment(FT_LEB, false, Sec), Value(&Value_), IsSigned(IsSigned_) { Contents.push_back(0); } @@ -416,7 +399,7 @@ public: MCDwarfLineAddrFragment(int64_t LineDelta, const MCExpr &AddrDelta, MCSection *Sec = nullptr) - : MCFragment(FT_Dwarf, false, 0, Sec), LineDelta(LineDelta), + : MCFragment(FT_Dwarf, false, Sec), LineDelta(LineDelta), AddrDelta(&AddrDelta) { Contents.push_back(0); } @@ -448,7 +431,7 @@ public: MCDwarfCallFrameFragment(const MCExpr &AddrDelta, MCSection *Sec = nullptr) - : MCFragment(FT_DwarfFrame, false, 0, Sec), AddrDelta(&AddrDelta) { + : MCFragment(FT_DwarfFrame, false, Sec), AddrDelta(&AddrDelta) { Contents.push_back(0); } @@ -472,7 +455,7 @@ public: MCSafeSEHFragment(const MCSymbol *Sym, MCSection *Sec = nullptr) - : MCFragment(FT_SafeSEH, false, 0, Sec), Sym(Sym) {} + : MCFragment(FT_SafeSEH, false, Sec), Sym(Sym) {} /// \name Accessors /// @{ @@ -508,7 +491,7 @@ const MCSymbol *FnEndSym, ArrayRef SecondaryFuncs, MCSection *Sec = nullptr) - : MCFragment(FT_CVInlineLines, false, 0, Sec), SiteFuncId(SiteFuncId), + : MCFragment(FT_CVInlineLines, false, Sec), SiteFuncId(SiteFuncId), StartFileId(StartFileId), StartLineNum(StartLineNum), FnStartSym(FnStartSym), FnEndSym(FnEndSym), SecondaryFuncs(SecondaryFuncs.begin(), SecondaryFuncs.end()) {} @@ -560,6 +543,66 @@ } }; +// If bundling is enabled, instruction fragments have to +// obey the bundling restrictions. BundleLockFrgaments have an adaptive size to +// satisfy padding requirements. With padding, we'll have: +// +// +// BundleLockFragment +// ||| +// ------------------------------------- +// Prev |##########| F | +// ------------------------------------- +// ^ +// | +// F->Offset +// +// The fragment's offset will point to after the padding, and its computed +// size won't include the padding. +// + +class MCBundleLockFragment : public MCFragment { + + /// \brief Get the padding size that must be inserted before this fragment. + /// Used for bundling. By default, no padding is inserted. + /// Note that padding size is restricted to 8 bits. This is an optimization + /// to reduce the amount of space used for each fragment. In practice, larger + /// padding should never be required. + unsigned BundlePadding : 7; + + /// \brief Should this fragment be placed at the end of an aligned bundle? + unsigned AlignToBundleEnd : 1; + +public: + MCBundleLockFragment(unsigned BundlePadding, bool AlignToBundleEnd, + MCSection *Sec = nullptr) + : MCFragment(FT_BundleLock, false, Sec), BundlePadding(BundlePadding), + AlignToBundleEnd(AlignToBundleEnd) {} + + unsigned getBundlePadding() const { return 1U << BundlePadding; } + + bool getAlignToBundleEnd() const { return AlignToBundleEnd != 0; } + + void setAlignToBundleEnd(bool Val = true) { AlignToBundleEnd = Val; } + + uint64_t getSize(MCAsmLayout const &Layout) const; + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_BundleLock; + } +}; + +class MCBundleUnlockFragment : public MCFragment { + +public: + MCBundleUnlockFragment(MCSection *Sec = nullptr) + : MCFragment(FT_BundleUnlock, false, Sec) {} + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_BundleUnlock; + } +}; + } // end namespace llvm #endif Index: include/llvm/MC/MCObjectStreamer.h =================================================================== --- include/llvm/MC/MCObjectStreamer.h +++ include/llvm/MC/MCObjectStreamer.h @@ -39,6 +39,9 @@ bool EmitEHFrame; bool EmitDebugFrame; SmallVector PendingLabels; + unsigned BundleDepth; + unsigned BundleAlignment; + MCBundleLockFragment * BundleCurrent; virtual void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo&) = 0; void EmitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override; Index: lib/MC/MCAssembler.cpp =================================================================== --- lib/MC/MCAssembler.cpp +++ lib/MC/MCAssembler.cpp @@ -68,7 +68,7 @@ MCAssembler::MCAssembler(MCContext &Context_, MCAsmBackend &Backend_, MCCodeEmitter &Emitter_, MCObjectWriter &Writer_) : Context(Context_), Backend(Backend_), Emitter(Emitter_), Writer(Writer_), - BundleAlignSize(0), RelaxAll(false), SubsectionsViaSymbols(false), + RelaxAll(false), SubsectionsViaSymbols(false), IncrementalLinkerCompatible(false), ELFHeaderEFlags(0) { VersionMinInfo.Major = 0; // Major version == 0 for "none specified" } @@ -84,7 +84,6 @@ LinkerOptions.clear(); FileNames.clear(); ThumbFuncs.clear(); - BundleAlignSize = 0; RelaxAll = false; SubsectionsViaSymbols = false; IncrementalLinkerCompatible = false; @@ -305,6 +304,10 @@ return cast(F).getContents().size(); case MCFragment::FT_CVDefRange: return cast(F).getContents().size(); + case MCFragment::FT_BundleLock: + return cast(F).getSize(Layout); + case MCFragment::FT_BundleUnlock: + return 0; case MCFragment::FT_Dummy: llvm_unreachable("Should not have been added"); } @@ -330,49 +333,6 @@ else F->Offset = 0; LastValidFragment[F->getParent()] = F; - - // If bundling is enabled and this fragment has instructions in it, it has to - // obey the bundling restrictions. With padding, we'll have: - // - // - // BundlePadding - // ||| - // ------------------------------------- - // Prev |##########| F | - // ------------------------------------- - // ^ - // | - // F->Offset - // - // The fragment's offset will point to after the padding, and its computed - // size won't include the padding. - // - // When the -mc-relax-all flag is used, we optimize bundling by writting the - // padding directly into fragments when the instructions are emitted inside - // the streamer. When the fragment is larger than the bundle size, we need to - // ensure that it's bundle aligned. This means that if we end up with - // multiple fragments, we must emit bundle padding between fragments. - // - // ".align N" is an example of a directive that introduces multiple - // fragments. We could add a special case to handle ".align N" by emitting - // within-fragment padding (which would produce less padding when N is less - // than the bundle size), but for now we don't. - // - if (Assembler.isBundlingEnabled() && F->hasInstructions()) { - assert(isa(F) && - "Only MCEncodedFragment implementations have instructions"); - uint64_t FSize = Assembler.computeFragmentSize(*this, *F); - - if (!Assembler.getRelaxAll() && FSize > Assembler.getBundleAlignSize()) - report_fatal_error("Fragment can't be larger than a bundle size"); - - uint64_t RequiredBundlePadding = computeBundlePadding(Assembler, F, - F->Offset, FSize); - if (RequiredBundlePadding > UINT8_MAX) - report_fatal_error("Padding cannot exceed 255 bytes"); - F->setBundlePadding(static_cast(RequiredBundlePadding)); - F->Offset += RequiredBundlePadding; - } } void MCAssembler::registerSymbol(const MCSymbol &Symbol, bool *Created) { @@ -385,36 +345,30 @@ } } -void MCAssembler::writeFragmentPadding(const MCFragment &F, uint64_t FSize, - MCObjectWriter *OW) const { - // Should NOP padding be written out before this fragment? - unsigned BundlePadding = F.getBundlePadding(); - if (BundlePadding > 0) { - assert(isBundlingEnabled() && - "Writing bundle padding with disabled bundling"); - assert(F.hasInstructions() && - "Writing bundle padding for a fragment without instructions"); - - unsigned TotalLength = BundlePadding + static_cast(FSize); - if (F.alignToBundleEnd() && TotalLength > getBundleAlignSize()) { - // If the padding itself crosses a bundle boundary, it must be emitted - // in 2 pieces, since even nop instructions must not cross boundaries. - // v--------------v <- BundleAlignSize - // v---------v <- BundlePadding - // ---------------------------- - // | Prev |####|####| F | - // ---------------------------- - // ^-------------------^ <- TotalLength - unsigned DistanceToBoundary = TotalLength - getBundleAlignSize(); - if (!getBackend().writeNopData(DistanceToBoundary, OW)) - report_fatal_error("unable to write NOP sequence of " + - Twine(DistanceToBoundary) + " bytes"); - BundlePadding -= DistanceToBoundary; - } - if (!getBackend().writeNopData(BundlePadding, OW)) - report_fatal_error("unable to write NOP sequence of " + - Twine(BundlePadding) + " bytes"); +void MCAssembler::writeNopData(MCAsmLayout const &Layout, uint64_t Offset, + uint64_t Count, MCObjectWriter *OW, + uint64_t BundleSize) const { + if (BundleSize == 0) { + getBackend().writeNopData(Count, OW); + return; } + // If the padding itself crosses a bundle boundary, it must be emitted + // in 2 pieces, since even nop instructions must not cross boundaries. + // v--------------v <- BundleAlignSize + // v---------v <- BundlePadding + // ---------------------------- + // | Prev |####|####| F | + // ---------------------------- + // ^-------------------^ <- TotalLength + while (Count > 0) { + uint64_t NextBoundary = ((Offset / BundleSize) + 1) * BundleSize; + uint64_t Amount = std::min(Count, NextBoundary - Offset); + Count -= Amount; + Offset += Amount; + if (!getBackend().writeNopData(Amount, OW)) + report_fatal_error("unable to write nop sequence of " + + Twine(Amount) + " bytes"); + } } /// \brief Write the fragment \p F to the output file. @@ -425,8 +379,6 @@ // FIXME: Embed in fragments instead? uint64_t FragmentSize = Asm.computeFragmentSize(Layout, F); - Asm.writeFragmentPadding(F, FragmentSize, OW); - // This variable (and its dummy usage) is to participate in the assert at // the end of the function. uint64_t Start = OW->getStream().tell(); @@ -456,9 +408,7 @@ // bytes left to fill use the Value and ValueSize to fill the rest. // If we are aligning with nops, ask that target to emit the right data. if (AF.hasEmitNops()) { - if (!Asm.getBackend().writeNopData(Count, OW)) - report_fatal_error("unable to write nop sequence of " + - Twine(Count) + " bytes"); + Asm.writeNopData(Layout, Layout.getFragmentOffset(&F), Count, OW); break; } @@ -475,6 +425,16 @@ break; } + case MCFragment::FT_BundleLock: { + uint64_t AlignSize = cast(F).getBundlePadding(); + Asm.writeNopData(Layout, Layout.getFragmentOffset(&F), FragmentSize, OW, + AlignSize); + break; + } + + case MCFragment::FT_BundleUnlock: + break; + case MCFragment::FT_Data: ++stats::EmittedDataFragments; OW->writeBytes(cast(F).getContents()); Index: lib/MC/MCELFStreamer.cpp =================================================================== --- lib/MC/MCELFStreamer.cpp +++ lib/MC/MCELFStreamer.cpp @@ -37,43 +37,11 @@ using namespace llvm; -bool MCELFStreamer::isBundleLocked() const { - return getCurrentSectionOnly()->isBundleLocked(); -} - MCELFStreamer::~MCELFStreamer() { } void MCELFStreamer::mergeFragment(MCDataFragment *DF, MCDataFragment *EF) { - MCAssembler &Assembler = getAssembler(); - - if (Assembler.isBundlingEnabled() && Assembler.getRelaxAll()) { - uint64_t FSize = EF->getContents().size(); - - if (FSize > Assembler.getBundleAlignSize()) - report_fatal_error("Fragment can't be larger than a bundle size"); - - uint64_t RequiredBundlePadding = computeBundlePadding( - Assembler, EF, DF->getContents().size(), FSize); - - if (RequiredBundlePadding > UINT8_MAX) - report_fatal_error("Padding cannot exceed 255 bytes"); - - if (RequiredBundlePadding > 0) { - SmallString<256> Code; - raw_svector_ostream VecOS(Code); - MCObjectWriter *OW = Assembler.getBackend().createObjectWriter(VecOS); - - EF->setBundlePadding(static_cast(RequiredBundlePadding)); - - Assembler.writeFragmentPadding(*EF, FSize, OW); - delete OW; - - DF->getContents().append(Code.begin(), Code.end()); - } - } - flushPendingLabels(DF, DF->getContents().size()); for (unsigned i = 0, e = EF->getFixups().size(); i != e; ++i) { @@ -123,24 +91,9 @@ llvm_unreachable("invalid assembler flag!"); } -// If bundle alignment is used and there are any instructions in the section, it -// needs to be aligned to at least the bundle size. -static void setSectionAlignmentForBundling(const MCAssembler &Assembler, - MCSection *Section) { - if (Section && Assembler.isBundlingEnabled() && Section->hasInstructions() && - Section->getAlignment() < Assembler.getBundleAlignSize()) - Section->setAlignment(Assembler.getBundleAlignSize()); -} - void MCELFStreamer::ChangeSection(MCSection *Section, const MCExpr *Subsection) { - MCSection *CurSection = getCurrentSectionOnly(); - if (CurSection && isBundleLocked()) - report_fatal_error("Unterminated .bundle_lock when changing a section"); - MCAssembler &Asm = getAssembler(); - // Ensure the previous section gets aligned if necessary. - setSectionAlignmentForBundling(Asm, CurSection); auto *SectionELF = static_cast(Section); const MCSymbol *Grp = SectionELF->getGroup(); if (Grp) @@ -344,8 +297,6 @@ void MCELFStreamer::EmitValueImpl(const MCExpr *Value, unsigned Size, SMLoc Loc) { - if (isBundleLocked()) - report_fatal_error("Emitting values inside a locked bundle is forbidden"); fixSymbolsInTLSFixups(Value); MCObjectStreamer::EmitValueImpl(Value, Size, Loc); } @@ -354,8 +305,6 @@ int64_t Value, unsigned ValueSize, unsigned MaxBytesToEmit) { - if (isBundleLocked()) - report_fatal_error("Emitting values inside a locked bundle is forbidden"); MCObjectStreamer::EmitValueToAlignment(ByteAlignment, Value, ValueSize, MaxBytesToEmit); } @@ -483,65 +432,8 @@ for (unsigned i = 0, e = Fixups.size(); i != e; ++i) fixSymbolsInTLSFixups(Fixups[i].getValue()); - // There are several possibilities here: - // - // If bundling is disabled, append the encoded instruction to the current data - // fragment (or create a new such fragment if the current fragment is not a - // data fragment). - // - // If bundling is enabled: - // - If we're not in a bundle-locked group, emit the instruction into a - // fragment of its own. If there are no fixups registered for the - // instruction, emit a MCCompactEncodedInstFragment. Otherwise, emit a - // MCDataFragment. - // - If we're in a bundle-locked group, append the instruction to the current - // data fragment because we want all the instructions in a group to get into - // the same fragment. Be careful not to do that for the first instruction in - // the group, though. - MCDataFragment *DF; + MCDataFragment *DF = getOrCreateDataFragment(); - if (Assembler.isBundlingEnabled()) { - MCSection &Sec = *getCurrentSectionOnly(); - if (Assembler.getRelaxAll() && isBundleLocked()) - // If the -mc-relax-all flag is used and we are bundle-locked, we re-use - // the current bundle group. - DF = BundleGroups.back(); - else if (Assembler.getRelaxAll() && !isBundleLocked()) - // When not in a bundle-locked group and the -mc-relax-all flag is used, - // we create a new temporary fragment which will be later merged into - // the current fragment. - DF = new MCDataFragment(); - else if (isBundleLocked() && !Sec.isBundleGroupBeforeFirstInst()) - // If we are bundle-locked, we re-use the current fragment. - // The bundle-locking directive ensures this is a new data fragment. - DF = cast(getCurrentFragment()); - else if (!isBundleLocked() && Fixups.size() == 0) { - // Optimize memory usage by emitting the instruction to a - // MCCompactEncodedInstFragment when not in a bundle-locked group and - // there are no fixups registered. - MCCompactEncodedInstFragment *CEIF = new MCCompactEncodedInstFragment(); - insert(CEIF); - CEIF->getContents().append(Code.begin(), Code.end()); - return; - } else { - DF = new MCDataFragment(); - insert(DF); - } - if (Sec.getBundleLockState() == MCSection::BundleLockedAlignToEnd) { - // If this fragment is for a group marked "align_to_end", set a flag - // in the fragment. This can happen after the fragment has already been - // created if there are nested bundle_align groups and an inner one - // is the one marked align_to_end. - DF->setAlignToBundleEnd(true); - } - - // We're now emitting an instruction in a bundle group, so this flag has - // to be turned off. - Sec.setBundleGroupBeforeFirstInst(false); - } else { - DF = getOrCreateDataFragment(); - } - // Add the fixups and data. for (unsigned i = 0, e = Fixups.size(); i != e; ++i) { Fixups[i].setOffset(Fixups[i].getOffset() + DF->getContents().size()); @@ -549,85 +441,9 @@ } DF->setHasInstructions(true); DF->getContents().append(Code.begin(), Code.end()); - - if (Assembler.isBundlingEnabled() && Assembler.getRelaxAll()) { - if (!isBundleLocked()) { - mergeFragment(getOrCreateDataFragment(), DF); - delete DF; - } - } } -void MCELFStreamer::EmitBundleAlignMode(unsigned AlignPow2) { - assert(AlignPow2 <= 30 && "Invalid bundle alignment"); - MCAssembler &Assembler = getAssembler(); - if (AlignPow2 > 0 && (Assembler.getBundleAlignSize() == 0 || - Assembler.getBundleAlignSize() == 1U << AlignPow2)) - Assembler.setBundleAlignSize(1U << AlignPow2); - else - report_fatal_error(".bundle_align_mode cannot be changed once set"); -} - -void MCELFStreamer::EmitBundleLock(bool AlignToEnd) { - MCSection &Sec = *getCurrentSectionOnly(); - - // Sanity checks - // - if (!getAssembler().isBundlingEnabled()) - report_fatal_error(".bundle_lock forbidden when bundling is disabled"); - - if (!isBundleLocked()) - Sec.setBundleGroupBeforeFirstInst(true); - - if (getAssembler().getRelaxAll() && !isBundleLocked()) { - // TODO: drop the lock state and set directly in the fragment - MCDataFragment *DF = new MCDataFragment(); - BundleGroups.push_back(DF); - } - - Sec.setBundleLockState(AlignToEnd ? MCSection::BundleLockedAlignToEnd - : MCSection::BundleLocked); -} - -void MCELFStreamer::EmitBundleUnlock() { - MCSection &Sec = *getCurrentSectionOnly(); - - // Sanity checks - if (!getAssembler().isBundlingEnabled()) - report_fatal_error(".bundle_unlock forbidden when bundling is disabled"); - else if (!isBundleLocked()) - report_fatal_error(".bundle_unlock without matching lock"); - else if (Sec.isBundleGroupBeforeFirstInst()) - report_fatal_error("Empty bundle-locked group is forbidden"); - - // When the -mc-relax-all flag is used, we emit instructions to fragments - // stored on a stack. When the bundle unlock is emitted, we pop a fragment - // from the stack a merge it to the one below. - if (getAssembler().getRelaxAll()) { - assert(!BundleGroups.empty() && "There are no bundle groups"); - MCDataFragment *DF = BundleGroups.back(); - - // FIXME: Use BundleGroups to track the lock state instead. - Sec.setBundleLockState(MCSection::NotBundleLocked); - - // FIXME: Use more separate fragments for nested groups. - if (!isBundleLocked()) { - mergeFragment(getOrCreateDataFragment(), DF); - BundleGroups.pop_back(); - delete DF; - } - - if (Sec.getBundleLockState() != MCSection::BundleLockedAlignToEnd) - getOrCreateDataFragment()->setAlignToBundleEnd(false); - } else - Sec.setBundleLockState(MCSection::NotBundleLocked); -} - void MCELFStreamer::FinishImpl() { - // Ensure the last section gets aligned if necessary. - MCSection *CurSection = getCurrentSectionOnly(); - setSectionAlignmentForBundling(getAssembler(), CurSection); - EmitFrames(nullptr); this->MCObjectStreamer::FinishImpl(); Index: lib/MC/MCFragment.cpp =================================================================== --- lib/MC/MCFragment.cpp +++ lib/MC/MCFragment.cpp @@ -188,14 +188,22 @@ return getSectionAddressSize(Sec); } -uint64_t llvm::computeBundlePadding(const MCAssembler &Assembler, - const MCFragment *F, - uint64_t FOffset, uint64_t FSize) { - uint64_t BundleSize = Assembler.getBundleAlignSize(); - assert(BundleSize > 0 && - "computeBundlePadding should only be called if bundling is enabled"); +uint64_t MCBundleLockFragment::getSize(MCAsmLayout const &Layout) const { + uint64_t BundleSize = getBundlePadding(); uint64_t BundleMask = BundleSize - 1; - uint64_t OffsetInBundle = FOffset & BundleMask; + uint64_t OffsetInBundle = Layout.getFragmentOffset(this) & BundleMask; + uint64_t FSize = 0; + for (ilist_iterator I = ++getIterator(), + N = getParent()->getFragmentList().end(); ; ++I) { + assert (I != N && "BundleLock without BundleUnlock"); + MCFragment const &Fragment = *I; + assert(Fragment.getKind() != FT_BundleLock && "BundleLock without Unlock"); + if (Fragment.getKind() == FT_BundleUnlock) + break; + FSize += Layout.getAssembler().computeFragmentSize(Layout, Fragment); + } + if(FSize > BundleSize) + report_fatal_error("Fragment can't be larger than a bundle size"); uint64_t EndOfFragment = OffsetInBundle + FSize; // There are two kinds of bundling restrictions: @@ -205,7 +213,7 @@ // 2) Otherwise, check if the fragment would cross a bundle boundary. If it // would, add padding until the end of the bundle so that the fragment // will start in a new one. - if (F->alignToBundleEnd()) { + if (AlignToBundleEnd) { // Three possibilities here: // // A) The fragment just happens to end at a bundle boundary, so we're good. @@ -235,17 +243,14 @@ V->destroy(); } -MCFragment::MCFragment() : Kind(FragmentType(~0)), HasInstructions(false), - AlignToBundleEnd(false), BundlePadding(0) { -} +MCFragment::MCFragment() : Kind(FragmentType(~0)), HasInstructions(false) {} MCFragment::~MCFragment() { } MCFragment::MCFragment(FragmentType Kind, bool HasInstructions, - uint8_t BundlePadding, MCSection *Parent) - : Kind(Kind), HasInstructions(HasInstructions), AlignToBundleEnd(false), - BundlePadding(BundlePadding), Parent(Parent), Atom(nullptr), - Offset(~UINT64_C(0)) { + MCSection *Parent) + : Kind(Kind), HasInstructions(HasInstructions), Parent(Parent), + Atom(nullptr), Offset(~UINT64_C(0)) { if (Parent && !isDummy()) Parent->getFragmentList().push_back(this); } @@ -294,6 +299,12 @@ case FT_CVDefRange: delete cast(this); return; + case FT_BundleLock: + delete cast(this); + return; + case FT_BundleUnlock: + delete cast(this); + return; case FT_Dummy: delete cast(this); return; @@ -334,13 +345,14 @@ case MCFragment::FT_SafeSEH: OS << "MCSafeSEHFragment"; break; case MCFragment::FT_CVInlineLines: OS << "MCCVInlineLineTableFragment"; break; case MCFragment::FT_CVDefRange: OS << "MCCVDefRangeTableFragment"; break; + case MCFragment::FT_BundleLock: OS << "MCBundleLockFragment"; break; + case MCFragment::FT_BundleUnlock: OS << "MCBundleUnlockFragment"; break; case MCFragment::FT_Dummy: OS << "MCDummyFragment"; break; } OS << "(getBundlePadding()) << ">"; + << " HasInstructions:" << hasInstructions(); switch (getKind()) { case MCFragment::FT_Align: { @@ -448,6 +460,15 @@ } break; } + case MCFragment::FT_BundleLock: { + const auto *F = cast(this); + OS << "\n "; + OS << "BundlePadding: "<< F->getBundlePadding(); + OS << "AlignBundleToEnd: " << F->getAlignToBundleEnd(); + break; + } + case MCFragment::FT_BundleUnlock: + break; case MCFragment::FT_Dummy: break; } Index: lib/MC/MCObjectStreamer.cpp =================================================================== --- lib/MC/MCObjectStreamer.cpp +++ lib/MC/MCObjectStreamer.cpp @@ -29,7 +29,8 @@ : MCStreamer(Context), Assembler(new MCAssembler(Context, TAB, *Emitter_, *TAB.createObjectWriter(OS))), - EmitEHFrame(true), EmitDebugFrame(false) {} + EmitEHFrame(true), EmitDebugFrame(false), BundleDepth(0), + BundleAlignment(0), BundleCurrent(nullptr) {} MCObjectStreamer::~MCObjectStreamer() { delete &Assembler->getBackend(); @@ -101,8 +102,7 @@ MCDataFragment *F = dyn_cast_or_null(getCurrentFragment()); // When bundling is enabled, we don't want to add data to a fragment that // already has instructions (see MCELFStreamer::EmitInstToData for details) - if (!F || (Assembler->isBundlingEnabled() && !Assembler->getRelaxAll() && - F->hasInstructions())) { + if (!F) { F = new MCDataFragment(); insert(F); } @@ -160,8 +160,7 @@ // Otherwise queue the label and set its fragment pointer when we emit the // next fragment. auto *F = dyn_cast_or_null(getCurrentFragment()); - if (F && !(getAssembler().isBundlingEnabled() && - getAssembler().getRelaxAll())) { + if (F) { Symbol->setFragment(F); Symbol->setOffset(F->getContents().size()); } else { @@ -194,6 +193,9 @@ void MCObjectStreamer::ChangeSection(MCSection *Section, const MCExpr *Subsection) { + if (BundleDepth != 0) + report_fatal_error("Unterminated .bundle_lock when changing a section"); + changeSectionImpl(Section, Subsection); } @@ -224,9 +226,24 @@ return Sec.hasInstructions(); } +namespace { +class BundleUnlocker { +public: + BundleUnlocker(MCObjectStreamer *Streamer) : Streamer(Streamer) {} + ~BundleUnlocker() { + if (Streamer) + Streamer->EmitBundleUnlock(); + } + MCObjectStreamer *Streamer; +}; +} + void MCObjectStreamer::EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) { + if (BundleAlignment != 0) + EmitBundleLock(false); MCStreamer::EmitInstruction(Inst, STI); + BundleUnlocker Unlocker(BundleAlignment != 0 ? this : nullptr); MCSection *Sec = getCurrentSectionOnly(); Sec->setHasInstructions(true); @@ -248,8 +265,7 @@ // - Bundling is enabled and this instruction is inside a bundle-locked // group. We want to emit all such instructions into the same data // fragment. - if (Assembler.getRelaxAll() || - (Assembler.isBundlingEnabled() && Sec->isBundleLocked())) { + if (Assembler.getRelaxAll()) { MCInst Relaxed; getAssembler().getBackend().relaxInstruction(Inst, Relaxed); while (getAssembler().getBackend().mayNeedRelaxation(Relaxed)) @@ -264,9 +280,6 @@ void MCObjectStreamer::EmitInstToFragment(const MCInst &Inst, const MCSubtargetInfo &STI) { - if (getAssembler().getRelaxAll() && getAssembler().isBundlingEnabled()) - llvm_unreachable("All instructions should have already been relaxed"); - // Always create a new, separate fragment here, because its size can change // during relaxation. MCRelaxableFragment *IF = new MCRelaxableFragment(Inst, STI); @@ -284,18 +297,6 @@ "Aligned bundling is not implemented for this object format"; #endif -void MCObjectStreamer::EmitBundleAlignMode(unsigned AlignPow2) { - llvm_unreachable(BundlingNotImplementedMsg); -} - -void MCObjectStreamer::EmitBundleLock(bool AlignToEnd) { - llvm_unreachable(BundlingNotImplementedMsg); -} - -void MCObjectStreamer::EmitBundleUnlock() { - llvm_unreachable(BundlingNotImplementedMsg); -} - void MCObjectStreamer::EmitDwarfLocDirective(unsigned FileNo, unsigned Line, unsigned Column, unsigned Flags, unsigned Isa, @@ -507,3 +508,35 @@ flushPendingLabels(nullptr); getAssembler().Finish(); } + +void MCObjectStreamer::EmitBundleAlignMode(unsigned AlignPow2) { + BundleAlignment = AlignPow2; +} + +void MCObjectStreamer::EmitBundleLock(bool AlignToEnd) { + if (BundleAlignment == 0) + report_fatal_error(".bundle_lock forbidden when bundling is disabled"); + if(BundleDepth++ != 0) { + assert(BundleCurrent != nullptr && "Nested but no current bundle"); + BundleCurrent->setAlignToBundleEnd(AlignToEnd || + BundleCurrent->getAlignToBundleEnd()); + return; + } + MCSection * Section = getCurrentSectionOnly(); + if(Section->getFragmentList().empty()) + Section->setAlignment(1U << BundleAlignment); + MCSection *CurSection = getCurrentSectionOnly(); + auto F = new MCBundleLockFragment(BundleAlignment, AlignToEnd); + BundleCurrent = F; + CurSection->getFragmentList().insert(CurInsertionPoint, F); + F->setParent(CurSection); +} + +void MCObjectStreamer::EmitBundleUnlock() { + if(BundleDepth == 0) + report_fatal_error(".bundle_unlock without matching lock"); + --BundleDepth; + if(BundleDepth != 0) + return; + insert (new MCBundleUnlockFragment); +} Index: test/MC/X86/AlignedBundling/misaligned-bundle-group.s =================================================================== --- test/MC/X86/AlignedBundling/misaligned-bundle-group.s +++ test/MC/X86/AlignedBundling/misaligned-bundle-group.s @@ -1,9 +1,9 @@ # RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - \ # RUN: | llvm-objdump -disassemble -no-show-raw-insn - \ -# RUN: | FileCheck -check-prefix=CHECK -check-prefix=CHECK-OPT %s +# RUN: | FileCheck -check-prefix=CHECK %s # RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -mc-relax-all %s -o - \ # RUN: | llvm-objdump -disassemble -no-show-raw-insn - \ -# RUN: | FileCheck -check-prefix=CHECK -check-prefix=CHECK-RELAX %s +# RUN: | FileCheck -check-prefix=CHECK %s .text foo: @@ -13,11 +13,7 @@ .bundle_lock align_to_end # CHECK: 1: nopw %cs:(%eax,%eax) # CHECK: 10: nopw %cs:(%eax,%eax) -# CHECK-RELAX: 1f: nop -# CHECK-RELAX: 20: nopw %cs:(%eax,%eax) -# CHECK-RELAX: 2f: nopw %cs:(%eax,%eax) -# CHECK-OPT: 1b: calll -4 -# CHECK-RELAX: 3b: calll -4 +# CHECK: 1b: calll -4 calll bar # 5 bytes .bundle_unlock ret # 1 byte Index: test/MC/X86/AlignedBundling/misaligned-bundle.s =================================================================== --- test/MC/X86/AlignedBundling/misaligned-bundle.s +++ test/MC/X86/AlignedBundling/misaligned-bundle.s @@ -1,9 +1,9 @@ # RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu %s -o - \ # RUN: | llvm-objdump -disassemble -no-show-raw-insn - \ -# RUN: | FileCheck -check-prefix=CHECK -check-prefix=CHECK-OPT %s +# RUN: | FileCheck %s # RUN: llvm-mc -filetype=obj -triple i686-pc-linux-gnu -mc-relax-all %s -o - \ # RUN: | llvm-objdump -disassemble -no-show-raw-insn - \ -# RUN: | FileCheck -check-prefix=CHECK -check-prefix=CHECK-RELAX %s +# RUN: | FileCheck %s .text foo: @@ -11,21 +11,16 @@ push %ebp # 1 byte .align 16 # CHECK: 1: nopw %cs:(%eax,%eax) -# CHECK-RELAX: 10: nopw %cs:(%eax,%eax) -# CHECK-RELAX: 1f: nop -# CHECK-OPT: 10: movl $1, (%esp) -# CHECK-RELAX: 20: movl $1, (%esp) +# CHECK: 10: movl $1, (%esp) movl $0x1, (%esp) # 7 bytes movl $0x1, (%esp) # 7 bytes -# CHECK-OPT: 1e: nop +# CHECK: 1e: nop movl $0x2, 0x1(%esp) # 8 bytes movl $0x2, 0x1(%esp) # 8 bytes -# CHECK-RELAX: 3e: nop -# CHECK-RELAX: 40: movl $2, 1(%esp) movl $0x2, 0x1(%esp) # 8 bytes movl $0x2, (%esp) # 7 bytes -# CHECK-OPT: 3f: nop -# CHECK-OPT: 40: movl $3, (%esp) +# CHECK: 3f: nop +# CHECK: 40: movl $3, (%esp) movl $0x3, (%esp) # 7 bytes movl $0x3, (%esp) # 7 bytes ret Index: test/MC/X86/AlignedBundling/nesting.s =================================================================== --- test/MC/X86/AlignedBundling/nesting.s +++ test/MC/X86/AlignedBundling/nesting.s @@ -31,7 +31,7 @@ callq foo # Check that the callqs get bundled together, and that the whole group is # align_to_end - .bundle_lock + .bundle_lock callq bar .bundle_lock align_to_end callq bar Index: test/MC/X86/AlignedBundling/relax-in-bundle-group.s =================================================================== --- test/MC/X86/AlignedBundling/relax-in-bundle-group.s +++ test/MC/X86/AlignedBundling/relax-in-bundle-group.s @@ -1,7 +1,7 @@ # RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu %s -o - \ # RUN: | llvm-objdump -disassemble - | FileCheck %s # RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu -mc-relax-all %s -o - \ -# RUN: | llvm-objdump -disassemble - | FileCheck %s +# RUN: | llvm-objdump -disassemble - | FileCheck --check-prefix=CHECK-RELAX %s # Test that instructions inside bundle-locked groups are relaxed even if their # fixup is short enough not to warrant relaxation on its own. @@ -23,16 +23,18 @@ jle .L_ELSE # This group would've started at 0x18 and is too long, so a chunky NOP padding # is inserted to push it to 0x20. -# CHECK: 18: {{[a-f0-9 ]+}} nopl +# CHECK-RELAX: 18: {{[a-f0-9 ]+}} nopl # The long encoding for JLE should be used here even though its target is close -# CHECK-NEXT: 20: 0f 8e +# CHECK: 18: 7e 06 +# CHECK-RELAX-NEXT: 20: 0f 8e addl %ebp, %eax jmp .L_RET # Same for the JMP -# CHECK: 28: e9 +# CHECK: 1c: eb +# CHECK-RELAX: 28: e9 .bundle_unlock