Index: llvm/include/llvm/MC/MCAsmBackend.h =================================================================== --- llvm/include/llvm/MC/MCAsmBackend.h +++ llvm/include/llvm/MC/MCAsmBackend.h @@ -51,6 +51,9 @@ const support::endianness Endian; + virtual void alignBranchesBegin(MCObjectStreamer &OS, const MCInst &Inst) {} + virtual void alignBranchesEnd(MCObjectStreamer &OS, const MCInst &Inst) {} + /// lifetime management virtual void reset() {} Index: llvm/include/llvm/MC/MCAssembler.h =================================================================== --- llvm/include/llvm/MC/MCAssembler.h +++ llvm/include/llvm/MC/MCAssembler.h @@ -195,7 +195,8 @@ bool relaxPaddingFragment(MCAsmLayout &Layout, MCPaddingFragment &PF); bool relaxLEB(MCAsmLayout &Layout, MCLEBFragment &IF); - + bool relaxBoundaryAlign(MCAsmLayout &Layout, + MCBoundaryAlignFragment &MF); bool relaxDwarfLineAddr(MCAsmLayout &Layout, MCDwarfLineAddrFragment &DF); bool relaxDwarfCallFrameFragment(MCAsmLayout &Layout, MCDwarfCallFrameFragment &DF); Index: llvm/include/llvm/MC/MCFragment.h =================================================================== --- llvm/include/llvm/MC/MCFragment.h +++ llvm/include/llvm/MC/MCFragment.h @@ -41,6 +41,7 @@ FT_Dwarf, FT_DwarfFrame, FT_LEB, + FT_BoundaryAlign, FT_Padding, FT_SymbolId, FT_CVInlineLines, @@ -656,6 +657,38 @@ } }; +/// Representing required padding such that a particular other fragment does +/// not cross a particular power-of-two boundary. The other fragment must +/// follow this one within the same section, but is not guaranteed to +/// immediately follow. +class MCBoundaryAlignFragment : public MCFragment { +private: + /// The size of the MCBoundaryAlignFragment. Lazily populated + /// during relaxation. + unsigned Size = 0; + /// The fragment which must be aligned so as not to cross boundary.. + const MCFragment *Fragment = nullptr; + /// The boundary which must not be crossed. Must be a power of two. + unsigned AlignBoundarySize = 0; + +public: + MCBoundaryAlignFragment(unsigned AlignBoundarySize, + MCSection *Sec = nullptr) + : MCFragment(FT_BoundaryAlign, false, Sec), + AlignBoundarySize(AlignBoundarySize) {} + + unsigned getBoundarySize() const { return AlignBoundarySize; } + + void setSize(unsigned Value) { Size = Value; } + uint64_t getSize() const { return Size; } + + void setFragment(const MCFragment *Target) { Fragment = Target; } + const MCFragment *getFragment() const { return Fragment; } + + static bool classof(const MCFragment *F) { + return F->getKind() == MCFragment::FT_BoundaryAlign; + } +}; } // end namespace llvm #endif // LLVM_MC_MCFRAGMENT_H Index: llvm/lib/MC/MCAssembler.cpp =================================================================== --- llvm/lib/MC/MCAssembler.cpp +++ llvm/lib/MC/MCAssembler.cpp @@ -313,6 +313,9 @@ case MCFragment::FT_LEB: return cast(F).getContents().size(); + case MCFragment::FT_BoundaryAlign: + return cast(F).getSize(); + case MCFragment::FT_Padding: return cast(F).getSize(); @@ -612,6 +615,15 @@ break; } + case MCFragment::FT_BoundaryAlign: { + if (FragmentSize == 0) + break; + if (!Asm.getBackend().writeNopData(OS, FragmentSize)) + report_fatal_error("unable to write nop sequence of " + + Twine(FragmentSize) + " bytes"); + break; + } + case MCFragment::FT_Padding: { if (!Asm.getBackend().writeNopData(OS, FragmentSize)) report_fatal_error("unable to write nop sequence of " + @@ -969,6 +981,69 @@ return OldSize != LF.getContents().size(); } +/// mayCrossBoundary - Check if the branch with given address and size crosses +/// the boundary. +static bool mayCrossBoundary(unsigned StartAddr, unsigned Size, + unsigned BoundarySize) { + unsigned EndAddr = StartAddr + Size; + return StartAddr / BoundarySize != ((EndAddr - 1) / BoundarySize); +} + +/// isAgainstBoundary - Check if the branch with given address and size is +/// against the boundary. +static bool isAgainstBoundary(unsigned StartAddr, unsigned Size, + unsigned BoundarySize) { + unsigned EndAddr = StartAddr + Size; + return EndAddr % BoundarySize == 0; +} + +/// needPadding - Check if the branch with given address and size needs padding. +static bool needPadding(unsigned StartAddr, unsigned Size, + unsigned BoundarySize) { + return mayCrossBoundary(StartAddr, Size, BoundarySize) || + isAgainstBoundary(StartAddr, Size, BoundarySize); +} + +/// getPaddingSize - Get how many bytes need to be padded to align branch with +/// given address if the branch cross or is against the boundary. +static unsigned getPaddingSize(unsigned StartAddr, unsigned BoundarySize) { + return BoundarySize - (StartAddr % BoundarySize); +} + +/// getInstSize - Get the size of encoded instruction in the fragment. +// Can't this be replaced w/computeFragmentSize? +static unsigned getInstSize(const MCFragment &F) { + switch (F.getKind()) { + default: + llvm_unreachable("Illegal fragment type"); + case MCFragment::FT_Data: + return cast(F).getContents().size(); + case MCFragment::FT_Relaxable: + return cast(F).getContents().size(); + case MCFragment::FT_CompactEncodedInst: + return cast(F).getContents().size(); + } +} + +bool MCAssembler::relaxBoundaryAlign(MCAsmLayout &Layout, + MCBoundaryAlignFragment &MF) { + auto *BranchFragment = MF.getFragment(); + if (!BranchFragment) + return false; + unsigned OldSize = MF.getSize(); + unsigned AlignedSize = getInstSize(*BranchFragment); + unsigned AlignedOffset = Layout.getFragmentOffset(BranchFragment); + AlignedOffset -= OldSize; + unsigned NewSize = 0; + unsigned BoundarySize = MF.getBoundarySize(); + if (needPadding(AlignedOffset, AlignedSize, BoundarySize)) { + NewSize = getPaddingSize(AlignedOffset, BoundarySize); + assert(NewSize < BoundarySize); + } + MF.setSize(NewSize); + return (NewSize != OldSize); +} + bool MCAssembler::relaxDwarfLineAddr(MCAsmLayout &Layout, MCDwarfLineAddrFragment &DF) { MCContext &Context = Layout.getAssembler().getContext(); @@ -1085,6 +1160,10 @@ case MCFragment::FT_LEB: RelaxedFrag = relaxLEB(Layout, *cast(I)); break; + case MCFragment::FT_BoundaryAlign: + RelaxedFrag = + relaxBoundaryAlign(Layout, *cast(I)); + break; case MCFragment::FT_Padding: RelaxedFrag = relaxPaddingFragment(Layout, *cast(I)); break; Index: llvm/lib/MC/MCFragment.cpp =================================================================== --- llvm/lib/MC/MCFragment.cpp +++ llvm/lib/MC/MCFragment.cpp @@ -275,6 +275,9 @@ case FT_LEB: delete cast(this); return; + case FT_BoundaryAlign: + delete cast(this); + return; case FT_Padding: delete cast(this); return; @@ -322,6 +325,7 @@ case MCFragment::FT_Dwarf: OS << "MCDwarfFragment"; break; case MCFragment::FT_DwarfFrame: OS << "MCDwarfCallFrameFragment"; break; case MCFragment::FT_LEB: OS << "MCLEBFragment"; break; + case MCFragment::FT_BoundaryAlign: OS<<"MCBoundaryAlignFragment"; break; case MCFragment::FT_Padding: OS << "MCPaddingFragment"; break; case MCFragment::FT_SymbolId: OS << "MCSymbolIdFragment"; break; case MCFragment::FT_CVInlineLines: OS << "MCCVInlineLineTableFragment"; break; @@ -422,6 +426,15 @@ OS << " Value:" << LF->getValue() << " Signed:" << LF->isSigned(); break; } + case MCFragment::FT_BoundaryAlign: { + const MCBoundaryAlignFragment *MF = + cast(this); + OS << "\n "; + OS << " Size:" << MF->getSize(); + OS << " AlignBoundarySize:" << MF->getBoundarySize(); + OS << " Fragment:" << MF->getFragment(); + break; + } case MCFragment::FT_Padding: { const MCPaddingFragment *F = cast(this); OS << "\n "; Index: llvm/lib/MC/MCObjectStreamer.cpp =================================================================== --- llvm/lib/MC/MCObjectStreamer.cpp +++ llvm/lib/MC/MCObjectStreamer.cpp @@ -330,7 +330,9 @@ void MCObjectStreamer::EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) { getAssembler().getBackend().handleCodePaddingInstructionBegin(Inst); + getAssembler().getBackend().alignBranchesBegin(*this, Inst); EmitInstructionImpl(Inst, STI); + getAssembler().getBackend().alignBranchesEnd(*this, Inst); getAssembler().getBackend().handleCodePaddingInstructionEnd(Inst); } Index: llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp =================================================================== --- llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp +++ llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp @@ -19,14 +19,18 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCFixupKindInfo.h" #include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCMachObjectWriter.h" +#include "llvm/MC/MCObjectStreamer.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCValue.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/Support/TargetRegistry.h" using namespace llvm; static unsigned getFixupKindSize(unsigned Kind) { @@ -64,6 +68,60 @@ } namespace { +class X86AlignBranchKind { +private: + uint8_t AlignBranchKind = 0; + +public: + enum Flag : uint8_t { + AlignBranchNone = 0, + AlignBranchJmp = 1U << 1, + AlignBranchCall = 1U << 2, + AlignBranchRet = 1U << 3, + AlignBranchIndirect = 1U << 4 + }; + + void operator=(const std::string &Val) { + if (Val.empty()) + return; + SmallVector BranchTypes; + StringRef(Val).split(BranchTypes, '+', -1, false); + for (auto BranchType : BranchTypes) { + if (BranchType == "jmp") + addKind(AlignBranchJmp); + else if (BranchType == "call") + addKind(AlignBranchCall); + else if (BranchType == "ret") + addKind(AlignBranchRet); + else if (BranchType == "indirect") + addKind(AlignBranchIndirect); + } + } + + operator uint8_t() const { return AlignBranchKind; } + void addKind(Flag Value) { AlignBranchKind |= Value; } +}; + +X86AlignBranchKind X86AlignBranchKindLoc; + +cl::opt X86AlignBranchBoundary( + "x86-align-branch-boundary", cl::init(0), cl::Hidden, + cl::desc("Control how the assembler should align branches with segment " + "prefixes or NOP. The boundary's size must be a power of 2. It " + "should be 0 or no less than 32. Branches will be aligned within " + "the boundary of specifies size. -x86-align-branch-boundary=0 " + "doesn't align branches.")); + +cl::opt> X86AlignBranch( + "x86-align-branch", + cl::desc("Specify types of branches to align (plus separated list of " + "types). The branches's types is combination of jmp, call, ret," + "and indirect."), + cl::Hidden, + cl::value_desc( + "jmp, which aligns unconditional jumps; call, which aligns calls; " + "ret, which aligns rets; indirect, which aligns indirect jumps."), + cl::location(X86AlignBranchKindLoc)); class X86ELFObjectWriter : public MCELFObjectTargetWriter { public: @@ -74,9 +132,25 @@ class X86AsmBackend : public MCAsmBackend { const MCSubtargetInfo &STI; + const MCInstrInfo &MCII; + X86AlignBranchKind AlignBranchType; + unsigned AlignBoundarySize = 0; + MCBoundaryAlignFragment *PendingBA = nullptr; + + bool hasVariantSymbol(const MCInst &MI) const; + bool needAlign(MCObjectStreamer &OS) const; + bool needAlignInst(const MCInst &Inst) const; + public: X86AsmBackend(const Target &T, const MCSubtargetInfo &STI) - : MCAsmBackend(support::little), STI(STI) {} + : MCAsmBackend(support::little), STI(STI), + MCII(*(T.createMCInstrInfo())) { + AlignBoundarySize = X86AlignBranchBoundary; + AlignBranchType = X86AlignBranchKindLoc; + } + + void alignBranchesBegin(MCObjectStreamer &OS, const MCInst &Inst) override; + void alignBranchesEnd(MCObjectStreamer &OS, const MCInst &Inst) override; unsigned getNumFixupKinds() const override { return X86::NumTargetFixupKinds; @@ -258,6 +332,96 @@ return getRelaxedOpcodeBranch(Inst, is16BitMode); } +/// hasVariantSymbol - Check if the instruction has variant symbol operand. +bool X86AsmBackend::hasVariantSymbol(const MCInst &MI) const { + + for (auto &Operand : MI) { + if (Operand.isExpr()) { + const MCExpr &Expr = *Operand.getExpr(); + if (Expr.getKind() == MCExpr::SymbolRef && + cast(*Operand.getExpr()).getKind() != + MCSymbolRefExpr::VK_None) + return true; + } + } + return false; +} + +bool X86AsmBackend::needAlign(MCObjectStreamer &OS) const { + if (AlignBoundarySize == 0 || + AlignBranchType == X86AlignBranchKind::AlignBranchNone) + return false; + + MCAssembler &Assembler = OS.getAssembler(); + MCSection *Sec = OS.getCurrentSectionOnly(); + // To be Done: Currently don't deal with Bundle cases. + if (Assembler.isBundlingEnabled() && Sec->isBundleLocked()) + return false; + + // Branches only need to be aligned in 32-bit or 64-bit mode. + if (!(STI.getFeatureBits()[X86::Mode64Bit] || + STI.getFeatureBits()[X86::Mode32Bit])) + return false; + + return true; +} + +/// needAlignInst - Check if the instruction operand needs to be aligned. +/// Padding is disabled before intruction which may be rewritten by linker(eg. +/// TLSCALL). +bool X86AsmBackend::needAlignInst(const MCInst &Inst) const { + // Linker may rewrite the instruction with variant symbol operand. + if(hasVariantSymbol(Inst)) return false; + + const MCInstrDesc &InstDesc = MCII.get(Inst.getOpcode()); + return (InstDesc.isUnconditionalBranch() && + (AlignBranchType & X86AlignBranchKind::AlignBranchJmp)) || + (InstDesc.isCall() && + (AlignBranchType & X86AlignBranchKind::AlignBranchCall)) || + (InstDesc.isReturn() && + (AlignBranchType & X86AlignBranchKind::AlignBranchRet)) || + (InstDesc.isIndirectBranch() && + (AlignBranchType & X86AlignBranchKind::AlignBranchIndirect)); +} + +/// alignBranchesBegin - Insert MCBoundaryAlignFragment before instructions +/// to align branches. +void X86AsmBackend::alignBranchesBegin(MCObjectStreamer &OS, + const MCInst &Inst) { + if (!needAlign(OS) || !needAlignInst(Inst)) + return; + + // Insert BranchPadding before the instruction need to be aligned. + auto *F = new MCBoundaryAlignFragment(AlignBoundarySize); + OS.insert(F); + PendingBA = F; +} + +void X86AsmBackend::alignBranchesEnd(MCObjectStreamer &OS, const MCInst &Inst) { + if (!needAlign(OS) || !needAlignInst(Inst)) + return; + + MCFragment *CF = OS.getCurrentFragment(); + if (!CF) + return; + + auto *F = PendingBA; + PendingBA = nullptr; + assert(F); + + // Link it for later relaxation - this allows the padding for alignment to + // actually be computed and emitted. + F->setFragment(CF); + + // Update the maximum alignment on the current section if necessary. + MCSection *Sec = OS.getCurrentSectionOnly(); + if (AlignBoundarySize > Sec->getAlignment()) + Sec->setAlignment(Align(AlignBoundarySize)); + + // Break the last fragment so that more instructions can't be pushed into it. + OS.insert(new MCDataFragment()); +} + Optional X86AsmBackend::getFixupKind(StringRef Name) const { if (STI.getTargetTriple().isOSBinFormatELF()) { if (STI.getTargetTriple().getArch() == Triple::x86_64) { Index: llvm/test/MC/X86/align-branch-64.s =================================================================== --- /dev/null +++ llvm/test/MC/X86/align-branch-64.s @@ -0,0 +1,87 @@ + # RUN: llvm-mc -filetype=obj -triple x86_64-pc-linux-gnu --x86-align-branch-boundary=32 --x86-align-branch=call+jmp+indirect+ret %s | llvm-objdump -d --no-show-raw-insn - | FileCheck %s + + # instruction sizes for reference: + # callq is 5 bytes long + # push %rax is 1 byte + # jmp is 2 bytes + # jmp is 5 bytes + # ret N is 2 bytes + + # These tests are checking the edge cases on the alignment computation + + .text + # CHECK: test1: + # CHECK: 20: callq + .globl test1 + .p2align 5 +test1: + .rept 29 + push %rax + .endr + callq bar + + # CHECK: test2: + # CHECK: 60: callq + .globl test2 + .p2align 5 +test2: + .rept 31 + push %rax + .endr + callq bar + + # CHECK: test3: + # CHECK: a0: callq + .globl test3 + .p2align 5 +test3: + .rept 27 + push %rax + .endr + callq bar + + # next couple check instruction type coverage + + # CHECK: test_jmp: + # CHECK: e0: jmp + .globl test_jmp + .p2align 5 +test_jmp: + .rept 31 + push %rax + .endr + jmp bar + + # CHECK: test_ret: + # CHECK: 120: retq + .globl test_ret + .p2align 5 +test_ret: + .rept 31 + push %rax + .endr + retq $0 + + # check a case with a relaxable instruction + + # CHECK: test_jmp_far: + # CHECK: 160: jmp + .globl test_jmp_far + .p2align 5 +test_jmp_far: + .rept 31 + push %rax + .endr + jmp baz + + + .p2align 4 + .type bar,@function +bar: + retq + + .section "unknown" + .p2align 4 + .type baz,@function +baz: + retq