Index: include/llvm/MC/MCStreamer.h =================================================================== --- include/llvm/MC/MCStreamer.h +++ include/llvm/MC/MCStreamer.h @@ -193,10 +193,6 @@ WinEH::FrameInfo *CurrentWinFrameInfo; - /// Retreive the current frame info if one is available and it is not yet - /// closed. Otherwise, issue an error and return null. - WinEH::FrameInfo *EnsureValidWinFrameInfo(SMLoc Loc); - /// Tracks an index to represent the order a symbol was emitted in. /// Zero means we did not emit that symbol. DenseMap SymbolOrdering; @@ -219,10 +215,6 @@ virtual void EmitCFIStartProcImpl(MCDwarfFrameInfo &Frame); virtual void EmitCFIEndProcImpl(MCDwarfFrameInfo &CurFrame); - /// When emitting an object file, create and emit a real label. When emitting - /// textual assembly, this should do nothing to avoid polluting our output. - virtual MCSymbol *EmitCFILabel(); - WinEH::FrameInfo *getCurrentWinFrameInfo() { return CurrentWinFrameInfo; } @@ -258,6 +250,14 @@ return TargetStreamer.get(); } + /// When emitting an object file, create and emit a real label. When emitting + /// textual assembly, this should do nothing to avoid polluting our output. + virtual MCSymbol *EmitCFILabel(); + + /// Retreive the current frame info if one is available and it is not yet + /// closed. Otherwise, issue an error and return null. + WinEH::FrameInfo *EnsureValidWinFrameInfo(SMLoc Loc); + unsigned getNumFrameInfos() { return DwarfFrameInfos.size(); } ArrayRef getDwarfFrameInfos() const { return DwarfFrameInfos; @@ -891,6 +891,11 @@ virtual void EmitWinCFIStartProc(const MCSymbol *Symbol, SMLoc Loc = SMLoc()); virtual void EmitWinCFIEndProc(SMLoc Loc = SMLoc()); + /// This is used on platforms, such as Windows on ARM64, that require function + /// or funclet sizes to be emitted in .xdata before the End marker is emitted + /// for the frame. We cannot use the End marker, as it is not set at the + /// point of emitting .xdata, in order to indicate that the frame is active. + virtual void EmitWinCFIFuncletOrFuncEnd(SMLoc Loc = SMLoc()); virtual void EmitWinCFIStartChained(SMLoc Loc = SMLoc()); virtual void EmitWinCFIEndChained(SMLoc Loc = SMLoc()); virtual void EmitWinCFIPushReg(unsigned Register, SMLoc Loc = SMLoc()); Index: include/llvm/MC/MCWin64EH.h =================================================================== --- include/llvm/MC/MCWin64EH.h +++ include/llvm/MC/MCWin64EH.h @@ -56,6 +56,14 @@ void Emit(MCStreamer &Streamer) const override; void EmitUnwindInfo(MCStreamer &Streamer, WinEH::FrameInfo *FI) const override; }; + +class ARM64UnwindEmitter : public WinEH::UnwindEmitter { +public: + void Emit(MCStreamer &Streamer) const override; + void EmitUnwindInfo(MCStreamer &Streamer, + WinEH::FrameInfo *FI) const override; +}; + } } // end namespace llvm Index: include/llvm/MC/MCWinEH.h =================================================================== --- include/llvm/MC/MCWinEH.h +++ include/llvm/MC/MCWinEH.h @@ -10,6 +10,7 @@ #ifndef LLVM_MC_MCWINEH_H #define LLVM_MC_MCWINEH_H +#include "llvm/ADT/MapVector.h" #include namespace llvm { @@ -20,9 +21,9 @@ namespace WinEH { struct Instruction { const MCSymbol *Label; - const unsigned Offset; - const unsigned Register; - const unsigned Operation; + unsigned Offset; + unsigned Register; + unsigned Operation; Instruction(unsigned Op, MCSymbol *L, unsigned Reg, unsigned Off) : Label(L), Offset(Off), Register(Reg), Operation(Op) {} @@ -31,6 +32,7 @@ struct FrameInfo { const MCSymbol *Begin = nullptr; const MCSymbol *End = nullptr; + const MCSymbol *FuncletOrFuncEnd = nullptr; const MCSymbol *ExceptionHandler = nullptr; const MCSymbol *Function = nullptr; const MCSymbol *PrologEnd = nullptr; @@ -43,6 +45,7 @@ int LastFrameInst = -1; const FrameInfo *ChainedParent = nullptr; std::vector Instructions; + MapVector> EpilogMap; FrameInfo() = default; FrameInfo(const MCSymbol *Function, const MCSymbol *BeginFuncEHLabel) Index: include/llvm/Support/Win64EH.h =================================================================== --- include/llvm/Support/Win64EH.h +++ include/llvm/Support/Win64EH.h @@ -33,7 +33,24 @@ UOP_SaveNonVolBig, UOP_SaveXMM128 = 8, UOP_SaveXMM128Big, - UOP_PushMachFrame + UOP_PushMachFrame, + // The following set of unwind opcodes is for ARM64. They are documented at + // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling + UOP_AllocMedium, + UOP_SaveFPLRX, + UOP_SaveFPLR, + UOP_SaveReg, + UOP_SaveRegX, + UOP_SaveRegP, + UOP_SaveRegPX, + UOP_SaveFReg, + UOP_SaveFRegX, + UOP_SaveFRegP, + UOP_SaveFRegPX, + UOP_SetFP, + UOP_AddFP, + UOP_Nop, + UOP_End }; /// UnwindCode - This union describes a single operation in a function prolog, Index: lib/MC/MCAsmStreamer.cpp =================================================================== --- lib/MC/MCAsmStreamer.cpp +++ lib/MC/MCAsmStreamer.cpp @@ -289,6 +289,7 @@ void EmitWinCFIStartProc(const MCSymbol *Symbol, SMLoc Loc) override; void EmitWinCFIEndProc(SMLoc Loc) override; + void EmitWinCFIFuncletOrFuncEnd(SMLoc Loc) override; void EmitWinCFIStartChained(SMLoc Loc) override; void EmitWinCFIEndChained(SMLoc Loc) override; void EmitWinCFIPushReg(unsigned Register, SMLoc Loc) override; @@ -1590,6 +1591,10 @@ EmitEOL(); } +// TODO: Implement +void MCAsmStreamer::EmitWinCFIFuncletOrFuncEnd(SMLoc Loc) { +} + void MCAsmStreamer::EmitWinCFIStartChained(SMLoc Loc) { MCStreamer::EmitWinCFIStartChained(Loc); Index: lib/MC/MCStreamer.cpp =================================================================== --- lib/MC/MCStreamer.cpp +++ lib/MC/MCStreamer.cpp @@ -609,6 +609,17 @@ CurFrame->End = Label; } +void MCStreamer::EmitWinCFIFuncletOrFuncEnd(SMLoc Loc) { + WinEH::FrameInfo *CurFrame = EnsureValidWinFrameInfo(Loc); + if (!CurFrame) + return; + if (CurFrame->ChainedParent) + getContext().reportError(Loc, "Not all chained regions terminated!"); + + MCSymbol *Label = EmitCFILabel(); + CurFrame->FuncletOrFuncEnd = Label; +} + void MCStreamer::EmitWinCFIStartChained(SMLoc Loc) { WinEH::FrameInfo *CurFrame = EnsureValidWinFrameInfo(Loc); if (!CurFrame) Index: lib/MC/MCWin64EH.cpp =================================================================== --- lib/MC/MCWin64EH.cpp +++ lib/MC/MCWin64EH.cpp @@ -11,6 +11,9 @@ #include "llvm/ADT/Twine.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" +#include "llvm/MC/MCObjectFileInfo.h" +#include "llvm/MC/MCObjectStreamer.h" +#include "llvm/MC/MCSectionCOFF.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/Support/Win64EH.h" @@ -23,6 +26,8 @@ uint8_t Count = 0; for (const auto &I : Insns) { switch (static_cast(I.Operation)) { + default: + assert(false && "Unsupported unwind code"); case Win64EH::UOP_PushNonVol: case Win64EH::UOP_AllocSmall: case Win64EH::UOP_SetFPReg: @@ -60,6 +65,9 @@ uint16_t w; b2 = (inst.Operation & 0x0F); switch (static_cast(inst.Operation)) { + default: + assert(false && "Unsupported unwind code"); + break; case Win64EH::UOP_PushNonVol: EmitAbsDifference(streamer, inst.Label, begin); b2 |= (inst.Register & 0x0F) << 4; @@ -242,3 +250,338 @@ ::EmitUnwindInfo(Streamer, info); } +static int64_t GetAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS, + const MCSymbol *RHS) { + MCContext &Context = Streamer.getContext(); + const MCExpr *Diff = + MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), + MCSymbolRefExpr::create(RHS, Context), Context); + MCObjectStreamer *OS = (MCObjectStreamer *)(&Streamer); + int64_t value; + Diff->evaluateAsAbsolute(value, OS->getAssembler()); + return value; +} + +static uint32_t +ARM64CountOfUnwindCodes(const std::vector &Insns) { + uint32_t Count = 0; + for (const auto &I : Insns) { + switch (static_cast(I.Operation)) { + default: + assert(false && "Unsupported ARM64 unwind code"); + break; + case Win64EH::UOP_AllocSmall: + Count += 1; + break; + case Win64EH::UOP_AllocMedium: + Count += 2; + break; + case Win64EH::UOP_AllocLarge: + Count += 4; + break; + case Win64EH::UOP_SaveFPLRX: + Count += 1; + break; + case Win64EH::UOP_SaveFPLR: + Count += 1; + break; + case Win64EH::UOP_SaveReg: + Count += 2; + break; + case Win64EH::UOP_SaveRegP: + Count += 2; + break; + case Win64EH::UOP_SaveRegPX: + Count += 2; + break; + case Win64EH::UOP_SaveRegX: + Count += 2; + break; + case Win64EH::UOP_SaveFReg: + Count += 2; + break; + case Win64EH::UOP_SaveFRegP: + Count += 2; + break; + case Win64EH::UOP_SaveFRegX: + Count += 2; + break; + case Win64EH::UOP_SaveFRegPX: + Count += 2; + break; + case Win64EH::UOP_SetFP: + Count += 1; + break; + case Win64EH::UOP_AddFP: + Count += 2; + break; + case Win64EH::UOP_Nop: + Count += 1; + break; + case Win64EH::UOP_End: + Count += 1; + break; + } + } + return Count; +} + +// Unwind opcode encodings and restrictions are documented at +// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling +static void ARM64EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin, + WinEH::Instruction &inst) { + uint8_t b, reg; + uint16_t w; + switch (static_cast(inst.Operation)) { + default: + assert(false && "Unsupported ARM64 unwind code"); + break; + case Win64EH::UOP_AllocSmall: + b = (inst.Offset >> 4) & 0x1F; + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_AllocMedium: + w = (inst.Offset >> 4) & 0x7FF; + b = 0xC0; + b |= (w >> 8); + streamer.EmitIntValue(b, 1); + b = w & 0xFF; + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_AllocLarge: + uint32_t dw; + dw = 0xC0000000; + dw |= (inst.Offset >> 4) & 0xFFFFFF; + streamer.EmitIntValue(dw, 4); + break; + case Win64EH::UOP_SetFP: + b = 0xE1; + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_AddFP: + b = 0xE2; + streamer.EmitIntValue(b, 1); + b = (inst.Offset >> 3); + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_Nop: + b = 0xE3; + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveFPLRX: + b = 0x80; + b |= ((inst.Offset - 1) >> 3) & 0x3F; + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveFPLR: + b = 0x40; + b |= (inst.Offset >> 3) & 0x3F; + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveReg: + assert(inst.Register >= 19 && "Saved reg must be >= 19"); + reg = inst.Register - 19; + b = 0xD0 | ((reg & 0xC) >> 2); + streamer.EmitIntValue(b, 1); + b = ((reg & 0x3) << 6) | (inst.Offset >> 3); + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveRegX: + assert(inst.Register >= 19 && "Saved reg must be >= 19"); + reg = inst.Register - 19; + b = 0xD4 | ((reg & 0x8) >> 3); + streamer.EmitIntValue(b, 1); + b = ((reg & 0x7) << 5) | ((inst.Offset - 1) >> 3); + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveRegP: + assert(inst.Register >= 19 && "Saved registers must be >= 19"); + reg = inst.Register - 19; + b = 0xC8 | ((reg & 0xC) >> 2); + streamer.EmitIntValue(b, 1); + b = ((reg & 0x3) << 6) | (inst.Offset >> 3); + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveRegPX: + assert(inst.Register >= 19 && "Saved registers must be >= 19"); + reg = inst.Register - 19; + b = 0xCC | ((reg & 0xC) >> 2); + streamer.EmitIntValue(b, 1); + b = ((reg & 0x3) << 6) | ((inst.Offset - 1) >> 3); + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveFReg: + assert(inst.Register >= 8 && "Saved dreg must be >= 8"); + reg = inst.Register - 8; + b = 0xDC | ((reg & 0x4) >> 2); + streamer.EmitIntValue(b, 1); + b = ((reg & 0x3) << 6) | (inst.Offset >> 3); + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveFRegX: + assert(inst.Register >= 8 && "Saved dreg must be >= 8"); + reg = inst.Register - 8; + b = 0xDE; + streamer.EmitIntValue(b, 1); + b = ((reg & 0x7) << 5) | ((inst.Offset - 1) >> 3); + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveFRegP: + assert(inst.Register >= 8 && "Saved dregs must be >= 8"); + reg = inst.Register - 8; + b = 0xD8 | ((reg & 0x4) >> 2); + streamer.EmitIntValue(b, 1); + b = ((reg & 0x3) << 6) | (inst.Offset >> 3); + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_SaveFRegPX: + assert(inst.Register >= 8 && "Saved dregs must be >= 8"); + reg = inst.Register - 8; + b = 0xDA | ((reg & 0x4) >> 2); + streamer.EmitIntValue(b, 1); + b = ((reg & 0x3) << 6) | ((inst.Offset - 1) >> 3); + streamer.EmitIntValue(b, 1); + break; + case Win64EH::UOP_End: + b = 0xE4; + streamer.EmitIntValue(b, 1); + break; + } +} + +// Populate the .xdata section. The format of .xdata on ARM64 is documented at +// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling +static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) { + // If this UNWIND_INFO already has a symbol, it's already been emitted. + if (info->Symbol) + return; + + MCContext &context = streamer.getContext(); + MCSymbol *Label = context.createTempSymbol(); + + streamer.EmitValueToAlignment(4); + streamer.EmitLabel(Label); + info->Symbol = Label; + + uint32_t FuncLength = 0x0; + FuncLength = (uint32_t)GetAbsDifference(streamer, info->FuncletOrFuncEnd, + info->Begin); + if (FuncLength) + FuncLength /= 4; + uint32_t PrologCodeBytes = ARM64CountOfUnwindCodes(info->Instructions); + uint32_t TotalCodeBytes = PrologCodeBytes; + + // Process epilogs. + MapVector EpilogInfo; + for (auto &I : info->EpilogMap) { + MCSymbol *EpilogStart = I.first; + auto &EpilogInstrs = I.second; + uint32_t CodeBytes = ARM64CountOfUnwindCodes(EpilogInstrs); + EpilogInfo[EpilogStart] = TotalCodeBytes; + TotalCodeBytes += CodeBytes; + } + + // Code Words, Epilog count, E, X, Vers, Function Length + uint32_t row1 = 0x0; + uint8_t CodeWords = TotalCodeBytes / 4; + uint8_t CodeWordsMod = TotalCodeBytes % 4; + if (CodeWordsMod) + CodeWords++; + uint32_t EpilogCount = info->EpilogMap.size(); + if (TotalCodeBytes < 124) + row1 |= (CodeWords & 0x1F) << 27; + if (EpilogCount < 31) + row1 |= EpilogCount << 22; + // X is always 1 right now, TODO: packed epilog setup + if (info->HandlesExceptions) // X + row1 |= 1 << 20; + row1 |= FuncLength & 0x3FFFF; + streamer.EmitIntValue(row1, 4); + + // Extended Code Words, Extended Epilog Count + uint32_t row2 = 0x0; + if (!(TotalCodeBytes < 124)) + row2 |= (CodeWords & 0xFF) << 16; + if (!(EpilogCount < 31)) + row2 |= (EpilogCount & 0xFFFF); + if (row2) + streamer.EmitIntValue(row2, 4); + + // Epilog Start Index, Epilog Start Offset + for (auto &I : EpilogInfo) { + MCSymbol *EpilogStart = I.first; + uint32_t EpilogIndex = I.second; + uint32_t EpilogOffset = + (uint32_t)GetAbsDifference(streamer, EpilogStart, info->Begin); + if (EpilogOffset) + EpilogOffset /= 4; + uint32_t row3 = EpilogOffset; + row3 |= (EpilogIndex & 0x3FF) << 22; + streamer.EmitIntValue(row3, 4); + } + + // Emit prolog unwind instructions (in reverse order). + uint8_t numInst = info->Instructions.size(); + for (uint8_t c = 0; c < numInst; ++c) { + WinEH::Instruction inst = info->Instructions.back(); + info->Instructions.pop_back(); + ARM64EmitUnwindCode(streamer, info->Begin, inst); + } + + // Emit epilog unwind instructions + for (auto &I : info->EpilogMap) { + auto &EpilogInstrs = I.second; + for (uint32_t i = 0; i < EpilogInstrs.size(); i++) { + WinEH::Instruction inst = EpilogInstrs[i]; + ARM64EmitUnwindCode(streamer, info->Begin, inst); + } + } + + int32_t BytesMod = CodeWords * 4 - TotalCodeBytes; + assert(BytesMod >= 0); + for (int i = 0; i < BytesMod; i++) + streamer.EmitIntValue(0xE3, 1); + + if (info->HandlesExceptions) + streamer.EmitValue( + MCSymbolRefExpr::create(info->ExceptionHandler, + MCSymbolRefExpr::VK_COFF_IMGREL32, context), + 4); +} + +static void ARM64EmitRuntimeFunction(MCStreamer &streamer, + const WinEH::FrameInfo *info) { + MCContext &context = streamer.getContext(); + + streamer.EmitValueToAlignment(4); + EmitSymbolRefWithOfs(streamer, info->Function, info->Begin); + streamer.EmitValue(MCSymbolRefExpr::create(info->Symbol, + MCSymbolRefExpr::VK_COFF_IMGREL32, + context), + 4); +} + +void llvm::Win64EH::ARM64UnwindEmitter::Emit(MCStreamer &Streamer) const { + // Emit the unwind info structs first. + for (const auto &CFI : Streamer.getWinFrameInfos()) { + MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); + Streamer.SwitchSection(XData); + ARM64EmitUnwindInfo(Streamer, CFI.get()); + } + + // Now emit RUNTIME_FUNCTION entries. + for (const auto &CFI : Streamer.getWinFrameInfos()) { + MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); + Streamer.SwitchSection(PData); + ARM64EmitRuntimeFunction(Streamer, CFI.get()); + } +} + +void llvm::Win64EH::ARM64UnwindEmitter::EmitUnwindInfo( + MCStreamer &Streamer, WinEH::FrameInfo *info) const { + // Switch sections (the static function above is meant to be called from + // here and from Emit(). + MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); + Streamer.SwitchSection(XData); + ARM64EmitUnwindInfo(Streamer, info); +} Index: lib/Target/AArch64/AArch64AsmPrinter.cpp =================================================================== --- lib/Target/AArch64/AArch64AsmPrinter.cpp +++ lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -21,6 +21,7 @@ #include "InstPrinter/AArch64InstPrinter.h" #include "MCTargetDesc/AArch64AddressingModes.h" #include "MCTargetDesc/AArch64MCTargetDesc.h" +#include "MCTargetDesc/AArch64TargetStreamer.h" #include "Utils/AArch64BaseInfo.h" #include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" @@ -559,6 +560,8 @@ OutStreamer->EmitLabel(LOHLabel); } + AArch64TargetStreamer *TS = + static_cast(OutStreamer->getTargetStreamer()); // Do any manual lowerings. switch (MI->getOpcode()) { default: @@ -683,6 +686,90 @@ case TargetOpcode::PATCHABLE_TAIL_CALL: LowerPATCHABLE_TAIL_CALL(*MI); return; + + case AArch64::SEH_StackAlloc: + TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm()); + return; + + case AArch64::SEH_SaveFPLR: + TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm()); + return; + + case AArch64::SEH_SaveFPLR_X: + TS->EmitARM64WinCFISaveFPLRX(MI->getOperand(0).getImm()); + return; + + case AArch64::SEH_SaveReg: + TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(), + MI->getOperand(1).getImm()); + return; + + case AArch64::SEH_SaveReg_X: + TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(), + MI->getOperand(1).getImm()); + return; + + case AArch64::SEH_SaveRegP: + assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && + "Non-consecutive registers not allowed for save_regp"); + TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(), + MI->getOperand(2).getImm()); + return; + + case AArch64::SEH_SaveRegP_X: + assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && + "Non-consecutive registers not allowed for save_regp_x"); + TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(), + MI->getOperand(2).getImm()); + return; + + case AArch64::SEH_SaveFReg: + TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(), + MI->getOperand(1).getImm()); + return; + + case AArch64::SEH_SaveFReg_X: + TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(), + MI->getOperand(1).getImm()); + return; + + case AArch64::SEH_SaveFRegP: + assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && + "Non-consecutive registers not allowed for save_regp"); + TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(), + MI->getOperand(2).getImm()); + return; + + case AArch64::SEH_SaveFRegP_X: + assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) && + "Non-consecutive registers not allowed for save_regp_x"); + TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(), + MI->getOperand(2).getImm()); + return; + + case AArch64::SEH_SetFP: + TS->EmitARM64WinCFISetFP(); + return; + + case AArch64::SEH_AddFP: + TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm()); + return; + + case AArch64::SEH_Nop: + TS->EmitARM64WinCFINop(); + return; + + case AArch64::SEH_PrologEnd: + TS->EmitARM64WinCFIPrologEnd(); + return; + + case AArch64::SEH_EpilogStart: + TS->EmitARM64WinCFIEpilogStart(); + return; + + case AArch64::SEH_EpilogEnd: + TS->EmitARM64WinCFIEpilogEnd(); + return; } // Finally, do the automated lowerings for everything else. Index: lib/Target/AArch64/AArch64InstrInfo.td =================================================================== --- lib/Target/AArch64/AArch64InstrInfo.td +++ lib/Target/AArch64/AArch64InstrInfo.td @@ -2983,6 +2983,28 @@ let hasNoSchedulingInfo = 1; } +//===----------------------------------------------------------------------===// +// Instructions used for emitting unwind opcodes on ARM64 Windows. +//===----------------------------------------------------------------------===// +let isPseudo = 1 in { + def SEH_StackAlloc : Pseudo<(outs), (ins i32imm:$size), []>, Sched<[]>; + def SEH_SaveFPLR : Pseudo<(outs), (ins i32imm:$offs), []>, Sched<[]>; + def SEH_SaveFPLR_X : Pseudo<(outs), (ins i32imm:$offs), []>, Sched<[]>; + def SEH_SaveReg : Pseudo<(outs), (ins i32imm:$reg, i32imm:$offs), []>, Sched<[]>; + def SEH_SaveReg_X : Pseudo<(outs), (ins i32imm:$reg, i32imm:$offs), []>, Sched<[]>; + def SEH_SaveRegP : Pseudo<(outs), (ins i32imm:$reg0, i32imm:$reg1, i32imm:$offs), []>, Sched<[]>; + def SEH_SaveRegP_X : Pseudo<(outs), (ins i32imm:$reg0, i32imm:$reg1, i32imm:$offs), []>, Sched<[]>; + def SEH_SaveFReg : Pseudo<(outs), (ins i32imm:$reg, i32imm:$offs), []>, Sched<[]>; + def SEH_SaveFReg_X : Pseudo<(outs), (ins i32imm:$reg, i32imm:$offs), []>, Sched<[]>; + def SEH_SaveFRegP : Pseudo<(outs), (ins i32imm:$reg0, i32imm:$reg1, i32imm:$offs), []>, Sched<[]>; + def SEH_SaveFRegP_X : Pseudo<(outs), (ins i32imm:$reg0, i32imm:$reg1, i32imm:$offs), []>, Sched<[]>; + def SEH_SetFP : Pseudo<(outs), (ins), []>, Sched<[]>; + def SEH_AddFP : Pseudo<(outs), (ins i32imm:$offs), []>, Sched<[]>; + def SEH_Nop : Pseudo<(outs), (ins), []>, Sched<[]>; + def SEH_PrologEnd : Pseudo<(outs), (ins), []>, Sched<[]>; + def SEH_EpilogStart : Pseudo<(outs), (ins), []>, Sched<[]>; + def SEH_EpilogEnd : Pseudo<(outs), (ins), []>, Sched<[]>; +} //===----------------------------------------------------------------------===// // Floating point immediate move. Index: lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp =================================================================== --- lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp +++ lib/Target/AArch64/MCTargetDesc/AArch64ELFStreamer.cpp @@ -227,7 +227,7 @@ if (TT.isOSBinFormatELF()) return new AArch64TargetELFStreamer(S); if (TT.isOSBinFormatCOFF()) - return new AArch64TargetWinCOFFStreamer(S); + return createAArch64WinCOFFTargetStreamer(S); return nullptr; } Index: lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h =================================================================== --- lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h +++ lib/Target/AArch64/MCTargetDesc/AArch64TargetStreamer.h @@ -33,6 +33,24 @@ /// Callback used to implement the .inst directive. virtual void emitInst(uint32_t Inst); + virtual void EmitARM64WinCFIAllocStack(unsigned Size) {} + virtual void EmitARM64WinCFISaveFPLR(int Offset) {} + virtual void EmitARM64WinCFISaveFPLRX(int Offset) {} + virtual void EmitARM64WinCFISaveReg(unsigned Reg, int Offset) {} + virtual void EmitARM64WinCFISaveRegX(unsigned Reg, int Offset) {} + virtual void EmitARM64WinCFISaveRegP(unsigned Reg, int Offset) {} + virtual void EmitARM64WinCFISaveRegPX(unsigned Reg, int Offset) {} + virtual void EmitARM64WinCFISaveFReg(unsigned Reg, int Offset) {} + virtual void EmitARM64WinCFISaveFRegX(unsigned Reg, int Offset) {} + virtual void EmitARM64WinCFISaveFRegP(unsigned Reg, int Offset) {} + virtual void EmitARM64WinCFISaveFRegPX(unsigned Reg, int Offset) {} + virtual void EmitARM64WinCFISetFP() {} + virtual void EmitARM64WinCFIAddFP(unsigned Size) {} + virtual void EmitARM64WinCFINop() {} + virtual void EmitARM64WinCFIPrologEnd() {} + virtual void EmitARM64WinCFIEpilogStart() {} + virtual void EmitARM64WinCFIEpilogEnd() {} + private: std::unique_ptr ConstantPools; }; Index: lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h =================================================================== --- lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h +++ lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h @@ -17,22 +17,10 @@ #include "AArch64TargetStreamer.h" #include "llvm/MC/MCWinCOFFStreamer.h" -namespace { -class AArch64WinCOFFStreamer; - -class AArch64TargetWinCOFFStreamer : public llvm::AArch64TargetStreamer { -private: - AArch64WinCOFFStreamer &getStreamer(); - -public: - AArch64TargetWinCOFFStreamer(llvm::MCStreamer &S) - : AArch64TargetStreamer(S) {} -}; - -} // end anonymous namespace - namespace llvm { +MCTargetStreamer *createAArch64WinCOFFTargetStreamer(MCStreamer &S); + MCWinCOFFStreamer *createAArch64WinCOFFStreamer( MCContext &Context, std::unique_ptr TAB, std::unique_ptr OW, std::unique_ptr Emitter, Index: lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp =================================================================== --- lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp +++ lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFStreamer.cpp @@ -11,12 +11,16 @@ #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCObjectWriter.h" +#include "llvm/MC/MCWin64EH.h" +#include "llvm/MC/MCWinCOFFStreamer.h" using namespace llvm; namespace { class AArch64WinCOFFStreamer : public MCWinCOFFStreamer { + Win64EH::ARM64UnwindEmitter EHStreamer; + public: friend class AArch64TargetWinCOFFStreamer; @@ -25,17 +29,208 @@ std::unique_ptr OW) : MCWinCOFFStreamer(C, std::move(AB), std::move(CE), std::move(OW)) {} + void EmitWinEHHandlerData(SMLoc Loc) override; + void EmitWindowsUnwindTables() override; void FinishImpl() override; }; +void AArch64WinCOFFStreamer::EmitWinEHHandlerData(SMLoc Loc) { + MCStreamer::EmitWinEHHandlerData(Loc); + + // We have to emit the unwind info now, because this directive + // actually switches to the .xdata section! + EHStreamer.EmitUnwindInfo(*this, getCurrentWinFrameInfo()); +} + +void AArch64WinCOFFStreamer::EmitWindowsUnwindTables() { + if (!getNumWinFrameInfos()) + return; + EHStreamer.Emit(*this); +} + void AArch64WinCOFFStreamer::FinishImpl() { EmitFrames(nullptr); + EmitWindowsUnwindTables(); MCWinCOFFStreamer::FinishImpl(); } + +class AArch64TargetWinCOFFStreamer : public llvm::AArch64TargetStreamer { +private: + // True if we are processing SEH directives in an epilogue. + bool InEpilogCFI = false; + + // Symbol of the current epilog for which we are processing SEH directives. + MCSymbol *CurrentEpilog = nullptr; +public: + AArch64TargetWinCOFFStreamer(llvm::MCStreamer &S) + : AArch64TargetStreamer(S) {} + + // The unwind codes on ARM64 Windows are documented at + // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling + void EmitARM64WinCFIAllocStack(unsigned Size) override; + void EmitARM64WinCFISaveFPLR(int Offset) override; + void EmitARM64WinCFISaveFPLRX(int Offset) override; + void EmitARM64WinCFISaveReg(unsigned Reg, int Offset) override; + void EmitARM64WinCFISaveRegX(unsigned Reg, int Offset) override; + void EmitARM64WinCFISaveRegP(unsigned Reg, int Offset) override; + void EmitARM64WinCFISaveRegPX(unsigned Reg, int Offset) override; + void EmitARM64WinCFISaveFReg(unsigned Reg, int Offset) override; + void EmitARM64WinCFISaveFRegX(unsigned Reg, int Offset) override; + void EmitARM64WinCFISaveFRegP(unsigned Reg, int Offset) override; + void EmitARM64WinCFISaveFRegPX(unsigned Reg, int Offset) override; + void EmitARM64WinCFISetFP() override; + void EmitARM64WinCFIAddFP(unsigned Size) override; + void EmitARM64WinCFINop() override; + void EmitARM64WinCFIPrologEnd() override; + void EmitARM64WinCFIEpilogStart() override; + void EmitARM64WinCFIEpilogEnd() override; +private: + void EmitARM64WinUnwindCode(unsigned UnwindCode, int Reg, int Offset); +}; + + +// Helper function to common out unwind code setup for those codes that can +// belong to both prolog and epilog. +// There are three types of Windows ARM64 SEH codes. They can +// 1) take no operands: SEH_Nop, SEH_PrologEnd, SEH_EpilogStart, SEH_EpilogEnd +// 2) take an offset: SEH_StackAlloc, SEH_SaveFPLR, SEH_SaveFPLR_X +// 3) take a register and an offset/size: all others +void AArch64TargetWinCOFFStreamer::EmitARM64WinUnwindCode(unsigned UnwindCode, + int Reg, + int Offset) { + auto &S = getStreamer(); + WinEH::FrameInfo *CurFrame = S.EnsureValidWinFrameInfo(SMLoc()); + if (!CurFrame) + return; + MCSymbol *Label = S.EmitCFILabel(); + auto Inst = WinEH::Instruction(UnwindCode, Label, Reg, Offset); + if (InEpilogCFI) + CurFrame->EpilogMap[CurrentEpilog].push_back(Inst); + else + CurFrame->Instructions.push_back(Inst); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFIAllocStack(unsigned Size) { + unsigned Op = Win64EH::UOP_AllocSmall; + if (Size >= 16384) + Op = Win64EH::UOP_AllocLarge; + else if (Size >= 512) + Op = Win64EH::UOP_AllocMedium; + EmitARM64WinUnwindCode(Op, -1, Size); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveFPLR(int Offset) { + EmitARM64WinUnwindCode(Win64EH::UOP_SaveFPLR, -1, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveFPLRX(int Offset) { + EmitARM64WinUnwindCode(Win64EH::UOP_SaveFPLRX, -1, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveReg(unsigned Reg, + int Offset) { + assert(Offset >= 0 && Offset <= 504 && + "Offset for save reg should be >= 0 && <= 504"); + EmitARM64WinUnwindCode(Win64EH::UOP_SaveReg, Reg, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveRegX(unsigned Reg, + int Offset) { + assert(Offset <= 0 && Offset >= -256 && + "Offset for pre-indexed save reg should be negative"); + Offset = -Offset; + EmitARM64WinUnwindCode(Win64EH::UOP_SaveRegX, Reg, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveRegP(unsigned Reg, int Offset) { + EmitARM64WinUnwindCode(Win64EH::UOP_SaveRegP, Reg, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveRegPX(unsigned Reg, int Offset) { + EmitARM64WinUnwindCode(Win64EH::UOP_SaveRegPX, Reg, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveFReg(unsigned Reg, int Offset) { + assert(Offset >= 0 && Offset <= 504 && + "Offset for save reg should be >= 0 && <= 504"); + EmitARM64WinUnwindCode(Win64EH::UOP_SaveFReg, Reg, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveFRegX(unsigned Reg, int Offset) { + assert(Offset <= 0 && Offset >= -256 && + "Offset for pre-indexed save reg should be negative"); + Offset = -Offset; + EmitARM64WinUnwindCode(Win64EH::UOP_SaveFRegX, Reg, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveFRegP(unsigned Reg, int Offset) { + EmitARM64WinUnwindCode(Win64EH::UOP_SaveFRegP, Reg, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISaveFRegPX(unsigned Reg, int Offset) { + Offset = -Offset; + EmitARM64WinUnwindCode(Win64EH::UOP_SaveFRegPX, Reg, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFISetFP() { + EmitARM64WinUnwindCode(Win64EH::UOP_SetFP, -1, 0); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFIAddFP(unsigned Offset) { + assert(Offset <= 2040 && "UOP_AddFP must have offset <= 2040"); + EmitARM64WinUnwindCode(Win64EH::UOP_AddFP, -1, Offset); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFINop() { + EmitARM64WinUnwindCode(Win64EH::UOP_Nop, -1, 0); +} + +// The functions below handle opcodes that can end up in either a prolog or +// an epilog, but not both. +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFIPrologEnd() { + auto &S = getStreamer(); + WinEH::FrameInfo *CurFrame = S.EnsureValidWinFrameInfo(SMLoc()); + if (!CurFrame) + return; + + MCSymbol *Label = S.EmitCFILabel(); + CurFrame->PrologEnd = Label; + WinEH::Instruction Inst = WinEH::Instruction(Win64EH::UOP_End, Label, -1, 0); + auto it = CurFrame->Instructions.begin(); + CurFrame->Instructions.insert(it, Inst); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFIEpilogStart() { + auto &S = getStreamer(); + WinEH::FrameInfo *CurFrame = S.EnsureValidWinFrameInfo(SMLoc()); + if (!CurFrame) + return; + + InEpilogCFI = true; + CurrentEpilog = S.EmitCFILabel(); +} + +void AArch64TargetWinCOFFStreamer::EmitARM64WinCFIEpilogEnd() { + auto &S = getStreamer(); + WinEH::FrameInfo *CurFrame = S.EnsureValidWinFrameInfo(SMLoc()); + if (!CurFrame) + return; + + InEpilogCFI = false; + MCSymbol *Label = S.EmitCFILabel(); + WinEH::Instruction Inst = WinEH::Instruction(Win64EH::UOP_End, Label, -1, 0); + CurFrame->EpilogMap[CurrentEpilog].push_back(Inst); + CurrentEpilog = nullptr; +} } // end anonymous namespace namespace llvm { + +MCTargetStreamer *createAArch64WinCOFFTargetStreamer(MCStreamer &S) { + return new AArch64TargetWinCOFFStreamer(S); +} + MCWinCOFFStreamer *createAArch64WinCOFFStreamer( MCContext &Context, std::unique_ptr MAB, std::unique_ptr OW, std::unique_ptr Emitter,