Index: llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -926,12 +926,15 @@ auto AFI = MF->getInfo(); unsigned Size = AFI->getJumpTableEntrySize(JTI); - if (Size == 4) { + if (Size == 4 && getSubtargetInfo().getTargetTriple().isOSBinFormatMachO()) { + // .word LBB - _func + const MCExpr *Base = + MCSymbolRefExpr::create(getSymbol(&MF->getFunction()), OutContext); + Value = MCBinaryExpr::createSub(Value, Base, OutContext); + } else if (Size == 4) { // .word LBB - LJTI - bool UseLinkerPrivateJT = - getSubtargetInfo().getTargetTriple().isOSBinFormatMachO(); - const MCExpr *Base = MCSymbolRefExpr::create( - MF->getJTISymbol(JTI, OutContext, UseLinkerPrivateJT), OutContext); + const MCExpr *Base = + MCSymbolRefExpr::create(MF->getJTISymbol(JTI, OutContext), OutContext); Value = MCBinaryExpr::createSub(Value, Base, OutContext); } else { // .byte (LBB - LBB) >> 2 (or .hword) @@ -1630,11 +1633,12 @@ case AArch64::JumpTableDest32: { // We want: // ldrsw xScratch, [xTable, xEntry, lsl #2] - // add xDest, xTable, xScratch + // add xDest, xBase, xScratch unsigned DestReg = MI->getOperand(0).getReg(), ScratchReg = MI->getOperand(1).getReg(), - TableReg = MI->getOperand(2).getReg(), - EntryReg = MI->getOperand(3).getReg(); + BaseReg = MI->getOperand(2).getReg(), + TableReg = MI->getOperand(3).getReg(), + EntryReg = MI->getOperand(4).getReg(); EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX) .addReg(ScratchReg) .addReg(TableReg) @@ -1643,7 +1647,7 @@ .addImm(1)); EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs) .addReg(DestReg) - .addReg(TableReg) + .addReg(BaseReg) .addReg(ScratchReg) .addImm(0)); return; Index: llvm/lib/Target/AArch64/AArch64CompressJumpTables.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64CompressJumpTables.cpp +++ llvm/lib/Target/AArch64/AArch64CompressJumpTables.cpp @@ -93,7 +93,7 @@ if (MI.getOpcode() != AArch64::JumpTableDest32) return false; - int JTIdx = MI.getOperand(4).getIndex(); + int JTIdx = MI.getOperand(5).getIndex(); auto &JTInfo = *MF->getJumpTableInfo(); const MachineJumpTableEntry &JT = JTInfo.getJumpTables()[JTIdx]; @@ -128,11 +128,13 @@ if (isUInt<8>(Span / 4)) { AFI->setJumpTableEntryInfo(JTIdx, 1, MinBlock->getSymbol()); MI.setDesc(TII->get(AArch64::JumpTableDest8)); + MI.RemoveOperand(2); ++NumJT8; return true; } else if (isUInt<16>(Span / 4)) { AFI->setJumpTableEntryInfo(JTIdx, 2, MinBlock->getSymbol()); MI.setDesc(TII->get(AArch64::JumpTableDest16)); + MI.RemoveOperand(2); ++NumJT16; return true; } Index: llvm/lib/Target/AArch64/AArch64ExpandHardenedPseudos.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ExpandHardenedPseudos.cpp +++ llvm/lib/Target/AArch64/AArch64ExpandHardenedPseudos.cpp @@ -88,6 +88,8 @@ // cmp xEntry, # ; if table size fits in 12-bit immediate // csel xEntry, xEntry, xzr, ls // ldrsw xScratch, [xTable, xEntry, lsl #2] ; kill xEntry, xScratch = xEntry + // adrp xBase, _func@PAGE + // add xBase, xBase, _func@PAGEOFF // add xDest, xTable, xScratch ; kill xTable, xDest = xTable // br xDest @@ -149,6 +151,19 @@ .addReg(AArch64::X16) .addImm(0) .addImm(1); + + if (STI.isTargetMachO()) { + // Table contains "LBB - _func" because it has to be resolved at + // compile-time in case function overflows available 24-bit addend. + Function *F = &MF.getFunction(); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADRP), AArch64::X17) + .addGlobalAddress(F, 0, AArch64II::MO_PAGE); + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXri), AArch64::X17) + .addUse(AArch64::X17) + .addGlobalAddress(F, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC) + .addImm(0); + } + BuildMI(MBB, MBBI, DL, TII->get(AArch64::ADDXrs), AArch64::X16) .addReg(AArch64::X17) .addReg(AArch64::X16) Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -6222,8 +6222,8 @@ // With aarch64-hardened-codegen, we only expand the full jump table dispatch // sequence later, to guarantee the integrity of the intermediate values. - if (DAG.getMachineFunction().getFunction() - .hasFnAttribute("jump-table-hardening") || + Function &F = DAG.getMachineFunction().getFunction(); + if (F.hasFnAttribute("jump-table-hardening") || Subtarget->getTargetTriple().getArchName() == "arm64e") { assert(Subtarget->isTargetMachO() && "hardened jump-table not yet supported on non-macho"); @@ -6235,9 +6235,16 @@ return SDValue(B, 0); } - SDNode *Dest = - DAG.getMachineNode(AArch64::JumpTableDest32, DL, MVT::i64, MVT::i64, JT, - Entry, DAG.getTargetJumpTable(JTI, MVT::i32)); + // It's more efficient to use the jump-table itself as the base for any + // offsets stored, MachO relocations are limited to 24-bit addends so there we + // unfortunately need to resolve all differences at compile-time in case the + // function is large. + SDValue Base = + Subtarget->isTargetMachO() ? DAG.getGlobalAddress(&F, DL, MVT::i64) : JT; + + SDNode *Dest = DAG.getMachineNode( + AArch64::JumpTableDest32, DL, MVT::i64, MVT::i64, + {Base, JT, Entry, DAG.getTargetJumpTable(JTI, MVT::i32)}); return DAG.getNode(ISD::BRIND, DL, MVT::Other, Op.getOperand(0), SDValue(Dest, 0)); } Index: llvm/lib/Target/AArch64/AArch64InstrInfo.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -132,9 +132,9 @@ NumBytes = 28; break; case AArch64::BR_JumpTable: - // 28 fixed + 16 variable, for table size materialization + // 28 fixed + 24 variable, for table size materialization // We could potentially model the variable size overhead more accurately. - NumBytes = 44; + NumBytes = 52; break; case AArch64::JumpTableDest32: case AArch64::JumpTableDest16: Index: llvm/lib/Target/AArch64/AArch64InstrInfo.td =================================================================== --- llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -710,7 +710,7 @@ // branch relaxation so be pessimistic. let Size = 12, Constraints = "@earlyclobber $dst,@earlyclobber $scratch" in { def JumpTableDest32 : Pseudo<(outs GPR64:$dst, GPR64sp:$scratch), - (ins GPR64:$table, GPR64:$entry, i32imm:$jti), []>, + (ins GPR64:$base, GPR64:$table, GPR64:$entry, i32imm:$jti), []>, Sched<[]>; def JumpTableDest16 : Pseudo<(outs GPR64:$dst, GPR64sp:$scratch), (ins GPR64:$table, GPR64:$entry, i32imm:$jti), []>, Index: llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp =================================================================== --- llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp +++ llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp @@ -2966,9 +2966,20 @@ Register TargetReg = MRI.createVirtualRegister(&AArch64::GPR64RegClass); Register ScratchReg = MRI.createVirtualRegister(&AArch64::GPR64spRegClass); - auto JumpTableInst = MIB.buildInstr(AArch64::JumpTableDest32, - {TargetReg, ScratchReg}, {JTAddr, Index}) - .addJumpTableIndex(JTI); + Register Base; + if (STI.isTargetMachO()) { + Function *F = &MF.getFunction(); + Base = MRI.createVirtualRegister(&AArch64::GPR64RegClass); + MIB.buildInstr(AArch64::MOVaddr) + .addDef(Base) + .addGlobalAddress(F, 0, AArch64II::MO_PAGE) + .addGlobalAddress(F, 0, AArch64II::MO_PAGEOFF | AArch64II::MO_NC); + } else + Base = JTAddr; + auto JumpTableInst = + MIB.buildInstr(AArch64::JumpTableDest32, {TargetReg, ScratchReg}, + {Base, JTAddr, Index}) + .addJumpTableIndex(JTI); // Build the indirect branch. MIB.buildInstr(AArch64::BR, {}, {TargetReg}); I.eraseFromParent(); Index: llvm/test/CodeGen/AArch64/GlobalISel/arm64e-jump-table-hardening.ll =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/arm64e-jump-table-hardening.ll +++ llvm/test/CodeGen/AArch64/GlobalISel/arm64e-jump-table-hardening.ll @@ -10,6 +10,8 @@ ; CHECK: adrp x17, lJTI0_0@PAGE ; CHECK: add x17, x17, lJTI0_0@PAGEOFF ; CHECK: ldrsw x16, [x17, x16, lsl #2] +; CHECK: adrp x17, _test_jumptable@PAGE +; CHECK: add x17, x17, _test_jumptable@PAGEOFF ; CHECK: add x16, x17, x16 ; CHECK: br x16 @@ -44,9 +46,9 @@ } ; CHECK: lJTI0_0: -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable Index: llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt-constrain.mir =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt-constrain.mir +++ llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt-constrain.mir @@ -31,7 +31,7 @@ ; CHECK: bb.1: ; CHECK: successors: %bb.2(0x40000000), %bb.3(0x40000000) ; CHECK: [[MOVaddrJT:%[0-9]+]]:gpr64 = MOVaddrJT target-flags(aarch64-page) %jump-table.0, target-flags(aarch64-pageoff, aarch64-nc) %jump-table.0 - ; CHECK: early-clobber %6:gpr64, early-clobber %7:gpr64sp = JumpTableDest32 [[MOVaddrJT]], [[SUBREG_TO_REG]], %jump-table.0 + ; CHECK: early-clobber %6:gpr64, early-clobber %7:gpr64sp = JumpTableDest32 [[MOVaddrJT]], [[MOVaddrJT]], [[SUBREG_TO_REG]], %jump-table.0 ; CHECK: BR %6 ; CHECK: bb.2: ; CHECK: successors: %bb.3(0x80000000) Index: llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt.mir =================================================================== --- llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt.mir +++ llvm/test/CodeGen/AArch64/GlobalISel/select-jump-table-brjt.mir @@ -66,7 +66,7 @@ ; CHECK: successors: %bb.3(0x2aaaaaab), %bb.4(0x2aaaaaab), %bb.2(0x2aaaaaab) ; CHECK: [[COPY2:%[0-9]+]]:gpr32 = COPY $wzr ; CHECK: [[MOVaddrJT:%[0-9]+]]:gpr64 = MOVaddrJT target-flags(aarch64-page) %jump-table.0, target-flags(aarch64-pageoff, aarch64-nc) %jump-table.0 - ; CHECK: early-clobber %18:gpr64, early-clobber %19:gpr64sp = JumpTableDest32 [[MOVaddrJT]], [[SUBREG_TO_REG]], %jump-table.0 + ; CHECK: early-clobber %18:gpr64, early-clobber %19:gpr64sp = JumpTableDest32 [[MOVaddrJT]], [[MOVaddrJT]], [[SUBREG_TO_REG]], %jump-table.0 ; CHECK: BR %18 ; CHECK: bb.2.sw.bb: ; CHECK: successors: %bb.4(0x80000000) Index: llvm/test/CodeGen/AArch64/arm64e-jump-table-hardening.ll =================================================================== --- llvm/test/CodeGen/AArch64/arm64e-jump-table-hardening.ll +++ llvm/test/CodeGen/AArch64/arm64e-jump-table-hardening.ll @@ -8,7 +8,9 @@ ; CHECK: adrp [[JTPAGE:x[0-9]+]], lJTI0_0@PAGE ; CHECK: add x[[JT:[0-9]+]], [[JTPAGE]], lJTI0_0@PAGEOFF ; CHECK: ldrsw [[OFFSET:x[0-9]+]], [x[[JT]], [[INDEX2]], lsl #2] -; CHECK: add [[DEST:x[0-9]+]], x[[JT]], [[OFFSET]] +; CHECK: adrp [[BASE:x[0-9]+]], _test_jumptable@PAGE +; CHECK: add [[BASE]], [[BASE]], _test_jumptable@PAGEOFF +; CHECK: add [[DEST:x[0-9]+]], [[BASE]], [[OFFSET]] ; CHECK: br [[DEST]] define i32 @test_jumptable(i32 %in) "jump-table-hardening" { @@ -42,9 +44,9 @@ } ; CHECK: lJTI0_0: -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 -; CHECK-NEXT: .long LBB{{[0-9_]+}}-lJTI0_0 +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable +; CHECK-NEXT: .long LBB{{[0-9_]+}}-_test_jumptable Index: llvm/test/CodeGen/AArch64/jti-correct-datatype.mir =================================================================== --- llvm/test/CodeGen/AArch64/jti-correct-datatype.mir +++ llvm/test/CodeGen/AArch64/jti-correct-datatype.mir @@ -38,7 +38,7 @@ $x8 = ADRP target-flags(aarch64-page) %jump-table.0 renamable $w9 = ORRWrs $wzr, killed renamable $w0, 0, implicit-def $x9 renamable $x8 = ADDXri killed $x8, target-flags(aarch64-pageoff, aarch64-nc) %jump-table.0, 0 - early-clobber renamable $x10, dead early-clobber renamable $x11 = JumpTableDest32 killed renamable $x8, killed renamable $x9, %jump-table.0 + early-clobber renamable $x10, dead early-clobber renamable $x11 = JumpTableDest32 killed renamable $x8, killed renamable $x8, killed renamable $x9, %jump-table.0 BR killed renamable $x10 bb.2 (align 65536): Index: llvm/test/CodeGen/AArch64/jump-table-compress.mir =================================================================== --- llvm/test/CodeGen/AArch64/jump-table-compress.mir +++ llvm/test/CodeGen/AArch64/jump-table-compress.mir @@ -53,7 +53,7 @@ ; CHECK-LABEL: bb.1 ; CHECK: JumpTableDest8 liveins: $x8 - early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x9, undef killed $x8, %jump-table.0 + early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x10, undef killed $x9, undef killed $x8, %jump-table.0 BR killed $x10 bb.2: @@ -63,7 +63,7 @@ bb.3: ; CHECK-LABEL: bb.3 ; CHECK: JumpTableDest16 - early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x9, undef killed $x8, %jump-table.1 + early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x10, undef killed $x9, undef killed $x8, %jump-table.1 BR killed $x10 bb.4: @@ -73,7 +73,7 @@ bb.5: ; CHECK-LABEL: bb.5 ; CHECK: JumpTableDest8 - early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x9, undef killed $x8, %jump-table.2 + early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x10, undef killed $x9, undef killed $x8, %jump-table.2 BR killed $x10 bb.6: @@ -83,7 +83,7 @@ bb.7: ; CHECK-LABEL: bb.7 ; CHECK: JumpTableDest32 - early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x9, undef killed $x8, %jump-table.3 + early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x10, undef killed $x9, undef killed $x8, %jump-table.3 BR killed $x10 bb.8: @@ -97,7 +97,7 @@ bb.10: ; CHECK-LABEL: bb.10 ; CHECK: JumpTableDest8 - early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x9, undef killed $x8, %jump-table.4 + early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x10, undef killed $x9, undef killed $x8, %jump-table.4 BR killed $x10 bb.11: @@ -107,6 +107,6 @@ bb.12: ; CHECK-LABEL: bb.12 ; CHECK: JumpTableDest32 - early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x9, undef killed $x8, %jump-table.5 + early-clobber $x10, dead early-clobber $x11 = JumpTableDest32 undef killed $x10, undef killed $x9, undef killed $x8, %jump-table.5 BR killed $x10 ... Index: llvm/test/CodeGen/AArch64/jump-table.ll =================================================================== --- llvm/test/CodeGen/AArch64/jump-table.ll +++ llvm/test/CodeGen/AArch64/jump-table.ll @@ -151,10 +151,12 @@ i32 4, label %lbl4 ] ; CHECK-IOS-LABEL: test_jumptable32: -; CHECK-IOS: adrp [[JTPAGE:x[0-9]+]], [[LJTI:lJTI[0-9]+_[0-9]+]]@PAGE -; CHECK-IOS: add x[[JT:[0-9]+]], [[JTPAGE]], [[LJTI]]@PAGEOFF +; CHECK-IOS-DAG: adrp [[JTPAGE:x[0-9]+]], [[LJTI:lJTI[0-9]+_[0-9]+]]@PAGE +; CHECK-IOS-DAG: add x[[JT:[0-9]+]], [[JTPAGE]], [[LJTI]]@PAGEOFF +; CHECK-IOS-DAG: adrp [[BASEPAGE:x[0-9]+]], _test_jumptable32@PAGE +; CHECK-IOS-DAG: add [[BASE:x[0-9]+]], [[BASEPAGE]], _test_jumptable32@PAGEOFF ; CHECK-IOS: ldrsw [[OFFSET:x[0-9]+]], [x[[JT]], {{x[0-9]+}}, lsl #2] -; CHECK-IOS: add [[DEST:x[0-9]+]], x[[JT]], [[OFFSET]] +; CHECK-IOS: add [[DEST:x[0-9]+]], [[BASE]], [[OFFSET]] ; CHECK-IOS: br [[DEST]] ; CHECK: .rodata @@ -163,7 +165,7 @@ ; CHECK-IOS-NOT .section __TEXT,__const ; CHECK-IOS: [[LJTI]]: -; CHECK-IOS: .long {{LBB[0-9]+_[0-9]+}}-[[LJTI]] +; CHECK-IOS: .long {{LBB[0-9]+_[0-9]+}}-_test_jumptable32 def: ret i32 0