Index: lld/MachO/Arch/ARM64.cpp =================================================================== --- lld/MachO/Arch/ARM64.cpp +++ lld/MachO/Arch/ARM64.cpp @@ -168,10 +168,10 @@ struct Ldr { uint8_t destRegister; uint8_t baseRegister; - uint8_t size; + uint8_t p2Size; bool isFloat; ExtendType extendType; - uint64_t offset; + int64_t offset; }; struct PerformedReloc { @@ -190,6 +190,7 @@ void applyAdrpAdrp(const OptimizationHint &); void applyAdrpLdr(const OptimizationHint &); void applyAdrpLdrGot(const OptimizationHint &); + void applyAdrpLdrGotLdr(const OptimizationHint &); private: uint8_t *buf; @@ -228,34 +229,35 @@ if ((insn & 0x3fc00000) == 0x39400000) { // LDR (immediate), LDRB (immediate), LDRH (immediate) - ldr.size = 1 << size; + ldr.p2Size = size; ldr.extendType = ZeroExtend; ldr.isFloat = false; } else if ((insn & 0x3f800000) == 0x39800000) { // LDRSB (immediate), LDRSH (immediate), LDRSW (immediate) - ldr.size = 1 << size; + ldr.p2Size = size; ldr.extendType = static_cast(opc); ldr.isFloat = false; } else if ((insn & 0x3f400000) == 0x3d400000) { // LDR (immediate, SIMD&FP) ldr.extendType = ZeroExtend; ldr.isFloat = true; - if (size == 2 && opc == 1) - ldr.size = 4; - else if (size == 3 && opc == 1) - ldr.size = 8; + if (opc == 1) + ldr.p2Size = size; else if (size == 0 && opc == 3) - ldr.size = 16; + ldr.p2Size = 4; else return false; } else { return false; } - ldr.offset = ((insn >> 10) & 0xfff) * ldr.size; + ldr.offset = ((insn >> 10) & 0xfff) << ldr.p2Size; return true; } +static bool isValidAdrOffset(int32_t delta) { return isInt<21>(delta); } + static void writeAdr(void *loc, uint32_t dest, int32_t delta) { + assert(isValidAdrOffset(delta)); uint32_t opcode = 0x10000000; uint32_t immHi = (delta & 0x001ffffc) << 3; uint32_t immLo = (delta & 0x00000003) << 29; @@ -264,26 +266,63 @@ static void writeNop(void *loc) { write32le(loc, 0xd503201f); } -static void writeLiteralLdr(void *loc, Ldr original, int32_t delta) { - uint32_t imm19 = (delta & 0x001ffffc) << 3; - uint32_t opcode = 0; - switch (original.size) { - case 4: - if (original.isFloat) +static bool isLiteralLdrEligible(const Ldr &ldr) { + return ldr.p2Size > 1 && isShiftedInt<19, 2>(ldr.offset); +} + +static void writeLiteralLdr(void *loc, const Ldr &ldr) { + assert(isLiteralLdrEligible(ldr)); + uint32_t imm19 = (ldr.offset & 0x001ffffc) << 3; + uint32_t opcode; + switch (ldr.p2Size) { + case 2: + if (ldr.isFloat) opcode = 0x1c000000; else - opcode = original.extendType == Sign64 ? 0x98000000 : 0x18000000; + opcode = ldr.extendType == Sign64 ? 0x98000000 : 0x18000000; break; - case 8: - opcode = original.isFloat ? 0x5c000000 : 0x58000000; + case 3: + opcode = ldr.isFloat ? 0x5c000000 : 0x58000000; break; - case 16: + case 4: opcode = 0x9c000000; break; default: - assert(false && "Invalid size for literal ldr"); + llvm_unreachable("Invalid literal ldr size"); } - write32le(loc, opcode | imm19 | original.destRegister); + write32le(loc, opcode | imm19 | ldr.destRegister); +} + +static bool isImmediateLdrEligible(const Ldr &ldr) { + // Note: We deviate from ld64 behavior, which converts to immediate loads + // only if ldr.offset < 4096, even though the offset is divided by the load's + // size in the 12-bit immediate operand. Only the unsigned offset variant is + // supported. + + uint32_t size = 1 << ldr.p2Size; + return ldr.offset >= 0 && (ldr.offset % size) == 0 && + isUInt<12>(ldr.offset >> ldr.p2Size); +} + +static void writeImmediateLdr(void *loc, const Ldr &ldr) { + assert(isImmediateLdrEligible(ldr)); + uint32_t opcode = 0x39000000; + if (ldr.isFloat) { + opcode |= 0x04000000; + assert(ldr.extendType == ZeroExtend); + } + opcode |= ldr.destRegister; + opcode |= ldr.baseRegister << 5; + uint8_t size, opc; + if (ldr.p2Size == 4) { + size = 0; + opc = 3; + } else { + opc = ldr.extendType; + size = ldr.p2Size; + } + uint32_t immBits = ldr.offset >> ldr.p2Size; + write32le(loc, opcode | (immBits << 10) | (opc << 22) | (size << 30)); } uint64_t OptimizationHintContext::getRelocTarget(const Reloc &reloc) { @@ -352,7 +391,7 @@ if (rel1->referentVA != rel2->referentVA) return; int64_t delta = rel1->referentVA - rel1->rel.offset - isec->getVA(); - if (delta >= (1 << 20) || delta < -(1 << 20)) + if (!isValidAdrOffset(delta)) return; writeAdr(buf + hint.offset0, add.destRegister, delta); @@ -413,16 +452,12 @@ return; if (ldr.offset != (rel1->referentVA & 0xfff)) return; - if ((rel1->referentVA & 3) != 0) - return; - if (ldr.size == 1 || ldr.size == 2) - return; - int64_t delta = rel1->referentVA - rel2->rel.offset - isec->getVA(); - if (delta >= (1 << 20) || delta < -(1 << 20)) + ldr.offset = rel1->referentVA - rel2->rel.offset - isec->getVA(); + if (!isLiteralLdrEligible(ldr)) return; writeNop(buf + hint.offset0); - writeLiteralLdr(buf + hint.offset0 + hint.delta[0], ldr, delta); + writeLiteralLdr(buf + hint.offset0 + hint.delta[0], ldr); } // GOT loads are emitted by the compiler as a pair of adrp and ldr instructions, @@ -438,6 +473,101 @@ applyAdrpLdr(hint); } +// Relaxes a load from an address specified in the GOT. +// If the symbol is external and the GOT is within a +/- 1 MiB range, the +// adrp + add used for loading the GOT is turned into adr + nop. +// If the symbol is local, the adrp + add + ldr sequence is turned into a +// literal ldr or adr(p) + ldr if possible. +void OptimizationHintContext::applyAdrpLdrGotLdr(const OptimizationHint &hint) { + uint32_t ins1 = read32le(buf + hint.offset0); + Adrp adrp; + if (!parseAdrp(ins1, adrp)) + return; + uint32_t ins3 = read32le(buf + hint.offset0 + hint.delta[1]); + Ldr ldr3; + if (!parseLdr(ins3, ldr3)) + return; + uint32_t ins2 = read32le(buf + hint.offset0 + hint.delta[0]); + Ldr ldr2; + Add add2; + + Optional rel1 = findPrimaryReloc(hint.offset0); + Optional rel2 = findReloc(hint.offset0 + hint.delta[0]); + if (!rel1 || !rel2) + return; + + if (parseAdd(ins2, add2)) { + // adrp x0, _foo@PAGE + // add x1, x0, _foo@PAGEOFF + // ldr x2, [x1, #off] + + if (adrp.destRegister != add2.srcRegister) + return; + if (add2.destRegister != ldr3.baseRegister) + return; + + // Load from the target address directly. + // nop + // nop + // ldr x2, [_foo + #off] + uint64_t rel3VA = hint.offset0 + hint.delta[1] + isec->getVA(); + Ldr literalLdr = ldr3; + literalLdr.offset += rel1->referentVA - rel3VA; + if (isLiteralLdrEligible(literalLdr)) { + writeNop(buf + hint.offset0); + writeNop(buf + hint.offset0 + hint.delta[0]); + writeLiteralLdr(buf + hint.offset0 + hint.delta[1], literalLdr); + return; + } + + // Load the target address into a register and load from there indirectly. + // adr x1, _foo + // nop + // ldr x2, [x1, #off] + int64_t adrOffset = rel1->referentVA - rel1->rel.offset - isec->getVA(); + if (isValidAdrOffset(adrOffset)) { + writeAdr(buf + hint.offset0, ldr3.baseRegister, adrOffset); + writeNop(buf + hint.offset0 + hint.delta[0]); + return; + } + + // Move adding the target's page offset into the ldr's immediate offset. + // adrp x0, _foo@PAGE + // nop + // ldr x2, [x0, _foo@PAGEOFF + #off] + Ldr immediateLdr = ldr3; + immediateLdr.baseRegister = adrp.destRegister; + immediateLdr.offset += add2.addend; + if (isImmediateLdrEligible(immediateLdr)) { + writeNop(buf + hint.offset0 + hint.delta[0]); + writeImmediateLdr(buf + hint.offset0 + hint.delta[1], immediateLdr); + return; + } + } else if (parseLdr(ins2, ldr2)) { + // adrp x1, _foo@GOTPAGE + // ldr x2, [x1, _foo@GOTPAGEOFF] + // ldr x3, [x2, #off] + if (ldr2.baseRegister != adrp.destRegister) + return; + if (ldr3.baseRegister != ldr2.destRegister) + return; + // Loads from the GOT must be pointer sized. + if (ldr2.p2Size != 3 || ldr2.isFloat) + return; + + // Load the GOT entry's address directly. + // nop + // ldr x2, _foo@GOTPAGE + _foo@GOTPAGEOFF + // ldr x3, [x2, #off] + Ldr literalLdr = ldr2; + literalLdr.offset = rel1->referentVA - rel2->rel.offset - isec->getVA(); + if (isLiteralLdrEligible(literalLdr)) { + writeNop(buf + hint.offset0); + writeLiteralLdr(buf + hint.offset0 + hint.delta[0], literalLdr); + } + } +} + void ARM64::applyOptimizationHints(uint8_t *buf, const ConcatInputSection *isec, ArrayRef relocTargets) const { assert(isec); @@ -457,7 +587,11 @@ ctx1.applyAdrpLdr(hint); break; case LOH_ARM64_ADRP_ADD_LDR: + // TODO: Implement this + break; case LOH_ARM64_ADRP_LDR_GOT_LDR: + ctx1.applyAdrpLdrGotLdr(hint); + break; case LOH_ARM64_ADRP_ADD_STR: case LOH_ARM64_ADRP_LDR_GOT_STR: // TODO: Implement these Index: lld/test/MachO/loh-adrp-ldr-got-ldr.s =================================================================== --- /dev/null +++ lld/test/MachO/loh-adrp-ldr-got-ldr.s @@ -0,0 +1,263 @@ +# REQUIRES: aarch64 + +# RUN: rm -rf %t; split-file %s %t +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/lib.s -o %t/lib.o +# RUN: %lld -arch arm64 -dylib -o %t/lib.dylib %t/lib.o +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/external.s -o %t/near-got.o +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/external.s -defsym=PADDING=1 -o %t/far-got.o +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/local.s -o %t/local.o +# RUN: %lld -arch arm64 %t/near-got.o %t/lib.dylib -o %t/NearGot +# RUN: %lld -arch arm64 %t/far-got.o %t/lib.dylib -o %t/FarGot +# RUN: %lld -arch arm64 %t/local.o -o %t/Local +# RUN: llvm-objdump -d --macho %t/NearGot | FileCheck %s -check-prefix=NEAR-GOT +# RUN: llvm-objdump -d --macho %t/FarGot | FileCheck %s -check-prefix=FAR-GOT +# RUN: llvm-objdump -d --macho %t/Local | FileCheck %s -check-prefix=LOCAL + +#--- external.s +.text +.align 2 +.globl _main +_main: + +## Basic test +L1: adrp x0, _external@GOTPAGE +L2: ldr x1, [x0, _external@GOTPAGEOFF] +L3: ldr x2, [x1] +# NEAR-GOT-LABEL: _main: +# NEAR-GOT-NEXT: nop +# NEAR-GOT-NEXT: ldr x1, #{{.*}} ; literal pool symbol address: _external +# NEAR-GOT-NEXT: ldr x2, [x1] +# FAR-GOT-LABEL: _main: +# FAR-GOT-NEXT: adrp x0 +# FAR-GOT-NEXT: ldr x1 +# FAR-GOT-NEXT: ldr x2, [x1] + +## The second load has an offset +L4: adrp x0, _external@GOTPAGE +L5: ldr x1, [x0, _external@GOTPAGEOFF] +L6: ldr q2, [x1, #16] +# NEAR-GOT-NEXT: nop +# NEAR-GOT-NEXT: ldr x1, #{{.*}} ; literal pool symbol address: _external +# NEAR-GOT-NEXT: ldr q2, [x1, #16] +# FAR-GOT-NEXT: adrp x0 +# FAR-GOT-NEXT: ldr x1 +# FAR-GOT-NEXT: ldr q2, [x1, #16] + +### Tests for invalid inputs +.ifndef PADDING +## Registers don't match +L7: adrp x0, _external@GOTPAGE +L8: ldr x1, [x1, _external@GOTPAGEOFF] +L9: ldr x2, [x1] +# NEAR-GOT-NEXT: adrp x0 +# NEAR-GOT-NEXT: ldr x1 +# NEAR-GOT-NEXT: ldr x2, [x1] + +## Registers don't match +L10: adrp x0, _external@GOTPAGE +L11: ldr x1, [x0, _external@GOTPAGEOFF] +L12: ldr x2, [x0] +# NEAR-GOT-NEXT: adrp x0 +# NEAR-GOT-NEXT: ldr x1 +# NEAR-GOT-NEXT: ldr x2, [x0] + +## Not an LDR (immediate) +L13: adrp x0, _external@GOTPAGE +L14: ldr x1, 0 +L15: ldr x2, [x1] +# NEAR-GOT-NEXT: adrp x0 +# NEAR-GOT-NEXT: ldr x1 +# NEAR-GOT-NEXT: ldr x2, [x1] + +.loh AdrpLdrGotLdr L7, L8, L9 +.loh AdrpLdrGotLdr L10, L11, L12 +.loh AdrpLdrGotLdr L13, L14, L15 +.endif + +.loh AdrpLdrGotLdr L1, L2, L3 +.loh AdrpLdrGotLdr L4, L5, L6 + +.ifdef PADDING +.space 1048576 +.endif +.data + + +#--- lib.s +.data +.align 4 +.globl _external +_external: + .zero 32 + +#--- local.s +.text +.align 2 +.globl _main +_main: + +### Transformation to a literal LDR +## Basic case +L1: adrp x0, _close@GOTPAGE +L2: ldr x1, [x0, _close@GOTPAGEOFF] +L3: ldr x2, [x1] +# LOCAL-LABEL: _main: +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2 + +## Load with offset +L4: adrp x0, _close@GOTPAGE +L5: ldr x1, [x0, _close@GOTPAGEOFF] +L6: ldr x2, [x1, #8] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2 + +## 32 bit load +L7: adrp x0, _close@GOTPAGE +L8: ldr x1, [x0, _close@GOTPAGEOFF] +L9: ldr w1, [x1] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr w1, _close + +## Floating point +L10: adrp x0, _close@GOTPAGE +L11: ldr x1, [x0, _close@GOTPAGEOFF] +L12: ldr s1, [x1] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr s1, _close + +L13: adrp x0, _close@GOTPAGE +L14: ldr x1, [x0, _close@GOTPAGEOFF] +L15: ldr d1, [x1, #8] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr d1, _close8 + +L16: adrp x0, _close@GOTPAGE +L17: ldr x1, [x0, _close@GOTPAGEOFF] +L18: ldr q0, [x1] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr q0, _close + + +### Transformation to ADR+LDR +## 1 byte floating point load +L19: adrp x0, _close@GOTPAGE +L20: ldr x1, [x0, _close@GOTPAGEOFF] +L21: ldr b2, [x1] +# LOCAL-NEXT: adr x1 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr b2, [x1] + +## 1 byte GPR load, zero extend +L22: adrp x0, _close@GOTPAGE +L23: ldr x1, [x0, _close@GOTPAGEOFF] +L24: ldrb w2, [x1] +# LOCAL-NEXT: adr x1 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldrb w2, [x1] + +## 1 byte GPR load, sign extend +L25: adrp x0, _close@GOTPAGE +L26: ldr x1, [x0, _close@GOTPAGEOFF] +L27: ldrsb x2, [x1] +# LOCAL-NEXT: adr x1 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldrsb x2, [x1] + +## Unaligned +L28: adrp x0, _unaligned@GOTPAGE +L29: ldr x1, [x0, _unaligned@GOTPAGEOFF] +L30: ldr x2, [x1] +# LOCAL-NEXT: adr x1 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2, [x1] + + +### Transformation to ADRP + imediate LDR +## Basic test: target is far +L31: adrp x0, _far@GOTPAGE +L32: ldr x1, [x0, _far@GOTPAGEOFF] +L33: ldr x2, [x1] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2 + +## With offset +L34: adrp x0, _far@GOTPAGE +L35: ldr x1, [x0, _far@GOTPAGEOFF] +L36: ldr x2, [x1, #8] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2 + +### No changes other than GOT relaxation +## Far and unaligned +L37: adrp x0, _far_unaligned@GOTPAGE +L38: ldr x1, [x0, _far_unaligned@GOTPAGEOFF] +L39: ldr x2, [x1] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: add x1, x0 +# LOCAL-NEXT: ldr x2, [x1] + +## Far with large offset (_far_offset@GOTPAGEOFF + #255 > 4095) +L40: adrp x0, _far_offset@GOTPAGE +L41: ldr x1, [x0, _far_offset@GOTPAGEOFF] +L42: ldrb w2, [x1, #255] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: add x1, x0 +# LOCAL-NEXT: ldrb w2, [x1, #255] + +### Tests for invalid inputs, only GOT relaxation should happen +## Registers don't match +L43: adrp x0, _far@GOTPAGE +L44: ldr x1, [x0, _far@GOTPAGEOFF] +L45: ldr x2, [x2] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: add x1, x0 +# LOCAL-NEXT: ldr x2, [x2] + +.data +.align 4 + .quad 0 +_close: + .quad 0 +_close8: + .quad 0 + .byte 0 +_unaligned: + .quad 0 + +.space 1048576 +.align 12 + .quad 0 +_far: + .quad 0 + .byte 0 +_far_unaligned: + .quad 0 +.space 4000 +_far_offset: + .byte 0 + + +.loh AdrpLdrGotLdr L1, L2, L3 +.loh AdrpLdrGotLdr L4, L5, L6 +.loh AdrpLdrGotLdr L7, L8, L9 +.loh AdrpLdrGotLdr L10, L11, L12 +.loh AdrpLdrGotLdr L13, L14, L15 +.loh AdrpLdrGotLdr L16, L17, L18 +.loh AdrpLdrGotLdr L19, L20, L21 +.loh AdrpLdrGotLdr L22, L23, L24 +.loh AdrpLdrGotLdr L25, L26, L27 +.loh AdrpLdrGotLdr L28, L29, L30 +.loh AdrpLdrGotLdr L31, L32, L33 +.loh AdrpLdrGotLdr L34, L35, L36 +.loh AdrpLdrGotLdr L37, L38, L39 +.loh AdrpLdrGotLdr L40, L41, L42 +.loh AdrpLdrGotLdr L43, L44, L45