Index: lld/MachO/Arch/ARM64.cpp =================================================================== --- lld/MachO/Arch/ARM64.cpp +++ lld/MachO/Arch/ARM64.cpp @@ -190,6 +190,7 @@ void applyAdrpAdrp(const OptimizationHint &); void applyAdrpLdr(const OptimizationHint &); void applyAdrpLdrGot(const OptimizationHint &); + void applyAdrpLdrGotLdr(const OptimizationHint &); private: uint8_t *buf; @@ -240,10 +241,8 @@ // LDR (immediate, SIMD&FP) ldr.extendType = ZeroExtend; ldr.isFloat = true; - if (size == 2 && opc == 1) - ldr.size = 4; - else if (size == 3 && opc == 1) - ldr.size = 8; + if (opc == 1) + ldr.size = 1 << size; else if (size == 0 && opc == 3) ldr.size = 16; else @@ -286,6 +285,43 @@ write32le(loc, opcode | imm19 | original.destRegister); } +static void writeImmediateLdr(void *loc, Ldr ldr) { + uint32_t opcode = 0x39000000; + if (ldr.isFloat) { + opcode |= 0x04000000; + assert(ldr.extendType == ZeroExtend); + } + opcode |= ldr.destRegister; + opcode |= ldr.baseRegister << 5; + uint8_t size, opc; + if (ldr.size == 16) { + size = 0; + opc = 3; + } else { + opc = static_cast(ldr.extendType); + switch (ldr.size) { + case 1: + size = 0; + break; + case 2: + size = 1; + break; + case 4: + size = 2; + assert(ldr.extendType != Sign32); + break; + case 8: + size = 3; + assert(ldr.extendType == ZeroExtend); + break; + default: + llvm_unreachable("Invalid immediate ldr size"); + } + } + uint32_t imm12 = (ldr.offset / ldr.size) << 10; + write32le(loc, opcode | imm12 | (opc << 22) | (size << 30)); +} + uint64_t OptimizationHintContext::getRelocTarget(const Reloc &reloc) { size_t relocIdx = &reloc - isec->relocs.data(); return relocTargets[relocIdx]; @@ -438,6 +474,98 @@ applyAdrpLdr(hint); } +// Relaxes a load from an address specified in the GOT. +// If the symbol is external and the GOT is within a +/- 1 MiB range, the +// adrp + add used for loading the GOT is turned into adr + nop. +// If the symbol is local, the adrp + add + ldr sequence is turned into a +// literal ldr or adr(p) + ldr if possible. +void OptimizationHintContext::applyAdrpLdrGotLdr(const OptimizationHint &hint) { + uint32_t ins1 = read32le(buf + hint.offset0); + Adrp adrp; + if (!parseAdrp(ins1, adrp)) + return; + uint32_t ins3 = read32le(buf + hint.offset0 + hint.delta[1]); + Ldr ldr3; + if (!parseLdr(ins3, ldr3)) + return; + uint32_t ins2 = read32le(buf + hint.offset0 + hint.delta[0]); + Ldr ldr2; + Add add2; + + Optional rel1 = findPrimaryReloc(hint.offset0); + Optional rel2 = findReloc(hint.offset0 + hint.delta[0]); + if (!rel1 || !rel2) + return; + + if (parseAdd(ins2, add2)) { + // adrp x0, _foo@PAGE + // add x1, x0, _foo@PAGEOFF + // ldr x2, [x1, #off] + + if (adrp.destRegister != add2.srcRegister) + return; + if (add2.destRegister != ldr3.baseRegister) + return; + + uint64_t rel3Offset = hint.offset0 + hint.delta[1]; + // _foo + #off offset from the third instruction + int64_t delta1 = + rel1->referentVA + ldr3.offset - rel3Offset - isec->getVA(); + // _foo's offset from the first instruction + int64_t delta2 = rel1->referentVA - rel1->rel.offset - isec->getVA(); + // _foo@PAGEOFF + #off value + int64_t delta3 = add2.addend + ldr3.offset; + + if ((ldr3.size != 1) && (ldr3.size != 2) && ((delta1 & 3) == 0) && + (-(1 << 20) <= delta1) && (delta1 < (1 << 20))) { + // nop + // nop + // ldr x2, [_foo + #off] + writeNop(buf + hint.offset0); + writeNop(buf + hint.offset0 + hint.delta[0]); + writeLiteralLdr(buf + hint.offset0 + hint.delta[1], ldr3, delta1); + } else if (-(1 << 20) <= delta2 && delta2 < (1 << 20)) { + // adr x1, _foo + // nop + // ldr x2, [x1, #off] + writeAdr(buf + hint.offset0, ldr3.baseRegister, delta2); + writeNop(buf + hint.offset0 + hint.delta[0]); + } else if ((delta3 % ldr3.size == 0) && (delta3 >= 0) && + (delta3 < 4096 * ldr3.size)) { + // Note: We deviate from ld64 behavior, which applies this only if + // delta3 < 4096, even though the offset is divided by the load's size + // in the 12-bit immediate operand. + + // adrp x0, _foo@PAGE + // nop + // ldr x2, [x0, _foo@PAGEOFF + #off] + writeNop(buf + hint.offset0 + hint.delta[0]); + ldr3.baseRegister = adrp.destRegister; + ldr3.offset = delta3; + writeImmediateLdr(buf + hint.offset0 + hint.delta[1], ldr3); + } + } else if (parseLdr(ins2, ldr2)) { + // adrp x1, _foo@GOTPAGE + // ldr x2, [x1, _foo@GOTPAGEOFF] + // ldr x3, [x2, #off] + if (ldr2.baseRegister != adrp.destRegister) + return; + if (ldr3.baseRegister != ldr2.destRegister) + return; + if (ldr2.size != 8 || ldr2.isFloat) + return; + + int64_t delta = rel1->referentVA - rel2->rel.offset - isec->getVA(); + if ((delta & 3) == 0 && (-(1 << 20) <= delta) && (delta < (1 << 20))) { + // nop + // ldr x2, _foo@GOTPAGE + _foo@GOTPAGEOFF + // ldr x3, [x2, #off] + writeNop(buf + hint.offset0); + writeLiteralLdr(buf + hint.offset0 + hint.delta[0], ldr2, delta); + } + } +} + void ARM64::applyOptimizationHints(uint8_t *buf, const ConcatInputSection *isec, ArrayRef relocTargets) const { assert(isec); @@ -457,7 +585,11 @@ ctx1.applyAdrpLdr(hint); break; case LOH_ARM64_ADRP_ADD_LDR: + // TODO: Implement this + break; case LOH_ARM64_ADRP_LDR_GOT_LDR: + ctx1.applyAdrpLdrGotLdr(hint); + break; case LOH_ARM64_ADRP_ADD_STR: case LOH_ARM64_ADRP_LDR_GOT_STR: // TODO: Implement these Index: lld/test/MachO/loh-adrp-ldr-got-ldr.s =================================================================== --- /dev/null +++ lld/test/MachO/loh-adrp-ldr-got-ldr.s @@ -0,0 +1,263 @@ +# REQUIRES: aarch64 + +# RUN: rm -rf %t; split-file %s %t +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/lib.s -o %t/lib.o +# RUN: %lld -arch arm64 -dylib -o %t/lib.dylib %t/lib.o +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/external.s -o %t/near-got.o +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/external.s -defsym=PADDING=1 -o %t/far-got.o +# RUN: llvm-mc -filetype=obj -triple=arm64-apple-darwin %t/local.s -o %t/local.o +# RUN: %lld -arch arm64 %t/near-got.o %t/lib.dylib -o %t/NearGot +# RUN: %lld -arch arm64 %t/far-got.o %t/lib.dylib -o %t/FarGot +# RUN: %lld -arch arm64 %t/local.o -o %t/Local +# RUN: llvm-objdump -d --macho %t/NearGot | FileCheck %s -check-prefix=NEAR-GOT +# RUN: llvm-objdump -d --macho %t/FarGot | FileCheck %s -check-prefix=FAR-GOT +# RUN: llvm-objdump -d --macho %t/Local | FileCheck %s -check-prefix=LOCAL + +#--- external.s +.text +.align 2 +.globl _main +_main: + +## Basic test +L1: adrp x0, _external@GOTPAGE +L2: ldr x1, [x0, _external@GOTPAGEOFF] +L3: ldr x2, [x1] +# NEAR-GOT-LABEL: _main: +# NEAR-GOT-NEXT: nop +# NEAR-GOT-NEXT: ldr x1, #{{.*}} ; literal pool symbol address: _external +# NEAR-GOT-NEXT: ldr x2, [x1] +# FAR-GOT-LABEL: _main: +# FAR-GOT-NEXT: adrp x0 +# FAR-GOT-NEXT: ldr x1 +# FAR-GOT-NEXT: ldr x2, [x1] + +## The second load has an offset +L4: adrp x0, _external@GOTPAGE +L5: ldr x1, [x0, _external@GOTPAGEOFF] +L6: ldr q2, [x1, #16] +# NEAR-GOT-NEXT: nop +# NEAR-GOT-NEXT: ldr x1, #{{.*}} ; literal pool symbol address: _external +# NEAR-GOT-NEXT: ldr q2, [x1, #16] +# FAR-GOT-NEXT: adrp x0 +# FAR-GOT-NEXT: ldr x1 +# FAR-GOT-NEXT: ldr q2, [x1, #16] + +### Tests for invalid inputs +.ifndef PADDING +## Registers don't match +L7: adrp x0, _external@GOTPAGE +L8: ldr x1, [x1, _external@GOTPAGEOFF] +L9: ldr x2, [x1] +# NEAR-GOT-NEXT: adrp x0 +# NEAR-GOT-NEXT: ldr x1 +# NEAR-GOT-NEXT: ldr x2, [x1] + +## Registers don't match +L10: adrp x0, _external@GOTPAGE +L11: ldr x1, [x0, _external@GOTPAGEOFF] +L12: ldr x2, [x0] +# NEAR-GOT-NEXT: adrp x0 +# NEAR-GOT-NEXT: ldr x1 +# NEAR-GOT-NEXT: ldr x2, [x0] + +## Not an LDR (immediate) +L13: adrp x0, _external@GOTPAGE +L14: ldr x1, 0 +L15: ldr x2, [x1] +# NEAR-GOT-NEXT: adrp x0 +# NEAR-GOT-NEXT: ldr x1 +# NEAR-GOT-NEXT: ldr x2, [x1] + +.loh AdrpLdrGotLdr L7, L8, L9 +.loh AdrpLdrGotLdr L10, L11, L12 +.loh AdrpLdrGotLdr L13, L14, L15 +.endif + +.loh AdrpLdrGotLdr L1, L2, L3 +.loh AdrpLdrGotLdr L4, L5, L6 + +.ifdef PADDING +.space 1048576 +.endif +.data + + +#--- lib.s +.data +.align 4 +.globl _external +_external: + .zero 32 + +#--- local.s +.text +.align 2 +.globl _main +_main: + +### Transformation to a literal LDR +## Basic case +L1: adrp x0, _close@GOTPAGE +L2: ldr x1, [x0, _close@GOTPAGEOFF] +L3: ldr x2, [x1] +# LOCAL-LABEL: _main: +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2 + +## Load with offset +L4: adrp x0, _close@GOTPAGE +L5: ldr x1, [x0, _close@GOTPAGEOFF] +L6: ldr x2, [x1, #8] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2 + +## 32 bit load +L7: adrp x0, _close@GOTPAGE +L8: ldr x1, [x0, _close@GOTPAGEOFF] +L9: ldr w1, [x1] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr w1, _close + +## Floating point +L10: adrp x0, _close@GOTPAGE +L11: ldr x1, [x0, _close@GOTPAGEOFF] +L12: ldr s1, [x1] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr s1, _close + +L13: adrp x0, _close@GOTPAGE +L14: ldr x1, [x0, _close@GOTPAGEOFF] +L15: ldr d1, [x1, #8] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr d1, _close8 + +L16: adrp x0, _close@GOTPAGE +L17: ldr x1, [x0, _close@GOTPAGEOFF] +L18: ldr q0, [x1] +# LOCAL-NEXT: nop +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr q0, _close + + +### Transformation to ADR+LDR +## 1 byte floating point load +L19: adrp x0, _close@GOTPAGE +L20: ldr x1, [x0, _close@GOTPAGEOFF] +L21: ldr b2, [x1] +# LOCAL-NEXT: adr x1 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr b2, [x1] + +## 1 byte GPR load, zero extend +L22: adrp x0, _close@GOTPAGE +L23: ldr x1, [x0, _close@GOTPAGEOFF] +L24: ldrb w2, [x1] +# LOCAL-NEXT: adr x1 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldrb w2, [x1] + +## 1 byte GPR load, sign extend +L25: adrp x0, _close@GOTPAGE +L26: ldr x1, [x0, _close@GOTPAGEOFF] +L27: ldrsb x2, [x1] +# LOCAL-NEXT: adr x1 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldrsb x2, [x1] + +## Unaligned +L28: adrp x0, _unaligned@GOTPAGE +L29: ldr x1, [x0, _unaligned@GOTPAGEOFF] +L30: ldr x2, [x1] +# LOCAL-NEXT: adr x1 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2, [x1] + + +### Transformation to ADRP + imediate LDR +## Basic test: target is far +L31: adrp x0, _far@GOTPAGE +L32: ldr x1, [x0, _far@GOTPAGEOFF] +L33: ldr x2, [x1] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2 + +## With offset +L34: adrp x0, _far@GOTPAGE +L35: ldr x1, [x0, _far@GOTPAGEOFF] +L36: ldr x2, [x1, #8] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: nop +# LOCAL-NEXT: ldr x2 + +### No changes other than GOT relaxation +## Far and unaligned +L37: adrp x0, _far_unaligned@GOTPAGE +L38: ldr x1, [x0, _far_unaligned@GOTPAGEOFF] +L39: ldr x2, [x1] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: add x1, x0 +# LOCAL-NEXT: ldr x2, [x1] + +## Far with large offset (_far_offset@GOTPAGEOFF + #255 > 4095) +L40: adrp x0, _far_offset@GOTPAGE +L41: ldr x1, [x0, _far_offset@GOTPAGEOFF] +L42: ldrb w2, [x1, #255] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: add x1, x0 +# LOCAL-NEXT: ldrb w2, [x1, #255] + +### Tests for invalid inputs, only GOT relaxation should happen +## Registers don't match +L43: adrp x0, _far@GOTPAGE +L44: ldr x1, [x0, _far@GOTPAGEOFF] +L45: ldr x2, [x2] +# LOCAL-NEXT: adrp x0 +# LOCAL-NEXT: add x1, x0 +# LOCAL-NEXT: ldr x2, [x2] + +.data +.align 4 + .quad 0 +_close: + .quad 0 +_close8: + .quad 0 + .byte 0 +_unaligned: + .quad 0 + +.space 1048576 +.align 12 + .quad 0 +_far: + .quad 0 + .byte 0 +_far_unaligned: + .quad 0 +.space 4000 +_far_offset: + .byte 0 + + +.loh AdrpLdrGotLdr L1, L2, L3 +.loh AdrpLdrGotLdr L4, L5, L6 +.loh AdrpLdrGotLdr L7, L8, L9 +.loh AdrpLdrGotLdr L10, L11, L12 +.loh AdrpLdrGotLdr L13, L14, L15 +.loh AdrpLdrGotLdr L16, L17, L18 +.loh AdrpLdrGotLdr L19, L20, L21 +.loh AdrpLdrGotLdr L22, L23, L24 +.loh AdrpLdrGotLdr L25, L26, L27 +.loh AdrpLdrGotLdr L28, L29, L30 +.loh AdrpLdrGotLdr L31, L32, L33 +.loh AdrpLdrGotLdr L34, L35, L36 +.loh AdrpLdrGotLdr L37, L38, L39 +.loh AdrpLdrGotLdr L40, L41, L42 +.loh AdrpLdrGotLdr L43, L44, L45