diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -2547,11 +2547,24 @@
 }
 
 template <class ELFT> void Writer<ELFT>::assignFileOffsetsBinary() {
-  uint64_t off = 0;
+  // Compute the minimum LMA of all non-empty non-NOBITS sections as minAddr.
+  auto needsOffset = [](OutputSection &sec) {
+    return sec.type != SHT_NOBITS && (sec.flags & SHF_ALLOC) && sec.size > 0;
+  };
+  uint64_t minAddr = UINT64_MAX;
   for (OutputSection *sec : outputSections)
-    if (sec->flags & SHF_ALLOC)
-      off = setFileOffset(sec, off);
-  fileSize = alignTo(off, config->wordsize);
+    if (needsOffset(*sec)) {
+      sec->offset = sec->getLMA();
+      minAddr = std::min(minAddr, sec->offset);
+    }
+
+  // Sections are laid out at LMA minus minAddr.
+  fileSize = 0;
+  for (OutputSection *sec : outputSections)
+    if (needsOffset(*sec)) {
+      sec->offset -= minAddr;
+      fileSize = std::max(fileSize, sec->offset + sec->size);
+    }
 }
 
 static std::string rangeToString(uint64_t addr, uint64_t len) {
diff --git a/lld/test/ELF/oformat-binary-ttext.s b/lld/test/ELF/oformat-binary-ttext.s
--- a/lld/test/ELF/oformat-binary-ttext.s
+++ b/lld/test/ELF/oformat-binary-ttext.s
@@ -4,9 +4,8 @@
 # RUN: ld.lld -N -Ttext 0x100 -o %t.out %t --oformat binary
 # RUN: od -t x1 -v %t.out | FileCheck %s --check-prefix=BIN
 
-# BIN:      0000000 90 00 00 00 00 00 00 00
-# BIN-NEXT: 0000010
-# BIN-NOT:  0000020
+# BIN:      0000000 90
+# BIN-NEXT: 0000001
 
 ## The same but without OMAGIC.
 # RUN: ld.lld -Ttext 0x100 -o %t.out %t --oformat binary
diff --git a/lld/test/ELF/oformat-binary.s b/lld/test/ELF/oformat-binary.s
--- a/lld/test/ELF/oformat-binary.s
+++ b/lld/test/ELF/oformat-binary.s
@@ -3,8 +3,8 @@
 
 # RUN: ld.lld -o %t.out %t --oformat binary
 # RUN: od -t x1 -v %t.out | FileCheck %s
-# CHECK: 000000 90 11 22 00 00 00 00 00
-# CHECK-NOT: 00000010
+# CHECK:      0000000 90 11 22
+# CHECK-NEXT: 0000003
 
 ## Check case when linkerscript is used.
 # RUN: echo "SECTIONS { . = 0x1000; }" > %t.script
@@ -15,6 +15,31 @@
 # RUN: ld.lld -o %t2.out --script %t.script %t --oformat binary
 # RUN: od -t x1 -v %t2.out | FileCheck %s
 
+## LMA(.text)=0x100, LMA(.mysec)=0x108. The minimum LMA of all non-empty sections is 0x100.
+## We place an output section at its LMA minus 0x100.
+# RUN: echo 'SECTIONS { .text 0x100 : {*(.text)} .mysec ALIGN(8) : {*(.mysec*)} }' > %talign.lds
+# RUN: ld.lld -T %talign.lds %t --oformat binary -o %talign
+# RUN: od -Ax -t x1 %talign | FileCheck %s --check-prefix=ALIGN --ignore-case
+
+# ALIGN:      000000 90 00 00 00 00 00 00 00 11 22
+# ALIGN-NEXT: 00000a
+
+## The empty section .data is ignored when computing the file size.
+# RUN: echo 'SECTIONS { .text : {*(.text .mysec*)} .data 0x100 : {keep = .;}}' > %tempty.lds
+# RUN: ld.lld -T %tempty.lds %t --oformat binary -o %tempty
+# RUN: od -t x1 %tempty | FileCheck %s
+
+## NOBITS sections are ignored as well.
+# RUN: echo 'SECTIONS { .text : {*(.text .mysec*)} .data 0x100 (NOLOAD) : {BYTE(0)}}' > %tnobits.lds
+# RUN: ld.lld -T %tnobits.lds %t --oformat binary -o %tnobits
+# RUN: od -t x1 %tnobits | FileCheck %s
+
+## FIXME .mysec should be placed at file offset 1.
+## This does not work because for a section without PT_LOAD, we consider LMA = VMA.
+# RUN: echo 'SECTIONS { .text : {*(.text)} .mysec 0x8 : AT(1) {*(.mysec*)} }' > %tlma.lds
+# RUN: ld.lld -T %tlma.lds %t --oformat binary -o %tlma
+# RUN: od -Ax -t x1 %tlma | FileCheck %s --check-prefix=ALIGN --ignore-case
+
 # RUN: not ld.lld -o /dev/null %t --oformat foo 2>&1 \
 # RUN:   | FileCheck %s --check-prefix ERR
 # ERR: unknown --oformat value: foo