diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -1889,26 +1889,44 @@
 // out to be empty.
 static void removeUnusedSyntheticSections() {
   // All input synthetic sections that can be empty are placed after
-  // all regular ones. We iterate over them all and exit at first
-  // non-synthetic.
-  for (InputSectionBase *s : llvm::reverse(inputSections)) {
-    SyntheticSection *ss = dyn_cast<SyntheticSection>(s);
-    if (!ss)
-      return;
-    OutputSection *os = ss->getParent();
-    if (!os || ss->isNeeded())
-      continue;
+  // all regular ones. Reverse iterate to find the first synthetic section
+  // after a non-synthetic one which will be our starting point.
+  auto Start = std::find_if(inputSections.rbegin(), inputSections.rend(),
+                            [](InputSectionBase *S) {
+                              return !isa<SyntheticSection>(S);
+                            })
+                   .base();
+
+  DenseSet<InputSectionDescription *> IsdSet;
+  // Mark unused synthetic sections for deletion
+  auto End = std::stable_partition(
+      Start, inputSections.end(), [&](InputSectionBase *S) {
+        SyntheticSection *Ss = dyn_cast<SyntheticSection>(S);
+        OutputSection *Os = Ss->getParent();
+        if (!Os || Ss->isNeeded())
+          return true;
 
-    // If we reach here, then ss is an unused synthetic section and we want to
-    // remove it from the corresponding input section description, and
-    // orphanSections.
-    for (BaseCommand *b : os->sectionCommands)
-      if (auto *isd = dyn_cast<InputSectionDescription>(b))
-        llvm::erase_if(isd->sections,
-                       [=](InputSection *isec) { return isec == ss; });
-    llvm::erase_if(script->orphanSections,
-                   [=](const InputSectionBase *isec) { return isec == ss; });
-  }
+        // If we reach here, then Ss is an unused synthetic section and we want
+        // to remove it from the corresponding input section description, and
+        // orphanSections.
+        for (BaseCommand *B : Os->sectionCommands)
+          if (auto *Isd = dyn_cast<InputSectionDescription>(B))
+            IsdSet.insert(Isd);
+
+        llvm::erase_if(
+            script->orphanSections,
+            [=](const InputSectionBase *Isec) { return Isec == Ss; });
+
+        return false;
+      });
+
+  DenseSet<InputSectionBase *> Unused(End, inputSections.end());
+  for (auto *Isd : IsdSet)
+    llvm::erase_if(Isd->sections,
+                   [=](InputSection *Isec) { return Unused.count(Isec); });
+
+  // Erase unused synthetic sections.
+  inputSections.erase(End, inputSections.end());
 }
 
 // Create output section objects and add them to OutputSections.
diff --git a/lld/test/ELF/shuffle-sections-init-fini.s b/lld/test/ELF/shuffle-sections-init-fini.s
--- a/lld/test/ELF/shuffle-sections-init-fini.s
+++ b/lld/test/ELF/shuffle-sections-init-fini.s
@@ -21,12 +21,12 @@
 # CHECK:      Hex dump of section '.init_array'
 # CHECK-NEXT: 0x{{[0-9a-f]+}} ff
 # ORDERED-SAME: 000102 03040506 0708090a 0b
-# SHUFFLED-SAME: 080301 04050907 0b020a06 00
+# SHUFFLED-SAME: 070201 0006090a 040b0503 08
 
 # CHECK:      Hex dump of section '.fini_array'
 # CHECK-NEXT: 0x{{[0-9a-f]+}} ff
 # ORDERED-SAME:  000102 03040506 0708090a 0b
-# SHUFFLED-SAME: 0a0405 08070b02 03090006 01
+# SHUFFLED-SAME: 070008 0a040209 03010b06 05
 
 ## With a SECTIONS command, SHT_INIT_ARRAY prirotities are ignored.
 ## All .init_array* are shuffled together.
@@ -42,7 +42,7 @@
 
 # CHECK2:       Hex dump of section '.init_array'
 # ORDERED2-NEXT:  0x{{[0-9a-f]+}} 00010203 04050607 08090a0b ff
-# SHUFFLED2-NEXT: 0x{{[0-9a-f]+}} 08030104 0509070b 02ff0a06 00
+# SHUFFLED2-NEXT: 0x{{[0-9a-f]+}} 07020100 06090a04 0b050308 ff
 
 .irp i,0,1,2,3,4,5,6,7,8,9,10,11
   .section .init,"ax",@progbits,unique,\i
diff --git a/lld/test/ELF/shuffle-sections.s b/lld/test/ELF/shuffle-sections.s
--- a/lld/test/ELF/shuffle-sections.s
+++ b/lld/test/ELF/shuffle-sections.s
@@ -10,7 +10,7 @@
 # RUN: ld.lld --shuffle-sections='*=1' %t.o -o %t1.out
 # RUN: llvm-readelf -x .text %t1.out | FileCheck %s --check-prefix=SHUFFLE1
 # SHUFFLE1: Hex dump of section '.text':
-# SHUFFLE1-NEXT: 0203cccc 0104
+# SHUFFLE1-NEXT: 01020403
 
 ## Test that --shuffle-sections= can be used with --symbol-ordering-file
 # RUN: echo "foo" > %t_order.txt
@@ -19,7 +19,7 @@
 # RUN: ld.lld --symbol-ordering-file %t_order.txt --shuffle-sections='*=2' %t.o -o %t2.out
 # RUN: llvm-readelf -x .text %t2.out | FileCheck %s --check-prefix=SHUFFLE2
 # SHUFFLE2: Hex dump of section '.text':
-# SHUFFLE2-NEXT: 02cccccc 010403
+# SHUFFLE2-NEXT: 02cccccc 010304
 
 # RUN: ld.lld --symbol-ordering-file %t_order.txt --shuffle-sections='*=3' %t.o -o %t3.out
 # RUN: llvm-readelf -x .text %t3.out | FileCheck %s --check-prefix=SHUFFLE3