diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp
--- a/lld/ELF/Arch/PPC64.cpp
+++ b/lld/ELF/Arch/PPC64.cpp
@@ -279,9 +279,6 @@
 template <typename ELFT>
 static std::pair<Defined *, int64_t>
 getRelaTocSymAndAddend(InputSectionBase *tocSec, uint64_t offset) {
-  if (tocSec->numRelocations == 0)
-    return {};
-
   // .rela.toc contains exclusively R_PPC64_ADDR64 relocations sorted by
   // r_offset: 0, 8, 16, etc. For a given Offset, Offset / 8 gives us the
   // relocation index in most cases.
@@ -291,7 +288,10 @@
   // points to a relocation with larger r_offset. Do a linear probe then.
   // Constants are extremely uncommon in .toc and the extra number of array
   // accesses can be seen as a small constant.
-  ArrayRef<typename ELFT::Rela> relas = tocSec->template relas<ELFT>();
+  ArrayRef<typename ELFT::Rela> relas =
+      tocSec->template relsOrRelas<ELFT>().relas;
+  if (relas.empty())
+    return {};
   uint64_t index = std::min<uint64_t>(offset / 8, relas.size() - 1);
   for (;;) {
     if (relas[index].r_offset == offset) {
diff --git a/lld/ELF/DWARF.cpp b/lld/ELF/DWARF.cpp
--- a/lld/ELF/DWARF.cpp
+++ b/lld/ELF/DWARF.cpp
@@ -137,9 +137,10 @@
 Optional<RelocAddrEntry> LLDDwarfObj<ELFT>::find(const llvm::DWARFSection &s,
                                                  uint64_t pos) const {
   auto &sec = static_cast<const LLDDWARFSection &>(s);
-  if (sec.sec->areRelocsRela)
-    return findAux(*sec.sec, pos, sec.sec->template relas<ELFT>());
-  return findAux(*sec.sec, pos, sec.sec->template rels<ELFT>());
+  const RelsOrRelas<ELFT> rels = sec.sec->template relsOrRelas<ELFT>();
+  if (rels.areRelocsRel())
+    return findAux(*sec.sec, pos, rels.rels);
+  return findAux(*sec.sec, pos, rels.relas);
 }
 
 template class elf::LLDDwarfObj<ELF32LE>;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1913,10 +1913,11 @@
 static void readSymbolPartitionSection(InputSectionBase *s) {
   // Read the relocation that refers to the partition's entry point symbol.
   Symbol *sym;
-  if (s->areRelocsRela)
-    sym = &s->getFile<ELFT>()->getRelocTargetSym(s->template relas<ELFT>()[0]);
+  const RelsOrRelas<ELFT> rels = s->template relsOrRelas<ELFT>();
+  if (rels.areRelocsRel())
+    sym = &s->getFile<ELFT>()->getRelocTargetSym(rels.rels[0]);
   else
-    sym = &s->getFile<ELFT>()->getRelocTargetSym(s->template rels<ELFT>()[0]);
+    sym = &s->getFile<ELFT>()->getRelocTargetSym(rels.relas[0]);
   if (!isa<Defined>(sym) || !sym->includeInDynsym())
     return;
 
diff --git a/lld/ELF/ICF.cpp b/lld/ELF/ICF.cpp
--- a/lld/ELF/ICF.cpp
+++ b/lld/ELF/ICF.cpp
@@ -239,6 +239,8 @@
 template <class RelTy>
 bool ICF<ELFT>::constantEq(const InputSection *secA, ArrayRef<RelTy> ra,
                            const InputSection *secB, ArrayRef<RelTy> rb) {
+  if (ra.size() != rb.size())
+    return false;
   for (size_t i = 0; i < ra.size(); ++i) {
     if (ra[i].r_offset != rb[i].r_offset ||
         ra[i].getType(config->isMips64EL) != rb[i].getType(config->isMips64EL))
@@ -312,8 +314,8 @@
 // except relocation targets.
 template <class ELFT>
 bool ICF<ELFT>::equalsConstant(const InputSection *a, const InputSection *b) {
-  if (a->numRelocations != b->numRelocations || a->flags != b->flags ||
-      a->getSize() != b->getSize() || a->data() != b->data())
+  if (a->flags != b->flags || a->getSize() != b->getSize() ||
+      a->data() != b->data())
     return false;
 
   // If two sections have different output sections, we cannot merge them.
@@ -321,10 +323,10 @@
   if (a->getParent() != b->getParent())
     return false;
 
-  if (a->areRelocsRela)
-    return constantEq(a, a->template relas<ELFT>(), b,
-                      b->template relas<ELFT>());
-  return constantEq(a, a->template rels<ELFT>(), b, b->template rels<ELFT>());
+  const RelsOrRelas<ELFT> ra = a->template relsOrRelas<ELFT>();
+  const RelsOrRelas<ELFT> rb = b->template relsOrRelas<ELFT>();
+  return ra.areRelocsRel() ? constantEq(a, ra.rels, b, rb.rels)
+                           : constantEq(a, ra.relas, b, rb.relas);
 }
 
 // Compare two lists of relocations. Returns true if all pairs of
@@ -369,10 +371,10 @@
 // Compare "moving" part of two InputSections, namely relocation targets.
 template <class ELFT>
 bool ICF<ELFT>::equalsVariable(const InputSection *a, const InputSection *b) {
-  if (a->areRelocsRela)
-    return variableEq(a, a->template relas<ELFT>(), b,
-                      b->template relas<ELFT>());
-  return variableEq(a, a->template rels<ELFT>(), b, b->template rels<ELFT>());
+  const RelsOrRelas<ELFT> ra = a->template relsOrRelas<ELFT>();
+  const RelsOrRelas<ELFT> rb = b->template relsOrRelas<ELFT>();
+  return ra.areRelocsRel() ? variableEq(a, ra.rels, b, rb.rels)
+                           : variableEq(a, ra.relas, b, rb.relas);
 }
 
 template <class ELFT> size_t ICF<ELFT>::findBoundary(size_t begin, size_t end) {
@@ -499,10 +501,11 @@
   // a large time complexity will have less work to do.
   for (unsigned cnt = 0; cnt != 2; ++cnt) {
     parallelForEach(sections, [&](InputSection *s) {
-      if (s->areRelocsRela)
-        combineRelocHashes<ELFT>(cnt, s, s->template relas<ELFT>());
+      const RelsOrRelas<ELFT> rels = s->template relsOrRelas<ELFT>();
+      if (rels.areRelocsRel())
+        combineRelocHashes<ELFT>(cnt, s, rels.rels);
       else
-        combineRelocHashes<ELFT>(cnt, s, s->template rels<ELFT>());
+        combineRelocHashes<ELFT>(cnt, s, rels.relas);
     });
   }
 
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -260,7 +260,8 @@
   void initializeJustSymbols();
 
   InputSectionBase *getRelocTarget(const Elf_Shdr &sec);
-  InputSectionBase *createInputSection(const Elf_Shdr &sec, StringRef shstrtab);
+  InputSectionBase *createInputSection(uint32_t idx, const Elf_Shdr &sec,
+                                       StringRef shstrtab);
 
   bool shouldMerge(const Elf_Shdr &sec, StringRef name);
 
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -612,7 +612,7 @@
               .second;
       if (keepGroup) {
         if (config->relocatable)
-          this->sections[i] = createInputSection(sec, shstrtab);
+          this->sections[i] = createInputSection(i, sec, shstrtab);
         selectedGroups.push_back(entries);
         continue;
       }
@@ -636,7 +636,7 @@
     case SHT_NULL:
       break;
     default:
-      this->sections[i] = createInputSection(sec, shstrtab);
+      this->sections[i] = createInputSection(i, sec, shstrtab);
     }
   }
 
@@ -653,7 +653,7 @@
     const Elf_Shdr &sec = objSections[i];
 
     if (sec.sh_type == SHT_REL || sec.sh_type == SHT_RELA)
-      this->sections[i] = createInputSection(sec, shstrtab);
+      this->sections[i] = createInputSection(i, sec, shstrtab);
 
     // A SHF_LINK_ORDER section with sh_link=0 is handled as if it did not have
     // the flag.
@@ -862,7 +862,8 @@
 }
 
 template <class ELFT>
-InputSectionBase *ObjFile<ELFT>::createInputSection(const Elf_Shdr &sec,
+InputSectionBase *ObjFile<ELFT>::createInputSection(uint32_t idx,
+                                                    const Elf_Shdr &sec,
                                                     StringRef shstrtab) {
   StringRef name = CHECK(getObj().getSectionName(sec, shstrtab), this);
 
@@ -952,22 +953,10 @@
       this->sections[sec.sh_info] = target;
     }
 
-    if (target->firstRelocation)
+    if (target->relSecIdx != 0)
       fatal(toString(this) +
             ": multiple relocation sections to one section are not supported");
-
-    if (sec.sh_type == SHT_RELA) {
-      ArrayRef<Elf_Rela> rels = CHECK(getObj().relas(sec), this);
-      target->firstRelocation = rels.begin();
-      target->numRelocations = rels.size();
-      target->areRelocsRela = true;
-    } else {
-      ArrayRef<Elf_Rel> rels = CHECK(getObj().rels(sec), this);
-      target->firstRelocation = rels.begin();
-      target->numRelocations = rels.size();
-      target->areRelocsRela = false;
-    }
-    assert(isUInt<31>(target->numRelocations));
+    target->relSecIdx = idx;
 
     // Relocation sections are usually removed from the output, so return
     // `nullptr` for the normal case. However, if -r or --emit-relocs is
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -33,6 +33,13 @@
 
 extern std::vector<Partition> partitions;
 
+// Returned by InputSectionBase::relsOrRelas. At least one member is empty.
+template <class ELFT> struct RelsOrRelas {
+  ArrayRef<typename ELFT::Rel> rels;
+  ArrayRef<typename ELFT::Rela> relas;
+  bool areRelocsRel() const { return rels.size(); }
+};
+
 // This is the base class of all sections that lld handles. Some are sections in
 // input files, some are sections in the produced output file and some exist
 // just as a convenience for implementing special ways of combining some
@@ -114,10 +121,8 @@
 
   static bool classof(const SectionBase *s) { return s->kind() != Output; }
 
-  // Relocations that refer to this section.
-  unsigned numRelocations : 31;
-  unsigned areRelocsRela : 1;
-  const void *firstRelocation = nullptr;
+  // Section index of the relocation section if exists.
+  uint32_t relSecIdx = 0;
 
   // The file which contains this section. Its dynamic type is always
   // ObjFile<ELFT>, but in order to avoid ELFT, we use InputFile as
@@ -170,19 +175,7 @@
   // used by --gc-sections.
   InputSectionBase *nextInSectionGroup = nullptr;
 
-  template <class ELFT> ArrayRef<typename ELFT::Rel> rels() const {
-    assert(!areRelocsRela);
-    return llvm::makeArrayRef(
-        static_cast<const typename ELFT::Rel *>(firstRelocation),
-        numRelocations);
-  }
-
-  template <class ELFT> ArrayRef<typename ELFT::Rela> relas() const {
-    assert(areRelocsRela);
-    return llvm::makeArrayRef(
-        static_cast<const typename ELFT::Rela *>(firstRelocation),
-        numRelocations);
-  }
+  template <class ELFT> RelsOrRelas<ELFT> relsOrRelas() const;
 
   // InputSections that are dependent on us (reverse dependency for GC)
   llvm::TinyPtrVector<InputSection *> dependentSections;
@@ -392,9 +385,9 @@
 };
 
 #ifdef _WIN32
-static_assert(sizeof(InputSection) <= 192, "InputSection is too big");
-#else
 static_assert(sizeof(InputSection) <= 184, "InputSection is too big");
+#else
+static_assert(sizeof(InputSection) <= 176, "InputSection is too big");
 #endif
 
 inline bool isDebugSection(const InputSectionBase &sec) {
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -70,9 +70,6 @@
   if (sectionKind == SectionBase::Merge && rawData.size() > UINT32_MAX)
     error(toString(this) + ": section too large");
 
-  numRelocations = 0;
-  areRelocsRela = false;
-
   // The ELF spec states that a value of 0 means the section has
   // no alignment constraints.
   uint32_t v = std::max<uint32_t>(alignment, 1);
@@ -162,6 +159,25 @@
   return secStart - fileStart;
 }
 
+template <class ELFT> RelsOrRelas<ELFT> InputSectionBase::relsOrRelas() const {
+  if (relSecIdx == 0)
+    return {};
+  RelsOrRelas<ELFT> ret;
+  const ELFFile<ELFT> obj = cast<ELFFileBase>(file)->getObj<ELFT>();
+  typename ELFT::Shdr shdr = cantFail(obj.sections())[relSecIdx];
+  if (shdr.sh_type == SHT_REL) {
+    ret.rels = makeArrayRef(reinterpret_cast<const typename ELFT::Rel *>(
+                                obj.base() + shdr.sh_offset),
+                            shdr.sh_size / sizeof(typename ELFT::Rel));
+  } else {
+    assert(shdr.sh_type == SHT_RELA);
+    ret.relas = makeArrayRef(reinterpret_cast<const typename ELFT::Rela *>(
+                                 obj.base() + shdr.sh_offset),
+                             shdr.sh_size / sizeof(typename ELFT::Rela));
+  }
+  return ret;
+}
+
 uint64_t SectionBase::getOffset(uint64_t offset) const {
   switch (kind()) {
   case Output: {
@@ -993,12 +1009,15 @@
   }
 
   auto *sec = cast<InputSection>(this);
-  if (config->relocatable)
+  if (config->relocatable) {
     relocateNonAllocForRelocatable(sec, buf);
-  else if (sec->areRelocsRela)
-    sec->relocateNonAlloc<ELFT>(buf, sec->template relas<ELFT>());
-  else
-    sec->relocateNonAlloc<ELFT>(buf, sec->template rels<ELFT>());
+  } else {
+    const RelsOrRelas<ELFT> rels = sec->template relsOrRelas<ELFT>();
+    if (rels.areRelocsRel())
+      sec->relocateNonAlloc<ELFT>(buf, rels.rels);
+    else
+      sec->relocateNonAlloc<ELFT>(buf, rels.relas);
+  }
 }
 
 void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) {
@@ -1312,10 +1331,11 @@
 // .eh_frame is a sequence of CIE or FDE records.
 // This function splits an input section into records and returns them.
 template <class ELFT> void EhInputSection::split() {
-  if (areRelocsRela)
-    split<ELFT>(relas<ELFT>());
+  const RelsOrRelas<ELFT> rels = relsOrRelas<ELFT>();
+  if (rels.areRelocsRel())
+    split<ELFT>(rels.rels);
   else
-    split<ELFT>(rels<ELFT>());
+    split<ELFT>(rels.relas);
 }
 
 template <class ELFT, class RelTy>
@@ -1452,6 +1472,11 @@
 template void InputSection::writeTo<ELF64LE>(uint8_t *);
 template void InputSection::writeTo<ELF64BE>(uint8_t *);
 
+template RelsOrRelas<ELF32LE> InputSectionBase::relsOrRelas<ELF32LE>() const;
+template RelsOrRelas<ELF32BE> InputSectionBase::relsOrRelas<ELF32BE>() const;
+template RelsOrRelas<ELF64LE> InputSectionBase::relsOrRelas<ELF64LE>() const;
+template RelsOrRelas<ELF64BE> InputSectionBase::relsOrRelas<ELF64BE>() const;
+
 template MergeInputSection::MergeInputSection(ObjFile<ELF32LE> &,
                                               const ELF32LE::Shdr &, StringRef);
 template MergeInputSection::MergeInputSection(ObjFile<ELF32BE> &,
diff --git a/lld/ELF/MarkLive.cpp b/lld/ELF/MarkLive.cpp
--- a/lld/ELF/MarkLive.cpp
+++ b/lld/ELF/MarkLive.cpp
@@ -252,13 +252,12 @@
     // referenced by .eh_frame sections, so we scan them for that here.
     if (auto *eh = dyn_cast<EhInputSection>(sec)) {
       eh->markLive();
-      if (!eh->numRelocations)
-        continue;
 
-      if (eh->areRelocsRela)
-        scanEhFrameSection(*eh, eh->template relas<ELFT>());
-      else
-        scanEhFrameSection(*eh, eh->template rels<ELFT>());
+      const RelsOrRelas<ELFT> rels = eh->template relsOrRelas<ELFT>();
+      if (rels.areRelocsRel())
+        scanEhFrameSection(*eh, rels.rels);
+      else if (rels.relas.size())
+        scanEhFrameSection(*eh, rels.relas);
     }
 
     if (sec->flags & SHF_GNU_RETAIN) {
@@ -288,13 +287,11 @@
   while (!queue.empty()) {
     InputSectionBase &sec = *queue.pop_back_val();
 
-    if (sec.areRelocsRela) {
-      for (const typename ELFT::Rela &rel : sec.template relas<ELFT>())
-        resolveReloc(sec, rel, false);
-    } else {
-      for (const typename ELFT::Rel &rel : sec.template rels<ELFT>())
-        resolveReloc(sec, rel, false);
-    }
+    const RelsOrRelas<ELFT> rels = sec.template relsOrRelas<ELFT>();
+    for (const typename ELFT::Rel &rel : rels.rels)
+      resolveReloc(sec, rel, false);
+    for (const typename ELFT::Rela &rel : rels.relas)
+      resolveReloc(sec, rel, false);
 
     for (InputSectionBase *isec : sec.dependentSections)
       enqueue(isec, 0);
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -1611,10 +1611,11 @@
 }
 
 template <class ELFT> void elf::scanRelocations(InputSectionBase &s) {
-  if (s.areRelocsRela)
-    scanRelocs<ELFT>(s, s.relas<ELFT>());
+  const RelsOrRelas<ELFT> rels = s.template relsOrRelas<ELFT>();
+  if (rels.areRelocsRel())
+    scanRelocs<ELFT>(s, rels.rels);
   else
-    scanRelocs<ELFT>(s, s.rels<ELFT>());
+    scanRelocs<ELFT>(s, rels.relas);
 }
 
 static bool mergeCmp(const InputSection *a, const InputSection *b) {
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -431,10 +431,11 @@
 void EhFrameSection::addSectionAux(EhInputSection *sec) {
   if (!sec->isLive())
     return;
-  if (sec->areRelocsRela)
-    addRecords<ELFT>(sec, sec->template relas<ELFT>());
+  const RelsOrRelas<ELFT> rels = sec->template relsOrRelas<ELFT>();
+  if (rels.areRelocsRel())
+    addRecords<ELFT>(sec, rels.rels);
   else
-    addRecords<ELFT>(sec, sec->template rels<ELFT>());
+    addRecords<ELFT>(sec, rels.relas);
 }
 
 void EhFrameSection::addSection(EhInputSection *sec) {
@@ -483,12 +484,11 @@
   DenseSet<size_t> ciesWithLSDA;
   for (EhInputSection *sec : sections) {
     ciesWithLSDA.clear();
-    if (sec->areRelocsRela)
-      iterateFDEWithLSDAAux<ELFT>(*sec, sec->template relas<ELFT>(),
-                                  ciesWithLSDA, fn);
+    const RelsOrRelas<ELFT> rels = sec->template relsOrRelas<ELFT>();
+    if (rels.areRelocsRel())
+      iterateFDEWithLSDAAux<ELFT>(*sec, rels.rels, ciesWithLSDA, fn);
     else
-      iterateFDEWithLSDAAux<ELFT>(*sec, sec->template rels<ELFT>(),
-                                  ciesWithLSDA, fn);
+      iterateFDEWithLSDAAux<ELFT>(*sec, rels.relas, ciesWithLSDA, fn);
   }
 }