[clang] Serialize source locations as 64-bit in PCH.

[simon_tatham]

Summary

This is part of a patch series working towards the ability to make
SourceLocation into a 64-bit type to handle larger translation units.

Unconditionally expanding the serialized encoding to 64 bits is the
simplest way to ensure that it will be big enough to handle an
expanded SourceLocation.

As I understand it, PCH files are already automatically versioned with
the exact revision of the compiler that built them, so this shouldn't
introduce any compatibility problem, because clang will already refuse
to load any PCH from a different version of itself.

Patch based on previous work by Mikhail Maltsev.

[lebedev.ri]

This review seems to be stuck/dead, consider abandoning if no longer relevant.

clang/include/clang/Serialization/ASTReader.h

private:
   /// the PCH chain must be eagerly deserialized.
   SmallVector<serialization::DeclID, 16> TentativeDefinitions;
 
   /// The IDs of all CXXRecordDecls stored in the chain whose VTables are
   /// used.
   ///
   /// CodeGen has to emit VTables for these records, so they have to be eagerly
   /// deserialized.
-  SmallVector<serialization::DeclID, 64> VTableUses;
+  SmallVector<uint64_t, 64> VTableUses;
 
   /// A snapshot of the pending instantiations in the chain.
   ///
   /// This record tracks the instantiations that Sema has to perform at the
   /// end of the TU. It consists of a pair of values for every pending
   /// instantiation where the first value is the ID of the decl and the second
   /// is the instantiation location.
-  SmallVector<serialization::DeclID, 64> PendingInstantiations;
+  SmallVector<uint64_t, 64> PendingInstantiations;
 
   //@}
 
   /// \name DiagnosticsEngine-relevant special data
   /// Fields containing data that is used for generating diagnostics
   //@{
 
   /// A snapshot of Sema's unused file-scoped variable tracking, for
   /// generating warnings.
   SmallVector<serialization::DeclID, 16> UnusedFileScopedDecls;
 
   /// A list of all the delegating constructors we've seen, to diagnose
   /// cycles.
   SmallVector<serialization::DeclID, 4> DelegatingCtorDecls;
 
   /// Method selectors used in a @selector expression. Used for
   /// implementation of -Wselector.
-  SmallVector<serialization::SelectorID, 64> ReferencedSelectorsData;
+  SmallVector<uint64_t, 64> ReferencedSelectorsData;
 
   /// A snapshot of Sema's weak undeclared identifier tracking, for
   /// generating warnings.
-  SmallVector<serialization::IdentifierID, 64> WeakUndeclaredIdentifiers;
+  SmallVector<uint64_t, 64> WeakUndeclaredIdentifiers;
 
   /// The IDs of type aliases for ext_vectors that exist in the chain.
   ///
   /// Used by Sema for finding sugared names for ext_vectors in diagnostics.
   SmallVector<serialization::DeclID, 4> ExtVectorDecls;
 
   //@}
 
private:
   /// Extensions required by an OpenCL declaration.
   llvm::DenseMap<const Decl *, std::set<std::string>> OpenCLDeclExtMap;
 
   /// A list of the namespaces we've seen.
   SmallVector<serialization::DeclID, 4> KnownNamespaces;
 
   /// A list of undefined decls with internal linkage followed by the
   /// SourceLocation of a matching ODR-use.
-  SmallVector<serialization::DeclID, 8> UndefinedButUsed;
+  SmallVector<uint64_t, 8> UndefinedButUsed;
 
   /// Delete expressions to analyze at the end of translation unit.
   SmallVector<uint64_t, 8> DelayedDeleteExprs;
 
   // A list of late parsed template function data with their module files.
   SmallVector<std::pair<ModuleFile *, SmallVector<uint64_t, 1>>, 4>
       LateParsedTemplates;
 

clang/lib/Serialization/ASTReader.cpp

while (Data < DataEnd) {
     if (!OM) {
       std::string Msg =
           "SourceLocation remap refers to unknown module, cannot find ";
       Msg.append(std::string(Name));
       Error(Msg);
       return;
     }
 
-    SourceLocation::UIntType SLocOffset =
-        endian::readNext<uint32_t, little, unaligned>(Data);
+    uint64_t SLocOffset = endian::readNext<uint64_t, little, unaligned>(Data);
     uint32_t IdentifierIDOffset =
         endian::readNext<uint32_t, little, unaligned>(Data);
     uint32_t MacroIDOffset =
         endian::readNext<uint32_t, little, unaligned>(Data);
     uint32_t PreprocessedEntityIDOffset =
         endian::readNext<uint32_t, little, unaligned>(Data);
     uint32_t SubmoduleIDOffset =
         endian::readNext<uint32_t, little, unaligned>(Data);
     uint32_t SelectorIDOffset =
         endian::readNext<uint32_t, little, unaligned>(Data);
     uint32_t DeclIDOffset =
         endian::readNext<uint32_t, little, unaligned>(Data);
     uint32_t TypeIndexOffset =
         endian::readNext<uint32_t, little, unaligned>(Data);
 
+    // SourceLocations are serialized as uint64_t, irrespective of
+    // CLANG_64_BIT_SOURCE_LOCATIONS. So we have to bounds-check them
+    // when reading back in.
+    if (SLocOffset > std::numeric_limits<SourceLocation::UIntType>::max()) {
+      Error("This version of clang cannot handle SourceLocations bigger than "
+            "32 bits");
+      return;
+    }
+
     auto mapOffset = [&](uint32_t Offset, uint32_t BaseOffset,
                          RemapBuilder &Remap) {
       constexpr uint32_t None = std::numeric_limits<uint32_t>::max();
       if (Offset != None)
         Remap.insert(std::make_pair(Offset,
                                     static_cast<int>(BaseOffset - Offset)));
     };
 

clang/lib/Serialization/ASTWriter.cpp

SmallString<2048> Buffer;
         using namespace llvm::support;
 
         endian::Writer LE(Out, little);
         LE.write<uint8_t>(static_cast<uint8_t>(M.Kind));
         StringRef Name = M.isModule() ? M.ModuleName : M.FileName;
         LE.write<uint16_t>(Name.size());
         Out.write(Name.data(), Name.size());
 
-        // Note: if a base ID was uint max, it would not be possible to load
-        // another module after it or have more than one entity inside it.
-        uint32_t None = std::numeric_limits<uint32_t>::max();
-
         auto writeBaseIDOrNone = [&](auto BaseID, bool ShouldWrite) {
-          assert(BaseID < std::numeric_limits<uint32_t>::max() && "base id too high");
+          using T = decltype(BaseID);
+          static_assert(std::is_same<T, uint32_t>::value ||
+                            std::is_same<T, uint64_t>::value,
+                        "Invalid BaseID type");
+          assert(BaseID < std::numeric_limits<T>::max() && "base id too high");
           if (ShouldWrite)
-            LE.write<uint32_t>(BaseID);
-          else
-            LE.write<uint32_t>(None);
+            LE.write<T>(BaseID);
+          else {
+            // Note: if a base ID was uint max, it would not be possible to load
+            // another module after it or have more than one entity inside it.
+            constexpr T None = std::numeric_limits<T>::max();
+            LE.write<T>(None);
+          }
         };
 
         // These values should be unique within a chain, since they will be read
         // as keys into ContinuousRangeMaps.
-        writeBaseIDOrNone(M.SLocEntryBaseOffset, M.LocalNumSLocEntries);
+
+        // SourceLocations are serialized as uint64_t, irrespective of
+        // CLANG_64_BIT_SOURCE_LOCATIONS
+        writeBaseIDOrNone(static_cast<uint64_t>(M.SLocEntryBaseOffset),
+                          M.LocalNumSLocEntries);
         writeBaseIDOrNone(M.BaseIdentifierID, M.LocalNumIdentifiers);
         writeBaseIDOrNone(M.BaseMacroID, M.LocalNumMacros);
         writeBaseIDOrNone(M.BasePreprocessedEntityID,
                           M.NumPreprocessedEntities);
         writeBaseIDOrNone(M.BaseSubmoduleID, M.LocalNumSubmodules);
         writeBaseIDOrNone(M.BaseSelectorID, M.LocalNumSelectors);
         writeBaseIDOrNone(M.BaseDeclID, M.LocalNumDecls);
         writeBaseIDOrNone(M.BaseTypeIndex, M.LocalNumTypes);