Store token literal data in PCH. Avoids disk read on PreProc expansion.

[v.g.vassilev]

Summary

Patch by Axel Naumann!

[v.g.vassilev]

What would be the best way to test this. Shall we put:

echo '#define LITERAL 42' > HEADER.h

clang++ -std=c++14 -x c++-header -o HEADER.h.pch HEADER.h

chmod a-r HEADER.h

echo 'void f(int); void g() { f(12); }' | clang++ -std=c++14 -Xclang
-fno-validate-pch -include-pch HEADER.h.pch -c -Xclang -emit-llvm -o -
-x c++ -
# works, we see the IR

echo 'void f(int); void g() { f(LITERAL); }' | clang++ -std=c++14
-Xclang -fno-validate-pch -include-pch HEADER.h.pch -c -Xclang
-emit-llvm -o - -x c++ -
fatal error: cannot open file
'/home/axel/build/tmp/clang-repro-PCH-touches-disk-for-literal/HEADER.h':
Permission denied
1 error generated.

[v.g.vassilev]

Fix StringRef allocation.

[rsmith]

Seems reasonable to store this with the token. Do you have performance data?

include/clang/Serialization/ASTReader.h

616

Remove this use pointers directly into blob data from the token record (or BumpPtrAllocate the strings using the preprocessor's allocator if there's some reason you can't use blob data).

lib/Serialization/ASTWriter.cpp

4360

Remove this FIXME

[v.g.vassilev]

Add a test case.

include/clang/Serialization/ASTReader.h

private:
   typedef std::pair<IdentifierInfo *, serialization::SubmoduleID>
       LoadedMacroInfo;
 
   /// \brief A set of #undef directives that we have loaded; used to
   /// deduplicate the same #undef information coming from multiple module
   /// files.
   llvm::DenseSet<LoadedMacroInfo> LoadedUndefs;
 
+  /// \brief Token literal data loaded and owned by us.
+  std::vector<std::string *> TokenLiteralDataLoaded;
+
   typedef ContinuousRangeMap<serialization::MacroID, ModuleFile *, 4>
     GlobalMacroMapType;
 
   /// \brief Mapping from global macro IDs to the module in which the
   /// macro resides along with the offset that should be added to the
   /// global macro ID to produce a local ID.
   GlobalMacroMapType GlobalMacroMap;
 

lib/Serialization/ASTReader.cpp

Token ASTReader::ReadToken(ModuleFile &F, const RecordDataImpl &Record,
   Token Tok;
   Tok.startToken();
   Tok.setLocation(ReadSourceLocation(F, Record, Idx));
   Tok.setLength(Record[Idx++]);
   if (IdentifierInfo *II = getLocalIdentifier(F, Record[Idx++]))
     Tok.setIdentifierInfo(II);
   Tok.setKind((tok::TokenKind)Record[Idx++]);
   Tok.setFlag((Token::TokenFlags)Record[Idx++]);
+  if (Tok.isLiteral()) {
+    const RecordData &RD = reinterpret_cast<const RecordData &>(Record);
+    std::string *Lit = new std::string(ReadString(RD, Idx));
+    TokenLiteralDataLoaded.push_back(Lit);
+    Tok.setLiteralData(Lit->c_str());
+  }
   return Tok;
 }
 
 MacroInfo *ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
   BitstreamCursor &Stream = F.MacroCursor;
 
   // Keep track of where we are in the stream, then jump back there
   // after reading this macro.
for (const auto &Ext : Extensions) {
 
     ModuleFileExtensions.insert({BlockName, Ext});
   }
 }
 
 ASTReader::~ASTReader() {
   if (OwnsDeserializationListener)
     delete DeserializationListener;
+  for (auto PStr : TokenLiteralDataLoaded) {
+    delete PStr;
+  }
 }
 
 IdentifierResolver &ASTReader::getIdResolver() {
   return SemaObj ? SemaObj->IdResolver : DummyIdResolver;
 }
 
 unsigned ASTRecordReader::readRecord(llvm::BitstreamCursor &Cursor,
                                      unsigned AbbrevID) {
   Idx = 0;
   Record.clear();
   return Cursor.readRecord(AbbrevID, Record);
 }

lib/Serialization/ASTWriter.cpp

#include "clang/Serialization/AttrPCHWrite.inc"
 
   }
 }
 
 void ASTWriter::AddToken(const Token &Tok, RecordDataImpl &Record) {
   AddSourceLocation(Tok.getLocation(), Record);
   Record.push_back(Tok.getLength());
 
   // FIXME: When reading literal tokens, reconstruct the literal pointer
   // if it is needed.
   AddIdentifierRef(Tok.getIdentifierInfo(), Record);
   // FIXME: Should translate token kind to a stable encoding.
   Record.push_back(Tok.getKind());
   // FIXME: Should translate token flags to a stable encoding.
   Record.push_back(Tok.getFlags());
+  if (Tok.isLiteral())
+    AddString(StringRef(Tok.getLiteralData(), Tok.getLength()), Record);
 }
 
 void ASTWriter::AddString(StringRef Str, RecordDataImpl &Record) {
   Record.push_back(Str.size());
   Record.insert(Record.end(), Str.begin(), Str.end());
 }
 
 bool ASTWriter::PreparePathForOutput(SmallVectorImpl<char> &Path) {