[ASTReader] Report error when accessing corrupt record data

[vsk]

Summary

If we try to read a corrupt pch, we can easily assert-fail or trigger invalid
memory accesses when manipulating ASTReader::RecordData objects. This problem
is easy enough to diagnose when assertions are enabled, but can cause trouble
otherwise.

Emit an error diagnostic when we detect a bad RecordData access.

[vsk]

Gentle ping.

For context, we've had a few situations where corrupt pch files trigger surprising assertion failures in our world builds.

[benlangmuir]

This LGTM, but I would like Richard to take a look too.

[rsmith]

Have you done any performance testing on this code? In modules-heavy builds, bitcode reading is *very* hot.

[vsk]

I haven't measured the performance overhead of this patch. The failure case it's supposed to protect against is exceptionally rare: we haven't seen it happen again. In light of that, and considering that it clutters up the code, I think it'd be better to drop this.

include/clang/Serialization/ASTReader.h

class ASTReader
   : public ExternalPreprocessorSource,
     public ExternalPreprocessingRecordSource,
     public ExternalHeaderFileInfoSource,
     public ExternalSemaSource,
     public IdentifierInfoLookup,
     public ExternalSLocEntrySource
 {
 public:
-  typedef SmallVector<uint64_t, 64> RecordData;
+  class RecordData : public SmallVector<uint64_t, 64> {
+  public:
+    RecordData(ASTReader *AR) : AR(AR) {}
+
+    reference operator[](size_type idx) {
+      if (AR && idx >= size()) {
+        AR->Error("Malformed record data, invalid access");
+        return Poison;
+      }
+      return SmallVector<uint64_t, 64>::operator[](idx);
+    }
+
+    const_reference operator[](size_type idx) const {
+      if (AR && idx >= size()) {
+        AR->Error("Malformed record data, invalid access");
+        return Poison;
+      }
+      return SmallVector<uint64_t, 64>::operator[](idx);
+    }
+
+  private:
+    ASTReader *AR;
+    uint64_t Poison;
+  };
+
   typedef SmallVectorImpl<uint64_t> RecordDataImpl;
 
   /// \brief The result of reading the control block of an AST file, which
   /// can fail for various reasons.
   enum ASTReadResult {
     /// \brief The control block was read successfully. Aside from failures,
     /// the AST file is safe to read into the current context.
     Success,

lib/Serialization/ASTReader.cpp

bool ASTReader::ReadLexicalDeclContextStorage(ModuleFile &M,
                                               BitstreamCursor &Cursor,
                                               uint64_t Offset,
                                               DeclContext *DC) {
   assert(Offset != 0);
 
   SavedStreamPosition SavedPosition(Cursor);
   Cursor.JumpToBit(Offset);
 
-  RecordData Record;
+  RecordData Record(this);
   StringRef Blob;
   unsigned Code = Cursor.ReadCode();
   unsigned RecCode = Cursor.readRecord(Code, Record, &Blob);
   if (RecCode != DECL_CONTEXT_LEXICAL) {
     Error("Expected lexical block");
     return true;
   }
 
bool ASTReader::ReadVisibleDeclContextStorage(ModuleFile &M,
                                               BitstreamCursor &Cursor,
                                               uint64_t Offset,
                                               DeclID ID) {
   assert(Offset != 0);
 
   SavedStreamPosition SavedPosition(Cursor);
   Cursor.JumpToBit(Offset);
 
-  RecordData Record;
+  RecordData Record(this);
   StringRef Blob;
   unsigned Code = Cursor.ReadCode();
   unsigned RecCode = Cursor.readRecord(Code, Record, &Blob);
   if (RecCode != DECL_CONTEXT_VISIBLE) {
     Error("Expected visible lookup table block");
     return true;
   }
 
bool ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
   }
 
   // Enter the source manager block.
   if (SLocEntryCursor.EnterSubBlock(SOURCE_MANAGER_BLOCK_ID)) {
     Error("malformed source manager block record in AST file");
     return true;
   }
 
-  RecordData Record;
+  RecordData Record(this);
   while (true) {
     llvm::BitstreamEntry E = SLocEntryCursor.advanceSkippingSubblocks();
     
     switch (E.Kind) {
     case llvm::BitstreamEntry::SubBlock: // Handled for us already.
     case llvm::BitstreamEntry::Error:
       Error("malformed block record in AST file");
       return true;
if (unsigned(-ID) - 2 >= getTotalNumSLocs() || ID > 0) {
     return true;
   }
 
   // Local helper to read the (possibly-compressed) buffer data following the
   // entry record.
   auto ReadBuffer = [this](
       BitstreamCursor &SLocEntryCursor,
       StringRef Name) -> std::unique_ptr<llvm::MemoryBuffer> {
-    RecordData Record;
+    RecordData Record(this);
     StringRef Blob;
     unsigned Code = SLocEntryCursor.ReadCode();
     unsigned RecCode = SLocEntryCursor.readRecord(Code, Record, &Blob);
 
     if (RecCode == SM_SLOC_BUFFER_BLOB_COMPRESSED) {
       SmallString<0> Uncompressed;
       if (llvm::zlib::uncompress(Blob, Uncompressed, Record[0]) !=
           llvm::zlib::StatusOK) {
bool ASTReader::ReadSLocEntry(int ID) {
 
   ++NumSLocEntriesRead;
   llvm::BitstreamEntry Entry = SLocEntryCursor.advance();
   if (Entry.Kind != llvm::BitstreamEntry::Record) {
     Error("incorrectly-formatted source location entry in AST file");
     return true;
   }
   
-  RecordData Record;
+  RecordData Record(this);
   StringRef Blob;
   switch (SLocEntryCursor.readRecord(Entry.ID, Record, &Blob)) {
   default:
     Error("incorrectly-formatted source location entry in AST file");
     return true;
 
   case SM_SLOC_FILE_ENTRY: {
     // We will detect whether a file changed and return 'Failure' for it, but

 MacroInfo *ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
   BitstreamCursor &Stream = F.MacroCursor;
 
   // Keep track of where we are in the stream, then jump back there
   // after reading this macro.
   SavedStreamPosition SavedPosition(Stream);
 
   Stream.JumpToBit(Offset);
-  RecordData Record;
+  RecordData Record(this);
   SmallVector<IdentifierInfo*, 16> MacroArgs;
   MacroInfo *Macro = nullptr;
 
   while (true) {
     // Advance to the next record, but if we get to the end of the block, don't
     // pop it (removing all the abbreviations from the cursor) since we want to
     // be able to reseek within the block and read entries.
     unsigned Flags = BitstreamCursor::AF_DontPopBlockAtEnd;
for (auto &I : llvm::reverse(ModuleMgr)) {
 
     // If there was no preprocessor block, skip this file.
     if (!MacroCursor.getBitStreamReader())
       continue;
 
     BitstreamCursor Cursor = MacroCursor;
     Cursor.JumpToBit(I->MacroStartOffset);
 
-    RecordData Record;
+    RecordData Record(this);
     while (true) {
       llvm::BitstreamEntry E = Cursor.advanceSkippingSubblocks();
       
       switch (E.Kind) {
       case llvm::BitstreamEntry::SubBlock: // Handled for us already.
       case llvm::BitstreamEntry::Error:
         Error("malformed block record in AST file");
         return;
struct ModuleMacroRecord {
     MacroInfo *MI;
     SmallVector<SubmoduleID, 8> Overrides;
   };
   llvm::SmallVector<ModuleMacroRecord, 8> ModuleMacros;
 
   // We expect to see a sequence of PP_MODULE_MACRO records listing exported
   // macros, followed by a PP_MACRO_DIRECTIVE_HISTORY record with the complete
   // macro histroy.
-  RecordData Record;
+  RecordData Record(this);
   while (true) {
     llvm::BitstreamEntry Entry =
         Cursor.advance(BitstreamCursor::AF_DontPopBlockAtEnd);
     if (Entry.Kind != llvm::BitstreamEntry::Record) {
       Error("malformed block record in AST file");
       return;
     }
 

 ASTReader::InputFileInfo
 ASTReader::readInputFileInfo(ModuleFile &F, unsigned ID) {
   // Go find this input file.
   BitstreamCursor &Cursor = F.InputFilesCursor;
   SavedStreamPosition SavedPosition(Cursor);
   Cursor.JumpToBit(F.InputFileOffsets[ID-1]);
 
   unsigned Code = Cursor.ReadCode();
-  RecordData Record;
+  RecordData Record(this);
   StringRef Blob;
 
   unsigned Result = Cursor.readRecord(Code, Record, &Blob);
   assert(static_cast<InputFileRecordTypes>(Result) == INPUT_FILE &&
          "invalid record type for input file");
   (void)Result;
 
   assert(Record[0] == ID && "Bogus stored ID or offset");

 ASTReader::ASTReadResult ASTReader::ReadOptionsBlock(
     BitstreamCursor &Stream, unsigned ClientLoadCapabilities,
     bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener,
     std::string &SuggestedPredefines) {
   if (Stream.EnterSubBlock(OPTIONS_BLOCK_ID))
     return Failure;
 
   // Read all of the records in the options block.
-  RecordData Record;
+  RecordData Record(nullptr);
   ASTReadResult Result = Success;
   while (1) {
     llvm::BitstreamEntry Entry = Stream.advance();
     
     switch (Entry.Kind) {
     case llvm::BitstreamEntry::Error:
     case llvm::BitstreamEntry::SubBlock:
       return Failure;
ASTReader::ReadControlBlock(ModuleFile &F,
   ASTReadResult Result = Success;
 
   if (Stream.EnterSubBlock(CONTROL_BLOCK_ID)) {
     Error("malformed block record in AST file");
     return Failure;
   }
 
   // Read all of the records and blocks in the control block.
-  RecordData Record;
+  RecordData Record(this);
   unsigned NumInputs = 0;
   unsigned NumUserInputs = 0;
   while (1) {
     llvm::BitstreamEntry Entry = Stream.advance();
     
     switch (Entry.Kind) {
     case llvm::BitstreamEntry::Error:
       Error("malformed block record in AST file");
ASTReader::ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
   BitstreamCursor &Stream = F.Stream;
 
   if (Stream.EnterSubBlock(AST_BLOCK_ID)) {
     Error("malformed block record in AST file");
     return Failure;
   }
 
   // Read all of the records and blocks for the AST file.
-  RecordData Record;
+  RecordData Record(this);
   while (1) {
     llvm::BitstreamEntry Entry = Stream.advance();
     
     switch (Entry.Kind) {
     case llvm::BitstreamEntry::Error:
       Error("error at end of module block in AST file");
       return Failure;
     case llvm::BitstreamEntry::EndBlock: {
static bool parseModuleFileExtensionMetadata(
   Metadata.UserInfo = std::string(Blob.data() + BlockNameLen,
                                   Blob.data() + BlockNameLen + UserInfoLen);
   return false;
 }
 
 ASTReader::ASTReadResult ASTReader::ReadExtensionBlock(ModuleFile &F) {
   BitstreamCursor &Stream = F.Stream;
 
-  RecordData Record;
+  RecordData Record(this);
   while (true) {
     llvm::BitstreamEntry Entry = Stream.advance();
     switch (Entry.Kind) {
     case llvm::BitstreamEntry::SubBlock:
       if (Stream.SkipBlock())
         return Failure;
 
       continue;
static ASTFileSignature readASTFileSignature(llvm::BitstreamReader &StreamFile){
   if (!startsWithASTFileMagic(Stream))
     return 0;
 
   // Scan for the CONTROL_BLOCK_ID block.
   if (SkipCursorToBlock(Stream, CONTROL_BLOCK_ID))
     return 0;
 
   // Scan for SIGNATURE inside the control block.
-  ASTReader::RecordData Record;
+  ASTReader::RecordData Record(nullptr);
   while (1) {
     llvm::BitstreamEntry Entry = Stream.advanceSkippingSubblocks();
     if (Entry.Kind == llvm::BitstreamEntry::EndBlock ||
         Entry.Kind != llvm::BitstreamEntry::Record)
       return 0;
 
     Record.clear();
     StringRef Blob;
std::string ASTReader::getOriginalSourceFile(
   
   // Scan for the CONTROL_BLOCK_ID block.
   if (SkipCursorToBlock(Stream, CONTROL_BLOCK_ID)) {
     Diags.Report(diag::err_fe_pch_malformed_block) << ASTFileName;
     return std::string();
   }
 
   // Scan for ORIGINAL_FILE inside the control block.
-  RecordData Record;
+  RecordData Record(nullptr);
   while (1) {
     llvm::BitstreamEntry Entry = Stream.advanceSkippingSubblocks();
     if (Entry.Kind == llvm::BitstreamEntry::EndBlock)
       return std::string();
     
     if (Entry.Kind != llvm::BitstreamEntry::Record) {
       Diags.Report(diag::err_fe_pch_malformed_block) << ASTFileName;
       return std::string();
bool ASTReader::readASTFileControlBlock(
   if (SkipCursorToBlock(Stream, CONTROL_BLOCK_ID))
     return true;
 
   bool NeedsInputFiles = Listener.needsInputFileVisitation();
   bool NeedsSystemInputFiles = Listener.needsSystemInputFileVisitation();
   bool NeedsImports = Listener.needsImportVisitation();
   BitstreamCursor InputFilesCursor;
 
-  RecordData Record;
+  RecordData Record(nullptr);
   std::string ModuleDir;
   bool DoneWithControlBlock = false;
   while (!DoneWithControlBlock) {
     llvm::BitstreamEntry Entry = Stream.advance();
 
     switch (Entry.Kind) {
     case llvm::BitstreamEntry::SubBlock: {
       switch (Entry.ID) {
case INPUT_FILE_OFFSETS: {
         if (isSystemFile && !NeedsSystemInputFiles)
           break; // the rest are system input files
 
         BitstreamCursor &Cursor = InputFilesCursor;
         SavedStreamPosition SavedPosition(Cursor);
         Cursor.JumpToBit(InputFileOffs[I]);
 
         unsigned Code = Cursor.ReadCode();
-        RecordData Record;
+        RecordData Record(nullptr);
         StringRef Blob;
         bool shouldContinue = false;
         switch ((InputFileRecordTypes)Cursor.readRecord(Code, Record, &Blob)) {
         case INPUT_FILE:
           bool Overridden = static_cast<bool>(Record[3]);
           std::string Filename = Blob;
           ResolveImportedPath(Filename, ModuleDir);
           shouldContinue = Listener.visitInputFile(
ASTReader::ReadSubmoduleBlock(ModuleFile &F, unsigned ClientLoadCapabilities) {
   if (F.Stream.EnterSubBlock(SUBMODULE_BLOCK_ID)) {
     Error("malformed submodule block record in AST file");
     return Failure;
   }
 
   ModuleMap &ModMap = PP.getHeaderSearchInfo().getModuleMap();
   bool First = true;
   Module *CurrentModule = nullptr;
-  RecordData Record;
+  RecordData Record(this);
   while (true) {
     llvm::BitstreamEntry Entry = F.Stream.advanceSkippingSubblocks();
     
     switch (Entry.Kind) {
     case llvm::BitstreamEntry::SubBlock: // Handled for us already.
     case llvm::BitstreamEntry::Error:
       Error("malformed block record in AST file");
       return Failure;
PreprocessedEntity *ASTReader::ReadPreprocessedEntity(unsigned Index) {
   if (Entry.Kind != llvm::BitstreamEntry::Record)
     return nullptr;
 
   // Read the record.
   SourceRange Range(ReadSourceLocation(M, PPOffs.Begin),
                     ReadSourceLocation(M, PPOffs.End));
   PreprocessingRecord &PPRec = *PP.getPreprocessingRecord();
   StringRef Blob;
-  RecordData Record;
+  RecordData Record(this);
   PreprocessorDetailRecordTypes RecType =
     (PreprocessorDetailRecordTypes)M.PreprocessorDetailCursor.readRecord(
                                           Entry.ID, Record, &Blob);
   switch (RecType) {
   case PPD_MACRO_EXPANSION: {
     bool isBuiltin = Record[0];
     IdentifierInfo *Name = nullptr;
     MacroDefinitionRecord *Def = nullptr;
QualType ASTReader::readTypeRecord(unsigned Index) {
 
   ReadingKindTracker ReadingKind(Read_Type, *this);
 
   // Note that we are loading a type record.
   Deserializing AType(this);
 
   unsigned Idx = 0;
   DeclsCursor.JumpToBit(Loc.Offset);
-  RecordData Record;
+  RecordData Record(this);
   unsigned Code = DeclsCursor.ReadCode();
   switch ((TypeCode)DeclsCursor.readRecord(Code, Record)) {
   case TYPE_EXT_QUAL: {
     if (Record.size() != 2) {
       Error("Incorrect encoding of extended qualifier type");
       return QualType();
     }
     QualType Base = readType(*Loc.F, Record, Idx);

 CXXCtorInitializer **
 ASTReader::GetExternalCXXCtorInitializers(uint64_t Offset) {
   RecordLocation Loc = getLocalBitOffset(Offset);
   BitstreamCursor &Cursor = Loc.F->DeclsCursor;
   SavedStreamPosition SavedPosition(Cursor);
   Cursor.JumpToBit(Loc.Offset);
   ReadingKindTracker ReadingKind(Read_Decl, *this);
 
-  RecordData Record;
+  RecordData Record(this);
   unsigned Code = Cursor.ReadCode();
   unsigned RecCode = Cursor.readRecord(Code, Record);
   if (RecCode != DECL_CXX_CTOR_INITIALIZERS) {
     Error("malformed AST file: missing C++ ctor initializers");
     return nullptr;
   }
 
   unsigned Idx = 0;

 }
 
 CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) {
   RecordLocation Loc = getLocalBitOffset(Offset);
   BitstreamCursor &Cursor = Loc.F->DeclsCursor;
   SavedStreamPosition SavedPosition(Cursor);
   Cursor.JumpToBit(Loc.Offset);
   ReadingKindTracker ReadingKind(Read_Decl, *this);
-  RecordData Record;
+  RecordData Record(this);
   unsigned Code = Cursor.ReadCode();
   unsigned RecCode = Cursor.readRecord(Code, Record);
   if (RecCode != DECL_CXX_BASE_SPECIFIERS) {
     Error("malformed AST file: missing C++ base specifiers");
     return nullptr;
   }
 
   unsigned Idx = 0;
for (SmallVectorImpl<std::pair<BitstreamCursor,
        I = CommentsCursors.begin(),
        E = CommentsCursors.end();
        I != E; ++I) {
     Comments.clear();
     BitstreamCursor &Cursor = I->first;
     serialization::ModuleFile &F = *I->second;
     SavedStreamPosition SavedPosition(Cursor);
 
-    RecordData Record;
+    RecordData Record(this);
     while (true) {
       llvm::BitstreamEntry Entry =
         Cursor.advanceSkippingSubblocks(BitstreamCursor::AF_DontPopBlockAtEnd);
 
       switch (Entry.Kind) {
       case llvm::BitstreamEntry::SubBlock: // Handled for us already.
       case llvm::BitstreamEntry::Error:
         Error("malformed block record in AST file");

lib/Serialization/ASTReaderDecl.cpp

   SavedStreamPosition SavedPosition(DeclsCursor);
 
   ReadingKindTracker ReadingKind(Read_Decl, *this);
 
   // Note that we are loading a declaration record.
   Deserializing ADecl(this);
 
   DeclsCursor.JumpToBit(Loc.Offset);
-  RecordData Record;
+  RecordData Record(this);
   unsigned Code = DeclsCursor.ReadCode();
   unsigned Idx = 0;
   ASTDeclReader Reader(*this, *Loc.F, ID, RawLocation, Record,Idx);
 
   Decl *D = nullptr;
   switch ((DeclCode)DeclsCursor.readRecord(Code, Record)) {
   case DECL_CONTEXT_LEXICAL:
   case DECL_CONTEXT_VISIBLE:

 
     bool WasInteresting = isConsumerInterestedIn(D, false);
     for (auto &FileAndOffset : UpdateOffsets) {
       ModuleFile *F = FileAndOffset.first;
       uint64_t Offset = FileAndOffset.second;
       llvm::BitstreamCursor &Cursor = F->DeclsCursor;
       SavedStreamPosition SavedPosition(Cursor);
       Cursor.JumpToBit(Offset);
-      RecordData Record;
+      RecordData Record(this);
       unsigned Code = Cursor.ReadCode();
       unsigned RecCode = Cursor.readRecord(Code, Record);
       (void)RecCode;
       assert(RecCode == DECL_UPDATES && "Expected DECL_UPDATES record!");
 
       unsigned Idx = 0;
       ASTDeclReader Reader(*this, *F, ID, 0, Record, Idx);
       Reader.UpdateDecl(D, *F, Record);

   // Load the list of other redeclarations from this module file.
   ModuleFile *M = getOwningModuleFile(FirstLocal);
   assert(M && "imported decl from no module file");
 
   llvm::BitstreamCursor &Cursor = M->DeclsCursor;
   SavedStreamPosition SavedPosition(Cursor);
   Cursor.JumpToBit(LocalOffset);
 
-  RecordData Record;
+  RecordData Record(this);
   unsigned Code = Cursor.ReadCode();
   unsigned RecCode = Cursor.readRecord(Code, Record);
   (void)RecCode;
   assert(RecCode == LOCAL_REDECLARATIONS && "expected LOCAL_REDECLARATIONS record!");
 
   // FIXME: We have several different dispatches on decl kind here; maybe
   // we should instead generate one loop per kind and dispatch up-front?
   Decl *MostRecent = FirstLocal;

lib/Serialization/ASTReaderStmt.cpp

   // Map of offset to previously deserialized stmt. The offset points
   /// just after the stmt record.
   llvm::DenseMap<uint64_t, Stmt *> StmtEntries;
 
 #ifndef NDEBUG
   unsigned PrevNumStmts = StmtStack.size();
 #endif
 
-  RecordData Record;
+  RecordData Record(this);
   unsigned Idx;
   ASTStmtReader Reader(*this, F, Cursor, Record, Idx);
   Stmt::EmptyShell Empty;
 
   while (true) {
     llvm::BitstreamEntry Entry = Cursor.advanceSkippingSubblocks();
     
     switch (Entry.Kind) {