[preprocessor] Assertions on the inferrable null pointers in Preprocessor befere dereference

[OikawaKirie]

Summary

This patch asserts on some smart pointers that can be statically inferred to be null pointers. For a smart pointer, when its nullability is checked, and the null branch can reach a dereference, we think it is a potential null pointer dereference. In this patch, I added an assertion before each reported dereference.

• clang/lib/Lex/PPDirectives.cpp:441, inferred from the loop body: e.g. line 481, check and continue
• clang/lib/Lex/PPDirectives.cpp:1385, inferred from line 911: check CurLexer
• clang/lib/Lex/PPDirectives.cpp:1723, inferred from line 911: check CurLexer
• clang/lib/Lex/PPDirectives.cpp:2540, inferred from line 911: check CurLexer
• clang/lib/Lex/PPLexerChange.cpp:320, inferred from line 311: check CurLexer
• clang/lib/Lex/Preprocessor.cpp:1226, inferred from recomputeCurLexerKind line 383: check CurLexer

Besides, the analyzer also reports the dereference in function SkipTokensWhileUsingPCH and Lex. I think these two reported dereference cannot actually happen, but I also add the asserts to make sure the value is correct.

clang/lib/Lex/PPDirectives.cpp

void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
   Token Tok;
   if (auto SkipLength =
           getSkippedRangeForExcludedConditionalBlock(HashTokenLoc)) {
     // Skip to the next '#endif' / '#else' / '#elif'.
     CurLexer->skipOver(*SkipLength);
   }
   SourceLocation endLoc;
   while (true) {
+    assert(CurLexer);
     CurLexer->Lex(Tok);
 
     if (Tok.is(tok::code_completion)) {
       if (CodeComplete)
         CodeComplete->CodeCompleteInConditionalExclusion();
       setCodeCompletionReached();
       continue;
     }
if (Callbacks) {
     Callbacks->FileChanged(CurPPLexer->getSourceLocation(), Reason, FileKind);
   }
 }
 
 /// HandleUserDiagnosticDirective - Handle a #warning or #error directive.
 ///
 void Preprocessor::HandleUserDiagnosticDirective(Token &Tok,
                                                  bool isWarning) {
+  assert(CurLexer);
+
   // Read the rest of the line raw.  We do this because we don't want macros
   // to be expanded and we don't require that the tokens be valid preprocessing
   // tokens.  For example, this is allowed: "#warning `   'foo".  GCC does
   // collapse multiple consecutive white space between tokens, but this isn't
   // specified by the standard.
   SmallString<128> Message;
   CurLexer->ReadToEndOfLine(&Message);
 
case ImportAction::ModuleImport:
     EnterAnnotationToken(SourceRange(HashLoc, EndLoc),
                          tok::annot_module_include, Action.ModuleForHeader);
     break;
   case ImportAction::Failure:
     assert(TheModuleLoader.HadFatalFailure &&
            "This should be an early exit only to a fatal error");
     TheModuleLoader.HadFatalFailure = true;
     IncludeTok.setKind(tok::eof);
+    assert(CurLexer);
     CurLexer->cutOffLexing();
     return;
   }
 }
 
 Optional<FileEntryRef> Preprocessor::LookupHeaderIncludeOrImport(
     const DirectoryLookup *&CurDir, StringRef& Filename,
     SourceLocation FilenameLoc, CharSourceRange FilenameRange,
MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody(
   MacroInfo *const MI = AllocateMacroInfo(MacroNameTok.getLocation());
 
   Token Tok;
   LexUnexpandedToken(Tok);
 
   // Ensure we consume the rest of the macro body if errors occur.
   auto _ = llvm::make_scope_exit([&]() {
     // The flag indicates if we are still waiting for 'eod'.
+    assert(CurLexer);
     if (CurLexer->ParsingPreprocessorDirective)
       DiscardUntilEndOfDirective();
   });
 
   // Used to un-poison and then re-poison identifiers of the __VA_ARGS__ ilk
   // within their appropriate context.
   VariadicMacroScopeGuard VariadicMacroScopeGuard(*this);
 

clang/lib/Lex/PPLexerChange.cpp

bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
   if ((LeavingSubmodule || IncludeMacroStack.empty()) &&
       !BuildingSubmoduleStack.empty() &&
       BuildingSubmoduleStack.back().IsPragma) {
     Diag(BuildingSubmoduleStack.back().ImportLoc,
          diag::err_pp_module_begin_without_module_end);
     Module *M = LeaveSubmodule(/*ForPragma*/true);
 
     Result.startToken();
+    assert(CurLexer && "Got EOF but no current lexer set!");
     const char *EndPos = getCurLexerEndPos();
     CurLexer->BufferPtr = EndPos;
     CurLexer->FormTokenWithChars(Result, EndPos, tok::annot_module_end);
     Result.setAnnotationEndLoc(Result.getLocation());
     Result.setAnnotationValue(M);
     return true;
   }
 

clang/lib/Lex/Preprocessor.cpp

void Preprocessor::SkipTokensWhileUsingPCH() {
   bool UsingPCHThroughHeader = SkippingUntilPCHThroughHeader;
   bool UsingPragmaHdrStop = SkippingUntilPragmaHdrStop;
   Token Tok;
   while (true) {
     bool InPredefines =
         (CurLexer && CurLexer->getFileID() == getPredefinesFileID());
     switch (CurLexerKind) {
     case CLK_Lexer:
+      assert(CurLexer && "CurLexer is empty while CurLexerKind is CLK_Lexer.");
       CurLexer->Lex(Tok);
      break;
     case CLK_TokenLexer:
+      assert(CurTokenLexer &&
+          "CurTokenLexer is empty while CurLexerKind is CLK_TokenLexer.");
       CurTokenLexer->Lex(Tok);
       break;
     case CLK_CachingLexer:
       CachingLex(Tok);
       break;
     case CLK_LexAfterModuleImport:
       LexAfterModuleImport(Tok);
       break;

 void Preprocessor::Lex(Token &Result) {
   ++LexLevel;
 
   // We loop here until a lex function returns a token; this avoids recursion.
   bool ReturnedToken;
   do {
     switch (CurLexerKind) {
     case CLK_Lexer:
+      assert(CurLexer && "CurLexer is empty while CurLexerKind is CLK_Lexer.");
       ReturnedToken = CurLexer->Lex(Result);
       break;
     case CLK_TokenLexer:
+      assert(CurTokenLexer &&
+          "CurTokenLexer is empty while CurLexerKind is CLK_TokenLexer.");
       ReturnedToken = CurTokenLexer->Lex(Result);
       break;
     case CLK_CachingLexer:
       CachingLex(Result);
       ReturnedToken = true;
       break;
     case CLK_LexAfterModuleImport:
       ReturnedToken = LexAfterModuleImport(Result);
case ImportAction::SkippedModuleImport:
       Suffix[0].setAnnotationEndLoc(Suffix[0].getLocation());
       Suffix[0].setAnnotationValue(Action.ModuleForHeader);
       // FIXME: Call the moduleImport callback?
       break;
     case ImportAction::Failure:
       assert(TheModuleLoader.HadFatalFailure &&
              "This should be an early exit only to a fatal error");
       Result.setKind(tok::eof);
+      assert(CurLexer);
       CurLexer->cutOffLexing();
       EnterTokens(Suffix);
       return true;
     }
 
     EnterTokens(Suffix);
     return false;
   }