Remove the enableIncrementalProcesing logic in favor of -fincremental-extensions

[v.g.vassilev]

Summary

As we have discussed in https://reviews.llvm.org/D127284 we would like to phase out enableIncrementalProcesing and use a better integrated option.

https://reviews.llvm.org/D127284 introduces a LangOpt called IncrementalExtensions, however, we also have a TU_Incremental kind. I am not entirely sure when we should use that because generally most of our actions (even if we pass -fincremental-extensions) we still get TU_Complete.

@aaron.ballman, @rsmith, do you have an idea how to harmonize this?

clang/include/clang/Lex/Preprocessor.h

void TypoCorrectToken(const Token &Tok) {
     if (CachedLexPos != 0 && isBacktrackEnabled())
       CachedTokens[CachedLexPos-1] = Tok;
   }
 
   /// Recompute the current lexer kind based on the CurLexer/
   /// CurTokenLexer pointers.
   void recomputeCurLexerKind();
 
-  /// Returns true if incremental processing is enabled
-  bool isIncrementalProcessingEnabled() const {
-    return getLangOpts().IncrementalExtensions;
-  }
-
-  /// Enables the incremental processing
-  void enableIncrementalProcessing(bool value = true) {
-    // FIXME: Drop this interface.
-    const_cast<LangOptions &>(getLangOpts()).IncrementalExtensions = value;
-  }
-
   /// Specify the point at which code-completion will be performed.
   ///
   /// \param File the file in which code completion should occur. If
   /// this file is included multiple times, code-completion will
   /// perform completion the first time it is included. If NULL, this
   /// function clears out the code-completion point.
   ///
   /// \param Line the line at which code completion should occur

clang/lib/Interpreter/IncrementalParser.cpp

static CodeGenerator *getCodeGen(FrontendAction *Act) {
   if (!WrappedAct->hasIRSupport())
     return nullptr;
   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
 }
 
 llvm::Expected<PartialTranslationUnit &>
 IncrementalParser::Parse(llvm::StringRef input) {
   Preprocessor &PP = CI->getPreprocessor();
-  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
+  assert(PP.getLangOpts().IncrementalExtensions && "Not in incremental mode!?");
 
   std::ostringstream SourceName;
   SourceName << "input_line_" << InputCount++;
 
   // Create an uninitialized memory buffer, copy code in and append "\n"
   size_t InputSize = input.size(); // don't include trailing 0
   // MemBuffer size should *not* include terminating zero
   std::unique_ptr<llvm::MemoryBuffer> MB(

clang/lib/Lex/PPLexerChange.cpp

bool Preprocessor::HandleEndOfFile(Token &Result, bool isEndOfMacro) {
   }
 
   if (creatingPCHWithThroughHeader() && !LeavingPCHThroughHeader) {
     // Reached the end of the compilation without finding the through header.
     Diag(CurLexer->getFileLoc(), diag::err_pp_through_header_not_seen)
         << PPOpts->PCHThroughHeader << 0;
   }
 
-  if (!isIncrementalProcessingEnabled())
+  if (!getLangOpts().IncrementalExtensions) {
     // We're done with lexing.
     CurLexer.reset();
-
-  if (!isIncrementalProcessingEnabled())
     CurPPLexer = nullptr;
+  }
 
   if (TUKind == TU_Complete) {
     // This is the end of the top-level file. 'WarnUnusedMacroLocs' has
     // collected all macro locations that we need to warn because they are not
     // used.
     for (WarnUnusedMacroLocsTy::iterator
            I=WarnUnusedMacroLocs.begin(), E=WarnUnusedMacroLocs.end();
            I!=E; ++I)

clang/lib/Parse/ParseDecl.cpp

#include "clang/Basic/OpenCLImageTypes.def"
 
   // C11 _Atomic
   case tok::kw__Atomic:
     return true;
   }
 }
 
 Parser::DeclGroupPtrTy Parser::ParseTopLevelStmtDecl() {
-  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode");
+  assert(PP.getLangOpts().IncrementalExtensions && "Not in incremental mode");
 
   // Parse a top-level-stmt.
   Parser::StmtVector Stmts;
   ParsedStmtContext SubStmtCtx = ParsedStmtContext();
   StmtResult R = ParseStatementOrDeclaration(Stmts, SubStmtCtx);
   if (!R.isUsable())
     return nullptr;
 

clang/lib/Parse/Parser.cpp

 ///           declaration
 /// [C++20]   module-import-declaration
 bool Parser::ParseTopLevelDecl(DeclGroupPtrTy &Result,
                                Sema::ModuleImportState &ImportState) {
   DestroyTemplateIdAnnotationsRAIIObj CleanupRAII(*this);
 
   // Skip over the EOF token, flagging end of previous input for incremental
   // processing
-  if (PP.isIncrementalProcessingEnabled() && Tok.is(tok::eof))
+  if (PP.getLangOpts().IncrementalExtensions && Tok.is(tok::eof))
     ConsumeToken();
 
   Result = nullptr;
   switch (Tok.getKind()) {
   case tok::annot_pragma_unused:
     HandlePragmaUnused();
     return false;
 
case tok::kw_module:
     return nullptr;
 
   default:
   dont_know:
     if (Tok.isEditorPlaceholder()) {
       ConsumeToken();
       return nullptr;
     }
-    if (PP.isIncrementalProcessingEnabled() &&
+    if (PP.getLangOpts().IncrementalExtensions &&
         !isDeclarationStatement(/*DisambiguatingWithExpression=*/true))
       return ParseTopLevelStmtDecl();
 
     // We can't tell whether this is a function-definition or declaration yet.
     if (!SingleDecl)
       return ParseDeclarationOrFunctionDefinition(Attrs, DeclSpecAttrs, DS);
   }
 

clang/lib/Sema/Sema.cpp

void Sema::ActOnEndOfTranslationUnit() {
   UnusedFileScopedDecls.erase(
       std::remove_if(UnusedFileScopedDecls.begin(nullptr, true),
                      UnusedFileScopedDecls.end(),
                      [this](const DeclaratorDecl *DD) {
                        return ShouldRemoveFromUnused(this, DD);
                      }),
       UnusedFileScopedDecls.end());
 
-  if (TUKind == TU_Prefix) {
-    // Translation unit prefixes don't need any of the checking below.
-    if (!PP.isIncrementalProcessingEnabled())
+  if (TUKind == TU_Prefix) {
+    if (!LangOpts.IncrementalExtensions)
       TUScope = nullptr;
     return;
   }
 
   // Check for #pragma weak identifiers that were never declared
   LoadExternalWeakUndeclaredIdentifiers();
   for (const auto &WeakIDs : WeakUndeclaredIdentifiers) {
     if (WeakIDs.second.empty())
void Sema::ActOnEndOfTranslationUnit() {
   }
 
   // Check we've noticed that we're no longer parsing the initializer for every
   // variable. If we miss cases, then at best we have a performance issue and
   // at worst a rejects-valid bug.
   assert(ParsingInitForAutoVars.empty() &&
          "Didn't unmark var as having its initializer parsed");
 
-  if (!PP.isIncrementalProcessingEnabled())
+  if (TUKind != TU_Incremental)
     TUScope = nullptr;
 }
 
 
 //===----------------------------------------------------------------------===//
 // Helper functions.
 //===----------------------------------------------------------------------===//
 

clang/unittests/Frontend/FrontendActionTest.cpp

 
 using namespace llvm;
 using namespace clang;
 
 namespace {
 
 class TestASTFrontendAction : public ASTFrontendAction {
 public:
-  TestASTFrontendAction(bool enableIncrementalProcessing = false,
-                        bool actOnEndOfTranslationUnit = false)
-    : EnableIncrementalProcessing(enableIncrementalProcessing),
-      ActOnEndOfTranslationUnit(actOnEndOfTranslationUnit) { }
+  TestASTFrontendAction(bool actOnEndOfTranslationUnit = false)
+      : ActOnEndOfTranslationUnit(actOnEndOfTranslationUnit) {}
 
-  bool EnableIncrementalProcessing;
   bool ActOnEndOfTranslationUnit;
   std::vector<std::string> decl_names;
 
-  bool BeginSourceFileAction(CompilerInstance &ci) override {
-    if (EnableIncrementalProcessing)
-      ci.getPreprocessor().enableIncrementalProcessing();
-
-    return ASTFrontendAction::BeginSourceFileAction(ci);
-  }
-
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override {
     return std::make_unique<Visitor>(CI, ActOnEndOfTranslationUnit,
                                       decl_names);
   }
 
 private:
   class Visitor : public ASTConsumer, public RecursiveASTVisitor<Visitor> {
TEST(ASTFrontendAction, IncrementalParsing) {
   auto invocation = std::make_shared<CompilerInvocation>();
   invocation->getPreprocessorOpts().addRemappedFile(
       "test.cc",
       MemoryBuffer::getMemBuffer("int main() { float x; }").release());
   invocation->getFrontendOpts().Inputs.push_back(
       FrontendInputFile("test.cc", Language::CXX));
   invocation->getFrontendOpts().ProgramAction = frontend::ParseSyntaxOnly;
   invocation->getTargetOpts().Triple = "i386-unknown-linux-gnu";
+  invocation->getLangOpts()->IncrementalExtensions = 1;
   CompilerInstance compiler;
   compiler.setInvocation(std::move(invocation));
   compiler.createDiagnostics();
 
-  TestASTFrontendAction test_action(/*enableIncrementalProcessing=*/true);
+  TestASTFrontendAction test_action;
   ASSERT_TRUE(compiler.ExecuteAction(test_action));
   ASSERT_EQ(2U, test_action.decl_names.size());
   EXPECT_EQ("main", test_action.decl_names[0]);
   EXPECT_EQ("x", test_action.decl_names[1]);
 }
 
 TEST(ASTFrontendAction, LateTemplateIncrementalParsing) {
   auto invocation = std::make_shared<CompilerInvocation>();
   invocation->getLangOpts()->CPlusPlus = true;
   invocation->getLangOpts()->DelayedTemplateParsing = true;
+  invocation->getLangOpts()->IncrementalExtensions = 1;
   invocation->getPreprocessorOpts().addRemappedFile(
     "test.cc", MemoryBuffer::getMemBuffer(
       "template<typename T> struct A { A(T); T data; };\n"
       "template<typename T> struct B: public A<T> {\n"
       "  B();\n"
       "  B(B const& b): A<T>(b.data) {}\n"
       "};\n"
       "B<char> c() { return B<char>(); }\n").release());
   invocation->getFrontendOpts().Inputs.push_back(
       FrontendInputFile("test.cc", Language::CXX));
   invocation->getFrontendOpts().ProgramAction = frontend::ParseSyntaxOnly;
   invocation->getTargetOpts().Triple = "i386-unknown-linux-gnu";
   CompilerInstance compiler;
   compiler.setInvocation(std::move(invocation));
   compiler.createDiagnostics();
 
-  TestASTFrontendAction test_action(/*enableIncrementalProcessing=*/true,
-                                    /*actOnEndOfTranslationUnit=*/true);
+  TestASTFrontendAction test_action(/*actOnEndOfTranslationUnit=*/true);
   ASSERT_TRUE(compiler.ExecuteAction(test_action));
   ASSERT_EQ(13U, test_action.decl_names.size());
   EXPECT_EQ("A", test_action.decl_names[0]);
   EXPECT_EQ("c", test_action.decl_names[12]);
 }
 
 struct TestPPCallbacks : public PPCallbacks {
   TestPPCallbacks() : SeenEnd(false) {}

lldb/source/Plugins/ExpressionParser/Clang/ClangModulesDeclVendor.cpp

std::vector<std::string> compiler_invocation_arguments = {
       "-fmodules",
       "-fimplicit-module-maps",
       "-fcxx-modules",
       "-fsyntax-only",
       "-femit-all-decls",
       "-target",
       arch.GetTriple().str(),
       "-fmodules-validate-system-headers",
-      "-Werror=non-modular-include-in-framework-module"};
+      "-Werror=non-modular-include-in-framework-module",
+      "-Xclang=-fincremental-extensions"};
 
   target.GetPlatform()->AddClangModuleCompilationOptions(
       &target, compiler_invocation_arguments);
 
   compiler_invocation_arguments.push_back(ModuleImportBufferName);
 
   // Add additional search paths with { "-I", path } or { "-F", path } here.
 
if (!instance->hasTarget())
     return nullptr;
 
   instance->getTarget().adjust(*diagnostics_engine, instance->getLangOpts());
 
   if (!action->BeginSourceFile(*instance,
                                instance->getFrontendOpts().Inputs[0]))
     return nullptr;
 
-  instance->getPreprocessor().enableIncrementalProcessing();
-
   instance->createASTReader();
 
   instance->createSema(action->getTranslationUnitKind(), nullptr);
 
   const bool skipFunctionBodies = false;
   std::unique_ptr<clang::Parser> parser(new clang::Parser(
       instance->getPreprocessor(), instance->getSema(), skipFunctionBodies));