Allow to set incremental processing from CIndex

[yvvan]

Summary

Make possible to parse translation Unit in CIndex with incremental processing turned on.

1. Why to do that? - Reparse with that option takes less time.
2. What's the affected area? - I don't know. Can someone explain me what incremental processing is and what I loose if I parse with that option?

include/clang-c/Index.h

enum CXTranslationUnit_Flags {
    * purposes of an IDE, this is undesirable behavior and as much information
    * as possible should be reported. Use this flag to enable this behavior.
    */
   CXTranslationUnit_KeepGoing = 0x200,
 
   /**
    * \brief Sets the preprocessor in a mode for parsing a single file only.
    */
-  CXTranslationUnit_SingleFileParse = 0x400
+  CXTranslationUnit_SingleFileParse = 0x400,
+
+  /**
+   * \brief Do incremental processing
+   */
+  CXTranslationUnit_IncrementalProcessing = 0x800
 };
 
 /**
  * \brief Returns the set of flags that is suitable for parsing a translation
  * unit that is being edited.
  *
  * The set of flags returned provide options for \c clang_parseTranslationUnit()
  * to indicate that the translation unit is likely to be reparsed many times,

include/clang/Frontend/ASTUnit.h

static ASTUnit *LoadFromCommandLine(
       bool RemappedFilesKeepOriginalName = true,
       unsigned PrecompilePreambleAfterNParses = 0,
       TranslationUnitKind TUKind = TU_Complete,
       bool CacheCodeCompletionResults = false,
       bool IncludeBriefCommentsInCodeCompletion = false,
       bool AllowPCHWithCompilerErrors = false, bool SkipFunctionBodies = false,
       bool SingleFileParse = false,
       bool UserFilesAreVolatile = false, bool ForSerialization = false,
+      bool IncrementalProcessing = false,
       llvm::Optional<StringRef> ModuleFormat = llvm::None,
       std::unique_ptr<ASTUnit> *ErrAST = nullptr,
       IntrusiveRefCntPtr<vfs::FileSystem> VFS = nullptr);
 
   /// \brief Reparse the source files using the same command-line options that
   /// were originally used to produce this translation unit.
   ///
   /// \param VFS - A vfs::FileSystem to be used for all file accesses. Note that

include/clang/Lex/Preprocessor.h

if (CachedLexPos != 0 && isBacktrackEnabled())
       CachedTokens[CachedLexPos-1] = Tok;
   }
 
   /// \brief Recompute the current lexer kind based on the CurLexer/CurPTHLexer/
   /// CurTokenLexer pointers.
   void recomputeCurLexerKind();
 
   /// \brief Returns true if incremental processing is enabled
-  bool isIncrementalProcessingEnabled() const { return IncrementalProcessing; }
+  bool isIncrementalProcessingEnabled() const;
 
   /// \brief Enables the incremental processing
-  void enableIncrementalProcessing(bool value = true) {
-    IncrementalProcessing = value;
-  }
+  void enableIncrementalProcessing(bool value = true);
   
   /// \brief Specify the point at which code-completion will be performed.
   ///
   /// \param File the file in which code completion should occur. If
   /// this file is included multiple times, code-completion will
   /// perform completion the first time it is included. If NULL, this
   /// function clears out the code-completion point.
   ///

include/clang/Lex/PreprocessorOptions.h

public:
   std::string ImplicitPTHInclude;
 
   /// If given, a PTH cache file to use for speeding up header parsing.
   std::string TokenCache;
 
   /// When enabled, preprocessor is in a mode for parsing a single file only.
   bool SingleFileParseMode = false;
 
+  /// Do incremental processing
+  bool IncrementalProcessing = false;
+
   /// \brief True if the SourceManager should report the original file name for
   /// contents of files that were remapped to other files. Defaults to true.
   bool RemappedFilesKeepOriginalName;
 
   /// \brief The set of file remappings, which take existing files on
   /// the system (the first part of each pair) and gives them the
   /// contents of other files on the system (the second part of each
   /// pair).

lib/Frontend/ASTUnit.cpp

ASTUnit *ASTUnit::LoadFromCommandLine(
     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
     IntrusiveRefCntPtr<DiagnosticsEngine> Diags, StringRef ResourceFilesPath,
     bool OnlyLocalDecls, bool CaptureDiagnostics,
     ArrayRef<RemappedFile> RemappedFiles, bool RemappedFilesKeepOriginalName,
     unsigned PrecompilePreambleAfterNParses, TranslationUnitKind TUKind,
     bool CacheCodeCompletionResults, bool IncludeBriefCommentsInCodeCompletion,
     bool AllowPCHWithCompilerErrors, bool SkipFunctionBodies,
     bool SingleFileParse, bool UserFilesAreVolatile, bool ForSerialization,
-    llvm::Optional<StringRef> ModuleFormat, std::unique_ptr<ASTUnit> *ErrAST,
-    IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
+    bool IncrementalProcessing, llvm::Optional<StringRef> ModuleFormat,
+    std::unique_ptr<ASTUnit> *ErrAST, IntrusiveRefCntPtr<vfs::FileSystem> VFS) {
   assert(Diags.get() && "no DiagnosticsEngine was provided");
 
   SmallVector<StoredDiagnostic, 4> StoredDiagnostics;
 
   std::shared_ptr<CompilerInvocation> CI;
 
   {
 
for (const auto &RemappedFile : RemappedFiles) {
     CI->getPreprocessorOpts().addRemappedFile(RemappedFile.first,
                                               RemappedFile.second);
   }
   PreprocessorOptions &PPOpts = CI->getPreprocessorOpts();
   PPOpts.RemappedFilesKeepOriginalName = RemappedFilesKeepOriginalName;
   PPOpts.AllowPCHWithCompilerErrors = AllowPCHWithCompilerErrors;
   PPOpts.GeneratePreamble = PrecompilePreambleAfterNParses != 0;
   PPOpts.SingleFileParseMode = SingleFileParse;
+  PPOpts.IncrementalProcessing = IncrementalProcessing;
   
   // Override the resources path.
   CI->getHeaderSearchOpts().ResourceDir = ResourceFilesPath;
 
   CI->getFrontendOpts().SkipFunctionBodies = SkipFunctionBodies;
 
   if (ModuleFormat)
     CI->getHeaderSearchOpts().ModuleFormat = ModuleFormat.getValue();

lib/Lex/Preprocessor.cpp

if ((*H)->HandleComment(*this, Comment))
       AnyPendingTokens = true;
   }
   if (!AnyPendingTokens || getCommentRetentionState())
     return false;
   Lex(result);
   return true;
 }
 
+bool Preprocessor::isIncrementalProcessingEnabled() const {
+  return PPOpts->IncrementalProcessing;
+}
+
+void Preprocessor::enableIncrementalProcessing(bool value) {
+  PPOpts->IncrementalProcessing = value;
+}
+
 ModuleLoader::~ModuleLoader() { }
 
 CommentHandler::~CommentHandler() { }
 
 CodeCompletionHandler::~CodeCompletionHandler() { }
 
 void Preprocessor::createPreprocessingRecord() {
   if (Record)
     return;
   
   Record = new PreprocessingRecord(getSourceManager());
   addPPCallbacks(std::unique_ptr<PPCallbacks>(Record));
 }

tools/libclang/CIndex.cpp

= (options & (CXTranslationUnit_Incomplete |
                   CXTranslationUnit_SingleFileParse))? TU_Prefix : TU_Complete;
   bool CacheCodeCompletionResults
     = options & CXTranslationUnit_CacheCompletionResults;
   bool IncludeBriefCommentsInCodeCompletion
     = options & CXTranslationUnit_IncludeBriefCommentsInCodeCompletion;
   bool SkipFunctionBodies = options & CXTranslationUnit_SkipFunctionBodies;
   bool SingleFileParse = options & CXTranslationUnit_SingleFileParse;
   bool ForSerialization = options & CXTranslationUnit_ForSerialization;
+  bool IncrementalProcessing = options & CXTranslationUnit_IncrementalProcessing;
 
   // Configure the diagnostics.
   IntrusiveRefCntPtr<DiagnosticsEngine>
     Diags(CompilerInstance::createDiagnostics(new DiagnosticOptions));
 
   if (options & CXTranslationUnit_KeepGoing)
     Diags->setSuppressAfterFatalError(false);
 
clang_parseTranslationUnit_Impl(CXIndex CIdx, const char *source_filename,
   std::unique_ptr<ASTUnit> Unit(ASTUnit::LoadFromCommandLine(
       Args->data(), Args->data() + Args->size(),
       CXXIdx->getPCHContainerOperations(), Diags,
       CXXIdx->getClangResourcesPath(), CXXIdx->getOnlyLocalDecls(),
       /*CaptureDiagnostics=*/true, *RemappedFiles.get(),
       /*RemappedFilesKeepOriginalName=*/true, PrecompilePreambleAfterNParses,
       TUKind, CacheCodeCompletionResults, IncludeBriefCommentsInCodeCompletion,
       /*AllowPCHWithCompilerErrors=*/true, SkipFunctionBodies, SingleFileParse,
-      /*UserFilesAreVolatile=*/true, ForSerialization,
+      /*UserFilesAreVolatile=*/true, ForSerialization, IncrementalProcessing,
       CXXIdx->getPCHContainerOperations()->getRawReader().getFormat(),
       &ErrUnit));
 
   // Early failures in LoadFromCommandLine may return with ErrUnit unset.
   if (!Unit && !ErrUnit)
     return CXError_ASTReadError;
 
   if (NumErrors != Diags->getClient()->getNumErrors()) {