[ClangRepl] support code completion at the repl

[capfredf]

Summary

This patch enabled code completion for ClangREPL. The feature was built upon two
existing Clang components: a list completer for LineEditor and SemaCodeCompletion.

The former serves the main entry point to trigger the latter. Because a
completion point for a compiler instance needs to be unchanged once it is set,
an incremental compiler instance for code completion is created. In addition to
completion points, it differs from a regular incremental compiler in the following ways:

1. It does not execute input.

2. To obtain declarations or bindings from previous input in the same REPL

session, it carries over AST context source from the main compiler, i.e. the one
used for the interpreter.

3. It contains a ReplCompletionConsumer, a subclass of

CodeCompletionConsumer. The consumer communicates completion results from
SemaCodeCompletion back to the list completer for the REPL.

clang/include/clang/Interpreter/CodeCompletion.h

+//===------ CodeCompletion.h - Code Completion for ClangRepl -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the classes which performs code completion at the REPL.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_INTERPRETER_CODE_COMPLETION_H
+#define LLVM_CLANG_INTERPRETER_CODE_COMPLETION_H
+#include "clang/Sema/CodeCompleteConsumer.h"
+#include "llvm/LineEditor/LineEditor.h"
+
+namespace clang {
+class Interpreter;
+class IncrementalCompilerBuilder;
+
+clang::CodeCompleteOptions getClangCompleteOpts();
+
+class ReplCompletionConsumer : public CodeCompleteConsumer {
+public:
+  ReplCompletionConsumer(std::vector<CodeCompletionResult> &Results)
+      : CodeCompleteConsumer(getClangCompleteOpts()),
+        CCAllocator(std::make_shared<GlobalCodeCompletionAllocator>()),
+        CCTUInfo(CCAllocator), Results(Results){};
+  void ProcessCodeCompleteResults(class Sema &S, CodeCompletionContext Context,
+                                  CodeCompletionResult *InResults,
+                                  unsigned NumResults) final;
+
+  clang::CodeCompletionAllocator &getAllocator() override {
+    return *CCAllocator;
+  }
+
+  clang::CodeCompletionTUInfo &getCodeCompletionTUInfo() override {
+    return CCTUInfo;
+  }
+
+private:
+  std::shared_ptr<GlobalCodeCompletionAllocator> CCAllocator;
+  CodeCompletionTUInfo CCTUInfo;
+  std::vector<CodeCompletionResult> &Results;
+};
+
+struct ReplListCompleter {
+  IncrementalCompilerBuilder &CB;
+  Interpreter &MainInterp;
+  ReplListCompleter(IncrementalCompilerBuilder &CB, Interpreter &Interp)
+      : CB(CB), MainInterp(Interp){};
+  std::vector<llvm::LineEditor::Completion> operator()(llvm::StringRef Buffer,
+                                                       size_t Pos) const;
+
+private:
+  std::vector<StringRef>
+  toCodeCompleteStrings(const std::vector<CodeCompletionResult> &Results) const;
+};
+
+} // namespace clang
+#endif

clang/include/clang/Interpreter/Interpreter.h

 namespace orc {
 class LLJIT;
 class ThreadSafeContext;
 } // namespace orc
 } // namespace llvm
 
 namespace clang {
 
+class CodeCompletionResult;
 class CompilerInstance;
 class IncrementalExecutor;
 class IncrementalParser;
+class ReplCompletionConsumer;
 
 /// Create a pre-configured \c CompilerInstance for incremental processing.
 class IncrementalCompilerBuilder {
 public:
   IncrementalCompilerBuilder() {}
 
   void SetCompilerArgs(const std::vector<const char *> &Args) {
     UserArgs = Args;

 /// Provides top-level interfaces for incremental compilation and execution.
 class Interpreter {
   std::unique_ptr<llvm::orc::ThreadSafeContext> TSCtx;
   std::unique_ptr<IncrementalParser> IncrParser;
   std::unique_ptr<IncrementalExecutor> IncrExecutor;
 
   // An optional parser for CUDA offloading
   std::unique_ptr<IncrementalParser> DeviceParser;
+  std::unique_ptr<ReplCompletionConsumer> CConsumer;
 
   Interpreter(std::unique_ptr<CompilerInstance> CI, llvm::Error &Err);
+  Interpreter(std::unique_ptr<CompilerInstance> CI, llvm::Error &Err,
+              std::vector<CodeCompletionResult> &CompResults,
+              const CompilerInstance *ParentCI = nullptr);
 
   llvm::Error CreateExecutor();
   unsigned InitPTUSize = 0;
 
   // This member holds the last result of the value printing. It's a class
   // member because we might want to access it after more inputs. If no value
   // printing happens, it's in an invalid state.
   Value LastValue;
 
 public:
   ~Interpreter();
+
   static llvm::Expected<std::unique_ptr<Interpreter>>
   create(std::unique_ptr<CompilerInstance> CI);
+
   static llvm::Expected<std::unique_ptr<Interpreter>>
   createWithCUDA(std::unique_ptr<CompilerInstance> CI,
                  std::unique_ptr<CompilerInstance> DCI);
+
+  static llvm::Expected<std::unique_ptr<Interpreter>>
+  createForCodeCompletion(IncrementalCompilerBuilder &CB,
+                          const CompilerInstance *ParentCI,
+                          std::vector<CodeCompletionResult> &CompResults);
+
   const ASTContext &getASTContext() const;
   ASTContext &getASTContext();
+  void CodeComplete(llvm::StringRef Input, size_t Col, size_t Line = 1);
   const CompilerInstance *getCompilerInstance() const;
   llvm::Expected<llvm::orc::LLJIT &> getExecutionEngine();
 
   llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Code);
   llvm::Error Execute(PartialTranslationUnit &T);
   llvm::Error ParseAndExecute(llvm::StringRef Code, Value *V = nullptr);
   llvm::Expected<llvm::orc::ExecutorAddr> CompileDtorCall(CXXRecordDecl *CXXRD);
 
public:
   enum InterfaceKind { NoAlloc, WithAlloc, CopyArray };
 
   const llvm::SmallVectorImpl<Expr *> &getValuePrintingInfo() const {
     return ValuePrintingInfo;
   }
 
   Expr *SynthesizeExpr(Expr *E);
 
+  std::string getAllInput() const;
+
 private:
   size_t getEffectivePTUSize() const;
 
   bool FindRuntimeInterface();
 
   llvm::DenseMap<CXXRecordDecl *, llvm::orc::ExecutorAddr> Dtors;
 
   llvm::SmallVector<Expr *, 3> ValuePrintingInfo;
 };
 } // namespace clang
 
 #endif // LLVM_CLANG_INTERPRETER_INTERPRETER_H

clang/lib/Interpreter/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
    core
    native
    MC
    Option
    OrcJit
    OrcShared
    OrcTargetProcess
    Support
    Target
    TargetParser
   )
 
 add_clang_library(clangInterpreter
+  CodeCompletion.cpp
   DeviceOffload.cpp
+  ExternalSource.cpp
   IncrementalExecutor.cpp
   IncrementalParser.cpp
   Interpreter.cpp
   InterpreterUtils.cpp
   Value.cpp
 
   DEPENDS
   intrinsics_gen

clang/lib/Interpreter/CodeCompletion.cpp

+//===------ CodeCompletion.cpp - Code Completion for ClangRepl -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the classes which performs code completion at the REPL.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Interpreter/CodeCompletion.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Interpreter/Interpreter.h"
+#include "clang/Lex/PreprocessorOptions.h"
+#include "clang/Sema/CodeCompleteOptions.h"
+#include "clang/Sema/Sema.h"
+
+namespace clang {
+
+clang::CodeCompleteOptions getClangCompleteOpts() {
+  clang::CodeCompleteOptions Opts;
+  Opts.IncludeCodePatterns = true;
+  Opts.IncludeMacros = true;
+  Opts.IncludeGlobals = true;
+  Opts.IncludeBriefComments = true;
+  return Opts;
+}
+
+void ReplCompletionConsumer::ProcessCodeCompleteResults(
+    class Sema &S, CodeCompletionContext Context,
+    CodeCompletionResult *InResults, unsigned NumResults) {
+  for (unsigned I = 0; I < NumResults; ++I) {
+    auto &Result = InResults[I];
+    switch (Result.Kind) {
+    case CodeCompletionResult::RK_Declaration:
+      if (Result.Declaration->getIdentifier()) {
+        Results.push_back(Result);
+      }
+      break;
+    default:
+      break;
+    case CodeCompletionResult::RK_Keyword:
+      Results.push_back(Result);
+      break;
+    }
+  }
+}
+
+std::vector<StringRef> ReplListCompleter::toCodeCompleteStrings(
+    const std::vector<CodeCompletionResult> &Results) const {
+  std::vector<StringRef> CompletionStrings;
+  for (auto Res : Results) {
+    switch (Res.Kind) {
+    case CodeCompletionResult::RK_Declaration:
+      if (auto *ID = Res.Declaration->getIdentifier()) {
+        CompletionStrings.push_back(ID->getName());
+      }
+      break;
+    case CodeCompletionResult::RK_Keyword:
+      CompletionStrings.push_back(Res.Keyword);
+      break;
+    default:
+      break;
+    }
+  }
+  return CompletionStrings;
+}
+
+std::vector<llvm::LineEditor::Completion>
+ReplListCompleter::operator()(llvm::StringRef Buffer, size_t Pos) const {
+  std::vector<llvm::LineEditor::Completion> Comps;
+  std::vector<CodeCompletionResult> Results;
+  auto Interp = Interpreter::createForCodeCompletion(
+      CB, MainInterp.getCompilerInstance(), Results);
+
+  if (auto Err = Interp.takeError()) {
+    // log the error and returns an empty vector;
+    llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: ");
+    return Comps;
+  }
+
+  std::string AllCodeText =
+      MainInterp.getAllInput() + "\nvoid dummy(){\n" + Buffer.str() + "}";
+
+  // We need to wrap our input because we need `Sema::CodeCompleteOrdinaryName`
+  // to work on code from the REPL in a statement completion context. By
+  // default, `Sema::CodeCompleteOrdinaryName` thinks the input is a regular c++
+  // file. For example,
+  // ```
+  // clang-repl> int foo = 42;
+  // clang-repl> f_
+  // ```
+  //
+  // `Sema::CodeCompleteOrdinaryName` treats the code as
+  //
+  // ```
+  // int foo = 42;
+  // f_
+  // ```
+  //
+  // Since top-level expressions are not supported, `foo` should not be an
+  // option. But in a REPL session, we should be allowed to use `foo` to make a
+  // statement like `foo + 84;`.
+
+  auto Lines = std::count(AllCodeText.begin(), AllCodeText.end(), '\n') + 1;
+
+  (*Interp)->CodeComplete(AllCodeText, Pos + 1, Lines);
+
+  size_t space_pos = Buffer.rfind(" ");
+  llvm::StringRef s;
+  if (space_pos == llvm::StringRef::npos) {
+    s = Buffer;
+  } else {
+    s = Buffer.substr(space_pos + 1);
+  }
+
+  for (auto c : toCodeCompleteStrings(Results)) {
+    if (c.startswith(s)) {
+      Comps.push_back(
+          llvm::LineEditor::Completion(c.substr(s.size()).str(), c.str()));
+    }
+  }
+  return Comps;
+}
+} // namespace clang

clang/lib/Interpreter/ExternalSource.h

+//==----- ExternalSource.h - External AST Source for Code Completion ------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines components that make declarations parsed and executed by
+// the interpreter visible to the context where code completion is being
+// triggered.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ExternalASTSource.h"
+
+namespace clang {
+class ASTContext;
+class FileManager;
+class ASTImporter;
+
+class ExternalSource : public clang::ExternalASTSource {
+  ASTContext &ChildASTCtxt;
+  TranslationUnitDecl *ChildTUDeclCtxt;
+  ASTContext &ParentASTCtxt;
+  TranslationUnitDecl *ParentTUDeclCtxt;
+
+  std::unique_ptr<ASTImporter> Importer;
+
+public:
+  ExternalSource(ASTContext &ChildASTCtxt, FileManager &ChildFM,
+                 ASTContext &ParentASTCtxt, FileManager &ParentFM);
+  bool FindExternalVisibleDeclsByName(const DeclContext *DC,
+                                      DeclarationName Name) override;
+  void
+  completeVisibleDeclsMap(const clang::DeclContext *childDeclContext) override;
+};
+} // namespace clang

clang/lib/Interpreter/ExternalSource.cpp

+//===--- ExternalSource.cpp - External AST Source for Code Completion ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The file implements classes that make declarations parsed and executed by the
+// interpreter visible to the context where code completion is being triggered.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ExternalSource.h"
+#include "clang/AST/ASTImporter.h"
+#include "clang/AST/DeclarationName.h"
+#include "clang/Basic/IdentifierTable.h"
+
+namespace clang {
+ExternalSource::ExternalSource(ASTContext &ChildASTCtxt, FileManager &ChildFM,
+                               ASTContext &ParentASTCtxt, FileManager &ParentFM)
+    : ChildASTCtxt(ChildASTCtxt),
+      ChildTUDeclCtxt(ChildASTCtxt.getTranslationUnitDecl()),
+      ParentASTCtxt(ParentASTCtxt),
+      ParentTUDeclCtxt(ParentASTCtxt.getTranslationUnitDecl()) {
+  ASTImporter *importer =
+      new ASTImporter(ChildASTCtxt, ChildFM, ParentASTCtxt, ParentFM,
+                      /*MinimalImport : ON*/ true);
+  Importer.reset(importer);
+}
+
+bool ExternalSource::FindExternalVisibleDeclsByName(const DeclContext *DC,
+                                                    DeclarationName Name) {
+  IdentifierTable &ParentIdTable = ParentASTCtxt.Idents;
+
+  auto ParentDeclName =
+      DeclarationName(&(ParentIdTable.get(Name.getAsString())));
+
+  DeclContext::lookup_result lookup_result =
+      ParentTUDeclCtxt->lookup(ParentDeclName);
+
+  if (!lookup_result.empty()) {
+    return true;
+  }
+  return false;
+}
+
+void ExternalSource::completeVisibleDeclsMap(
+    const DeclContext *ChildDeclContext) {
+  assert(ChildDeclContext && ChildDeclContext == ChildTUDeclCtxt &&
+         "No child decl context!");
+
+  if (!ChildDeclContext->hasExternalVisibleStorage())
+    return;
+
+  for (auto *DeclCtxt = ParentTUDeclCtxt; DeclCtxt != nullptr;
+       DeclCtxt = DeclCtxt->getPreviousDecl()) {
+    for (auto &IDeclContext : DeclCtxt->decls()) {
+      if (NamedDecl *Decl = llvm::dyn_cast<NamedDecl>(IDeclContext)) {
+        if (auto DeclOrErr = Importer->Import(Decl)) {
+          if (NamedDecl *importedNamedDecl =
+                  llvm::dyn_cast<NamedDecl>(*DeclOrErr)) {
+            SetExternalVisibleDeclsForName(ChildDeclContext,
+                                           importedNamedDecl->getDeclName(),
+                                           importedNamedDecl);
+          }
+
+        } else {
+          llvm::consumeError(DeclOrErr.takeError());
+        }
+      }
+    }
+    ChildDeclContext->setHasExternalLexicalStorage(false);
+  }
+}
+
+} // namespace clang

clang/lib/Interpreter/IncrementalParser.h

 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 
 #include <list>
 #include <memory>
 namespace llvm {
 class LLVMContext;
-}
+} // namespace llvm
 
 namespace clang {
 class ASTConsumer;
 class CodeGenerator;
 class CompilerInstance;
 class IncrementalAction;
 class Interpreter;
 class Parser;
protected:
 
   /// List containing every information about every incrementally parsed piece
   /// of code.
   std::list<PartialTranslationUnit> PTUs;
 
   IncrementalParser();
 
 public:
+  // This is not necessarily needed.
+  // We can probably use external source to replace `AllInput`
+  std::string AllInput;
+
   IncrementalParser(Interpreter &Interp,
                     std::unique_ptr<CompilerInstance> Instance,
-                    llvm::LLVMContext &LLVMCtx, llvm::Error &Err);
+                    llvm::LLVMContext &LLVMCtx, llvm::Error &Err,
+                    const CompilerInstance *ParentCI = nullptr);
   virtual ~IncrementalParser();
 
   CompilerInstance *getCI() { return CI.get(); }
   CodeGenerator *getCodeGen() const;
 
   /// Parses incremental input by creating an in-memory file.
   ///\returns a \c PartialTranslationUnit which holds information about the
   /// \c TranslationUnitDecl and \c llvm::Module corresponding to the input.
   virtual llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Input);
+  void ParseForCodeCompletion(llvm::StringRef Input, size_t Col, size_t Line);
 
   /// Uses the CodeGenModule mangled name cache and avoids recomputing.
   ///\returns the mangled name of a \c GD.
   llvm::StringRef GetMangledName(GlobalDecl GD) const;
 
   void CleanUpPTU(PartialTranslationUnit &PTU);
 
   std::list<PartialTranslationUnit> &getPTUs() { return PTUs; }
 
   std::unique_ptr<llvm::Module> GenModule();
 
 private:
   llvm::Expected<PartialTranslationUnit &> ParseOrWrapTopLevelDecl();
+
+  llvm::Expected<PartialTranslationUnit &> ParseForPTU(FileID FID,
+                                                       SourceLocation SrcLoc);
+
+  std::pair<FileID, SourceLocation> createSourceFile(llvm::StringRef SourceName,
+                                                     llvm::StringRef Input);
 };
 } // end namespace clang
 
 #endif // LLVM_CLANG_LIB_INTERPRETER_INCREMENTALPARSER_H

clang/lib/Interpreter/IncrementalParser.cpp

 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the class which performs incremental code compilation.
 //
 //===----------------------------------------------------------------------===//
 
 #include "IncrementalParser.h"
+#include "ExternalSource.h"
 #include "clang/AST/DeclContextInternals.h"
 #include "clang/CodeGen/BackendUtil.h"
 #include "clang/CodeGen/CodeGenAction.h"
 #include "clang/CodeGen/ModuleBuilder.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendAction.h"
 #include "clang/FrontendTool/Utils.h"
 #include "clang/Interpreter/Interpreter.h"

 /// such as \p Preprocessor and \p Sema assuming no further input will come.
 ///
 /// \p IncrementalAction ensures it keep its underlying action's objects alive
 /// as long as the \p IncrementalParser needs them.
 ///
 class IncrementalAction : public WrapperFrontendAction {
 private:
   bool IsTerminating = false;
+  const CompilerInstance *ParentCI;
 
 public:
   IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
-                    llvm::Error &Err)
+                    llvm::Error &Err,
+                    const CompilerInstance *ParentCI = nullptr)
       : WrapperFrontendAction([&]() {
           llvm::ErrorAsOutParameter EAO(&Err);
           std::unique_ptr<FrontendAction> Act;
           switch (CI.getFrontendOpts().ProgramAction) {
           default:
             Err = llvm::createStringError(
                 std::errc::state_not_recoverable,
                 "Driver initialization failed. "
IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx,
             [[fallthrough]];
           case frontend::PrintPreprocessedInput:
             [[fallthrough]];
           case frontend::EmitLLVMOnly:
             Act.reset(new EmitLLVMOnlyAction(&LLVMCtx));
             break;
           }
           return Act;
-        }()) {}
+        }()),
+        ParentCI(ParentCI) {}
   FrontendAction *getWrapped() const { return WrappedAction.get(); }
   TranslationUnitKind getTranslationUnitKind() override {
     return TU_Incremental;
   }
   void ExecuteAction() override {
     CompilerInstance &CI = getCompilerInstance();
     assert(CI.hasPreprocessor() && "No PP!");
 
     // FIXME: Move the truncation aspect of this into Sema, we delayed this till
     // here so the source manager would be initialized.
     if (hasCodeCompletionSupport() &&
         !CI.getFrontendOpts().CodeCompletionAt.FileName.empty())
       CI.createCodeCompletionConsumer();
 
     // Use a code completion consumer?
     CodeCompleteConsumer *CompletionConsumer = nullptr;
     if (CI.hasCodeCompletionConsumer())
       CompletionConsumer = &CI.getCodeCompletionConsumer();
 
     Preprocessor &PP = CI.getPreprocessor();
     PP.EnterMainSourceFile();
 
+    if (ParentCI) {
+      ExternalSource *myExternalSource = new ExternalSource(
+          CI.getASTContext(), CI.getFileManager(), ParentCI->getASTContext(),
+          ParentCI->getFileManager());
+      llvm::IntrusiveRefCntPtr<ExternalASTSource> astContextExternalSource(
+          myExternalSource);
+      CI.getASTContext().setExternalSource(astContextExternalSource);
+      CI.getASTContext().getTranslationUnitDecl()->setHasExternalVisibleStorage(
+          true);
+    }
+
     if (!CI.hasSema())
       CI.createSema(getTranslationUnitKind(), CompletionConsumer);
   }
 
   // Do not terminate after processing the input. This allows us to keep various
   // clang objects alive and to incrementally grow the current TU.
   void EndSourceFile() override {
     // The WrappedAction can be nullptr if we issued an error in the ctor.
CodeGenerator *IncrementalParser::getCodeGen() const {
   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
 }
 
 IncrementalParser::IncrementalParser() {}
 
 IncrementalParser::IncrementalParser(Interpreter &Interp,
                                      std::unique_ptr<CompilerInstance> Instance,
                                      llvm::LLVMContext &LLVMCtx,
-                                     llvm::Error &Err)
+                                     llvm::Error &Err,
+                                     const CompilerInstance *ParentCI)
     : CI(std::move(Instance)) {
   llvm::ErrorAsOutParameter EAO(&Err);
-  Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err);
+  Act = std::make_unique<IncrementalAction>(*CI, LLVMCtx, Err, ParentCI);
   if (Err)
     return;
   CI->ExecuteAction(*Act);
   std::unique_ptr<ASTConsumer> IncrConsumer =
       std::make_unique<IncrementalASTConsumer>(Interp, CI->takeASTConsumer());
   CI->setASTConsumer(std::move(IncrConsumer));
   Consumer = &CI->getASTConsumer();
   P.reset(
IncrementalParser::ParseOrWrapTopLevelDecl() {
   LocalInstantiations.perform();
   GlobalInstantiations.perform();
 
   Consumer->HandleTranslationUnit(C);
 
   return LastPTU;
 }
 
-llvm::Expected<PartialTranslationUnit &>
-IncrementalParser::Parse(llvm::StringRef input) {
+void IncrementalParser::ParseForCodeCompletion(llvm::StringRef input,
+                                               size_t Col, size_t Line) {
   Preprocessor &PP = CI->getPreprocessor();
   assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
 
   std::ostringstream SourceName;
-  SourceName << "input_line_" << InputCount++;
+  SourceName << "input_line_[Completion]";
+
+  auto [FID, SrcLoc] = createSourceFile(SourceName.str(), input);
+  auto FE = CI->getSourceManager().getFileEntryRefForID(FID);
+  // auto Entry = PP.getFileManager().getFile(DummyFN);
+  // if (!Entry) {
+  //   std::cout << "Entry invalid \n";
+  //   return;
+  // }
+  if (FE) {
+    PP.SetCodeCompletionPoint(*FE, Line, Col);
+
+    // NewLoc only used for diags.
+    if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, SrcLoc))
+      return;
+
+    auto PTU = ParseOrWrapTopLevelDecl();
+    if (auto Err = PTU.takeError()) {
+      consumeError(std::move(Err));
+      return;
+    }
+
+    return;
+  }
+}
 
+std::pair<FileID, SourceLocation>
+IncrementalParser::createSourceFile(llvm::StringRef SourceName,
+                                    llvm::StringRef Input) {
   // Create an uninitialized memory buffer, copy code in and append "\n"
-  size_t InputSize = input.size(); // don't include trailing 0
+  size_t InputSize = Input.size(); // don't include trailing 0
   // MemBuffer size should *not* include terminating zero
   std::unique_ptr<llvm::MemoryBuffer> MB(
       llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
                                                         SourceName.str()));
   char *MBStart = const_cast<char *>(MB->getBufferStart());
-  memcpy(MBStart, input.data(), InputSize);
+  memcpy(MBStart, Input.data(), InputSize);
   MBStart[InputSize] = '\n';
 
   SourceManager &SM = CI->getSourceManager();
 
   // FIXME: Create SourceLocation, which will allow clang to order the overload
   // candidates for example
   SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
 
   // Create FileID for the current buffer.
-  FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
-                               /*LoadedOffset=*/0, NewLoc);
+  // FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
+  //                              /*LoadedOffset=*/0, NewLoc);
+
+  const clang::FileEntry *FE = SM.getFileManager().getVirtualFile(
+      SourceName.str(), InputSize, 0 /* mod time*/);
+  SM.overrideFileContents(FE, std::move(MB));
+  FileID FID = SM.createFileID(FE, NewLoc, SrcMgr::C_User);
+  return {FID, NewLoc};
+}
+
+llvm::Expected<PartialTranslationUnit &>
+IncrementalParser::ParseForPTU(FileID FID, SourceLocation SrcLoc) {
+  // Create an uninitialized memory buffer, copy code in and append "\n"
+  Preprocessor &PP = CI->getPreprocessor();
+  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
 
   // NewLoc only used for diags.
-  if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
+  if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, SrcLoc))
     return llvm::make_error<llvm::StringError>("Parsing failed. "
                                                "Cannot enter source file.",
                                                std::error_code());
 
   auto PTU = ParseOrWrapTopLevelDecl();
   if (!PTU)
-    return PTU.takeError();
+    return std::move(PTU.takeError());
+  return *PTU;
+}
+
+llvm::Expected<PartialTranslationUnit &>
+IncrementalParser::Parse(llvm::StringRef input) {
+  Preprocessor &PP = CI->getPreprocessor();
+  std::ostringstream SourceName;
+  SourceName << "input_line_" << InputCount++;
+
+  auto [FID, SrcLoc] = createSourceFile(SourceName.str(), input);
+  auto PTU = ParseForPTU(FID, SrcLoc);
+
+  if (!PTU) {
+    return std::move(PTU.takeError());
+  }
 
   if (PP.getLangOpts().DelayedTemplateParsing) {
     // Microsoft-specific:
     // Late parsed templates can leave unswallowed "macro"-like tokens.
     // They will seriously confuse the Parser when entering the next
     // source file. So lex until we are EOF.
     Token Tok;
     do {

clang/lib/Interpreter/Interpreter.cpp

 //===------ Interpreter.cpp - Incremental Compilation and Execution -------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the component which performs incremental code
 // compilation and execution.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Interpreter/Interpreter.h"
 
 #include "DeviceOffload.h"
+#include "ExternalSource.h"
 #include "IncrementalExecutor.h"
 #include "IncrementalParser.h"
 
 #include "InterpreterUtils.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/Mangle.h"
 #include "clang/AST/TypeVisitor.h"
 #include "clang/Basic/DiagnosticSema.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/CodeGen/CodeGenAction.h"
 #include "clang/CodeGen/ModuleBuilder.h"
 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/Job.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/TextDiagnosticBuffer.h"
+#include "clang/Interpreter/CodeCompletion.h"
 #include "clang/Interpreter/Value.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Sema/Lookup.h"
 #include "llvm/ExecutionEngine/JITSymbol.h"
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/IR/Module.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
CreateCI(const llvm::opt::ArgStringList &Argv) {
   Clang->getTarget().adjust(Clang->getDiagnostics(), Clang->getLangOpts());
 
   // Don't clear the AST before backend codegen since we do codegen multiple
   // times, reusing the same AST.
   Clang->getCodeGenOpts().ClearASTBeforeBackend = false;
 
   Clang->getFrontendOpts().DisableFree = false;
   Clang->getCodeGenOpts().DisableFree = false;
-
   return std::move(Clang);
 }
 
 } // anonymous namespace
 
 llvm::Expected<std::unique_ptr<CompilerInstance>>
 IncrementalCompilerBuilder::create(std::vector<const char *> &ClangArgv) {
 
Interpreter::Interpreter(std::unique_ptr<CompilerInstance> CI,
                          llvm::Error &Err) {
   llvm::ErrorAsOutParameter EAO(&Err);
   auto LLVMCtx = std::make_unique<llvm::LLVMContext>();
   TSCtx = std::make_unique<llvm::orc::ThreadSafeContext>(std::move(LLVMCtx));
   IncrParser = std::make_unique<IncrementalParser>(*this, std::move(CI),
                                                    *TSCtx->getContext(), Err);
 }
 
+Interpreter::Interpreter(std::unique_ptr<CompilerInstance> CI, llvm::Error &Err,
+                         std::vector<CodeCompletionResult> &CompResults,
+                         const CompilerInstance *ParentCI) {
+  llvm::ErrorAsOutParameter EAO(&Err);
+  auto LLVMCtx = std::make_unique<llvm::LLVMContext>();
+  TSCtx = std::make_unique<llvm::orc::ThreadSafeContext>(std::move(LLVMCtx));
+  auto *CConsumer = new ReplCompletionConsumer(CompResults);
+  CI->setCodeCompletionConsumer(CConsumer);
+  IncrParser = std::make_unique<IncrementalParser>(
+      *this, std::move(CI), *TSCtx->getContext(), Err, ParentCI);
+}
+
 Interpreter::~Interpreter() {
   if (IncrExecutor) {
     if (llvm::Error Err = IncrExecutor->cleanUp())
       llvm::report_fatal_error(
           llvm::Twine("Failed to clean up IncrementalExecutor: ") +
           toString(std::move(Err)));
   }
 }
Interpreter::create(std::unique_ptr<CompilerInstance> CI) {
   // FIXME: This is a ugly hack. Undo command checks its availability by looking
   // at the size of the PTU list. However we have parsed something in the
   // beginning of the REPL so we have to mark them as 'Irrevocable'.
   Interp->InitPTUSize = Interp->IncrParser->getPTUs().size();
   return std::move(Interp);
 }
 
 llvm::Expected<std::unique_ptr<Interpreter>>
+Interpreter::createForCodeCompletion(
+    IncrementalCompilerBuilder &CB, const CompilerInstance *ParentCI,
+    std::vector<CodeCompletionResult> &CompResults) {
+  auto CI = CB.CreateCpp();
+  if (auto Err = CI.takeError()) {
+    return std::move(Err);
+  }
+
+  (*CI)->getPreprocessorOpts().SingleFileParseMode = true;
+
+  (*CI)->getLangOpts().SpellChecking = false;
+  (*CI)->getLangOpts().DelayedTemplateParsing = false;
+
+  auto &FrontendOpts = (*CI)->getFrontendOpts();
+  FrontendOpts.CodeCompleteOpts = getClangCompleteOpts();
+
+  llvm::Error Err = llvm::Error::success();
+  auto Interp = std::unique_ptr<Interpreter>(
+      new Interpreter(std::move(*CI), Err, CompResults, ParentCI));
+
+  if (Err)
+    return std::move(Err);
+
+  Interp->InitPTUSize = Interp->IncrParser->getPTUs().size();
+  return std::move(Interp);
+}
+
+llvm::Expected<std::unique_ptr<Interpreter>>
 Interpreter::createWithCUDA(std::unique_ptr<CompilerInstance> CI,
                             std::unique_ptr<CompilerInstance> DCI) {
   // avoid writing fat binary to disk using an in-memory virtual file system
   llvm::IntrusiveRefCntPtr<llvm::vfs::InMemoryFileSystem> IMVFS =
       std::make_unique<llvm::vfs::InMemoryFileSystem>();
   llvm::IntrusiveRefCntPtr<llvm::vfs::OverlayFileSystem> OverlayVFS =
       std::make_unique<llvm::vfs::OverlayFileSystem>(
           llvm::vfs::getRealFileSystem());
Expr *Interpreter::SynthesizeExpr(Expr *E) {
 
   ExprResult Result = Builder.getCall();
   // It could fail, like printing an array type in C. (not supported)
   if (Result.isInvalid())
     return E;
   return Result.get();
 }
 
+std::string Interpreter::getAllInput() const { return IncrParser->AllInput; }
+
+void Interpreter::CodeComplete(llvm::StringRef Input, size_t Col, size_t Line) {
+  IncrParser->ParseForCodeCompletion(Input, Col, Line);
+}
+
 // Temporary rvalue struct that need special care.
 REPL_EXTERNAL_VISIBILITY void *
 __clang_Interpreter_SetValueWithAlloc(void *This, void *OutVal,
                                       void *OpaqueType) {
   Value &VRef = *(Value *)OutVal;
   VRef = Value(static_cast<Interpreter *>(This), OpaqueType);
   return VRef.getPtr();
 }

clang/tools/clang-repl/ClangRepl.cpp

 //===--- tools/clang-repl/ClangRepl.cpp - clang-repl - the Clang REPL -----===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 //  This file implements a REPL tool on top of clang.
 //
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Interpreter/CodeCompletion.h"
 #include "clang/Interpreter/Interpreter.h"
 
 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
 #include "llvm/LineEditor/LineEditor.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ManagedStatic.h" // llvm_shutdown
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/TargetSelect.h"
for (const std::string &input : OptInputs) {
     if (auto Err = Interp->ParseAndExecute(input))
       llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: ");
   }
 
   bool HasError = false;
 
   if (OptInputs.empty()) {
     llvm::LineEditor LE("clang-repl");
-    // FIXME: Add LE.setListCompleter
     std::string Input;
+    LE.setListCompleter(clang::ReplListCompleter(CB, *Interp));
     while (std::optional<std::string> Line = LE.readLine()) {
       llvm::StringRef L = *Line;
       L = L.trim();
       if (L.endswith("\\")) {
         // FIXME: Support #ifdef X \ ...
         Input += L.drop_back(1);
         LE.setPrompt("clang-repl...   ");
         continue;
       }
 
       Input += L;
-
       if (Input == R"(%quit)") {
         break;
-      } else if (Input == R"(%undo)") {
+      }
+      if (Input == R"(%undo)") {
         if (auto Err = Interp->Undo()) {
           llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: ");
           HasError = true;
         }
       } else if (Input.rfind("%lib ", 0) == 0) {
         if (auto Err = Interp->LoadDynamicLibrary(Input.data() + 5)) {
           llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: ");
           HasError = true;

clang/unittests/Interpreter/CMakeLists.txt

 set(LLVM_LINK_COMPONENTS
   ${LLVM_TARGETS_TO_BUILD}
   Core
   OrcJIT
   Support
   TargetParser
   )
 
 add_clang_unittest(ClangReplInterpreterTests
   IncrementalProcessingTest.cpp
   InterpreterTest.cpp
+  CodeCompletionTest.cpp
   )
 target_link_libraries(ClangReplInterpreterTests PUBLIC
   clangAST
   clangBasic
   clangInterpreter
   clangFrontend
   clangSema
   )
 
 # Exceptions on Windows are not yet supported.
 if(NOT WIN32)
   add_subdirectory(ExceptionTests)
 endif()
 
 export_executable_symbols(ClangReplInterpreterTests)

clang/unittests/Interpreter/CodeCompletionTest.cpp

+#include "clang/Interpreter/CodeCompletion.h"
+#include "clang/Interpreter/Interpreter.h"
+
+#include "llvm/LineEditor/LineEditor.h"
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "llvm/Support/Error.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace clang;
+namespace {
+auto CB = clang::IncrementalCompilerBuilder();
+
+static std::unique_ptr<Interpreter> createInterpreter() {
+  auto CI = cantFail(CB.CreateCpp());
+  return cantFail(clang::Interpreter::create(std::move(CI)));
+}
+
+TEST(CodeCompletionTest, Sanity) {
+  auto Interp = createInterpreter();
+  if (auto R = Interp->ParseAndExecute("int foo = 12;")) {
+    consumeError(std::move(R));
+    return;
+  }
+  auto Completer = ReplListCompleter(CB, *Interp);
+  std::vector<llvm::LineEditor::Completion> comps =
+      Completer(std::string("f"), 1);
+  EXPECT_EQ((size_t)2, comps.size()); // foo and float
+  EXPECT_EQ(comps[0].TypedText, std::string("oo"));
+}
+
+TEST(CodeCompletionTest, SanityNoneValid) {
+  auto Interp = createInterpreter();
+  if (auto R = Interp->ParseAndExecute("int foo = 12;")) {
+    consumeError(std::move(R));
+    return;
+  }
+  auto Completer = ReplListCompleter(CB, *Interp);
+  std::vector<llvm::LineEditor::Completion> comps =
+      Completer(std::string("babanana"), 8);
+  EXPECT_EQ((size_t)0, comps.size()); // foo and float
+}
+
+TEST(CodeCompletionTest, TwoDecls) {
+  auto Interp = createInterpreter();
+  if (auto R = Interp->ParseAndExecute("int application = 12;")) {
+    consumeError(std::move(R));
+    return;
+  }
+  if (auto R = Interp->ParseAndExecute("int apple = 12;")) {
+    consumeError(std::move(R));
+    return;
+  }
+  auto Completer = ReplListCompleter(CB, *Interp);
+  std::vector<llvm::LineEditor::Completion> comps =
+      Completer(std::string("app"), 3);
+  EXPECT_EQ((size_t)2, comps.size());
+}
+} // anonymous namespace