[clang-repl] Add basic multiline input support

[sunho]

Summary

Add basic multi-line input support to clang-repl by using newly available SourceFileGrower interface of Preprocessor.

Compared to cling, this change uses fully fledged clang preprocessor (not MetaLexer) so it supports very complicated macro expressions.

NOTE: this is the related to https://discourse.llvm.org/t/rfc-flexible-lexer-buffering-for-handling-incomplete-input-in-interactive-c-c/64180

[sunho]

This comment was removed by sunho.

[sunho]

This comment was removed by sunho.

[sunho]

Update

[sunho]

Update

[sunho]

Update

[sunho]

Update

[v.g.vassilev]

clang/lib/Interpreter/IncrementalParser.cpp

260

I think we can simplify the heuristics here by asking if the end source location of the last created declaration is the same as the parser/lexer current token that is observed before EOF. If that's the case, then we are done. If not we can request more input.

clang/include/clang/Interpreter/Interpreter.h

 class Interpreter {
   std::unique_ptr<llvm::orc::ThreadSafeContext> TSCtx;
   std::unique_ptr<IncrementalParser> IncrParser;
   std::unique_ptr<IncrementalExecutor> IncrExecutor;
 
   Interpreter(std::unique_ptr<CompilerInstance> CI, llvm::Error &Err);
 
 public:
+  using ReceiveAdditionalLine = std::function<std::optional<std::string>()>;
   ~Interpreter();
   static llvm::Expected<std::unique_ptr<Interpreter>>
   create(std::unique_ptr<CompilerInstance> CI);
   const CompilerInstance *getCompilerInstance() const;
   const llvm::orc::LLJIT *getExecutionEngine() const;
-  llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Code);
+  ///\param RecvLine a callback that will be called to get additional lines
+  /// required to finish parsing.
+  llvm::Expected<PartialTranslationUnit &>
+  Parse(llvm::StringRef Code, ReceiveAdditionalLine RecvLine = nullptr);
   llvm::Error Execute(PartialTranslationUnit &T);
-  llvm::Error ParseAndExecute(llvm::StringRef Code) {
-    auto PTU = Parse(Code);
+  ///\param RecvLine a callback that will be called to get additional lines
+  /// required to finish parsing.
+  llvm::Error ParseAndExecute(llvm::StringRef Code,
+                              ReceiveAdditionalLine RecvLine = nullptr) {
+    auto PTU = Parse(Code, RecvLine);
     if (!PTU)
       return PTU.takeError();
     if (PTU->TheModule)
       return Execute(*PTU);
     return llvm::Error::success();
   }
 
   /// Undo N previous incremental inputs.

clang/lib/Interpreter/IncrementalParser.h

class IncrementalParser {
   /// Counts the number of direct user input lines that have been parsed.
   unsigned InputCount = 0;
 
   /// List containing every information about every incrementally parsed piece
   /// of code.
   std::list<PartialTranslationUnit> PTUs;
 
 public:
+  using ReceiveAdditionalLine = std::function<std::optional<std::string>()>;
   IncrementalParser(std::unique_ptr<CompilerInstance> Instance,
                     llvm::LLVMContext &LLVMCtx, llvm::Error &Err);
   ~IncrementalParser();
 
   const CompilerInstance *getCI() const { return CI.get(); }
 
-  /// Parses incremental input by creating an in-memory file.
-  ///\returns a \c PartialTranslationUnit which holds information about the
+  ///\param RecvLine a callback that will be called to get additional lines
+  /// required to finish parsing. \returns a \c PartialTranslationUnit which
+  /// holds information about the
   /// \c TranslationUnitDecl and \c llvm::Module corresponding to the input.
-  llvm::Expected<PartialTranslationUnit &> Parse(llvm::StringRef Input);
+  llvm::Expected<PartialTranslationUnit &>
+  Parse(llvm::StringRef Input, ReceiveAdditionalLine RecvLine = nullptr);
 
   /// Uses the CodeGenModule mangled name cache and avoids recomputing.
   ///\returns the mangled name of a \c GD.
   llvm::StringRef GetMangledName(GlobalDecl GD) const;
 
   void CleanUpPTU(PartialTranslationUnit &PTU);
 
   std::list<PartialTranslationUnit> &getPTUs() { return PTUs; }
 
 private:
+  llvm::Expected<FileID>
+  ReceiveCompleteSourceInput(ReceiveAdditionalLine RecvLine,
+                             StringRef InitialCode, StringRef SourceName);
   llvm::Expected<PartialTranslationUnit &> ParseOrWrapTopLevelDecl();
 };
 } // end namespace clang
 
 #endif // LLVM_CLANG_LIB_INTERPRETER_INCREMENTALPARSER_H

clang/lib/Interpreter/IncrementalParser.cpp

 //===--------- IncrementalParser.cpp - Incremental Compilation  -----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 //
 // This file implements the class which performs incremental code compilation.
 //
 //===----------------------------------------------------------------------===//
 
 #include "IncrementalParser.h"
 
 #include "clang/AST/DeclContextInternals.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/TokenKinds.h"
 #include "clang/CodeGen/BackendUtil.h"
 #include "clang/CodeGen/CodeGenAction.h"
 #include "clang/CodeGen/ModuleBuilder.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendAction.h"
 #include "clang/FrontendTool/Utils.h"
+#include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorLexer.h"
 #include "clang/Parse/Parser.h"
 #include "clang/Sema/Sema.h"
 
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/CrashRecoveryContext.h"
 #include "llvm/Support/Error.h"
+#include "llvm/Support/MemoryBufferRef.h"
 #include "llvm/Support/Timer.h"
 
 #include <sstream>
 
 namespace clang {
 
 /// A custom action enabling the incremental processing functionality.
 ///

 static CodeGenerator *getCodeGen(FrontendAction *Act) {
   IncrementalAction *IncrAct = static_cast<IncrementalAction *>(Act);
   FrontendAction *WrappedAct = IncrAct->getWrapped();
   if (!WrappedAct->hasIRSupport())
     return nullptr;
   return static_cast<CodeGenAction *>(WrappedAct)->getCodeGenerator();
 }
 
-llvm::Expected<PartialTranslationUnit &>
-IncrementalParser::Parse(llvm::StringRef input) {
-  Preprocessor &PP = CI->getPreprocessor();
-  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
-
-  std::ostringstream SourceName;
-  SourceName << "input_line_" << InputCount++;
-
+static std::unique_ptr<llvm::MemoryBuffer>
+CreateMemoryBuffer(llvm::StringRef SourceName, llvm::StringRef Str) {
   // Create an uninitialized memory buffer, copy code in and append "\n"
-  size_t InputSize = input.size(); // don't include trailing 0
+  size_t StrSize = Str.size(); // don't include trailing 0
   // MemBuffer size should *not* include terminating zero
   std::unique_ptr<llvm::MemoryBuffer> MB(
-      llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1,
-                                                        SourceName.str()));
+      llvm::WritableMemoryBuffer::getNewUninitMemBuffer(StrSize + 1,
+                                                        SourceName));
   char *MBStart = const_cast<char *>(MB->getBufferStart());
-  memcpy(MBStart, input.data(), InputSize);
-  MBStart[InputSize] = '\n';
+  memcpy(MBStart, Str.data(), StrSize);
+  MBStart[StrSize] = '\n';
+  return MB;
+}
 
-  SourceManager &SM = CI->getSourceManager();
+class IncrementalInputReceiver : public SourceFileGrower {
+public:
+  IncrementalInputReceiver(Preprocessor &PP, SourceManager &SM, IncrementalParser::ReceiveAdditionalLine RecvLine): 
+    PP(PP), SM(SM), RecvLine(RecvLine) {
+  }
 
-  // FIXME: Create SourceLocation, which will allow clang to order the overload
-  // candidates for example
-  SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID());
-
-  // Create FileID for the current buffer.
-  FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0,
-                               /*LoadedOffset=*/0, NewLoc);
+  llvm::Expected<FileID>
+  Receive(StringRef InitialCode,
+                                                StringRef SourceName) {
+    // Buffer holding collected source code
+    CodeBuffer = InitialCode.str();
+    CodeBuffer += '\n';
+    const clang::FileEntry* FE
+      = SM.getFileManager().getVirtualFile(SourceName, CodeBuffer.size(),
+                                           0 /* mod time*/);
+    CurFileID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User);
+    PP.setSourceFileGrower(this);
+    SM.overrideFileContents(FE, llvm::MemoryBufferRef(CodeBuffer, ""));
 
-  // NewLoc only used for diags.
-  if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc))
+    SourceLocation NewLoc = SM.getLocForStartOfFile(CurFileID);
+    if (PP.EnterSourceFile(CurFileID, /*DirLookup=*/nullptr, NewLoc))
       return llvm::make_error<llvm::StringError>("Parsing failed. "
                                                  "Cannot enter source file.",
                                                  std::error_code());
+    Token Tok;
+    do {
+      if (Err) {
+        PP.EndSourceFile();
+        PP.setSourceFileGrower(nullptr);
+        return std::move(Err);
+      }
+      PP.Lex(Tok);
+      switch(Tok.getKind()) {
+        case tok::l_brace:
+        case tok::l_paren:
+        case tok::l_square: {
+          BraceLevel[Tok.getKind()]++;
+          break;
+        }
+        case tok::r_brace:
+        case tok::r_paren:
+        case tok::r_square: {
+          if (BraceLevel[ToLBracket(Tok.getKind())] == 0) {
+            PP.EndSourceFile();
+            PP.setSourceFileGrower(nullptr);
+            return llvm::make_error<llvm::StringError>("Parsing failed. "
+                                                      "Unmathced braces.",
+                                                      std::error_code());
+          }
+          BraceLevel[ToLBracket(Tok.getKind())]--;
+          break;
+        }
+        default:
+          break;
+      }
+    } while (Tok.isNot(tok::eof));
+    PP.EndSourceFile();
+    PP.setSourceFileGrower(nullptr);
+
+    return SM.createFileID(CreateMemoryBuffer(SourceName, CodeBuffer),
+                         SrcMgr::C_User, /*LoadedID=*/0,
+                         /*LoadedOffset=*/0);
+  }
+
+  tok::TokenKind ToLBracket(tok::TokenKind Tok) {
+    switch (Tok) {
+      case tok::r_brace:
+        return tok::l_brace;
+      case tok::r_paren:
+        return tok::l_paren;
+      case tok::r_square:
+        return tok::l_square;
+      default:
+        return tok::unknown;
+    }
+  }
+
+  ~IncrementalInputReceiver() = default;
+
+  bool TryGrowFile(FileID FileID) override {
+    if (FileID != CurFileID)
+      return false;
+    if (IsBraceLevelZero())
+      return false;
+     if (auto Line = RecvLine()) {
+        CodeBuffer += *Line;
+        CodeBuffer += '\n';
+        const FileEntry* Entry = SM.getFileEntryForID(CurFileID);
+        assert(Entry);
+        SM.overrideFileContents(Entry, llvm::MemoryBufferRef(CodeBuffer, ""));
+        return true;
+      } 
+      Err = llvm::make_error<llvm::StringError>("Parsing failed. "
+                                                  "Truncated code.",
+                                                  std::error_code());
+      return false;            
+  }
+
+  bool IsBraceLevelZero() {
+    PreprocessorLexer* CurLexer = PP.getCurrentLexer();
+    if (CurLexer && CurLexer->conditional_begin() != CurLexer->conditional_end())
+      return false;
+    for (auto& It : BraceLevel) {
+      if (It.second)
+        return false;
+    }
+    return true;
+  }
+private:
+  Preprocessor &PP;
+  SourceManager &SM;
+  llvm::Error Err = llvm::Error::success();
+  FileID CurFileID;
+  std::string CodeBuffer;
+  IncrementalParser::ReceiveAdditionalLine RecvLine;
+  llvm::DenseMap<unsigned short, unsigned> BraceLevel;
+};
 
+// If RecvLine is not null, this will repeatedly call RecvLine function to fetch
+// the additional lines required to finish a cut-off multiline function
+// definition.
+llvm::Expected<FileID>
+IncrementalParser::ReceiveCompleteSourceInput(ReceiveAdditionalLine RecvLine,
+                                              StringRef InitialCode,
+                                              StringRef SourceName) {
+  IncrementalInputReceiver InputReciever(CI->getPreprocessor(), CI->getSourceManager(), RecvLine);
+  return InputReciever.Receive(InitialCode, SourceName);
+}
+
+llvm::Expected<PartialTranslationUnit &>
+  IncrementalParser::Parse(llvm::StringRef Input,
+                          ReceiveAdditionalLine RecvLine) {
+  Preprocessor &PP = CI->getPreprocessor();
+  assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?");
+
+  auto SourceName = ("input_line_" + llvm::Twine(InputCount++)).str();
+
+  SourceManager &SM = CI->getSourceManager();
+  auto FID = ReceiveCompleteSourceInput(RecvLine, Input, SourceName);
+  if (!FID)
+    return FID.takeError();
+
+  SourceLocation NewLoc = SM.getLocForStartOfFile(*FID);
+  if (PP.EnterSourceFile(*FID, /*DirLookup=*/nullptr, NewLoc))
+    return llvm::make_error<llvm::StringError>("Parsing failed. "
+                                            "Cannot enter source file.",
+                                              std::error_code());
   auto PTU = ParseOrWrapTopLevelDecl();
   if (!PTU)
     return PTU.takeError();
 
   if (PP.getLangOpts().DelayedTemplateParsing) {
     // Microsoft-specific:
     // Late parsed templates can leave unswallowed "macro"-like tokens.
     // They will seriously confuse the Parser when entering the next

clang/lib/Interpreter/Interpreter.cpp

 
 const llvm::orc::LLJIT *Interpreter::getExecutionEngine() const {
   if (IncrExecutor)
     return IncrExecutor->getExecutionEngine();
   return nullptr;
 }
 
 llvm::Expected<PartialTranslationUnit &>
-Interpreter::Parse(llvm::StringRef Code) {
-  return IncrParser->Parse(Code);
+Interpreter::Parse(llvm::StringRef Code, ReceiveAdditionalLine RecvLine) {
+  return IncrParser->Parse(Code, RecvLine);
 }
 
 llvm::Error Interpreter::Execute(PartialTranslationUnit &T) {
   assert(T.TheModule);
   if (!IncrExecutor) {
     const clang::TargetInfo &TI =
         getCompilerInstance()->getASTContext().getTargetInfo();
     llvm::Error Err = llvm::Error::success();

clang/test/Interpreter/multiline-func-macro-brace-error.cpp

+// REQUIRES: host-supports-jit
+// UNSUPPORTED: system-aix
+// RUN: cat %s | clang-repl 2>&1 | FileCheck %s
+// Check invalid brace counts are detected.
+// Brace counting is done to support multiline functions.
+extern "C" int printf(const char *, ...);
+
+#define ___DEFINED_SYMBOL
+
+int test_multiline_brace_count_error() {
+#ifdef ___UNDEFINED_SYMBOL
+  printf("Simple multiline brace count\n");
+#else
+}
+#endif
+}
+// CHECK: error: Parsing failed. Unmathced braces.
+
+#define A 1
+#define B 3
+#define C (A + B)
+#define D C
+
+int test_multiline_brace_count_complex_error() {
+#if A + B - 2 * C + D == 0
+}
+#else
+  printf("Complex multiline brace count\n");
+#endif
+}
+// CHECK-NEXT: error: Parsing failed. Unmathced braces.
+
+% quit
 No newline at end of file

clang/test/Interpreter/multiline-func-macro-brace.cpp

+// REQUIRES: host-supports-jit
+// UNSUPPORTED: system-aix
+// RUN: cat %s | clang-repl | FileCheck %s
+// Check a brace counting is not broken by introduction of macro directives.
+// Brace counting is done to support multiline functions.
+extern "C" int printf(const char *, ...);
+
+#define ___DEFINED_SYMBOL
+
+int test_multiline_brace_count_simple() {
+#ifdef ___UNDEFINED_SYMBOL
+  {
+#else
+  printf("Simple multiline brace count\n");
+#endif
+    return 0;
+  }
+  auto r1 = test_multiline_brace_count_simple();
+  // CHECK: Simple multiline brace count
+
+  int test_multiline_brace_count_defined() {
+#ifdef ___DEFINED_SYMBOL
+    printf("Defined multiline brace count\n");
+#else
+}
+}
+}
+}
+}
+}
+#endif
+    return 0;
+  }
+  auto r2 = test_multiline_brace_count_defined();
+  // CHECK-NEXT: Defined multiline brace count
+
+#define A 1
+#define B 3
+#define C (A + B)
+#define D C
+
+  int test_multiline_brace_count_complex() {
+#if A + B - 2 * C + D == 0
+    printf("Complex multiline brace count\n");
+#else
+  {
+#endif
+    return 0;
+  }
+  auto r3 = test_multiline_brace_count_complex();
+  // CHECK-NEXT: Complex multiline brace count
+
+% quit
 No newline at end of file

clang/test/Interpreter/multiline-func.cpp

+// REQUIRES: host-supports-jit
+// UNSUPPORTED: system-aix
+// RUN: cat %s | clang-repl | FileCheck %s
+// Check a multiline function is parsed and executed correctly.
+extern "C" int printf(const char *, ...);
+int test_multiline_function() {
+  printf("Multiline\n");
+  printf("Function\n");
+  return 0;
+}
+
+auto r1 = test_multiline_function();
+// CHECK: Multiline
+// CHECK-NEXT: Function
+
+% quit
 No newline at end of file

clang/test/Interpreter/multiline-ifdef.cpp

+// REQUIRES: host-supports-jit
+// UNSUPPORTED: system-aix
+// RUN: cat %s | clang-repl 2>&1 | FileCheck %s
+// Check if complex multiline ifdef directive is working.
+extern "C" int printf(const char *, ...);
+
+#define ___DEFINED_SYMBOL
+
+#ifdef ___DEFINED_SYMBOL
+int test_ifdef_one() {
+  printf("Simple multiline ifdef\n");
+}
+#endif
+// CHECK: Simple multiline ifdef
+
+#ifdef ___DEFINED_SYMBOL
+#ifndef ___DEFINED_SYMBOL
+int test_ifdef_two() {
+  printf("Wrong\n");
+}
+#endif
+#ifdef ___DEFINED_SYMBOL
+int test_ifdef_two() {
+  printf("Multiline ifdef two stack\n");
+}
+#endif
+#endif
+// CHECK-NEXT: Multiline ifdef two stack
+
+#ifdef ___DEFINED_SYMBOL
+#ifdef ___DEFINED_SYMBOL
+#ifdef ___DEFINED_SYMBOL
+#ifdef ___DEFINED_SYMBOL
+int test_ifdef_four() {
+  printf("Multiline ifdef four stack\n");
+}
+#endif
+#endif
+#endif
+#endif
+// CHECK-NEXT: Multiline ifdef four stack
+
+%quit
 No newline at end of file

clang/tools/clang-repl/ClangRepl.cpp

while (std::optional<std::string> Line = LE.readLine()) {
       if (*Line == R"(%undo)") {
         if (auto Err = Interp->Undo()) {
           llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: ");
           HasError = true;
         }
         continue;
       }
 
-      if (auto Err = Interp->ParseAndExecute(*Line)) {
+      if (auto Err =
+              Interp->ParseAndExecute(*Line, [&]() { return LE.readLine(); })) {
         llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: ");
         HasError = true;
       }
     }
   }
 
   // Our error handler depends on the Diagnostics object, which we're
   // potentially about to delete. Uninstall the handler now so that any
   // later errors use the default handling behavior instead.
   llvm::remove_fatal_error_handler();
 
   return checkDiagErrors(Interp->getCompilerInstance(), HasError);
 }