diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h --- a/clang/include/clang/Interpreter/Interpreter.h +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -53,15 +53,22 @@ Interpreter(std::unique_ptr CI, llvm::Error &Err); public: + using ReceiveAdditionalLine = std::function()>; ~Interpreter(); static llvm::Expected> create(std::unique_ptr CI); const CompilerInstance *getCompilerInstance() const; const llvm::orc::LLJIT *getExecutionEngine() const; - llvm::Expected Parse(llvm::StringRef Code); + ///\param RecvLine a callback that will be called to get additional lines + /// required to finish parsing. + llvm::Expected + Parse(llvm::StringRef Code, ReceiveAdditionalLine RecvLine = nullptr); llvm::Error Execute(PartialTranslationUnit &T); - llvm::Error ParseAndExecute(llvm::StringRef Code) { - auto PTU = Parse(Code); + ///\param RecvLine a callback that will be called to get additional lines + /// required to finish parsing. + llvm::Error ParseAndExecute(llvm::StringRef Code, + ReceiveAdditionalLine RecvLine = nullptr) { + auto PTU = Parse(Code, RecvLine); if (!PTU) return PTU.takeError(); if (PTU->TheModule) diff --git a/clang/lib/Interpreter/IncrementalParser.h b/clang/lib/Interpreter/IncrementalParser.h --- a/clang/lib/Interpreter/IncrementalParser.h +++ b/clang/lib/Interpreter/IncrementalParser.h @@ -57,16 +57,19 @@ std::list PTUs; public: + using ReceiveAdditionalLine = std::function()>; IncrementalParser(std::unique_ptr Instance, llvm::LLVMContext &LLVMCtx, llvm::Error &Err); ~IncrementalParser(); const CompilerInstance *getCI() const { return CI.get(); } - /// Parses incremental input by creating an in-memory file. - ///\returns a \c PartialTranslationUnit which holds information about the + ///\param RecvLine a callback that will be called to get additional lines + /// required to finish parsing. \returns a \c PartialTranslationUnit which + /// holds information about the /// \c TranslationUnitDecl and \c llvm::Module corresponding to the input. - llvm::Expected Parse(llvm::StringRef Input); + llvm::Expected + Parse(llvm::StringRef Input, ReceiveAdditionalLine RecvLine = nullptr); /// Uses the CodeGenModule mangled name cache and avoids recomputing. ///\returns the mangled name of a \c GD. @@ -77,6 +80,9 @@ std::list &getPTUs() { return PTUs; } private: + llvm::Expected + ReceiveCompleteSourceInput(ReceiveAdditionalLine RecvLine, + StringRef InitialCode, StringRef SourceName); llvm::Expected ParseOrWrapTopLevelDecl(); }; } // end namespace clang diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp --- a/clang/lib/Interpreter/IncrementalParser.cpp +++ b/clang/lib/Interpreter/IncrementalParser.cpp @@ -13,18 +13,24 @@ #include "IncrementalParser.h" #include "clang/AST/DeclContextInternals.h" +#include "clang/Basic/SourceLocation.h" +#include "clang/Basic/TokenKinds.h" #include "clang/CodeGen/BackendUtil.h" #include "clang/CodeGen/CodeGenAction.h" #include "clang/CodeGen/ModuleBuilder.h" #include "clang/Frontend/CompilerInstance.h" #include "clang/Frontend/FrontendAction.h" #include "clang/FrontendTool/Utils.h" +#include "clang/Lex/PPCallbacks.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Lex/PreprocessorLexer.h" #include "clang/Parse/Parser.h" #include "clang/Sema/Sema.h" #include "llvm/Option/ArgList.h" #include "llvm/Support/CrashRecoveryContext.h" #include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBufferRef.h" #include "llvm/Support/Timer.h" #include @@ -213,40 +219,167 @@ return static_cast(WrappedAct)->getCodeGenerator(); } -llvm::Expected -IncrementalParser::Parse(llvm::StringRef input) { - Preprocessor &PP = CI->getPreprocessor(); - assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?"); - - std::ostringstream SourceName; - SourceName << "input_line_" << InputCount++; - +static std::unique_ptr +CreateMemoryBuffer(llvm::StringRef SourceName, llvm::StringRef Str) { // Create an uninitialized memory buffer, copy code in and append "\n" - size_t InputSize = input.size(); // don't include trailing 0 + size_t StrSize = Str.size(); // don't include trailing 0 // MemBuffer size should *not* include terminating zero std::unique_ptr MB( - llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1, - SourceName.str())); + llvm::WritableMemoryBuffer::getNewUninitMemBuffer(StrSize + 1, + SourceName)); char *MBStart = const_cast(MB->getBufferStart()); - memcpy(MBStart, input.data(), InputSize); - MBStart[InputSize] = '\n'; + memcpy(MBStart, Str.data(), StrSize); + MBStart[StrSize] = '\n'; + return MB; +} - SourceManager &SM = CI->getSourceManager(); +class IncrementalInputReceiver : public SourceFileGrower { +public: + IncrementalInputReceiver(Preprocessor &PP, SourceManager &SM, IncrementalParser::ReceiveAdditionalLine RecvLine): + PP(PP), SM(SM), RecvLine(RecvLine) { + } + + llvm::Expected + Receive(StringRef InitialCode, + StringRef SourceName) { + // Buffer holding collected source code + CodeBuffer = InitialCode.str(); + CodeBuffer += '\n'; + const clang::FileEntry* FE + = SM.getFileManager().getVirtualFile(SourceName, CodeBuffer.size(), + 0 /* mod time*/); + CurFileID = SM.createFileID(FE, SourceLocation(), SrcMgr::C_User); + PP.setSourceFileGrower(this); + SM.overrideFileContents(FE, llvm::MemoryBufferRef(CodeBuffer, "")); + + SourceLocation NewLoc = SM.getLocForStartOfFile(CurFileID); + if (PP.EnterSourceFile(CurFileID, /*DirLookup=*/nullptr, NewLoc)) + return llvm::make_error("Parsing failed. " + "Cannot enter source file.", + std::error_code()); + Token Tok; + do { + if (Err) { + PP.EndSourceFile(); + PP.setSourceFileGrower(nullptr); + return std::move(Err); + } + PP.Lex(Tok); + switch(Tok.getKind()) { + case tok::l_brace: + case tok::l_paren: + case tok::l_square: { + BraceLevel[Tok.getKind()]++; + break; + } + case tok::r_brace: + case tok::r_paren: + case tok::r_square: { + if (BraceLevel[ToLBracket(Tok.getKind())] == 0) { + PP.EndSourceFile(); + PP.setSourceFileGrower(nullptr); + return llvm::make_error("Parsing failed. " + "Unmathced braces.", + std::error_code()); + } + BraceLevel[ToLBracket(Tok.getKind())]--; + break; + } + default: + break; + } + } while (Tok.isNot(tok::eof)); + PP.EndSourceFile(); + PP.setSourceFileGrower(nullptr); - // FIXME: Create SourceLocation, which will allow clang to order the overload - // candidates for example - SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID()); + return SM.createFileID(CreateMemoryBuffer(SourceName, CodeBuffer), + SrcMgr::C_User, /*LoadedID=*/0, + /*LoadedOffset=*/0); + } - // Create FileID for the current buffer. - FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0, - /*LoadedOffset=*/0, NewLoc); + tok::TokenKind ToLBracket(tok::TokenKind Tok) { + switch (Tok) { + case tok::r_brace: + return tok::l_brace; + case tok::r_paren: + return tok::l_paren; + case tok::r_square: + return tok::l_square; + default: + return tok::unknown; + } + } - // NewLoc only used for diags. - if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc)) - return llvm::make_error("Parsing failed. " - "Cannot enter source file.", - std::error_code()); + ~IncrementalInputReceiver() = default; + + bool TryGrowFile(FileID FileID) override { + if (FileID != CurFileID) + return false; + if (IsBraceLevelZero()) + return false; + if (auto Line = RecvLine()) { + CodeBuffer += *Line; + CodeBuffer += '\n'; + const FileEntry* Entry = SM.getFileEntryForID(CurFileID); + assert(Entry); + SM.overrideFileContents(Entry, llvm::MemoryBufferRef(CodeBuffer, "")); + return true; + } + Err = llvm::make_error("Parsing failed. " + "Truncated code.", + std::error_code()); + return false; + } + + bool IsBraceLevelZero() { + PreprocessorLexer* CurLexer = PP.getCurrentLexer(); + if (CurLexer && CurLexer->conditional_begin() != CurLexer->conditional_end()) + return false; + for (auto& It : BraceLevel) { + if (It.second) + return false; + } + return true; + } +private: + Preprocessor &PP; + SourceManager &SM; + llvm::Error Err = llvm::Error::success(); + FileID CurFileID; + std::string CodeBuffer; + IncrementalParser::ReceiveAdditionalLine RecvLine; + llvm::DenseMap BraceLevel; +}; +// If RecvLine is not null, this will repeatedly call RecvLine function to fetch +// the additional lines required to finish a cut-off multiline function +// definition. +llvm::Expected +IncrementalParser::ReceiveCompleteSourceInput(ReceiveAdditionalLine RecvLine, + StringRef InitialCode, + StringRef SourceName) { + IncrementalInputReceiver InputReciever(CI->getPreprocessor(), CI->getSourceManager(), RecvLine); + return InputReciever.Receive(InitialCode, SourceName); +} + +llvm::Expected + IncrementalParser::Parse(llvm::StringRef Input, + ReceiveAdditionalLine RecvLine) { + Preprocessor &PP = CI->getPreprocessor(); + assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?"); + + auto SourceName = ("input_line_" + llvm::Twine(InputCount++)).str(); + + SourceManager &SM = CI->getSourceManager(); + auto FID = ReceiveCompleteSourceInput(RecvLine, Input, SourceName); + if (!FID) + return FID.takeError(); + + SourceLocation NewLoc = SM.getLocForStartOfFile(*FID); + if (PP.EnterSourceFile(*FID, /*DirLookup=*/nullptr, NewLoc)) + return llvm::make_error("Parsing failed. " + "Cannot enter source file.", + std::error_code()); auto PTU = ParseOrWrapTopLevelDecl(); if (!PTU) return PTU.takeError(); diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp --- a/clang/lib/Interpreter/Interpreter.cpp +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -210,8 +210,8 @@ } llvm::Expected -Interpreter::Parse(llvm::StringRef Code) { - return IncrParser->Parse(Code); +Interpreter::Parse(llvm::StringRef Code, ReceiveAdditionalLine RecvLine) { + return IncrParser->Parse(Code, RecvLine); } llvm::Error Interpreter::Execute(PartialTranslationUnit &T) { diff --git a/clang/test/Interpreter/multiline-func-macro-brace-error.cpp b/clang/test/Interpreter/multiline-func-macro-brace-error.cpp new file mode 100644 --- /dev/null +++ b/clang/test/Interpreter/multiline-func-macro-brace-error.cpp @@ -0,0 +1,33 @@ +// REQUIRES: host-supports-jit +// UNSUPPORTED: system-aix +// RUN: cat %s | clang-repl 2>&1 | FileCheck %s +// Check invalid brace counts are detected. +// Brace counting is done to support multiline functions. +extern "C" int printf(const char *, ...); + +#define ___DEFINED_SYMBOL + +int test_multiline_brace_count_error() { +#ifdef ___UNDEFINED_SYMBOL + printf("Simple multiline brace count\n"); +#else +} +#endif +} +// CHECK: error: Parsing failed. Unmathced braces. + +#define A 1 +#define B 3 +#define C (A + B) +#define D C + +int test_multiline_brace_count_complex_error() { +#if A + B - 2 * C + D == 0 +} +#else + printf("Complex multiline brace count\n"); +#endif +} +// CHECK-NEXT: error: Parsing failed. Unmathced braces. + +% quit \ No newline at end of file diff --git a/clang/test/Interpreter/multiline-func-macro-brace.cpp b/clang/test/Interpreter/multiline-func-macro-brace.cpp new file mode 100644 --- /dev/null +++ b/clang/test/Interpreter/multiline-func-macro-brace.cpp @@ -0,0 +1,53 @@ +// REQUIRES: host-supports-jit +// UNSUPPORTED: system-aix +// RUN: cat %s | clang-repl | FileCheck %s +// Check a brace counting is not broken by introduction of macro directives. +// Brace counting is done to support multiline functions. +extern "C" int printf(const char *, ...); + +#define ___DEFINED_SYMBOL + +int test_multiline_brace_count_simple() { +#ifdef ___UNDEFINED_SYMBOL + { +#else + printf("Simple multiline brace count\n"); +#endif + return 0; + } + auto r1 = test_multiline_brace_count_simple(); + // CHECK: Simple multiline brace count + + int test_multiline_brace_count_defined() { +#ifdef ___DEFINED_SYMBOL + printf("Defined multiline brace count\n"); +#else +} +} +} +} +} +} +#endif + return 0; + } + auto r2 = test_multiline_brace_count_defined(); + // CHECK-NEXT: Defined multiline brace count + +#define A 1 +#define B 3 +#define C (A + B) +#define D C + + int test_multiline_brace_count_complex() { +#if A + B - 2 * C + D == 0 + printf("Complex multiline brace count\n"); +#else + { +#endif + return 0; + } + auto r3 = test_multiline_brace_count_complex(); + // CHECK-NEXT: Complex multiline brace count + +% quit \ No newline at end of file diff --git a/clang/test/Interpreter/multiline-func.cpp b/clang/test/Interpreter/multiline-func.cpp new file mode 100644 --- /dev/null +++ b/clang/test/Interpreter/multiline-func.cpp @@ -0,0 +1,16 @@ +// REQUIRES: host-supports-jit +// UNSUPPORTED: system-aix +// RUN: cat %s | clang-repl | FileCheck %s +// Check a multiline function is parsed and executed correctly. +extern "C" int printf(const char *, ...); +int test_multiline_function() { + printf("Multiline\n"); + printf("Function\n"); + return 0; +} + +auto r1 = test_multiline_function(); +// CHECK: Multiline +// CHECK-NEXT: Function + +% quit \ No newline at end of file diff --git a/clang/test/Interpreter/multiline-ifdef.cpp b/clang/test/Interpreter/multiline-ifdef.cpp new file mode 100644 --- /dev/null +++ b/clang/test/Interpreter/multiline-ifdef.cpp @@ -0,0 +1,43 @@ +// REQUIRES: host-supports-jit +// UNSUPPORTED: system-aix +// RUN: cat %s | clang-repl 2>&1 | FileCheck %s +// Check if complex multiline ifdef directive is working. +extern "C" int printf(const char *, ...); + +#define ___DEFINED_SYMBOL + +#ifdef ___DEFINED_SYMBOL +int test_ifdef_one() { + printf("Simple multiline ifdef\n"); +} +#endif +// CHECK: Simple multiline ifdef + +#ifdef ___DEFINED_SYMBOL +#ifndef ___DEFINED_SYMBOL +int test_ifdef_two() { + printf("Wrong\n"); +} +#endif +#ifdef ___DEFINED_SYMBOL +int test_ifdef_two() { + printf("Multiline ifdef two stack\n"); +} +#endif +#endif +// CHECK-NEXT: Multiline ifdef two stack + +#ifdef ___DEFINED_SYMBOL +#ifdef ___DEFINED_SYMBOL +#ifdef ___DEFINED_SYMBOL +#ifdef ___DEFINED_SYMBOL +int test_ifdef_four() { + printf("Multiline ifdef four stack\n"); +} +#endif +#endif +#endif +#endif +// CHECK-NEXT: Multiline ifdef four stack + +%quit \ No newline at end of file diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp --- a/clang/tools/clang-repl/ClangRepl.cpp +++ b/clang/tools/clang-repl/ClangRepl.cpp @@ -124,7 +124,8 @@ continue; } - if (auto Err = Interp->ParseAndExecute(*Line)) { + if (auto Err = + Interp->ParseAndExecute(*Line, [&]() { return LE.readLine(); })) { llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); HasError = true; }