Index: clang/include/clang/Interpreter/Interpreter.h =================================================================== --- clang/include/clang/Interpreter/Interpreter.h +++ clang/include/clang/Interpreter/Interpreter.h @@ -53,15 +53,22 @@ Interpreter(std::unique_ptr CI, llvm::Error &Err); public: + using ReceiveAdditionalLine = std::function()>; ~Interpreter(); static llvm::Expected> create(std::unique_ptr CI); const CompilerInstance *getCompilerInstance() const; const llvm::orc::LLJIT *getExecutionEngine() const; - llvm::Expected Parse(llvm::StringRef Code); + ///\param RecvLine a callback that will be called to get additional lines + ///required to finish parsing. + llvm::Expected + Parse(llvm::StringRef Code, ReceiveAdditionalLine RecvLine = nullptr); llvm::Error Execute(PartialTranslationUnit &T); - llvm::Error ParseAndExecute(llvm::StringRef Code) { - auto PTU = Parse(Code); + ///\param RecvLine a callback that will be called to get additional lines + ///required to finish parsing. + llvm::Error ParseAndExecute(llvm::StringRef Code, + ReceiveAdditionalLine RecvLine = nullptr) { + auto PTU = Parse(Code, RecvLine); if (!PTU) return PTU.takeError(); if (PTU->TheModule) Index: clang/lib/Interpreter/IncrementalDiagnosticBuffer.h =================================================================== --- /dev/null +++ clang/lib/Interpreter/IncrementalDiagnosticBuffer.h @@ -0,0 +1,86 @@ +//===-- IncrementalDiagnosticBuffer.h - Incremental Diagnostics --*- C++ +//-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the class which collects diagnostics messages. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INTERPRETER_INCREMENTALDIAGNOSTICBUFFER_H +#define LLVM_CLANG_LIB_INTERPRETER_INCREMENTALDIAGNOSTICBUFFER_H + +#include "clang/Basic/DiagnosticLex.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" + +#include "llvm/Support/Error.h" + +namespace clang { + +/// Collect diagnostics messages to buffer that can be flushed or +/// cleared based on the circumstances. +/// While collecting diagnostics messages, this looks whether +/// certain interesting diagnostic kinds have happend. +class IncrementalDiagnosticBuffer : public DiagnosticConsumer { +public: + IncrementalDiagnosticBuffer(DiagnosticOptions &DiagOpts) + : TextStream(TextBuffer), + TextPrinter( + std::make_unique(TextStream, &DiagOpts)) {} + ~IncrementalDiagnosticBuffer() = default; + + /// Whether err_pp_unterminated_conditional has happend since the last + /// flush/clear. If true, it indicates that #ifdef or #if directive was not + /// finished off by #endif. + bool HasTruncatedConditionalDirective() const { + return TruncatedConditionalDirective; + } + + /// Clear the collected diagnostic messages. + void ClearDiagnostics() { + TruncatedConditionalDirective = false; + TextBuffer.clear(); + } + + /// Get the collected diagnostic messages as a string buffer. + /// This also clears the collected diagnostic message buffer. + std::string FlushDiagnostics() { + std::string Result = std::move(TextBuffer); + ClearDiagnostics(); + return Result; + } + + void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel, + const Diagnostic &Info) override { + switch (Info.getID()) { + case diag::err_pp_unterminated_conditional: { + TruncatedConditionalDirective = true; + break; + } + default: + break; + } + TextPrinter->HandleDiagnostic(DiagLevel, Info); + } + + void BeginSourceFile(const clang::LangOptions &LO, + const clang::Preprocessor *PP) override { + TextPrinter->BeginSourceFile(LO, PP); + } + + void EndSourceFile() override { TextPrinter->EndSourceFile(); } + +private: + bool TruncatedConditionalDirective = false; + std::string TextBuffer; + llvm::raw_string_ostream TextStream; + std::unique_ptr TextPrinter; +}; + +} // namespace clang + +#endif \ No newline at end of file Index: clang/lib/Interpreter/IncrementalParser.h =================================================================== --- clang/lib/Interpreter/IncrementalParser.h +++ clang/lib/Interpreter/IncrementalParser.h @@ -21,6 +21,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Support/Error.h" +#include "IncrementalDiagnosticBuffer.h" + #include #include namespace llvm { @@ -57,16 +59,21 @@ std::list PTUs; public: + using ReceiveAdditionalLine = std::function()>; IncrementalParser(std::unique_ptr Instance, + IncrementalDiagnosticBuffer &DiagsBuffer, llvm::LLVMContext &LLVMCtx, llvm::Error &Err); ~IncrementalParser(); const CompilerInstance *getCI() const { return CI.get(); } /// Parses incremental input by creating an in-memory file. - ///\returns a \c PartialTranslationUnit which holds information about the + ///\param RecvLine a callback that will be called to get additional lines + ///required to finish parsing. \returns a \c PartialTranslationUnit which + ///holds information about the /// \c TranslationUnitDecl and \c llvm::Module corresponding to the input. - llvm::Expected Parse(llvm::StringRef Input); + llvm::Expected + Parse(llvm::StringRef Input, ReceiveAdditionalLine RecvLine = nullptr); /// Uses the CodeGenModule mangled name cache and avoids recomputing. ///\returns the mangled name of a \c GD. @@ -78,6 +85,10 @@ private: llvm::Expected ParseOrWrapTopLevelDecl(); + llvm::Expected + ReceiveCompleteSourceInput(ReceiveAdditionalLine RecvLine, + StringRef InitialCode, StringRef SourceName); + IncrementalDiagnosticBuffer &DiagsBuffer; }; } // end namespace clang Index: clang/lib/Interpreter/IncrementalParser.cpp =================================================================== --- clang/lib/Interpreter/IncrementalParser.cpp +++ clang/lib/Interpreter/IncrementalParser.cpp @@ -120,9 +120,10 @@ }; IncrementalParser::IncrementalParser(std::unique_ptr Instance, + IncrementalDiagnosticBuffer &DiagsBuffer, llvm::LLVMContext &LLVMCtx, llvm::Error &Err) - : CI(std::move(Instance)) { + : CI(std::move(Instance)), DiagsBuffer(DiagsBuffer) { llvm::ErrorAsOutParameter EAO(&Err); Act = std::make_unique(*CI, LLVMCtx, Err); if (Err) @@ -174,9 +175,12 @@ // is due to a top-level semicolon, an action override, or a parse error // skipping something. if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get())) - return llvm::make_error("Parsing failed. " - "The consumer rejected a decl", - std::error_code()); + return llvm::make_error( + (llvm::Twine("Parsing failed. " + "The consumer rejected a decl\n") + + DiagsBuffer.FlushDiagnostics()) + .str(), + std::error_code()); } DiagnosticsEngine &Diags = getCI()->getDiagnostics(); @@ -187,8 +191,10 @@ Diags.Reset(/*soft=*/true); Diags.getClient()->clear(); - return llvm::make_error("Parsing failed.", - std::error_code()); + return llvm::make_error( + (llvm::Twine("Parsing failed. \n") + DiagsBuffer.FlushDiagnostics()) + .str(), + std::error_code()); } // Process any TopLevelDecls generated by #pragma weak. @@ -213,40 +219,119 @@ return static_cast(WrappedAct)->getCodeGenerator(); } -llvm::Expected -IncrementalParser::Parse(llvm::StringRef input) { - Preprocessor &PP = CI->getPreprocessor(); - assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?"); - - std::ostringstream SourceName; - SourceName << "input_line_" << InputCount++; - +static std::unique_ptr +CreateMemoryBuffer(llvm::StringRef SourceName, llvm::StringRef Str) { // Create an uninitialized memory buffer, copy code in and append "\n" - size_t InputSize = input.size(); // don't include trailing 0 + size_t StrSize = Str.size(); // don't include trailing 0 // MemBuffer size should *not* include terminating zero std::unique_ptr MB( - llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1, - SourceName.str())); + llvm::WritableMemoryBuffer::getNewUninitMemBuffer(StrSize + 1, + SourceName)); char *MBStart = const_cast(MB->getBufferStart()); - memcpy(MBStart, input.data(), InputSize); - MBStart[InputSize] = '\n'; + memcpy(MBStart, Str.data(), StrSize); + MBStart[StrSize] = '\n'; + return MB; +} +// If RecvLine is not null, this will repeatedly call RecvLine function to fetch +// the additional lines required to finish a cut-off multiline function +// definition. +llvm::Expected +IncrementalParser::ReceiveCompleteSourceInput(ReceiveAdditionalLine RecvLine, + StringRef InitialCode, + StringRef SourceName) { + // FIXME: Optimize this to O(n) by teaching Lexer to ask for additional bytes. + // FIXME: Support string literal cutoff. + Preprocessor &PP = CI->getPreprocessor(); SourceManager &SM = CI->getSourceManager(); - - // FIXME: Create SourceLocation, which will allow clang to order the overload - // candidates for example + DiagnosticsEngine &Diags = getCI()->getDiagnostics(); SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID()); - // Create FileID for the current buffer. - FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0, - /*LoadedOffset=*/0, NewLoc); + // Buffer holding collected source code + auto CodeBuffer = InitialCode.str(); + CodeBuffer += '\n'; + + while (true) { + // Create FileID for the current buffer. + FileID FID = SM.createFileID(llvm::MemoryBufferRef(CodeBuffer, ""), + SrcMgr::C_User, /*LoadedID=*/0, + /*LoadedOffset=*/0, NewLoc); + if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc)) + return llvm::make_error("Parsing failed. " + "Cannot enter source file.", + std::error_code()); - // NewLoc only used for diags. - if (PP.EnterSourceFile(FID, /*DirLookup=*/nullptr, NewLoc)) + // Count brace level to determine if additional lines are needed. + Token Tok; + unsigned BraceLevel = 0; + do { + PP.Lex(Tok); + if (Tok.getKind() == tok::l_brace) + ++BraceLevel; + else if (Tok.getKind() == tok::r_brace) { + if (BraceLevel == 0) + return llvm::make_error("Parsing failed. " + "Unmathced braces.", + std::error_code()); + --BraceLevel; + } + } while (Tok.isNot(tok::eof)); + PP.EndSourceFile(); + + if (BraceLevel != 0 || DiagsBuffer.HasTruncatedConditionalDirective()) { + // We can't finish off the function definition either because of + // truncated #ifdef/if directive or lacking closing braces. + // Try to get additional lines. + + // If we can't fetch additional lines, this is just an error. + if (!RecvLine) + return llvm::make_error("Parsing failed. " + "Unmathced braces.", + std::error_code()); + + // Clear the diagnostic messages that might contain incorrect + // diagnostics. + Diags.Reset(/*soft=*/true); + Diags.getClient()->clear(); + DiagsBuffer.ClearDiagnostics(); + + // Fetch additional lines. + if (auto Line = RecvLine()) { + CodeBuffer += *Line; + CodeBuffer += '\n'; + } else + return llvm::make_error("Parsing failed. " + "Truncated code.", + std::error_code()); + } else { + // Brace count is sane and it has no truncated #ifdef/if directive. + // We're good to go. + break; + } + } + return SM.createFileID(CreateMemoryBuffer(SourceName, CodeBuffer), + SrcMgr::C_User, /*LoadedID=*/0, + /*LoadedOffset=*/0, NewLoc); +} + +llvm::Expected +IncrementalParser::Parse(llvm::StringRef Input, + ReceiveAdditionalLine RecvLine) { + Preprocessor &PP = CI->getPreprocessor(); + assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?"); + + auto SourceName = ("input_line_" + llvm::Twine(InputCount++)).str(); + + SourceManager &SM = CI->getSourceManager(); + auto FID = ReceiveCompleteSourceInput(RecvLine, Input, SourceName); + if (!FID) + return FID.takeError(); + + SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID()); + if (PP.EnterSourceFile(*FID, /*DirLookup=*/nullptr, NewLoc)) return llvm::make_error("Parsing failed. " "Cannot enter source file.", std::error_code()); - auto PTU = ParseOrWrapTopLevelDecl(); if (!PTU) return PTU.takeError(); Index: clang/lib/Interpreter/Interpreter.cpp =================================================================== --- clang/lib/Interpreter/Interpreter.cpp +++ clang/lib/Interpreter/Interpreter.cpp @@ -74,8 +74,9 @@ // Buffer diagnostics from argument parsing so that we can output them using // a well formed diagnostic object. IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); - TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; - DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); + TextDiagnosticBuffer *TextDiagsBuffer = new TextDiagnosticBuffer; + DiagnosticsEngine Diags(DiagID, &*DiagOpts, TextDiagsBuffer); + bool Success = CompilerInvocation::CreateFromArgs( Clang->getInvocation(), llvm::makeArrayRef(Argv.begin(), Argv.size()), Diags); @@ -87,13 +88,14 @@ CompilerInvocation::GetResourcesPath(Argv[0], nullptr); // Create the actual diagnostics engine. - Clang->createDiagnostics(); + IncrementalDiagnosticBuffer *DiagsBuffer = + new IncrementalDiagnosticBuffer(*DiagOpts); + Clang->createDiagnostics(DiagsBuffer); if (!Clang->hasDiagnostics()) return llvm::createStringError(llvm::errc::not_supported, "Initialization failed. " "Unable to create diagnostics engine"); - - DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); + TextDiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); if (!Success) return llvm::createStringError(llvm::errc::not_supported, "Initialization failed. " @@ -179,8 +181,11 @@ llvm::ErrorAsOutParameter EAO(&Err); auto LLVMCtx = std::make_unique(); TSCtx = std::make_unique(std::move(LLVMCtx)); - IncrParser = std::make_unique(std::move(CI), - *TSCtx->getContext(), Err); + IncrParser = std::make_unique( + std::move(CI), + reinterpret_cast( + CI->getDiagnosticClient()), + *TSCtx->getContext(), Err); } Interpreter::~Interpreter() {} @@ -206,8 +211,8 @@ } llvm::Expected -Interpreter::Parse(llvm::StringRef Code) { - return IncrParser->Parse(Code); +Interpreter::Parse(llvm::StringRef Code, ReceiveAdditionalLine RecvLine) { + return IncrParser->Parse(Code, RecvLine); } llvm::Error Interpreter::Execute(PartialTranslationUnit &T) { Index: clang/test/Interpreter/multiline-func-macro-brace-error.cpp =================================================================== --- /dev/null +++ clang/test/Interpreter/multiline-func-macro-brace-error.cpp @@ -0,0 +1,33 @@ +// REQUIRES: host-supports-jit +// UNSUPPORTED: system-aix +// RUN: cat %s | clang-repl 2>&1 | FileCheck %s +// Check invalid brace counts are detected. +// Brace counting is done to support multiline functions. +extern "C" int printf(const char *, ...); + +#define ___DEFINED_SYMBOL + +int test_multiline_brace_count_error() { +#ifdef ___UNDEFINED_SYMBOL + printf("Simple multiline brace count\n"); +#else +} +#endif +} +// CHECK: error: Parsing failed. Unmathced braces. + +#define A 1 +#define B 3 +#define C (A + B) +#define D C + +int test_multiline_brace_count_complex_error() { +#if A + B - 2 * C + D == 0 +} +#else + printf("Complex multiline brace count\n"); +#endif +} +// CHECK-NEXT: error: Parsing failed. Unmathced braces. + +% quit Index: clang/test/Interpreter/multiline-func-macro-brace.cpp =================================================================== --- /dev/null +++ clang/test/Interpreter/multiline-func-macro-brace.cpp @@ -0,0 +1,53 @@ +// REQUIRES: host-supports-jit +// UNSUPPORTED: system-aix +// RUN: cat %s | clang-repl | FileCheck %s +// Check a brace counting is not broken by introduction of macro directives. +// Brace counting is done to support multiline functions. +extern "C" int printf(const char *, ...); + +#define ___DEFINED_SYMBOL + +int test_multiline_brace_count_simple() { +#ifdef ___UNDEFINED_SYMBOL + { +#else + printf("Simple multiline brace count\n"); +#endif + return 0; + } + auto r1 = test_multiline_brace_count_simple(); + // CHECK: Simple multiline brace count + + int test_multiline_brace_count_defined() { +#ifdef ___DEFINED_SYMBOL + printf("Defined multiline brace count\n"); +#else +} +} +} +} +} +} +#endif + return 0; + } + auto r2 = test_multiline_brace_count_defined(); + // CHECK-NEXT: Defined multiline brace count + +#define A 1 +#define B 3 +#define C (A + B) +#define D C + + int test_multiline_brace_count_complex() { +#if A + B - 2 * C + D == 0 + printf("Complex multiline brace count\n"); +#else + { +#endif + return 0; + } + auto r3 = test_multiline_brace_count_complex(); + // CHECK-NEXT: Complex multiline brace count + + % quit Index: clang/test/Interpreter/multiline-func.cpp =================================================================== --- /dev/null +++ clang/test/Interpreter/multiline-func.cpp @@ -0,0 +1,16 @@ +// REQUIRES: host-supports-jit +// UNSUPPORTED: system-aix +// RUN: cat %s | clang-repl | FileCheck %s +// Check a multiline function is parsed and executed correctly. +extern "C" int printf(const char *, ...); +int test_multiline_function() { + printf("Multiline\n"); + printf("Function\n"); + return 0; +} + +auto r1 = test_multiline_function(); +// CHECK: Multiline +// CHECK-NEXT: Function + +% quit Index: clang/tools/clang-repl/ClangRepl.cpp =================================================================== --- clang/tools/clang-repl/ClangRepl.cpp +++ clang/tools/clang-repl/ClangRepl.cpp @@ -117,7 +117,8 @@ continue; } - if (auto Err = Interp->ParseAndExecute(*Line)) + if (auto Err = + Interp->ParseAndExecute(*Line, [&]() { return LE.readLine(); })) llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); } }