diff --git a/clang/include/clang/CodeGen/CodeGenAction.h b/clang/include/clang/CodeGen/CodeGenAction.h --- a/clang/include/clang/CodeGen/CodeGenAction.h +++ b/clang/include/clang/CodeGen/CodeGenAction.h @@ -19,6 +19,7 @@ namespace clang { class BackendConsumer; +class CodeGenerator; class CodeGenAction : public ASTFrontendAction { private: @@ -77,6 +78,8 @@ /// Take the LLVM context used by this action. llvm::LLVMContext *takeLLVMContext(); + CodeGenerator *getCodeGenerator() const; + BackendConsumer *BEConsumer; }; diff --git a/clang/include/clang/Frontend/FrontendAction.h b/clang/include/clang/Frontend/FrontendAction.h --- a/clang/include/clang/Frontend/FrontendAction.h +++ b/clang/include/clang/Frontend/FrontendAction.h @@ -234,7 +234,7 @@ /// Perform any per-file post processing, deallocate per-file /// objects, and run statistics and output file cleanup code. - void EndSourceFile(); + virtual void EndSourceFile(); /// @} }; @@ -302,15 +302,16 @@ /// some existing action's behavior. It implements every virtual method in /// the FrontendAction interface by forwarding to the wrapped action. class WrapperFrontendAction : public FrontendAction { +protected: std::unique_ptr WrappedAction; -protected: bool PrepareToExecuteAction(CompilerInstance &CI) override; std::unique_ptr CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override; bool BeginInvocation(CompilerInstance &CI) override; bool BeginSourceFileAction(CompilerInstance &CI) override; void ExecuteAction() override; + void EndSourceFile() override; void EndSourceFileAction() override; bool shouldEraseOutputFiles() override; diff --git a/clang/include/clang/Interpreter/Interpreter.h b/clang/include/clang/Interpreter/Interpreter.h new file mode 100644 --- /dev/null +++ b/clang/include/clang/Interpreter/Interpreter.h @@ -0,0 +1,71 @@ +//===--- Interpreter.h - Incremental Compilation and Execution---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the component which performs incremental code +// compilation and execution. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INTERPRETER_INTERPRETER_H +#define LLVM_CLANG_INTERPRETER_INTERPRETER_H + +#include "clang/Interpreter/Transaction.h" + +#include "llvm/Support/Error.h" + +#include +#include + +namespace llvm { +namespace orc { +class ThreadSafeContext; +} +class Module; +} // namespace llvm + +namespace clang { + +class CompilerInstance; +class DeclGroupRef; +class IncrementalExecutor; +class IncrementalParser; + +/// Create a pre-configured \c CompilerInstance for incremental processing. +class IncrementalCompilerBuilder { +public: + static llvm::Expected> + create(std::vector &ClangArgv); +}; + +/// Provides top-level interfaces for incremental compilation and execution. +class Interpreter { + std::unique_ptr TSCtx; + std::unique_ptr IncrParser; + std::unique_ptr IncrExecutor; + + Interpreter(std::unique_ptr CI, llvm::Error &Err); + +public: + ~Interpreter(); + static llvm::Expected> + create(std::unique_ptr CI); + const CompilerInstance *getCompilerInstance() const; + llvm::Expected Parse(llvm::StringRef Code); + llvm::Error Execute(Transaction &T); + llvm::Error ParseAndExecute(llvm::StringRef Code) { + auto ErrOrTransaction = Parse(Code); + if (auto Err = ErrOrTransaction.takeError()) + return Err; + if (ErrOrTransaction->TheModule) + return Execute(*ErrOrTransaction); + return llvm::Error::success(); + } +}; +} // namespace clang + +#endif // LLVM_CLANG_INTERPRETER_INTERPRETER_H diff --git a/clang/include/clang/Interpreter/Transaction.h b/clang/include/clang/Interpreter/Transaction.h new file mode 100644 --- /dev/null +++ b/clang/include/clang/Interpreter/Transaction.h @@ -0,0 +1,39 @@ +//===--- Transaction.h - Incremental Compilation and Execution---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines utilities tracking the incrementally processed pieces of +// code. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INTERPRETER_TRANSACTION_H +#define LLVM_CLANG_INTERPRETER_TRANSACTION_H + +#include +#include + +namespace llvm { +class Module; +} + +namespace clang { + +class DeclGroupRef; + +/// The class keeps track of various objects created as part of processing +/// incremental inputs. +struct Transaction { + /// The decls created for the input. + std::vector Decls; + + /// The llvm IR produced for the input. + std::unique_ptr TheModule; +}; +} // namespace clang + +#endif // LLVM_CLANG_INTERPRETER_TRANSACTION_H diff --git a/clang/lib/CMakeLists.txt b/clang/lib/CMakeLists.txt --- a/clang/lib/CMakeLists.txt +++ b/clang/lib/CMakeLists.txt @@ -25,3 +25,4 @@ add_subdirectory(StaticAnalyzer) add_subdirectory(Format) add_subdirectory(Testing) +add_subdirectory(Interpreter) diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp --- a/clang/lib/CodeGen/CodeGenAction.cpp +++ b/clang/lib/CodeGen/CodeGenAction.cpp @@ -885,6 +885,10 @@ return VMContext; } +CodeGenerator *CodeGenAction::getCodeGenerator() const { + return BEConsumer->getCodeGenerator(); +} + static std::unique_ptr GetOutputStream(CompilerInstance &CI, StringRef InFile, BackendAction Action) { switch (Action) { diff --git a/clang/lib/Frontend/FrontendAction.cpp b/clang/lib/Frontend/FrontendAction.cpp --- a/clang/lib/Frontend/FrontendAction.cpp +++ b/clang/lib/Frontend/FrontendAction.cpp @@ -1087,6 +1087,7 @@ void WrapperFrontendAction::ExecuteAction() { WrappedAction->ExecuteAction(); } +void WrapperFrontendAction::EndSourceFile() { WrappedAction->EndSourceFile(); } void WrapperFrontendAction::EndSourceFileAction() { WrappedAction->EndSourceFileAction(); } diff --git a/clang/lib/Interpreter/CMakeLists.txt b/clang/lib/Interpreter/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang/lib/Interpreter/CMakeLists.txt @@ -0,0 +1,22 @@ +set(LLVM_LINK_COMPONENTS + core + native + OrcJit + Target + ) + +add_clang_library(clangInterpreter + IncrementalExecutor.cpp + IncrementalParser.cpp + Interpreter.cpp + + LINK_LIBS + clangAST + clangAnalysis + clangBasic + clangEdit + clangLex + clangSema + clangCodeGen + clangFrontendTool + ) diff --git a/clang/lib/Interpreter/IncrementalExecutor.h b/clang/lib/Interpreter/IncrementalExecutor.h new file mode 100644 --- /dev/null +++ b/clang/lib/Interpreter/IncrementalExecutor.h @@ -0,0 +1,46 @@ +//===--- IncrementalExecutor.h - Incremental Execution ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the class which performs incremental code execution. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H +#define LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" + +#include + +namespace llvm { +class Error; +class Module; +namespace orc { +class LLJIT; +class ThreadSafeContext; +} // namespace orc +} // namespace llvm + +namespace clang { +class IncrementalExecutor { + using CtorDtorIterator = llvm::orc::CtorDtorIterator; + std::unique_ptr Jit; + llvm::orc::ThreadSafeContext &TSCtx; + +public: + IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, llvm::Error &Err); + ~IncrementalExecutor(); + + llvm::Error addModule(std::unique_ptr M); + llvm::Error runCtors() const; +}; + +} // end namespace clang + +#endif // LLVM_CLANG_LIB_INTERPRETER_INCREMENTALEXECUTOR_H diff --git a/clang/lib/Interpreter/IncrementalExecutor.cpp b/clang/lib/Interpreter/IncrementalExecutor.cpp new file mode 100644 --- /dev/null +++ b/clang/lib/Interpreter/IncrementalExecutor.cpp @@ -0,0 +1,61 @@ +//===--- IncrementalExecutor.cpp - Incremental Execution --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the class which performs incremental code execution. +// +//===----------------------------------------------------------------------===// + +#include "IncrementalExecutor.h" + +#include "llvm/ExecutionEngine/ExecutionEngine.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/TargetSelect.h" + +namespace clang { + +IncrementalExecutor::IncrementalExecutor(llvm::orc::ThreadSafeContext &TSC, + llvm::Error &Err) + : TSCtx(TSC) { + using namespace llvm::orc; + llvm::ErrorAsOutParameter EAO(&Err); + + if (auto JitOrErr = LLJITBuilder().create()) + Jit = std::move(*JitOrErr); + else { + Err = JitOrErr.takeError(); + return; + } + + const char Pref = Jit->getDataLayout().getGlobalPrefix(); + // Discover symbols from the process as a fallback. + if (auto PSGOrErr = DynamicLibrarySearchGenerator::GetForCurrentProcess(Pref)) + Jit->getMainJITDylib().addGenerator(std::move(*PSGOrErr)); + else { + Err = PSGOrErr.takeError(); + return; + } +} + +IncrementalExecutor::~IncrementalExecutor() {} + +llvm::Error IncrementalExecutor::addModule(std::unique_ptr M) { + return Jit->addIRModule(llvm::orc::ThreadSafeModule(std::move(M), TSCtx)); +} + +llvm::Error IncrementalExecutor::runCtors() const { + return Jit->initialize(Jit->getMainJITDylib()); +} + +} // end namespace clang diff --git a/clang/lib/Interpreter/IncrementalParser.h b/clang/lib/Interpreter/IncrementalParser.h new file mode 100644 --- /dev/null +++ b/clang/lib/Interpreter/IncrementalParser.h @@ -0,0 +1,77 @@ +//===--- IncrementalParser.h - Incremental Compilation ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the class which performs incremental code compilation. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_LIB_INTERPRETER_INCREMENTALPARSER_H +#define LLVM_CLANG_LIB_INTERPRETER_INCREMENTALPARSER_H + +#include "clang/Interpreter/Transaction.h" + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +#include +#include +namespace llvm { +class LLVMContext; +} + +namespace clang { +class ASTConsumer; +class CompilerInstance; +class CodeGenerator; +class DeclGroupRef; +class FrontendAction; +class IncrementalAction; +class Parser; + +/// Provides support for incremental compilation. Keeps track of the state +/// changes between the subsequent incremental input. +/// +class IncrementalParser { + /// Long-lived, incremental parsing action. + std::unique_ptr Act; + + /// Compiler instance performing the incremental compilation. + std::unique_ptr CI; + + /// Parser. + std::unique_ptr P; + + /// Consumer to process the produced top level decls. Owned by Act. + ASTConsumer *Consumer = nullptr; + + /// Counts the number of direct user input lines that have been parsed. + unsigned InputCount = 0; + + /// List containing every information about every incrementally parsed piece + /// of code. + std::list Transactions; + +public: + IncrementalParser(std::unique_ptr Instance, + llvm::LLVMContext &LLVMCtx, llvm::Error &Err); + ~IncrementalParser(); + + const CompilerInstance *getCI() const { return CI.get(); } + + /// Parses incremental input by creating an in-memory file. + ///\returns a \c Transaction which holds information about the \c Decls and + /// \c llvm::Module corresponding to the input. + llvm::Expected Parse(llvm::StringRef Input); + +private: + llvm::Expected ParseOrWrapTopLevelDecl(); +}; +} // end namespace clang + +#endif // LLVM_CLANG_LIB_INTERPRETER_INCREMENTALPARSER_H diff --git a/clang/lib/Interpreter/IncrementalParser.cpp b/clang/lib/Interpreter/IncrementalParser.cpp new file mode 100644 --- /dev/null +++ b/clang/lib/Interpreter/IncrementalParser.cpp @@ -0,0 +1,254 @@ +//===--------- IncrementalParser.cpp - Incremental Compilation -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the class which performs incremental code compilation. +// +//===----------------------------------------------------------------------===// + +#include "IncrementalParser.h" + +#include "clang/CodeGen/BackendUtil.h" +#include "clang/CodeGen/CodeGenAction.h" +#include "clang/CodeGen/ModuleBuilder.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendAction.h" +#include "clang/FrontendTool/Utils.h" +#include "clang/Parse/Parser.h" +#include "clang/Sema/Sema.h" + +#include "llvm/Option/ArgList.h" +#include "llvm/Support/CrashRecoveryContext.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/Timer.h" + +#include + +namespace clang { + +/// A custom action enabling the incremental processing functionality. +/// +/// The usual \p FrontendAction expects one call to ExecuteAction and once it +/// sees a call to \p EndSourceFile it deletes some of the important objects +/// such as \p Preprocessor and \p Sema assuming no further input will come. +/// +/// \p IncrementalAction ensures it keep its underlying action's objects alive +/// as long as the \p IncrementalParser needs them. +/// +class IncrementalAction : public WrapperFrontendAction { +private: + bool IsTerminating = false; + +public: + IncrementalAction(CompilerInstance &CI, llvm::LLVMContext &LLVMCtx, + llvm::Error &Err) + : WrapperFrontendAction([&]() { + llvm::ErrorAsOutParameter EAO(&Err); + std::unique_ptr Act; + switch (CI.getFrontendOpts().ProgramAction) { + default: + Err = llvm::createStringError( + std::errc::state_not_recoverable, + "Driver initialization failed. " + "Incremental mode for action is not supported"); + return Act; + case frontend::ASTDump: + LLVM_FALLTHROUGH; + case frontend::ASTPrint: + LLVM_FALLTHROUGH; + case frontend::ParseSyntaxOnly: + Act = CreateFrontendAction(CI); + break; + case frontend::EmitObj: + LLVM_FALLTHROUGH; + case frontend::EmitLLVMOnly: + Act.reset(new EmitLLVMOnlyAction(&LLVMCtx)); + break; + } + return Act; + }()) {} + FrontendAction *getWrapped() const { return WrappedAction.get(); } + void ExecuteAction() override { + CompilerInstance &CI = getCompilerInstance(); + assert(CI.hasPreprocessor() && "No PP!"); + + // FIXME: Move the truncation aspect of this into Sema, we delayed this till + // here so the source manager would be initialized. + if (hasCodeCompletionSupport() && + !CI.getFrontendOpts().CodeCompletionAt.FileName.empty()) + CI.createCodeCompletionConsumer(); + + // Use a code completion consumer? + CodeCompleteConsumer *CompletionConsumer = nullptr; + if (CI.hasCodeCompletionConsumer()) + CompletionConsumer = &CI.getCodeCompletionConsumer(); + + Preprocessor &PP = CI.getPreprocessor(); + PP.enableIncrementalProcessing(); + PP.EnterMainSourceFile(); + + if (!CI.hasSema()) + CI.createSema(getTranslationUnitKind(), CompletionConsumer); + } + + // Do not terminate after processing the input. This allows us to keep various + // clang objects alive and to incrementally grow the current TU. + void EndSourceFile() override { + // The WrappedAction can be nullptr if we issued an error in the ctor. + if (IsTerminating && getWrapped()) + WrapperFrontendAction::EndSourceFile(); + } + + void FinalizeAction() { + assert(!IsTerminating && "Already finalized!"); + IsTerminating = true; + EndSourceFile(); + } +}; + +IncrementalParser::IncrementalParser(std::unique_ptr Instance, + llvm::LLVMContext &LLVMCtx, + llvm::Error &Err) + : CI(std::move(Instance)) { + llvm::ErrorAsOutParameter EAO(&Err); + Act = std::make_unique(*CI, LLVMCtx, Err); + if (Err) + return; + CI->ExecuteAction(*Act); + Consumer = &CI->getASTConsumer(); + P.reset( + new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies=*/false)); + P->Initialize(); +} + +IncrementalParser::~IncrementalParser() { Act->FinalizeAction(); } + +llvm::Expected IncrementalParser::ParseOrWrapTopLevelDecl() { + DiagnosticsEngine &Diags = getCI()->getDiagnostics(); + + if (Diags.hasErrorOccurred()) + llvm::report_fatal_error("Previous input had errors, " + "recovery not yet supported", + /*GenCrashDiag=*/false); + + // Recover resources if we crash before exiting this method. + Sema &S = CI->getSema(); + llvm::CrashRecoveryContextCleanupRegistrar CleanupSema(&S); + Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled=*/true); + Sema::LocalEagerInstantiationScope LocalInstantiations(S); + + // Skip previous eof due to last incremental input. + if (P->getCurToken().is(tok::eof)) + P->ConsumeToken(); + + Transactions.emplace_back(Transaction()); + Transaction &LastTransaction = Transactions.back(); + + Parser::DeclGroupPtrTy ADecl; + for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl); !AtEOF; + AtEOF = P->ParseTopLevelDecl(ADecl)) { + // If we got a null return and something *was* parsed, ignore it. This + // is due to a top-level semicolon, an action override, or a parse error + // skipping something. + if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get())) + return llvm::make_error("Parsing failed. " + "The consumer rejected a decl", + std::error_code()); + LastTransaction.Decls.push_back(ADecl.get()); + } + + // Process any TopLevelDecls generated by #pragma weak. + for (Decl *D : S.WeakTopLevelDecls()) { + DeclGroupRef DGR(D); + LastTransaction.Decls.push_back(DGR); + Consumer->HandleTopLevelDecl(DGR); + } + + LocalInstantiations.perform(); + GlobalInstantiations.perform(); + + Consumer->HandleTranslationUnit(S.getASTContext()); + + if (Diags.hasErrorOccurred()) + return llvm::make_error("Parsing failed.", + std::error_code()); + + return LastTransaction; +} + +static CodeGenerator *getCodeGen(FrontendAction *Act) { + IncrementalAction *IncrAct = static_cast(Act); + FrontendAction *WrappedAct = IncrAct->getWrapped(); + if (!WrappedAct->hasIRSupport()) + return nullptr; + return static_cast(WrappedAct)->getCodeGenerator(); +} + +llvm::Expected IncrementalParser::Parse(llvm::StringRef input) { + Preprocessor &PP = CI->getPreprocessor(); + assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?"); + + std::ostringstream SourceName; + SourceName << "input_line_" << InputCount++; + + // Create an uninitialized memory buffer, copy code in and append "\n" + size_t InputSize = input.size(); // don't include trailing 0 + // MemBuffer size should *not* include terminating zero + std::unique_ptr MB( + llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1, + SourceName.str())); + char *MBStart = const_cast(MB->getBufferStart()); + memcpy(MBStart, input.data(), InputSize); + MBStart[InputSize] = '\n'; + + SourceManager &SM = CI->getSourceManager(); + + // FIXME: Create SourceLocation, which will allow clang to order the overload + // candidates for example + SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID()); + + // Create FileID for the current buffer. + FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID=*/0, + /*LoadedOffset=*/0, NewLoc); + + // NewLoc only used for diags. + if (PP.EnterSourceFile(FID, /*DirLookup=*/0, NewLoc)) + return llvm::make_error("Parsing failed. " + "Cannot enter source file.", + std::error_code()); + + auto ErrOrTransaction = ParseOrWrapTopLevelDecl(); + if (auto Err = ErrOrTransaction.takeError()) + return std::move(Err); + + if (PP.getLangOpts().DelayedTemplateParsing) { + // Microsoft-specific: + // Late parsed templates can leave unswallowed "macro"-like tokens. + // They will seriously confuse the Parser when entering the next + // source file. So lex until we are EOF. + Token Tok; + do { + PP.Lex(Tok); + } while (Tok.isNot(tok::eof)); + } + + Token AssertTok; + PP.Lex(AssertTok); + assert(AssertTok.is(tok::eof) && + "Lexer must be EOF when starting incremental parse!"); + + if (CodeGenerator *CG = getCodeGen(Act.get())) { + std::unique_ptr M(CG->ReleaseModule()); + CG->StartModule("incr_module_" + std::to_string(Transactions.size()), + M->getContext()); + + ErrOrTransaction->TheModule = std::move(M); + } + + return ErrOrTransaction; +} +} // end namespace clang diff --git a/clang/lib/Interpreter/Interpreter.cpp b/clang/lib/Interpreter/Interpreter.cpp new file mode 100644 --- /dev/null +++ b/clang/lib/Interpreter/Interpreter.cpp @@ -0,0 +1,220 @@ +//===------ Interpreter.cpp - Incremental Compilation and Execution -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the component which performs incremental code +// compilation and execution. +// +//===----------------------------------------------------------------------===// + +#include "clang/Interpreter/Interpreter.h" + +#include "IncrementalExecutor.h" +#include "IncrementalParser.h" + +#include "clang/Basic/TargetInfo.h" +#include "clang/CodeGen/ModuleBuilder.h" +#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Job.h" +#include "clang/Driver/Options.h" +#include "clang/Driver/Tool.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/TextDiagnosticBuffer.h" +#include "clang/Lex/PreprocessorOptions.h" + +#include "llvm/IR/Module.h" +#include "llvm/Support/Host.h" + +using namespace clang; + +// FIXME: Figure out how to unify with namespace init_convenience from +// tools/clang-import-test/clang-import-test.cpp and +// examples/clang-interpreter/main.cpp +namespace { +/// Retrieves the clang CC1 specific flags out of the compilation's jobs. +/// \returns NULL on error. +static llvm::Expected +GetCC1Arguments(DiagnosticsEngine *Diagnostics, + driver::Compilation *Compilation) { + // We expect to get back exactly one Command job, if we didn't something + // failed. Extract that job from the Compilation. + const driver::JobList &Jobs = Compilation->getJobs(); + if (!Jobs.size() || !isa(*Jobs.begin())) + return llvm::createStringError(std::errc::state_not_recoverable, + "Driver initialization failed. " + "Unable to create a driver job"); + + // The one job we find should be to invoke clang again. + const driver::Command *Cmd = cast(&(*Jobs.begin())); + if (llvm::StringRef(Cmd->getCreator().getName()) != "clang") + return llvm::createStringError(std::errc::state_not_recoverable, + "Driver initialization failed"); + + return &Cmd->getArguments(); +} + +static llvm::Expected> +CreateCI(const llvm::opt::ArgStringList &Argv) { + std::unique_ptr Clang(new CompilerInstance()); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + + // Register the support for object-file-wrapped Clang modules. + // FIXME: Clang should register these container operations automatically. + auto PCHOps = Clang->getPCHContainerOperations(); + PCHOps->registerWriter(std::make_unique()); + PCHOps->registerReader(std::make_unique()); + + // Buffer diagnostics from argument parsing so that we can output them using + // a well formed diagnostic object. + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); + bool Success = CompilerInvocation::CreateFromArgs( + Clang->getInvocation(), llvm::makeArrayRef(Argv.begin(), Argv.size()), + Diags); + + // Infer the builtin include path if unspecified. + if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && + Clang->getHeaderSearchOpts().ResourceDir.empty()) + Clang->getHeaderSearchOpts().ResourceDir = + CompilerInvocation::GetResourcesPath(Argv[0], nullptr); + + // Create the actual diagnostics engine. + Clang->createDiagnostics(); + if (!Clang->hasDiagnostics()) + return llvm::createStringError(std::errc::state_not_recoverable, + "Initialization failed. " + "Unable to create diagnostics engine"); + + DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); + if (!Success) + return llvm::createStringError(std::errc::state_not_recoverable, + "Initialization failed. " + "Unable to flush diagnostics"); + + // FIXME: Merge with CompilerInstance::ExecuteAction. + llvm::MemoryBuffer *MB = llvm::MemoryBuffer::getMemBuffer("").release(); + Clang->getPreprocessorOpts().addRemappedFile("<<< inputs >>>", MB); + + Clang->setTarget(TargetInfo::CreateTargetInfo( + Clang->getDiagnostics(), Clang->getInvocation().TargetOpts)); + if (!Clang->hasTarget()) + return llvm::createStringError(std::errc::state_not_recoverable, + "Initialization failed. " + "Target is missing"); + + Clang->getTarget().adjust(Clang->getLangOpts()); + + return std::move(Clang); +} + +} // anonymous namespace + +llvm::Expected> +IncrementalCompilerBuilder::create(std::vector &ClangArgv) { + + // If we don't know ClangArgv0 or the address of main() at this point, try + // to guess it anyway (it's possible on some platforms). + std::string MainExecutableName = + llvm::sys::fs::getMainExecutable(nullptr, nullptr); + + ClangArgv.insert(ClangArgv.begin(), MainExecutableName.c_str()); + + // Prepending -c to force the driver to do something if no action was + // specified. By prepending we allow users to override the default + // action and use other actions in incremental mode. + // FIXME: Print proper driver diagnostics if the driver flags are wrong. + ClangArgv.insert(ClangArgv.begin() + 1, "-c"); + + if (!llvm::is_contained(ClangArgv, " -x")) { + // We do C++ by default; append right after argv[0] if no "-x" given + ClangArgv.push_back("-x"); + ClangArgv.push_back("c++"); + } + + // Put a dummy C++ file on to ensure there's at least one compile job for the + // driver to construct. + ClangArgv.push_back("<<< inputs >>>"); + + CompilerInvocation Invocation; + // Buffer diagnostics from argument parsing so that we can output them using a + // well formed diagnostic object. + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); + unsigned MissingArgIndex, MissingArgCount; + const llvm::opt::OptTable &Opts = driver::getDriverOptTable(); + llvm::opt::InputArgList ParsedArgs = + Opts.ParseArgs(ArrayRef(ClangArgv).slice(1), + MissingArgIndex, MissingArgCount); + ParseDiagnosticArgs(*DiagOpts, ParsedArgs, &Diags); + + driver::Driver Driver(/*MainBinaryName=*/ClangArgv[0], + llvm::sys::getDefaultTargetTriple(), Diags); + Driver.setCheckInputsExist(false); // the input comes from mem buffers + llvm::ArrayRef RF = llvm::makeArrayRef(ClangArgv); + std::unique_ptr Compilation(Driver.BuildCompilation(RF)); + + if (Compilation->getArgs().hasArg(driver::options::OPT_v)) + Compilation->getJobs().Print(llvm::errs(), "\n", /*Quote=*/false); + + auto ErrOrCC1Args = GetCC1Arguments(&Diags, Compilation.get()); + if (auto Err = ErrOrCC1Args.takeError()) + return std::move(Err); + + return CreateCI(**ErrOrCC1Args); +} + +Interpreter::Interpreter(std::unique_ptr CI, + llvm::Error &Err) { + llvm::ErrorAsOutParameter EAO(&Err); + auto LLVMCtx = std::make_unique(); + TSCtx = std::make_unique(std::move(LLVMCtx)); + IncrParser = std::make_unique(std::move(CI), + *TSCtx->getContext(), Err); +} + +Interpreter::~Interpreter() {} + +llvm::Expected> +Interpreter::create(std::unique_ptr CI) { + llvm::Error Err = llvm::Error::success(); + auto Interp = + std::unique_ptr(new Interpreter(std::move(CI), Err)); + if (Err) + return std::move(Err); + return std::move(Interp); +} + +const CompilerInstance *Interpreter::getCompilerInstance() const { + return IncrParser->getCI(); +} + +llvm::Expected Interpreter::Parse(llvm::StringRef Code) { + return IncrParser->Parse(Code); +} + +llvm::Error Interpreter::Execute(Transaction &T) { + assert(T.TheModule); + if (!IncrExecutor) { + llvm::Error Err = llvm::Error::success(); + IncrExecutor = std::make_unique(*TSCtx, Err); + if (Err) + return Err; + } + // FIXME: Add a callback to retain the llvm::Module once the JIT is done. + if (auto Err = IncrExecutor->addModule(std::move(T.TheModule))) + return Err; + + if (auto Err = IncrExecutor->runCtors()) + return Err; + + return llvm::Error::success(); +} diff --git a/clang/test/CMakeLists.txt b/clang/test/CMakeLists.txt --- a/clang/test/CMakeLists.txt +++ b/clang/test/CMakeLists.txt @@ -68,6 +68,7 @@ clang-import-test clang-rename clang-refactor + clang-repl clang-diff clang-scan-deps diagtool diff --git a/clang/test/Interpreter/execute.cpp b/clang/test/Interpreter/execute.cpp new file mode 100644 --- /dev/null +++ b/clang/test/Interpreter/execute.cpp @@ -0,0 +1,14 @@ +// RUN: cat %s | clang-repl | FileCheck %s +// REQUIRES: host-supports-jit + +extern "C" int printf(const char *, ...); +int i = 42; +auto r1 = printf("i = %d\n", i); +// CHECK: i = 42 + +struct S { float f = 1.0; S *m = nullptr;} s; + +auto r2 = printf("S[f=%f, m=0x%llx]\n", s.f, reinterpret_cast(s.m)); +// CHECK-NEXT: S[f=1.000000, m=0x0] + +quit diff --git a/clang/test/Interpreter/sanity.c b/clang/test/Interpreter/sanity.c new file mode 100644 --- /dev/null +++ b/clang/test/Interpreter/sanity.c @@ -0,0 +1,18 @@ +// RUN: cat %s | \ +// RUN: clang-repl -Xcc -fno-color-diagnostics -Xcc -Xclang -Xcc -ast-dump \ +// RUN: -Xcc -Xclang -Xcc -ast-dump-filter -Xcc -Xclang -Xcc Test 2>&1| \ +// RUN: FileCheck %s + +int TestVar = 12; +// CHECK: Dumping TestVar: +// CHECK-NEXT: VarDecl [[var_ptr:0x[0-9a-f]+]] <{{.*}} TestVar 'int' cinit +// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 12 + +void TestFunc() { ++TestVar; } +// CHECK: Dumping TestFunc: +// CHECK-NEXT: FunctionDecl {{.*}} TestFunc 'void ()' +// CHECK-NEXT: CompoundStmt{{.*}} +// CHECK-NEXT: UnaryOperator{{.*}} 'int' lvalue prefix '++' +// CHECK-NEXT: DeclRefExpr{{.*}} 'int' lvalue Var [[var_ptr]] 'TestVar' 'int' + +quit diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py --- a/clang/test/lit.cfg.py +++ b/clang/test/lit.cfg.py @@ -63,7 +63,7 @@ tool_dirs = [config.clang_tools_dir, config.llvm_tools_dir] tools = [ - 'apinotes-test', 'c-index-test', 'clang-diff', 'clang-format', + 'apinotes-test', 'c-index-test', 'clang-diff', 'clang-format', 'clang-repl', 'clang-tblgen', 'opt', 'llvm-ifs', 'yaml2obj', ToolSubst('%clang_extdef_map', command=FindTool( 'clang-extdef-mapping'), unresolved='ignore'), @@ -73,6 +73,28 @@ config.available_features.add('examples') tools.append('clang-interpreter') +def have_host_jit_support(): + clang_repl_exe = lit.util.which('clang-repl', config.clang_tools_dir) + + if not clang_repl_exe: + print('clang-repl not found') + return False + + try: + clang_repl_cmd = subprocess.Popen( + [clang_repl_exe, '--host-supports-jit'], stdout=subprocess.PIPE) + except OSError: + print('could not exec clang-repl') + return False + + clang_repl_out = clang_repl_cmd.stdout.read().decode('ascii') + clang_repl_cmd.wait() + + return 'true' in clang_repl_out + +if have_host_jit_support(): + config.available_features.add('host-supports-jit') + if config.clang_staticanalyzer: config.available_features.add('staticanalyzer') tools.append('clang-check') diff --git a/clang/tools/CMakeLists.txt b/clang/tools/CMakeLists.txt --- a/clang/tools/CMakeLists.txt +++ b/clang/tools/CMakeLists.txt @@ -11,6 +11,7 @@ add_clang_subdirectory(clang-offload-bundler) add_clang_subdirectory(clang-offload-wrapper) add_clang_subdirectory(clang-scan-deps) +add_clang_subdirectory(clang-repl) add_clang_subdirectory(c-index-test) diff --git a/clang/tools/clang-repl/CMakeLists.txt b/clang/tools/clang-repl/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang/tools/clang-repl/CMakeLists.txt @@ -0,0 +1,19 @@ +set( LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + Option + Support + ) + +add_clang_executable(clang-repl + EXCLUDE_FROM_ALL + ClangRepl.cpp + ) + +target_link_libraries(clang-repl PUBLIC + clangInterpreter + clangTooling + LLVMLineEditor + ) + +install(TARGETS clang-repl + RUNTIME DESTINATION bin) diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp new file mode 100644 --- /dev/null +++ b/clang/tools/clang-repl/ClangRepl.cpp @@ -0,0 +1,98 @@ +//===--- tools/clang-repl/ClangRepl.cpp - clang-repl - the Clang REPL -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a REPL tool on top of clang. +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Diagnostic.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Interpreter/Interpreter.h" + +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/LineEditor/LineEditor.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ManagedStatic.h" // llvm_shutdown +#include "llvm/Support/Signals.h" +#include "llvm/Support/TargetSelect.h" // llvm::Initialize* + +static llvm::cl::list + ClangArgs("Xcc", llvm::cl::ZeroOrMore, + llvm::cl::desc("Argument to pass to the CompilerInvocation"), + llvm::cl::CommaSeparated); +static llvm::cl::opt OptHostSupportsJit("host-supports-jit", + llvm::cl::Hidden); + +static void LLVMErrorHandler(void *UserData, const std::string &Message, + bool GenCrashDiag) { + auto &Diags = *static_cast(UserData); + + Diags.Report(clang::diag::err_fe_error_backend) << Message; + + // Run the interrupt handlers to make sure any special cleanups get done, in + // particular that we remove files registered with RemoveFileOnSignal. + llvm::sys::RunInterruptHandlers(); + + // We cannot recover from llvm errors. When reporting a fatal error, exit + // with status 70 to generate crash diagnostics. For BSD systems this is + // defined as an internal software error. Otherwise, exit with status 1. + + exit(GenCrashDiag ? 70 : 1); +} + +llvm::ExitOnError ExitOnErr; +int main(int argc, const char **argv) { + ExitOnErr.setBanner("clang-repl: "); + llvm::cl::ParseCommandLineOptions(argc, argv); + + std::vector ClangArgv(ClangArgs.size()); + std::transform(ClangArgs.begin(), ClangArgs.end(), ClangArgv.begin(), + [](const std::string &s) -> const char * { return s.data(); }); + llvm::InitializeNativeTarget(); + llvm::InitializeNativeTargetAsmPrinter(); + + if (OptHostSupportsJit) { + auto J = llvm::orc::LLJITBuilder().create(); + if (J) + llvm::outs() << "true\n"; + else { + llvm::consumeError(J.takeError()); + llvm::outs() << "false\n"; + } + return 0; + } + + // FIXME: Investigate if we could use runToolOnCodeWithArgs from tooling. It + // can replace the boilerplate code for creation of the compiler instance. + auto CI = ExitOnErr(clang::IncrementalCompilerBuilder::create(ClangArgv)); + + // Set an error handler, so that any LLVM backend diagnostics go through our + // error handler. + llvm::install_fatal_error_handler(LLVMErrorHandler, + static_cast(&CI->getDiagnostics())); + + auto Interp = ExitOnErr(clang::Interpreter::create(std::move(CI))); + llvm::LineEditor LE("clang-repl"); + // FIXME: Add LE.setListCompleter + while (llvm::Optional Line = LE.readLine()) { + if (*Line == "quit") + break; + if (auto Err = Interp->ParseAndExecute(*Line)) + llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); + } + + // Our error handler depends on the Diagnostics object, which we're + // potentially about to delete. Uninstall the handler now so that any + // later errors use the default handling behavior instead. + llvm::remove_fatal_error_handler(); + + llvm::llvm_shutdown(); + + return 0; +} diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt --- a/clang/unittests/CMakeLists.txt +++ b/clang/unittests/CMakeLists.txt @@ -35,6 +35,7 @@ add_subdirectory(Rewrite) add_subdirectory(Sema) add_subdirectory(CodeGen) +add_subdirectory(Interpreter) # FIXME: libclang unit tests are disabled on Windows due # to failures, mostly in libclang.VirtualFileOverlay_*. if(NOT WIN32 AND CLANG_TOOL_LIBCLANG_BUILD) diff --git a/clang/unittests/CodeGen/CMakeLists.txt b/clang/unittests/CodeGen/CMakeLists.txt --- a/clang/unittests/CodeGen/CMakeLists.txt +++ b/clang/unittests/CodeGen/CMakeLists.txt @@ -6,7 +6,6 @@ add_clang_unittest(ClangCodeGenTests BufferSourceTest.cpp CodeGenExternalTest.cpp - IncrementalProcessingTest.cpp TBAAMetadataTest.cpp CheckTargetFeaturesTest.cpp ) @@ -17,6 +16,7 @@ clangBasic clangCodeGen clangFrontend + clangInterpreter clangLex clangParse clangSerialization diff --git a/clang/unittests/CodeGen/IncrementalProcessingTest.cpp b/clang/unittests/CodeGen/IncrementalProcessingTest.cpp deleted file mode 100644 --- a/clang/unittests/CodeGen/IncrementalProcessingTest.cpp +++ /dev/null @@ -1,155 +0,0 @@ -//=== unittests/CodeGen/IncrementalProcessingTest.cpp - IncrementalCodeGen ===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "TestCompiler.h" - -#include "clang/AST/ASTConsumer.h" -#include "clang/AST/ASTContext.h" -#include "clang/AST/RecursiveASTVisitor.h" -#include "clang/Basic/TargetInfo.h" -#include "clang/CodeGen/ModuleBuilder.h" -#include "clang/Frontend/CompilerInstance.h" -#include "clang/Lex/Preprocessor.h" -#include "clang/Parse/Parser.h" -#include "clang/Sema/Sema.h" -#include "llvm/ADT/Triple.h" -#include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/Host.h" -#include "llvm/Support/MemoryBuffer.h" -#include "gtest/gtest.h" - -#include - -using namespace llvm; -using namespace clang; - -namespace { - -// Incremental processing produces several modules, all using the same "main -// file". Make sure CodeGen can cope with that, e.g. for static initializers. -const char TestProgram1[] = - "extern \"C\" int funcForProg1() { return 17; }\n" - "struct EmitCXXGlobalInitFunc1 {\n" - " EmitCXXGlobalInitFunc1() {}\n" - "} test1;"; - -const char TestProgram2[] = - "extern \"C\" int funcForProg2() { return 42; }\n" - "struct EmitCXXGlobalInitFunc2 {\n" - " EmitCXXGlobalInitFunc2() {}\n" - "} test2;"; - - -/// An incremental version of ParseAST(). -static std::unique_ptr -IncrementalParseAST(CompilerInstance& CI, Parser& P, - CodeGenerator& CG, const char* code) { - static int counter = 0; - struct IncreaseCounterOnRet { - ~IncreaseCounterOnRet() { - ++counter; - } - } ICOR; - - Sema& S = CI.getSema(); - clang::SourceManager &SM = S.getSourceManager(); - if (!code) { - // Main file - SM.setMainFileID(SM.createFileID( - llvm::MemoryBuffer::getMemBuffer(" "), clang::SrcMgr::C_User)); - - S.getPreprocessor().EnterMainSourceFile(); - P.Initialize(); - } else { - FileID FID = SM.createFileID( - llvm::MemoryBuffer::getMemBuffer(code), clang::SrcMgr::C_User); - SourceLocation MainStartLoc = SM.getLocForStartOfFile(SM.getMainFileID()); - SourceLocation InclLoc = MainStartLoc.getLocWithOffset(counter); - S.getPreprocessor().EnterSourceFile(FID, 0, InclLoc); - } - - ExternalASTSource *External = S.getASTContext().getExternalSource(); - if (External) - External->StartTranslationUnit(&CG); - - Parser::DeclGroupPtrTy ADecl; - for (bool AtEOF = P.ParseFirstTopLevelDecl(ADecl); !AtEOF; - AtEOF = P.ParseTopLevelDecl(ADecl)) { - // If we got a null return and something *was* parsed, ignore it. This - // is due to a top-level semicolon, an action override, or a parse error - // skipping something. - if (ADecl && !CG.HandleTopLevelDecl(ADecl.get())) - return nullptr; - } - - // Process any TopLevelDecls generated by #pragma weak. - for (Decl *D : S.WeakTopLevelDecls()) - CG.HandleTopLevelDecl(DeclGroupRef(D)); - - CG.HandleTranslationUnit(S.getASTContext()); - - std::unique_ptr M(CG.ReleaseModule()); - // Switch to next module. - CG.StartModule("incremental-module-" + std::to_string(counter), - M->getContext()); - return M; -} - -const Function* getGlobalInit(llvm::Module& M) { - for (const auto& Func: M) - if (Func.hasName() && Func.getName().startswith("_GLOBAL__sub_I_")) - return &Func; - - return nullptr; -} - -TEST(IncrementalProcessing, EmitCXXGlobalInitFunc) { - clang::LangOptions LO; - LO.CPlusPlus = 1; - LO.CPlusPlus11 = 1; - TestCompiler Compiler(LO); - clang::CompilerInstance &CI = Compiler.compiler; - CI.getPreprocessor().enableIncrementalProcessing(); - CI.setASTConsumer(std::move(Compiler.CG)); - clang::CodeGenerator& CG = - static_cast(CI.getASTConsumer()); - CI.createSema(clang::TU_Prefix, nullptr); - - Sema& S = CI.getSema(); - - std::unique_ptr ParseOP(new Parser(S.getPreprocessor(), S, - /*SkipFunctionBodies*/ false)); - Parser &P = *ParseOP.get(); - - std::array, 3> M; - M[0] = IncrementalParseAST(CI, P, CG, nullptr); - ASSERT_TRUE(M[0]); - - M[1] = IncrementalParseAST(CI, P, CG, TestProgram1); - ASSERT_TRUE(M[1]); - ASSERT_TRUE(M[1]->getFunction("funcForProg1")); - - M[2] = IncrementalParseAST(CI, P, CG, TestProgram2); - ASSERT_TRUE(M[2]); - ASSERT_TRUE(M[2]->getFunction("funcForProg2")); - // First code should not end up in second module: - ASSERT_FALSE(M[2]->getFunction("funcForProg1")); - - // Make sure global inits exist and are unique: - const Function* GlobalInit1 = getGlobalInit(*M[1]); - ASSERT_TRUE(GlobalInit1); - - const Function* GlobalInit2 = getGlobalInit(*M[2]); - ASSERT_TRUE(GlobalInit2); - - ASSERT_FALSE(GlobalInit1->getName() == GlobalInit2->getName()); - -} - -} // end anonymous namespace diff --git a/clang/unittests/Interpreter/CMakeLists.txt b/clang/unittests/Interpreter/CMakeLists.txt new file mode 100644 --- /dev/null +++ b/clang/unittests/Interpreter/CMakeLists.txt @@ -0,0 +1,11 @@ +set(LLVM_LINK_COMPONENTS + ) + +add_clang_unittest(ClangReplInterpreterTests + IncrementalProcessingTest.cpp + InterpreterTest.cpp + ) +target_link_libraries(ClangReplInterpreterTests PUBLIC + clangInterpreter + clangFrontend + ) diff --git a/clang/unittests/Interpreter/IncrementalProcessingTest.cpp b/clang/unittests/Interpreter/IncrementalProcessingTest.cpp new file mode 100644 --- /dev/null +++ b/clang/unittests/Interpreter/IncrementalProcessingTest.cpp @@ -0,0 +1,80 @@ +//=== unittests/CodeGen/IncrementalProcessingTest.cpp - IncrementalCodeGen ===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/AST/ASTConsumer.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/RecursiveASTVisitor.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/CodeGen/ModuleBuilder.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Interpreter/Interpreter.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Parse/Parser.h" +#include "clang/Sema/Sema.h" +#include "llvm/ADT/Triple.h" +#include "llvm/IR/LLVMContext.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/MemoryBuffer.h" +#include "gtest/gtest.h" + +#include + +using namespace llvm; +using namespace clang; + +namespace { + +// Incremental processing produces several modules, all using the same "main +// file". Make sure CodeGen can cope with that, e.g. for static initializers. +const char TestProgram1[] = "extern \"C\" int funcForProg1() { return 17; }\n" + "struct EmitCXXGlobalInitFunc1 {\n" + " EmitCXXGlobalInitFunc1() {}\n" + "} test1;"; + +const char TestProgram2[] = "extern \"C\" int funcForProg2() { return 42; }\n" + "struct EmitCXXGlobalInitFunc2 {\n" + " EmitCXXGlobalInitFunc2() {}\n" + "} test2;"; + +const Function *getGlobalInit(llvm::Module *M) { + for (const auto &Func : *M) + if (Func.hasName() && Func.getName().startswith("_GLOBAL__sub_I_")) + return &Func; + + return nullptr; +} + +TEST(IncrementalProcessing, EmitCXXGlobalInitFunc) { + std::vector ClangArgv = {"-Xclang", "-emit-llvm-only"}; + auto CI = llvm::cantFail(IncrementalCompilerBuilder::create(ClangArgv)); + auto Interp = llvm::cantFail(Interpreter::create(std::move(CI))); + + std::array Transactions; + + Transactions[0] = &llvm::cantFail(Interp->Parse(TestProgram1)); + ASSERT_TRUE(Transactions[0]->TheModule); + ASSERT_TRUE(Transactions[0]->TheModule->getFunction("funcForProg1")); + + Transactions[1] = &llvm::cantFail(Interp->Parse(TestProgram2)); + ASSERT_TRUE(Transactions[1]->TheModule); + ASSERT_TRUE(Transactions[1]->TheModule->getFunction("funcForProg2")); + // First code should not end up in second module: + ASSERT_FALSE(Transactions[1]->TheModule->getFunction("funcForProg1")); + + // Make sure global inits exist and are unique: + const Function *GlobalInit1 = getGlobalInit(Transactions[0]->TheModule.get()); + ASSERT_TRUE(GlobalInit1); + + const Function *GlobalInit2 = getGlobalInit(Transactions[1]->TheModule.get()); + ASSERT_TRUE(GlobalInit2); + + ASSERT_FALSE(GlobalInit1->getName() == GlobalInit2->getName()); +} + +} // end anonymous namespace diff --git a/clang/unittests/Interpreter/InterpreterTest.cpp b/clang/unittests/Interpreter/InterpreterTest.cpp new file mode 100644 --- /dev/null +++ b/clang/unittests/Interpreter/InterpreterTest.cpp @@ -0,0 +1,122 @@ +//===- unittests/Interpreter/InterpreterTest.cpp --- Interpreter tests ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Unit tests for Clang's Interpreter library. +// +//===----------------------------------------------------------------------===// + +#include "clang/Interpreter/Interpreter.h" + +#include "clang/AST/Decl.h" +#include "clang/AST/DeclGroup.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" + +#include "llvm/ADT/ArrayRef.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace clang; + +namespace { +using Args = std::vector; +static std::unique_ptr +createInterpreter(const Args &ExtraArgs = {}, + DiagnosticConsumer *Client = nullptr) { + Args ClangArgs = {"-Xclang", "-emit-llvm-only"}; + ClangArgs.insert(ClangArgs.end(), ExtraArgs.begin(), ExtraArgs.end()); + auto CI = cantFail(clang::IncrementalCompilerBuilder::create(ClangArgs)); + if (Client) + CI->getDiagnostics().setClient(Client, /*ShouldOwnClient=*/false); + return cantFail(clang::Interpreter::create(std::move(CI))); +} + +TEST(InterpreterTest, Sanity) { + std::unique_ptr Interp = createInterpreter(); + Transaction &R1(cantFail(Interp->Parse("void g(); void g() {}"))); + EXPECT_EQ(2U, R1.Decls.size()); + + Transaction &R2(cantFail(Interp->Parse("int i;"))); + EXPECT_EQ(1U, R2.Decls.size()); +} + +static std::string DeclToString(DeclGroupRef DGR) { + return llvm::cast(DGR.getSingleDecl())->getQualifiedNameAsString(); +} + +TEST(InterpreterTest, IncrementalInputTopLevelDecls) { + std::unique_ptr Interp = createInterpreter(); + auto R1OrErr = Interp->Parse("int var1 = 42; int f() { return var1; }"); + // gtest doesn't expand into explicit bool conversions. + EXPECT_TRUE(!!R1OrErr); + auto R1 = R1OrErr->Decls; + EXPECT_EQ(2U, R1.size()); + EXPECT_EQ("var1", DeclToString(R1[0])); + EXPECT_EQ("f", DeclToString(R1[1])); + + auto R2OrErr = Interp->Parse("int var2 = f();"); + EXPECT_TRUE(!!R2OrErr); + auto R2 = R2OrErr->Decls; + EXPECT_EQ(1U, R2.size()); + EXPECT_EQ("var2", DeclToString(R2[0])); +} + +TEST(InterpreterTest, Errors) { + Args ExtraArgs = {"-Xclang", "-diagnostic-log-file", "-Xclang", "-"}; + + // Create the diagnostic engine with unowned consumer. + std::string DiagnosticOutput; + llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput); + auto DiagPrinter = std::make_unique( + DiagnosticsOS, new DiagnosticOptions()); + + auto Interp = createInterpreter(ExtraArgs, DiagPrinter.get()); + auto Err = Interp->Parse("intentional_error v1 = 42; ").takeError(); + using ::testing::HasSubstr; + EXPECT_THAT(DiagnosticsOS.str(), + HasSubstr("error: unknown type name 'intentional_error'")); + EXPECT_EQ("Parsing failed.", llvm::toString(std::move(Err))); + +#ifdef GTEST_HAS_DEATH_TEST + EXPECT_DEATH((void)Interp->Parse("int var1 = 42;"), ""); +#endif +} + +// Here we test whether the user can mix declarations and statements. The +// interpreter should be smart enough to recognize the declarations from the +// statements and wrap the latter into a declaration, producing valid code. +TEST(InterpreterTest, DeclsAndStatements) { + Args ExtraArgs = {"-Xclang", "-diagnostic-log-file", "-Xclang", "-"}; + + // Create the diagnostic engine with unowned consumer. + std::string DiagnosticOutput; + llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput); + auto DiagPrinter = std::make_unique( + DiagnosticsOS, new DiagnosticOptions()); + + auto Interp = createInterpreter(ExtraArgs, DiagPrinter.get()); + auto R1OrErr = Interp->Parse( + "int var1 = 42; extern \"C\" int printf(const char*, ...);"); + // gtest doesn't expand into explicit bool conversions. + EXPECT_TRUE(!!R1OrErr); + + auto R1 = R1OrErr->Decls; + EXPECT_EQ(2U, R1.size()); + + // FIXME: Add support for wrapping and running statements. + auto R2OrErr = Interp->Parse("var1++; printf(\"var1 value %d\\n\", var1);"); + EXPECT_FALSE(!!R2OrErr); + using ::testing::HasSubstr; + EXPECT_THAT(DiagnosticsOS.str(), + HasSubstr("error: unknown type name 'var1'")); + auto Err = R2OrErr.takeError(); + EXPECT_EQ("Parsing failed.", llvm::toString(std::move(Err))); +} + +} // end anonymous namespace