Index: clang/include/clang/CodeGen/CodeGenAction.h =================================================================== --- clang/include/clang/CodeGen/CodeGenAction.h +++ clang/include/clang/CodeGen/CodeGenAction.h @@ -19,6 +19,7 @@ namespace clang { class BackendConsumer; +class CodeGenerator; class CodeGenAction : public ASTFrontendAction { private: @@ -77,6 +78,8 @@ /// Take the LLVM context used by this action. llvm::LLVMContext *takeLLVMContext(); + CodeGenerator *getCodeGenerator() const; + BackendConsumer *BEConsumer; }; Index: clang/include/clang/Frontend/FrontendAction.h =================================================================== --- clang/include/clang/Frontend/FrontendAction.h +++ clang/include/clang/Frontend/FrontendAction.h @@ -227,14 +227,15 @@ /// /// \return True on success; on failure the compilation of this file should /// be aborted and neither Execute() nor EndSourceFile() should be called. - bool BeginSourceFile(CompilerInstance &CI, const FrontendInputFile &Input); + virtual bool BeginSourceFile(CompilerInstance &CI, + const FrontendInputFile &Input); /// Set the source manager's main input file, and run the action. llvm::Error Execute(); /// Perform any per-file post processing, deallocate per-file /// objects, and run statistics and output file cleanup code. - void EndSourceFile(); + virtual void EndSourceFile(); /// @} }; @@ -302,9 +303,9 @@ /// some existing action's behavior. It implements every virtual method in /// the FrontendAction interface by forwarding to the wrapped action. class WrapperFrontendAction : public FrontendAction { +protected: std::unique_ptr WrappedAction; -protected: bool PrepareToExecuteAction(CompilerInstance &CI) override; std::unique_ptr CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override; Index: clang/include/clang/Interpreter/Interpreter.h =================================================================== --- /dev/null +++ clang/include/clang/Interpreter/Interpreter.h @@ -0,0 +1,66 @@ +//===--- Interpreter.h - Incremental Compilation and Execution---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the component which performs incremental code +// compilation and execution. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INTERPRETER_INTERPRETER_H +#define LLVM_CLANG_INTERPRETER_INTERPRETER_H + +#include "clang/Interpreter/Transaction.h" + +#include "llvm/Support/Error.h" + +#include +#include + +namespace llvm { +class Module; +} + +namespace clang { + +class CompilerInstance; +class DeclGroupRef; +class IncrementalExecutor; +class IncrementalParser; + +/// Create a pre-configured \c CompilerInstance for incremental processing. +class IncrementalCompilerBuilder { +public: + static llvm::Expected> + create(std::vector &ClangArgv); +}; + +/// Provides top-level interfaces for incremental compilation and execution. +class Interpreter { + std::unique_ptr IncrParser; + std::unique_ptr IncrExecutor; + + Interpreter(std::unique_ptr CI, llvm::Error &Err); +public: + ~Interpreter(); + static llvm::Expected> + create(std::unique_ptr CI); + const CompilerInstance *getCompilerInstance() const; + llvm::Expected Parse(llvm::StringRef Code); + llvm::Error Execute(Transaction &T); + llvm::Error ParseAndExecute(llvm::StringRef Code) { + auto ErrOrTransaction = Parse(Code); + if (auto Err = ErrOrTransaction.takeError()) + return Err; + if (ErrOrTransaction->TheModule) + return Execute(*ErrOrTransaction); + return llvm::Error::success(); + } +}; +} + +#endif // LLVM_CLANG_INTERPRETER_INTERPRETER_H Index: clang/include/clang/Interpreter/Transaction.h =================================================================== --- /dev/null +++ clang/include/clang/Interpreter/Transaction.h @@ -0,0 +1,33 @@ +//===--- Transaction.h - Incremental Compilation and Execution---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines utilities tracking the incrementally processed pieces of +// code. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INTERPRETER_TRANSACTION_H +#define LLVM_CLANG_INTERPRETER_TRANSACTION_H + +#include +#include + +namespace llvm { +class Module; +} + +namespace clang { + +class DeclGroupRef; +struct Transaction { + std::vector Decls; + std::unique_ptr TheModule; +}; +} + +#endif // LLVM_CLANG_INTERPRETER_TRANSACTION_H Index: clang/lib/CMakeLists.txt =================================================================== --- clang/lib/CMakeLists.txt +++ clang/lib/CMakeLists.txt @@ -25,3 +25,4 @@ add_subdirectory(StaticAnalyzer) add_subdirectory(Format) add_subdirectory(Testing) +add_subdirectory(Interpreter) Index: clang/lib/CodeGen/CodeGenAction.cpp =================================================================== --- clang/lib/CodeGen/CodeGenAction.cpp +++ clang/lib/CodeGen/CodeGenAction.cpp @@ -905,6 +905,10 @@ return VMContext; } +CodeGenerator *CodeGenAction::getCodeGenerator() const { + return BEConsumer->getCodeGenerator(); +} + static std::unique_ptr GetOutputStream(CompilerInstance &CI, StringRef InFile, BackendAction Action) { switch (Action) { Index: clang/lib/Interpreter/CMakeLists.txt =================================================================== --- /dev/null +++ clang/lib/Interpreter/CMakeLists.txt @@ -0,0 +1,22 @@ +set(LLVM_LINK_COMPONENTS + core + native + OrcJit + Target + ) + +add_clang_library(clangInterpreter + IncrementalExecutor.cpp + IncrementalParser.cpp + Interpreter.cpp + + LINK_LIBS + clangAST + clangAnalysis + clangBasic + clangEdit + clangLex + clangSema + clangCodeGen + clangFrontendTool + ) Index: clang/lib/Interpreter/IncrementalExecutor.h =================================================================== --- /dev/null +++ clang/lib/Interpreter/IncrementalExecutor.h @@ -0,0 +1,39 @@ +//===--- IncrementalExecutor.h - Incremental Execution ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the class which performs incremental code execution. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" + +#include + +namespace llvm { +class Error; +class Module; +namespace orc { + class LLJIT; +} +} + +namespace clang { +class IncrementalExecutor { + using CtorDtorIterator = llvm::orc::CtorDtorIterator; + std::unique_ptr Jit; + +public: + IncrementalExecutor(llvm::Error &Err); + ~IncrementalExecutor(); + + llvm::Error addModule(std::unique_ptr M); + llvm::Error runCtors() const; +}; + +} // end namespace clang Index: clang/lib/Interpreter/IncrementalExecutor.cpp =================================================================== --- /dev/null +++ clang/lib/Interpreter/IncrementalExecutor.cpp @@ -0,0 +1,65 @@ +//===--- IncrementalExecutor.cpp - Incremental Execution --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the class which performs incremental code execution. +// +//===----------------------------------------------------------------------===// + +#include "IncrementalExecutor.h" + +#include "llvm/ExecutionEngine/ExecutionEngine.h" +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/TargetSelect.h" + +namespace clang { + +IncrementalExecutor::IncrementalExecutor(llvm::Error &Err) { + using namespace llvm::orc; + llvm::ErrorAsOutParameter EAO(&Err); + auto JitOrErr = LLJITBuilder().create(); + if (auto Err2 = JitOrErr.takeError()) { + Err = std::move(Err2); + return; + } + + // Discover symbols from the process as a fallback. + const llvm::DataLayout &DL = (*JitOrErr)->getDataLayout(); + auto ProcessSymbolsGenerator = + DynamicLibrarySearchGenerator::GetForCurrentProcess(DL.getGlobalPrefix()); + + if (!ProcessSymbolsGenerator) { + Err = ProcessSymbolsGenerator.takeError(); + return; + } + + Jit = std::move(*JitOrErr); + + llvm::sys::DynamicLibrary::LoadLibraryPermanently(nullptr); + JITDylib &MainJD = Jit->getMainJITDylib(); + MainJD.addGenerator(std::move(*ProcessSymbolsGenerator)); +} + +IncrementalExecutor::~IncrementalExecutor() { } + +llvm::Error IncrementalExecutor::addModule(std::unique_ptr M) { + llvm::orc::ThreadSafeContext TSCtx(std::make_unique()); + return Jit->addIRModule(llvm::orc::ThreadSafeModule(std::move(M), TSCtx)); +} + +llvm::Error IncrementalExecutor::runCtors() const { + return Jit->initialize(Jit->getMainJITDylib()); +} + +} // end namespace clang Index: clang/lib/Interpreter/IncrementalParser.h =================================================================== --- /dev/null +++ clang/lib/Interpreter/IncrementalParser.h @@ -0,0 +1,68 @@ +//===--- IncrementalParser.h - Incremental Compilation ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the class which performs incremental code compilation. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_INTERPRETER_INCREMENTALPARSER_H +#define LLVM_CLANG_INTERPRETER_INCREMENTALPARSER_H + +#include "clang/Interpreter/Transaction.h" + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +#include +#include + +namespace clang { +class ASTConsumer; +class CompilerInstance; +class CodeGenerator; +class DeclGroupRef; +class FrontendAction; +class Parser; + +/// Provides support for incremental compilation. Keeps track of the state +/// changes between the subsequent incremental input. +/// +class IncrementalParser { + /// Long-lived, incremental parsing action. + std::unique_ptr Act; + + /// Compiler instance performing the incremental compilation. + std::unique_ptr CI; + + /// Parser. + std::unique_ptr P; + + /// Consumer to process the produced top level decls. Owned by Act. + ASTConsumer *Consumer = nullptr; + + /// Incremental input counter. + unsigned InputCount = 0; + + /// List containing every information about every incrementally parsed piece + /// of code. + std::list Transactions; + +public: + IncrementalParser(std::unique_ptr Instance); + ~IncrementalParser(); + + const CompilerInstance *getCI() const { return CI.get(); } + llvm::Expected Parse(llvm::StringRef Input); + +private: + llvm::Expected ParseOrWrapTopLevelDecl(); +}; +} // end namespace clang + +#endif // LLVM_CLANG_INTERPRETER_INCREMENTALPARSER_H Index: clang/lib/Interpreter/IncrementalParser.cpp =================================================================== --- /dev/null +++ clang/lib/Interpreter/IncrementalParser.cpp @@ -0,0 +1,220 @@ +//===--------- IncrementalParser.cpp - Incremental Compilation -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the class which performs incremental code compilation. +// +//===----------------------------------------------------------------------===// + +#include "IncrementalParser.h" + +#include "clang/CodeGen/BackendUtil.h" +#include "clang/CodeGen/CodeGenAction.h" +#include "clang/CodeGen/ModuleBuilder.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendAction.h" +#include "clang/FrontendTool/Utils.h" +#include "clang/Parse/Parser.h" +#include "clang/Sema/Sema.h" + +#include "llvm/Option/ArgList.h" +#include "llvm/Support/CrashRecoveryContext.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/Timer.h" + +#include + +using namespace clang; + +/// A custom action enabling the incremental processing functionality. +/// +/// The usual \p FrontendAction expects one call to ExecuteAction and once it +/// sees a call to \p EndSourceFile it deletes some of the important objects +/// such as \p Preprocessor and \p Sema assuming no further input will come. +/// +/// \p IncrementalAction ensures it keep its underlying action's objects alive +/// as long as the \p IncrementalParser needs them. +/// +class IncrementalAction : public WrapperFrontendAction { +private: + bool IsTerminating = false; + +public: + IncrementalAction(CompilerInstance &CI) + : WrapperFrontendAction(CreateFrontendAction(CI)) {} + FrontendAction *getWrapped() const { return WrappedAction.get(); } + void ExecuteAction() override { + CompilerInstance &CI = getCompilerInstance(); + assert(CI.hasPreprocessor() && "No PP!"); + + // FIXME: Move the truncation aspect of this into Sema, we delayed this till + // here so the source manager would be initialized. + if (hasCodeCompletionSupport() && + !CI.getFrontendOpts().CodeCompletionAt.FileName.empty()) + CI.createCodeCompletionConsumer(); + + // Use a code completion consumer? + CodeCompleteConsumer *CompletionConsumer = nullptr; + if (CI.hasCodeCompletionConsumer()) + CompletionConsumer = &CI.getCodeCompletionConsumer(); + + if (!CI.hasSema()) + CI.createSema(getTranslationUnitKind(), CompletionConsumer); + + Preprocessor &PP = CI.getPreprocessor(); + PP.enableIncrementalProcessing(); + PP.EnterMainSourceFile(); + } + + void EndSourceFile() override { + if (IsTerminating) { + WrapperFrontendAction::EndSourceFile(); + } + } + + void FinalizeAction() { + assert(!IsTerminating && "Already finalized!"); + IsTerminating = true; + EndSourceFile(); + } +}; + +IncrementalParser::IncrementalParser(std::unique_ptr Instance) + : CI(std::move(Instance)) { + Act = std::make_unique(*CI); + CI->ExecuteAction(*Act); + Consumer = &CI->getASTConsumer(); + P.reset( + new Parser(CI->getPreprocessor(), CI->getSema(), /*SkipBodies*/ false)); + P->Initialize(); +} + +IncrementalParser::~IncrementalParser() { + ((IncrementalAction *)Act.get())->FinalizeAction(); + // Our error handler depends on the Diagnostics object, which we're + // potentially about to delete. Uninstall the handler now so that any + // later errors use the default handling behavior instead. + llvm::remove_fatal_error_handler(); +} + +llvm::Expected IncrementalParser::ParseOrWrapTopLevelDecl() { + // Recover resources if we crash before exiting this method. + Sema &S = CI->getSema(); + llvm::CrashRecoveryContextCleanupRegistrar CleanupSema(&S); + Sema::GlobalEagerInstantiationScope GlobalInstantiations(S, /*Enabled*/ true); + Sema::LocalEagerInstantiationScope LocalInstantiations(S); + + // Skip previous eof due to last incremental input. + if (P->getCurToken().is(tok::eof)) + P->ConsumeToken(); + + Transactions.emplace_back(Transaction()); + Transaction &LastTransaction = Transactions.back(); + + Parser::DeclGroupPtrTy ADecl; + for (bool AtEOF = P->ParseFirstTopLevelDecl(ADecl); !AtEOF; + AtEOF = P->ParseTopLevelDecl(ADecl)) { + // If we got a null return and something *was* parsed, ignore it. This + // is due to a top-level semicolon, an action override, or a parse error + // skipping something. + if (ADecl && !Consumer->HandleTopLevelDecl(ADecl.get())) + return llvm::make_error("Parsing failed. " + "The consumer rejected a decl", + std::error_code()); + LastTransaction.Decls.push_back(ADecl.get()); + } + + // Process any TopLevelDecls generated by #pragma weak. + for (Decl *D : S.WeakTopLevelDecls()) { + DeclGroupRef DGR(D); + LastTransaction.Decls.push_back(DGR); + Consumer->HandleTopLevelDecl(DGR); + } + + LocalInstantiations.perform(); + GlobalInstantiations.perform(); + + Consumer->HandleTranslationUnit(S.getASTContext()); + + DiagnosticsEngine &Diags = getCI()->getDiagnostics(); + if (Diags.hasErrorOccurred()) { + Diags.Reset(); // FIXME: Mind the resetting of pragma handlers. + return llvm::make_error("Parsing failed.", + std::error_code()); + } + return LastTransaction; +} + +static CodeGenerator *getCodeGen(FrontendAction *Act) { + IncrementalAction *IncrAct = static_cast(Act); + FrontendAction *WrappedAct = IncrAct->getWrapped(); + if (!WrappedAct->hasIRSupport()) + return nullptr; + return static_cast(WrappedAct)->getCodeGenerator(); +} + +llvm::Expected IncrementalParser::Parse(llvm::StringRef input) { + Preprocessor &PP = CI->getPreprocessor(); + assert(PP.isIncrementalProcessingEnabled() && "Not in incremental mode!?"); + + std::ostringstream SourceName; + SourceName << "input_line_" << InputCount++; + + // Create an uninitialized memory buffer, copy code in and append "\n" + size_t InputSize = input.size(); // don't include trailing 0 + // MemBuffer size should *not* include terminating zero + std::unique_ptr MB( + llvm::WritableMemoryBuffer::getNewUninitMemBuffer(InputSize + 1, + SourceName.str())); + char *MBStart = const_cast(MB->getBufferStart()); + memcpy(MBStart, input.data(), InputSize); + MBStart[InputSize] = '\n'; + + SourceManager &SM = CI->getSourceManager(); + + // Create SourceLocation, which will allow clang to order the overload + // candidates for example + SourceLocation NewLoc = SM.getLocForStartOfFile(SM.getMainFileID()) + .getLocWithOffset(InputCount + 2); + + // Create FileID for the current buffer. + FileID FID = SM.createFileID(std::move(MB), SrcMgr::C_User, /*LoadedID*/ 0, + /*LoadedOffset*/ 0, NewLoc); + + // NewLoc only used for diags. + PP.EnterSourceFile(FID, /*DirLookup*/ 0, NewLoc); + + auto ErrOrTransaction = ParseOrWrapTopLevelDecl(); + if (auto Err = ErrOrTransaction.takeError()) + return std::move(Err); + + if (PP.getLangOpts().DelayedTemplateParsing) { + // Microsoft-specific: + // Late parsed templates can leave unswallowed "macro"-like tokens. + // They will seriously confuse the Parser when entering the next + // source file. So lex until we are EOF. + Token Tok; + do { + PP.Lex(Tok); + } while (Tok.isNot(tok::eof)); + } + + Token AssertTok; + PP.Lex(AssertTok); + assert(AssertTok.is(tok::eof) && + "Lexer must be EOF when starting incremental parse!"); + + if (CodeGenerator *CG = getCodeGen(Act.get())) { + std::unique_ptr M(CG->ReleaseModule()); + CG->StartModule("incr_module_" + std::to_string(Transactions.size()), + M->getContext()); + + ErrOrTransaction->TheModule = std::move(M); + } + + return ErrOrTransaction; +} Index: clang/lib/Interpreter/Interpreter.cpp =================================================================== --- /dev/null +++ clang/lib/Interpreter/Interpreter.cpp @@ -0,0 +1,231 @@ +//===------ Interpreter.cpp - Incremental Compilation and Execution -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the component which performs incremental code +// compilation and execution. +// +//===----------------------------------------------------------------------===// + +#include "clang/Interpreter/Interpreter.h" + +#include "clang/CodeGen/ModuleBuilder.h" +#include "clang/Frontend/CompilerInstance.h" + +#include "llvm/IR/Module.h" + +#include "IncrementalParser.h" +#include "IncrementalExecutor.h" + +// +#include "clang/Basic/TargetInfo.h" + +#include "clang/CodeGen/ObjectFilePCHContainerOperations.h" + +#include "clang/Driver/Compilation.h" +#include "clang/Driver/Driver.h" +#include "clang/Driver/Job.h" +#include "clang/Driver/Options.h" +#include "clang/Driver/Tool.h" +#include "clang/Frontend/TextDiagnosticBuffer.h" + + +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Frontend/VerifyDiagnosticConsumer.h" +#include "clang/Lex/PreprocessorOptions.h" + +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Host.h" +// + +using namespace clang; + +// FIXME: Figure out how to unify with namespace init_convenience from +// tools/clang-import-test/clang-import-test.cpp and +// examples/clang-interpreter/main.cpp +namespace { +/// Retrieves the clang CC1 specific flags out of the compilation's jobs. +/// \returns NULL on error. +static llvm::Expected +GetCC1Arguments(DiagnosticsEngine *Diagnostics, + driver::Compilation *Compilation) { + // We expect to get back exactly one Command job, if we didn't something + // failed. Extract that job from the Compilation. + const driver::JobList &Jobs = Compilation->getJobs(); + if (!Jobs.size() || !isa(*Jobs.begin())) + return llvm::createStringError(std::errc::state_not_recoverable, + "Driver initialization failed: " + "Unable to create a driver job"); + + // The one job we find should be to invoke clang again. + const driver::Command *Cmd = cast(&(*Jobs.begin())); + if (llvm::StringRef(Cmd->getCreator().getName()) != "clang") + return llvm::createStringError(std::errc::state_not_recoverable, + "Driver initialization failed"); + + return &Cmd->getArguments(); +} + + +static llvm::Expected> +CreateCI(const llvm::opt::ArgStringList &Argv) { + std::unique_ptr Clang(new CompilerInstance()); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + + // Register the support for object-file-wrapped Clang modules. + auto PCHOps = Clang->getPCHContainerOperations(); + PCHOps->registerWriter(std::make_unique()); + PCHOps->registerReader(std::make_unique()); + + // Buffer diagnostics from argument parsing so that we can output them using + // a well formed diagnostic object. + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); + bool Success = CompilerInvocation::CreateFromArgs( + Clang->getInvocation(), llvm::makeArrayRef(Argv.begin(), Argv.size()), Diags); + + // Infer the builtin include path if unspecified. + if (Clang->getHeaderSearchOpts().UseBuiltinIncludes && + Clang->getHeaderSearchOpts().ResourceDir.empty()) + Clang->getHeaderSearchOpts().ResourceDir = + CompilerInvocation::GetResourcesPath(Argv[0], nullptr); + + // Create the actual diagnostics engine. + Clang->createDiagnostics(); + if (!Clang->hasDiagnostics()) + return llvm::createStringError(std::errc::state_not_recoverable, + "Initialization failed: " + "Unable to create diagnostics engine"); + + DiagsBuffer->FlushDiagnostics(Clang->getDiagnostics()); + if (!Success) + return llvm::createStringError(std::errc::state_not_recoverable, + "Initialization failed: " + "Unable to flush diagnostics"); + + // FIXME: Merge with CompilerInstance::ExecuteAction. + llvm::MemoryBuffer *MB = llvm::MemoryBuffer::getMemBuffer("").release(); + Clang->getPreprocessorOpts().addRemappedFile("<<< inputs >>>", MB); + + Clang->setTarget(TargetInfo::CreateTargetInfo( + Clang->getDiagnostics(), Clang->getInvocation().TargetOpts)); + if (!Clang->hasTarget()) + return llvm::createStringError(std::errc::state_not_recoverable, + "Initialization failed: " + "Target is missing"); + + Clang->getTarget().adjust(Clang->getLangOpts()); + + return std::move(Clang); +} + + +} // anonymous namespace + +llvm::Expected> +IncrementalCompilerBuilder::create(std::vector &ClangArgv) { + + // If we don't know ClangArgv0 or the address of main() at this point, try + // to guess it anyway (it's possible on some platforms). + std::string MainExecutableName = + llvm::sys::fs::getMainExecutable(nullptr, nullptr); + + ClangArgv.insert(ClangArgv.begin(), MainExecutableName.c_str()); + + if (llvm::find(ClangArgv, " -x") == ClangArgv.end()) { + // We do C++ by default; append right after argv[0] if no "-x" given + ClangArgv.push_back("-x"); + ClangArgv.push_back("c++"); + } + // By adding -c, we force the driver to treat compilation as the last phase. + // It will then issue warnings via Diagnostics about un-used options that + // would have been used for linking. If the user provided a compiler name as + // the original argv[0], this will be treated as a linker input thanks to + // insertng a new argv[0] above. All un-used options get collected by + // UnusedInputdiagConsumer and get stripped out later. + ClangArgv.push_back("-c"); + + // Put a dummy C++ file on to ensure there's at least one compile job for the + // driver to construct. If the user specified some other argument that + // prevents compilation, e.g. -E or something like -version, we may still end + // up with no jobs but then this is the user's fault. + ClangArgv.push_back("<<< inputs >>>"); + + CompilerInvocation Invocation; + // Buffer diagnostics from argument parsing so that we can output them using a + // well formed diagnostic object. + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticBuffer *DiagsBuffer = new TextDiagnosticBuffer; + DiagnosticsEngine Diags(DiagID, &*DiagOpts, DiagsBuffer); + unsigned MissingArgIndex, MissingArgCount; + const llvm::opt::OptTable &Opts = driver::getDriverOptTable(); + llvm::opt::InputArgList ParsedArgs + = Opts.ParseArgs(ArrayRef(ClangArgv).slice(1), + MissingArgIndex, MissingArgCount); + ParseDiagnosticArgs(*DiagOpts, ParsedArgs, &Diags); + + driver::Driver Driver(/*MainBinaryName*/ ClangArgv[0], + llvm::sys::getDefaultTargetTriple(), Diags); + Driver.setCheckInputsExist(false); // the input comes from mem buffers + llvm::ArrayRef RF = llvm::makeArrayRef(ClangArgv); + std::unique_ptr Compilation(Driver.BuildCompilation(RF)); + + if (Compilation->getArgs().hasArg(driver::options::OPT_v)) + Compilation->getJobs().Print(llvm::errs(), "\n", /*Quote*/ false); + + auto ErrOrCC1Args = GetCC1Arguments(&Diags, Compilation.get()); + if (auto Err = ErrOrCC1Args.takeError()) + return std::move(Err); + + return CreateCI(**ErrOrCC1Args); +} + +Interpreter::Interpreter(std::unique_ptr CI, + llvm::Error &Err) { + llvm::ErrorAsOutParameter EAO(&Err); + IncrParser = std::make_unique(std::move(CI)); +} + +Interpreter::~Interpreter() {} + +llvm::Expected> +Interpreter::create(std::unique_ptr CI) { + llvm::Error Err = llvm::Error::success(); + auto Interp = + std::unique_ptr(new Interpreter(std::move(CI), Err)); + if (Err) + return std::move(Err); + return std::move(Interp); +} + +const CompilerInstance *Interpreter::getCompilerInstance() const { + return IncrParser->getCI(); +} + +llvm::Expected Interpreter::Parse(llvm::StringRef Code) { + return IncrParser->Parse(Code); +} + +llvm::Error Interpreter::Execute(Transaction &T) { + assert(T.TheModule); + if (!IncrExecutor) { + llvm::Error Err = llvm::Error::success(); + IncrExecutor = std::make_unique(Err); + if (Err) + return Err; + } + // FIXME: Add a callback to retain the llvm::Module once the JIT is done. + if (auto Err = IncrExecutor->addModule(std::move(T.TheModule))) + return Err; + + if (auto Err = IncrExecutor->runCtors()) + return Err; + + return llvm::Error::success(); +} Index: clang/test/CMakeLists.txt =================================================================== --- clang/test/CMakeLists.txt +++ clang/test/CMakeLists.txt @@ -68,6 +68,7 @@ clang-import-test clang-rename clang-refactor + clang-repl clang-diff clang-scan-deps diagtool Index: clang/test/Interpreter/execute.c =================================================================== --- /dev/null +++ clang/test/Interpreter/execute.c @@ -0,0 +1,13 @@ +// RUN: cat %s | clang-repl | FileCheck %s + +extern "C" int printf(const char*, ...); +int i = 42; +auto r1 = printf("i = %d\n", i); +// CHECK: i = 42 + +struct S { float f = 1.0; S *m = nullptr;} s; +auto r2 = printf("S[f=%f, m=%p]\n", s.f, s.m); +// CHECK-NEXT: S[f=1.000000, m=(nil)] + +quit + Index: clang/test/Interpreter/sanity.c =================================================================== --- /dev/null +++ clang/test/Interpreter/sanity.c @@ -0,0 +1,18 @@ +// RUN: cat %s | \ +// RUN: clang-repl -Xcc -fno-color-diagnostics -Xcc -Xclang -Xcc -ast-dump \ +// RUN: -Xcc -Xclang -Xcc -ast-dump-filter -Xcc -Xclang -Xcc Test | \ +// RUN: FileCheck %s + +int TestVar = 12; +// CHECK: Dumping TestVar: +// CHECK-NEXT: VarDecl [[var_ptr:0x[0-9a-f]+]] <{{.*}} TestVar 'int' cinit +// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 12 + +void TestFunc() { ++TestVar; } +// CHECK: Dumping TestFunc: +// CHECK-NEXT: FunctionDecl {{.*}} TestFunc 'void ()' +// CHECK-NEXT: CompoundStmt{{.*}} +// CHECK-NEXT: UnaryOperator{{.*}} 'int' lvalue prefix '++' +// CHECK-NEXT: DeclRefExpr{{.*}} 'int' lvalue Var [[var_ptr]] 'TestVar' 'int' + +quit Index: clang/tools/CMakeLists.txt =================================================================== --- clang/tools/CMakeLists.txt +++ clang/tools/CMakeLists.txt @@ -11,6 +11,7 @@ add_clang_subdirectory(clang-offload-bundler) add_clang_subdirectory(clang-offload-wrapper) add_clang_subdirectory(clang-scan-deps) +add_clang_subdirectory(clang-repl) add_clang_subdirectory(c-index-test) Index: clang/tools/clang-repl/CMakeLists.txt =================================================================== --- /dev/null +++ clang/tools/clang-repl/CMakeLists.txt @@ -0,0 +1,16 @@ +set( LLVM_LINK_COMPONENTS + Support + ) + +add_clang_tool(clang-repl + ClangRepl.cpp + ) + +target_link_libraries(clang-repl PUBLIC + clangInterpreter + clangTooling + LLVMLineEditor + ) + +install(TARGETS clang-repl + RUNTIME DESTINATION bin) Index: clang/tools/clang-repl/ClangRepl.cpp =================================================================== --- /dev/null +++ clang/tools/clang-repl/ClangRepl.cpp @@ -0,0 +1,86 @@ +//===--- tools/clang-repl/ClangRepl.cpp - clang-repl - the Clang REPL -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements a REPL tool on top of clang. +// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Diagnostic.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Interpreter/Interpreter.h" + +#include "llvm/LineEditor/LineEditor.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/ManagedStatic.h" // llvm_shutdown +#include "llvm/Support/Signals.h" +#include "llvm/Support/TargetSelect.h" // llvm::Initialize* + +static llvm::cl::list + ClangArgs("Xcc", llvm::cl::ZeroOrMore, + llvm::cl::desc("Argument to pass to the CompilerInvocation"), + llvm::cl::CommaSeparated); + +static void LLVMErrorHandler(void *UserData, const std::string &Message, + bool GenCrashDiag) { + auto &Diags = *static_cast(UserData); + + Diags.Report(clang::diag::err_fe_error_backend) << Message; + + // Run the interrupt handlers to make sure any special cleanups get done, in + // particular that we remove files registered with RemoveFileOnSignal. + llvm::sys::RunInterruptHandlers(); + + // We cannot recover from llvm errors. When reporting a fatal error, exit + // with status 70 to generate crash diagnostics. For BSD systems this is + // defined as an internal software error. Otherwise, exit with status 1. + + exit(GenCrashDiag ? 70 : 1); +} + +llvm::ExitOnError ExitOnErr; +int main(int argc, const char **argv) { + ExitOnErr.setBanner("clang-repl"); + llvm::cl::ParseCommandLineOptions(argc, argv); + + // Initialize targets first, so that --version shows registered targets. + llvm::InitializeAllTargets(); + llvm::InitializeAllTargetMCs(); + llvm::InitializeAllAsmPrinters(); + llvm::InitializeAllAsmParsers(); + + if (ClangArgs.empty()) { + ClangArgs.push_back("-Xclang"); + ClangArgs.push_back("-emit-llvm-only"); + } + std::vector ClangArgv(ClangArgs.size()); + std::transform(ClangArgs.begin(), ClangArgs.end(), ClangArgv.begin(), + [](const std::string &s) -> const char * { return s.data(); }); + // FIXME: Investigate if we could use runToolOnCodeWithArgs from tooling. It + // can replace the boilerplate code for creation of the compiler instance. + auto CI = ExitOnErr(clang::IncrementalCompilerBuilder::create(ClangArgv)); + + // Set an error handler, so that any LLVM backend diagnostics go through our + // error handler. + llvm::install_fatal_error_handler( + LLVMErrorHandler, static_cast(&CI->getDiagnostics())); + + auto Interp = ExitOnErr(clang::Interpreter::create(std::move(CI))); + llvm::LineEditor LE("clang-repl"); + // FIXME: Add LE.setListCompleter + while (llvm::Optional Line = LE.readLine()) { + if (*Line == "quit") + break; + if (auto Err = Interp->ParseAndExecute(*Line)) + llvm::logAllUnhandledErrors(std::move(Err), llvm::errs(), "error: "); + } + + llvm::llvm_shutdown(); + + return 0; +} Index: clang/unittests/CMakeLists.txt =================================================================== --- clang/unittests/CMakeLists.txt +++ clang/unittests/CMakeLists.txt @@ -34,6 +34,7 @@ add_subdirectory(Rewrite) add_subdirectory(Sema) add_subdirectory(CodeGen) +add_subdirectory(Interpreter) # FIXME: libclang unit tests are disabled on Windows due # to failures, mostly in libclang.VirtualFileOverlay_*. if(NOT WIN32 AND CLANG_TOOL_LIBCLANG_BUILD) Index: clang/unittests/CodeGen/CMakeLists.txt =================================================================== --- clang/unittests/CodeGen/CMakeLists.txt +++ clang/unittests/CodeGen/CMakeLists.txt @@ -17,6 +17,7 @@ clangBasic clangCodeGen clangFrontend + clangInterpreter clangLex clangParse clangSerialization Index: clang/unittests/CodeGen/IncrementalProcessingTest.cpp =================================================================== --- clang/unittests/CodeGen/IncrementalProcessingTest.cpp +++ clang/unittests/CodeGen/IncrementalProcessingTest.cpp @@ -6,14 +6,13 @@ // //===----------------------------------------------------------------------===// -#include "TestCompiler.h" - #include "clang/AST/ASTConsumer.h" #include "clang/AST/ASTContext.h" #include "clang/AST/RecursiveASTVisitor.h" #include "clang/Basic/TargetInfo.h" #include "clang/CodeGen/ModuleBuilder.h" #include "clang/Frontend/CompilerInstance.h" +#include "clang/Interpreter/Interpreter.h" #include "clang/Lex/Preprocessor.h" #include "clang/Parse/Parser.h" #include "clang/Sema/Sema.h" @@ -45,64 +44,8 @@ " EmitCXXGlobalInitFunc2() {}\n" "} test2;"; - -/// An incremental version of ParseAST(). -static std::unique_ptr -IncrementalParseAST(CompilerInstance& CI, Parser& P, - CodeGenerator& CG, const char* code) { - static int counter = 0; - struct IncreaseCounterOnRet { - ~IncreaseCounterOnRet() { - ++counter; - } - } ICOR; - - Sema& S = CI.getSema(); - clang::SourceManager &SM = S.getSourceManager(); - if (!code) { - // Main file - SM.setMainFileID(SM.createFileID( - llvm::MemoryBuffer::getMemBuffer(" "), clang::SrcMgr::C_User)); - - S.getPreprocessor().EnterMainSourceFile(); - P.Initialize(); - } else { - FileID FID = SM.createFileID( - llvm::MemoryBuffer::getMemBuffer(code), clang::SrcMgr::C_User); - SourceLocation MainStartLoc = SM.getLocForStartOfFile(SM.getMainFileID()); - SourceLocation InclLoc = MainStartLoc.getLocWithOffset(counter); - S.getPreprocessor().EnterSourceFile(FID, 0, InclLoc); - } - - ExternalASTSource *External = S.getASTContext().getExternalSource(); - if (External) - External->StartTranslationUnit(&CG); - - Parser::DeclGroupPtrTy ADecl; - for (bool AtEOF = P.ParseFirstTopLevelDecl(ADecl); !AtEOF; - AtEOF = P.ParseTopLevelDecl(ADecl)) { - // If we got a null return and something *was* parsed, ignore it. This - // is due to a top-level semicolon, an action override, or a parse error - // skipping something. - if (ADecl && !CG.HandleTopLevelDecl(ADecl.get())) - return nullptr; - } - - // Process any TopLevelDecls generated by #pragma weak. - for (Decl *D : S.WeakTopLevelDecls()) - CG.HandleTopLevelDecl(DeclGroupRef(D)); - - CG.HandleTranslationUnit(S.getASTContext()); - - std::unique_ptr M(CG.ReleaseModule()); - // Switch to next module. - CG.StartModule("incremental-module-" + std::to_string(counter), - M->getContext()); - return M; -} - -const Function* getGlobalInit(llvm::Module& M) { - for (const auto& Func: M) +const Function* getGlobalInit(llvm::Module* M) { + for (const auto& Func: *M) if (Func.hasName() && Func.getName().startswith("_GLOBAL__sub_I_")) return &Func; @@ -110,46 +53,30 @@ } TEST(IncrementalProcessing, EmitCXXGlobalInitFunc) { - clang::LangOptions LO; - LO.CPlusPlus = 1; - LO.CPlusPlus11 = 1; - TestCompiler Compiler(LO); - clang::CompilerInstance &CI = Compiler.compiler; - CI.getPreprocessor().enableIncrementalProcessing(); - CI.setASTConsumer(std::move(Compiler.CG)); - clang::CodeGenerator& CG = - static_cast(CI.getASTConsumer()); - CI.createSema(clang::TU_Prefix, nullptr); - - Sema& S = CI.getSema(); - - std::unique_ptr ParseOP(new Parser(S.getPreprocessor(), S, - /*SkipFunctionBodies*/ false)); - Parser &P = *ParseOP.get(); - - std::array, 3> M; - M[0] = IncrementalParseAST(CI, P, CG, nullptr); - ASSERT_TRUE(M[0]); - - M[1] = IncrementalParseAST(CI, P, CG, TestProgram1); - ASSERT_TRUE(M[1]); - ASSERT_TRUE(M[1]->getFunction("funcForProg1")); - - M[2] = IncrementalParseAST(CI, P, CG, TestProgram2); - ASSERT_TRUE(M[2]); - ASSERT_TRUE(M[2]->getFunction("funcForProg2")); - // First code should not end up in second module: - ASSERT_FALSE(M[2]->getFunction("funcForProg1")); - - // Make sure global inits exist and are unique: - const Function* GlobalInit1 = getGlobalInit(*M[1]); - ASSERT_TRUE(GlobalInit1); - - const Function* GlobalInit2 = getGlobalInit(*M[2]); - ASSERT_TRUE(GlobalInit2); - - ASSERT_FALSE(GlobalInit1->getName() == GlobalInit2->getName()); + std::vector ClangArgv = {"-Xclang", "-emit-llvm-only"}; + auto CI = llvm::cantFail(IncrementalCompilerBuilder::create(ClangArgv)); + auto Interp = llvm::cantFail(Interpreter::create(std::move(CI))); + + std::array Transactions; + + Transactions[0] = &llvm::cantFail(Interp->Parse(TestProgram1)); + ASSERT_TRUE(Transactions[0]->TheModule); + ASSERT_TRUE(Transactions[0]->TheModule->getFunction("funcForProg1")); + + Transactions[1] = &llvm::cantFail(Interp->Parse(TestProgram2)); + ASSERT_TRUE(Transactions[1]->TheModule); + ASSERT_TRUE(Transactions[1]->TheModule->getFunction("funcForProg2")); + // First code should not end up in second module: + ASSERT_FALSE(Transactions[1]->TheModule->getFunction("funcForProg1")); + + // Make sure global inits exist and are unique: + const Function* GlobalInit1 = getGlobalInit(Transactions[0]->TheModule.get()); + ASSERT_TRUE(GlobalInit1); + + const Function* GlobalInit2 = getGlobalInit(Transactions[1]->TheModule.get()); + ASSERT_TRUE(GlobalInit2); + ASSERT_FALSE(GlobalInit1->getName() == GlobalInit2->getName()); } } // end anonymous namespace Index: clang/unittests/Interpreter/CMakeLists.txt =================================================================== --- /dev/null +++ clang/unittests/Interpreter/CMakeLists.txt @@ -0,0 +1,10 @@ +set(LLVM_LINK_COMPONENTS + ) + +add_clang_unittest(ClangReplInterpreterTests + InterpreterTest.cpp + ) +target_link_libraries(ClangReplInterpreterTests PUBLIC + clangInterpreter + clangFrontend + ) Index: clang/unittests/Interpreter/InterpreterTest.cpp =================================================================== --- /dev/null +++ clang/unittests/Interpreter/InterpreterTest.cpp @@ -0,0 +1,93 @@ +//===- unittests/Interpreter/InterpreterTest.cpp --- Interpreter tests ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Unit tests for our Interpreter library. +// +//===----------------------------------------------------------------------===// + +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Interpreter/Interpreter.h" + +#include "clang/AST/Decl.h" +#include "clang/AST/DeclGroup.h" + +#include "llvm/ADT/ArrayRef.h" + +#include "gtest/gtest.h" + +using namespace clang; + +namespace { + +static std::unique_ptr createInterpreter() { + std::vector ClangArgs = {"-Xclang", "-emit-llvm-only"}; + auto CI = cantFail(clang::IncrementalCompilerBuilder::create(ClangArgs)); + return std::move(cantFail(clang::Interpreter::create(std::move(CI)))); +} + +TEST(InterpreterTest, Sanity) { + std::unique_ptr Interp = createInterpreter(); + Transaction &R1(cantFail(Interp->Parse("void g(); void g() {}"))); + EXPECT_EQ(2U, R1.Decls.size()); + + Transaction &R2(cantFail(Interp->Parse("int i;"))); + EXPECT_EQ(1U, R2.Decls.size()); +} + +static std::string DeclToString(DeclGroupRef DGR) { + return llvm::cast(DGR.getSingleDecl())->getQualifiedNameAsString(); +} + +TEST(InterpreterTest, IncrementalInputTopLevelDecls) { + std::unique_ptr Interp = createInterpreter(); + auto R1OrErr = Interp->Parse("int var1 = 42; int f() { return var1; }"); + // gtest doesn't expand into explicit bool conversions. + EXPECT_TRUE(!!R1OrErr); + auto R1 = R1OrErr->Decls; + EXPECT_EQ(2U, R1.size()); + EXPECT_EQ("var1", DeclToString(R1[0])); + EXPECT_EQ("f", DeclToString(R1[1])); + + auto R2OrErr = Interp->Parse("int var2 = f();"); + EXPECT_TRUE(!!R2OrErr); + auto R2 = R2OrErr->Decls; + EXPECT_EQ(1U, R2.size()); + EXPECT_EQ("var2", DeclToString(R2[0])); +} + + +TEST(InterpreterTest, Errors) { + std::unique_ptr Interp = createInterpreter(); + auto Err = Interp->Parse("intentional_error v1 = 42; ").takeError(); + EXPECT_EQ("Parsing failed.", llvm::toString(std::move(Err))); + + EXPECT_DEATH((void)Interp->Parse("int var1 = 42;"), ""); +} + +// Here we test whether the user can mix declarations and statements. The +// interpreter should be smart enough to recognize the declarations from the +// statements and wrap the latter into a declaration, producing valid code. +TEST(InterpreterTest, DeclsAndStatements) { + std::unique_ptr Interp = createInterpreter(); + auto R1OrErr = Interp->Parse( + "int var1 = 42; extern \"C\" int printf(const char*, ...);"); + // gtest doesn't expand into explicit bool conversions. + EXPECT_TRUE(!!R1OrErr); + + auto R1 = R1OrErr->Decls; + EXPECT_EQ(2U, R1.size()); + + // FIXME: Add support for wrapping and running statements. + auto R2OrErr = + Interp->Parse("var1++; printf(\"var1 value is %d\\n\", var1);"); + EXPECT_FALSE(!!R2OrErr); + auto Err = R2OrErr.takeError(); + EXPECT_EQ("Parsing failed.", llvm::toString(std::move(Err))); +} + +} // end anonymous namespace