Index: clang/include/clang/Driver/Options.td =================================================================== --- clang/include/clang/Driver/Options.td +++ clang/include/clang/Driver/Options.td @@ -2822,6 +2822,13 @@ NegFlag, BothFlags<[], [ClangOption, CC1Option], " __declspec as a keyword">>, Group; + +def fmodule_build_daemon : Flag<["-"], "fmodule-build-daemon">, Group, + Flags<[NoXarchOption]>, + Visibility<[ClangOption, CC1Option]>, + HelpText<"Enables module build daemon functionality">, + MarshallingInfoFlag>; + def fmodules_cache_path : Joined<["-"], "fmodules-cache-path=">, Group, Flags<[NoXarchOption]>, Visibility<[ClangOption, CC1Option]>, MetaVarName<"">, Index: clang/include/clang/Frontend/FrontendOptions.h =================================================================== --- clang/include/clang/Frontend/FrontendOptions.h +++ clang/include/clang/Frontend/FrontendOptions.h @@ -347,6 +347,9 @@ /// Whether to share the FileManager when building modules. unsigned ModulesShareFileManager : 1; + /// Connect to module build daemon + unsigned ModuleBuildDaemon : 1; + CodeCompleteOptions CodeCompleteOpts; /// Specifies the output format of the AST. Index: clang/include/clang/Tooling/ModuleBuildDaemon/Protocol.h =================================================================== --- /dev/null +++ clang/include/clang/Tooling/ModuleBuildDaemon/Protocol.h @@ -0,0 +1,109 @@ +//===---------------------------- Protocall.h -----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_PROTOCAL_H +#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_PROTOCAL_H + +#include "clang/Frontend/CompilerInstance.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/YAMLParser.h" +#include "llvm/Support/YAMLTraits.h" + +#define MAX_BUFFER 4096 +#define SOCKET_FILE_NAME "mbd.sock" +#define STDOUT_FILE_NAME "mbd.out" +#define STDERR_FILE_NAME "mbd.err" + +using namespace clang; +using namespace llvm; + +namespace cc1modbuildd { + +enum class ActionType { REGISTER, HANDSHAKE }; +enum class StatusType { REQUEST, SUCCESS, FAILURE }; + +struct SocketMsg { + ActionType MsgAction; + StatusType MsgStatus; + std::optional WorkingDirectory; + // First element needs to be path to compiler + std::optional> Argv0PlusCC1CommandLine; + + SocketMsg() = default; + + SocketMsg(ActionType Action, StatusType Status, + const std::optional &CurrentWD, + const std::optional> &CommandLine) + : MsgAction(Action), MsgStatus(Status), WorkingDirectory(CurrentWD), + Argv0PlusCC1CommandLine(CommandLine) {} + + SocketMsg(ActionType Action, StatusType Status) + : MsgAction(Action), MsgStatus(Status), WorkingDirectory(std::nullopt), + Argv0PlusCC1CommandLine(std::nullopt) {} +}; + +// Create unbuffered STDOUT stream so that any logging done by module build +// daemon can be viewed without having to terminate the process +raw_fd_ostream &ub_outs(); + +SmallString<128> getBasePath(); + +bool daemonExists(StringRef BasePath); + +std::string getBufferFromSocketMsg(SocketMsg Command); + +Expected getSocketMsgFromBuffer(char *Buffer); + +Expected connectToSocketAndHandshake(StringRef SocketPath); + +llvm::Error attemptHandshake(int SocketFD); + +llvm::Error getModuleBuildDaemon(const char *Argv0, StringRef BasePath); + +llvm::Error spawnModuleBuildDaemon(StringRef BasePath, const char *Argv0); + +llvm::Error registerTranslationUnit(CompilerInstance &Clang, StringRef Argv0, + StringRef BasePath); + +// Work in progress. Eventually function will modify CC1 command line to include +// path to modules already built by the daemon +void updateCC1WithModuleBuildDaemon(CompilerInstance &Clang, const char *Argv0); + +llvm::Error scanTranslationUnit(SocketMsg Command); + +} // namespace cc1modbuildd + +template <> +struct llvm::yaml::ScalarEnumerationTraits { + static void enumeration(IO &io, cc1modbuildd::StatusType &value) { + io.enumCase(value, "REQUEST", cc1modbuildd::StatusType::REQUEST); + io.enumCase(value, "SUCCESS", cc1modbuildd::StatusType::SUCCESS); + io.enumCase(value, "FAILURE", cc1modbuildd::StatusType::FAILURE); + } +}; + +template <> +struct llvm::yaml::ScalarEnumerationTraits { + static void enumeration(IO &io, cc1modbuildd::ActionType &value) { + io.enumCase(value, "REGISTER", cc1modbuildd::ActionType::REGISTER); + io.enumCase(value, "HANDSHAKE", cc1modbuildd::ActionType::HANDSHAKE); + } +}; + +template <> struct llvm::yaml::MappingTraits { + static void mapping(IO &io, cc1modbuildd::SocketMsg &info) { + io.mapRequired("Action", info.MsgAction); + io.mapRequired("Status", info.MsgStatus); + io.mapOptional("WorkingDirectory", info.WorkingDirectory); + io.mapOptional("FullCommandLine", info.Argv0PlusCC1CommandLine); + } +}; + +#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_PROTOCAL_H Index: clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h =================================================================== --- /dev/null +++ clang/include/clang/Tooling/ModuleBuildDaemon/SocketSupport.h @@ -0,0 +1,31 @@ +//===-------------------------- SocketSupport.h ---------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETSUPPORT_H +#define LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETSUPPORT_H + +#include "clang/Frontend/CompilerInstance.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/YAMLParser.h" +#include "llvm/Support/YAMLTraits.h" + +using namespace clang; +using namespace llvm; + +namespace cc1modbuildd { + +Expected createSocket(); +Expected connectToSocket(StringRef SocketPath); +Expected connectAndWriteToSocket(std::string Buffer, StringRef SocketPath); +Expected> readFromSocket(int FD); +llvm::Error writeToSocket(std::string Buffer, int WriteFD); + +} // namespace cc1modbuildd + +#endif // LLVM_CLANG_TOOLING_MODULEBUILDDAEMON_SOCKETSUPPORT_H \ No newline at end of file Index: clang/lib/Driver/ToolChains/Clang.cpp =================================================================== --- clang/lib/Driver/ToolChains/Clang.cpp +++ clang/lib/Driver/ToolChains/Clang.cpp @@ -5,7 +5,6 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// - #include "Clang.h" #include "AMDGPU.h" #include "Arch/AArch64.h" @@ -3737,6 +3736,10 @@ Std->containsValue("c++latest") || Std->containsValue("gnu++latest")); bool HaveModules = HaveStdCXXModules; + // -fmodule-build-daemon enables the module build daemon functionality + if (Args.hasArg(options::OPT_fmodule_build_daemon)) + Args.AddLastArg(CmdArgs, options::OPT_fmodule_build_daemon); + // -fmodules enables the use of precompiled modules (off by default). // Users can pass -fno-cxx-modules to turn off modules support for // C++/Objective-C++ programs. Index: clang/lib/Tooling/CMakeLists.txt =================================================================== --- clang/lib/Tooling/CMakeLists.txt +++ clang/lib/Tooling/CMakeLists.txt @@ -13,6 +13,7 @@ add_subdirectory(Syntax) add_subdirectory(DependencyScanning) add_subdirectory(Transformer) +add_subdirectory(ModuleBuildDaemon) # Replace the last lib component of the current binary directory with include string(FIND ${CMAKE_CURRENT_BINARY_DIR} "/lib/" PATH_LIB_START REVERSE) Index: clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt =================================================================== --- /dev/null +++ clang/lib/Tooling/ModuleBuildDaemon/CMakeLists.txt @@ -0,0 +1,8 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_library(clangModuleBuildDaemon + Protocol.cpp + SocketSupport.cpp + ) \ No newline at end of file Index: clang/lib/Tooling/ModuleBuildDaemon/Protocol.cpp =================================================================== --- /dev/null +++ clang/lib/Tooling/ModuleBuildDaemon/Protocol.cpp @@ -0,0 +1,277 @@ +//===---------------------------- Protocol.cpp ----------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/ModuleBuildDaemon/Protocol.h" +#include "clang/Basic/Version.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h" +#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/BLAKE3.h" + +// TODO: Make portable +#if LLVM_ON_UNIX + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace clang; +using namespace llvm; + +static std::string getWorkingDir() { + return std::filesystem::current_path().string(); +} + +raw_fd_ostream &cc1modbuildd::ub_outs() { + static raw_fd_ostream S(STDOUT_FILENO, false, true); + return S; +} + +Expected cc1modbuildd::connectToSocketAndHandshake(StringRef SocketPath) { + + Expected ConnectedFD = connectToSocket(SocketPath); + if (!ConnectedFD) + return std::move(ConnectedFD.takeError()); + + llvm::Error Err = attemptHandshake(std::move(*ConnectedFD)); + if (Err) + return std::move(Err); + + return ConnectedFD; +} + +bool cc1modbuildd::daemonExists(StringRef BasePath) { + + SmallString<128> SocketPath = BasePath; + llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME); + + if (!llvm::sys::fs::exists(SocketPath)) + return false; + + Expected ConnectedFD = connectToSocketAndHandshake(SocketPath); + if (ConnectedFD) { + close(std::move(*ConnectedFD)); + return true; + } + + consumeError(ConnectedFD.takeError()); + return false; +} + +llvm::Error cc1modbuildd::spawnModuleBuildDaemon(StringRef BasePath, + const char *Argv0) { + std::string BasePathStr = BasePath.str(); + const char *Args[] = {Argv0, "-cc1modbuildd", BasePathStr.c_str(), nullptr}; + pid_t pid; + int EC = posix_spawn(&pid, Args[0], + /*file_actions*/ nullptr, + /*spawnattr*/ nullptr, const_cast(Args), + /*envp*/ nullptr); + if (EC) + return createStringError(std::error_code(EC, std::generic_category()), + "failed to spawn module build daemon process"); + + return llvm::Error::success(); +} + +SmallString<128> cc1modbuildd::getBasePath() { + llvm::BLAKE3 Hash; + Hash.update(getClangFullVersion()); + auto HashResult = Hash.final(); + uint64_t HashValue = + llvm::support::endian::read( + HashResult.data()); + std::string Key = toString(llvm::APInt(64, HashValue), 36, /*Signed*/ false); + + // set paths + SmallString<128> BasePath; + llvm::sys::path::system_temp_directory(/*erasedOnReboot*/ true, BasePath); + llvm::sys::path::append(BasePath, "clang-" + Key); + return BasePath; +} + +llvm::Error cc1modbuildd::attemptHandshake(int SocketFD) { + + cc1modbuildd::SocketMsg Request{ActionType::HANDSHAKE, StatusType::REQUEST}; + std::string Buffer = cc1modbuildd::getBufferFromSocketMsg(Request); + + if (llvm::Error Err = writeToSocket(Buffer, SocketFD)) + return std::move(Err); + + return llvm::Error::success(); +} + +std::string cc1modbuildd::getBufferFromSocketMsg(SocketMsg SocketMsg) { + + std::string Buffer; + llvm::raw_string_ostream OS(Buffer); + llvm::yaml::Output YamlOut(OS); + + YamlOut << SocketMsg; + return Buffer; +} + +Expected +cc1modbuildd::getSocketMsgFromBuffer(char *Buffer) { + + SocketMsg ClientRequest; + llvm::yaml::Input YamlIn(Buffer); + YamlIn >> ClientRequest; + + if (YamlIn.error()) { + std::string Msg = "Syntax or semantic error during YAML parsing"; + return llvm::make_error(Msg, inconvertibleErrorCode()); + } + + return ClientRequest; +} + +llvm::Error cc1modbuildd::getModuleBuildDaemon(const char *Argv0, + StringRef BasePath) { + + // If module build daemon already exist return success + if (cc1modbuildd::daemonExists(BasePath)) { + return llvm::Error::success(); + } + + if (llvm::Error Err = cc1modbuildd::spawnModuleBuildDaemon(BasePath, Argv0)) + return std::move(Err); + + sleep(10); + + // Confirm that module build daemon was created + if (cc1modbuildd::daemonExists(BasePath)) + return llvm::Error::success(); + + return llvm::make_error( + "Module build daemon did not exist after spawn attempt", + inconvertibleErrorCode()); +} + +void cc1modbuildd::updateCC1WithModuleBuildDaemon(CompilerInstance &Clang, + const char *Argv0) { + + SmallString<128> BasePath = cc1modbuildd::getBasePath(); + + llvm::Error DaemonErr = cc1modbuildd::getModuleBuildDaemon(Argv0, BasePath); + if (DaemonErr) { + handleAllErrors(std::move(DaemonErr), [&](ErrorInfoBase &EIB) { + errs() << "Connect to daemon failed: " << EIB.message() << "\n"; + }); + return; + } + + llvm::Error RegisterErr = + cc1modbuildd::registerTranslationUnit(Clang, Argv0, BasePath); + if (RegisterErr) { + handleAllErrors(std::move(RegisterErr), [&](ErrorInfoBase &EIB) { + errs() << "Register translation unit failed: " << EIB.message() << "\n"; + }); + return; + } + + return; +} + +llvm::Error cc1modbuildd::registerTranslationUnit(CompilerInstance &Clang, + StringRef Argv0, + StringRef BasePath) { + + std::vector CC1Cmd = Clang.getInvocation().getCC1CommandLine(); + CC1Cmd.insert(CC1Cmd.begin(), Argv0.str()); + cc1modbuildd::SocketMsg Request{ActionType::REGISTER, StatusType::REQUEST, + getWorkingDir(), CC1Cmd}; + + std::string Buffer = getBufferFromSocketMsg(Request); + + // FIXME: Should not need to append again here + SmallString<128> SocketPath = BasePath; + llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME); + + Expected MaybeServerFD = connectAndWriteToSocket(Buffer, SocketPath); + if (!MaybeServerFD) + return std::move(MaybeServerFD.takeError()); + + // Blocks cc1 invocation until module build daemon is done processing + // translation unit. Currently receives a SUCCESS message and returns + // llvm::Error::success() but will eventually recive updated cc1 command line + Expected> MaybeResponseBuffer = + readFromSocket(std::move(*MaybeServerFD)); + if (!MaybeResponseBuffer) + return std::move(MaybeResponseBuffer.takeError()); + + // Wait for response from module build daemon + Expected MaybeResponse = + getSocketMsgFromBuffer(std::move(*MaybeResponseBuffer).get()); + if (!MaybeResponse) + return std::move(MaybeResponse.takeError()); + SocketMsg ServerResponse = std::move(*MaybeResponse); + + assert(ServerResponse.MsgAction == ActionType::REGISTER && + "ActionType should only be REGISTER"); + if (ServerResponse.MsgStatus == StatusType::SUCCESS) + return llvm::Error::success(); + + return llvm::make_error( + "Daemon failed to processes registered translation unit", + inconvertibleErrorCode()); +} + +llvm::Error cc1modbuildd::scanTranslationUnit(SocketMsg Request) { + + tooling::dependencies::DependencyScanningService Service( + tooling::dependencies::ScanningMode::DependencyDirectivesScan, + tooling::dependencies::ScanningOutputFormat::Full, + /*OptimizeArgs*/ false, + /*EagerLoadModules*/ false); + + tooling::dependencies::DependencyScanningTool Tool(Service); + + llvm::DenseSet AlreadySeenModules; + auto LookupOutput = [&](const tooling::dependencies::ModuleID &MID, + tooling::dependencies::ModuleOutputKind MOK) { + return MID.ContextHash; + }; + + auto MaybeFile = Tool.getTranslationUnitDependencies( + Request.Argv0PlusCC1CommandLine.value(), Request.WorkingDirectory.value(), + AlreadySeenModules, LookupOutput); + + if (!MaybeFile) + return std::move(MaybeFile.takeError()); + + tooling::dependencies::TranslationUnitDeps TUDeps = std::move(*MaybeFile); + + // For now write dependencies to log file + for (auto const &Dep : TUDeps.FileDeps) { + cc1modbuildd::ub_outs() << Dep << '\n'; + } + + if (!TUDeps.ModuleGraph.empty()) + errs() << "Warning: translation unit contained modules. Module build " + "daemon not yet able to build modules" + << '\n'; + + return llvm::Error::success(); +} + +#endif // LLVM_ON_UNIX \ No newline at end of file Index: clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp =================================================================== --- /dev/null +++ clang/lib/Tooling/ModuleBuildDaemon/SocketSupport.cpp @@ -0,0 +1,106 @@ +//===------------------------- SocketSupport.cpp --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h" +#include "clang/Basic/Version.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Tooling/ModuleBuildDaemon/Protocol.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/BLAKE3.h" + +// TODO: Make portable +#if LLVM_ON_UNIX + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +Expected cc1modbuildd::createSocket() { + int FD; + if ((FD = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { + std::string Msg = "socket create error: " + std::string(strerror(errno)); + return createStringError(inconvertibleErrorCode(), Msg); + } + return FD; +} + +Expected cc1modbuildd::connectToSocket(StringRef SocketPath) { + + Expected MaybeFD = cc1modbuildd::createSocket(); + if (!MaybeFD) + return std::move(MaybeFD.takeError()); + + int FD = std::move(*MaybeFD); + + struct sockaddr_un Addr; + memset(&Addr, 0, sizeof(Addr)); + Addr.sun_family = AF_UNIX; + strncpy(Addr.sun_path, SocketPath.str().c_str(), sizeof(Addr.sun_path) - 1); + + if (connect(FD, (struct sockaddr *)&Addr, sizeof(Addr)) == -1) { + close(FD); + std::string msg = "socket connect error: " + std::string(strerror(errno)); + return createStringError(inconvertibleErrorCode(), msg); + } + return FD; +} + +Expected cc1modbuildd::connectAndWriteToSocket(std::string Buffer, + StringRef SocketPath) { + + Expected MaybeConnectedFD = connectToSocket(SocketPath); + if (!MaybeConnectedFD) + return std::move(MaybeConnectedFD.takeError()); + + int ConnectedFD = std::move(*MaybeConnectedFD); + llvm::Error Err = writeToSocket(Buffer, ConnectedFD); + if (Err) + return std::move(Err); + + return ConnectedFD; +} + +Expected> cc1modbuildd::readFromSocket(int FD) { + + std::unique_ptr Buffer(new char[MAX_BUFFER]); + memset(Buffer.get(), 0, MAX_BUFFER); + int n = read(FD, Buffer.get(), MAX_BUFFER); + + if (n < 0) { + std::string Msg = "socket read error: " + std::string(strerror(errno)); + return llvm::make_error(Msg, inconvertibleErrorCode()); + } + if (n == 0) + return llvm::make_error("EOF", inconvertibleErrorCode()); + + return Buffer; +} + +llvm::Error cc1modbuildd::writeToSocket(std::string Buffer, int WriteFD) { + + ssize_t MessageSize = static_cast(Buffer.size()); + + if (write(WriteFD, Buffer.c_str(), Buffer.size()) != MessageSize) { + std::string Msg = "socket write error: " + std::string(strerror(errno)); + return llvm::make_error(Msg, inconvertibleErrorCode()); + } + return llvm::Error::success(); +} + +#endif // LLVM_ON_UNIX \ No newline at end of file Index: clang/tools/driver/CMakeLists.txt =================================================================== --- clang/tools/driver/CMakeLists.txt +++ clang/tools/driver/CMakeLists.txt @@ -28,6 +28,7 @@ cc1_main.cpp cc1as_main.cpp cc1gen_reproducer_main.cpp + cc1modbuildd_main.cpp DEPENDS intrinsics_gen @@ -39,9 +40,11 @@ PRIVATE clangBasic clangCodeGen + clangDependencyScanning clangDriver clangFrontend clangFrontendTool + clangModuleBuildDaemon clangSerialization ) Index: clang/tools/driver/cc1_main.cpp =================================================================== --- clang/tools/driver/cc1_main.cpp +++ clang/tools/driver/cc1_main.cpp @@ -25,6 +25,7 @@ #include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/Frontend/Utils.h" #include "clang/FrontendTool/Utils.h" +#include "clang/Tooling/ModuleBuildDaemon/Protocol.h" #include "llvm/ADT/Statistic.h" #include "llvm/Config/llvm-config.h" #include "llvm/LinkAllPasses.h" @@ -45,6 +46,7 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include +#include #ifdef CLANG_HAVE_RLIMITS #include @@ -243,6 +245,12 @@ return 1; } +#if LLVM_ON_UNIX + // handle module build daemon functionality if enabled + if (Clang->getInvocation().getFrontendOpts().ModuleBuildDaemon) + cc1modbuildd::updateCC1WithModuleBuildDaemon(*Clang, Argv0); +#endif + // Execute the frontend actions. { llvm::TimeTraceScope TimeScope("ExecuteCompiler"); Index: clang/tools/driver/cc1modbuildd_main.cpp =================================================================== --- /dev/null +++ clang/tools/driver/cc1modbuildd_main.cpp @@ -0,0 +1,265 @@ +//===------- cc1modbuildd_main.cpp - Clang CC1 Module Build Daemon --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/Tooling/ModuleBuildDaemon/Protocol.h" +#include "clang/Tooling/ModuleBuildDaemon/SocketSupport.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/ThreadPool.h" +#include "llvm/Support/Threading.h" +#include "llvm/Support/YAMLParser.h" +#include "llvm/Support/YAMLTraits.h" + +// TODO: Make portable +#if LLVM_ON_UNIX + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace llvm; +using namespace clang; + +namespace { +class ModuleBuildDaemonServer { +public: + SmallString<128> BasePath; + SmallString<128> SocketPath; + SmallString<128> PidPath; + + ModuleBuildDaemonServer(SmallString<128> Path) + : BasePath(Path), SocketPath(Path) { + llvm::sys::path::append(SocketPath, SOCKET_FILE_NAME); + } + + ~ModuleBuildDaemonServer() { Shutdown(SIGTERM); } + + int Fork(); + int Launch(); + int Listen(); + static llvm::Error Service(int Client); + + void Shutdown(int signal) { + unlink(SocketPath.c_str()); + shutdown(ListenSocketFD, SHUT_RD); + close(ListenSocketFD); + exit(EXIT_SUCCESS); + } + +private: + pid_t Pid = -1; + int ListenSocketFD = -1; +}; + +// Required to handle SIGTERM by calling Shutdown +ModuleBuildDaemonServer *DaemonPtr = nullptr; +void HandleSignal(int signal) { + if (DaemonPtr != nullptr) { + DaemonPtr->Shutdown(signal); + } +} +} // namespace + +// Forks and detaches process, creating module build daemon +int ModuleBuildDaemonServer::Fork() { + + pid_t pid = fork(); + + if (pid < 0) { + exit(EXIT_FAILURE); + } + if (pid > 0) { + exit(EXIT_SUCCESS); + } + + Pid = getpid(); + + close(STDIN_FILENO); + close(STDOUT_FILENO); + close(STDERR_FILENO); + + SmallString<128> STDOUT = BasePath; + llvm::sys::path::append(STDOUT, STDOUT_FILE_NAME); + freopen(STDOUT.c_str(), "a", stdout); + + SmallString<128> STDERR = BasePath; + llvm::sys::path::append(STDERR, STDERR_FILE_NAME); + freopen(STDERR.c_str(), "a", stderr); + + if (signal(SIGTERM, HandleSignal) == SIG_ERR) { + errs() << "failed to handle SIGTERM" << '\n'; + exit(EXIT_FAILURE); + } + if (signal(SIGHUP, SIG_IGN) == SIG_ERR) { + errs() << "failed to ignore SIGHUP" << '\n'; + exit(EXIT_FAILURE); + } + if (setsid() == -1) { + errs() << "setsid failed" << '\n'; + exit(EXIT_FAILURE); + } + + return EXIT_SUCCESS; +} + +// Creates unix socket for IPC with module build daemon +int ModuleBuildDaemonServer::Launch() { + + // new socket + if ((ListenSocketFD = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) { + std::perror("Socket create error: "); + exit(EXIT_FAILURE); + } + + struct sockaddr_un addr; + memset(&addr, 0, sizeof(struct sockaddr_un)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, SocketPath.c_str(), sizeof(addr.sun_path) - 1); + + // bind to local address + if (bind(ListenSocketFD, (struct sockaddr *)&addr, sizeof(addr)) == -1) { + + // If the socket address is already in use, exit because another module + // build daemon has successfully launched. When translation units are + // compiled in parallel, until the socket file is created, all clang + // invocations will spawn a module build daemon. + if (errno == EADDRINUSE) { + close(ListenSocketFD); + exit(EXIT_SUCCESS); + } + std::perror("Socket bind error: "); + exit(EXIT_FAILURE); + } + + // set socket to accept incoming connection request + unsigned MaxBacklog = llvm::hardware_concurrency().compute_thread_count(); + if (listen(ListenSocketFD, MaxBacklog) == -1) { + std::perror("Socket listen error: "); + exit(EXIT_FAILURE); + } + + cc1modbuildd::ub_outs() << "daemon initialization complete!" << '\n'; + return 0; +} + +// Function submitted to thread pool with each client connection. Not +// responsible for closing client connections +llvm::Error ModuleBuildDaemonServer::Service(int Client) { + + // Read buffer from client connection + Expected> MaybeClientBuffer = + cc1modbuildd::readFromSocket(Client); + if (!MaybeClientBuffer) + return std::move(MaybeClientBuffer.takeError()); + std::unique_ptr ClientBuffer = std::move(*MaybeClientBuffer); + + // Convert buffer into SocketMsg + Expected MaybeClientRequest = + cc1modbuildd::getSocketMsgFromBuffer(ClientBuffer.get()); + if (!MaybeClientRequest) + return std::move(MaybeClientRequest.takeError()); + cc1modbuildd::SocketMsg ClientRequest = std::move(*MaybeClientRequest); + + // Handle HANDSHAKE + // Currently cc1 invocations do not wait for response to HANDSHAKE SocketMsg + if (ClientRequest.MsgAction == cc1modbuildd::ActionType::HANDSHAKE) + return llvm::Error::success(); + + // Handle REGISTER + if (ClientRequest.MsgAction == cc1modbuildd::ActionType::REGISTER) { + llvm::Error ScanErr = cc1modbuildd::scanTranslationUnit(ClientRequest); + if (ScanErr) + return std::move(ScanErr); + + std::string ServerResponse = cc1modbuildd::getBufferFromSocketMsg( + {cc1modbuildd::ActionType::REGISTER, + cc1modbuildd::StatusType::SUCCESS}); + // Unblocks cc1 invocation + llvm::Error WriteErr = cc1modbuildd::writeToSocket(ServerResponse, Client); + if (WriteErr) + return std::move(WriteErr); + + return llvm::Error::success(); + } + + // Conditional should exist for each ActionType + llvm_unreachable("Unrecognized Action"); +} + +int ModuleBuildDaemonServer::Listen() { + + llvm::ThreadPool Pool; + int Client; + + while (true) { + + if ((Client = accept(ListenSocketFD, NULL, NULL)) == -1) { + std::perror("Socket accept error: "); + continue; + } + + // FIXME: Error messages will be over written as results are returned + std::shared_future result = Pool.async(Service, Client); + llvm::Error Err = std::move(const_cast(result.get())); + + if (Err) { + handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { + errs() << "Error while scanning: " << EIB.message() << '\n'; + }); + } + + close(Client); + } + return 0; +} + +// Module build daemon is spawned with the following command line: +// +// clang -cc1modbuildd +// +// defines the location of all files created by the module build daemon +// and should follow the format /path/to/dir. For example, `clang -cc1modbuildd +// /tmp/` creates a socket file at `/tmp/mbd.sock`. /tmp is also valid. +// +// When module build daemons are spawned by cc1 invocations, follows the +// format /tmp/clang- +// +int cc1modbuildd_main(ArrayRef Argv) { + + if (Argv.size() < 1) { + outs() << "spawning a module build daemon requies a command line format of " + "`clang -cc1modbuildd `. defines where the module " + "build daemon will create files" + << '\n'; + return 1; + } + + // TODO: Add check to confirm BasePath is of correct format + SmallString<128> BasePath(Argv[0]); + llvm::sys::fs::create_directories(BasePath); + ModuleBuildDaemonServer Daemon(BasePath); + + // Used to handle signals + DaemonPtr = &Daemon; + + Daemon.Fork(); + Daemon.Launch(); + Daemon.Listen(); + + return 0; +} + +#endif // LLVM_ON_UNIX \ No newline at end of file Index: clang/tools/driver/driver.cpp =================================================================== --- clang/tools/driver/driver.cpp +++ clang/tools/driver/driver.cpp @@ -213,6 +213,9 @@ extern int cc1gen_reproducer_main(ArrayRef Argv, const char *Argv0, void *MainAddr, const llvm::ToolContext &); +#if LLVM_ON_UNIX +extern int cc1modbuildd_main(ArrayRef Argv); +#endif static void insertTargetAndModeArgs(const ParsedClangName &NameParts, SmallVectorImpl &ArgVector, @@ -369,9 +372,20 @@ if (Tool == "-cc1gen-reproducer") return cc1gen_reproducer_main(ArrayRef(ArgV).slice(2), ArgV[0], GetExecutablePathVP, ToolContext); - // Reject unknown tools. +#if LLVM_ON_UNIX + if (Tool == "-cc1modbuildd") + return cc1modbuildd_main(ArrayRef(ArgV).slice(2)); + + // FIXME: Once cc1modbuildd becomes portable unify llvm::errs messages llvm::errs() << "error: unknown integrated tool '" << Tool << "'. " - << "Valid tools include '-cc1' and '-cc1as'.\n"; + << "Valid tools include '-cc1', '-cc1as', '-cc1gen-reproducer', " + << "and '-cc1modbuildd'.\n"; +#else + // Reject unknown tools + llvm::errs() + << "error: unknown integrated tool '" << Tool << "'. " + << "Valid tools include '-cc1', '-cc1as', and '-cc1gen-reproducer'.\n"; +#endif return 1; }