Index: clang-tools-extra/clang-tidy/ClangTidy.h =================================================================== --- clang-tools-extra/clang-tidy/ClangTidy.h +++ clang-tools-extra/clang-tidy/ClangTidy.h @@ -82,6 +82,10 @@ bool ApplyAnyFix, bool EnableCheckProfile = false, llvm::StringRef StoreCheckProfile = StringRef()); +/// Run the multi-pass project-level "compact" routine of checks that support +/// such on the data available in the Context. +void runClangTidyCompactPhase(clang::tidy::ClangTidyContext &Context); + /// Controls what kind of fixes clang-tidy is allowed to apply. enum FixBehaviour { /// Don't try to apply any fix. Index: clang-tools-extra/clang-tidy/ClangTidy.cpp =================================================================== --- clang-tools-extra/clang-tidy/ClangTidy.cpp +++ clang-tools-extra/clang-tidy/ClangTidy.cpp @@ -305,35 +305,55 @@ unsigned WarningsAsErrors; }; +using CheckVec = std::vector>; + class ClangTidyASTConsumer : public MultiplexConsumer { public: ClangTidyASTConsumer(std::vector> Consumers, std::unique_ptr Profiling, std::unique_ptr Finder, - std::vector> Checks) + CheckVec Checks, MultipassProjectPhase MultipassPhase) : MultiplexConsumer(std::move(Consumers)), Profiling(std::move(Profiling)), Finder(std::move(Finder)), - Checks(std::move(Checks)) {} + Checks(std::move(Checks)), MultipassPhase(MultipassPhase) {} + + ~ClangTidyASTConsumer() { + if (MultipassPhase == MultipassProjectPhase::Collect) + // Allow the checks to write their data at the end of execution. + for (auto &Check : Checks) + Check->runPostCollect(); + } private: // Destructor order matters! Profiling must be destructed last. // Or at least after Finder. std::unique_ptr Profiling; std::unique_ptr Finder; - std::vector> Checks; + CheckVec Checks; + MultipassProjectPhase MultipassPhase; }; } // namespace +static void instantiateTidyModules(ClangTidyCheckFactories &CheckFactories) { + for (ClangTidyModuleRegistry::entry E : ClangTidyModuleRegistry::entries()) { + std::unique_ptr M = E.instantiate(); + M->addCheckFactories(CheckFactories); + } +} + +static CheckVec instantiateChecks(ClangTidyContext &Context, + ClangTidyCheckFactories &CheckFactories) { + CheckVec Checks = CheckFactories.createChecks(&Context); + return Checks; +} + ClangTidyASTConsumerFactory::ClangTidyASTConsumerFactory( ClangTidyContext &Context, IntrusiveRefCntPtr OverlayFS) : Context(Context), OverlayFS(std::move(OverlayFS)), CheckFactories(new ClangTidyCheckFactories) { - for (ClangTidyModuleRegistry::entry E : ClangTidyModuleRegistry::entries()) { - std::unique_ptr Module = E.instantiate(); - Module->addCheckFactories(*CheckFactories); - } + instantiateTidyModules(*CheckFactories); } #if CLANG_TIDY_ENABLE_STATIC_ANALYZER @@ -401,8 +421,7 @@ if (WorkingDir) Context.setCurrentBuildDirectory(WorkingDir.get()); - std::vector> Checks = - CheckFactories->createChecks(&Context); + CheckVec Checks = instantiateChecks(Context, *CheckFactories); llvm::erase_if(Checks, [&](std::unique_ptr &Check) { return !Check->isLanguageVersionSupported(Context.getLangOpts()); @@ -458,7 +477,7 @@ #endif // CLANG_TIDY_ENABLE_STATIC_ANALYZER return std::make_unique( std::move(Consumers), std::move(Profiling), std::move(Finder), - std::move(Checks)); + std::move(Checks), Context.getGlobalOptions().MultipassPhase); } std::vector ClangTidyASTConsumerFactory::getCheckNames() { @@ -480,8 +499,7 @@ ClangTidyOptions::OptionMap ClangTidyASTConsumerFactory::getCheckOptions() { ClangTidyOptions::OptionMap Options; - std::vector> Checks = - CheckFactories->createChecks(&Context); + CheckVec Checks = instantiateChecks(Context, *CheckFactories); for (const auto &Check : Checks) Check->storeOptions(Options); return Options; @@ -509,6 +527,15 @@ return Factory.getCheckOptions(); } +void runClangTidyCompactPhase(clang::tidy::ClangTidyContext &Context) { + std::unique_ptr CheckFactories( + new ClangTidyCheckFactories()); + instantiateTidyModules(*CheckFactories); + CheckVec Checks = instantiateChecks(Context, *CheckFactories); + for (auto &Check : Checks) + Check->runCompact(); +} + std::vector runClangTidy(clang::tidy::ClangTidyContext &Context, const CompilationDatabase &Compilations, Index: clang-tools-extra/clang-tidy/ClangTidyCheck.h =================================================================== --- clang-tools-extra/clang-tidy/ClangTidyCheck.h +++ clang-tools-extra/clang-tidy/ClangTidyCheck.h @@ -111,6 +111,26 @@ /// work in here. virtual void check(const ast_matchers::MatchFinder::MatchResult &Result) {} + /// ``ClangTidyChecks`` that register ASTMatchers should do the actual work + /// for registering per-TU data to the project-level collection in here. + virtual void collect(const ast_matchers::MatchFinder::MatchResult &Result) {} + + /// Checks that performed ``collect`` should write their data to the output to + /// the file in here. + virtual void postCollect(StringRef OutFile) {} + + /// Execute the ``postCollect()`` function to the right, automatically + /// generated filename. + void runPostCollect(); + + /// In the ``compact`` phase, checks that are supporting such should reduce + /// the data received from all the TUs into a single file. + virtual void compact(const std::vector &PerTUCollectedDataFiles, + StringRef OutFile) {} + + /// Gather the list of per-TU files from the context, and execute ``compact``. + void runCompact(); + /// Add a diagnostic with the check's name. DiagnosticBuilder diag(SourceLocation Loc, StringRef Description, DiagnosticIDs::Level Level = DiagnosticIDs::Warning); @@ -422,6 +442,18 @@ bool areDiagsSelfContained() const { return Context->areDiagsSelfContained(); } + /// Returns the path where the current check should write collected data to. + std::string getCollectPath(); + /// Returns the file where the current check should write or read compacted + /// data to/from. + StringRef getCompactedDataPath() const { + return Context->getCompactedDataPath(CheckName); + } + /// Returns the current phase of execution in a multi-pass project-level + /// system. + MultipassProjectPhase getPhase() const { + return Context->getGlobalOptions().MultipassPhase; + } }; /// Read a named option from the ``Context`` and parse it as a bool. Index: clang-tools-extra/clang-tidy/ClangTidyCheck.cpp =================================================================== --- clang-tools-extra/clang-tidy/ClangTidyCheck.cpp +++ clang-tools-extra/clang-tidy/ClangTidyCheck.cpp @@ -42,7 +42,63 @@ // For historical reasons, checks don't implement the MatchFinder run() // callback directly. We keep the run()/check() distinction to avoid interface // churn, and to allow us to add cross-cutting logic in the future. - check(Result); + + switch (getPhase()) { + case MultipassProjectPhase::Diagnose: + check(Result); + break; + case MultipassProjectPhase::Collect: + collect(Result); + break; + case MultipassProjectPhase::Compact: + llvm_unreachable("AST Matchers should not have run in compact mode."); + } +} + +void ClangTidyCheck::runCompact() { + using namespace llvm::sys::fs; + using namespace llvm::sys::path; + + std::vector Inputs; + StringRef OutputPath = getCompactedDataPath(); + StringRef OutputFilename = filename(OutputPath); + + std::error_code EC; + for (auto It = directory_iterator( + Context->getGlobalOptions().MultipassDirectory, EC); + It != directory_iterator(); It.increment(EC)) { + if (EC) + continue; + + StringRef Filename = filename(It->path()); + if (Filename.startswith(this->CheckName) && Filename != OutputFilename) + Inputs.push_back(It->path()); + } + + compact(Inputs, OutputPath); +} + +void ClangTidyCheck::runPostCollect() { + // FIXME: Hash does not respect multiple compilations of the same file with + // different compile flags. + assert(getPhase() == MultipassProjectPhase::Collect && + "postCollect() in wrong phase."); + postCollect(getCollectPath()); +} + +std::string ClangTidyCheck::getCollectPath() { + using namespace llvm::sys::path; + assert(getPhase() == MultipassProjectPhase::Collect && + "getCollectPath() in wrong phase."); + + SmallString<256> Filename; + llvm::raw_svector_ostream OS{Filename}; + StringRef CurrentFile = Context->getCurrentFile(); + OS << Context->getGlobalOptions().MultipassDirectory << get_separator() + << CheckName << '.' << filename(CurrentFile) << '.' + << hash_value(CurrentFile) << ".yaml"; + + return Filename.str().str(); } ClangTidyCheck::OptionsView::OptionsView( Index: clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h =================================================================== --- clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h +++ clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.h @@ -201,6 +201,10 @@ DiagEngine->getDiagnosticIDs()->getDescription(DiagnosticID))); } + /// Returns the file where a check should write or read compacted data + /// to/from. + StringRef getCompactedDataPath(StringRef CheckName); + private: // Writes to Stats. friend class ClangTidyDiagnosticConsumer; @@ -230,6 +234,8 @@ bool SelfContainedDiags; NoLintDirectiveHandler NoLintHandler; + + llvm::DenseMap CompactedDataPaths; }; /// Gets the Fix attached to \p Diagnostic. Index: clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp =================================================================== --- clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp +++ clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp @@ -234,6 +234,35 @@ LangOpts = Context->getLangOpts(); } +StringRef ClangTidyContext::getCompactedDataPath(StringRef CheckName) { + using namespace llvm::sys::fs; + using namespace llvm::sys::path; + + MultipassProjectPhase Phase = getGlobalOptions().MultipassPhase; + if (Phase == MultipassProjectPhase::Collect) + llvm_unreachable("Invalid phase 'Collect' for accessing compacted data"); + + auto InsertResult = CompactedDataPaths.try_emplace(CheckName, ""); + std::string &FilePath = InsertResult.first->second; + if (!InsertResult.second) + return FilePath; + + SmallString<128> OutputFile; + llvm::raw_svector_ostream OS{OutputFile}; + OS << getGlobalOptions().MultipassDirectory << get_separator() << CheckName + << ".yaml"; + + if (Phase == MultipassProjectPhase::Compact) + // The file will be written, and thus, created. + FilePath = OutputFile.str().str(); + else if (Phase == MultipassProjectPhase::Diagnose) + if (llvm::sys::fs::exists(OutputFile)) + // If the file doesn't exist, leave it as empty string. + FilePath = OutputFile.str().str(); + + return FilePath; +} + const ClangTidyGlobalOptions &ClangTidyContext::getGlobalOptions() const { return OptionsProvider->getGlobalOptions(); } Index: clang-tools-extra/clang-tidy/ClangTidyOptions.h =================================================================== --- clang-tools-extra/clang-tidy/ClangTidyOptions.h +++ clang-tools-extra/clang-tidy/ClangTidyOptions.h @@ -25,6 +25,19 @@ namespace clang { namespace tidy { +/// When executing multi-pass project-level analysis, identifies the phase the +/// current process is configured for. +enum class MultipassProjectPhase { + /// In the collection phase, checks capable for project-level analysis emit + /// analysis data. + Collect, + /// Create a full project-level data file that diagnosis can use. + Compact, + /// Emit diagnostics from the checks. (This is the default value in + /// single-phase mode.) + Diagnose +}; + /// Contains a list of line ranges in a single file. struct FileFilter { /// File name. @@ -43,6 +56,10 @@ /// Output warnings from certain line ranges of certain files only. /// If empty, no warnings will be filtered. std::vector LineFilter; + + MultipassProjectPhase MultipassPhase; + /// The directory where multi-pass project-level analysis stores its data to. + std::string MultipassDirectory; }; /// Contains options for clang-tidy. These options may be read from Index: clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp =================================================================== --- clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp +++ clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp @@ -129,10 +129,10 @@ cl::init(false), cl::cat(ClangTidyCategory)); static cl::opt FixNotes("fix-notes", cl::desc(R"( -If a warning has no fix, but a single fix can -be found through an associated diagnostic note, -apply the fix. -Specifying this flag will implicitly enable the +If a warning has no fix, but a single fix can +be found through an associated diagnostic note, +apply the fix. +Specifying this flag will implicitly enable the '--fix' flag. )"), cl::init(false), cl::cat(ClangTidyCategory)); @@ -257,9 +257,55 @@ )"), cl::init(false), cl::cat(ClangTidyCategory)); +static cl::opt MultipassPhase( + "multipass-phase", cl::desc(R"( +When executing project-level analysis, specify +which phase of the analysis to run. Multi-pass +project-level analysis requires the execution +of 3 passes in sequence. Not all checks support +this feature. +)"), + cl::values( + clEnumValN(clang::tidy::MultipassProjectPhase::Collect, "collect", R"( +Collect per-TU analysis data from checks that are +capable of multi-pass analysis. +This pass can be executed in parallel. +)"), + clEnumValN(clang::tidy::MultipassProjectPhase::Compact, "compact", R"( +Transform the per-TU data into a single project-level +data to be consumed for diagnostics. +This pass CAN NOT be executed in parallel! +)"), + clEnumValN(clang::tidy::MultipassProjectPhase::Diagnose, "diagnose", R"( +Emit diagnostics of the code, using the previously +collected and compacted data, or with per-TU data +only for single-pass analysis analysis. +This pass can be executed in parallel. +)")), + cl::init(clang::tidy::MultipassProjectPhase::Diagnose), + cl::cat(ClangTidyCategory)); + +static cl::opt MultipassDirectory("multipass-dir", cl::desc(R"( +When executing project-level analysis, specify +a directory where data can be stored inbetween +phases. +)"), + cl::cat(ClangTidyCategory)); + namespace clang { namespace tidy { +static SmallString<256> makeAbsolute(const std::string &Input) { + if (Input.empty()) + return {}; + SmallString<256> AbsolutePath(Input); + if (std::error_code EC = llvm::sys::fs::make_absolute(AbsolutePath)) { + llvm::errs() << "Can't make absolute path from " << Input << ": " + << EC.message() << "\n"; + } + return AbsolutePath; +} + static void printStats(const ClangTidyStats &Stats) { if (Stats.errorsIgnored()) { llvm::errs() << "Suppressed " << Stats.errorsIgnored() << " warnings ("; @@ -297,6 +343,12 @@ return nullptr; } + GlobalOptions.MultipassPhase = MultipassPhase; + + SmallString<256> MultipassDirAbsolute = makeAbsolute(MultipassDirectory); + GlobalOptions.MultipassDirectory = + makeAbsolute(MultipassDirectory).str().str(); + ClangTidyOptions DefaultOptions; DefaultOptions.Checks = DefaultChecks; DefaultOptions.WarningsAsErrors = ""; @@ -416,18 +468,7 @@ if (!OptionsProvider) return 1; - auto MakeAbsolute = [](const std::string &Input) -> SmallString<256> { - if (Input.empty()) - return {}; - SmallString<256> AbsolutePath(Input); - if (std::error_code EC = llvm::sys::fs::make_absolute(AbsolutePath)) { - llvm::errs() << "Can't make absolute path from " << Input << ": " - << EC.message() << "\n"; - } - return AbsolutePath; - }; - - SmallString<256> ProfilePrefix = MakeAbsolute(StoreCheckProfile); + SmallString<256> ProfilePrefix = makeAbsolute(StoreCheckProfile); StringRef FileName("dummy"); auto PathList = OptionsParser->getSourcePathList(); @@ -435,7 +476,7 @@ FileName = PathList.front(); } - SmallString<256> FilePath = MakeAbsolute(std::string(FileName)); + SmallString<256> FilePath = makeAbsolute(std::string(FileName)); ClangTidyOptions EffectiveOptions = OptionsProvider->getOptions(FilePath); std::vector EnabledChecks = @@ -484,12 +525,34 @@ return 1; } + MultipassProjectPhase Phase = + OptionsProvider->getGlobalOptions().MultipassPhase; + if (Phase == MultipassProjectPhase::Compact) { + ClangTidyContext Context(std::move(OwningOptionsProvider), + AllowEnablingAnalyzerAlphaCheckers); + runClangTidyCompactPhase(Context); + return 0; + } + if (PathList.empty()) { llvm::errs() << "Error: no input files specified.\n"; llvm::cl::PrintHelpMessage(/*Hidden=*/false, /*Categorized=*/true); return 1; } + const std::string &MultipassDir = + OptionsProvider->getGlobalOptions().MultipassDirectory; + if (Phase == MultipassProjectPhase::Collect || + Phase == MultipassProjectPhase::Compact) { + if (MultipassDir.empty()) { + llvm::errs() << "Error: --multipass-phase 'collect' or 'compact' given, " + "but no '--multipass-dir' set.\n"; + return 1; + } + if (!llvm::sys::fs::is_directory(MultipassDir)) + llvm::sys::fs::create_directory(MultipassDir); + } + llvm::InitializeAllTargetInfos(); llvm::InitializeAllTargetMCs(); llvm::InitializeAllAsmParsers(); Index: clang-tools-extra/docs/ReleaseNotes.rst =================================================================== --- clang-tools-extra/docs/ReleaseNotes.rst +++ clang-tools-extra/docs/ReleaseNotes.rst @@ -106,6 +106,10 @@ means it is advised to use YAML's block style initiated by the pipe character `|` for the `Checks` section in order to benefit from the easier syntax that works without commas. +- Added the infrastructure support for checks to be executed in multiple phases + to facilitate analysis with project-level information. Such analysis is not + the default, and not all checks support it. + New checks ^^^^^^^^^^ Index: clang-tools-extra/docs/clang-tidy/index.rst =================================================================== --- clang-tools-extra/docs/clang-tidy/index.rst +++ clang-tools-extra/docs/clang-tidy/index.rst @@ -227,6 +227,32 @@ complete list of passes, use the :option:`--list-checks` and :option:`-load` options together. + --multipass-dir= - + When executing project-level analysis, specify + a directory where data can be stored inbetween + phases. + --multipass-phase= - + When executing project-level analysis, specify + which phase of the analysis to run. Multi-pass + project-level analysis requires the execution + of 3 passes in sequence. Not all checks support + this feature. + =collect - + Collect per-TU analysis data from checks that are + capable of multi-pass analysis. + + This pass can be executed in parallel. + =compact - + Transform the per-TU data into a single project-level + data to be consumed for diagnostics. + + This pass CAN NOT be executed in parallel! + =diagnose - + Emit diagnostics of the code, using the previously + collected and compacted data, or with per-TU data + only for single-pass analysis analysis. + + This pass can be executed in parallel. -p= - Build path --quiet - Run clang-tidy in quiet mode. This suppresses @@ -418,5 +444,44 @@ :program:`clang-tidy` will generate a ``clang-tidy-nolint`` error diagnostic if any ``NOLINTBEGIN``/``NOLINTEND`` comment violates these requirements. +Project-level analysis +====================== + +By default, Clang-Tidy runs checks on every translation unit of the project +separately. +Some checks, however, might benefit from and give better or more meaningful +results, or only work, when executed not for a single file, but for the entire +project. +The **multi-pass** analysis can be used in this case, with which checks can first +`collect` information into a temporary data location (``--multipass-dir``) on the +disk, `compact` per-TU data into project-level data, and `diagnose` with the +project-level data in mind. +The phase is selected by passing the appropriate value to the +``--multipass-phase`` command-line parameter. + +Whether a check supports project-level analysis, and how project-level data is +stored and transformed from per-TU to "global" values is specific to each +individual check. + +.. code-block:: bash + $ clang-tidy --checks=... file1.cpp file2.cpp + + $ clang-tidy --checks=... --multipass-phase=collect --multipass-dir="TidyData" file1.cpp file2.cpp + # No output to the terminal. + $ ls ./TidyData + blah.file1.cpp.01234567.yaml + blah.file2.cpp.89abcdef.yaml + + $ clang-tidy --checks=... --multipass-phase=compact --multipass-dir="TidyData" + # No file list required to be specified. + # No output to the terminal. + $ ls ./TidyData + blah.file1.cpp.01234567.yaml + blah.file2.cpp.89abcdef.yaml + blah.yaml + + $ clang-tidy --checks=... --multipass-phase=compact --multipass-dir="TidyData" file1.cpp file2.cpp + # Diagnostics that use project-level data. + .. _LibTooling: https://clang.llvm.org/docs/LibTooling.html .. _How To Setup Tooling For LLVM: https://clang.llvm.org/docs/HowToSetupToolingForLLVM.html