Index: include/clang/CrossTU/CrossTranslationUnit.h =================================================================== --- include/clang/CrossTU/CrossTranslationUnit.h +++ include/clang/CrossTU/CrossTranslationUnit.h @@ -44,7 +44,8 @@ failed_to_generate_usr, triple_mismatch, lang_mismatch, - lang_dialect_mismatch + lang_dialect_mismatch, + load_threshold_reached }; class IndexError : public llvm::ErrorInfo { @@ -111,7 +112,9 @@ /// \p CrossTUDir directory, called \p IndexName. In case the declaration is /// found in the index the corresponding AST file will be loaded and the /// definition of the function will be merged into the original AST using - /// the AST Importer. + /// the AST Importer. \p CTULoadTreshold should serve as an upper limit to the + /// number of TUs imported in order to reduce the memory footprint of CTU + /// analysis. /// /// \return The declaration with the definition will be returned. /// If no suitable definition is found in the index file or multiple @@ -120,7 +123,8 @@ /// Note that the AST files should also be in the \p CrossTUDir. llvm::Expected getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir, - StringRef IndexName, bool DisplayCTUProgress = false); + StringRef IndexName, bool DisplayCTUProgress, + unsigned CTULoadThreshold); /// This function loads a function definition from an external AST /// file. @@ -128,7 +132,8 @@ /// A function definition with the same declaration will be looked up in the /// index file which should be in the \p CrossTUDir directory, called /// \p IndexName. In case the declaration is found in the index the - /// corresponding AST file will be loaded. + /// corresponding AST file will be loaded. If the number of TUs imported + /// reaches \p CTULoadTreshold, no loading is performed. /// /// \return Returns a pointer to the ASTUnit that contains the definition of /// the looked up function or an Error. @@ -138,7 +143,8 @@ llvm::Expected loadExternalAST(StringRef LookupName, StringRef CrossTUDir, StringRef IndexName, - bool DisplayCTUProgress = false); + bool DisplayCTUProgress, + unsigned CTULoadThreshold); /// This function merges a definition from a separate AST Unit into /// the current one which was created by the compiler instance that @@ -167,6 +173,7 @@ CompilerInstance &CI; ASTContext &Context; std::unique_ptr LookupTable; + unsigned NumASTLoaded{0u}; }; } // namespace cross_tu Index: include/clang/StaticAnalyzer/Core/AnalyzerOptions.def =================================================================== --- include/clang/StaticAnalyzer/Core/AnalyzerOptions.def +++ include/clang/StaticAnalyzer/Core/AnalyzerOptions.def @@ -287,6 +287,14 @@ "the analyzer's progress related to ctu.", false) +ANALYZER_OPTION(unsigned, CTUImportThreshold, "ctu-import-threshold", + "The maximal amount of translation units that is considered " + "for import when inlining functions during CTU analysis. " + "Lowering this threshold can alleviate the memory burder of " + "analysis with many interdependent definitions located in " + "various translation units.", + 100u) + //===----------------------------------------------------------------------===// // Unsinged analyzer options. //===----------------------------------------------------------------------===// Index: lib/CrossTU/CrossTranslationUnit.cpp =================================================================== --- lib/CrossTU/CrossTranslationUnit.cpp +++ lib/CrossTU/CrossTranslationUnit.cpp @@ -43,6 +43,8 @@ STATISTIC(NumTripleMismatch, "The # of triple mismatches"); STATISTIC(NumLangMismatch, "The # of language mismatches"); STATISTIC(NumLangDialectMismatch, "The # of language dialect mismatches"); +STATISTIC(NumASTLoadThresholdReached, + "The # of ASTs not loaded because of threshold"); // Same as Triple's equality operator, but we check a field only if that is // known in both instances. @@ -102,6 +104,8 @@ return "Language mismatch"; case index_error_code::lang_dialect_mismatch: return "Language dialect mismatch"; + case index_error_code::load_threshold_reached: + return "Load threshold reached"; } llvm_unreachable("Unrecognized index_error_code."); } @@ -197,7 +201,8 @@ CrossTranslationUnitContext::getCrossTUDefinition(const FunctionDecl *FD, StringRef CrossTUDir, StringRef IndexName, - bool DisplayCTUProgress) { + bool DisplayCTUProgress, + unsigned CTULoadThreshold) { assert(FD && "FD is missing, bad call to this function!"); assert(!FD->hasBody() && "FD has a definition in current translation unit!"); ++NumGetCTUCalled; @@ -206,7 +211,8 @@ return llvm::make_error( index_error_code::failed_to_generate_usr); llvm::Expected ASTUnitOrError = - loadExternalAST(LookupFnName, CrossTUDir, IndexName, DisplayCTUProgress); + loadExternalAST(LookupFnName, CrossTUDir, IndexName, DisplayCTUProgress, + CTULoadThreshold); if (!ASTUnitOrError) return ASTUnitOrError.takeError(); ASTUnit *Unit = *ASTUnitOrError; @@ -293,11 +299,18 @@ llvm::Expected CrossTranslationUnitContext::loadExternalAST( StringRef LookupName, StringRef CrossTUDir, StringRef IndexName, - bool DisplayCTUProgress) { + bool DisplayCTUProgress, unsigned CTULoadThreshold) { // FIXME: The current implementation only supports loading functions with // a lookup name from a single translation unit. If multiple // translation units contains functions with the same lookup name an // error will be returned. + + if (NumASTLoaded >= CTULoadThreshold) { + ++NumASTLoadThresholdReached; + return llvm::make_error( + index_error_code::load_threshold_reached); + } + ASTUnit *Unit = nullptr; auto FnUnitCacheEntry = FunctionASTUnitMap.find(LookupName); if (FnUnitCacheEntry == FunctionASTUnitMap.end()) { @@ -335,6 +348,7 @@ ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts())); Unit = LoadedUnit.get(); FileASTUnitMap[ASTFileName] = std::move(LoadedUnit); + ++NumASTLoaded; if (DisplayCTUProgress) { llvm::errs() << "CTU loaded AST file: " << ASTFileName << "\n"; Index: lib/StaticAnalyzer/Core/CallEvent.cpp =================================================================== --- lib/StaticAnalyzer/Core/CallEvent.cpp +++ lib/StaticAnalyzer/Core/CallEvent.cpp @@ -572,7 +572,8 @@ *Engine.getCrossTranslationUnitContext(); llvm::Expected CTUDeclOrError = CTUCtx.getCrossTUDefinition(FD, Opts.CTUDir, Opts.CTUIndexName, - Opts.DisplayCTUProgress); + Opts.DisplayCTUProgress, + Opts.CTUImportThreshold); if (!CTUDeclOrError) { handleAllErrors(CTUDeclOrError.takeError(), Index: test/Analysis/analyzer-config.c =================================================================== --- test/Analysis/analyzer-config.c +++ test/Analysis/analyzer-config.c @@ -20,6 +20,7 @@ // CHECK-NEXT: cfg-temporary-dtors = true // CHECK-NEXT: crosscheck-with-z3 = false // CHECK-NEXT: ctu-dir = "" +// CHECK-NEXT: ctu-import-threshold = 100 // CHECK-NEXT: ctu-index-name = externalDefMap.txt // CHECK-NEXT: display-ctu-progress = false // CHECK-NEXT: eagerly-assume = true @@ -52,4 +53,4 @@ // CHECK-NEXT: unroll-loops = false // CHECK-NEXT: widen-loops = false // CHECK-NEXT: [stats] -// CHECK-NEXT: num-entries = 49 +// CHECK-NEXT: num-entries = 50 Index: test/Analysis/ctu-import-threshold.c =================================================================== --- /dev/null +++ test/Analysis/ctu-import-threshold.c @@ -0,0 +1,5 @@ +// Ensure analyzer option 'ctu-import-threshold' is a recognized option. +// +// RUN: %clang_cc1 -analyze -analyzer-config ctu-import-threshold=30 -verify %s +// +// expected-no-diagnostics Index: unittests/CrossTU/CrossTranslationUnitTest.cpp =================================================================== --- unittests/CrossTU/CrossTranslationUnitTest.cpp +++ unittests/CrossTU/CrossTranslationUnitTest.cpp @@ -23,8 +23,9 @@ class CTUASTConsumer : public clang::ASTConsumer { public: - explicit CTUASTConsumer(clang::CompilerInstance &CI, bool *Success) - : CTU(CI), Success(Success) {} + explicit CTUASTConsumer(clang::CompilerInstance &CI, bool *Success, + unsigned ImportLimit) + : CTU(CI), Success(Success), ImportLimit(ImportLimit) {} void HandleTranslationUnit(ASTContext &Ctx) { const TranslationUnitDecl *TU = Ctx.getTranslationUnitDecl(); @@ -70,41 +71,54 @@ EXPECT_TRUE(llvm::sys::fs::exists(ASTFileName)); // Load the definition from the AST file. - llvm::Expected NewFDorError = - CTU.getCrossTUDefinition(FD, "", IndexFileName); - EXPECT_TRUE((bool)NewFDorError); - const FunctionDecl *NewFD = *NewFDorError; + llvm::Expected NewFDorError = handleExpected( + CTU.getCrossTUDefinition(FD, "", IndexFileName, false, ImportLimit), + []() { return nullptr; }, [](IndexError &) {}); - *Success = NewFD && NewFD->hasBody() && !OrigFDHasBody; + if (NewFDorError) { + const FunctionDecl *NewFD = *NewFDorError; + *Success = NewFD && NewFD->hasBody() && !OrigFDHasBody; + } } private: CrossTranslationUnitContext CTU; bool *Success; + unsigned ImportLimit; }; class CTUAction : public clang::ASTFrontendAction { public: - CTUAction(bool *Success) : Success(Success) {} + CTUAction(bool *Success, unsigned ImportLimit) + : Success(Success), ImportLimit(ImportLimit) {} protected: std::unique_ptr CreateASTConsumer(clang::CompilerInstance &CI, StringRef) override { - return llvm::make_unique(CI, Success); + return llvm::make_unique(CI, Success, ImportLimit); } private: bool *Success; + unsigned ImportLimit; }; } // end namespace TEST(CrossTranslationUnit, CanLoadFunctionDefinition) { bool Success = false; - EXPECT_TRUE(tooling::runToolOnCode(new CTUAction(&Success), "int f(int);")); + EXPECT_TRUE( + tooling::runToolOnCode(new CTUAction(&Success, 1u), "int f(int);")); EXPECT_TRUE(Success); } +TEST(CrossTranslationUnit, RespectsLoadThreshold) { + bool Success = false; + EXPECT_TRUE( + tooling::runToolOnCode(new CTUAction(&Success, 0u), "int f(int);")); + EXPECT_FALSE(Success); +} + TEST(CrossTranslationUnit, IndexFormatCanBeParsed) { llvm::StringMap Index; Index["a"] = "/b/f1";