Index: include/clang/AST/ASTContext.h =================================================================== --- include/clang/AST/ASTContext.h +++ include/clang/AST/ASTContext.h @@ -62,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -77,12 +78,14 @@ } // end namespace llvm namespace clang { - +class ASTImporter; class ASTMutationListener; class ASTRecordLayout; +class ASTUnit; class AtomicExpr; class BlockExpr; class CharUnits; +class CompilerInstance; class CXXABI; class DiagnosticsEngine; class Expr; @@ -1893,6 +1896,23 @@ } //===--------------------------------------------------------------------===// + // Cross-translation unit support + //===--------------------------------------------------------------------===// +private: + llvm::StringMap FileASTUnitMap; + llvm::StringMap FunctionAstUnitMap; + llvm::StringMap FunctionFileMap; + llvm::DenseMap ASTUnitImporterMap; + llvm::DenseMap ImportMap; + ASTImporter &getOrCreateASTImporter(ASTContext &From); + +public: + const FunctionDecl * + getXTUDefinition(const FunctionDecl *FD, CompilerInstance &CI, + StringRef XTUDir, DiagnosticsEngine &Diags, + std::function Loader); + + //===--------------------------------------------------------------------===// // Type Sizing and Analysis //===--------------------------------------------------------------------===// Index: include/clang/AST/Decl.h =================================================================== --- include/clang/AST/Decl.h +++ include/clang/AST/Decl.h @@ -50,6 +50,7 @@ class TypeLoc; class UnresolvedSetImpl; class VarTemplateDecl; +class CompilerInstance; /// \brief A container of type source information. /// Index: include/clang/AST/Mangle.h =================================================================== --- include/clang/AST/Mangle.h +++ include/clang/AST/Mangle.h @@ -52,6 +52,11 @@ ASTContext &Context; DiagnosticsEngine &Diags; const ManglerKind Kind; + // Used for cross translation unit analysis. + // To reduce the risk of function name collision in C projects, we force + // name mangling for C functions when generating lookup identifiers for + // the static analyzer. + bool ShouldForceMangleProto; llvm::DenseMap GlobalBlockIds; llvm::DenseMap LocalBlockIds; @@ -87,6 +92,11 @@ return Result.first->second; } + bool shouldForceMangleProto() const { return ShouldForceMangleProto; } + void setShouldForceMangleProto(bool ForceMangleArguments) { + ShouldForceMangleProto = ForceMangleArguments; + } + /// @name Mangler Entry Points /// @{ Index: include/clang/StaticAnalyzer/Core/AnalyzerOptions.h =================================================================== --- include/clang/StaticAnalyzer/Core/AnalyzerOptions.h +++ include/clang/StaticAnalyzer/Core/AnalyzerOptions.h @@ -272,6 +272,9 @@ /// \sa shouldDisplayNotesAsEvents Optional DisplayNotesAsEvents; + /// \sa getXTUDir + Optional XTUDir; + /// A helper function that retrieves option for a given full-qualified /// checker name. /// Options for checkers can be specified via 'analyzer-config' command-line @@ -548,6 +551,9 @@ /// to false when unset. bool shouldDisplayNotesAsEvents(); + /// Returns the directory containing the XTU related files. + StringRef getXTUDir(); + public: AnalyzerOptions() : AnalysisStoreOpt(RegionStoreModel), Index: include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h =================================================================== --- include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h +++ include/clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h @@ -384,7 +384,7 @@ // Iterator access to formal parameters and their types. private: typedef std::const_mem_fun_t get_type_fun; - + public: /// Return call's formal parameters. /// @@ -436,20 +436,7 @@ return cast(CallEvent::getDecl()); } - RuntimeDefinition getRuntimeDefinition() const override { - const FunctionDecl *FD = getDecl(); - // Note that the AnalysisDeclContext will have the FunctionDecl with - // the definition (if one exists). - if (FD) { - AnalysisDeclContext *AD = - getLocationContext()->getAnalysisDeclContext()-> - getManager()->getContext(FD); - if (AD->getBody()) - return RuntimeDefinition(AD->getDecl()); - } - - return RuntimeDefinition(); - } + RuntimeDefinition getRuntimeDefinition() const override; bool argumentsMayEscape() const override; Index: include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h =================================================================== --- include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h +++ include/clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h @@ -56,6 +56,8 @@ }; private: + CompilerInstance &CI; + AnalysisManager &AMgr; AnalysisDeclContextManager &AnalysisDeclContexts; @@ -97,9 +99,8 @@ InliningModes HowToInline; public: - ExprEngine(AnalysisManager &mgr, bool gcEnabled, - SetOfConstDecls *VisitedCalleesIn, - FunctionSummariesTy *FS, + ExprEngine(CompilerInstance &CI, AnalysisManager &mgr, bool gcEnabled, + SetOfConstDecls *VisitedCalleesIn, FunctionSummariesTy *FS, InliningModes HowToInlineIn); ~ExprEngine() override; @@ -132,6 +133,8 @@ BugReporter& getBugReporter() { return BR; } + CompilerInstance &getCompilerInstance() { return CI; } + const NodeBuilderContext &getBuilderContext() { assert(currBldrCtx); return *currBldrCtx; Index: lib/AST/ASTContext.cpp =================================================================== --- lib/AST/ASTContext.cpp +++ lib/AST/ASTContext.cpp @@ -13,6 +13,7 @@ #include "clang/AST/ASTContext.h" #include "CXXABI.h" +#include "clang/AST/ASTImporter.h" #include "clang/AST/ASTMutationListener.h" #include "clang/AST/Attr.h" #include "clang/AST/CharUnits.h" @@ -34,11 +35,15 @@ #include "clang/Basic/Builtins.h" #include "clang/Basic/SourceManager.h" #include "clang/Basic/TargetInfo.h" +#include "clang/Frontend/ASTUnit.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/StaticAnalyzer/Core/AnalyzerOptions.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/Support/Capacity.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" +#include #include using namespace clang; @@ -1418,6 +1423,132 @@ } } + +//===----------------------------------------------------------------------===// +// Cross-translation unit support +//===----------------------------------------------------------------------===// + +std::string getMangledName(const NamedDecl *ND, MangleContext *MangleCtx) { + std::string MangledName; + llvm::raw_string_ostream OS(MangledName); + if (const auto *CCD = dyn_cast(ND)) + // FIXME: Use correct Ctor/DtorType + MangleCtx->mangleCXXCtor(CCD, Ctor_Complete, OS); + else if (const auto *CDD = dyn_cast(ND)) + MangleCtx->mangleCXXDtor(CDD, Dtor_Complete, OS); + else + MangleCtx->mangleName(ND, OS); + ASTContext &Ctx = ND->getASTContext(); + // We are not going to support vendor and don't support OS and environment. + // FIXME: support OS and environment correctly + llvm::Triple::ArchType T = Ctx.getTargetInfo().getTriple().getArch(); + if (T == llvm::Triple::thumb) + T = llvm::Triple::arm; + OS << "@" << Ctx.getTargetInfo().getTriple().getArchTypeName(T); + return OS.str(); +} + +const FunctionDecl* iterateContextDecls(const DeclContext *DC, + const std::string &MangledFnName, + std::unique_ptr &MangleCtx) { + //FIXME: Use ASTMatcher instead. + if (!DC) + return nullptr; + for (const Decl *D : DC->decls()) { + const auto *SubDC = dyn_cast(D); + if (const auto *FD = iterateContextDecls(SubDC, MangledFnName, MangleCtx)) + return FD; + + const auto *ND = dyn_cast(D); + const FunctionDecl *ResultDecl; + if (!ND || !ND->hasBody(ResultDecl)) { + continue; + } + std::string LookupMangledName = getMangledName(ResultDecl, MangleCtx.get()); + // We are already sure that the triple is correct here. + if (LookupMangledName != MangledFnName) + continue; + return ResultDecl; + } + return nullptr; +} + +const FunctionDecl * +ASTContext::getXTUDefinition(const FunctionDecl *FD, CompilerInstance &CI, + StringRef XTUDir, DiagnosticsEngine &Diags, + std::function Loader) { + assert(!FD->hasBody() && "FD has a definition in current translation unit!"); + if (!FD->getType()->getAs()) + return nullptr; // Cannot even mangle that. + auto FoundImport = ImportMap.find(FD); + if (FoundImport != ImportMap.end()) + return FoundImport->second; + + std::unique_ptr MangleCtx( + ItaniumMangleContext::create(FD->getASTContext(), Diags)); + MangleCtx->setShouldForceMangleProto(true); + std::string MangledFnName = getMangledName(FD, MangleCtx.get()); + ASTUnit *Unit = nullptr; + StringRef ASTFileName; + auto FnUnitCacheEntry = FunctionAstUnitMap.find(MangledFnName); + if (FnUnitCacheEntry == FunctionAstUnitMap.end()) { + if (FunctionFileMap.empty()) { + std::string ExternalFunctionMap = (XTUDir + "/externalFnMap.txt").str(); + std::ifstream ExternalFnMapFile(ExternalFunctionMap); + std::string FunctionName, FileName; + while (ExternalFnMapFile >> FunctionName >> FileName) + FunctionFileMap[FunctionName] = (XTUDir + "/" + FileName).str(); + } + + auto It = FunctionFileMap.find(MangledFnName); + if (It != FunctionFileMap.end()) + ASTFileName = It->second; + else // No definition found even in some other build unit. + return nullptr; + auto ASTCacheEntry = FileASTUnitMap.find(ASTFileName); + if (ASTCacheEntry == FileASTUnitMap.end()) { + Unit = Loader(ASTFileName); + FileASTUnitMap[ASTFileName] = Unit; + FunctionAstUnitMap[MangledFnName] = Unit; + } else { + Unit = ASTCacheEntry->second; + FunctionAstUnitMap[MangledFnName] = Unit; + } + } else { + Unit = FnUnitCacheEntry->second; + } + + if (!Unit) + return nullptr; + assert(&Unit->getFileManager() == + &Unit->getASTContext().getSourceManager().getFileManager()); + ASTImporter &Importer = getOrCreateASTImporter(Unit->getASTContext()); + TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl(); + if (const FunctionDecl *ResultDecl = + iterateContextDecls(TU, MangledFnName, MangleCtx)) { + llvm::errs() << "Importing function " << MangledFnName << " from " + << ASTFileName << "\n"; + // FIXME: Refactor const_cast + auto *ToDecl = cast( + Importer.Import(const_cast(ResultDecl))); + assert(ToDecl->hasBody()); + ImportMap[FD] = ToDecl; + return ToDecl; + } + return nullptr; +} + +ASTImporter &ASTContext::getOrCreateASTImporter(ASTContext &From) { + auto I = ASTUnitImporterMap.find(From.getTranslationUnitDecl()); + if (I != ASTUnitImporterMap.end()) + return *I->second; + ASTImporter *NewImporter = new ASTImporter( + *this, getSourceManager().getFileManager(), + From, From.getSourceManager().getFileManager(), false); + ASTUnitImporterMap[From.getTranslationUnitDecl()] = NewImporter; + return *NewImporter; +} + CharUnits ASTContext::getDeclAlign(const Decl *D, bool ForAlignof) const { unsigned Align = Target->getCharWidth(); Index: lib/AST/ASTImporter.cpp =================================================================== --- lib/AST/ASTImporter.cpp +++ lib/AST/ASTImporter.cpp @@ -3206,6 +3206,8 @@ if (ToD) return ToD; + const FunctionDecl *FoundWithoutBody = nullptr; + // Try to find a function in our own ("to") context with the same name, same // type, and in the same context as the function we're importing. if (!LexicalDC->isFunctionOrMethod()) { @@ -3217,12 +3219,19 @@ if (!FoundDecls[I]->isInIdentifierNamespace(IDNS)) continue; - if (FunctionDecl *FoundFunction = dyn_cast(FoundDecls[I])) { + if (auto *FoundFunction = dyn_cast(FoundDecls[I])) { if (FoundFunction->hasExternalFormalLinkage() && D->hasExternalFormalLinkage()) { if (Importer.IsStructurallyEquivalent(D->getType(), FoundFunction->getType())) { // FIXME: Actually try to merge the body and other attributes. + const FunctionDecl *FromBodyDecl = nullptr; + D->hasBody(FromBodyDecl); + if (D == FromBodyDecl && !FoundFunction->hasBody()) { + // This function is needed to merge completely. + FoundWithoutBody = FoundFunction; + break; + } return Importer.Imported(D, FoundFunction); } @@ -3261,8 +3270,7 @@ QualType FromTy = D->getType(); bool usedDifferentExceptionSpec = false; - if (const FunctionProtoType * - FromFPT = D->getType()->getAs()) { + if (const auto *FromFPT = D->getType()->getAs()) { FunctionProtoType::ExtProtoInfo FromEPI = FromFPT->getExtProtoInfo(); // FunctionProtoType::ExtProtoInfo's ExceptionSpecDecl can point to the // FunctionDecl that we are importing the FunctionProtoType for. @@ -3297,7 +3305,7 @@ TypeSourceInfo *TInfo = Importer.Import(D->getTypeSourceInfo()); FunctionDecl *ToFunction = nullptr; SourceLocation InnerLocStart = Importer.Import(D->getInnerLocStart()); - if (CXXConstructorDecl *FromConstructor = dyn_cast(D)) { + if (auto *FromConstructor = dyn_cast(D)) { ToFunction = CXXConstructorDecl::Create(Importer.getToContext(), cast(DC), InnerLocStart, @@ -3329,8 +3337,7 @@ NameInfo, T, TInfo, D->isInlineSpecified(), D->isImplicit()); - } else if (CXXConversionDecl *FromConversion - = dyn_cast(D)) { + } else if (auto *FromConversion = dyn_cast(D)) { ToFunction = CXXConversionDecl::Create(Importer.getToContext(), cast(DC), InnerLocStart, @@ -3339,7 +3346,7 @@ FromConversion->isExplicit(), D->isConstexpr(), Importer.Import(D->getLocEnd())); - } else if (CXXMethodDecl *Method = dyn_cast(D)) { + } else if (auto *Method = dyn_cast(D)) { ToFunction = CXXMethodDecl::Create(Importer.getToContext(), cast(DC), InnerLocStart, @@ -3373,6 +3380,12 @@ } ToFunction->setParams(Parameters); + if (FoundWithoutBody) { + auto *Recent = const_cast( + FoundWithoutBody->getMostRecentDecl()); + ToFunction->setPreviousDecl(Recent); + } + if (usedDifferentExceptionSpec) { // Update FunctionProtoType::ExtProtoInfo. QualType T = Importer.Import(D->getType()); Index: lib/AST/ItaniumMangle.cpp =================================================================== --- lib/AST/ItaniumMangle.cpp +++ lib/AST/ItaniumMangle.cpp @@ -79,6 +79,17 @@ if (FD->isExternC()) return FD->getASTContext().getTranslationUnitDecl(); + // Avoid infinite recursion with code like: + // void f(struct S* p); + // Where this is the first declaration of S. This is only valid for C. + // For some tools it makes sense to mangle C functions (e.g. avoid collisions + // when indexing). It might be nicer to check whether the Decl is in + // FunctionPrototypeScope, but this information is lost after the Sema is + // done. + if (!D->getASTContext().getLangOpts().CPlusPlus && DC->isFunctionOrMethod() && + isa(D)) + return D->getASTContext().getTranslationUnitDecl(); + return DC->getRedeclContext(); } @@ -653,9 +664,11 @@ // ::= // Don't mangle in the type if this isn't a decl we should typically mangle. - if (!Context.shouldMangleDeclName(FD)) { - mangleName(FD); - return; + if (!Context.shouldMangleDeclName(FD) && + !(Context.shouldForceMangleProto() && + FD->getType()->getAs())){ + mangleName(FD); + return; } AbiTagList ReturnTypeAbiTags = makeFunctionReturnTypeTags(FD); Index: lib/Basic/SourceManager.cpp =================================================================== --- lib/Basic/SourceManager.cpp +++ lib/Basic/SourceManager.cpp @@ -2025,6 +2025,8 @@ } /// \brief Determines the order of 2 source locations in the translation unit. +/// FIXME: It also works when two locations are from different translation +/// units. In that case it will return *some* order. /// /// \returns true if LHS source location comes before RHS, false otherwise. bool SourceManager::isBeforeInTranslationUnit(SourceLocation LHS, @@ -2121,7 +2123,8 @@ return LIsScratch; return LOffs.second < ROffs.second; } - llvm_unreachable("Unsortable locations found"); + // FIXME: Source locations from different translation units. + return LOffs.first < ROffs.first; } void SourceManager::PrintStats() const { Index: lib/StaticAnalyzer/Core/AnalyzerOptions.cpp =================================================================== --- lib/StaticAnalyzer/Core/AnalyzerOptions.cpp +++ lib/StaticAnalyzer/Core/AnalyzerOptions.cpp @@ -370,3 +370,10 @@ getBooleanOption("notes-as-events", /*Default=*/false); return DisplayNotesAsEvents.getValue(); } + +StringRef AnalyzerOptions::getXTUDir() { + if (!XTUDir.hasValue()) + XTUDir = getOptionAsString("xtu-dir", ""); + return XTUDir.getValue(); +} + Index: lib/StaticAnalyzer/Core/CMakeLists.txt =================================================================== --- lib/StaticAnalyzer/Core/CMakeLists.txt +++ lib/StaticAnalyzer/Core/CMakeLists.txt @@ -48,6 +48,7 @@ clangAST clangAnalysis clangBasic + clangFrontend clangLex clangRewrite ) Index: lib/StaticAnalyzer/Core/CallEvent.cpp =================================================================== --- lib/StaticAnalyzer/Core/CallEvent.cpp +++ lib/StaticAnalyzer/Core/CallEvent.cpp @@ -16,6 +16,9 @@ #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h" #include "clang/AST/ParentMap.h" #include "clang/Analysis/ProgramPoint.h" +#include "clang/Frontend/ASTUnit.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" #include "clang/StaticAnalyzer/Core/PathSensitive/DynamicTypeMap.h" #include "llvm/ADT/SmallSet.h" @@ -355,6 +358,44 @@ D->parameters()); } +RuntimeDefinition AnyFunctionCall::getRuntimeDefinition() const { + const FunctionDecl *FD = getDecl(); + // Note that the AnalysisDeclContext will have the FunctionDecl with + // the definition (if one exists). + if (!FD) + return RuntimeDefinition(); + + AnalysisDeclContext *AD = + getLocationContext()->getAnalysisDeclContext()-> + getManager()->getContext(FD); + if (AD->getBody()) + return RuntimeDefinition(AD->getDecl()); + + auto Engine = static_cast( + getState()->getStateManager().getOwningEngine()); + CompilerInstance &CI = Engine->getCompilerInstance(); + + auto ASTLoader = [&](StringRef ASTFileName) { + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticPrinter *DiagClient = + new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + IntrusiveRefCntPtr Diags( + new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient)); + return ASTUnit::LoadFromASTFile( + ASTFileName, CI.getPCHContainerOperations()->getRawReader(), + Diags, CI.getFileSystemOpts()) + .release(); + }; + + const FunctionDecl *XTUDecl = AD->getASTContext().getXTUDefinition( + FD, CI, Engine->getAnalysisManager().options.getXTUDir(), + CI.getDiagnostics(), ASTLoader); + + return RuntimeDefinition(XTUDecl); +} + + bool AnyFunctionCall::argumentsMayEscape() const { if (CallEvent::argumentsMayEscape() || hasVoidPointerToNonConstArg()) return true; Index: lib/StaticAnalyzer/Core/ExprEngine.cpp =================================================================== --- lib/StaticAnalyzer/Core/ExprEngine.cpp +++ lib/StaticAnalyzer/Core/ExprEngine.cpp @@ -67,25 +67,19 @@ static const char* TagProviderName = "ExprEngine"; -ExprEngine::ExprEngine(AnalysisManager &mgr, bool gcEnabled, - SetOfConstDecls *VisitedCalleesIn, - FunctionSummariesTy *FS, - InliningModes HowToInlineIn) - : AMgr(mgr), - AnalysisDeclContexts(mgr.getAnalysisDeclContextManager()), - Engine(*this, FS), - G(Engine.getGraph()), - StateMgr(getContext(), mgr.getStoreManagerCreator(), - mgr.getConstraintManagerCreator(), G.getAllocator(), - this), - SymMgr(StateMgr.getSymbolManager()), - svalBuilder(StateMgr.getSValBuilder()), - currStmtIdx(0), currBldrCtx(nullptr), - ObjCNoRet(mgr.getASTContext()), - ObjCGCEnabled(gcEnabled), BR(mgr, *this), - VisitedCallees(VisitedCalleesIn), - HowToInline(HowToInlineIn) -{ +ExprEngine::ExprEngine(CompilerInstance &CI, AnalysisManager &mgr, + bool gcEnabled, SetOfConstDecls *VisitedCalleesIn, + FunctionSummariesTy *FS, InliningModes HowToInlineIn) + : CI(CI), AMgr(mgr), + AnalysisDeclContexts(mgr.getAnalysisDeclContextManager()), + Engine(*this, FS), G(Engine.getGraph()), + StateMgr(getContext(), mgr.getStoreManagerCreator(), + mgr.getConstraintManagerCreator(), G.getAllocator(), this), + SymMgr(StateMgr.getSymbolManager()), + svalBuilder(StateMgr.getSValBuilder()), currStmtIdx(0), + currBldrCtx(nullptr), ObjCNoRet(mgr.getASTContext()), + ObjCGCEnabled(gcEnabled), BR(mgr, *this), + VisitedCallees(VisitedCalleesIn), HowToInline(HowToInlineIn) { unsigned TrimInterval = mgr.options.getGraphTrimInterval(); if (TrimInterval != 0) { // Enable eager node reclaimation when constructing the ExplodedGraph. Index: lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp =================================================================== --- lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp +++ lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp @@ -40,9 +40,17 @@ #include "llvm/Support/Program.h" #include "llvm/Support/Timer.h" #include "llvm/Support/raw_ostream.h" +#include #include #include #include +#include +#include +#include +#include +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/AST/Mangle.h" +#include "clang/Basic/TargetInfo.h" using namespace clang; using namespace ento; @@ -168,6 +176,7 @@ AnalyzerOptionsRef Opts; ArrayRef Plugins; CodeInjector *Injector; + CompilerInstance &CI; /// \brief Stores the declarations from the local translation unit. /// Note, we pre-compute the local declarations at parse time as an @@ -192,12 +201,12 @@ /// translation unit. FunctionSummariesTy FunctionSummaries; - AnalysisConsumer(const Preprocessor &pp, const std::string &outdir, - AnalyzerOptionsRef opts, ArrayRef plugins, - CodeInjector *injector) + AnalysisConsumer(CompilerInstance &CI, const Preprocessor &pp, + const std::string &outdir, AnalyzerOptionsRef opts, + ArrayRef plugins, CodeInjector *injector) : RecVisitorMode(0), RecVisitorBR(nullptr), Ctx(nullptr), PP(pp), OutDir(outdir), Opts(std::move(opts)), Plugins(plugins), - Injector(injector) { + Injector(injector), CI(CI) { DigestAnalyzerOptions(); if (Opts->PrintStats) { llvm::EnableStatistics(false); @@ -415,6 +424,21 @@ } } +extern std::string getMangledName(const NamedDecl *ND, + MangleContext *MangleCtx); + +void lockedWrite(const std::string &fileName, const std::string &content) { + if (content.empty()) + return; + int fd = open(fileName.c_str(), O_CREAT|O_WRONLY|O_APPEND, 0666); + flock(fd, LOCK_EX); + ssize_t written = write(fd, content.c_str(), content.length()); + assert(written == static_cast(content.length())); + (void)written; + flock(fd, LOCK_UN); + close(fd); +} + static bool shouldSkipFunction(const Decl *D, const SetOfConstDecls &Visited, const SetOfConstDecls &VisitedAsTopLevel) { @@ -706,7 +730,8 @@ if (!Mgr->getAnalysisDeclContext(D)->getAnalysis()) return; - ExprEngine Eng(*Mgr, ObjCGCEnabled, VisitedCallees, &FunctionSummaries,IMode); + ExprEngine Eng(CI, *Mgr, ObjCGCEnabled, VisitedCallees, &FunctionSummaries, + IMode); // Set the graph auditor. std::unique_ptr Auditor; @@ -764,7 +789,7 @@ bool hasModelPath = analyzerOpts->Config.count("model-path") > 0; return llvm::make_unique( - CI.getPreprocessor(), CI.getFrontendOpts().OutputFile, analyzerOpts, + CI, CI.getPreprocessor(), CI.getFrontendOpts().OutputFile, analyzerOpts, CI.getFrontendOpts().Plugins, hasModelPath ? new ModelInjector(CI) : nullptr); } Index: test/Analysis/Inputs/externalFnMap.txt =================================================================== --- /dev/null +++ test/Analysis/Inputs/externalFnMap.txt @@ -0,0 +1,14 @@ +_Z7h_chaini@x86_64 xtu-chain.cpp.ast +_ZN4chns4chf2Ei@x86_64 xtu-chain.cpp.ast +_ZN4chns5chcls4chf4Ei@x86_64 xtu-chain.cpp.ast +_Z1fi@x86_64 xtu-other.cpp.ast +_Z1gi@x86_64 xtu-other.cpp.ast +_Z1hi@x86_64 xtu-other.cpp.ast +_ZN4myns9embed_cls4feclEi@x86_64 xtu-other.cpp.ast +_ZN4myns3fnsEi@x86_64 xtu-other.cpp.ast +_ZN5mycls10embed_cls25fecl2Ei@x86_64 xtu-other.cpp.ast +_ZN5mycls4fsclEi@x86_64 xtu-other.cpp.ast +_ZN4chns4chf3Ei@x86_64 xtu-other.cpp.ast +_ZN5mycls3fclEi@x86_64 xtu-other.cpp.ast +_ZN4chns4chf1Ei@x86_64 xtu-other.cpp.ast +_ZN4myns8embed_ns4fensEi@x86_64 xtu-other.cpp.ast Index: test/Analysis/Inputs/xtu-chain.cpp =================================================================== --- /dev/null +++ test/Analysis/Inputs/xtu-chain.cpp @@ -0,0 +1,20 @@ +int h_chain(int x) { + return x * 2; +} + +namespace chns { +int chf3(int x); + +int chf2(int x) { + return chf3(x); +} + +class chcls { +public: + int chf4(int x); +}; + +int chcls::chf4(int x) { + return x * 3; +} +} Index: test/Analysis/Inputs/xtu-other.cpp =================================================================== --- /dev/null +++ test/Analysis/Inputs/xtu-other.cpp @@ -0,0 +1,67 @@ +int callback_to_main(int x); +int f(int x) { + return x - 1; +} + +int g(int x) { + return callback_to_main(x) + 1; +} + +int h_chain(int); + +int h(int x) { + return 2 * h_chain(x); +} + +namespace myns { +int fns(int x) { + return x + 7; +} + +namespace embed_ns { +int fens(int x) { + return x - 3; +} +} + +class embed_cls { +public: + int fecl(int x) { + return x - 7; + } +}; +} + +class mycls { +public: + int fcl(int x) { + return x + 5; + } + static int fscl(int x) { + return x + 6; + } + + class embed_cls2 { + public: + int fecl2(int x) { + return x - 11; + } + }; +}; + +namespace chns { +int chf2(int x); + +class chcls { +public: + int chf4(int x); +}; + +int chf3(int x) { + return chcls().chf4(x); +} + +int chf1(int x) { + return chf2(x); +} +} Index: test/Analysis/xtu-main.cpp =================================================================== --- /dev/null +++ test/Analysis/xtu-main.cpp @@ -0,0 +1,58 @@ +// RUN: mkdir -p %T/xtudir +// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-pch -o %T/xtudir/xtu-other.cpp.ast %S/Inputs/xtu-other.cpp +// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -emit-pch -o %T/xtudir/xtu-chain.cpp.ast %S/Inputs/xtu-chain.cpp +// RUN: cp %S/Inputs/externalFnMap.txt %T/xtudir/ +// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fsyntax-only -analyze -analyzer-checker=core,debug.ExprInspection -analyzer-config xtu-dir=%T/xtudir -analyzer-config reanalyze-xtu-visited=true -verify %s + +void clang_analyzer_eval(int); + +int f(int); +int g(int); +int h(int); + +int callback_to_main(int x) { return x + 1; } + +namespace myns { +int fns(int x); + +namespace embed_ns { +int fens(int x); +} + +class embed_cls { +public: + int fecl(int x); +}; +} + +class mycls { +public: + int fcl(int x); + static int fscl(int x); + + class embed_cls2 { + public: + int fecl2(int x); + }; +}; + +namespace chns { +int chf1(int x); +} + +int main() { + clang_analyzer_eval(f(3) == 2); // expected-warning{{TRUE}} + clang_analyzer_eval(f(4) == 3); // expected-warning{{TRUE}} + clang_analyzer_eval(f(5) == 3); // expected-warning{{FALSE}} + clang_analyzer_eval(g(4) == 6); // expected-warning{{TRUE}} + clang_analyzer_eval(h(2) == 8); // expected-warning{{TRUE}} + + clang_analyzer_eval(myns::fns(2) == 9); // expected-warning{{TRUE}} + clang_analyzer_eval(myns::embed_ns::fens(2) == -1); // expected-warning{{TRUE}} + clang_analyzer_eval(mycls().fcl(1) == 6); // expected-warning{{TRUE}} + clang_analyzer_eval(mycls::fscl(1) == 7); // expected-warning{{TRUE}} + clang_analyzer_eval(myns::embed_cls().fecl(1) == -6); // expected-warning{{TRUE}} + clang_analyzer_eval(mycls::embed_cls2().fecl2(0) == -11); // expected-warning{{TRUE}} + + clang_analyzer_eval(chns::chf1(4) == 12); // expected-warning{{TRUE}} +} Index: tools/CMakeLists.txt =================================================================== --- tools/CMakeLists.txt +++ tools/CMakeLists.txt @@ -19,6 +19,8 @@ add_clang_subdirectory(clang-check) add_clang_subdirectory(scan-build) add_clang_subdirectory(scan-view) + add_clang_subdirectory(clang-func-mapping) + add_clang_subdirectory(clang-cmdline-arch-extractor) endif() # We support checking out the clang-tools-extra repository into the 'extra' Index: tools/clang-cmdline-arch-extractor/CMakeLists.txt =================================================================== --- /dev/null +++ tools/clang-cmdline-arch-extractor/CMakeLists.txt @@ -0,0 +1,20 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + asmparser + support + mc + ) + +add_clang_executable(clang-cmdline-arch-extractor + ClangCmdlineArchExtractor.cpp + ) + +target_link_libraries(clang-cmdline-arch-extractor + clangTooling + clangBasic + clangFrontend + clangRewriteFrontend + ) + +install(TARGETS clang-cmdline-arch-extractor + RUNTIME DESTINATION bin) Index: tools/clang-cmdline-arch-extractor/ClangCmdlineArchExtractor.cpp =================================================================== --- /dev/null +++ tools/clang-cmdline-arch-extractor/ClangCmdlineArchExtractor.cpp @@ -0,0 +1,73 @@ +//===- ClangCmdlineArchExtractor.cpp ------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--------------------------------------------------------------------===// +// +// Clang tool which prints architecture type for a given command line. +// +//===--------------------------------------------------------------------===// + +#include "clang/Basic/LLVM.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/Utils.h" +#include "clang/Tooling/CommonOptionsParser.h" +#include "llvm/Support/PrettyStackTrace.h" +#include "llvm/Support/Signals.h" +#include +#include + +using namespace llvm; +using namespace clang; + +static std::string getTripleSuffix(const Triple &Triple) { + // We are not going to support vendor and don't support OS and environment. + // FIXME: support OS and environment correctly + Triple::ArchType T = Triple.getArch(); + if (T == Triple::thumb) + T = Triple::arm; + return Triple.getArchTypeName(T); +} + +int main(int argc, const char **argv) { + // Print a stack trace if we signal out. + sys::PrintStackTraceOnErrorSignal(argv[0], true); + PrettyStackTraceProgram X(argc, argv); + + SmallVector Sources; + std::vector Args; + const StringRef cppFile = ".cpp", ccFile = ".cc", cFile = ".c", + cxxFile = ".cxx"; + for (int i = 1; i < argc; i++) { + StringRef Arg = argv[i]; + if (Arg.endswith(cppFile) || Arg.endswith(ccFile) || Arg.endswith(cFile) || + Arg.endswith(cxxFile)) { + Sources.push_back(argv[i]); + } else { + Args.push_back(argv[i]); + } + } + + if (Sources.empty()) + return 1; + + Args.push_back(Sources[0].data()); + std::unique_ptr CI( + createInvocationFromCommandLine(Args)); + + const std::string Suffix = + "@" + getTripleSuffix(llvm::Triple(CI->getTargetOpts().Triple)); + + for (StringRef SourceFile : Sources) { + char *Path = realpath(SourceFile.data(), nullptr); + if (Path) + outs() << Path << Suffix << " "; + free(Path); + } + + return 0; +} Index: tools/clang-func-mapping/CMakeLists.txt =================================================================== --- /dev/null +++ tools/clang-func-mapping/CMakeLists.txt @@ -0,0 +1,21 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + asmparser + support + mc + ) + +add_clang_executable(clang-func-mapping + ClangFnMapGen.cpp + ) + +target_link_libraries(clang-func-mapping + clangTooling + clangAST + clangBasic + clangFrontend + clangRewriteFrontend + ) + +install(TARGETS clang-func-mapping + RUNTIME DESTINATION bin) Index: tools/clang-func-mapping/ClangFnMapGen.cpp =================================================================== --- /dev/null +++ tools/clang-func-mapping/ClangFnMapGen.cpp @@ -0,0 +1,257 @@ +//===- ClangFnMapGen.cpp -----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--------------------------------------------------------------------===// +// +// Clang tool which creates a list of defined functions and the files in which +// they are defined. +// +//===--------------------------------------------------------------------===// + +#include "clang/AST/ASTConsumer.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/GlobalDecl.h" +#include "clang/AST/Mangle.h" +#include "clang/AST/StmtVisitor.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace llvm; +using namespace clang; +using namespace clang::tooling; + +typedef StringSet<> StrSet; +typedef StringMap CallGraph; + +static cl::OptionCategory ClangFnMapGenCategory("clang-fnmapgen options"); +static cl::opt XTUDir( + "xtu-dir", + cl::desc( + "Directory that contains the XTU related files (e.g.: AST dumps)."), + cl::init(""), cl::cat(ClangFnMapGenCategory)); + +static void lockedWrite(const std::string &fileName, + const std::string &content) { + if (!content.empty()) { + int fd = open(fileName.c_str(), O_CREAT | O_WRONLY | O_APPEND, 0666); + flock(fd, LOCK_EX); + ssize_t written = write(fd, content.c_str(), content.length()); + assert(written == static_cast(content.length())); + (void)written; + flock(fd, LOCK_UN); + close(fd); + } +} + +static std::string getTripleSuffix(ASTContext &Ctx) { + // We are not going to support vendor and don't support OS and environment. + // FIXME: support OS and environment correctly. + Triple::ArchType T = Ctx.getTargetInfo().getTriple().getArch(); + if (T == Triple::thumb) + T = Triple::arm; + return Ctx.getTargetInfo().getTriple().getArchTypeName(T); +} + +class MapFunctionNamesConsumer : public ASTConsumer { +private: + ASTContext &Ctx; + ItaniumMangleContext *ItaniumCtx; + std::stringstream DefinedFuncsStr; + std::stringstream ExternFuncStr; + CallGraph CG; + const std::string Triple; + +public: + MapFunctionNamesConsumer(ASTContext &Context, ItaniumMangleContext *MangleCtx) + : Ctx(Context), ItaniumCtx(MangleCtx), + Triple(std::string("@") + getTripleSuffix(Context)) {} + std::string CurrentFileName; + + ~MapFunctionNamesConsumer(); + virtual void HandleTranslationUnit(ASTContext &Ctx) { + handleDecl(Ctx.getTranslationUnitDecl()); + } + +private: + std::string getMangledName(const FunctionDecl *FD, MangleContext *Ctx); + std::string getMangledName(const FunctionDecl *FD) { + return getMangledName(FD, ItaniumCtx); + } + + bool isCLibraryFunction(const FunctionDecl *FD); + void handleDecl(const Decl *D); + + class WalkAST : public ConstStmtVisitor { + MapFunctionNamesConsumer &Parent; + std::string CurrentFuncName; + MangleContext *MangleCtx; + const std::string Triple; + + public: + WalkAST(MapFunctionNamesConsumer &parent, const std::string &FuncName, + MangleContext *Ctx, const std::string &triple) + : Parent(parent), CurrentFuncName(FuncName), MangleCtx(Ctx), + Triple(triple) {} + void VisitCallExpr(const CallExpr *CE); + void VisitStmt(const Stmt *S) { VisitChildren(S); } + void VisitChildren(const Stmt *S); + }; +}; + +std::string MapFunctionNamesConsumer::getMangledName(const FunctionDecl *FD, + MangleContext *Ctx) { + std::string MangledName; + llvm::raw_string_ostream os(MangledName); + if (const auto *CCD = dyn_cast(FD)) + // FIXME: Use correct Ctor/DtorType. + Ctx->mangleCXXCtor(CCD, Ctor_Complete, os); + else if (const auto *CDD = dyn_cast(FD)) + Ctx->mangleCXXDtor(CDD, Dtor_Complete, os); + else + Ctx->mangleName(FD, os); + os.flush(); + return MangledName; +} + +void MapFunctionNamesConsumer::handleDecl(const Decl *D) { + if (!D) + return; + + if (const auto *FD = dyn_cast(D)) { + if (const Stmt *Body = FD->getBody()) { + std::string MangledName = getMangledName(FD); + const SourceManager &SM = Ctx.getSourceManager(); + if (CurrentFileName.empty()) { + StringRef SMgrName = + SM.getFileEntryForID(SM.getMainFileID())->getName(); + char *Path = realpath(SMgrName.str().c_str(), nullptr); + CurrentFileName = Path; + free(Path); + } + + std::string FileName = + std::string("/ast/") + getTripleSuffix(Ctx) + CurrentFileName; + std::string FullName = MangledName + Triple; + + switch (FD->getLinkageInternal()) { + case ExternalLinkage: + case VisibleNoLinkage: + case UniqueExternalLinkage: + if (SM.isInMainFile(Body->getLocStart())) + DefinedFuncsStr << "!"; + DefinedFuncsStr << FullName << " " << FileName << "\n"; + default: + break; + } + + WalkAST Walker(*this, FullName, ItaniumCtx, Triple); + Walker.Visit(Body); + } else if (!FD->getBody() && !FD->getBuiltinID()) { + std::string MangledName = getMangledName(FD); + ExternFuncStr << MangledName << Triple << "\n"; + } + } + + if (const auto *DC = dyn_cast(D)) + for (const Decl *D : DC->decls()) + handleDecl(D); +} + +bool MapFunctionNamesConsumer::isCLibraryFunction(const FunctionDecl *FD) { + SourceManager &SM = Ctx.getSourceManager(); + if (!FD) + return false; + SourceLocation Loc = FD->getLocation(); + if (Loc.isValid()) + return SM.isInSystemHeader(Loc); + return true; +} + +MapFunctionNamesConsumer::~MapFunctionNamesConsumer() { + // Flush results to files. + std::string BuildDir = XTUDir; + lockedWrite(BuildDir + "/externalFns.txt", ExternFuncStr.str()); + lockedWrite(BuildDir + "/definedFns.txt", DefinedFuncsStr.str()); + std::ostringstream CFGStr; + for (auto &Entry : CG) { + CFGStr << CurrentFileName << Triple << "::" << Entry.getKey().data(); + for (auto &E : Entry.getValue()) + CFGStr << ' ' << E.getKey().data(); + CFGStr << '\n'; + } + + lockedWrite(BuildDir + "/cfg.txt", CFGStr.str()); +} + +void MapFunctionNamesConsumer::WalkAST::VisitChildren(const Stmt *S) { + for (const Stmt *CS : S->children()) + if (CS) + Visit(CS); +} + +void MapFunctionNamesConsumer::WalkAST::VisitCallExpr(const CallExpr *CE) { + const auto *FD = dyn_cast_or_null(CE->getCalleeDecl()); + if (FD && !FD->getBuiltinID()) { + std::string FuncName = (FD->hasBody() ? "::" : "") + + Parent.getMangledName(FD, MangleCtx) + Triple; + Parent.CG[CurrentFuncName].insert(FuncName); + } + VisitChildren(CE); +} + +class MapFunctionNamesAction : public ASTFrontendAction { +protected: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + llvm::StringRef) { + ItaniumMangleContext *ItaniumCtx = + ItaniumMangleContext::create(CI.getASTContext(), CI.getDiagnostics()); + ItaniumCtx->setShouldForceMangleProto(true); + std::unique_ptr PFC( + new MapFunctionNamesConsumer(CI.getASTContext(), ItaniumCtx)); + return PFC; + } +}; + +int main(int argc, const char **argv) { + // Print a stack trace if we signal out. + sys::PrintStackTraceOnErrorSignal(argv[0], false); + PrettyStackTraceProgram X(argc, argv); + + SmallVector Sources; + CommonOptionsParser OptionsParser(argc, argv, ClangFnMapGenCategory, + cl::ZeroOrMore); + + if (XTUDir.getNumOccurrences() != 1) { + errs() << "Exactly one XTU dir should be provided\n"; + return 1; + } + const StringRef cppFile = ".cpp", ccFile = ".cc", cFile = ".c", + cxxFile = ".cxx"; + for (int i = 1; i < argc; i++) { + StringRef arg = argv[i]; + if (arg.endswith(cppFile) || arg.endswith(ccFile) || arg.endswith(cFile) || + arg.endswith(cxxFile)) { + Sources.push_back(arg); + } + } + ClangTool Tool(OptionsParser.getCompilations(), Sources); + Tool.run(newFrontendActionFactory().get()); + return 0; +} Index: tools/scan-build-py/libscanbuild/runner.py =================================================================== --- tools/scan-build-py/libscanbuild/runner.py +++ tools/scan-build-py/libscanbuild/runner.py @@ -162,7 +162,8 @@ def target(): """ Creates output file name for reports. """ - if opts['output_format'] in {'plist', 'plist-html'}: + if opts['output_format'] in {'plist', 'plist-html', + 'plist-multi-file'}: (handle, name) = tempfile.mkstemp(prefix='report-', suffix='.plist', dir=opts['output_dir']) Index: tools/xtu-analysis/xtu-analyze.py =================================================================== --- /dev/null +++ tools/xtu-analysis/xtu-analyze.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python + +import argparse +import io +import json +import multiprocessing +import os +import re +import shutil +import signal +import subprocess +import string +import sys +import threading +import time +import uuid +import sys + +reload(sys) +sys.setdefaultencoding('utf8') + +sys.path.append(os.path.join(os.path.dirname(__file__), + '..', '..', 'utils', 'analyzer')) +try: + import MergeCoverage +except: + raise + +threading_factor = int(multiprocessing.cpu_count() * 1.5) +analyser_output_formats = ['plist-multi-file', 'plist', 'plist-html', + 'html', 'text'] +analyser_output_format = analyser_output_formats[0] +gcov_outdir = 'gcov' +gcov_tmpdir = gcov_outdir + '_tmp' + +parser = argparse.ArgumentParser( + description='Executes 2nd pass of XTU analysis') +parser.add_argument('-b', required=True, dest='buildlog', metavar='build.json', + help='Use a JSON Compilation Database') +parser.add_argument('-p', metavar='preanalyze-dir', dest='xtuindir', + help='Use directory for reading preanalyzation data ' + '(default=".xtu")', + default='.xtu') +parser.add_argument('-o', metavar='output-dir', dest='xtuoutdir', + help='Use directory for output analyzation results ' + '(default=".xtu-out")', + default='.xtu-out') +parser.add_argument('-e', metavar='enabled-checker', nargs='+', + dest='enabled_checkers', + help='List all enabled checkers') +parser.add_argument('-d', metavar='disabled-checker', nargs='+', + dest='disabled_checkers', + help='List all disabled checkers') +parser.add_argument('-j', metavar='threads', dest='threads', + help='Number of threads used (default=' + + str(threading_factor) + ')', + default=threading_factor) +parser.add_argument('-v', dest='verbose', action='store_true', + help='Verbose output of every command executed') +parser.add_argument('--clang-path', metavar='clang-path', dest='clang_path', + help='Set path of clang binaries to be used (default ' + 'taken from CLANG_PATH environment variable)', + default=os.environ.get('CLANG_PATH')) +parser.add_argument('--analyze-cc-path', metavar='analyze-cc-path', + dest='analyze_path', + help='Set path of analyze-cc to be used ' + '(default is taken from CLANG_ANALYZE_CC_PATH ' + 'environment variable)', + default=os.environ.get('CLANG_ANALYZE_CC_PATH')) +parser.add_argument('--output-format', metavar='format', + choices=analyser_output_formats, + default=analyser_output_format, + help='Format for analysis reports ' + '(one of %s; default is "%s").' % + (', '.join(analyser_output_formats), + analyser_output_format)) +parser.add_argument('--no-xtu', dest='no_xtu', action='store_true', + help='Do not use XTU at all, ' + 'only do normal static analysis') +parser.add_argument('--record-coverage', dest='record_coverage', + action='store_true', + help='Generate coverage information during analysis') +mainargs = parser.parse_args() + +concurrent_threads = 0 +concurrent_thread_times = [0.0] +concurrent_thread_last_clock = time.time() + +if mainargs.clang_path is None: + clang_path = '' +else: + clang_path = os.path.abspath(mainargs.clang_path) +if mainargs.verbose: + print 'XTU uses clang dir: ' + (clang_path if clang_path != '' + else '') + +if mainargs.analyze_path is None: + analyze_path = '' +else: + analyze_path = os.path.abspath(mainargs.analyze_path) +if mainargs.verbose: + print 'XTU uses analyze-cc dir: ' + (analyze_path if analyze_path != '' + else '') + +analyzer_params = [] +if mainargs.enabled_checkers: + analyzer_params += ['-analyzer-checker', mainargs.enabled_checkers] +if mainargs.disabled_checkers: + analyzer_params += ['-analyzer-disable-checker', mainargs.disable_checkers] +if not mainargs.no_xtu: + analyzer_params += ['-analyzer-config', + 'xtu-dir=' + os.path.abspath(mainargs.xtuindir)] +analyzer_params += ['-analyzer-config', 'reanalyze-xtu-visited=true'] +if mainargs.record_coverage: + gcov_tmppath = os.path.abspath(os.path.join(mainargs.xtuoutdir, + gcov_tmpdir)) + gcov_finalpath = os.path.abspath(os.path.join(mainargs.xtuoutdir, + gcov_outdir)) + shutil.rmtree(gcov_tmppath, True) + # analyzer_params += ['-analyzer-config', + # 'record-coverage=' + gcov_tmppath] +analyzer_params += ['-analyzer-stats'] +# analyzer_params += ['-analyzer-output ' + mainargs.output_format] +passthru_analyzer_params = [] +for param in analyzer_params: + passthru_analyzer_params += ['-Xanalyzer'] + passthru_analyzer_params += [param] +passthru_analyzer_params += ['--analyzer-output ' + mainargs.output_format] + +analyzer_env = os.environ.copy() +analyzer_env['ANALYZE_BUILD_CLANG'] = os.path.join(clang_path, 'clang') +analyzer_env['ANALYZE_BUILD_REPORT_DIR'] = os.path.abspath(mainargs.xtuoutdir) +analyzer_env['ANALYZE_BUILD_PARAMETERS'] = ' '.join(passthru_analyzer_params) +analyzer_env['ANALYZE_BUILD_REPORT_FORMAT'] = mainargs.output_format +# analyzer_env['ANALYZE_BUILD_VERBOSE'] = 'DEBUG' + +graph_lock = threading.Lock() + +buildlog_file = open(mainargs.buildlog, 'r') +buildlog = json.load(buildlog_file) +buildlog_file.close() + +src_pattern = re.compile('.*\.(C|c|cc|cpp|cxx|ii|m|mm)$', re.IGNORECASE) +dircmd_separator = ': ' +dircmd_2_orders = {} +src_build_steps = 0 +for step in buildlog: + if src_pattern.match(step['file']): + uid = step['directory'] + dircmd_separator + step['command'] + if uid not in dircmd_2_orders: + dircmd_2_orders[uid] = [src_build_steps] + else: + dircmd_2_orders[uid].append(src_build_steps) + src_build_steps += 1 + + +def get_compiler_and_arguments(cmd): + had_command = False + args = [] + for arg in cmd.split(): + if had_command: + args.append(arg) + if not had_command and arg.find('=') == -1: + had_command = True + compiler = arg + return compiler, args + + +def analyze(directory, command): + compiler, args = get_compiler_and_arguments(command) + + last_src = None + for cmdpart in command.split(): + if src_pattern.match(cmdpart): + last_src = cmdpart + tu_name = '' + if last_src: + tu_name += last_src.split(os.sep)[-1] + tu_name += '_' + str(uuid.uuid4()) + + cmdenv = analyzer_env.copy() + cmdenv['ANALYZE_BUILD_CC'] = compiler + cmdenv['ANALYZE_BUILD_CXX'] = compiler + if mainargs.record_coverage: + cmdenv['ANALYZE_BUILD_PARAMETERS'] += \ + ' -Xanalyzer -analyzer-config -Xanalyzer record-coverage=' + \ + os.path.join(gcov_tmppath, tu_name) + analyze_cmd = os.path.join(analyze_path, 'analyze-cc') + \ + ' ' + string.join(args, ' ') + if mainargs.verbose: + print analyze_cmd + + # Buffer output of subprocess and dump it out at the end, so that + # the subprocess doesn't continue to write output after the user + # sends SIGTERM + runOK = True + out = '******* Error running command' + try: + po = subprocess.Popen(analyze_cmd, shell=True, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + universal_newlines=True, + cwd=directory, + env=cmdenv) + out, _ = po.communicate() + runOK = not po.returncode + except OSError: + runOK = False + if mainargs.verbose: + sys.stdout.write(out) + if not runOK: + prefix = os.path.join(os.path.abspath(mainargs.xtuoutdir), "fails") + else: + prefix = os.path.join(os.path.abspath(mainargs.xtuoutdir), "passes") + with open(os.path.join(prefix, "%s.out" % tu_name), "w") as f: + f.write("%s\n%s" % (analyze_cmd, out)) + + +def analyze_work(): + global concurrent_threads + global concurrent_thread_times + global concurrent_thread_last_clock + global graph_lock + global dircmd_2_orders + while len(dircmd_2_orders) > 0: + graph_lock.acquire() + found_dircmd_orders = None + found_dircmd = None + found_orders = None + for dircmd_orders in dircmd_2_orders.items(): + dircmd = dircmd_orders[0].split(dircmd_separator, 2) + orders = dircmd_orders[1] + assert len(dircmd) == 2 and len(dircmd[0]) > 0 and \ + len(dircmd[1]) > 0 + assert len(orders) > 0 + found_dircmd_orders = dircmd_orders + found_dircmd = dircmd + found_orders = orders + break + if found_dircmd_orders is not None: + del dircmd_2_orders[found_dircmd_orders[0]] + + concurrent_thread_current_clock = time.time() + concurrent_thread_times[concurrent_threads] += \ + concurrent_thread_current_clock - concurrent_thread_last_clock + concurrent_thread_last_clock = concurrent_thread_current_clock + concurrent_threads += 1 + if len(concurrent_thread_times) == concurrent_threads: + concurrent_thread_times.append(0.0) + + graph_lock.release() + analyze(found_dircmd[0], found_dircmd[1]) + graph_lock.acquire() + + concurrent_thread_current_clock = time.time() + concurrent_thread_times[concurrent_threads] += \ + concurrent_thread_current_clock - concurrent_thread_last_clock + concurrent_thread_last_clock = concurrent_thread_current_clock + concurrent_threads -= 1 + assert concurrent_threads >= 0 + + graph_lock.release() + else: + graph_lock.release() + time.sleep(0.125) + +try: + os.makedirs(os.path.abspath(mainargs.xtuoutdir)) +except OSError: + print 'Output directory %s already exists!' % \ + os.path.abspath(mainargs.xtuoutdir) + sys.exit(1) + +os.makedirs(os.path.join(os.path.abspath(mainargs.xtuoutdir), "passes")) +os.makedirs(os.path.join(os.path.abspath(mainargs.xtuoutdir), "fails")) + +original_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) +signal.signal(signal.SIGINT, original_handler) + +analyze_workers = [] +for i in range(int(mainargs.threads)): + analyze_workers.append(threading.Thread(target=analyze_work)) +for worker in analyze_workers: + worker.start() +try: + for worker in analyze_workers: + worker.join(9999999999) +except KeyboardInterrupt: + exit(1) + +try: + os.removedirs(os.path.abspath(mainargs.xtuoutdir)) + print 'Removing directory %s because it contains no reports' % \ + os.path.abspath(mainargs.xtuoutdir) +except OSError: + pass + +if mainargs.record_coverage: + MergeCoverage.main(gcov_tmppath, gcov_finalpath) + shutil.rmtree(gcov_tmppath, True) + +assert concurrent_threads == 0 +concurrent_thread_times[0] += time.time() - concurrent_thread_last_clock +sumtime = 0.0 +for i in range(len(concurrent_thread_times)): + sumtime += concurrent_thread_times[i] +print '--- Total running time: %.2fs' % sumtime +for i in range(len(concurrent_thread_times)): + print '----- ' + \ + (('using %d processes' % i) if i != 0 else 'self time') + \ + ' for %.2fs (%.0f%%)' % (concurrent_thread_times[i], + concurrent_thread_times[i] * 100.0 / sumtime) Index: tools/xtu-analysis/xtu-build.py =================================================================== --- /dev/null +++ tools/xtu-analysis/xtu-build.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python + +import argparse +import io +import json +import multiprocessing +import os +import re +import signal +import subprocess +import string + +threading_factor = int(multiprocessing.cpu_count() * 1.5) +timeout = 86400 + +parser = argparse.ArgumentParser( + description='Executes 1st pass of XTU analysis') +parser.add_argument('-b', required=True, dest='buildlog', + metavar='build.json', + help='Use a JSON Compilation Database') +parser.add_argument('-p', metavar='preanalyze-dir', dest='xtuindir', + help='Use directory for generating preanalyzation data ' + '(default=".xtu")', + default='.xtu') +parser.add_argument('-j', metavar='threads', dest='threads', + help='Number of threads used (default=' + + str(threading_factor) + ')', + default=threading_factor) +parser.add_argument('-v', dest='verbose', action='store_true', + help='Verbose output of every command executed') +parser.add_argument('--clang-path', metavar='clang-path', dest='clang_path', + help='Set path of clang binaries to be used ' + '(default taken from CLANG_PATH envvar)', + default=os.environ.get('CLANG_PATH')) +parser.add_argument('--timeout', metavar='N', + help='Timeout for build in seconds (default: %d)' % + timeout, + default=timeout) +mainargs = parser.parse_args() + +if mainargs.clang_path is None: + clang_path = '' +else: + clang_path = os.path.abspath(mainargs.clang_path) +if mainargs.verbose: + print 'XTU uses clang dir: ' + \ + (clang_path if clang_path != '' else '') + +buildlog_file = open(mainargs.buildlog, 'r') +buildlog = json.load(buildlog_file) +buildlog_file.close() + +src_pattern = re.compile('.*\.(C|c|cc|cpp|cxx|ii|m|mm)$', re.IGNORECASE) +src_2_dir = {} +src_2_cmd = {} +src_order = [] +cmd_2_src = {} +cmd_order = [] +for step in buildlog: + if src_pattern.match(step['file']): + if step['file'] not in src_2_dir: + src_2_dir[step['file']] = step['directory'] + if step['file'] not in src_2_cmd: + src_2_cmd[step['file']] = step['command'] + src_order.append(step['file']) + if step['command'] not in cmd_2_src: + cmd_2_src[step['command']] = [step['file']] + cmd_order.append(step['command']) + else: + cmd_2_src[step['command']].append(step['file']) + + +def clear_file(filename): + try: + os.remove(filename) + except OSError: + pass + + +def get_command_arguments(cmd): + had_command = False + args = [] + for arg in cmd.split(): + if had_command and not src_pattern.match(arg): + args.append(arg) + if not had_command and arg.find('=') == -1: + had_command = True + return args + + +def generate_ast(source): + cmd = src_2_cmd[source] + args = get_command_arguments(cmd) + arch_command = os.path.join(clang_path, 'clang-cmdline-arch-extractor') + \ + ' ' + string.join(args, ' ') + ' ' + source + if mainargs.verbose: + print arch_command + arch_output = subprocess.check_output(arch_command, shell=True) + arch = arch_output[arch_output.rfind('@')+1:].strip() + ast_joined_path = os.path.join(mainargs.xtuindir, + os.path.join('/ast/' + arch, + os.path.realpath(source)[1:] + + '.ast')[1:]) + ast_path = os.path.abspath(ast_joined_path) + try: + os.makedirs(os.path.dirname(ast_path)) + except OSError: + if os.path.isdir(os.path.dirname(ast_path)): + pass + else: + raise + dir_command = 'cd ' + src_2_dir[source] + ast_command = os.path.join(clang_path, 'clang') + ' -emit-ast ' + \ + string.join(args, ' ') + ' -w ' + source + ' -o ' + ast_path + if mainargs.verbose: + print dir_command + " && " + ast_command + subprocess.call(dir_command + " && " + ast_command, shell=True) + + +def map_functions(command): + args = get_command_arguments(command) + sources = cmd_2_src[command] + dir_command = 'cd ' + src_2_dir[sources[0]] + funcmap_command = os.path.join(clang_path, 'clang-func-mapping') + \ + ' --xtu-dir ' + os.path.abspath(mainargs.xtuindir) + ' ' + \ + string.join(sources, ' ') + ' -- ' + string.join(args, ' ') + if mainargs.verbose: + print funcmap_command + subprocess.call(dir_command + " && " + funcmap_command, shell=True) + +clear_file(os.path.join(mainargs.xtuindir, 'cfg.txt')) +clear_file(os.path.join(mainargs.xtuindir, 'definedFns.txt')) +clear_file(os.path.join(mainargs.xtuindir, 'externalFns.txt')) +clear_file(os.path.join(mainargs.xtuindir, 'externalFnMap.txt')) + +original_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) +ast_workers = multiprocessing.Pool(processes=int(mainargs.threads)) +signal.signal(signal.SIGINT, original_handler) +try: + res = ast_workers.map_async(generate_ast, src_order) + # Block with timeout so that signals don't get ignored, python bug 8296 + res.get(mainargs.timeout) +except KeyboardInterrupt: + ast_workers.terminate() + ast_workers.join() + exit(1) +else: + ast_workers.close() + ast_workers.join() + +original_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) +funcmap_workers = multiprocessing.Pool(processes=int(mainargs.threads)) +signal.signal(signal.SIGINT, original_handler) +try: + res = funcmap_workers.map_async(map_functions, cmd_order) + res.get(mainargs.timeout) +except KeyboardInterrupt: + funcmap_workers.terminate() + funcmap_workers.join() + exit(1) +else: + funcmap_workers.close() + funcmap_workers.join() + + +# Generate externalFnMap.txt + +func_2_file = {} +extfunc_2_file = {} +func_2_fileset = {} + +defined_fns_filename = os.path.join(mainargs.xtuindir, 'definedFns.txt') +with open(defined_fns_filename, 'r') as defined_fns_file: + for line in defined_fns_file: + funcname, filename = line.strip().split(' ') + if funcname.startswith('!'): + funcname = funcname[1:] # main function + if funcname not in func_2_file.keys(): + func_2_fileset[funcname] = set([filename]) + else: + func_2_fileset[funcname].add(filename) + func_2_file[funcname] = filename + +extern_fns_filename = os.path.join(mainargs.xtuindir, 'externalFns.txt') +with open(extern_fns_filename, 'r') as extern_fns_file: + for line in extern_fns_file: + line = line.strip() + if line in func_2_file and line not in extfunc_2_file: + extfunc_2_file[line] = func_2_file[line] + +extern_fns_map_filename = os.path.join(mainargs.xtuindir, 'externalFnMap.txt') +with open(extern_fns_map_filename, 'w') as out_file: + for func, fname in extfunc_2_file.items(): + if len(func_2_fileset[func]) == 1: + out_file.write('%s %s.ast\n' % (func, fname))