Index: include/clang/Basic/DiagnosticFrontendKinds.td =================================================================== --- include/clang/Basic/DiagnosticFrontendKinds.td +++ include/clang/Basic/DiagnosticFrontendKinds.td @@ -225,4 +225,8 @@ def warn_option_invalid_ocl_version : Warning< "OpenCL version %0 does not support the option '%1'">, InGroup; + +def err_fnmap_parsing : Error< + "error parsing CrossTU index file: '%0' line: %1 'USR filename' format " + "expected">; } Index: include/clang/CrossTU/CrossTranslationUnit.h =================================================================== --- /dev/null +++ include/clang/CrossTU/CrossTranslationUnit.h @@ -0,0 +1,81 @@ +//===--- CrossTranslationUnit.h - -------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the CrossTranslationUnit interface. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CLANG_TOOLING_CROSSTRANSLATIONUNIT_H +#define LLVM_CLANG_TOOLING_CROSSTRANSLATIONUNIT_H + +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" + +namespace clang { +class CompilerInstance; +class ASTContext; +class ASTImporter; +class ASTUnit; +class DeclContext; +class FunctionDecl; +class NamedDecl; +class TranslationUnitDecl; + +namespace crossTU { + +/// \brief This class can be used for tools that requires cross translation +/// unit capability. +/// +/// This class can load function definitions from external AST files. +/// The loaded definition will be merged back to the original AST using the +/// AST Importer. +/// In order to use this class, an index file is required that describes +/// the locations of the AST files for each function definition. +/// +/// Note that this class also implements caching. +class CrossTranslationUnit { +public: + CrossTranslationUnit(CompilerInstance &CI); + ~CrossTranslationUnit(); + + /// \brief This function can load a function definition from an external AST + /// file and merge it into the original AST. + /// + /// This method should only be used on functions that have no definitions in + /// the current translation unit. A function definition with the same + /// declaration will be looked up in the index file which should be in the + /// \p CrossTUDir directory, called \p IndexName. In case the declaration is + /// found in the index the corresponding AST file will be loaded and the + /// definition of the function will be merged into the original AST using + /// the AST Importer. The declaration with the definition will be returned. + /// + /// Note that the AST files should also be in the \p CrossTUDir. + const FunctionDecl *getCrossTUDefinition(const FunctionDecl *FD, + StringRef CrossTUDir, + StringRef IndexName); + +private: + ASTImporter &getOrCreateASTImporter(ASTContext &From); + std::string getLookupName(const NamedDecl *ND); + const FunctionDecl *findFunctionInDeclContext(const DeclContext *DC, + StringRef LookupFnName); + + llvm::StringMap> FileASTUnitMap; + llvm::StringMap FunctionAstUnitMap; + llvm::StringMap FunctionFileMap; + llvm::DenseMap> + ASTUnitImporterMap; + CompilerInstance &CI; + ASTContext &Context; +}; + +} // namespace tooling +} // namespace clang + +#endif // LLVM_CLANG_TOOLING_CROSSTRANSLATIONUNIT_H Index: lib/AST/ASTImporter.cpp =================================================================== --- lib/AST/ASTImporter.cpp +++ lib/AST/ASTImporter.cpp @@ -1841,6 +1841,8 @@ if (ToD) return ToD; + const FunctionDecl *FoundWithoutBody = nullptr; + // Try to find a function in our own ("to") context with the same name, same // type, and in the same context as the function we're importing. if (!LexicalDC->isFunctionOrMethod()) { @@ -1858,6 +1860,13 @@ if (Importer.IsStructurallyEquivalent(D->getType(), FoundFunction->getType())) { // FIXME: Actually try to merge the body and other attributes. + const FunctionDecl *FromBodyDecl = nullptr; + D->hasBody(FromBodyDecl); + if (D == FromBodyDecl && !FoundFunction->hasBody()) { + // This function is needed to merge completely. + FoundWithoutBody = FoundFunction; + break; + } return Importer.Imported(D, FoundFunction); } @@ -2008,6 +2017,12 @@ } ToFunction->setParams(Parameters); + if (FoundWithoutBody) { + auto *Recent = const_cast( + FoundWithoutBody->getMostRecentDecl()); + ToFunction->setPreviousDecl(Recent); + } + if (usedDifferentExceptionSpec) { // Update FunctionProtoType::ExtProtoInfo. QualType T = Importer.Import(D->getType()); Index: lib/CrossTU/CMakeLists.txt =================================================================== --- /dev/null +++ lib/CrossTU/CMakeLists.txt @@ -0,0 +1,13 @@ +set(LLVM_LINK_COMPONENTS + Support + ) + +add_clang_library(clangCrossTU + CrossTranslationUnit.cpp + + LINK_LIBS + clangAST + clangBasic + clangFrontend + clangIndex + ) Index: lib/CrossTU/CrossTranslationUnit.cpp =================================================================== --- /dev/null +++ lib/CrossTU/CrossTranslationUnit.cpp @@ -0,0 +1,163 @@ +//===--- CrossTranslationUnit.cpp - -----------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file provides an interface to load binary AST dumps on demand. This +// feature can be utilized for tools that require cross translation unit +// support. +// +//===----------------------------------------------------------------------===// +#include "clang/CrossTU/CrossTranslationUnit.h" +#include "clang/AST/ASTImporter.h" +#include "clang/AST/Decl.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/Frontend/ASTUnit.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Frontend/TextDiagnosticPrinter.h" +#include "clang/Index/USRGeneration.h" +#include "llvm/ADT/Triple.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" +#include + +namespace clang { +namespace crossTU { + +CrossTranslationUnit::CrossTranslationUnit(CompilerInstance &CI) + : CI(CI), Context(CI.getASTContext()) {} + +CrossTranslationUnit::~CrossTranslationUnit() {} + +std::string CrossTranslationUnit::getLookupName(const NamedDecl *ND) { + SmallString<128> DeclUSR; + bool Ret = index::generateUSRForDecl(ND, DeclUSR); + assert(!Ret); + return DeclUSR.str(); +} + +/// Recursively visit the funtion decls of a DeclContext, and looks up a +/// function based on mangled name. +const FunctionDecl * +CrossTranslationUnit::findFunctionInDeclContext(const DeclContext *DC, + StringRef LookupFnName) { + if (!DC) + return nullptr; + for (const Decl *D : DC->decls()) { + const auto *SubDC = dyn_cast(D); + if (const auto *FD = findFunctionInDeclContext(SubDC, LookupFnName)) + return FD; + + const auto *ND = dyn_cast(D); + const FunctionDecl *ResultDecl; + if (!ND || !ND->hasBody(ResultDecl)) + continue; + if (getLookupName(ResultDecl) != LookupFnName) + continue; + return ResultDecl; + } + return nullptr; +} + +const FunctionDecl *CrossTranslationUnit::getCrossTUDefinition( + const FunctionDecl *FD, StringRef CrossTUDir, StringRef IndexName) { + assert(!FD->hasBody() && "FD has a definition in current translation unit!"); + + std::string LookupFnName = getLookupName(FD); + if (LookupFnName.empty()) + return nullptr; + ASTUnit *Unit = nullptr; + auto FnUnitCacheEntry = FunctionAstUnitMap.find(LookupFnName); + if (FnUnitCacheEntry == FunctionAstUnitMap.end()) { + if (FunctionFileMap.empty()) { + SmallString<256> ExternalFunctionMap = CrossTUDir; + llvm::sys::path::append(ExternalFunctionMap, IndexName); + std::ifstream ExternalFnMapFile(ExternalFunctionMap.c_str()); + if (!ExternalFnMapFile) { + Context.getDiagnostics().Report(diag::err_fe_error_opening) + << ExternalFunctionMap << "required by the CrossTU functionality"; + return nullptr; + } + + StringRef FunctionName, FileName; + std::string Line; + unsigned LineNo = 0; + while (std::getline(ExternalFnMapFile, Line)) { + size_t Pos = Line.find(" "); + StringRef LineRef{Line}; + if (Pos > 0 && Pos != std::string::npos) { + FunctionName = LineRef.substr(0, Pos); + FileName = LineRef.substr(Pos + 1); + SmallString<256> FilePath = CrossTUDir; + llvm::sys::path::append(FilePath, FileName); + FunctionFileMap[FunctionName] = FilePath.str().str(); + } else { + Context.getDiagnostics().Report(diag::err_fnmap_parsing) + << ExternalFunctionMap << (LineNo + 1); + } + LineNo++; + } + } + + StringRef ASTFileName; + auto It = FunctionFileMap.find(LookupFnName); + if (It == FunctionFileMap.end()) + return nullptr; // No definition found even in some other build unit. + ASTFileName = It->second; + auto ASTCacheEntry = FileASTUnitMap.find(ASTFileName); + if (ASTCacheEntry == FileASTUnitMap.end()) { + IntrusiveRefCntPtr DiagOpts = new DiagnosticOptions(); + TextDiagnosticPrinter *DiagClient = + new TextDiagnosticPrinter(llvm::errs(), &*DiagOpts); + IntrusiveRefCntPtr DiagID(new DiagnosticIDs()); + IntrusiveRefCntPtr Diags( + new DiagnosticsEngine(DiagID, &*DiagOpts, DiagClient)); + + std::unique_ptr LoadedUnit(ASTUnit::LoadFromASTFile( + ASTFileName, CI.getPCHContainerOperations()->getRawReader(), + ASTUnit::LoadEverything, Diags, CI.getFileSystemOpts())); + Unit = LoadedUnit.get(); + FileASTUnitMap[ASTFileName] = std::move(LoadedUnit); + } else { + Unit = ASTCacheEntry->second.get(); + } + FunctionAstUnitMap[LookupFnName] = Unit; + } else { + Unit = FnUnitCacheEntry->second; + } + + if (!Unit) + return nullptr; + assert(&Unit->getFileManager() == + &Unit->getASTContext().getSourceManager().getFileManager()); + ASTImporter &Importer = getOrCreateASTImporter(Unit->getASTContext()); + TranslationUnitDecl *TU = Unit->getASTContext().getTranslationUnitDecl(); + if (const FunctionDecl *ResultDecl = + findFunctionInDeclContext(TU, LookupFnName)) { + auto *ToDecl = cast( + Importer.Import(const_cast(ResultDecl))); + assert(ToDecl->hasBody()); + assert(FD->hasBody() && "Functions already imported should have body."); + return ToDecl; + } + return nullptr; +} + +ASTImporter &CrossTranslationUnit::getOrCreateASTImporter(ASTContext &From) { + auto I = ASTUnitImporterMap.find(From.getTranslationUnitDecl()); + if (I != ASTUnitImporterMap.end()) + return *I->second; + ASTImporter *NewImporter = + new ASTImporter(Context, Context.getSourceManager().getFileManager(), + From, From.getSourceManager().getFileManager(), false); + ASTUnitImporterMap[From.getTranslationUnitDecl()].reset(NewImporter); + return *NewImporter; +} + +} // namespace tooling +} // namespace clang Index: lib/Tooling/CMakeLists.txt =================================================================== --- lib/Tooling/CMakeLists.txt +++ lib/Tooling/CMakeLists.txt @@ -27,6 +27,7 @@ clangDriver clangFormat clangFrontend + clangIndex clangLex clangRewrite clangToolingCore Index: test/Analysis/func-mapping-test.cpp =================================================================== --- /dev/null +++ test/Analysis/func-mapping-test.cpp @@ -0,0 +1,7 @@ +// RUN: %clang_func_map %s -- | FileCheck %s + +int f(int) { + return 0; +} + +// CHECK: c:@F@f#I# Index: test/CMakeLists.txt =================================================================== --- test/CMakeLists.txt +++ test/CMakeLists.txt @@ -47,6 +47,7 @@ clang-tblgen clang-offload-bundler clang-import-test + clang-func-mapping ) if(CLANG_ENABLE_STATIC_ANALYZER) Index: test/lit.cfg =================================================================== --- test/lit.cfg +++ test/lit.cfg @@ -271,6 +271,7 @@ ' --driver-mode=cl ')) config.substitutions.append( ('%clangxx', ' ' + config.clang + ' --driver-mode=g++ ')) +config.substitutions.append( ('%clang_func_map', ' ' + lit.util.which('clang-func-mapping', config.environment['PATH']) + ' ') ) config.substitutions.append( ('%clang', ' ' + config.clang + ' ') ) config.substitutions.append( ('%test_debuginfo', ' ' + config.llvm_src_root + '/utils/test_debuginfo.pl ') ) config.substitutions.append( ('%itanium_abi_triple', makeItaniumABITriple(config.target_triple)) ) Index: tools/CMakeLists.txt =================================================================== --- tools/CMakeLists.txt +++ tools/CMakeLists.txt @@ -19,6 +19,7 @@ add_clang_subdirectory(clang-check) add_clang_subdirectory(scan-build) add_clang_subdirectory(scan-view) + add_clang_subdirectory(clang-func-mapping) endif() # We support checking out the clang-tools-extra repository into the 'extra' Index: tools/clang-func-mapping/CMakeLists.txt =================================================================== --- /dev/null +++ tools/clang-func-mapping/CMakeLists.txt @@ -0,0 +1,21 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + asmparser + support + mc + ) + +add_clang_executable(clang-func-mapping + ClangFnMapGen.cpp + ) + +target_link_libraries(clang-func-mapping + clangAST + clangBasic + clangFrontend + clangIndex + clangTooling + ) + +install(TARGETS clang-func-mapping + RUNTIME DESTINATION bin) Index: tools/clang-func-mapping/ClangFnMapGen.cpp =================================================================== --- /dev/null +++ tools/clang-func-mapping/ClangFnMapGen.cpp @@ -0,0 +1,127 @@ +//===- ClangFnMapGen.cpp -----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===--------------------------------------------------------------------===// +// +// Clang tool which creates a list of defined functions and the files in which +// they are defined. +// +//===--------------------------------------------------------------------===// + +#include "clang/AST/ASTConsumer.h" +#include "clang/AST/ASTContext.h" +#include "clang/AST/GlobalDecl.h" +#include "clang/AST/Mangle.h" +#include "clang/AST/StmtVisitor.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Basic/TargetInfo.h" +#include "clang/Frontend/CompilerInstance.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Index/USRGeneration.h" +#include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Signals.h" +#include +#include +#include + +using namespace llvm; +using namespace clang; +using namespace clang::tooling; + +static cl::OptionCategory ClangFnMapGenCategory("clang-fnmapgen options"); + +class MapFunctionNamesConsumer : public ASTConsumer { +public: + MapFunctionNamesConsumer(ASTContext &Context) : Ctx(Context) {} + + ~MapFunctionNamesConsumer() { + // Flush results to standard output. + llvm::outs() << DefinedFuncsStr.str(); + } + + virtual void HandleTranslationUnit(ASTContext &Ctx) { + handleDecl(Ctx.getTranslationUnitDecl()); + } + +private: + std::string getLookupName(const FunctionDecl *FD); + void handleDecl(const Decl *D); + + ASTContext &Ctx; + std::stringstream DefinedFuncsStr; + std::string CurrentFileName; +}; + +void MapFunctionNamesConsumer::handleDecl(const Decl *D) { + if (!D) + return; + + if (const auto *FD = dyn_cast(D)) { + if (FD->isThisDeclarationADefinition()) { + if (const Stmt *Body = FD->getBody()) { + SmallString<128> LookupName; + bool Res = index::generateUSRForDecl(D, LookupName); + assert(!Res); + const SourceManager &SM = Ctx.getSourceManager(); + if (CurrentFileName.empty()) { + StringRef SMgrName = + SM.getFileEntryForID(SM.getMainFileID())->getName(); + char *Path = realpath(SMgrName.str().c_str(), nullptr); + CurrentFileName = Path; + free(Path); + } + + switch (FD->getLinkageInternal()) { + case ExternalLinkage: + case VisibleNoLinkage: + case UniqueExternalLinkage: + if (SM.isInMainFile(Body->getLocStart())) + DefinedFuncsStr << LookupName.str().str() << " " << CurrentFileName + << "\n"; + default: + break; + } + } + } + } + + if (const auto *DC = dyn_cast(D)) + for (const Decl *D : DC->decls()) + handleDecl(D); +} + +class MapFunctionNamesAction : public ASTFrontendAction { +protected: + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + llvm::StringRef) { + std::unique_ptr PFC( + new MapFunctionNamesConsumer(CI.getASTContext())); + return PFC; + } +}; + +static cl::extrahelp CommonHelp(CommonOptionsParser::HelpMessage); + +int main(int argc, const char **argv) { + // Print a stack trace if we signal out. + sys::PrintStackTraceOnErrorSignal(argv[0], false); + PrettyStackTraceProgram X(argc, argv); + + const char *Overview = "\nThis tool collects the USR name and location " + "of all functions definitions in the source files " + "(excluding headers).\n"; + CommonOptionsParser OptionsParser(argc, argv, ClangFnMapGenCategory, + cl::ZeroOrMore, Overview); + + ClangTool Tool(OptionsParser.getCompilations(), + OptionsParser.getSourcePathList()); + Tool.run(newFrontendActionFactory().get()); + return 0; +} Index: unittests/CrossTU/CMakeLists.txt =================================================================== --- /dev/null +++ unittests/CrossTU/CMakeLists.txt @@ -0,0 +1,16 @@ +set(LLVM_LINK_COMPONENTS + ${LLVM_TARGETS_TO_BUILD} + Support + ) + +add_clang_unittest(CrossTUTests + CrossTranslationUnitTest.cpp + ) + +target_link_libraries(ToolingTests + clangAST + clangBasic + clangCrossTU + clangFrontend + clangTooling + ) Index: unittests/CrossTU/CrossTranslationUnitTest.cpp =================================================================== --- /dev/null +++ unittests/CrossTU/CrossTranslationUnitTest.cpp @@ -0,0 +1,98 @@ +//===- unittest/Tooling/CrossTranslationUnitTest.cpp - Tooling unit tests -===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "clang/AST/ASTConsumer.h" +#include "clang/CrossTU/CrossTranslationUnit.h" +#include "clang/Frontend/FrontendAction.h" +#include "clang/Tooling/Tooling.h" +#include "llvm/Config/llvm-config.h" +#include "llvm/Support/Path.h" +#include "gtest/gtest.h" +#include + +namespace clang { +namespace crossTU { + +namespace { +StringRef IndexFileName = "index.txt"; +StringRef ASTFileName = "f.ast"; +StringRef DefinitionFileName = "input.cc"; + +class CTUASTConsumer : public clang::ASTConsumer { +public: + explicit CTUASTConsumer(clang::CompilerInstance &CI, bool *Success) + : CTU(CI), Success(Success) {} + + void HandleTranslationUnit(ASTContext &Ctx) { + const TranslationUnitDecl *TU = Ctx.getTranslationUnitDecl(); + const FunctionDecl *FD = nullptr; + for (const Decl *D : TU->decls()) { + FD = dyn_cast(D); + if (FD && FD->getName() == "f") + break; + } + assert(FD); + bool OrigFDHasBody = FD->hasBody(); + + // Prepare the index file and the AST file. + std::error_code EC; + llvm::raw_fd_ostream OS(IndexFileName, EC, llvm::sys::fs::F_Text); + OS << "c:@F@f#I# " << ASTFileName << "\n"; + OS.flush(); + StringRef SourceText = "int f(int) { return 0; }\n"; + // This file must exist since the saved ASTFile will reference it. + llvm::raw_fd_ostream OS2(DefinitionFileName, EC, llvm::sys::fs::F_Text); + OS2 << SourceText; + OS2.flush(); + std::unique_ptr ASTWithDefinition = + tooling::buildASTFromCode(SourceText); + ASTWithDefinition->Save(ASTFileName); + + // Load the definition from the AST file. + const FunctionDecl *NewFD = + CTU.getCrossTUDefinition(FD, ".", IndexFileName); + + *Success = NewFD && NewFD->hasBody() && !OrigFDHasBody; + } + +private: + CrossTranslationUnit CTU; + bool *Success; +}; + +class CTUAction : public clang::ASTFrontendAction { +public: + CTUAction(bool *Success) : Success(Success) {} + +protected: + std::unique_ptr + CreateASTConsumer(clang::CompilerInstance &CI, StringRef) override { + return llvm::make_unique(CI, Success); + } + +private: + bool *Success; +}; + +} // end namespace + +TEST(CrossTranslationUnit, CanLoadFunctionDefinition) { + bool Success = false; + EXPECT_TRUE(tooling::runToolOnCode(new CTUAction(&Success), "int f(int);")); + EXPECT_TRUE(Success); + EXPECT_TRUE(llvm::sys::fs::exists(IndexFileName)); + EXPECT_FALSE((bool)llvm::sys::fs::remove(IndexFileName)); + EXPECT_TRUE(llvm::sys::fs::exists(ASTFileName)); + EXPECT_FALSE((bool)llvm::sys::fs::remove(ASTFileName)); + EXPECT_TRUE(llvm::sys::fs::exists(DefinitionFileName)); + EXPECT_FALSE((bool)llvm::sys::fs::remove(DefinitionFileName)); +} + +} // end namespace tooling +} // end namespace clang