[sancov] skip dead files from computations

[aizatsky]

Summary

Should improve memory consumption

[aizatsky]

format

[vitalybuka]

test/tools/sancov/symbolize.test

7–8

more convenient would be to have just one file for this tests and use
prefixies, e.g. :

RUN: sancov -symbolize -skip-dead-files=1 -strip_path_prefix="llvm/" %p/Inputs/test-linux_x86_64 %p/Inputs/test-linux_x86_64.0.sancov | FileCheck %s
RUN: sancov -symbolize -skip-dead-files=0 -strip_path_prefix="llvm/" %p/Inputs/test-linux_x86_64 %p/Inputs/test-linux_x86_64.0.sancov | FileCheck %s --check-prefixes=CHECK,CHECK-KEEPDEAD

and replace removed lines with:
CHECK-KEEPDEAD-NEXT: ...

[vitalybuka]

LGTM either way

[aizatsky]

I actually prefer to have separate lit tests. When you have multiple tests in a single file and it fails it is sometimes very hard to determine the failure cause.

test/tools/sancov/not_covered_functions.test

 REQUIRES: x86_64-linux
-RUN: sancov -not-covered-functions %p/Inputs/test-linux_x86_64 %p/Inputs/test-linux_x86_64.0.sancov | FileCheck %s
+RUN: sancov -skip-dead-files=0 -not-covered-functions %p/Inputs/test-linux_x86_64 %p/Inputs/test-linux_x86_64.0.sancov | FileCheck %s
 RUN: sancov -not-covered-functions %p/Inputs/test-linux_x86_64 %p/Inputs/test-linux_x86_64.1.sancov | FileCheck --check-prefix=CHECK1 --allow-empty %s
 
 CHECK: Inputs{{[/\\]}}foo.cpp:5 foo()
 CHECK-NOT: {{.*__sanitizer.*}}
 CHECK1-NOT: {{.+}}
 

test/tools/sancov/stats.test

 REQUIRES: x86_64-linux
 RUN: sancov -print-coverage-stats %p/Inputs/test-linux_x86_64 %p/Inputs/test-linux_x86_64.0.sancov | FileCheck %s
 
-CHECK: all-edges: 9
+CHECK: all-edges: 8
 CHECK: cov-edges: 5
-CHECK: all-functions: 3
+CHECK: all-functions: 2
 CHECK: cov-functions: 2
 
 

test/tools/sancov/symbolize.test

 REQUIRES: x86_64-linux
 RUN: sancov -symbolize -strip_path_prefix="llvm/" %p/Inputs/test-linux_x86_64 %p/Inputs/test-linux_x86_64.0.sancov | FileCheck %s
 
 CHECK: {
 CHECK-NEXT:  "covered-points" : ["4e132b", "4e1472", "4e1520", "4e1553", "4e1586"],
 CHECK-NEXT:  "binary-hash" : "BB3CDD5045AED83906F6ADCC1C4DAF7E2596A6B5",
 CHECK-NEXT:  "point-symbol-info" : {
-CHECK-NEXT:    "test/tools/sancov/Inputs/foo.cpp" : {
-CHECK-NEXT:      "foo()" : {
-CHECK-NEXT:        "4e178c" : "5:0"
-CHECK-NEXT:     }
-CHECK-NEXT:   },
 CHECK-NEXT:   "test/tools/sancov/Inputs/test.cpp" : {
 CHECK-NEXT:     "bar(std::string)" : {
 CHECK-NEXT:       "4e132b" : "12:0"
 CHECK-NEXT:      },
 CHECK-NEXT:      "main" : {
 CHECK-NEXT:        "4e1472" : "14:0",
 CHECK-NEXT:        "4e14c2" : "16:9",
 CHECK-NEXT:        "4e1520" : "17:5",
 CHECK-NEXT:        "4e1553" : "17:5",
 CHECK-NEXT:        "4e1586" : "17:5",
 CHECK-NEXT:        "4e1635" : "19:1",
 CHECK-NEXT:        "4e1690" : "17:5"
 CHECK-NEXT:      }
 CHECK-NEXT:    }
 CHECK-NEXT:  }
 CHECK-NEXT:}
 

test/tools/sancov/symbolize_noskip_dead_files.test

 REQUIRES: x86_64-linux
-RUN: sancov -symbolize -strip_path_prefix="llvm/" %p/Inputs/test-linux_x86_64 %p/Inputs/test-linux_x86_64.0.sancov | FileCheck %s
+RUN: sancov -symbolize -skip-dead-files=0 -strip_path_prefix="llvm/" %p/Inputs/test-linux_x86_64 %p/Inputs/test-linux_x86_64.0.sancov | FileCheck %s
 
 CHECK: {
 CHECK-NEXT:  "covered-points" : ["4e132b", "4e1472", "4e1520", "4e1553", "4e1586"],
 CHECK-NEXT:  "binary-hash" : "BB3CDD5045AED83906F6ADCC1C4DAF7E2596A6B5",
 CHECK-NEXT:  "point-symbol-info" : {
 CHECK-NEXT:    "test/tools/sancov/Inputs/foo.cpp" : {
 CHECK-NEXT:      "foo()" : {
 CHECK-NEXT:        "4e178c" : "5:0"

tools/sancov/sancov.cc

 
 static cl::list<std::string>
     ClInputFiles(cl::Positional, cl::OneOrMore,
                  cl::desc("(<binary file>|<.sancov file>)..."));
 
 static cl::opt<bool> ClDemangle("demangle", cl::init(true),
                                 cl::desc("Print demangled function name."));
 
+static cl::opt<bool> ClSkipDeadFiles("skip-dead-files", cl::init(true),
+                                cl::desc("Do not list dead source files in reports."));
+
 static cl::opt<std::string> ClStripPathPrefix(
     "strip_path_prefix", cl::init(""),
     cl::desc("Strip this prefix from file paths in reports."));
 
 static cl::opt<std::string>
     ClBlacklist("blacklist", cl::init(""),
                 cl::desc("Blacklist file (sanitizer blacklist format)."));
 
static std::unique_ptr<SpecialCaseList> createUserBlacklist() {
     return SpecialCaseList::createOrDie({{ClBlacklist}});
   }
   std::unique_ptr<SpecialCaseList> DefaultBlacklist;
   std::unique_ptr<SpecialCaseList> UserBlacklist;
 };
 
 static std::vector<CoveragePoint>
 getCoveragePoints(const std::string &ObjectFile,
-                  const std::set<uint64_t> &Addrs, bool InlinedCode) {
+                  const std::set<uint64_t> &Addrs,
+                  const std::set<uint64_t> &CoveredAddrs) {
   std::vector<CoveragePoint> Result;
   auto Symbolizer(createSymbolizer());
   Blacklists B;
 
+  std::set<std::string> CoveredFiles;
+  if (ClSkipDeadFiles) {
+    for (auto Addr : CoveredAddrs) {
+      auto LineInfo = Symbolizer->symbolizeCode(ObjectFile, Addr);
+      failIfError(LineInfo);
+      CoveredFiles.insert(LineInfo->FileName);
+      auto InliningInfo = Symbolizer->symbolizeInlinedCode(ObjectFile, Addr);
+      failIfError(InliningInfo);
+      for (uint32_t I = 0; I < InliningInfo->getNumberOfFrames(); ++I) {
+        auto FrameInfo = InliningInfo->getFrame(I);
+        CoveredFiles.insert(FrameInfo.FileName);
+      }
+    }
+  }
+
   for (auto Addr : Addrs) {
     std::set<DILineInfo> Infos; // deduplicate debug info.
 
     auto LineInfo = Symbolizer->symbolizeCode(ObjectFile, Addr);
     failIfError(LineInfo);
+    if (ClSkipDeadFiles && CoveredFiles.find(LineInfo->FileName) == CoveredFiles.end())
+      continue;
     LineInfo->FileName = normalizeFilename(LineInfo->FileName);
     if (B.isBlacklisted(*LineInfo))
       continue;
 
     auto Id = utohexstr(Addr, true);
     auto Point = CoveragePoint(Id);
     Infos.insert(*LineInfo);
     Point.Locs.push_back(*LineInfo);
 
-    if (InlinedCode) {
-      auto InliningInfo = Symbolizer->symbolizeInlinedCode(ObjectFile, Addr);
-      failIfError(InliningInfo);
-      for (uint32_t I = 0; I < InliningInfo->getNumberOfFrames(); ++I) {
-        auto FrameInfo = InliningInfo->getFrame(I);
-        FrameInfo.FileName = normalizeFilename(FrameInfo.FileName);
-        if (B.isBlacklisted(FrameInfo))
-          continue;
-        if (Infos.find(FrameInfo) == Infos.end()) {
-          Infos.insert(FrameInfo);
-          Point.Locs.push_back(FrameInfo);
-        }
-      }
-    }
+      if (ClSkipDeadFiles && CoveredFiles.find(FrameInfo.FileName) == CoveredFiles.end())
+        continue;
 
     Result.push_back(Point);
   }
 
   return Result;
 }
 
 static bool isCoveragePointSymbol(StringRef Name) {
static ErrorOr<bool> isCoverageFile(const std::string &FileName) {
   return Header->Magic == BinCoverageMagic;
 }
 
 static bool isSymbolizedCoverageFile(const std::string &FileName) {
   auto ShortFileName = llvm::sys::path::filename(FileName);
   return SymcovFileRegex.match(ShortFileName);
 }
 
+static void filterDeadFiles(std::set<uint64_t>* CoveragePoints, 
+    const std::set<uint64_t>& CoveredPoints) {
+}
+
 static std::unique_ptr<SymbolizedCoverage>
 symbolize(const RawCoverage &Data, const std::string ObjectFile) {
   auto Coverage = make_unique<SymbolizedCoverage>();
 
   ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
       MemoryBuffer::getFile(ObjectFile);
   failIfError(BufOrErr);
   SHA1 Hasher;
   Hasher.update((*BufOrErr)->getBuffer());
   Coverage->BinaryHash = toHex(Hasher.final());
 
   for (uint64_t Addr : *Data.Addrs) {
     Coverage->CoveredIds.insert(utohexstr(Addr, true));
   }
 
   std::set<uint64_t> AllAddrs = findCoveragePointAddrs(ObjectFile);
   if (!std::includes(AllAddrs.begin(), AllAddrs.end(), Data.Addrs->begin(),
                      Data.Addrs->end())) {
     fail("Coverage points in binary and .sancov file do not match.");
   }
-  Coverage->Points = getCoveragePoints(ObjectFile, AllAddrs, true);
+  if (ClSkipDeadFiles) {
+    filterDeadFiles(&AllAddrs, *Data.Addrs);
+  }
+  Coverage->Points = getCoveragePoints(ObjectFile, AllAddrs, *Data.Addrs);
   return Coverage;
 }
 
 struct FileFn {
   bool operator<(const FileFn &RHS) const {
     return std::tie(FileName, FunctionName) <
            std::tie(RHS.FileName, RHS.FunctionName);
   }
for (const auto &FileName : CovFiles) {
       }
 
       CoverageByObjFile[Iter->second].push_back(FileName);
     };
 
     // Read raw coverage and symbolize it.
     for (const auto &Pair : CoverageByObjFile) {
       if (findSanitizerCovFunctions(Pair.first).empty()) {
-        for (const auto &FileName : Pair.second) {
-          CovFiles.erase(FileName);
-        }
-
         errs()
             << "Ignoring " << Pair.first
             << " and its coverage because  __sanitizer_cov* functions were not "
                "found.\n";
         continue;
       }
 
       for (const std::string &CoverageFile : Pair.second) {