[clangd] Publish diagnostics from stale preambles

[kadircet]

Summary

Diagnostics emitted while parsing main-file is easy, as they already
see new source locations. We just need to make sure the ones that can be
attributed to preamble section of the file is patched. That's achieved in two
ways:

• New include is actually patched with a #line-directive, so most clang features already point at the right line.
• Patching of main-file-includes inside the ParsedAST, which is used by replay preamble and include-cleaner analysis. This patch fixes/improve patching in this case.

The second problem is patching includes that were published when building the
preamble. Unfortunately because we don't have the full edit history in place,
it's hard to perform an exact translation. This patch tries to just preserve
diagnostics that are contained to lines containing #include or #define
directives and moves them around with the new file contents. It's granularity is
per-line, it doesn't handle column-wide changes.

Depends on D142890.

[kadircet]

Well, turns out presumed location patching was not handled in clangd at all for
diagnostics. So add that to translate ranges, at least when we have ranges or
locations that point to preamble patch.

[kadircet]

split into D143093, D143095 and D143096

clang-tools-extra/clangd/Diagnostics.cpp

if (!R.isValid() || M.getFileID(R.getBegin()) != M.getFileID(R.getEnd()) ||
     return false;
   return L != R.getEnd() && M.isPointWithin(L, R.getBegin(), R.getEnd());
 }
 
 // Clang diags have a location (shown as ^) and 0 or more ranges (~~~~).
 // LSP needs a single range.
 Range diagnosticRange(const clang::Diagnostic &D, const LangOptions &L) {
   auto &M = D.getSourceManager();
+  auto PatchedRange = [&M](CharSourceRange &R) {
+    // FIXME: Should we handle all presumed locations?
+    R.setBegin(translatePreamblePatchLocation(R.getBegin(), M));
+    R.setEnd(translatePreamblePatchLocation(R.getEnd(), M));
+    return R;
+  };
   auto Loc = M.getFileLoc(D.getLocation());
   for (const auto &CR : D.getRanges()) {
     auto R = Lexer::makeFileCharRange(CR, M, L);
     if (locationInRange(Loc, R, M))
-      return halfOpenToRange(M, R);
+      return halfOpenToRange(M, PatchedRange(R));
   }
   // The range may be given as a fixit hint instead.
   for (const auto &F : D.getFixItHints()) {
     auto R = Lexer::makeFileCharRange(F.RemoveRange, M, L);
     if (locationInRange(Loc, R, M))
-      return halfOpenToRange(M, R);
+      return halfOpenToRange(M, PatchedRange(R));
   }
   // If the token at the location is not a comment, we use the token.
   // If we can't get the token at the location, fall back to using the location
   auto R = CharSourceRange::getCharRange(Loc);
   Token Tok;
   if (!Lexer::getRawToken(Loc, Tok, M, L, true) && Tok.isNot(tok::comment)) {
     R = CharSourceRange::getTokenRange(Tok.getLocation(), Tok.getEndLoc());
   }
-  return halfOpenToRange(M, R);
+  return halfOpenToRange(M, PatchedRange(R));
 }
 
 // Try to find a location in the main-file to report the diagnostic D.
 // Returns a description like "in included file", or nullptr on failure.
 const char *getMainFileRange(const Diag &D, const SourceManager &SM,
                              SourceLocation DiagLoc, Range &R) {
   // Look for a note in the main file indicating template instantiation.
   for (const auto &N : D.Notes) {
bool tryMoveToMainFile(Diag &D, FullSourceLoc DiagLoc) {
   D.File = SM.getFileEntryRefForID(SM.getMainFileID())->getName().str();
   D.Range = std::move(R);
   D.InsideMainFile = true;
   // Update message to mention original file.
   D.Message = llvm::formatv("{0}: {1}", Prefix, D.Message);
   return true;
 }
 
-bool isInsideMainFile(const clang::Diagnostic &D) {
-  if (!D.hasSourceManager())
-    return false;
-
-  return clangd::isInsideMainFile(D.getLocation(), D.getSourceManager());
-}
-
 bool isNote(DiagnosticsEngine::Level L) {
   return L == DiagnosticsEngine::Note || L == DiagnosticsEngine::Remark;
 }
 
 llvm::StringRef diagLeveltoString(DiagnosticsEngine::Level Lvl) {
   switch (Lvl) {
   case DiagnosticsEngine::Ignored:
     return "ignored";
if (!LangOpts || !Info.hasSourceManager()) {
     return;
   }
 
   SourceManager &SM = Info.getSourceManager();
 
   auto FillDiagBase = [&](DiagBase &D) {
     fillNonLocationData(DiagLevel, Info, D);
 
-    D.InsideMainFile = isInsideMainFile(Info);
+    auto PLoc = translatePreamblePatchLocation(Info.getLocation(), SM);
+    D.InsideMainFile = isInsideMainFile(PLoc, SM);
     D.Range = diagnosticRange(Info, *LangOpts);
-    D.File = std::string(SM.getFilename(Info.getLocation()));
-    D.AbsFile = getCanonicalPath(
-        SM.getFileEntryForID(SM.getFileID(Info.getLocation())), SM);
+    auto FID = SM.getFileID(Info.getLocation());
+    if (auto *FE = SM.getFileEntryForID(FID)) {
+      D.File = FE->getName().str();
+      D.AbsFile = getCanonicalPath(FE, SM);
+    }
     D.ID = Info.getID();
     return D;
   };
 
   auto AddFix = [&](bool SyntheticMessage) -> bool {
     assert(!Info.getFixItHints().empty() &&
            "diagnostic does not have attached fix-its");
     // No point in generating fixes, if the diagnostic is for a different file.

clang-tools-extra/clangd/ParsedAST.cpp

ParsedAST::build(llvm::StringRef Filename, const ParseInputs &Inputs,
   ASTDiags.EndSourceFile();
 
   std::optional<std::vector<Diag>> Diags;
   // FIXME: Also skip generation of diagnostics alltogether to speed up ast
   // builds when we are patching a stale preamble.
   if (PreserveDiags) {
     Diags = CompilerInvocationDiags;
     // Add diagnostics from the preamble, if any.
-    if (Preamble)
-      Diags->insert(Diags->end(), Preamble->Diags.begin(),
-                    Preamble->Diags.end());
+    if (Preamble) {
+      auto PDiags = Patch->patchedDiags();
+      Diags->insert(Diags->end(), PDiags.begin(), PDiags.end());
+    }
     // Finally, add diagnostics coming from the AST.
     {
       std::vector<Diag> D = ASTDiags.take(&*CTContext);
       Diags->insert(Diags->end(), D.begin(), D.end());
     }
   }
   ParsedAST Result(Filename, Inputs.Version, std::move(Preamble),
                    std::move(Clang), std::move(Action), std::move(Tokens),

clang-tools-extra/clangd/Preamble.h

public:
 
   /// Returns preamble bounds for the Modified.
   PreambleBounds modifiedBounds() const { return ModifiedBounds; }
 
   /// Returns textual patch contents.
   llvm::StringRef text() const { return PatchContents; }
 
   /// Whether diagnostics generated using this patch are trustable.
-  bool preserveDiagnostics() const { return PatchContents.empty(); }
+  bool preserveDiagnostics() const;
+
+  /// Returns diag locations for Modified contents, only contains diags attached
+  /// to an #include or #define directive.
+  std::vector<Diag> patchedDiags() const;
+  static constexpr llvm::StringLiteral HeaderName = "__preamble_patch__.h";
 
 private:
   static PreamblePatch create(llvm::StringRef FileName,
                               const ParseInputs &Modified,
                               const PreambleData &Baseline,
                               PatchType PatchType);
 
   PreamblePatch() = default;
   std::string PatchContents;
   std::string PatchFileName;
-  /// Includes that are present in both \p Baseline and \p Modified. Used for
-  /// patching includes of baseline preamble.
+  // Includes that are present in both \p Baseline and \p Modified. Used for
+  // patching includes of baseline preamble.
   std::vector<Inclusion> PreambleIncludes;
+  // Diags that were attached to an #include or a #define directive.
+  std::vector<Diag> PatchedDiags;
   PreambleBounds ModifiedBounds = {0, false};
 };
 
-/// Translates locations inside preamble patch to their main-file equivalent
-/// using presumed locations. Returns \p Loc if it isn't inside preamble patch.
-SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
-                                              const SourceManager &SM);
-
 } // namespace clangd
 } // namespace clang
 
 #endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_PREAMBLE_H

clang-tools-extra/clangd/Preamble.cpp

 //===--- Preamble.cpp - Reusing expensive parts of the AST ----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Preamble.h"
 #include "Compiler.h"
 #include "Config.h"
+#include "Diagnostics.h"
 #include "Headers.h"
 #include "SourceCode.h"
 #include "clang-include-cleaner/Record.h"
 #include "support/Logger.h"
 #include "support/ThreadsafeFS.h"
 #include "support/Trace.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/Basic/Diagnostic.h"

 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/raw_ostream.h"
-#include <iterator>
+#include <map>
 #include <memory>
 #include <string>
 #include <system_error>
 #include <utility>
 #include <vector>
 
 namespace clang {
 namespace clangd {
 namespace {
-constexpr llvm::StringLiteral PreamblePatchHeaderName = "__preamble_patch__.h";
 
 bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
                              const tooling::CompileCommand &RHS) {
   // We don't check for Output, it should not matter to clangd.
   return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
          llvm::ArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
 }
 

 
 // Represents directives other than includes, where basic textual information is
 // enough.
 struct TextualPPDirective {
   unsigned DirectiveLine;
   // Full text that's representing the directive, including the `#`.
   std::string Text;
   unsigned Offset;
+  tok::PPKeywordKind Directive;
+  // Name of the macro being defined in the case of a #define directive.
+  std::string MacroName;
 
   bool operator==(const TextualPPDirective &RHS) const {
     return std::tie(DirectiveLine, Offset, Text) ==
            std::tie(RHS.DirectiveLine, RHS.Offset, RHS.Text);
   }
 };
 
 // Formats a PP directive consisting of Prefix (e.g. "#define ") and Body ("X
struct DirectiveCollector : public PPCallbacks {
   }
 
   void MacroDefined(const Token &MacroNameTok,
                     const MacroDirective *MD) override {
     if (!InMainFile)
       return;
     TextualDirectives.emplace_back();
     TextualPPDirective &TD = TextualDirectives.back();
+    TD.Directive = tok::pp_define;
+    TD.MacroName = MacroNameTok.getIdentifierInfo()->getName().str();
 
     const auto *MI = MD->getMacroInfo();
     TD.Text =
         spellDirective("#define ",
                        CharSourceRange::getTokenRange(
                            MI->getDefinitionLoc(), MI->getDefinitionEndLoc()),
                        LangOpts, SM, TD.DirectiveLine, TD.Offset);
   }
const char *spellingForIncDirective(tok::PPKeywordKind IncludeDirective) {
   case tok::pp_include_next:
     return "include_next";
   default:
     break;
   }
   llvm_unreachable("not an include directive");
 }
 
-// Checks whether \p FileName is a valid spelling of main file.
-bool isMainFile(llvm::StringRef FileName, const SourceManager &SM) {
-  auto FE = SM.getFileManager().getFile(FileName);
-  return FE && *FE == SM.getFileEntryForID(SM.getMainFileID());
-}
-
 // Accumulating wall time timer. Similar to llvm::Timer, but much cheaper,
 // it only tracks wall time.
 // Since this is a generic timer, We may want to move this to support/ if we
 // find a use case outside of FS time tracking.
 class WallTimer {
 public:
   WallTimer() : TotalTime(std::chrono::steady_clock::duration::zero()) {}
   // [Re-]Start the timer.
public:
   }
 
   double getTime() { return Timer.getTime(); }
 
 private:
   WallTimer Timer;
 };
 
+// Represents an include as spelled in the source code, without taking resolved
+// path or location into account.
+struct IncludeKey {
+  tok::PPKeywordKind Directive;
+  llvm::StringRef Written;
+
+  bool operator<(const IncludeKey &RHS) const {
+    return std::tie(Directive, Written) < std::tie(RHS.Directive, RHS.Written);
+  }
+  bool operator==(const IncludeKey &RHS) const {
+    return std::tie(Directive, Written) == std::tie(RHS.Directive, RHS.Written);
+  }
+};
 } // namespace
 
 std::shared_ptr<const PreambleData>
 buildPreamble(PathRef FileName, CompilerInvocation CI,
               const ParseInputs &Inputs, bool StoreInMemory,
               PreambleParsedCallback PreambleCallback,
               PreambleBuildStats *Stats) {
   // Note that we don't need to copy the input contents, preamble can live
case '"':
       break;
     default:
       break;
     }
     OS << C;
   }
 }
 
+// Checks if all pointers in \p D are for the same line of the main file.
+static bool diagReferencesMultipleLines(const Diag &D) {
+  int Line = D.Range.start.line;
+  if (D.Range.end.line != Line)
+    return true;
+  bool NotePointsToOutside = llvm::any_of(D.Notes, [&](const Note &N) {
+    return N.File == D.File &&
+           (N.Range.start.line != Line || N.Range.end.line != Line);
+  });
+  if (NotePointsToOutside)
+    return true;
+  bool FixPointsToOutside = llvm::any_of(D.Fixes, [&](const Fix &F) {
+    for (auto &E : F.Edits)
+      if (E.range.start.line != Line || E.range.end.line != Line)
+        return true;
+    return false;
+  });
+  if (FixPointsToOutside)
+    return true;
+  return false;
+}
+
+// Move all references in \p D to \p NewLine. This assumes none of the
+// references to the main file are multi-line.
+static Diag translateDiag(const Diag &D, int NewLine) {
+  assert(!diagReferencesMultipleLines(D));
+  Diag Translated = D;
+  Translated.Range.start.line = Translated.Range.end.line = NewLine;
+  for (auto &N : Translated.Notes) {
+    if (N.File != D.File)
+      continue;
+    N.Range.start.line = N.Range.end.line = NewLine;
+  }
+  for (auto &F : Translated.Fixes) {
+    for (auto &E : F.Edits)
+      E.range.start.line = E.range.end.line = NewLine;
+  }
+  return Translated;
+}
+
+static llvm::DenseMap<int, llvm::SmallVector<const Diag *>>
+mapDiagsToLines(llvm::ArrayRef<Diag> Diags) {
+  llvm::DenseMap<int, llvm::SmallVector<const Diag *>> LineToDiags;
+  for (auto &D : Diags) {
+    if (diagReferencesMultipleLines(D))
+      continue;
+    LineToDiags[D.Range.start.line].emplace_back(&D);
+  }
+  return LineToDiags;
+}
+
+static std::map<IncludeKey, const Inclusion *>
+getExistingIncludes(const ScannedPreamble &BaselineScan,
+                    llvm::ArrayRef<Inclusion> MFI) {
+  std::map<IncludeKey, const Inclusion *> ExistingIncludes;
+  // There might be includes coming from disabled regions, record these for
+  // exclusion. note that we don't have resolved paths for those.
+  for (const auto &Inc : BaselineScan.Includes)
+    ExistingIncludes[{Inc.Directive, Inc.Written}] = nullptr;
+  for (const auto &Inc : MFI)
+    ExistingIncludes[{Inc.Directive, Inc.Written}] = &Inc;
+  return ExistingIncludes;
+}
+
 PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
                                     const ParseInputs &Modified,
                                     const PreambleData &Baseline,
                                     PatchType PatchType) {
   trace::Span Tracer("CreatePreamblePatch");
   SPAN_ATTACH(Tracer, "File", FileName);
   assert(llvm::sys::path::is_absolute(FileName) && "relative FileName!");
   // First scan preprocessor directives in Baseline and Modified. These will be
PreamblePatch PreamblePatch::create(llvm::StringRef FileName,
   if ((PatchType == PatchType::MacroDirectives || !IncludesChanged) &&
       !DirectivesChanged)
     return PreamblePatch::unmodified(Baseline);
 
   PreamblePatch PP;
   // This shouldn't coincide with any real file name.
   llvm::SmallString<128> PatchName;
   llvm::sys::path::append(PatchName, llvm::sys::path::parent_path(FileName),
-                          PreamblePatchHeaderName);
+                          PreamblePatch::HeaderName);
   PP.PatchFileName = PatchName.str().str();
   PP.ModifiedBounds = ModifiedScan->Bounds;
 
   llvm::raw_string_ostream Patch(PP.PatchContents);
   // Set default filename for subsequent #line directives
   Patch << "#line 0 \"";
   // FileName part of a line directive is subject to backslash escaping, which
   // might lead to problems on windows especially.
   escapeBackslashAndQuotes(FileName, Patch);
   Patch << "\"\n";
 
+  // List of diagnostics associated to a particular line in the baseline.
+  auto LineToDiags = mapDiagsToLines(Baseline.Diags);
+  auto MoveDiagsBetweenLines = [&LineToDiags, &PP](int OldLine, int NewLine) {
+    auto DiagsForInclude = LineToDiags.find(OldLine);
+    if (DiagsForInclude == LineToDiags.end())
+      return;
+    for (auto *D : DiagsForInclude->second)
+      PP.PatchedDiags.emplace_back(translateDiag(*D, NewLine));
+  };
+  // Map from an include to its line in the Modified contents.
+  std::map<IncludeKey, int> IncludeToPatchedLine;
   if (IncludesChanged && PatchType == PatchType::All) {
     // We are only interested in newly added includes, record the ones in
     // Baseline for exclusion.
-    llvm::DenseMap<std::pair<tok::PPKeywordKind, llvm::StringRef>,
-                   /*Resolved=*/llvm::StringRef>
-        ExistingIncludes;
-    for (const auto &Inc : Baseline.Includes.MainFileIncludes)
-      ExistingIncludes[{Inc.Directive, Inc.Written}] = Inc.Resolved;
-    // There might be includes coming from disabled regions, record these for
-    // exclusion too. note that we don't have resolved paths for those.
-    for (const auto &Inc : BaselineScan->Includes)
-      ExistingIncludes.try_emplace({Inc.Directive, Inc.Written});
+    auto ExistingIncludes =
+        getExistingIncludes(*BaselineScan, Baseline.Includes.MainFileIncludes);
     // Calculate extra includes that needs to be inserted.
     for (auto &Inc : ModifiedScan->Includes) {
       auto It = ExistingIncludes.find({Inc.Directive, Inc.Written});
       // Include already present in the baseline preamble. Set resolved path and
       // put into preamble includes.
       if (It != ExistingIncludes.end()) {
-        Inc.Resolved = It->second.str();
+        if (It->second) {
+          // Copy everything from existing include, apart from the location.
+          Inc.Resolved = It->second->Resolved;
+          Inc.HeaderID = It->second->HeaderID;
+          Inc.BehindPragmaKeep = It->second->BehindPragmaKeep;
+          Inc.FileKind = It->second->FileKind;
+        }
         PP.PreambleIncludes.push_back(Inc);
+        IncludeToPatchedLine[It->first] = Inc.HashLine;
         continue;
       }
       // Include is new in the modified preamble. Inject it into the patch and
       // use #line to set the presumed location to where it is spelled.
       auto LineCol = offsetToClangLineColumn(Modified.Contents, Inc.HashOffset);
       Patch << llvm::formatv("#line {0}\n", LineCol.first);
       Patch << llvm::formatv(
           "#{0} {1}\n", spellingForIncDirective(Inc.Directive), Inc.Written);
     }
+  } else {
+    // Copy include information from baseline preamble if nothing has changed.
+    PP.PreambleIncludes = Baseline.Includes.MainFileIncludes;
   }
 
+  // Patch locations for diagnostics on baseline includes.
+  for (const auto &Inc : Baseline.Includes.MainFileIncludes) {
+    // Use existing locations if includes haven't changed.
+    size_t NewLine = Inc.HashLine;
+    if (IncludesChanged) {
+      auto PatchedLine =
+          IncludeToPatchedLine.find({Inc.Directive, Inc.Written});
+      // Ignore diags for deleted includes.
+      if (PatchedLine == IncludeToPatchedLine.end())
+        continue;
+      NewLine = PatchedLine->second;
+    }
+    MoveDiagsBetweenLines(Inc.HashLine, NewLine);
+  }
+
+  // Maps a directive from its spelling to its location in the Modified
+  // contents.
+  llvm::StringMap<int> DirectiveToPatchedLine;
   if (DirectivesChanged) {
     // We need to patch all the directives, since they are order dependent. e.g:
     // #define BAR(X) NEW(X) // Newly introduced in Modified
     // #define BAR(X) OLD(X) // Exists in the Baseline
     //
     // If we've patched only the first directive, the macro definition would've
     // been wrong for the rest of the file, since patch is applied after the
     // baseline preamble.
     //
     // Note that we deliberately ignore conditional directives and undefs to
     // reduce complexity. The former might cause problems because scanning is
     // imprecise and might pick directives from disabled regions.
     for (const auto &TD : ModifiedScan->TextualDirectives) {
+      // Introduce an #undef directive before #defines to suppress any
+      // re-definition warnings.
+      if (TD.Directive == tok::pp_define)
+        Patch << "#undef " << TD.MacroName << '\n';
       Patch << "#line " << TD.DirectiveLine << '\n';
       Patch << TD.Text << '\n';
-    }
+      DirectiveToPatchedLine[TD.Text] = TD.DirectiveLine;
+    }
+  } else {
+    // Take existing directives as-is, if they're the same.
+    for (const auto &TD : BaselineScan->TextualDirectives)
+      DirectiveToPatchedLine[TD.Text] = TD.DirectiveLine;
+
+  // Patch locations for diagnositcs on baseline macros.
+  for (const auto &TD : BaselineScan->TextualDirectives) {
+    // Textual directive lines are 1-based.
+    auto NewLine = DirectiveToPatchedLine.find(TD.Text);
+    if (NewLine == DirectiveToPatchedLine.end())
+      continue;
+    MoveDiagsBetweenLines(TD.DirectiveLine - 1, NewLine->second - 1);
   }
+
   dlog("Created preamble patch: {0}", Patch.str());
   Patch.flush();
   return PP;
 }
 
 PreamblePatch PreamblePatch::createFullPatch(llvm::StringRef FileName,
                                              const ParseInputs &Modified,
                                              const PreambleData &Baseline) {

 std::vector<Inclusion> PreamblePatch::preambleIncludes() const {
   return PreambleIncludes;
 }
 
 PreamblePatch PreamblePatch::unmodified(const PreambleData &Preamble) {
   PreamblePatch PP;
   PP.PreambleIncludes = Preamble.Includes.MainFileIncludes;
   PP.ModifiedBounds = Preamble.Preamble.getBounds();
+  PP.PatchedDiags = Preamble.Diags;
   return PP;
 }
 
-SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
-                                              const SourceManager &SM) {
-  auto DefFile = SM.getFileID(Loc);
-  if (auto FE = SM.getFileEntryRefForID(DefFile)) {
-    auto IncludeLoc = SM.getIncludeLoc(DefFile);
-    // Preamble patch is included inside the builtin file.
-    if (IncludeLoc.isValid() && SM.isWrittenInBuiltinFile(IncludeLoc) &&
-        FE->getName().endswith(PreamblePatchHeaderName)) {
-      auto Presumed = SM.getPresumedLoc(Loc);
-      // Check that line directive is pointing at main file.
-      if (Presumed.isValid() && Presumed.getFileID().isInvalid() &&
-          isMainFile(Presumed.getFilename(), SM)) {
-        Loc = SM.translateLineCol(SM.getMainFileID(), Presumed.getLine(),
-                                  Presumed.getColumn());
-      }
-    }
-  }
-  return Loc;
-}
-
+bool PreamblePatch::preserveDiagnostics() const {
+  return PatchContents.empty() ||
+         Config::current().Diagnostics.Mode != Config::DiagnosticsMode::Strict;
+std::vector<Diag> PreamblePatch::patchedDiags() const { return PatchedDiags; }
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/SourceCode.h

 
 /// Returns true if Name is reserved, like _Foo or __Vector_base.
 inline bool isReservedName(llvm::StringRef Name) {
   // This doesn't catch all cases, but the most common.
   return Name.size() >= 2 && Name[0] == '_' &&
          (isUppercase(Name[1]) || Name[1] == '_');
 }
 
+/// Translates locations inside preamble patch to their main-file equivalent
+/// using presumed locations. Returns \p Loc if it isn't inside preamble patch.
+SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
+                                              const SourceManager &SM);
 } // namespace clangd
 } // namespace clang
 #endif

clang-tools-extra/clangd/SourceCode.cpp

bool isProtoFile(SourceLocation Loc, const SourceManager &SM) {
   auto FID = SM.getFileID(Loc);
   // All proto generated headers should start with this line.
   static const char *ProtoHeaderComment =
       "// Generated by the protocol buffer compiler.  DO NOT EDIT!";
   // Double check that this is an actual protobuf header.
   return SM.getBufferData(FID).startswith(ProtoHeaderComment);
 }
 
+SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
+                                              const SourceManager &SM) {
+  // Checks whether \p FileName is a valid spelling of main file.
+  auto IsMainFile = [](llvm::StringRef FileName, const SourceManager &SM) {
+    auto FE = SM.getFileManager().getFile(FileName);
+    return FE && *FE == SM.getFileEntryForID(SM.getMainFileID());
+  };
+
+  auto DefFile = SM.getFileID(Loc);
+  if (auto FE = SM.getFileEntryRefForID(DefFile)) {
+    auto IncludeLoc = SM.getIncludeLoc(DefFile);
+    // Preamble patch is included inside the builtin file.
+    if (IncludeLoc.isValid() && SM.isWrittenInBuiltinFile(IncludeLoc) &&
+        FE->getName().endswith(PreamblePatch::HeaderName)) {
+      auto Presumed = SM.getPresumedLoc(Loc);
+      // Check that line directive is pointing at main file.
+      if (Presumed.isValid() && Presumed.getFileID().isInvalid() &&
+          IsMainFile(Presumed.getFilename(), SM)) {
+        Loc = SM.translateLineCol(SM.getMainFileID(), Presumed.getLine(),
+                                  Presumed.getColumn());
+      }
+    }
+  }
+  return Loc;
+}
 } // namespace clangd
 } // namespace clang

clang-tools-extra/clangd/unittests/PreambleTests.cpp

 //===--- PreambleTests.cpp --------------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
 
 #include "Annotations.h"
 #include "Compiler.h"
+#include "Config.h"
+#include "Diagnostics.h"
 #include "Headers.h"
 #include "Hover.h"
+#include "ParsedAST.h"
 #include "Preamble.h"
+#include "Protocol.h"
 #include "SourceCode.h"
 #include "TestFS.h"
 #include "TestTU.h"
 #include "XRefs.h"
 #include "clang/Format/Format.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/PrecompiledPreamble.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/VirtualFileSystem.h"
+#include "llvm/Testing/Support/SupportHelpers.h"
 #include "gmock/gmock.h"
 #include "gtest/gtest.h"
 #include <memory>
 #include <optional>
 #include <string>
 #include <vector>
 
 using testing::Contains;
+using testing::ElementsAre;
 using testing::Field;
+using testing::IsEmpty;
 using testing::Matcher;
 using testing::MatchesRegex;
+using testing::Not;
+using testing::UnorderedElementsAre;
+using testing::UnorderedElementsAreArray;
 
 namespace clang {
 namespace clangd {
 namespace {
 
 MATCHER_P2(Distance, File, D, "") {
   return arg.first() == File && arg.second == D;
 }
auto PP = PreamblePatch::createFullPatch(testPath(TU.Filename), TU.inputs(FS),
                                            *BaselinePreamble);
   // Only a.h should exists in the preamble, as c.h has been dropped and b.h was
   // newly introduced.
   EXPECT_THAT(PP.preambleIncludes(),
               ElementsAre(AllOf(Field(&Inclusion::Written, "\"a.h\""),
                                 Field(&Inclusion::Resolved, testPath("a.h")))));
 }
 
-std::optional<ParsedAST> createPatchedAST(llvm::StringRef Baseline,
-                                          llvm::StringRef Modified) {
-  auto BaselinePreamble = TestTU::withCode(Baseline).preamble();
+std::optional<ParsedAST>
+createPatchedAST(llvm::StringRef Baseline, llvm::StringRef Modified,
+                 llvm::StringMap<std::string> AdditionalFiles = {}) {
+  auto PreambleTU = TestTU::withCode(Baseline);
+  PreambleTU.AdditionalFiles = AdditionalFiles;
+  auto BaselinePreamble = PreambleTU.preamble();
   if (!BaselinePreamble) {
     ADD_FAILURE() << "Failed to build baseline preamble";
     return std::nullopt;
   }
 
   IgnoreDiagnostics Diags;
   MockFS FS;
   auto TU = TestTU::withCode(Modified);
+  TU.AdditionalFiles = std::move(AdditionalFiles);
   auto CI = buildCompilerInvocation(TU.inputs(FS), Diags);
   if (!CI) {
     ADD_FAILURE() << "Failed to build compiler invocation";
     return std::nullopt;
   }
   return ParsedAST::build(testPath(TU.Filename), TU.inputs(FS), std::move(CI),
                           {}, BaselinePreamble);
 }
struct {
     const char *const Contents;
     const char *const ExpectedPatch;
   } Cases[] = {
       {
           R"cpp(
         #define BAR
         [[BAR]])cpp",
           R"cpp(#line 0 ".*main.cpp"
+#undef BAR
 #line 2
 #define         BAR
 )cpp",
       },
       // multiline macro
       {
           R"cpp(
         #define BAR \
 
         [[BAR]])cpp",
           R"cpp(#line 0 ".*main.cpp"
+#undef BAR
 #line 2
 #define         BAR
 )cpp",
       },
       // multiline macro
       {
           R"cpp(
         #define \
                 BAR
         [[BAR]])cpp",
           R"cpp(#line 0 ".*main.cpp"
+#undef BAR
 #line 3
 #define         BAR
 )cpp",
       },
   };
 
   for (const auto &Case : Cases) {
     SCOPED_TRACE(Case.Contents);
TEST(PreamblePatchTest, OrderingPreserved) {
   llvm::StringLiteral Baseline = "#define BAR(X) X";
   Annotations Modified(R"cpp(
     #define BAR(X, Y) X Y
     #define BAR(X) X
     [[BAR]](int y);
   )cpp");
 
   llvm::StringLiteral ExpectedPatch(R"cpp(#line 0 ".*main.cpp"
+#undef BAR
 #line 2
 #define     BAR\(X, Y\) X Y
+#undef BAR
 #line 3
 #define     BAR\(X\) X
 )cpp");
   EXPECT_THAT(getPreamblePatch(Baseline, Modified.code()),
               MatchesRegex(ExpectedPatch.str()));
 
   auto AST = createPatchedAST(Baseline, Modified.code());
   ASSERT_TRUE(AST);
TEST(PreamblePatch, NoopWhenNotRequested) {
   ASSERT_TRUE(BaselinePreamble);
 
   TU.Code = Modified.str();
   MockFS FS;
   auto PP = PreamblePatch::createMacroPatch(testPath(TU.Filename),
                                             TU.inputs(FS), *BaselinePreamble);
   EXPECT_TRUE(PP.text().empty());
 }
+
+std::vector<clangd::Range> mainFileDiagRanges(const ParsedAST &AST) {
+  std::vector<clangd::Range> Result;
+  auto AddRangeIfInMainfile = [&Result](const DiagBase &DB) {
+    if (DB.InsideMainFile)
+      Result.emplace_back(DB.Range);
+  };
+  for (auto &D : *AST.getDiagnostics()) {
+    AddRangeIfInMainfile(D);
+    for (auto &N : D.Notes)
+      AddRangeIfInMainfile(N);
+  }
+  return Result;
+}
+
+TEST(PreamblePatch, DiagnosticsFromMainASTAreInRightPlace) {
+  Config Cfg;
+  Cfg.Diagnostics.Mode = Config::DiagnosticsMode::Fast;
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+
+  llvm::StringLiteral BaselinePreamble = "#define FOO\n";
+  {
+    // Check with removals from preamble.
+    Annotations Code("[[x]];/* error-ok */");
+    auto AST = createPatchedAST(BaselinePreamble, Code.code());
+    EXPECT_THAT(mainFileDiagRanges(*AST), ElementsAre(Code.range()));
+  }
+  {
+    // Check with additions to preamble.
+    Annotations Code(
+        (BaselinePreamble + "#define BAR\n[[x]];/* error-ok */").str());
+    auto AST = createPatchedAST(BaselinePreamble, Code.code());
+    EXPECT_THAT(mainFileDiagRanges(*AST), ElementsAre(Code.range()));
+  }
+}
+
+TEST(PreamblePatch, DiagnosticsToPreamble) {
+  Config Cfg;
+  Cfg.Diagnostics.Mode = Config::DiagnosticsMode::Fast;
+  Cfg.Diagnostics.UnusedIncludes = Config::UnusedIncludesPolicy::Strict;
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+
+  llvm::StringMap<std::string> AdditionalFiles;
+  AdditionalFiles["foo.h"] = "#pragma once";
+  AdditionalFiles["bar.h"] = "#pragma once";
+  llvm::StringLiteral BaselinePreamble("#define FOO 1\n[[#include \"foo.h\"]]");
+  {
+    // Check with removals from preamble.
+    Annotations NewCode("[[#  include \"foo.h\"]]");
+    auto AST = createPatchedAST(Annotations(BaselinePreamble).code(),
+                                NewCode.code(), AdditionalFiles);
+    EXPECT_THAT(mainFileDiagRanges(*AST), ElementsAre(NewCode.range()));
+  }
+  {
+    // Check with additions to preamble.
+    Annotations NewCode(("[[#include \"bar.h\"]]\n" + BaselinePreamble).str());
+    auto AST = createPatchedAST(Annotations(BaselinePreamble).code(),
+                                NewCode.code(), AdditionalFiles);
+    EXPECT_THAT(mainFileDiagRanges(*AST),
+                UnorderedElementsAreArray(NewCode.ranges()));
+  }
+  {
+    llvm::StringLiteral BaselinePreamble = "#define [[FOO]] 1\n";
+    // Check ranges for notes.
+    Annotations NewCode(("#define BARXYZ 1\n" + BaselinePreamble +
+                         "void foo();\n#define [[FOO]] 2")
+                            .str());
+    auto AST = createPatchedAST(Annotations(BaselinePreamble).code(),
+                                NewCode.code(), AdditionalFiles);
+    EXPECT_THAT(mainFileDiagRanges(*AST),
+                UnorderedElementsAreArray(NewCode.ranges()));
+  }
+}
+
+TEST(PreamblePatch, TranslatesIncludeDiagnosticsInPreamble) {
+  Config Cfg;
+  Cfg.Diagnostics.Mode = Config::DiagnosticsMode::Fast;
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+
+  llvm::StringLiteral BaselinePreamble("#include [[<foo>]]\n");
+  {
+    // Check with additions to preamble.
+    Annotations Code(BaselinePreamble);
+    Annotations NewCode(("#define BAR\n" + BaselinePreamble).str());
+    auto AST = createPatchedAST(Code.code(), NewCode.code());
+    EXPECT_THAT(mainFileDiagRanges(*AST), ElementsAre(NewCode.range()));
+  }
+  {
+    // Check with removals from preamble.
+    Annotations Code(("#define BAR\n" + BaselinePreamble).str());
+    Annotations NewCode(BaselinePreamble);
+    auto AST = createPatchedAST(Code.code(), NewCode.code());
+    EXPECT_THAT(mainFileDiagRanges(*AST), ElementsAre(NewCode.range()));
+  }
+  {
+    // Drop line with diags.
+    Annotations Code(BaselinePreamble);
+    Annotations NewCode("#define BAR\n#include [[<bar>]]\n");
+    auto AST = createPatchedAST(Code.code(), NewCode.code());
+    EXPECT_THAT(mainFileDiagRanges(*AST), ElementsAre(NewCode.range()));
+  }
+}
+
+TEST(PreamblePatch, TranslatesNonIncludeDiagnosticsInPreamble) {
+  Config Cfg;
+  Cfg.Diagnostics.Mode = Config::DiagnosticsMode::Fast;
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+
+  llvm::StringLiteral BaselinePreamble("#define [[__LINE__]]\n");
+  ASSERT_THAT(TestTU::withCode(Annotations(BaselinePreamble).code())
+                  .build()
+                  .getDiagnostics(),
+              Not(llvm::ValueIs(IsEmpty())));
+  {
+    // Check with additions to preamble.
+    Annotations Code(BaselinePreamble);
+    Annotations NewCode(("#define BAR\n" + BaselinePreamble).str());
+    auto AST = createPatchedAST(Code.code(), NewCode.code());
+    EXPECT_THAT(mainFileDiagRanges(*AST), ElementsAre(NewCode.range()));
+  }
+  {
+    // Check with removals from preamble.
+    Annotations Code(("#define BAR\n" + BaselinePreamble).str());
+    Annotations NewCode(BaselinePreamble);
+    auto AST = createPatchedAST(Code.code(), NewCode.code());
+    EXPECT_THAT(mainFileDiagRanges(*AST), ElementsAre(NewCode.range()));
+  }
+  {
+    // Drop line with diags.
+    Annotations Code(BaselinePreamble);
+    Annotations NewCode("#define BAR\n#include [[<bar>]]\n#include <baz>\n");
+    auto AST = createPatchedAST(Code.code(), NewCode.code());
+    EXPECT_THAT(mainFileDiagRanges(*AST), ElementsAre(NewCode.range()));
+  }
+}
 } // namespace
 } // namespace clangd
 } // namespace clang