diff --git a/clang-tools-extra/clangd/XRefs.cpp b/clang-tools-extra/clangd/XRefs.cpp
--- a/clang-tools-extra/clangd/XRefs.cpp
+++ b/clang-tools-extra/clangd/XRefs.cpp
@@ -563,23 +563,49 @@
     assert(SM.getFileID(Loc) == File && "spelled token in wrong file?");
     unsigned Line = SM.getSpellingLineNumber(Loc);
     if (Line > WordLine)
-      return 1 + llvm::Log2_64(Line - WordLine);
+      return 1 + Line - WordLine;
     if (Line < WordLine)
-      return 2 + llvm::Log2_64(WordLine - Line);
+      return 2 + WordLine - Line;
     return 0;
   };
   const syntax::Token *BestTok = nullptr;
   // Search bounds are based on word length: 2^N lines forward.
-  unsigned BestCost = Word.Text.size() + 1;
+  unsigned BestCost = 1 << std::min(Word.Text.size(), sizeof(unsigned) * 8 - 1);
+  auto Code = SM.getBuffer(File)->getBuffer();
+  unsigned FileLines = std::count(Code.begin(), Code.end(), '\n');
+  auto TruncUnsigned = [](unsigned U) {
+    return std::min<unsigned>(U, std::numeric_limits<int>::max());
+  };
+  // Min position:
+  // - Line: max(0, WordLine - 2^N)
+  // - Column: 0
+  unsigned LineMin =
+      TruncUnsigned(WordLine < BestCost ? 0 : WordLine - BestCost);
+  auto OffsetMin = positionToOffset(Code, {static_cast<int>(LineMin), 0},
+                                    /*AllowColumnBeyondLineLength=*/true);
+  assert(OffsetMin);
+  auto LocMin = SM.getLocForStartOfFile(File).getLocWithOffset(*OffsetMin);
+  // Max position:
+  // - Line: min(<number of lines in the file>, WordLine + 2^N + 1)
+  // - Column: 0
+  unsigned LineMax = TruncUnsigned(
+      std::min(WordLine > std::numeric_limits<unsigned>::max() - BestCost - 1
+                   ? std::numeric_limits<unsigned>::max()
+                   : WordLine + BestCost + 1,
+               FileLines));
+  auto OffsetMax = positionToOffset(Code, {static_cast<int>(LineMax), 0},
+                                    /*AllowColumnBeyondLineLength=*/true);
+  assert(OffsetMax);
+  auto LocMax = SM.getLocForStartOfFile(File).getLocWithOffset(*OffsetMax);
 
   // Updates BestTok and BestCost if Tok is a good candidate.
   // May return true if the cost is too high for this token.
   auto Consider = [&](const syntax::Token &Tok) {
+    if (Tok.location() < LocMin || Tok.location() > LocMax)
+      return true; // we are too far from the word, break the outer loop.
     if (!(Tok.kind() == tok::identifier && Tok.text(SM) == Word.Text))
       return false;
-    // No point guessing the same location we started with.
-    if (Tok.location() == Word.Location)
-      return false;
+
     // We've done cheap checks, compute cost so we can break the caller's loop.
     unsigned TokCost = Cost(Tok.location());
     if (TokCost >= BestCost)