[clang][lex] Change skipped range to use offsets

[sunho]

Summary

As part with https://discourse.llvm.org/t/rfc-flexible-lexer-buffering-for-handling-incomplete-input-in-interactive-c-c/64180

NOTE: This patch is WIP.

[sunho]

This comment was removed by sunho.

[sunho]

This comment was removed by sunho.

[sunho]

This comment was removed by sunho.

[sunho]

Update

clang/include/clang/Lex/Preprocessor.h

private:
   ///
   /// See comments at the use-site for more context about why it is needed.
   bool SkippingExcludedConditionalBlock = false;
 
   /// Keeps track of skipped range mappings that were recorded while skipping
   /// excluded conditional directives. It maps the source buffer pointer at
   /// the beginning of a skipped block, to the number of bytes that should be
   /// skipped.
-  llvm::DenseMap<const char *, unsigned> RecordedSkippedRanges;
+  llvm::DenseMap<FileID, llvm::DenseMap<unsigned , unsigned>> RecordedSkippedRanges;
 
   void updateOutOfDateIdentifier(IdentifierInfo &II) const;
 
 public:
   Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
                DiagnosticsEngine &diags, LangOptions &opts, SourceManager &SM,
                HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
                IdentifierInfoLookup *IILookup = nullptr,

clang/lib/Lex/PPDirectives.cpp

void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
   Token Tok;
   SourceLocation endLoc;
 
   /// Keeps track and caches skipped ranges and also retrieves a prior skipped
   /// range if the same block is re-visited.
   struct SkippingRangeStateTy {
     Preprocessor &PP;
 
-    const char *BeginPtr = nullptr;
+    std::optional<unsigned> BeginOffset;
     unsigned *SkipRangePtr = nullptr;
 
     SkippingRangeStateTy(Preprocessor &PP) : PP(PP) {}
 
     void beginLexPass() {
-      if (BeginPtr)
+      if (BeginOffset)
         return; // continue skipping a block.
 
       // Initiate a skipping block and adjust the lexer if we already skipped it
       // before.
-      BeginPtr = PP.CurLexer->getBufferLocation();
-      SkipRangePtr = &PP.RecordedSkippedRanges[BeginPtr];
+      BeginOffset = PP.CurLexer->getCurrentBufferOffset();
+      SkipRangePtr = &PP.RecordedSkippedRanges[PP.CurLexer->getFileID()][*BeginOffset];
       if (*SkipRangePtr) {
         PP.CurLexer->seek(PP.CurLexer->getCurrentBufferOffset() + *SkipRangePtr,
                           /*IsAtStartOfLine*/ true);
       }
     }
 
-    void endLexPass(const char *Hashptr) {
-      if (!BeginPtr) {
+    void endLexPass(unsigned HashOffset) {
+      if (!BeginOffset) {
         // Not doing normal lexing.
         assert(PP.CurLexer->isDependencyDirectivesLexer());
         return;
       }
-
       // Finished skipping a block, record the range if it's first time visited.
       if (!*SkipRangePtr) {
-        *SkipRangePtr = Hashptr - BeginPtr;
+        *SkipRangePtr = HashOffset - *BeginOffset;
       }
-      assert(*SkipRangePtr == Hashptr - BeginPtr);
-      BeginPtr = nullptr;
+      assert(*SkipRangePtr == HashOffset - *BeginOffset);
+      BeginOffset = std::nullopt;
       SkipRangePtr = nullptr;
     }
   } SkippingRangeState(*this);
 
   while (true) {
     if (CurLexer->isDependencyDirectivesLexer()) {
       CurLexer->LexDependencyDirectiveTokenWhileSkipping(Tok);
     } else {
while (true) {
 
     // We just parsed a # character at the start of a line, so we're in
     // directive mode.  Tell the lexer this so any newlines we see will be
     // converted into an EOD token (this terminates the macro).
     CurPPLexer->ParsingPreprocessorDirective = true;
     if (CurLexer) CurLexer->SetKeepWhitespaceMode(false);
 
     assert(Tok.is(tok::hash));
-    const char *Hashptr = CurLexer->getBufferLocation() - Tok.getLength();
-    assert(CurLexer->getSourceLocation(Hashptr - CurLexer->BufferStart) == Tok.getLocation());
+    unsigned HashOffset = CurLexer->getCurrentBufferOffset() - Tok.getLength();
+    assert(CurLexer->getSourceLocation(HashOffset) == Tok.getLocation());
 
     // Read the next token, the directive flavor.
     LexUnexpandedToken(Tok);
 
     // If this isn't an identifier directive (e.g. is "# 1\n" or "#\n", or
     // something bogus), skip it.
     if (Tok.isNot(tok::raw_identifier)) {
       CurPPLexer->ParsingPreprocessorDirective = false;
if (Directive.startswith("if")) {
         PPConditionalInfo CondInfo;
         CondInfo.WasSkipping = true; // Silence bogus warning.
         bool InCond = CurPPLexer->popConditionalLevel(CondInfo);
         (void)InCond;  // Silence warning in no-asserts mode.
         assert(!InCond && "Can't be skipping if not in a conditional!");
 
         // If we popped the outermost skipping block, we're done skipping!
         if (!CondInfo.WasSkipping) {
-          SkippingRangeState.endLexPass(Hashptr);
+          SkippingRangeState.endLexPass(HashOffset);
           // Restore the value of LexingRawMode so that trailing comments
           // are handled correctly, if we've reached the outermost block.
           CurPPLexer->LexingRawMode = false;
           endLoc = CheckEndOfDirective("endif");
           CurPPLexer->LexingRawMode = true;
           if (Callbacks)
             Callbacks->Endif(Tok.getLocation(), CondInfo.IfLoc);
           break;
         } else {
           DiscardUntilEndOfDirective();
         }
       } else if (Sub == "lse") { // "else".
         // #else directive in a skipping conditional.  If not in some other
         // skipping conditional, and if #else hasn't already been seen, enter it
         // as a non-skipping conditional.
         PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
 
         if (!CondInfo.WasSkipping)
-          SkippingRangeState.endLexPass(Hashptr);
+          SkippingRangeState.endLexPass(HashOffset);
 
         // If this is a #else with a #else before it, report the error.
         if (CondInfo.FoundElse)
           Diag(Tok, diag::pp_err_else_after_else);
 
         // Note that we've seen a #else in this conditional.
         CondInfo.FoundElse = true;
 
if (Directive.startswith("if")) {
           break;
         } else {
           DiscardUntilEndOfDirective();  // C99 6.10p4.
         }
       } else if (Sub == "lif") {  // "elif".
         PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
 
         if (!CondInfo.WasSkipping)
-          SkippingRangeState.endLexPass(Hashptr);
+          SkippingRangeState.endLexPass(HashOffset);
 
         // If this is a #elif with a #else before it, report the error.
         if (CondInfo.FoundElse)
           Diag(Tok, diag::pp_err_elif_after_else) << PED_Elif;
 
         // If this is in a skipping block or if we're already handled this #if
         // block, don't bother parsing the condition.
         if (CondInfo.WasSkipping || CondInfo.FoundNonSkip) {
if (Directive.startswith("if")) {
         }
       } else if (Sub == "lifdef" ||  // "elifdef"
                  Sub == "lifndef") { // "elifndef"
         bool IsElifDef = Sub == "lifdef";
         PPConditionalInfo &CondInfo = CurPPLexer->peekConditionalLevel();
         Token DirectiveToken = Tok;
 
         if (!CondInfo.WasSkipping)
-          SkippingRangeState.endLexPass(Hashptr);
+          SkippingRangeState.endLexPass(HashOffset);
 
         // Warn if using `#elifdef` & `#elifndef` in not C2x & C++2b mode even
         // if this branch is in a skipping block.
         unsigned DiagID;
         if (LangOpts.CPlusPlus)
           DiagID = LangOpts.CPlusPlus2b ? diag::warn_cxx2b_compat_pp_directive
                                         : diag::ext_cxx2b_pp_directive;
         else