Fix issues with using clangd with C++ modules

[kuganv]

Summary

Fixes following incompatibility issues with c++ modules
(clangd with c++ modules results in assertion failure and pch_langopt_mismatch #1105)

1:
Preamble generated by clangd is generated with WriteCommentListToPCH = false;
However, if we are using precompiled modules that have comments in the
serialised AST, following sanity check in the clangd fails.

Sanity check that the comment does not come from the PCH. We choose to not
write them into PCH, because they are racy and slow to load.
assert(!Ctx.getSourceManager().isLoadedSourceLocation(RC→getBeginLoc()));

In this patch when we are generating Preamble with
WriteCommentListToPCH, we do not load the comments from AST.

2:
If we have modules that are built with RetainCommentsFromSystemHeaders
difference, that would prevent modules from getting loaded. Since this
difference is not critical that should be stopping modules from being loaded,
this patch changes it to be a COMPATIBLE_LANGOPT.

clang-tools-extra/clangd/test/modules-options-compatablity-test/A.h

+// Comment for void a()
+void a() {}

clang-tools-extra/clangd/test/modules-options-compatablity-test/B.h

+// Comment for B
+// ORDINARY COMMENT
+#include "A.h"

clang-tools-extra/clangd/test/modules-options-compatablity-test/compile_commands.json

+[{
+  "directory" : ".",
+  "command" : "clang -emit-obj DIRECTORY/use.c -fno-modules-global -fmodules "
+              "-fmodules-cache-path=/DOES/NOT/EXIST  -fno-builtin-module-map "
+              "-fno-implicit-modules  -fimplicit-module-maps "
+              "-fprebuilt-module-path=DIRECTORY/prebuilt",
+  "file" : "DIRECTORY/use.c"
+}]

clang-tools-extra/clangd/test/modules-options-compatablity-test/definition.jsonrpc

+{
+  "jsonrpc": "2.0",
+    "id": 0,
+    "method": "initialize",
+    "params": {
+      "processId": 123,
+      "rootPath": "clangd",
+      "capabilities": { "window": { "workDoneProgress": true, "implicitWorkDoneProgressCreate": true} },
+      "trace": "off"
+    }
+}
+---
+{
+  "jsonrpc": "2.0",
+    "method": "textDocument/didOpen",
+    "params": {
+      "textDocument": {
+        "uri": "file://DIRECTORY/use.c",
+        "languageId": "cpp",
+        "version": 1,
+        "text": "#include \"B.h\"\nvoid use(){\na();\n}"
+      }
+    }
+}
+---
+{"jsonrpc":"2.0","id":3,"method":"shutdown"}
+---
+{"jsonrpc":"2.0","method":"exit"}

clang-tools-extra/clangd/test/modules-options-compatablity-test/module.modulemap

+/* module.modulemap */
+module A {
+  header "A.h"
+}
+module B {
+  header "B.h"
+  export *
+}

clang-tools-extra/clangd/test/modules-options-compatablity-test/use.c

+/* use */
+#include <A.h>
+#include <B.h>
+
+void use() { a(); }

clang-tools-extra/clangd/test/modules-options-compatablity.test

+
+# RUN: rm -rf %t && mkdir -p %t
+# RUN: cp -r %S/modules-options-compatablity-test/* %t
+# RUN: mkdir -p %t/prebuilt
+# RUN: sed -i -e "s|DIRECTORY|%t|g" %t/definition.jsonrpc
+# RUN: sed -i -e "s|DIRECTORY|%t|g" %t/compile_commands.json
+# RUN: %clang -cc1 -emit-module -o %t/prebuilt/A.pcm -fmodules %t/module.modulemap -fmodule-name=A
+# RUN: %clang -cc1 -fretain-comments-from-system-headers -emit-module -o %t/prebuilt/B.pcm -fmodules %t/module.modulemap -fmodule-name=B -fprebuilt-module-path=%t/prebuilt
+# RUN: rm %t/*.h
+# RUN: clangd -background-index -lit-test < %t/definition.jsonrpc

clang/include/clang/Basic/LangOptions.def

 
 LANGOPT(XLPragmaPack, 1, 0, "IBM XL #pragma pack handling")
 
-LANGOPT(RetainCommentsFromSystemHeaders, 1, 0, "retain documentation comments from system headers in the AST")
+COMPATIBLE_LANGOPT(RetainCommentsFromSystemHeaders, 1, 0, "retain documentation comments from system headers in the AST")
 
 LANGOPT(SanitizeAddressFieldPadding, 2, 0, "controls how aggressive is ASan "
                                            "field padding (0: none, 1:least "

clang/lib/Serialization/ASTReader.cpp

           return Err;
         if (llvm::Error Err = ReadBlockAbbrevs(C, COMMENTS_BLOCK_ID))
           return Err;
+        if (PP.getPreprocessorOpts().GeneratePreamble &&
+            !PP.getPreprocessorOpts().WriteCommentListToPCH) {
+          break;
+        }
         CommentsCursors.push_back(std::make_pair(C, &F));
         break;
       }