[libclang] Added clang_getRealSpellingLocation to compensate for clang_getSpellingLocation returning the expansion location

[cameron314]

Summary

All the libclang functions for expanding a location, namely clang_getExpansionLocation, clang_getPresumedLocation, clang_getInstantiationLocation, and most surprisingly clang_getSpellingLocation return expansion locations, not spelling locations. As it turns out, there is no way to get a spelling location via libclang. This patch adds such a function.

Note that there's a FIXME in clang_getSpellingLocation about this, but changing clang_getSpellingLocation to actually return a spelling location would almost certainly break a large body of existing (third-party) code, mostly in subtle ways. I think it's better to introduce a new function, though I know it's ugly. But, this is what we've been using for the past year, and we've gotten quite comfortable with it.

[rsmith]

Ugh. Yes. I suppose it's too late to make this actually do the right thing. @akyrtzi, what do you think?

Please add a test for this (see unittests/libclang/LibclangTest.cpp).

[cameron314]

Ah, I was wondering where the tests were. I found this in the CMake file:

# FIXME: libclang unit tests are disabled on Windows due
# to failures, mostly in libclang.VirtualFileOverlay_*.
if(NOT WIN32 AND CLANG_TOOL_LIBCLANG_BUILD) 
  add_subdirectory(libclang)
endif()

I'll ignore the VFO failures for now, and add a new test for clang_getRealSpellingLocation :-)

[cameron314]

I've added a unit test.

include/clang-c/Index.h

CINDEX_LINKAGE void clang_getSpellingLocation(CXSourceLocation location,
                                               unsigned *line,
                                               unsigned *column,
                                               unsigned *offset);
 
 /**
  * \brief Retrieve the file, line, column, and offset represented by
  * the given source location.
  *
+ * If the location refers into a macro instantiation, return where the
+ * location was originally spelled in the source file (i.e. where the token
+ * text is located). clang_getSpellingLocation() is supposed to do this
+ * but doesn't, hence this function.
+ *
+ * \param location the location within a source file that will be decomposed
+ * into its parts.
+ *
+ * \param file [out] if non-NULL, will be set to the file to which the given
+ * source location points.
+ *
+ * \param line [out] if non-NULL, will be set to the line to which the given
+ * source location points.
+ *
+ * \param column [out] if non-NULL, will be set to the column to which the given
+ * source location points.
+ *
+ * \param offset [out] if non-NULL, will be set to the offset into the
+ * buffer to which the given source location points.
+ */
+CINDEX_LINKAGE void clang_getRealSpellingLocation(CXSourceLocation location,
+                                                  CXFile *file,
+                                                  unsigned *line,
+                                                  unsigned *column,
+                                                  unsigned *offset);
+
+/**
+ * \brief Retrieve the file, line, column, and offset represented by
+ * the given source location.
+ *
  * If the location refers into a macro expansion, return where the macro was
  * expanded or where the macro argument was written, if the location points at
  * a macro argument.
  *
  * \param location the location within a source file that will be decomposed
  * into its parts.
  *
  * \param file [out] if non-NULL, will be set to the file to which the given

tools/libclang/CXSourceLocation.cpp

void clang_getSpellingLocation(CXSourceLocation location,
   if (line)
     *line = SM.getLineNumber(FID, FileOffset);
   if (column)
     *column = SM.getColumnNumber(FID, FileOffset);
   if (offset)
     *offset = FileOffset;
 }
 
+void clang_getRealSpellingLocation(CXSourceLocation location,
+                                   CXFile *file,
+                                   unsigned *line,
+                                   unsigned *column,
+                                   unsigned *offset) {
+
+  if (!isASTUnitSourceLocation(location)) {
+    CXLoadedDiagnostic::decodeLocation(location, file, line, column, offset);
+    return;
+  }
+
+  SourceLocation Loc = SourceLocation::getFromRawEncoding(location.int_data);
+
+  if (!location.ptr_data[0] || Loc.isInvalid())
+    return createNullLocation(file, line, column, offset);
+
+  const SourceManager &SM =
+  *static_cast<const SourceManager*>(location.ptr_data[0]);
+  SourceLocation SpellLoc = SM.getSpellingLoc(Loc);
+  std::pair<FileID, unsigned> LocInfo = SM.getDecomposedLoc(SpellLoc);
+  FileID FID = LocInfo.first;
+  unsigned FileOffset = LocInfo.second;
+
+  if (FID.isInvalid())
+    return createNullLocation(file, line, column, offset);
+
+  if (file)
+    *file = const_cast<FileEntry *>(SM.getFileEntryForID(FID));
+  if (line)
+    *line = SM.getLineNumber(FID, FileOffset);
+  if (column)
+    *column = SM.getColumnNumber(FID, FileOffset);
+  if (offset)
+    *offset = FileOffset;
+}
+
 void clang_getFileLocation(CXSourceLocation location,
                            CXFile *file,
                            unsigned *line,
                            unsigned *column,
                            unsigned *offset) {
   if (!isASTUnitSourceLocation(location)) {
     CXLoadedDiagnostic::decodeLocation(location, file, line,
                                            column, offset);

unittests/libclang/LibclangTest.cpp

 #include "clang-c/Index.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include "gtest/gtest.h"
 #include <fstream>
 #include <set>
+#include <map>
+#include <memory>
+#include <functional>
 #define DEBUG_TYPE "libclang-test"
 
 TEST(libclang, clang_parseTranslationUnit2_InvalidArgs) {
   EXPECT_EQ(CXError_InvalidArguments,
             clang_parseTranslationUnit2(nullptr, nullptr, nullptr, 0, nullptr,
                                         0, 0, nullptr));
 }
 
TEST(libclang, ModuleMapDescriptor) {
   unsigned BufSize;
   clang_ModuleMapDescriptor_writeToBuffer(MMD, 0, &BufPtr, &BufSize);
   std::string BufStr(BufPtr, BufSize);
   EXPECT_STREQ(Contents, BufStr.c_str());
   clang_free(BufPtr);
   clang_ModuleMapDescriptor_dispose(MMD);
 }
 
-class LibclangReparseTest : public ::testing::Test {
+class LibclangParseTest : public ::testing::Test {
   std::set<std::string> Files;
+  typedef std::unique_ptr<std::string> fixed_addr_string;
+  std::map<fixed_addr_string, fixed_addr_string> UnsavedFileContents;
 public:
   std::string TestDir;
   CXIndex Index;
   CXTranslationUnit ClangTU;
   unsigned TUFlags;
+  std::vector<CXUnsavedFile> UnsavedFiles;
 
   void SetUp() override {
     llvm::SmallString<256> Dir;
     ASSERT_FALSE(llvm::sys::fs::createUniqueDirectory("libclang-test", Dir));
     TestDir = Dir.str();
     TUFlags = CXTranslationUnit_DetailedPreprocessingRecord |
-              clang_defaultEditingTranslationUnitOptions();
     Index = clang_createIndex(0, 0);
+    ClangTU = nullptr;
   }
   void TearDown() override {
     clang_disposeTranslationUnit(ClangTU);
     clang_disposeIndex(Index);
     for (const std::string &Path : Files)
       llvm::sys::fs::remove(Path);
     llvm::sys::fs::remove(TestDir);
   }
   void WriteFile(std::string &Filename, const std::string &Contents) {
     if (!llvm::sys::path::is_absolute(Filename)) {
       llvm::SmallString<256> Path(TestDir);
       llvm::sys::path::append(Path, Filename);
       Filename = Path.str();
       Files.insert(Filename);
     }
     llvm::sys::fs::create_directories(llvm::sys::path::parent_path(Filename));
     std::ofstream OS(Filename);
     OS << Contents;
     assert(OS.good());
   }
+  void MapUnsavedFile(const char* name, const std::string &contents) {
+    auto it = UnsavedFileContents.emplace(
+        fixed_addr_string(new std::string(name)),
+        fixed_addr_string(new std::string(contents)));
+    UnsavedFiles.push_back({
+        it.first->first->c_str(),   // filename
+        it.first->second->c_str(),  // contents
+        it.first->second->size()    // length
+    });
+  }
+  template<typename F>
+  void Traverse(const F &TraversalFunctor) {
+    CXCursor TuCursor = clang_getTranslationUnitCursor(ClangTU);
+    std::reference_wrapper<const F> FunctorRef = std::cref(TraversalFunctor);
+    clang_visitChildren(TuCursor,
+        &TraverseStateless<std::reference_wrapper<const F>>,
+        &FunctorRef);
+  }
+
+private:
+  template<typename TState>
+  static CXChildVisitResult TraverseStateless(CXCursor cx, CXCursor parent,
+      CXClientData data) {
+    TState *State = static_cast<TState*>(data);
+    return State->get()(cx, parent);
+  }
+};
+
+TEST_F(LibclangParseTest, SpellingLocation) {
+  MapUnsavedFile("main.cpp",
+    "#define BANANAS 4011\n"
+    "int plu = BANANAS;");
+
+  ClangTU = clang_parseTranslationUnit(Index, "main.cpp", nullptr, 0,
+    UnsavedFiles.data(), UnsavedFiles.size(), TUFlags);
+
+  bool sawInt;
+  Traverse([&](CXCursor cx, CXCursor) {
+    if (cx.kind == CXCursor_IntegerLiteral) {
+      sawInt = true;
+      auto cxl = clang_getCursorLocation(cx);
+      CXFile expansionFile, spellingFile;
+      unsigned line, column, offset;
+      clang_getSpellingLocation(cxl, &expansionFile, &line, nullptr, nullptr);
+      EXPECT_EQ(2, line);  // getSpellingLocation returns expansion location
+      clang_getRealSpellingLocation(cxl, &spellingFile, &line, &column, &offset);
+      EXPECT_TRUE(clang_File_isEqual(expansionFile, spellingFile));
+      EXPECT_EQ(1, line);
+      EXPECT_EQ(17, column);
+      EXPECT_EQ(16, offset);
+    }
+    return CXChildVisit_Recurse;
+  });
+  EXPECT_TRUE(sawInt);
+}
+
+class LibclangReparseTest : public LibclangParseTest {
+public:
   void DisplayDiagnostics() {
     unsigned NumDiagnostics = clang_getNumDiagnostics(ClangTU);
     for (unsigned i = 0; i < NumDiagnostics; ++i) {
       auto Diag = clang_getDiagnostic(ClangTU, i);
       DEBUG(llvm::dbgs() << clang_getCString(clang_formatDiagnostic(
           Diag, clang_defaultDiagnosticDisplayOptions())) << "\n");
       clang_disposeDiagnostic(Diag);
     }