Index: test/Index/c-index-tokenizer.c
===================================================================
--- /dev/null
+++ test/Index/c-index-tokenizer.c
@@ -0,0 +1,16 @@
+/*
+ * Checks that clang_tokenize doesn't return one-past-the-end token.
+ */
+
+// RUN: c-index-test -tokenize-at=%s:15:9 %s | FileCheck -check-prefix=CHECK-1 %s
+// CHECK-1: Identifier: "FOO"
+// CHECK-1: Literal: "2"
+// CHECK-1: <end>
+
+// RUN: c-index-test -tokenize-at=%s:16:9 %s | FileCheck -check-prefix=CHECK-2 %s
+// CHECK-2: Identifier: "BAR"
+// CHECK-2: Identifier: "BAZ"
+// CHECK-2: <end>
+
+#define FOO 2
+#define BAR BAZ
Index: tools/c-index-test/c-index-test.c
===================================================================
--- tools/c-index-test/c-index-test.c
+++ tools/c-index-test/c-index-test.c
@@ -2496,6 +2496,32 @@
     }
 }
 
+static void inspect_tokenize_cursor(CXCursor Cursor) {
+  CXSourceRange CursorExtent = clang_getCursorExtent(Cursor);
+  CXTranslationUnit TU = clang_Cursor_getTranslationUnit(Cursor);
+  unsigned count;
+  unsigned i;
+  CXToken* tokens;
+  clang_tokenize(TU, CursorExtent, &tokens, &count);
+
+  for (i = 0; i < count; ++i) {
+    const char *kind = "<unknown>";
+    CXString spelling = clang_getTokenSpelling(TU, tokens[i]);
+    switch (clang_getTokenKind(tokens[i])) {
+      case CXToken_Keyword: kind = "Keyword"; break;
+      case CXToken_Literal: kind = "Literal"; break;
+      case CXToken_Comment: kind = "Comment"; break;
+      case CXToken_Punctuation: kind = "Punctuation"; break;
+      case CXToken_Identifier: kind = "Identifier"; break;
+    }
+    printf("%s: \"%s\"\n", kind, clang_getCString(spelling));
+    clang_disposeString(spelling);
+  }
+  printf("<end>\n");
+
+  clang_disposeTokens(TU, tokens, count);
+}
+
 static void inspect_evaluate_cursor(CXCursor Cursor) {
   CXSourceLocation CursorLoc = clang_getCursorLocation(Cursor);
   CXString Spelling;
@@ -4314,6 +4340,9 @@
   if (argc > 2 && strstr(argv[1], "-get-macro-info-cursor-at=") == argv[1])
     return inspect_cursor_at(argc, argv, "-get-macro-info-cursor-at=",
                              inspect_macroinfo_cursor);
+  if (argc > 2 && strstr(argv[1], "-tokenize-at=") == argv[1])
+    return inspect_cursor_at(argc, argv, "-tokenize-at=",
+                             inspect_tokenize_cursor);
   if (argc > 2 && strstr(argv[1], "-file-refs-at=") == argv[1])
     return find_file_refs_at(argc, argv);
   if (argc > 2 && strstr(argv[1], "-file-includes-in=") == argv[1])
Index: tools/libclang/CIndex.cpp
===================================================================
--- tools/libclang/CIndex.cpp
+++ tools/libclang/CIndex.cpp
@@ -6156,7 +6156,7 @@
     }
     CXTokens.push_back(CXTok);
     previousWasAt = Tok.is(tok::at);
-  } while (Lex.getBufferLocation() <= EffectiveBufferEnd);
+  } while (Lex.getBufferLocation() < EffectiveBufferEnd);
 }
 
 void clang_tokenize(CXTranslationUnit TU, CXSourceRange Range,