[msan] Fix return type of mbrtowc

[vitalybuka]

Summary

Fixes https://github.com/google/oss-fuzz/issues/1009

[eugenis]

thanks!

lib/msan/msan_interceptors.cc

 
 INTERCEPTOR(int, mbtowc, wchar_t *dest, const char *src, SIZE_T n) {
   ENSURE_MSAN_INITED();
   int res = REAL(mbtowc)(dest, src, n);
   if (res != -1 && dest) __msan_unpoison(dest, sizeof(wchar_t));
   return res;
 }
 
-INTERCEPTOR(int, mbrtowc, wchar_t *dest, const char *src, SIZE_T n, void *ps) {
+INTERCEPTOR(SIZE_T, mbrtowc, wchar_t *dest, const char *src, SIZE_T n,
+            void *ps) {
   ENSURE_MSAN_INITED();
   SIZE_T res = REAL(mbrtowc)(dest, src, n, ps);
   if (res != (SIZE_T)-1 && dest) __msan_unpoison(dest, sizeof(wchar_t));
   return res;
 }
 
 // wchar_t *wmemcpy(wchar_t *dest, const wchar_t *src, SIZE_T n);
 INTERCEPTOR(wchar_t *, wmemcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) {

lib/msan/tests/msan_test.cc

TEST(MemorySanitizer, mbtowc) {
   const char *x = "abc";
   wchar_t wx;
   int res = mbtowc(&wx, x, 3);
   EXPECT_GT(res, 0);
   EXPECT_NOT_POISONED(wx);
 }
 
 TEST(MemorySanitizer, mbrtowc) {
-  const char *x = "abc";
-  wchar_t wx;
-  mbstate_t mbs;
-  memset(&mbs, 0, sizeof(mbs));
-  int res = mbrtowc(&wx, x, 3, &mbs);
-  EXPECT_GT(res, 0);
-  EXPECT_NOT_POISONED(wx);
+  mbstate_t mbs = {};
+
+  wchar_t wc;
+  size_t res = mbrtowc(&wc, "\377", 1, &mbs);
+  EXPECT_EQ(res, -1ULL);
+
+  res = mbrtowc(&wc, "abc", 3, &mbs);
+  EXPECT_GT(res, 0ULL);
+  EXPECT_NOT_POISONED(wc);
 }
 
 TEST(MemorySanitizer, wcsftime) {
   wchar_t x[100];
   time_t t = time(NULL);
   struct tm tms;
   struct tm *tmres = localtime_r(&t, &tms);
   ASSERT_NE((void *)0, tmres);