[Sanitizers] Solaris largefile fixes

[ro]

Summary

While testing the Solaris libsanitizer port on GCC mainline, I found that I'd messed up
the largefile checks in various ways, some of which showed as compile failures
(wrong structure sizes and member offsets), others at runtime, some of those only
on sparc as a big-endian target.

This patch fixes all of them:

• OFF_T is now correctly defined for 32-bit largefile and traditional environments, and 64-bit.

• The definition of __sanitizer_dirent now checks the correct conditionals.

• sanitizer_procmaps_solaris.cc undefines `_FILE_OFFSET_BITS``: before Solaris 11.4 `<procfs.h>` doesn't even compile with largefile support enabled, but the use at hand doesn't need it anyway while g++ 9 will define`_FILE_OFFSET_BITS=64`` out of the box.

• With full largefile support enabled, one needs to use e.g. `mmap64`` instead of `mmap``; this is hidden behind macros.

With this patch I could bootstrap gcc mainline on both sparc-sun-solaris2.11 and
i386-pc-solaris2.11. In addition, I've successfully built llvm on i386-pc-solaris2.11.

compiler-rt/trunk/lib/sanitizer_common/sanitizer_internal_defs.h

 #if SANITIZER_SOLARIS && !defined(_LP64)
 typedef long pid_t;
 #else
 typedef int pid_t;
 #endif
 
 #if SANITIZER_FREEBSD || SANITIZER_NETBSD || \
     SANITIZER_OPENBSD || SANITIZER_MAC || \
+    (SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \
     (SANITIZER_LINUX && defined(__x86_64__))
 typedef u64 OFF_T;
 #else
 typedef uptr OFF_T;
 #endif
 typedef u64  OFF64_T;
 
 #if (SANITIZER_WORDSIZE == 64) || SANITIZER_MAC

compiler-rt/trunk/lib/sanitizer_common/sanitizer_platform_limits_solaris.h

struct __sanitizer_msghdr {
   int msg_flags;
 };
 struct __sanitizer_cmsghdr {
   unsigned cmsg_len;
   int cmsg_level;
   int cmsg_type;
 };
 
-#if SANITIZER_SOLARIS32 && 0
-// FIXME: need to deal with large file and non-large file cases
+#if SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)
 struct __sanitizer_dirent {
   unsigned long long d_ino;
   long long d_off;
   unsigned short d_reclen;
   // more fields that we don't care about
 };
 #else
 struct __sanitizer_dirent {

compiler-rt/trunk/lib/sanitizer_common/sanitizer_procmaps_solaris.cc

 //===-- sanitizer_procmaps_solaris.cc -------------------------------------===//
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 //
 // Information about the process mappings (Solaris-specific parts).
 //===----------------------------------------------------------------------===//
 
 #include "sanitizer_platform.h"
 #if SANITIZER_SOLARIS
 #include "sanitizer_common.h"
 #include "sanitizer_procmaps.h"
 
+// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment.
+#undef _FILE_OFFSET_BITS
 #include <procfs.h>
 #include <limits.h>
 
 namespace __sanitizer {
 
 void ReadProcMaps(ProcSelfMapsBuff *proc_maps) {
   ReadFileToBuffer("/proc/self/xmap", &proc_maps->data, &proc_maps->mmaped_size,
                    &proc_maps->len);

compiler-rt/trunk/lib/sanitizer_common/sanitizer_solaris.cc

 
 #define _REAL(func) _ ## func
 #define DECLARE__REAL(ret_type, func, ...) \
   extern "C" ret_type _REAL(func)(__VA_ARGS__)
 #define DECLARE__REAL_AND_INTERNAL(ret_type, func, ...) \
   DECLARE__REAL(ret_type, func, __VA_ARGS__); \
   ret_type internal_ ## func(__VA_ARGS__)
 
+#if !defined(_LP64) && _FILE_OFFSET_BITS == 64
+#define _REAL64(func) _ ## func ## 64
+#else
+#define _REAL64(func) _REAL(func)
+#endif
+#define DECLARE__REAL64(ret_type, func, ...) \
+  extern "C" ret_type _REAL64(func)(__VA_ARGS__)
+#define DECLARE__REAL_AND_INTERNAL64(ret_type, func, ...) \
+  DECLARE__REAL64(ret_type, func, __VA_ARGS__); \
+  ret_type internal_ ## func(__VA_ARGS__)
+
 // ---------------------- sanitizer_libc.h
-DECLARE__REAL_AND_INTERNAL(uptr, mmap, void *addr, uptr /*size_t*/ length,
+DECLARE__REAL_AND_INTERNAL64(uptr, mmap, void *addr, uptr /*size_t*/ length,
                              int prot, int flags, int fd, OFF_T offset) {
-  return (uptr)_REAL(mmap)(addr, length, prot, flags, fd, offset);
+  return (uptr)_REAL64(mmap)(addr, length, prot, flags, fd, offset);
 }
 
 DECLARE__REAL_AND_INTERNAL(uptr, munmap, void *addr, uptr length) {
   return _REAL(munmap)(addr, length);
 }
 
 DECLARE__REAL_AND_INTERNAL(int, mprotect, void *addr, uptr length, int prot) {
   return _REAL(mprotect)(addr, length, prot);
 }
 
 DECLARE__REAL_AND_INTERNAL(uptr, close, fd_t fd) {
   return _REAL(close)(fd);
 }
 
-extern "C" int _REAL(open)(const char *, int, ...);
+extern "C" int _REAL64(open)(const char *, int, ...);
 
 uptr internal_open(const char *filename, int flags) {
-  return _REAL(open)(filename, flags);
+  return _REAL64(open)(filename, flags);
 }
 
 uptr internal_open(const char *filename, int flags, u32 mode) {
-  return _REAL(open)(filename, flags, mode);
+  return _REAL64(open)(filename, flags, mode);
 }
 
 uptr OpenFile(const char *filename, bool write) {
   return internal_open(filename,
       write ? O_WRONLY | O_CREAT : O_RDONLY, 0660);
 }
 
 DECLARE__REAL_AND_INTERNAL(uptr, read, fd_t fd, void *buf, uptr count) {
   return _REAL(read)(fd, buf, count);
 }
 
 DECLARE__REAL_AND_INTERNAL(uptr, write, fd_t fd, const void *buf, uptr count) {
   return _REAL(write)(fd, buf, count);
 }
 
 // FIXME: There's only _ftruncate64 beginning with Solaris 11.
 DECLARE__REAL_AND_INTERNAL(uptr, ftruncate, fd_t fd, uptr size) {
   return ftruncate(fd, size);
 }
 
-DECLARE__REAL_AND_INTERNAL(uptr, stat, const char *path, void *buf) {
-  return _REAL(stat)(path, (struct stat *)buf);
+DECLARE__REAL_AND_INTERNAL64(uptr, stat, const char *path, void *buf) {
+  return _REAL64(stat)(path, (struct stat *)buf);
 }
 
-DECLARE__REAL_AND_INTERNAL(uptr, lstat, const char *path, void *buf) {
-  return _REAL(lstat)(path, (struct stat *)buf);
+DECLARE__REAL_AND_INTERNAL64(uptr, lstat, const char *path, void *buf) {
+  return _REAL64(lstat)(path, (struct stat *)buf);
 }
 
-DECLARE__REAL_AND_INTERNAL(uptr, fstat, fd_t fd, void *buf) {
-  return _REAL(fstat)(fd, (struct stat *)buf);
+DECLARE__REAL_AND_INTERNAL64(uptr, fstat, fd_t fd, void *buf) {
+  return _REAL64(fstat)(fd, (struct stat *)buf);
 }
 
 uptr internal_filesize(fd_t fd) {
   struct stat st;
   if (internal_fstat(fd, &st))
     return -1;
   return (uptr)st.st_size;
 }
DECLARE__REAL_AND_INTERNAL(uptr, waitpid, int pid, int *status, int options) {
   return _REAL(waitpid)(pid, status, options);
 }
 
 DECLARE__REAL_AND_INTERNAL(uptr, getpid, void) {
   return _REAL(getpid)();
 }
 
 // FIXME: This might be wrong: _getdents doesn't take a struct linux_dirent *.
-DECLARE__REAL_AND_INTERNAL(uptr, getdents, fd_t fd, struct linux_dirent *dirp,
+DECLARE__REAL_AND_INTERNAL64(uptr, getdents, fd_t fd, struct linux_dirent *dirp,
                              unsigned int count) {
-  return _REAL(getdents)(fd, dirp, count);
+  return _REAL64(getdents)(fd, dirp, count);
 }
 
-DECLARE__REAL_AND_INTERNAL(uptr, lseek, fd_t fd, OFF_T offset, int whence) {
-  return _REAL(lseek)(fd, offset, whence);
+DECLARE__REAL_AND_INTERNAL64(uptr, lseek, fd_t fd, OFF_T offset, int whence) {
+  return _REAL64(lseek)(fd, offset, whence);
 }
 
 // FIXME: This might be wrong: _sigfillset doesn't take a
 // __sanitizer_sigset_t *.
 DECLARE__REAL_AND_INTERNAL(void, sigfillset, __sanitizer_sigset_t *set) {
   _REAL(sigfillset)(set);
 }