[scudo] Application & platform compatibility changes

[cryptoad]

Summary

This patch changes a few (small) things around for compatibility purposes for
the current Android & Fuchsia work:

• realloc'ing some memory that was not allocated with malloc, calloc or realloc, while UB according to http://pubs.opengroup.org/onlinepubs/009695399/functions/realloc.html is more common that one would think. We now only check this if DeallocationTypeMismatch is set; change the "mismatch" error messages to be more homogeneous;
• some sketchily written but widely used libraries expect a call to realloc to copy the usable size of the old chunk to the new one instead of the requested size. We have to begrundingly abide by this de-facto standard. This doesn't seem to impact security either way, unless someone comes up with something we didn't think about;
• the CRC32 intrinsics for 64-bit take a 64-bit first argument. This is misleading as the upper 32 bits end up being ignored. This was also raising -Wconversion errors. Change things to take a u32 as first argument. This also means we were (and are) only using 32 bits of the Cookie - not a big thing, but worth mentioning.
• Includes-wise: prefer stddef.h to cstddef, move scudo_flags.h where it is actually needed.
• Add tests for the memalign-realloc case, and the realloc-usable-size one.

(Edited typos)

[alekseyshl]

What are these libraries? Can you mention some examples here?

lib/scudo/scudo_allocator.h

 //===-- scudo_allocator.h ---------------------------------------*- C++ -*-===//
 //
 //                     The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
 //===----------------------------------------------------------------------===//
 ///
 /// Header for scudo_allocator.cpp.
 ///
 //===----------------------------------------------------------------------===//
 
 #ifndef SCUDO_ALLOCATOR_H_
 #define SCUDO_ALLOCATOR_H_
 
-#include "scudo_flags.h"
-
 #include "sanitizer_common/sanitizer_allocator.h"
 
 #if !SANITIZER_LINUX
 # error "The Scudo hardened allocator is currently only supported on Linux."
 #endif
 
 namespace __scudo {
 

lib/scudo/scudo_allocator.cpp

 /// It uses the sanitizer_common allocator as a base and aims at mitigating
 /// heap corruption vulnerabilities. It provides a checksum-guarded chunk
 /// header, a delayed free list, and additional sanity checks.
 ///
 //===----------------------------------------------------------------------===//
 
 #include "scudo_allocator.h"
 #include "scudo_crc32.h"
+#include "scudo_flags.h"
 #include "scudo_tls.h"
 #include "scudo_utils.h"
 
 #include "sanitizer_common/sanitizer_allocator_checks.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
 #include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_quarantine.h"
 
 #include <string.h>
 
 namespace __scudo {
 
 // Global static cookie, initialized at start-up.
 static uptr Cookie;
 
 // We default to software CRC32 if the alternatives are not supported, either
 // at compilation or at runtime.
 static atomic_uint8_t HashAlgorithm = { CRC32Software };
 
-INLINE u32 computeCRC32(uptr Crc, uptr Value, uptr *Array, uptr ArraySize) {
+INLINE u32 computeCRC32(u32 Crc, uptr Value, uptr *Array, uptr ArraySize) {
   // If the hardware CRC32 feature is defined here, it was enabled everywhere,
   // as opposed to only for scudo_crc32.cpp. This means that other hardware
   // specific instructions were likely emitted at other places, and as a
   // result there is no reason to not use it here.
 #if defined(__SSE4_2__) || defined(__ARM_FEATURE_CRC32)
   Crc = CRC32_INTRINSIC(Crc, Value);
   for (uptr i = 0; i < ArraySize; i++)
     Crc = CRC32_INTRINSIC(Crc, Array[i]);
struct ScudoChunk : UnpackedHeader {
   }
 
   // Compute the checksum of the Chunk pointer and its ChunkHeader.
   u16 computeChecksum(UnpackedHeader *Header) const {
     UnpackedHeader ZeroChecksumHeader = *Header;
     ZeroChecksumHeader.Checksum = 0;
     uptr HeaderHolder[sizeof(UnpackedHeader) / sizeof(uptr)];
     memcpy(&HeaderHolder, &ZeroChecksumHeader, sizeof(HeaderHolder));
-    u32 Crc = computeCRC32(Cookie, reinterpret_cast<uptr>(this), HeaderHolder,
+    u32 Crc = computeCRC32(static_cast<u32>(Cookie),
+                           reinterpret_cast<uptr>(this), HeaderHolder,
                            ARRAY_SIZE(HeaderHolder));
     return static_cast<u16>(Crc);
   }
 
   // Checks the validity of a chunk by verifying its checksum. It doesn't
   // incur termination in the event of an invalid chunk.
   bool isValid() {
     UnpackedHeader NewUnpackedHeader;
if (UNLIKELY(Header.State != ChunkAllocated)) {
       dieWithMessage("ERROR: invalid chunk state when deallocating address "
                      "%p\n", UserPtr);
     }
     if (DeallocationTypeMismatch) {
       // The deallocation type has to match the allocation one.
       if (Header.AllocType != Type) {
         // With the exception of memalign'd Chunks, that can be still be free'd.
         if (Header.AllocType != FromMemalign || Type != FromMalloc) {
-          dieWithMessage("ERROR: allocation type mismatch on address %p\n",
-                         UserPtr);
+          dieWithMessage("ERROR: allocation type mismatch when deallocating "
+                         "address %p\n", UserPtr);
         }
       }
     }
     uptr Size = Header.FromPrimary ? Header.SizeOrUnusedBytes :
         Chunk->getUsableSize(&Header) - Header.SizeOrUnusedBytes;
     if (DeleteSizeMismatch) {
       if (DeleteSize && DeleteSize != Size) {
         dieWithMessage("ERROR: invalid sized delete on chunk at address %p\n",
void *reallocate(void *OldPtr, uptr NewSize) {
     }
     ScudoChunk *Chunk = getScudoChunk(UserBeg);
     UnpackedHeader OldHeader;
     Chunk->loadHeader(&OldHeader);
     if (UNLIKELY(OldHeader.State != ChunkAllocated)) {
       dieWithMessage("ERROR: invalid chunk state when reallocating address "
                      "%p\n", OldPtr);
     }
+    if (DeallocationTypeMismatch) {
       if (UNLIKELY(OldHeader.AllocType != FromMalloc)) {
-      dieWithMessage("ERROR: invalid chunk type when reallocating address %p\n",
-                     OldPtr);
+        dieWithMessage("ERROR: allocation type mismatch when reallocating "
+                       "address %p\n", OldPtr);
+      }
     }
     uptr UsableSize = Chunk->getUsableSize(&OldHeader);
     // The new size still fits in the current chunk, and the size difference
     // is reasonable.
     if (NewSize <= UsableSize &&
         (UsableSize - NewSize) < (SizeClassMap::kMaxSize / 2)) {
       UnpackedHeader NewHeader = OldHeader;
       NewHeader.SizeOrUnusedBytes =
                 OldHeader.FromPrimary ? NewSize : UsableSize - NewSize;
       Chunk->compareExchangeHeader(&NewHeader, &OldHeader);
       return OldPtr;
     }
     // Otherwise, we have to allocate a new chunk and copy the contents of the
     // old one.
     void *NewPtr = allocate(NewSize, MinAlignment, FromMalloc);
     if (NewPtr) {
       uptr OldSize = OldHeader.FromPrimary ? OldHeader.SizeOrUnusedBytes :
           UsableSize - OldHeader.SizeOrUnusedBytes;
-      memcpy(NewPtr, OldPtr, Min(NewSize, OldSize));
+      memcpy(NewPtr, OldPtr, Min(NewSize, UsableSize));
       quarantineOrDeallocateChunk(Chunk, &OldHeader, OldSize);
     }
     return NewPtr;
   }
 
   // Helper function that returns the actual usable size of a chunk.
   uptr getUsableSize(const void *Ptr) {
     initThreadMaybe();

lib/scudo/scudo_new_delete.cpp

 /// Interceptors for operators new and delete.
 ///
 //===----------------------------------------------------------------------===//
 
 #include "scudo_allocator.h"
 
 #include "interception/interception.h"
 
-#include <cstddef>
+#include <stddef.h>
 
 using namespace __scudo;
 
 #define CXX_OPERATOR_ATTRIBUTE INTERCEPTOR_ATTRIBUTE
 
 // Fake std::nothrow_t to avoid including <new>.
 namespace std {
 struct nothrow_t {};

test/scudo/mismatch.cpp

 // RUN: %clang_scudo %s -o %t
-// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=1 not %run %t mallocdel   2>&1 | FileCheck %s
-// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=0     %run %t mallocdel   2>&1
-// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=1 not %run %t newfree     2>&1 | FileCheck %s
-// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=0     %run %t newfree     2>&1
-// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=1 not %run %t memaligndel 2>&1 | FileCheck %s
-// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=0     %run %t memaligndel 2>&1
+// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=1 not %run %t mallocdel       2>&1 | FileCheck --check-prefix=CHECK-dealloc %s
+// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=0     %run %t mallocdel       2>&1
+// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=1 not %run %t newfree         2>&1 | FileCheck --check-prefix=CHECK-dealloc %s
+// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=0     %run %t newfree         2>&1
+// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=1 not %run %t memaligndel     2>&1 | FileCheck --check-prefix=CHECK-dealloc %s
+// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=0     %run %t memaligndel     2>&1
+// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=1 not %run %t memalignrealloc 2>&1 | FileCheck --check-prefix=CHECK-realloc %s
+// RUN: SCUDO_OPTIONS=DeallocationTypeMismatch=0     %run %t memalignrealloc 2>&1
 
 // Tests that type mismatches between allocation and deallocation functions are
 // caught when the related option is set.
 
 #include <assert.h>
 #include <malloc.h>
 #include <stdlib.h>
 #include <string.h>
if (!strcmp(argv[1], "newfree")) {
     assert(p);
     free((void *)p);
   }
   if (!strcmp(argv[1], "memaligndel")) {
     int *p = (int *)memalign(16, 16);
     assert(p);
     delete p;
   }
+  if (!strcmp(argv[1], "memalignrealloc")) {
+    void *p = memalign(16, 16);
+    assert(p);
+    p = realloc(p, 32);
+    free(p);
+  }
   return 0;
 }
 
-// CHECK: ERROR: allocation type mismatch on address
+// CHECK-dealloc: ERROR: allocation type mismatch when deallocating address
+// CHECK-realloc: ERROR: allocation type mismatch when reallocating address

test/scudo/options.cpp

 int main(int argc, char **argv)
 {
   int *p = (int *)malloc(16);
   assert(p);
   delete p;
   return 0;
 }
 
-// CHECK: ERROR: allocation type mismatch on address
+// CHECK: ERROR: allocation type mismatch when deallocating address

test/scudo/realloc.cpp

 // RUN: %clang_scudo %s -lstdc++ -o %t
-// RUN:     %run %t pointers 2>&1
-// RUN:     %run %t contents 2>&1
-// RUN: not %run %t memalign 2>&1 | FileCheck %s
+// RUN: %run %t pointers 2>&1
+// RUN: %run %t contents 2>&1
+// RUN: %run %t usablesize 2>&1
 
 // Tests that our reallocation function returns the same pointer when the
 // requested size can fit into the previously allocated chunk. Also tests that
 // a new chunk is returned if the size is greater, and that the contents of the
-// chunk are left unchanged.
-// As a final test, make sure that a chunk allocated by memalign cannot be
-// reallocated.
+// chunk are left unchanged. Finally, checks that realloc copies the usable
+// size of the old chunk to the new one (as opposed to the requested size).
 
 #include <assert.h>
 #include <malloc.h>
 #include <string.h>
 
 #include <vector>
 
 int main(int argc, char **argv)
 {
   void *p, *old_p;
   // Those sizes will exercise both allocators (Primary & Secondary).
   std::vector<size_t> sizes{1, 16, 1024, 32768, 1 << 16, 1 << 17, 1 << 20};
 
   assert(argc == 2);
 
+  if (!strcmp(argv[1], "usablesize")) {
+    // This tests a sketchy behavior inherited from poorly written libraries
+    // that have become somewhat standard. When realloc'ing a chunk, the
+    // copied contents should span the usable size of the chunk, not the
+    // requested size.
+    size_t size = 496, usable_size;
+    p = nullptr;
+    // Make sure we get a chunk with a usable size actually larger than size.
+    do {
+      if (p) free(p);
+      size += 16;
+      p = malloc(size);
+      usable_size = malloc_usable_size(p);
+      assert(usable_size >= size);
+    } while (usable_size == size);
+    for (int i = 0; i < usable_size; i++)
+      reinterpret_cast<char *>(p)[i] = 'A';
+    old_p = p;
+    // Make sure we get a different chunk so that the data is actually copied.
+    do {
+      size *= 2;
+      p = realloc(p, size);
+      assert(p);
+    } while (p == old_p);
+    // The contents of the new chunk must match the old one up to usable_size.
+    for (int i = 0; i < usable_size; i++)
+      assert(reinterpret_cast<char *>(p)[i] == 'A');
+    free(p);
+  } else {
     for (size_t size : sizes) {
       if (!strcmp(argv[1], "pointers")) {
         old_p = p = realloc(nullptr, size);
         assert(p);
         size = malloc_usable_size(p);
         // Our realloc implementation will return the same pointer if the size
         // requested is lower than or equal to the usable size of the associated
         // chunk.
         p = realloc(p, size - 1);
         assert(p == old_p);
         p = realloc(p, size);
         assert(p == old_p);
         // And a new one if the size is greater.
         p = realloc(p, size + 1);
         assert(p != old_p);
         // A size of 0 will free the chunk and return nullptr.
         p = realloc(p, 0);
         assert(!p);
         old_p = nullptr;
       }
       if (!strcmp(argv[1], "contents")) {
         p = realloc(nullptr, size);
         assert(p);
         for (int i = 0; i < size; i++)
           reinterpret_cast<char *>(p)[i] = 'A';
         p = realloc(p, size + 1);
         // The contents of the reallocated chunk must match the original one.
         for (int i = 0; i < size; i++)
           assert(reinterpret_cast<char *>(p)[i] == 'A');
       }
-    if (!strcmp(argv[1], "memalign")) {
-      // A chunk coming from memalign cannot be reallocated.
-      p = memalign(16, size);
-      assert(p);
-      p = realloc(p, size);
-      free(p);
     }
   }
   return 0;
 }
 
 // CHECK: ERROR: invalid chunk type when reallocating address