Index: lib/sanitizer_common/sanitizer_allocator.h =================================================================== --- lib/sanitizer_common/sanitizer_allocator.h +++ lib/sanitizer_common/sanitizer_allocator.h @@ -21,6 +21,7 @@ #include "sanitizer_local_address_space_view.h" #include "sanitizer_mutex.h" #include "sanitizer_procmaps.h" +#include "sanitizer_remote_address_space_view.h" #include "sanitizer_type_traits.h" namespace __sanitizer { @@ -67,6 +68,31 @@ *rand_state = state; } +#define SANITIZER_COPY_FIELD(FIELD, DEST_PTR, SRC_PTR) \ + { \ + decltype(DEST_PTR) dst_ptr = DEST_PTR; \ + decltype(SRC_PTR) src_ptr = SRC_PTR; \ + internal_memcpy(&(dst_ptr->FIELD), &(src_ptr->FIELD), \ + sizeof(decltype(dst_ptr->FIELD))); \ + static_assert( \ + sizeof(decltype(dst_ptr->FIELD)) == sizeof(decltype(src_ptr->FIELD)), \ + "Sizes do not match for field " #FIELD); \ + } + +template +const SrcTy *CopyFromTargetCommon(DstTy *dst, const SrcTy *src, + bool src_needs_load, bool dst_needs_zeroing) { + CHECK_NE(dst, nullptr); + CHECK_NE(src, nullptr); + // Zero destination buffer so any padding gets zero-ed out. + if (dst_needs_zeroing) + internal_memset(dst, 0, sizeof(decltype(*dst))); + const SrcTy *src_cpy = src; + if (src_needs_load) + src_cpy = RemoteAddressSpaceView::Load(src); + return src_cpy; +} + #include "sanitizer_allocator_size_class_map.h" #include "sanitizer_allocator_stats.h" #include "sanitizer_allocator_primary64.h" @@ -76,6 +102,8 @@ #include "sanitizer_allocator_secondary.h" #include "sanitizer_allocator_combined.h" +#undef SANITIZER_COPY_FIELD + } // namespace __sanitizer #endif // SANITIZER_ALLOCATOR_H Index: lib/sanitizer_common/sanitizer_allocator_bytemap.h =================================================================== --- lib/sanitizer_common/sanitizer_allocator_bytemap.h +++ lib/sanitizer_common/sanitizer_allocator_bytemap.h @@ -18,6 +18,9 @@ class FlatByteMap { public: using AddressSpaceView = AddressSpaceViewTy; + template + using ThisTASVT = FlatByteMap; + void Init() { internal_memset(map_, 0, sizeof(map_)); } @@ -32,6 +35,27 @@ // FIXME: CHECK may be too expensive here. return map_[idx]; } + + // For `CopyFromTarget(...)`. + template + friend class FlatByteMap; + + static void CopyFromTarget(ThisTASVT *dst, + const ThisTASVT *this_ptr, + bool this_ptr_is_remote = true, + bool dst_needs_zeroing = true) { + // C++ does not guarantee that `ThisTASVT` and + // `ThisTASVT` will have the same data layout so copy the + // whole object across and then copy the fields individually into the + // destination buffer. + using ThisT = ThisTASVT; + const ThisT *this_cpy = CopyFromTargetCommon( + dst, this_ptr, this_ptr_is_remote, dst_needs_zeroing); + + // Keep this in sync with the fields of this class! + SANITIZER_COPY_FIELD(map_, dst, this_cpy); + } + private: u8 map_[kSize]; }; @@ -47,6 +71,10 @@ class TwoLevelByteMap { public: using AddressSpaceView = AddressSpaceViewTy; + template + using ThisTASVT = + TwoLevelByteMap; + void Init() { internal_memset(map1_, 0, sizeof(map1_)); mu_.Init(); @@ -80,6 +108,27 @@ return *value_ptr; } + // For `CopyFromTarget(...)`. + template + friend class TwoLevelByteMap; + + static void CopyFromTarget(ThisTASVT *dst, + const ThisTASVT *this_ptr, + bool this_ptr_is_remote = true, + bool dst_needs_zeroing = true) { + // C++ does not guarantee that `ThisTASVT` and + // `ThisTASVT` will have the same data layout so copy the + // whole object across and then copy the fields individually into the + // destination buffer. + using ThisT = ThisTASVT; + const ThisT *this_cpy = CopyFromTargetCommon( + dst, this_ptr, this_ptr_is_remote, dst_needs_zeroing); + + // Keep this in sync with the fields of this class! + SANITIZER_COPY_FIELD(map1_, dst, this_cpy); + SANITIZER_COPY_FIELD(mu_, dst, this_cpy); + } + private: u8 *Get(uptr idx) const { CHECK_LT(idx, kSize1); Index: lib/sanitizer_common/sanitizer_allocator_combined.h =================================================================== --- lib/sanitizer_common/sanitizer_allocator_combined.h +++ lib/sanitizer_common/sanitizer_allocator_combined.h @@ -25,6 +25,13 @@ class CombinedAllocator { public: using AddressSpaceView = AddressSpaceViewTy; + template + using ThisTASVT = CombinedAllocator< + typename PrimaryAllocator::template ThisTASVT, + typename AllocatorCache::template ThisTASVT, + typename SecondaryAllocator::template ThisTASVT, + OtherViewTy>; + static_assert(is_same::value, "PrimaryAllocator is using wrong AddressSpaceView"); @@ -44,6 +51,33 @@ secondary_.Init(); } + // For `CopyFromTarget(...)`. + template + friend class CombinedAllocator; + + static void CopyFromTarget(ThisTASVT *dst, + const ThisTASVT *this_ptr, + bool this_ptr_is_remote = true, + bool dst_needs_zeroing = true) { + // C++ does not guarantee that `ThisTASVT` and + // `ThisTASVT` will have the same data layout so copy the + // whole object across and then copy the fields individually into the + // destination buffer. + using ThisT = ThisTASVT; + const ThisT *this_cpy = CopyFromTargetCommon( + dst, this_ptr, this_ptr_is_remote, dst_needs_zeroing); + + // Keep this in sync with the fields of this class! + PrimaryAllocator::CopyFromTarget(&(dst->primary_), &(this_cpy->primary_), + /*this_ptr_is_remote=*/false, + /*dst_needs_zeroing=*/false); + SecondaryAllocator::CopyFromTarget(&(dst->secondary_), + &(this_cpy->secondary_), + /*this_ptr_is_remote=*/false, + /*dst_needs_zeroing=*/false); + SANITIZER_COPY_FIELD(stats_, dst, this_cpy); + } + void *Allocate(AllocatorCache *cache, uptr size, uptr alignment) { // Returning 0 on malloc(0) may break a lot of code. if (size == 0) Index: lib/sanitizer_common/sanitizer_allocator_local_cache.h =================================================================== --- lib/sanitizer_common/sanitizer_allocator_local_cache.h +++ lib/sanitizer_common/sanitizer_allocator_local_cache.h @@ -16,15 +16,22 @@ // Objects of this type should be used as local caches for SizeClassAllocator64 // or SizeClassAllocator32. Since the typical use of this class is to have one // object per thread in TLS, is has to be POD. -template -struct SizeClassAllocatorLocalCache - : SizeClassAllocator::AllocatorCache {}; +template +struct SizeClassAllocatorLocalCache : SizeClassAllocator::AllocatorCache { + template + using ThisTASVT = SizeClassAllocatorLocalCache< + typename SizeClassAllocator::template ThisTASVT>; +}; // Cache used by SizeClassAllocator64. template struct SizeClassAllocator64LocalCache { typedef SizeClassAllocator Allocator; + template + using ThisTASVT = SizeClassAllocator64LocalCache< + typename SizeClassAllocator::template ThisTASVT>; + void Init(AllocatorGlobalStats *s) { stats_.Init(); if (s) Index: lib/sanitizer_common/sanitizer_allocator_primary32.h =================================================================== --- lib/sanitizer_common/sanitizer_allocator_primary32.h +++ lib/sanitizer_common/sanitizer_allocator_primary32.h @@ -48,6 +48,8 @@ class SizeClassAllocator32 { public: using AddressSpaceView = AddressSpaceViewTy; + template + using ThisTASVT = SizeClassAllocator32; static const uptr kSpaceBeg = Params::kSpaceBeg; static const u64 kSpaceSize = Params::kSpaceSize; static const uptr kMetadataSize = Params::kMetadataSize; @@ -116,6 +118,30 @@ internal_memset(size_class_info_array, 0, sizeof(size_class_info_array)); } + // For `CopyFromTarget(...)`. + template + friend class SizeClassAllocator32; + + static void CopyFromTarget(ThisTASVT *dst, + const ThisTASVT *this_ptr, + bool this_ptr_is_remote = true, + bool dst_needs_zeroing = true) { + // C++ does not guarantee that `ThisTASVT` and + // `ThisTASVT` will have the same data layout so copy the + // whole object across and then copy the fields individually into the + // destination buffer. + using ThisT = ThisTASVT; + const ThisT *this_cpy = CopyFromTargetCommon( + dst, this_ptr, this_ptr_is_remote, dst_needs_zeroing); + + // Keep this in sync with the fields of this class! + ByteMap::CopyFromTarget(&(dst->possible_regions), + &(this_cpy->possible_regions), + /*this_ptr_is_remote=*/false, + /*dst_needs_zeroing=*/false); + SANITIZER_COPY_FIELD(size_class_info_array, dst, this_cpy); + } + s32 ReleaseToOSIntervalMs() const { return kReleaseToOSIntervalNever; } Index: lib/sanitizer_common/sanitizer_allocator_primary64.h =================================================================== --- lib/sanitizer_common/sanitizer_allocator_primary64.h +++ lib/sanitizer_common/sanitizer_allocator_primary64.h @@ -46,6 +46,8 @@ class SizeClassAllocator64 { public: using AddressSpaceView = AddressSpaceViewTy; + template + using ThisTASVT = SizeClassAllocator64; static const uptr kSpaceBeg = Params::kSpaceBeg; static const uptr kSpaceSize = Params::kSpaceSize; static const uptr kMetadataSize = Params::kMetadataSize; @@ -85,6 +87,28 @@ DCHECK_EQ(SpaceEnd() % kCacheLineSize, 0); } + // For `CopyFromTarget(...)`. + template + friend class SizeClassAllocator64; + + static void CopyFromTarget(ThisTASVT *dst, + const ThisTASVT *this_ptr, + bool this_ptr_is_remote = true, + bool dst_needs_zeroing = true) { + // C++ does not guarantee that `ThisTASVT` and + // `ThisTASVT` will have the same data layout so copy the + // whole object across and then copy the fields individually into the + // destination buffer. + using ThisT = ThisTASVT; + const ThisT *this_cpy = CopyFromTargetCommon( + dst, this_ptr, this_ptr_is_remote, dst_needs_zeroing); + + // Keep this in sync with the fields of this class! + SANITIZER_COPY_FIELD(address_range, dst, this_cpy); + SANITIZER_COPY_FIELD(NonConstSpaceBeg, dst, this_cpy); + SANITIZER_COPY_FIELD(release_to_os_interval_ms_, dst, this_cpy); + } + s32 ReleaseToOSIntervalMs() const { return atomic_load(&release_to_os_interval_ms_, memory_order_relaxed); } Index: lib/sanitizer_common/sanitizer_allocator_secondary.h =================================================================== --- lib/sanitizer_common/sanitizer_allocator_secondary.h +++ lib/sanitizer_common/sanitizer_allocator_secondary.h @@ -72,6 +72,10 @@ class LargeMmapAllocator { public: using AddressSpaceView = AddressSpaceViewTy; + template + using ThisTASVT = + LargeMmapAllocator; + void InitLinkerInitialized() { page_size_ = GetPageSizeCached(); chunks_ = reinterpret_cast(ptr_array_.Init()); @@ -82,6 +86,32 @@ InitLinkerInitialized(); } + // For `CopyFromTarget(...)`. + template + friend class LargeMmapAllocator; + + static void CopyFromTarget(ThisTASVT *dst, + const ThisTASVT *this_ptr, + bool this_ptr_is_remote = true, + bool dst_needs_zeroing = true) { + // C++ does not guarantee that `ThisTASVT` and + // `ThisTASVT` will have the same data layout so copy the + // whole object across and then copy the fields individually into the + // destination buffer. + using ThisT = ThisTASVT; + const ThisT *this_cpy = CopyFromTargetCommon( + dst, this_ptr, this_ptr_is_remote, dst_needs_zeroing); + + // Keep this in sync with the fields of this class! + SANITIZER_COPY_FIELD(page_size_, dst, this_cpy); + SANITIZER_COPY_FIELD(chunks_, dst, this_cpy); + SANITIZER_COPY_FIELD(ptr_array_, dst, this_cpy); + SANITIZER_COPY_FIELD(n_chunks_, dst, this_cpy); + SANITIZER_COPY_FIELD(chunks_sorted_, dst, this_cpy); + SANITIZER_COPY_FIELD(stats, dst, this_cpy); + SANITIZER_COPY_FIELD(mutex_, dst, this_cpy); + } + void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) { CHECK(IsPowerOfTwo(alignment)); uptr map_size = RoundUpMapSize(size);