diff --git a/compiler-rt/lib/scudo/standalone/CMakeLists.txt b/compiler-rt/lib/scudo/standalone/CMakeLists.txt --- a/compiler-rt/lib/scudo/standalone/CMakeLists.txt +++ b/compiler-rt/lib/scudo/standalone/CMakeLists.txt @@ -73,6 +73,7 @@ memtag.h mem_map.h mem_map_base.h + mem_map_fuchsia.h mutex.h options.h platform.h @@ -107,6 +108,7 @@ fuchsia.cpp linux.cpp mem_map.cpp + mem_map_fuchsia.cpp release.cpp report.cpp rss_limit_checker.cpp diff --git a/compiler-rt/lib/scudo/standalone/mem_map.h b/compiler-rt/lib/scudo/standalone/mem_map.h --- a/compiler-rt/lib/scudo/standalone/mem_map.h +++ b/compiler-rt/lib/scudo/standalone/mem_map.h @@ -21,6 +21,8 @@ #include "linux.h" #include "trusty.h" +#include "mem_map_fuchsia.h" + namespace scudo { // This will be deprecated when every allocator has been supported by each @@ -74,7 +76,7 @@ using ReservedMemoryT = ReservedMemoryDefault; using MemMapT = ReservedMemoryT::MemMapT; #elif SCUDO_FUCHSIA -using ReservedMemoryT = ReservedMemoryDefault; +using ReservedMemoryT = ReservedMemoryFuchsia; using MemMapT = ReservedMemoryT::MemMapT; #elif SCUDO_TRUSTY using ReservedMemoryT = ReservedMemoryDefault; diff --git a/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.h b/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.h new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.h @@ -0,0 +1,75 @@ +//===-- mem_map_fuchsia.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SCUDO_MEM_MAP_FUCHSIA_H_ +#define SCUDO_MEM_MAP_FUCHSIA_H_ + +#include "mem_map_base.h" + +#if SCUDO_FUCHSIA + +#include +#include + +namespace scudo { + +class MemMapFuchsia final : public MemMapBase { +public: + constexpr MemMapFuchsia() = default; + + // Impls for base functions. + bool mapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags); + void unmapImpl(uptr Addr, uptr Size); + bool remapImpl(uptr Addr, uptr Size, const char *Name, uptr Flags); + void setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags); + void releasePagesToOSImpl(uptr From, uptr Size) { + return releaseAndZeroPagesToOSImpl(From, Size); + } + void releaseAndZeroPagesToOSImpl(uptr From, uptr Size); + uptr getBaseImpl() { return WindowBase; } + uptr getCapacityImpl() { return WindowSize; } + +private: + friend class ReservedMemoryFuchsia; + + // Used by ReservedMemoryFuchsia::dispatch. + MemMapFuchsia(uptr Base, uptr Capacity); + + // Virtual memory address corresponding to VMO offset 0. + uptr MapAddr = 0; + + // Virtual memory base address and size of the VMO subrange that is still in + // use. unmapImpl() can shrink this range, either at the beginning or at the + // end. + uptr WindowBase = 0; + uptr WindowSize = 0; + + zx_handle_t Vmo = ZX_HANDLE_INVALID; +}; + +class ReservedMemoryFuchsia final + : public ReservedMemory { +public: + constexpr ReservedMemoryFuchsia() = default; + + bool createImpl(uptr Addr, uptr Size, const char *Name, uptr Flags); + void releaseImpl(); + MemMapT dispatchImpl(uptr Addr, uptr Size); + uptr getBaseImpl() { return Base; } + uptr getCapacityImpl() { return Capacity; } + +private: + uptr Base = 0; + uptr Capacity = 0; +}; + +} // namespace scudo + +#endif // SCUDO_FUCHSIA + +#endif // SCUDO_MEM_MAP_FUCHSIA_H_ diff --git a/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.cpp b/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.cpp new file mode 100644 --- /dev/null +++ b/compiler-rt/lib/scudo/standalone/mem_map_fuchsia.cpp @@ -0,0 +1,253 @@ +//===-- mem_map_fuchsia.cpp -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "mem_map_fuchsia.h" + +#include "atomic_helpers.h" +#include "common.h" +#include "string_utils.h" + +#if SCUDO_FUCHSIA + +#include +#include +#include + +namespace scudo { + +static void NORETURN dieOnError(zx_status_t Status, const char *FnName, + uptr Size) { + char Error[128]; + formatString(Error, sizeof(Error), + "SCUDO ERROR: %s failed with size %zuKB (%s)", FnName, + Size >> 10, _zx_status_get_string(Status)); + outputRaw(Error); + die(); +} + +static void setVmoName(zx_handle_t Vmo, const char *Name) { + size_t Len = strlen(Name); + DCHECK_LT(Len, ZX_MAX_NAME_LEN); + zx_status_t Status = _zx_object_set_property(Vmo, ZX_PROP_NAME, Name, Len); + CHECK_EQ(Status, ZX_OK); +} + +// Returns the (cached) base address of the root VMAR. +static uptr getRootVmarBase() { + static atomic_uptr CachedResult = {0}; + + uptr Result = atomic_load_relaxed(&CachedResult); + if (UNLIKELY(!Result)) { + zx_info_vmar_t VmarInfo; + zx_status_t Status = + _zx_object_get_info(_zx_vmar_root_self(), ZX_INFO_VMAR, &VmarInfo, + sizeof(VmarInfo), nullptr, nullptr); + CHECK_EQ(Status, ZX_OK); + CHECK_NE(VmarInfo.base, 0); + + atomic_store_relaxed(&CachedResult, VmarInfo.base); + Result = VmarInfo.base; + } + + return Result; +} + +// Lazily creates and then always returns the same zero-sized VMO. +static zx_handle_t getPlaceholderVmo() { + static atomic_u32 StoredVmo = {ZX_HANDLE_INVALID}; + + zx_handle_t Vmo = atomic_load_relaxed(&StoredVmo); + if (UNLIKELY(Vmo == ZX_HANDLE_INVALID)) { + // Create a zero-sized placeholder VMO. + zx_status_t Status = _zx_vmo_create(0, 0, &Vmo); + if (UNLIKELY(Status != ZX_OK)) + dieOnError(Status, "zx_vmo_create", 0); + + setVmoName(Vmo, "scudo:reserved"); + + // Atomically store its handle. If some other thread wins the race, use its + // handle and discard ours. + zx_handle_t OldValue = + atomic_compare_exchange(&StoredVmo, ZX_HANDLE_INVALID, Vmo); + if (OldValue != ZX_HANDLE_INVALID) { + Status = _zx_handle_close(Vmo); + CHECK_EQ(Status, ZX_OK); + + Vmo = OldValue; + } + } + + return Vmo; +} + +MemMapFuchsia::MemMapFuchsia(uptr Base, uptr Capacity) + : MapAddr(Base), WindowBase(Base), WindowSize(Capacity) { + // Create the VMO. + zx_status_t Status = _zx_vmo_create(Capacity, 0, &Vmo); + if (UNLIKELY(Status != ZX_OK)) + dieOnError(Status, "zx_vmo_create", Capacity); +} + +bool MemMapFuchsia::mapImpl(UNUSED uptr Addr, uptr Size, const char *Name, + uptr Flags) { + const bool AllowNoMem = !!(Flags & MAP_ALLOWNOMEM); + const bool PreCommit = !!(Flags & MAP_PRECOMMIT); + const bool NoAccess = !!(Flags & MAP_NOACCESS); + + // Create the VMO. + zx_status_t Status = _zx_vmo_create(Size, 0, &Vmo); + if (UNLIKELY(Status != ZX_OK)) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnError(Status, "zx_vmo_create", Size); + return false; + } + + if (Name != nullptr) + setVmoName(Vmo, Name); + + // Map it. + zx_vm_option_t MapFlags = ZX_VM_ALLOW_FAULTS; + if (!NoAccess) + MapFlags |= ZX_VM_PERM_READ | ZX_VM_PERM_WRITE; + Status = + _zx_vmar_map(_zx_vmar_root_self(), MapFlags, 0, Vmo, 0, Size, &MapAddr); + if (UNLIKELY(Status != ZX_OK)) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnError(Status, "zx_vmar_map", Size); + + Status = _zx_handle_close(Vmo); + CHECK_EQ(Status, ZX_OK); + + MapAddr = 0; + Vmo = ZX_HANDLE_INVALID; + return false; + } + + if (PreCommit) { + Status = _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_COMMIT, MapAddr, + Size, nullptr, 0); + CHECK_EQ(Status, ZX_OK); + } + + WindowBase = MapAddr; + WindowSize = Size; + return true; +} + +void MemMapFuchsia::unmapImpl(uptr Addr, uptr Size) { + zx_status_t Status; + + if (Size == WindowSize) { + // NOTE: Closing first and then unmapping seems slightly faster than doing + // the same operations in the opposite order. + Status = _zx_handle_close(Vmo); + CHECK_EQ(Status, ZX_OK); + Status = _zx_vmar_unmap(_zx_vmar_root_self(), Addr, Size); + CHECK_EQ(Status, ZX_OK); + + MapAddr = WindowBase = WindowSize = 0; + Vmo = ZX_HANDLE_INVALID; + } else { + // Unmap the subrange. + Status = _zx_vmar_unmap(_zx_vmar_root_self(), Addr, Size); + CHECK_EQ(Status, ZX_OK); + + // Decommit the pages that we just unmapped. + Status = _zx_vmo_op_range(Vmo, ZX_VMO_OP_DECOMMIT, Addr - MapAddr, Size, + nullptr, 0); + CHECK_EQ(Status, ZX_OK); + + if (Addr == WindowBase) + WindowBase += Size; + WindowSize -= Size; + } +} + +bool MemMapFuchsia::remapImpl(uptr Addr, uptr Size, const char *Name, + uptr Flags) { + const bool AllowNoMem = !!(Flags & MAP_ALLOWNOMEM); + const bool PreCommit = !!(Flags & MAP_PRECOMMIT); + const bool NoAccess = !!(Flags & MAP_NOACCESS); + + // NOTE: This will rename the *whole* VMO, not only the requested portion of + // it. But we cannot do better than this given the MemMap API. In practice, + // the upper layers of Scudo always pass the same Name for a given MemMap. + if (Name != nullptr) + setVmoName(Vmo, Name); + + uptr MappedAddr; + zx_vm_option_t MapFlags = ZX_VM_ALLOW_FAULTS | ZX_VM_SPECIFIC_OVERWRITE; + if (!NoAccess) + MapFlags |= ZX_VM_PERM_READ | ZX_VM_PERM_WRITE; + zx_status_t Status = + _zx_vmar_map(_zx_vmar_root_self(), MapFlags, Addr - getRootVmarBase(), + Vmo, Addr - MapAddr, Size, &MappedAddr); + if (UNLIKELY(Status != ZX_OK)) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnError(Status, "zx_vmar_map", Size); + return false; + } + DCHECK_EQ(Addr, MappedAddr); + + if (PreCommit) { + Status = _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_COMMIT, MapAddr, + Size, nullptr, 0); + CHECK_EQ(Status, ZX_OK); + } + + return true; +} + +void MemMapFuchsia::releaseAndZeroPagesToOSImpl(uptr From, uptr Size) { + zx_status_t Status = _zx_vmo_op_range(Vmo, ZX_VMO_OP_DECOMMIT, From - MapAddr, + Size, nullptr, 0); + CHECK_EQ(Status, ZX_OK); +} + +void MemMapFuchsia::setMemoryPermissionImpl(uptr Addr, uptr Size, uptr Flags) { + const bool NoAccess = !!(Flags & MAP_NOACCESS); + zx_vm_option_t MapFlags = 0; + if (!NoAccess) { + MapFlags |= ZX_VM_PERM_READ | ZX_VM_PERM_WRITE; + } + + zx_status_t Status = + _zx_vmar_protect(_zx_vmar_root_self(), MapFlags, Addr, Size); + CHECK_EQ(Status, ZX_OK); +} + +bool ReservedMemoryFuchsia::createImpl(UNUSED uptr Addr, uptr Size, + UNUSED const char *Name, uptr Flags) { + const bool AllowNoMem = !!(Flags & MAP_ALLOWNOMEM); + + // Reserve memory by mapping the placeholder VMO without any permission. + zx_status_t Status = _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_ALLOW_FAULTS, 0, + getPlaceholderVmo(), 0, Size, &Base); + if (UNLIKELY(Status != ZX_OK)) { + if (Status != ZX_ERR_NO_MEMORY || !AllowNoMem) + dieOnError(Status, "zx_vmar_map", Size); + return false; + } + + Capacity = Size; + return true; +} + +void ReservedMemoryFuchsia::releaseImpl() { + zx_status_t Status = _zx_vmar_unmap(_zx_vmar_root_self(), Base, Capacity); + CHECK_EQ(Status, ZX_OK); +} + +ReservedMemoryFuchsia::MemMapT ReservedMemoryFuchsia::dispatchImpl(uptr Addr, + uptr Size) { + return ReservedMemoryFuchsia::MemMapT(Addr, Size); +} + +} // namespace scudo + +#endif // SCUDO_FUCHSIA