From 68acd137e91c5e20f1e2d738f575497b824b558d Mon Sep 17 00:00:00 2001 From: pknowles Date: Thu, 8 May 2025 22:00:17 -0700 Subject: [PATCH 1/2] resizable_memory: use mprotect instead of mmap to resize on linux --- .../decodeless/detail/mappedfile_linux.hpp | 74 +++++-------------- 1 file changed, 19 insertions(+), 55 deletions(-) diff --git a/include/decodeless/detail/mappedfile_linux.hpp b/include/decodeless/detail/mappedfile_linux.hpp index 12b793c..441ce86 100644 --- a/include/decodeless/detail/mappedfile_linux.hpp +++ b/include/decodeless/detail/mappedfile_linux.hpp @@ -254,72 +254,36 @@ class ResizableMappedMemory { ResizableMappedMemory(ResizableMappedMemory&& other) noexcept = default; ResizableMappedMemory(size_t initialSize, size_t maxSize) : m_reserved(nullptr, maxSize, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0) { - if (initialSize) - map(initialSize); + resize(initialSize); } ResizableMappedMemory& operator=(const ResizableMappedMemory& other) = delete; void* data() const { return m_size ? m_reserved.address() : nullptr; } size_t size() const { return m_size; } size_t capacity() const { return m_reserved.size(); } void resize(size_t size) { - size = throwIfAbove(size, m_reserved.size()); - if (size) - map(size); + if (size > capacity()) + throw std::bad_alloc(); + + // Align to the next page boundary + size_t ps = pageSize(); + size_t newMappedSize = ((size + ps - 1) / ps) * ps; + + // Add/remove just the new range + if (newMappedSize > m_mappedSize) + protect(m_mappedSize, newMappedSize - m_mappedSize, PROT_READ | PROT_WRITE); + else if (newMappedSize < m_mappedSize) + protect(newMappedSize, m_mappedSize - newMappedSize, PROT_NONE); + m_mappedSize = newMappedSize; + m_size = size; } ResizableMappedMemory& operator=(ResizableMappedMemory&& other) noexcept = default; private: - void map(size_t size) { - // TODO: if m_mapped shrinks, does m_reserved instead need to be - // recreated to fill the gap? - size_t ps = pageSize(); - size_t mappedSize = ((size + ps - 1) / ps) * ps; // TODO: cache? - if (mappedSize == 0) { - // Cannot move, as it's reserved, and we throw if we try to exceed. - if (mmap(const_cast(m_reserved.address()), mappedSize, PROT_READ | PROT_WRITE, - MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) == MAP_FAILED) { - throw LastError(); - } - } else { -#if 1 - // Map any additional pages needed. Don't remap existing pages or - // they get zeroed. Another idea might be a memory fd. - if (mappedSize > m_mappedSize) { - // Cannot move, as it's reserved, and we throw if we try to exceed. - if (mmap(reinterpret_cast(uintptr_t(m_reserved.address()) + m_mappedSize), - mappedSize - m_mappedSize, PROT_READ | PROT_WRITE, - MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0) == MAP_FAILED) - throw LastError(); - m_mappedSize = mappedSize; - } -#else - #if 1 - // Remap? Specs say old and new addresses cannot be the same, - // although it only errors out when remapping more than a page. - void* addr = - mremap(m_reserved.address(), m_size, size, - MREMAP_MAYMOVE | MREMAP_FIXED | MREMAP_DONTUNMAP, m_reserved.address()); - #else - // Remap without MREMAP_FIXED - immediately fails because it can't - // replace the reserved mapping. - void* addr = mremap(m_reserved.address(), m_size, size, 0); - #endif - if (addr == MAP_FAILED) { - throw LastError(); - } else if (addr != m_reserved.address()) { - // Unrecoverable - fprintf(stderr, "fatal: mremap() moved the mapping"); - std::terminate(); - } -#endif - } - m_size = size; - } - static size_t throwIfAbove(size_t v, size_t limit) { - if (v > limit) - throw std::bad_alloc(); - return v; + void protect(size_t offset, size_t size, int prot) { + if (mprotect(reinterpret_cast(uintptr_t(m_reserved.address()) + offset), size, + prot) != 0) + throw LastError(); } detail::MemoryMap m_reserved; size_t m_size = 0; From 977ad2a75c2358ba36c98a4a32fb06dc7197e228 Mon Sep 17 00:00:00 2001 From: pknowles Date: Sun, 3 Aug 2025 17:19:10 -0700 Subject: [PATCH 2/2] resizable_memory: release pages with MADV_DONTNEED on linux --- .../decodeless/detail/mappedfile_linux.hpp | 40 +++++++----- test/src/mappedfile.cpp | 62 ++++++++++++++++++- 2 files changed, 84 insertions(+), 18 deletions(-) diff --git a/include/decodeless/detail/mappedfile_linux.hpp b/include/decodeless/detail/mappedfile_linux.hpp index 441ce86..f3f2a3c 100644 --- a/include/decodeless/detail/mappedfile_linux.hpp +++ b/include/decodeless/detail/mappedfile_linux.hpp @@ -88,6 +88,7 @@ class MemoryMap { static constexpr bool ProtNone = MemoryProtection == PROT_NONE; static constexpr bool Writable = (MemoryProtection & PROT_WRITE) != 0; using address_type = std::conditional_t; + using byte_type = std::conditional_t; MemoryMap(address_type addr, size_t length, int flags, int fd, off_t offset) : m_size(length) , m_address(mmap(const_cast(addr), length, MemoryProtection, flags, fd, offset)) @@ -123,6 +124,9 @@ class MemoryMap { unmap(); } address_type address() const { return m_address; } + address_type address(size_t offset) const { + return static_cast(static_cast(m_address) + offset); + } size_t size() const { return m_size; } void sync(int flags = MS_SYNC | MS_INVALIDATE) requires Writable @@ -135,12 +139,7 @@ class MemoryMap { throw LastError(); } void resize(size_t size) { -#if 0 - void* addr = mremap(m_address, m_size, size, - MREMAP_MAYMOVE | MREMAP_FIXED | MREMAP_DONTUNMAP, m_address); -#else void* addr = mremap(m_address, m_size, size, 0); -#endif if (addr == MAP_FAILED) { throw LastError(); } else if (addr != m_address) { @@ -268,11 +267,27 @@ class ResizableMappedMemory { size_t ps = pageSize(); size_t newMappedSize = ((size + ps - 1) / ps) * ps; - // Add/remove just the new range - if (newMappedSize > m_mappedSize) - protect(m_mappedSize, newMappedSize - m_mappedSize, PROT_READ | PROT_WRITE); - else if (newMappedSize < m_mappedSize) - protect(newMappedSize, m_mappedSize - newMappedSize, PROT_NONE); + if (newMappedSize > m_mappedSize) { + // Add just the new range + if (mprotect(m_reserved.address(m_mappedSize), newMappedSize - m_mappedSize, + PROT_READ | PROT_WRITE) != 0) + throw LastError(); + } else if (newMappedSize < m_mappedSize) { + // Release unused range to the OS. mprotect() will not do this. + // Using MADV_DONTNEED is many times faster than mmap(). +#if 1 + if (mprotect(m_reserved.address(newMappedSize), m_mappedSize - newMappedSize, + PROT_NONE) != 0) + throw LastError(); + if (madvise(m_reserved.address(newMappedSize), m_mappedSize - newMappedSize, + MADV_DONTNEED) != 0) + throw LastError(); +#else + if (mmap(m_reserved.address(newMappedSize), m_mappedSize - newMappedSize, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED, -1, 0) == MAP_FAILED) + throw LastError(); +#endif + } m_mappedSize = newMappedSize; m_size = size; } @@ -280,11 +295,6 @@ class ResizableMappedMemory { ResizableMappedMemory& operator=(ResizableMappedMemory&& other) noexcept = default; private: - void protect(size_t offset, size_t size, int prot) { - if (mprotect(reinterpret_cast(uintptr_t(m_reserved.address()) + offset), size, - prot) != 0) - throw LastError(); - } detail::MemoryMap m_reserved; size_t m_size = 0; size_t m_mappedSize = 0; diff --git a/test/src/mappedfile.cpp b/test/src/mappedfile.cpp index 5c7619e..c5871e9 100644 --- a/test/src/mappedfile.cpp +++ b/test/src/mappedfile.cpp @@ -1,10 +1,11 @@ // Copyright (c) 2024 Pyarelal Knowles, MIT License +#include #include +#include #include #include #include -#include #include #include @@ -226,7 +227,7 @@ TEST_F(MappedFileFixture, LinuxResize) { #endif -TEST_F(MappedFileFixture, ResizeMemory) { +TEST(MappedMemory, ResizeMemory) { const char str[] = "hello world!"; { resizable_memory file(0, 10000); @@ -270,7 +271,7 @@ TEST_F(MappedFileFixture, ResizeMemory) { } } -TEST_F(MappedFileFixture, ResizeMemoryExtended) { +TEST(MappedMemory, ResizeMemoryExtended) { size_t nextBytes = 1; resizable_memory memory(nextBytes, 1llu << 32); // 4gb of virtual memory pls void* data = memory.data(); @@ -425,3 +426,58 @@ TEST_F(MappedFileFixture, Readme) { fs::remove(tmpFile2); EXPECT_FALSE(fs::exists(tmpFile2)); } + +#ifndef _WIN32 +std::vector getResidency(void* base, size_t size) { + std::vector result(size / getpagesize(), 0u); + int ret = mincore(base, size, result.data()); + if (ret != 0) + throw detail::LastError(); + return result; +} + +TEST(MappedMemory, PageResidencyAfterDecommit) { + const size_t page_size = getpagesize(); + const size_t reserve_size = page_size * 64; // 64 pages total + const size_t commit_size = page_size * 4; // We'll use 4 pages + + // Reserve virtual address space (uncommitted, inaccessible) + void* base = + mmap(nullptr, reserve_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + ASSERT_NE(base, MAP_FAILED) << "Failed to mmap reserved space"; + EXPECT_TRUE(std::ranges::all_of(getResidency(base, commit_size), + [](uint8_t c) { return (c & 1u) == 0; })); + + // Commit a portion with PROT_READ | PROT_WRITE + int prot_result = mprotect(base, commit_size, PROT_READ | PROT_WRITE); + ASSERT_EQ(prot_result, 0) << "Failed to mprotect committed region"; + EXPECT_TRUE(std::ranges::all_of(getResidency(base, commit_size), + [](uint8_t c) { return (c & 1u) == 0; })); + + // Touch the memory to ensure it's backed by RAM + std::span committed(static_cast(base), commit_size); + std::ranges::fill(committed, std::byte(0xAB)); + + // Verify pages are resident using mincore + EXPECT_TRUE(std::ranges::all_of(getResidency(base, commit_size), + [](uint8_t c) { return (c & 1u) == 1; })); + + // Decommit + #if 0 + void* remap = mmap(base, commit_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED, -1, 0); + ASSERT_EQ(remap, base) << "Failed to remap to decommit pages"; + #else + // See MADV_FREE discussion here: https://github.com/golang/go/issues/42330 + prot_result = mprotect(base, commit_size, PROT_NONE); + ASSERT_EQ(prot_result, 0) << "Failed to mprotect committed region back to PROT_NONE"; + int madvise_result = madvise(base, commit_size, MADV_DONTNEED); + ASSERT_EQ(madvise_result, 0) << "Failed to release pages with madvise"; + #endif + EXPECT_TRUE(std::ranges::all_of(getResidency(base, commit_size), + [](uint8_t c) { return (c & 1u) == 0; })); + + // Cleanup + munmap(base, reserve_size); +} +#endif