From 27c56be1fa0c91bae2cbd08c58298c1246552e2f Mon Sep 17 00:00:00 2001 From: evoskuil Date: Wed, 1 Jul 2026 16:30:07 -0400 Subject: [PATCH 01/19] Fix disk full space handling across mmaps. --- .../database/impl/memory/mmap_private.ipp | 41 ++++++++++++------- include/bitcoin/database/memory/mmap.hpp | 11 ++++- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/include/bitcoin/database/impl/memory/mmap_private.ipp b/include/bitcoin/database/impl/memory/mmap_private.ipp index 104ba85e3..278ed8d76 100644 --- a/include/bitcoin/database/impl/memory/mmap_private.ipp +++ b/include/bitcoin/database/impl/memory/mmap_private.ipp @@ -32,6 +32,22 @@ namespace database { // ---------------------------------------------------------------------------- // private +TEMPLATE +template +size_t CLASS::space_one_(size_t rows) const NOEXCEPT +{ + return system::floored_subtract(to_capacity(to_bytes(rows)), + std::get(capacity_)); +} + +TEMPLATE +template +size_t CLASS::space_all_(size_t rows, + std::index_sequence) const NOEXCEPT +{ + return (space_one_(rows) + ...); +} + TEMPLATE template bool CLASS::map_all_(std::index_sequence) NOEXCEPT @@ -57,7 +73,8 @@ TEMPLATE template bool CLASS::remap_all_(size_t logical, std::index_sequence) NOEXCEPT { - return (remap_(to_bytes(logical)) && ...); + const auto space = space_all_(logical, std::index_sequence{}); + return (remap_(to_bytes(logical), space) && ...); } // mman wrappers, not thread safe. @@ -139,12 +156,11 @@ template bool CLASS::map_() NOEXCEPT { auto size = to_bytes(logical_); - - // Cannot map empty file, and want mininum capacity, so expand as required. - // disk_full: space is set but no code is set with false return. const auto minimum = to_bytes(minimum_); - if ((size < minimum) && !resize_((size = minimum))) - return false; + + if ((size < minimum) && !resize_((size = minimum), + space_one_(logical_))) + return false; memory_map_[Column] = system::pointer_cast( ::mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_SHARED, @@ -157,7 +173,7 @@ bool CLASS::map_() NOEXCEPT // Remapping has no effect on logical size, sets map_/capacity_. TEMPLATE template -bool CLASS::remap_(size_t size) NOEXCEPT +bool CLASS::remap_(size_t size, size_t space) NOEXCEPT { BC_ASSERT(size >= to_bytes(logical_)); @@ -170,16 +186,13 @@ bool CLASS::remap_(size_t size) NOEXCEPT if (!unmap_()) return false; - // disk_full: unmap(ok), resize(fail for space), map(ok), return false. - // disk_full: if second unmap fails then code is set, and false return. - if (!resize_(size)) + if (!resize_(size, space)) { /* bool */ map_(); return false; } #else - // disk_full: space is set but no code is set with false return. - if (!resize_(size)) + if (!resize_(size, space)) return false; #endif @@ -211,7 +224,7 @@ bool CLASS::remap_(size_t size) NOEXCEPT // disk_full: space is set but no code is set with false return. TEMPLATE template -bool CLASS::resize_(size_t size) NOEXCEPT +bool CLASS::resize_(size_t size, size_t space) NOEXCEPT { // Disk full detection, any other failure is an abort. #if !defined (WITHOUT_FALLOCATE) @@ -224,7 +237,7 @@ bool CLASS::resize_(size_t size) NOEXCEPT // Disk full is the only restartable store failure (leave mapped). if (errno == ENOSPC) { - set_disk_space(size - capacity_[Column]); + set_disk_space(space); return false; } diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index 2518a4652..a0aebc1fd 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -192,6 +192,13 @@ class mmap static constexpr auto fail = -1; using access = accessor; using sequence = std::make_index_sequence; + + // dispatch utils. + template + size_t space_one_(size_t rows) const NOEXCEPT; + template + size_t space_all_(size_t rows, + std::index_sequence) const NOEXCEPT; // mman dispatch, not thread safe. template @@ -211,9 +218,9 @@ class mmap template bool map_() NOEXCEPT; template - bool remap_(size_t size) NOEXCEPT; + bool remap_(size_t size, size_t space) NOEXCEPT; template - bool resize_(size_t size) NOEXCEPT; + bool resize_(size_t size, size_t space) NOEXCEPT; template bool finalize_(size_t size) NOEXCEPT; From ac1b6acbda526c139725da5c5567a882bba612ba Mon Sep 17 00:00:00 2001 From: evoskuil Date: Wed, 1 Jul 2026 16:30:25 -0400 Subject: [PATCH 02/19] Line length (style). --- include/bitcoin/database/impl/memory/mmap_private.ipp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/bitcoin/database/impl/memory/mmap_private.ipp b/include/bitcoin/database/impl/memory/mmap_private.ipp index 278ed8d76..3d97eeadb 100644 --- a/include/bitcoin/database/impl/memory/mmap_private.ipp +++ b/include/bitcoin/database/impl/memory/mmap_private.ipp @@ -206,7 +206,8 @@ bool CLASS::remap_(size_t size, size_t space) NOEXCEPT #elif defined(MREMAP_MAYMOVE) memory_map_[Column] = system::pointer_cast( - ::mremap(memory_map_[Column], capacity_[Column], size, MREMAP_MAYMOVE)); + ::mremap(memory_map_[Column], capacity_[Column], size, + MREMAP_MAYMOVE)); #else From edf9a2c0ea0563b272f43f3b6e537c3b81f17d82 Mon Sep 17 00:00:00 2001 From: evoskuil Date: Wed, 1 Jul 2026 19:04:06 -0400 Subject: [PATCH 03/19] Rename to_bytes() to to_width(). --- .../database/impl/memory/mmap_dispatch.ipp | 8 ++++---- .../database/impl/memory/mmap_private.ipp | 16 ++++++++-------- include/bitcoin/database/memory/mmap.hpp | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp index ac4356a17..ee546190b 100644 --- a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp +++ b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp @@ -58,8 +58,7 @@ memory_ptr CLASS::set_column(size_t offset, size_t size, return {}; // Fill new capacity as offset may not be at end due to expansion. - // Scalar-map path: column 0 in bytes (width 1, identity). - const auto first = to_bytes(logical_); + const auto first = to_width(logical_); std::fill_n( std::next(std::get(memory_map_), first), std::get(capacity_) - first, backfill); @@ -80,7 +79,7 @@ memory_ptr CLASS::get_column(size_t offset) const NOEXCEPT // increases. Close zeroizes capacity but file must be unloaded to do so. // Truncate can reduce logical, but capacity is not affected. It is always // safe to write past current logical within current capacity. - const auto allocated = to_bytes(size()); + const auto allocated = to_width(size()); // Takes a shared lock on remap_mutex_ until destruct, blocking remap. const auto ptr = std::make_shared(remap_mutex_); @@ -110,6 +109,7 @@ memory_ptr CLASS::capacity_column(size_t offset) const NOEXCEPT ptr->assign( std::next(std::get(memory_map_), offset), std::next(std::get(memory_map_), allocated)); + return ptr; } @@ -118,7 +118,7 @@ template > memory::iterator CLASS::raw_column(size_t offset) const NOEXCEPT { // Pointer is otherwise unguarded, not remap safe (use for table heads). - if (offset > to_bytes(size())) + if (offset > to_width(size())) return nullptr; return std::next(std::get(memory_map_), offset); diff --git a/include/bitcoin/database/impl/memory/mmap_private.ipp b/include/bitcoin/database/impl/memory/mmap_private.ipp index 3d97eeadb..bbe37c9d6 100644 --- a/include/bitcoin/database/impl/memory/mmap_private.ipp +++ b/include/bitcoin/database/impl/memory/mmap_private.ipp @@ -36,7 +36,7 @@ TEMPLATE template size_t CLASS::space_one_(size_t rows) const NOEXCEPT { - return system::floored_subtract(to_capacity(to_bytes(rows)), + return system::floored_subtract(to_capacity(to_width(rows)), std::get(capacity_)); } @@ -74,7 +74,7 @@ template bool CLASS::remap_all_(size_t logical, std::index_sequence) NOEXCEPT { const auto space = space_all_(logical, std::index_sequence{}); - return (remap_(to_bytes(logical), space) && ...); + return (remap_(to_width(logical), space) && ...); } // mman wrappers, not thread safe. @@ -90,7 +90,7 @@ bool CLASS::flush_() NOEXCEPT // unmap (and therefore msync) must be called before ftruncate. // "To flush all the dirty pages plus the metadata for the file and ensure // that they are physically written to disk..." - const auto size = to_bytes(logical_); + const auto size = to_width(logical_); const auto success = (::msync(memory_map_[Column], size, MS_SYNC) != fail) && (::fsync(opened_[Column]) != fail); @@ -122,7 +122,7 @@ TEMPLATE template bool CLASS::unmap_() NOEXCEPT { - const auto logical = to_bytes(logical_); + const auto logical = to_width(logical_); #if defined(HAVE_MSC) const auto success = @@ -155,8 +155,8 @@ TEMPLATE template bool CLASS::map_() NOEXCEPT { - auto size = to_bytes(logical_); - const auto minimum = to_bytes(minimum_); + auto size = to_width(logical_); + const auto minimum = to_width(minimum_); if ((size < minimum) && !resize_((size = minimum), space_one_(logical_))) @@ -175,11 +175,11 @@ TEMPLATE template bool CLASS::remap_(size_t size, size_t space) NOEXCEPT { - BC_ASSERT(size >= to_bytes(logical_)); + BC_ASSERT(size >= to_width(logical_)); // Cannot remap empty file, so expand to minimum capacity if zero. if (is_zero(size)) - size = to_bytes(minimum_); + size = to_width(minimum_); #if !defined(HAVE_MSC) && !defined(MREMAP_MAYMOVE) // macOS: unmap before ftruncate sets new size. diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index a0aebc1fd..49c6547f9 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -167,7 +167,7 @@ class mmap protected: /// Row<->byte transpose by the constexpr column width (folds for width 1). template - static constexpr size_t to_bytes(size_t offset) NOEXCEPT + static constexpr size_t to_width(size_t offset) NOEXCEPT { return offset * std::get(widths); } @@ -235,7 +235,7 @@ class mmap // These are protected by field_mutex_. // Shared allocation state and per-column file/capacity state. // Fields require field_mutex_ exclusive for write, shared for flush/read. - // logical_ is the shared row count across all columns. + // logical_ is the shared row count across all columns (bytes for width 1). std::array opened_; sizes capacity_{}; size_t logical_{}; From 0a658bd77c24be0113e3ecf8da8a231637ce6fd2 Mon Sep 17 00:00:00 2001 From: evoskuil Date: Wed, 1 Jul 2026 22:55:15 -0400 Subject: [PATCH 04/19] Style, comments. --- include/bitcoin/database/impl/primitives/nomap.ipp | 1 + include/bitcoin/database/memory/mmap.hpp | 11 ++++------- include/bitcoin/database/primitives/arraymap.hpp | 2 +- include/bitcoin/database/primitives/nomap.hpp | 2 +- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/include/bitcoin/database/impl/primitives/nomap.ipp b/include/bitcoin/database/impl/primitives/nomap.ipp index f020c9415..77041d5ce 100644 --- a/include/bitcoin/database/impl/primitives/nomap.ipp +++ b/include/bitcoin/database/impl/primitives/nomap.ipp @@ -242,6 +242,7 @@ inline Link CLASS::put_link(const Element& element) NOEXCEPT return put_link(link, element) ? link : Link{}; } +/// NOT THREAD SAFE (used only for height index with writer ordering). TEMPLATE template > inline bool CLASS::commit(const Element& element) NOEXCEPT diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index 49c6547f9..fc9636c04 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -76,14 +76,12 @@ class mmap /// ----------------------------------------------------------------------- /// Scalar construction (columns == 1): unchanged signature and codegen. - mmap(const std::filesystem::path& filename, size_t minimum=1, - size_t expansion=0, bool random=true) NOEXCEPT - requires (columns == one); + mmap(const path& filename, size_t minimum=1, size_t expansion=0, + bool random=true) NOEXCEPT requires (columns == one); /// Aggregate construction (columns > 1): one file per column, shared guards. mmap(const paths& filenames, size_t minimum=1, size_t expansion=0, - bool random=true) NOEXCEPT - requires (columns > one); + bool random=true) NOEXCEPT requires (columns > one); /// Destruct for debug assertion only. virtual ~mmap() NOEXCEPT; @@ -248,8 +246,7 @@ class mmap mutable std::shared_mutex remap_mutex_{}; }; -/// Scalar map: single column, width 1. Source and codegen identical to the -/// prior non-aggregate map; existing usage binds the unchanged constructor. +// TODO: rename mmap<> to mmaps<> and map to mmap. using map = mmap<1>; } // namespace database diff --git a/include/bitcoin/database/primitives/arraymap.hpp b/include/bitcoin/database/primitives/arraymap.hpp index dae69115d..303b3083f 100644 --- a/include/bitcoin/database/primitives/arraymap.hpp +++ b/include/bitcoin/database/primitives/arraymap.hpp @@ -115,7 +115,7 @@ class arraymap private: static constexpr auto is_slab = (RowSize == max_size_t); using head = database::arrayhead; - using body = database::manager, RowSize>; + using body = database::manager, RowSize>; // Thread safe (index/top/push). // Not thread safe (create/open/close/backup/restore). diff --git a/include/bitcoin/database/primitives/nomap.hpp b/include/bitcoin/database/primitives/nomap.hpp index 8eb8a4492..05fd50bcc 100644 --- a/include/bitcoin/database/primitives/nomap.hpp +++ b/include/bitcoin/database/primitives/nomap.hpp @@ -128,7 +128,7 @@ class nomap template = true> inline Link put_link(const Element& element) NOEXCEPT; - /// NOT THREAD SAFE + /// NOT THREAD SAFE (used only for height index with writer ordering). /// Write element to reserved next position and commit to logical space. template = true> inline bool commit(const Element& element) NOEXCEPT; From 0cdfb5dd97d4ed157a18aed8cce38d1b3e302aeb Mon Sep 17 00:00:00 2001 From: evoskuil Date: Wed, 1 Jul 2026 22:57:30 -0400 Subject: [PATCH 05/19] Add get_at(col) to storage interface, simplify mmap. --- .../database/impl/memory/mmap_dispatch.ipp | 98 +++++++++++-------- .../database/impl/memory/mmap_storage.ipp | 27 ----- .../database/memory/interfaces/storage.hpp | 19 ++-- include/bitcoin/database/memory/mmap.hpp | 30 ++---- test/mocks/chunk_storage.cpp | 52 +++++----- test/mocks/chunk_storage.hpp | 13 ++- 6 files changed, 116 insertions(+), 123 deletions(-) diff --git a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp index ee546190b..bb1b166b8 100644 --- a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp +++ b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp @@ -20,7 +20,6 @@ #define LIBBITCOIN_DATABASE_MEMORY_MMAP_DISPATCH_IPP #include -#include #include #include #include @@ -35,8 +34,33 @@ namespace database { // ---------------------------------------------------------------------------- TEMPLATE -template > -memory_ptr CLASS::set_column(size_t offset, size_t size, +memory_ptr CLASS::get_capacity(size_t offset) const NOEXCEPT +{ + // Same as get() but limited by capacity() vs. size(). + const auto allocated = std::get(capacity_); + const auto ptr = std::make_shared(remap_mutex_); + if (!loaded_ || is_null(ptr)) + return nullptr; + + ptr->assign( + std::next(std::get(memory_map_), offset), + std::next(std::get(memory_map_), allocated)); + + return ptr; +} + +TEMPLATE +memory::iterator CLASS::get_raw(size_t offset) const NOEXCEPT +{ + // Pointer is otherwise unguarded, not remap safe (use for table heads). + if (offset > to_width(size())) + return nullptr; + + return std::next(std::get(memory_map_), offset); +} + +TEMPLATE +memory_ptr CLASS::set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT { { @@ -58,70 +82,62 @@ memory_ptr CLASS::set_column(size_t offset, size_t size, return {}; // Fill new capacity as offset may not be at end due to expansion. - const auto first = to_width(logical_); - std::fill_n( - std::next(std::get(memory_map_), first), - std::get(capacity_) - first, backfill); + const auto first = to_width(logical_); + std::fill_n(std::next(std::get(memory_map_), first), + std::get(capacity_) - first, backfill); } logical_ = end; } - return get_column(offset); + return get(offset); } +// This avoids forwarding to get_at() for the tiny runtime performance benefit. TEMPLATE -template > -memory_ptr CLASS::get_column(size_t offset) const NOEXCEPT +memory_ptr CLASS::get(size_t offset) const NOEXCEPT { - // Obtaining size before access prevents mutual mutex wait (deadlock). - // The store could remap between here and next line, but capacity only - // increases. Close zeroizes capacity but file must be unloaded to do so. - // Truncate can reduce logical, but capacity is not affected. It is always - // safe to write past current logical within current capacity. - const auto allocated = to_width(size()); - - // Takes a shared lock on remap_mutex_ until destruct, blocking remap. + const auto allocated = to_width(size()); const auto ptr = std::make_shared(remap_mutex_); - - // loaded_ update is precluded by remap_mutex_, making this read atomic. if (!loaded_ || is_null(ptr)) - return nullptr; + return {}; - // With offset > size the assignment is negative (stream is exhausted). ptr->assign( - std::next(std::get(memory_map_), offset), - std::next(std::get(memory_map_), allocated)); + std::next(std::get(memory_map_), offset), + std::next(std::get(memory_map_), allocated)); return ptr; } TEMPLATE -template > -memory_ptr CLASS::capacity_column(size_t offset) const NOEXCEPT +memory_ptr CLASS::get_at(size_t column, size_t offset) const NOEXCEPT { - // Same as get() but limited by capacity() vs. size(). - const auto allocated = std::get(capacity_); + BC_PUSH_WARNING(NO_ARRAY_INDEXING) + + // Invalid column yields null (bounds check on runtime column index). + if (column >= columns) + return {}; + + // Obtaining size before access prevents mutual mutex wait (deadlock). + // Capacity only increases; safe to bound by logical (rows) transposed to + // this column's bytes. widths_[column] is the runtime column stride. + const auto allocated = size() * widths[column]; + + // Takes a shared lock on remap_mutex_ until destruct, blocking remap. const auto ptr = std::make_shared(remap_mutex_); + + // loaded_ update is precluded by remap_mutex_, making this read atomic. if (!loaded_ || is_null(ptr)) - return nullptr; + return {}; + // With offset > size the assignment is negative (stream is exhausted). ptr->assign( - std::next(std::get(memory_map_), offset), - std::next(std::get(memory_map_), allocated)); + std::next(memory_map_[column], offset), + std::next(memory_map_[column], allocated)); return ptr; -} - -TEMPLATE -template > -memory::iterator CLASS::raw_column(size_t offset) const NOEXCEPT -{ - // Pointer is otherwise unguarded, not remap safe (use for table heads). - if (offset > to_width(size())) - return nullptr; - return std::next(std::get(memory_map_), offset); + BC_POP_WARNING() } } // namespace database diff --git a/include/bitcoin/database/impl/memory/mmap_storage.ipp b/include/bitcoin/database/impl/memory/mmap_storage.ipp index dadfdefae..d2de6ad9e 100644 --- a/include/bitcoin/database/impl/memory/mmap_storage.ipp +++ b/include/bitcoin/database/impl/memory/mmap_storage.ipp @@ -364,33 +364,6 @@ size_t CLASS::allocate(size_t chunk) NOEXCEPT return end; } -// dispatch -// ---------------------------------------------------------------------------- - -TEMPLATE -memory_ptr CLASS::set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT -{ - return set_column(offset, size, backfill); -} - -TEMPLATE -memory_ptr CLASS::get(size_t offset) const NOEXCEPT -{ - return get_column(offset); -} - -TEMPLATE -memory_ptr CLASS::get_capacity(size_t offset) const NOEXCEPT -{ - return capacity_column(offset); -} - -TEMPLATE -memory::iterator CLASS::get_raw(size_t offset) const NOEXCEPT -{ - return raw_column(offset); -} - } // namespace database } // namespace libbitcoin diff --git a/include/bitcoin/database/memory/interfaces/storage.hpp b/include/bitcoin/database/memory/interfaces/storage.hpp index a3044e00b..cdbcb1cb5 100644 --- a/include/bitcoin/database/memory/interfaces/storage.hpp +++ b/include/bitcoin/database/memory/interfaces/storage.hpp @@ -86,6 +86,14 @@ class storage /// Increase logical by specified bytes, return offset to first (or eof). virtual size_t allocate(size_t chunk) NOEXCEPT = 0; + /// Get remap-protected r/w access to start/offset of memory map (or null). + /// Pointer is constrained to starting write within full capacity. + virtual memory_ptr get_capacity(size_t offset=zero) const NOEXCEPT = 0; + + /// Get unprotected r/w access to start/offset of memory map (or null). + /// Pointer is constrained to starting write within full capacity. + virtual memory::iterator get_raw(size_t offset=zero) const NOEXCEPT = 0; + /// Get remap-protected r/w access to offset (or null) allocated to size. virtual memory_ptr set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT = 0; @@ -94,13 +102,10 @@ class storage /// Pointer is constrained to starting write within logical allocation. virtual memory_ptr get(size_t offset=zero) const NOEXCEPT = 0; - /// Get remap-protected r/w access to start/offset of memory map (or null). - /// Pointer is constrained to starting write within full capacity. - virtual memory_ptr get_capacity(size_t offset=zero) const NOEXCEPT = 0; - - /// Get unprotected r/w access to start/offset of memory map (or null). - /// Pointer is constrained to starting write within full capacity. - virtual memory::iterator get_raw(size_t offset=zero) const NOEXCEPT = 0; + /// Same as get() but within specified column (or null for invalid column). + /// Pointer is constrained to starting write within logical allocation. + virtual memory_ptr get_at(size_t column, + size_t offset=zero) const NOEXCEPT = 0; }; } // namespace database diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index fc9636c04..a9134a377 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -56,22 +56,6 @@ class mmap /// Bytes per logical row across the aggregate (sum of column widths). static constexpr size_t stride = (Widths + ...); - /// Dispatch. - /// ----------------------------------------------------------------------- - - template = true> - memory_ptr set_column(size_t offset, size_t size, - uint8_t backfill) NOEXCEPT; - - template = true> - memory_ptr get_column(size_t offset=zero) const NOEXCEPT; - - template = true> - memory_ptr capacity_column(size_t offset=zero) const NOEXCEPT; - - template = true> - memory::iterator raw_column(size_t offset=zero) const NOEXCEPT; - /// Constructors. /// ----------------------------------------------------------------------- @@ -149,6 +133,12 @@ class mmap /// Increase logical by specified, return offset to first (or eof). size_t allocate(size_t chunk) NOEXCEPT override; + /// Remap-protected r/w access to start/offset (or null), within capacity. + memory_ptr get_capacity(size_t offset=zero) const NOEXCEPT override; + + /// Unprotected r/w access to start/offset (or null), within logical. + memory::iterator get_raw(size_t offset=zero) const NOEXCEPT override; + /// Remap-protected r/w access to offset (or null) allocated to size. memory_ptr set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT override; @@ -156,11 +146,9 @@ class mmap /// Remap-protected r/w access to start/offset (or null), within logical. memory_ptr get(size_t offset=zero) const NOEXCEPT override; - /// Remap-protected r/w access to start/offset (or null), within capacity. - memory_ptr get_capacity(size_t offset=zero) const NOEXCEPT override; - - /// Unprotected r/w access to start/offset (or null), within logical. - memory::iterator get_raw(size_t offset=zero) const NOEXCEPT override; + /// Same as get() but within specified column (or null for invalid column). + memory_ptr get_at(size_t column, + size_t offset=zero) const NOEXCEPT override; protected: /// Row<->byte transpose by the constexpr column width (folds for width 1). diff --git a/test/mocks/chunk_storage.cpp b/test/mocks/chunk_storage.cpp index ceaeb092e..0dc8634ef 100644 --- a/test/mocks/chunk_storage.cpp +++ b/test/mocks/chunk_storage.cpp @@ -45,6 +45,16 @@ chunk_storage::chunk_storage(const std::filesystem::path& filename, { } +code chunk_storage::get_fault() const NOEXCEPT +{ + return {}; +} + +size_t chunk_storage::get_space() const NOEXCEPT +{ + return {}; +} + system::data_chunk& chunk_storage::buffer() NOEXCEPT { return buffer_; @@ -177,6 +187,21 @@ size_t chunk_storage::allocate(size_t chunk) NOEXCEPT return link; } +memory_ptr chunk_storage::get_capacity(size_t offset) const NOEXCEPT +{ + const auto ptr = std::make_shared>(map_mutex_); + + // With offset > capacity the assignment is negative (stream is exhausted). + ptr->assign(get_raw(offset), get_raw(capacity())); + + return ptr; +} + +memory::iterator chunk_storage::get_raw(size_t offset) const NOEXCEPT +{ + return std::next(buffer_.data(), offset); +} + memory_ptr chunk_storage::set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT { @@ -214,29 +239,12 @@ memory_ptr chunk_storage::get(size_t offset) const NOEXCEPT return ptr; } -memory_ptr chunk_storage::get_capacity(size_t offset) const NOEXCEPT -{ - const auto ptr = std::make_shared>(map_mutex_); - - // With offset > capacity the assignment is negative (stream is exhausted). - ptr->assign(get_raw(offset), get_raw(capacity())); - - return ptr; -} - -memory::iterator chunk_storage::get_raw(size_t offset) const NOEXCEPT -{ - return std::next(buffer_.data(), offset); -} - -code chunk_storage::get_fault() const NOEXCEPT -{ - return {}; -} - -size_t chunk_storage::get_space() const NOEXCEPT +// TODO: extend support for multiple column SoA tables. +memory_ptr chunk_storage::get_at(size_t BC_DEBUG_ONLY(column), + size_t offset) const NOEXCEPT { - return {}; + BC_ASSERT(is_zero(column)); + return get(offset); } BC_POP_WARNING() diff --git a/test/mocks/chunk_storage.hpp b/test/mocks/chunk_storage.hpp index f315e4db7..e034b4d9c 100644 --- a/test/mocks/chunk_storage.hpp +++ b/test/mocks/chunk_storage.hpp @@ -54,6 +54,8 @@ class chunk_storage system::data_chunk& buffer() NOEXCEPT; // storage interface. + code get_fault() const NOEXCEPT override; + size_t get_space() const NOEXCEPT override; code create() const NOEXCEPT override; code open() NOEXCEPT override; code close() NOEXCEPT override; @@ -70,12 +72,13 @@ class chunk_storage bool expand(size_t size) NOEXCEPT override; bool reserve(size_t chunk) NOEXCEPT override; size_t allocate(size_t chunk) NOEXCEPT override; - memory_ptr set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT override; + memory_ptr get_capacity(size_t offset = zero) const NOEXCEPT override; + memory::iterator get_raw(size_t offset = zero) const NOEXCEPT override; + memory_ptr set(size_t offset, size_t size, + uint8_t backfill) NOEXCEPT override; memory_ptr get(size_t offset=zero) const NOEXCEPT override; - memory_ptr get_capacity(size_t offset=zero) const NOEXCEPT override; - memory::iterator get_raw(size_t offset=zero) const NOEXCEPT override; - code get_fault() const NOEXCEPT override; - size_t get_space() const NOEXCEPT override; + memory_ptr get_at(size_t column, + size_t offset=zero) const NOEXCEPT override; private: // These are protected by mutex. From a8c89d73b254dda7b83840ad604dd40eebb06b2a Mon Sep 17 00:00:00 2001 From: evoskuil Date: Sun, 5 Jul 2026 13:03:21 -0400 Subject: [PATCH 06/19] Integrate SoA single lock tables. --- .../libbitcoin-database-test.vcxproj | 1 - .../libbitcoin-database-test.vcxproj.filters | 3 - .../libbitcoin-database.vcxproj | 4 +- .../libbitcoin-database.vcxproj.filters | 12 +- .../libbitcoin-database-test.vcxproj | 1 - .../libbitcoin-database-test.vcxproj.filters | 3 - .../libbitcoin-database.vcxproj | 4 +- .../libbitcoin-database.vcxproj.filters | 12 +- include/bitcoin/database.hpp | 3 +- include/bitcoin/database/impl/memory/mmap.ipp | 35 +- .../database/impl/memory/mmap_dispatch.ipp | 53 ++- .../database/impl/memory/mmap_private.ipp | 53 +-- .../database/impl/memory/mmap_storage.ipp | 35 +- .../database/impl/primitives/arraymap.ipp | 8 +- .../database/impl/primitives/hashmap.ipp | 8 +- .../database/impl/primitives/manager.ipp | 194 +++++++---- .../database/impl/primitives/nomap.ipp | 47 +-- .../database/impl/primitives/nomaps.ipp | 198 +++++++++++ .../database/impl/query/batch/ecdsa.ipp | 38 +-- .../database/impl/query/batch/schnorr.ipp | 32 +- .../database/impl/query/batch/silent.ipp | 18 +- .../database/memory/interfaces/storage.hpp | 26 +- include/bitcoin/database/memory/maps.hpp | 135 -------- include/bitcoin/database/memory/memory.hpp | 2 +- include/bitcoin/database/memory/mmap.hpp | 54 ++- include/bitcoin/database/memory/mmaps.hpp | 88 +++++ .../bitcoin/database/primitives/arraymap.hpp | 6 +- .../bitcoin/database/primitives/column.hpp | 76 +++++ .../bitcoin/database/primitives/hashhead.hpp | 2 +- .../bitcoin/database/primitives/hashmap.hpp | 6 +- .../bitcoin/database/primitives/manager.hpp | 99 +++--- include/bitcoin/database/primitives/nomap.hpp | 34 +- .../bitcoin/database/primitives/nomaps.hpp | 159 +++++---- .../database/primitives/primitives.hpp | 1 + include/bitcoin/database/store.hpp | 88 ++--- .../bitcoin/database/tables/caches/ecdsa.hpp | 84 ++--- .../database/tables/caches/schnorr.hpp | 76 +---- .../bitcoin/database/tables/caches/silent.hpp | 37 +-- test/memory/mmap.cpp | 2 +- test/mocks/chunk_storage.cpp | 252 -------------- test/mocks/chunk_storage.hpp | 312 +++++++++++++++--- test/mocks/chunk_store.hpp | 146 +++----- test/mocks/map_store.hpp | 146 +++----- test/primitives/arrayhead.cpp | 6 +- test/primitives/hashhead.cpp | 6 +- test/primitives/nohead.cpp | 6 +- test/query/archive/chain_writer.cpp | 4 +- test/store/store.cpp | 24 +- test/store/store_close.cpp | 4 +- test/store/store_open.cpp | 4 +- test/store/store_prune.cpp | 4 +- test/store/store_snapshot.cpp | 4 +- test/tables/caches/ecdsa.cpp | 21 +- test/tables/caches/schnorr.cpp | 21 +- test/tables/caches/silent.cpp | 21 +- 55 files changed, 1404 insertions(+), 1314 deletions(-) create mode 100644 include/bitcoin/database/impl/primitives/nomaps.ipp delete mode 100644 include/bitcoin/database/memory/maps.hpp create mode 100644 include/bitcoin/database/memory/mmaps.hpp create mode 100644 include/bitcoin/database/primitives/column.hpp delete mode 100644 test/mocks/chunk_storage.cpp diff --git a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj index 0c048ca25..1fa17717a 100644 --- a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj +++ b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj @@ -133,7 +133,6 @@ $(IntDir)test_memory_utilities.obj - diff --git a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters index 7696c6c34..7fa9e4d5d 100644 --- a/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters +++ b/builds/msvc/vs2022/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters @@ -99,9 +99,6 @@ src\mocks - - src\mocks - src\primitives diff --git a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj index 581c581de..975320042 100644 --- a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj +++ b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj @@ -155,15 +155,16 @@ - + + @@ -236,6 +237,7 @@ + diff --git a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters index b8cac1d60..e6d4b5e27 100644 --- a/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters +++ b/builds/msvc/vs2022/libbitcoin-database/libbitcoin-database.vcxproj.filters @@ -182,9 +182,6 @@ include\bitcoin\database\memory\interfaces - - include\bitcoin\database\memory - include\bitcoin\database\memory @@ -194,6 +191,9 @@ include\bitcoin\database\memory + + include\bitcoin\database\memory + include\bitcoin\database\memory @@ -209,6 +209,9 @@ include\bitcoin\database\primitives + + include\bitcoin\database\primitives + include\bitcoin\database\primitives @@ -421,6 +424,9 @@ include\bitcoin\database\impl\primitives + + include\bitcoin\database\impl\primitives + include\bitcoin\database\impl\query\address diff --git a/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj b/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj index 06284fa82..cdcacca69 100644 --- a/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj +++ b/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj @@ -133,7 +133,6 @@ $(IntDir)test_memory_utilities.obj - diff --git a/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters b/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters index 7696c6c34..7fa9e4d5d 100644 --- a/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters +++ b/builds/msvc/vs2026/libbitcoin-database-test/libbitcoin-database-test.vcxproj.filters @@ -99,9 +99,6 @@ src\mocks - - src\mocks - src\primitives diff --git a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj index 07bb02cde..15d03af47 100644 --- a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj +++ b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj @@ -155,15 +155,16 @@ - + + @@ -236,6 +237,7 @@ + diff --git a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters index b8cac1d60..e6d4b5e27 100644 --- a/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters +++ b/builds/msvc/vs2026/libbitcoin-database/libbitcoin-database.vcxproj.filters @@ -182,9 +182,6 @@ include\bitcoin\database\memory\interfaces - - include\bitcoin\database\memory - include\bitcoin\database\memory @@ -194,6 +191,9 @@ include\bitcoin\database\memory + + include\bitcoin\database\memory + include\bitcoin\database\memory @@ -209,6 +209,9 @@ include\bitcoin\database\primitives + + include\bitcoin\database\primitives + include\bitcoin\database\primitives @@ -421,6 +424,9 @@ include\bitcoin\database\impl\primitives + + include\bitcoin\database\impl\primitives + include\bitcoin\database\impl\query\address diff --git a/include/bitcoin/database.hpp b/include/bitcoin/database.hpp index 55187bb95..8aa84be17 100644 --- a/include/bitcoin/database.hpp +++ b/include/bitcoin/database.hpp @@ -31,10 +31,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include diff --git a/include/bitcoin/database/impl/memory/mmap.ipp b/include/bitcoin/database/impl/memory/mmap.ipp index 2e627a3d6..3103a841d 100644 --- a/include/bitcoin/database/impl/memory/mmap.ipp +++ b/include/bitcoin/database/impl/memory/mmap.ipp @@ -34,9 +34,9 @@ namespace database { TEMPLATE CLASS::mmap(const path& filename, size_t minimum, size_t expansion, bool random) NOEXCEPT - requires (columns == one) + requires (is_one(columns)) : filenames_{ filename }, - minimum_(minimum), + minimum_(to_rows(minimum)), expansion_(expansion), random_(random), opened_{ file::invalid } @@ -48,7 +48,7 @@ CLASS::mmap(const paths& filenames, size_t minimum, size_t expansion, bool random) NOEXCEPT requires (columns > one) : filenames_(filenames), - minimum_(minimum), + minimum_(to_rows(minimum)), expansion_(expansion), random_(random), opened_{} @@ -59,17 +59,13 @@ CLASS::mmap(const paths& filenames, size_t minimum, size_t expansion, TEMPLATE CLASS::~mmap() NOEXCEPT { - BC_ASSERT_MSG(!loaded_, "..."); - BC_ASSERT_MSG(is_zero(logical_), "..."); - - BC_ASSERT_MSG(std::ranges::all_of(memory_map_, - [](auto map) NOEXCEPT { return is_null(map); }), "..."); - - BC_ASSERT_MSG(std::ranges::all_of(capacity_, - [](auto closed) NOEXCEPT { return is_zero(closed); }), "..."); - - BC_ASSERT_MSG(std::ranges::all_of(opened_, - [](auto opened) NOEXCEPT { return opened == file::invalid; }), "..."); + BC_ASSERT(!loaded_); + BC_ASSERT(is_zero(logical_)); + BC_ASSERT(is_zero(capacity_)); + BC_ASSERT(std::ranges::all_of(memory_map_, + [](auto map) NOEXCEPT { return is_null(map); })); + BC_ASSERT(std::ranges::all_of(opened_, + [](auto opened) NOEXCEPT { return opened == file::invalid; })); } TEMPLATE @@ -92,13 +88,10 @@ bool CLASS::is_loaded() const NOEXCEPT TEMPLATE size_t CLASS::to_capacity(size_t required) const NOEXCEPT { - BC_PUSH_WARNING(NO_STATIC_CAST) - const auto resize = required * ((expansion_ + 100.0) / 100.0); - const auto target = std::max(minimum_, static_cast(resize)); - BC_POP_WARNING() - - BC_ASSERT(target >= required); - return target; + // Covert required rows to capacity-padded rows. + using namespace system; + const auto growth = ceilinged_multiply(required, expansion_) / 100u; + return std::max(minimum_, ceilinged_add(required, growth)); } // Read-write protected by atomic, write-write protected by remap_mutex. diff --git a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp index bb1b166b8..016f597df 100644 --- a/include/bitcoin/database/impl/memory/mmap_dispatch.ipp +++ b/include/bitcoin/database/impl/memory/mmap_dispatch.ipp @@ -37,15 +37,13 @@ TEMPLATE memory_ptr CLASS::get_capacity(size_t offset) const NOEXCEPT { // Same as get() but limited by capacity() vs. size(). - const auto allocated = std::get(capacity_); + const auto allocated = to_width(capacity_); const auto ptr = std::make_shared(remap_mutex_); if (!loaded_ || is_null(ptr)) return nullptr; - ptr->assign( - std::next(std::get(memory_map_), offset), - std::next(std::get(memory_map_), allocated)); - + auto data = std::get(memory_map_); + ptr->assign(std::next(data, offset), std::next(data, allocated)); return ptr; } @@ -60,8 +58,7 @@ memory::iterator CLASS::get_raw(size_t offset) const NOEXCEPT } TEMPLATE -memory_ptr CLASS::set(size_t offset, size_t size, - uint8_t backfill) NOEXCEPT +memory_ptr CLASS::set(size_t offset, size_t size, uint8_t backfill) NOEXCEPT { { std::unique_lock field_lock(field_mutex_); @@ -70,21 +67,23 @@ memory_ptr CLASS::set(size_t offset, size_t size, return {}; const auto end = std::max(logical_, offset + size); - if (end > capacity_rows(capacity_)) + if (end > capacity_) { - const auto capacity = to_capacity(end); + const auto extended = to_capacity(end); // TODO: Could loop over a try lock here and log deadlock warning. std::unique_lock remap_lock(remap_mutex_); // Disk full condition leaves store in valid state despite null. - if (!remap_all_(capacity, sequence{})) + if (!remap_all_(extended, sequence{})) return {}; // Fill new capacity as offset may not be at end due to expansion. - const auto first = to_width(logical_); - std::fill_n(std::next(std::get(memory_map_), first), - std::get(capacity_) - first, backfill); + auto data = std::get(memory_map_); + const auto logical = to_width(logical_); + const auto capacity = to_width(capacity_); + const auto start = std::next(data, logical); + std::fill_n(start, capacity - logical, backfill); } logical_ = end; @@ -93,27 +92,17 @@ memory_ptr CLASS::set(size_t offset, size_t size, return get(offset); } -// This avoids forwarding to get_at() for the tiny runtime performance benefit. TEMPLATE memory_ptr CLASS::get(size_t offset) const NOEXCEPT { - const auto allocated = to_width(size()); - const auto ptr = std::make_shared(remap_mutex_); - if (!loaded_ || is_null(ptr)) - return {}; - - ptr->assign( - std::next(std::get(memory_map_), offset), - std::next(std::get(memory_map_), allocated)); - - return ptr; + return get_at(zero, offset); } +BC_PUSH_WARNING(NO_ARRAY_INDEXING) + TEMPLATE memory_ptr CLASS::get_at(size_t column, size_t offset) const NOEXCEPT { - BC_PUSH_WARNING(NO_ARRAY_INDEXING) - // Invalid column yields null (bounds check on runtime column index). if (column >= columns) return {}; @@ -126,20 +115,18 @@ memory_ptr CLASS::get_at(size_t column, size_t offset) const NOEXCEPT // Takes a shared lock on remap_mutex_ until destruct, blocking remap. const auto ptr = std::make_shared(remap_mutex_); - // loaded_ update is precluded by remap_mutex_, making this read atomic. + // loaded_ update is precluded by above lock, making this read atomic. if (!loaded_ || is_null(ptr)) return {}; // With offset > size the assignment is negative (stream is exhausted). - ptr->assign( - std::next(memory_map_[column], offset), - std::next(memory_map_[column], allocated)); - + auto data = memory_map_[column]; + ptr->assign(std::next(data, offset), std::next(data, allocated)); return ptr; - - BC_POP_WARNING() } +BC_POP_WARNING() + } // namespace database } // namespace libbitcoin diff --git a/include/bitcoin/database/impl/memory/mmap_private.ipp b/include/bitcoin/database/impl/memory/mmap_private.ipp index bbe37c9d6..f5a123f43 100644 --- a/include/bitcoin/database/impl/memory/mmap_private.ipp +++ b/include/bitcoin/database/impl/memory/mmap_private.ipp @@ -36,8 +36,9 @@ TEMPLATE template size_t CLASS::space_one_(size_t rows) const NOEXCEPT { - return system::floored_subtract(to_capacity(to_width(rows)), - std::get(capacity_)); + return system::floored_subtract( + to_width(rows), + to_width(capacity_)); } TEMPLATE @@ -52,14 +53,20 @@ TEMPLATE template bool CLASS::map_all_(std::index_sequence) NOEXCEPT { - return (map_() && ...); + if (!(map_() && ...)) + return false; + + capacity_ = std::max(logical_, minimum_); + return true; } TEMPLATE template bool CLASS::unmap_all_(std::index_sequence) NOEXCEPT { - return (unmap_() && ...); + const auto success = (unmap_() && ...); + capacity_ = zero; + return success; } TEMPLATE @@ -71,10 +78,14 @@ bool CLASS::flush_all_(std::index_sequence) NOEXCEPT TEMPLATE template -bool CLASS::remap_all_(size_t logical, std::index_sequence) NOEXCEPT +bool CLASS::remap_all_(size_t capacity, std::index_sequence) NOEXCEPT { - const auto space = space_all_(logical, std::index_sequence{}); - return (remap_(to_width(logical), space) && ...); + const auto space = space_all_(capacity, std::index_sequence{}); + if (!(remap_(to_width(capacity), space) && ...)) + return false; + + capacity_ = capacity; + return true; } // mman wrappers, not thread safe. @@ -127,7 +138,7 @@ bool CLASS::unmap_() NOEXCEPT #if defined(HAVE_MSC) const auto success = (::msync(memory_map_[Column], logical, MS_SYNC) != fail) - && (::munmap(memory_map_[Column], capacity_[Column]) != fail) + && (::munmap(memory_map_[Column], to_width(capacity_)) != fail) && (::ftruncate(opened_[Column], logical) != fail) && (::fsync(opened_[Column]) != fail); #else @@ -138,13 +149,12 @@ bool CLASS::unmap_() NOEXCEPT #else && (::fsync(opened_[Column]) != fail) #endif - && (::munmap(memory_map_[Column], capacity_[Column]) != fail); + && (::munmap(memory_map_[Column], to_width(capacity_)) != fail); #endif if (!success) set_first_code(error::munmap_failure); loaded_ = false; - capacity_[Column] = zero; memory_map_[Column] = {}; return success; } @@ -200,13 +210,13 @@ bool CLASS::remap_(size_t size, size_t space) NOEXCEPT // mman-win32 mremap hack (umap/map) requires flags and file descriptor. memory_map_[Column] = system::pointer_cast( - ::mremap_(memory_map_[Column], capacity_[Column], size, + ::mremap_(memory_map_[Column], to_width(capacity_), size, PROT_READ | PROT_WRITE, MAP_SHARED, opened_[Column])); #elif defined(MREMAP_MAYMOVE) memory_map_[Column] = system::pointer_cast( - ::mremap(memory_map_[Column], capacity_[Column], size, + ::mremap(memory_map_[Column], to_width(capacity_), size, MREMAP_MAYMOVE)); #else @@ -229,8 +239,8 @@ bool CLASS::resize_(size_t size, size_t space) NOEXCEPT { // Disk full detection, any other failure is an abort. #if !defined (WITHOUT_FALLOCATE) - if (::fallocate(opened_[Column], 0, capacity_[Column], - size - capacity_[Column]) == fail) + const auto capacity = to_width(capacity_); + if (::fallocate(opened_[Column], 0, capacity, size - capacity) == fail) #else if (::ftruncate(opened_[Column], size) == fail) #endif @@ -253,12 +263,16 @@ bool CLASS::resize_(size_t size, size_t space) NOEXCEPT // Finalize failure results in unmapped. TEMPLATE template -bool CLASS::finalize_(size_t size) NOEXCEPT +bool CLASS::finalize_(size_t + #if !defined (WITHOUT_MADVISE) && !defined(HAVE_MSC) + size + #endif +) NOEXCEPT { if (memory_map_[Column] == MAP_FAILED) { loaded_ = false; - capacity_[Column] = zero; + capacity_ = zero; memory_map_[Column] = {}; // mmap or mremap failure (not mapped). @@ -266,8 +280,7 @@ bool CLASS::finalize_(size_t size) NOEXCEPT return false; } -#if !defined (WITHOUT_MADVISE) -#if !defined(HAVE_MSC) +#if !defined (WITHOUT_MADVISE) && !defined(HAVE_MSC) // Get page size (usually 4KB). const int page_size = ::sysconf(_SC_PAGESIZE); const auto page = system::possible_narrow_sign_cast(page_size); @@ -303,11 +316,9 @@ bool CLASS::finalize_(size_t size) NOEXCEPT return false; } } -#endif // !HAVE_MSC -#endif // !WITHOUT_MADVISE +#endif // !WITHOUT_MADVISE && !HAVE_MSC loaded_ = true; - capacity_[Column] = size; return true; } diff --git a/include/bitcoin/database/impl/memory/mmap_storage.ipp b/include/bitcoin/database/impl/memory/mmap_storage.ipp index d2de6ad9e..682cad2bc 100644 --- a/include/bitcoin/database/impl/memory/mmap_storage.ipp +++ b/include/bitcoin/database/impl/memory/mmap_storage.ipp @@ -275,55 +275,54 @@ TEMPLATE size_t CLASS::capacity() const NOEXCEPT { std::shared_lock field_lock(field_mutex_); - return capacity_rows(capacity_); + return capacity_; } TEMPLATE -bool CLASS::truncate(size_t size) NOEXCEPT +bool CLASS::truncate(size_t count) NOEXCEPT { std::unique_lock field_lock(field_mutex_); - if (size > logical_) + if (count > logical_) return false; - logical_ = size; + logical_ = count; return true; } TEMPLATE -bool CLASS::expand(size_t size) NOEXCEPT +bool CLASS::expand(size_t count) NOEXCEPT { std::unique_lock field_lock(field_mutex_); if (fault_ || !loaded_) return false; - if (size <= logical_) + if (count <= logical_) return true; - // Column 0 capacity (bytes) transposed to rows for the row-space compare. - if (size > capacity_rows(capacity_)) + if (count > capacity_) { - const auto capacity = to_capacity(size); + const auto capacity = to_capacity(count); std::unique_lock remap_lock(remap_mutex_); if (!remap_all_(capacity, sequence{})) return false; } - logical_ = size; + logical_ = count; return true; } TEMPLATE -bool CLASS::reserve(size_t chunk) NOEXCEPT +bool CLASS::reserve(size_t count) NOEXCEPT { std::unique_lock field_lock(field_mutex_); - if (fault_ || !loaded_ || system::is_add_overflow(logical_, chunk)) + if (fault_ || !loaded_ || system::is_add_overflow(logical_, count)) return false; - const auto end = logical_ + chunk; - if (end > capacity_rows(capacity_)) + const auto end = logical_ + count; + if (end > capacity_) { const auto capacity = to_capacity(end); std::unique_lock remap_lock(remap_mutex_); @@ -339,15 +338,15 @@ bool CLASS::reserve(size_t chunk) NOEXCEPT // pointer is waiting on allocation. Lock safety requires that access pointers // are short-lived and do not block on allocation. TEMPLATE -size_t CLASS::allocate(size_t chunk) NOEXCEPT +size_t CLASS::allocate(size_t count) NOEXCEPT { std::unique_lock field_lock(field_mutex_); - if (fault_ || !loaded_ || system::is_add_overflow(logical_, chunk)) + if (fault_ || !loaded_ || system::is_add_overflow(logical_, count)) return storage::eof; - const auto current = capacity_rows(capacity_); - auto end = logical_ + chunk; + const auto current = capacity_; + auto end = logical_ + count; if (end > current) { const auto capacity = to_capacity(end); diff --git a/include/bitcoin/database/impl/primitives/arraymap.ipp b/include/bitcoin/database/impl/primitives/arraymap.ipp index ad84e471d..b37a898b1 100644 --- a/include/bitcoin/database/impl/primitives/arraymap.ipp +++ b/include/bitcoin/database/impl/primitives/arraymap.ipp @@ -110,15 +110,15 @@ size_t CLASS::body_size() const NOEXCEPT } TEMPLATE -Link CLASS::count() const NOEXCEPT +size_t CLASS::capacity() const NOEXCEPT { - return body_.count(); + return body_.capacity(); } TEMPLATE -size_t CLASS::capacity() const NOEXCEPT +Link CLASS::count() const NOEXCEPT { - return body_.capacity(); + return body_.count(); } TEMPLATE diff --git a/include/bitcoin/database/impl/primitives/hashmap.ipp b/include/bitcoin/database/impl/primitives/hashmap.ipp index 85a39473c..f15e87b6e 100644 --- a/include/bitcoin/database/impl/primitives/hashmap.ipp +++ b/include/bitcoin/database/impl/primitives/hashmap.ipp @@ -99,15 +99,15 @@ size_t CLASS::body_size() const NOEXCEPT } TEMPLATE -Link CLASS::count() const NOEXCEPT +size_t CLASS::capacity() const NOEXCEPT { - return body_.count(); + return body_.capacity(); } TEMPLATE -size_t CLASS::capacity() const NOEXCEPT +Link CLASS::count() const NOEXCEPT { - return body_.capacity(); + return body_.count(); } TEMPLATE diff --git a/include/bitcoin/database/impl/primitives/manager.ipp b/include/bitcoin/database/impl/primitives/manager.ipp index 9021d37bb..834b6a4e1 100644 --- a/include/bitcoin/database/impl/primitives/manager.ipp +++ b/include/bitcoin/database/impl/primitives/manager.ipp @@ -23,29 +23,71 @@ namespace libbitcoin { namespace database { - + TEMPLATE -CLASS::manager(storage& file) NOEXCEPT - : file_(file) +template +inline memory_ptr CLASS::get() const NOEXCEPT { + if constexpr (is_one(columns)) + return files_.get(); + else + return files_.get_at(Column); } TEMPLATE -inline size_t CLASS::size() const NOEXCEPT +template +inline memory_ptr CLASS::get(const Link& link) const NOEXCEPT { - return file_.size(); + if (link.is_terminal()) + return nullptr; + + const auto position = link_to_position(link); + + // memory.size() may be negative (stream treats as exhausted). + if constexpr (is_one(columns)) + return files_.get(position); + else + return files_.get_at(Column, position); } TEMPLATE -inline Link CLASS::count() const NOEXCEPT +template > +inline memory_ptr CLASS::get_capacity(const Link& link) const NOEXCEPT +{ + if (link.is_terminal()) + return nullptr; + + return files_.get_capacity(link_to_position(link)); +} + +TEMPLATE +CLASS::managers(storage& body) NOEXCEPT + : files_(body) { - return position_to_link(size()); +} + +TEMPLATE +inline size_t CLASS::size() const NOEXCEPT +{ + if constexpr (is_one(columns)) + return files_.size(); + else + return files_.size() * strides(std::make_index_sequence{}); } TEMPLATE inline size_t CLASS::capacity() const NOEXCEPT { - return file_.capacity(); + if constexpr (is_one(columns)) + return files_.capacity(); + else + return files_.capacity() * strides(std::make_index_sequence{}); +} + +TEMPLATE +inline Link CLASS::count() const NOEXCEPT +{ + return elements_to_link(files_.size()); } TEMPLATE @@ -54,8 +96,8 @@ bool CLASS::truncate(const Link& count) NOEXCEPT if (count.is_terminal()) return false; - // Truncate to count visible records (absolute). - return file_.truncate(link_to_position(count)); + // Truncate to count visible records (absolute, shared row count). + return files_.truncate(link_to_elements(count)); } TEMPLATE @@ -64,8 +106,8 @@ bool CLASS::expand(const Link& count) NOEXCEPT if (count.is_terminal()) return false; - // Expand to count visible records (absolute). - return file_.expand(link_to_position(count)); + // Expand to count visible records (absolute, shared row count). + return files_.expand(link_to_elements(count)); } TEMPLATE @@ -74,8 +116,8 @@ bool CLASS::reserve(const Link& count) NOEXCEPT if (count.is_terminal()) return false; - // Expand by count INVISIBLE records (relative). - return file_.reserve(link_to_position(count)); + // Expand by count invisible records (relative, shared row count). + return files_.reserve(link_to_elements(count)); } TEMPLATE @@ -84,115 +126,141 @@ Link CLASS::allocate(const Link& count) NOEXCEPT if (count.is_terminal()) return count; - // Expand by count visible records (relative), return absolute position. - const auto start = file_.allocate(link_to_position(count)); + // One shared allocation across all columns (one lock). Expand by count + // visible records (relative), return absolute record link of first. + const auto start = files_.allocate(link_to_elements(count)); - // Guards addition overflow in position_to_link (start must be valid). + // Guards addition overflow in cast_link (start must be valid). The eof + // sentinel is in file denomination, so must be detected before conversion. if (start == storage::eof) return Link::terminal; - // Callers (nomap and hashmap) handle terminal allocation. - return position_to_link(start); + // Callers (nomaps) handle terminal allocation. + return elements_to_link(start); } +// Errors. +// ---------------------------------------------------------------------------- + TEMPLATE -inline memory_ptr CLASS::get() const NOEXCEPT +code CLASS::get_fault() const NOEXCEPT { - return file_.get(); + return files_.get_fault(); } TEMPLATE -inline memory_ptr CLASS::get(const Link& value) const NOEXCEPT +size_t CLASS::get_space() const NOEXCEPT { - if (value.is_terminal()) - return nullptr; - - // memory.size() may be negative (stream treats as exhausted). - return file_.get(link_to_position(value)); + return files_.get_space(); } TEMPLATE -inline memory_ptr CLASS::get_capacity(const Link& value) const NOEXCEPT +code CLASS::reload() NOEXCEPT { - if (value.is_terminal()) - return nullptr; - - // memory.size() may be negative (stream treats as exhausted). - return file_.get_capacity(link_to_position(value)); + return files_.load(); } -// Errors. +// static // ---------------------------------------------------------------------------- +// private TEMPLATE -code CLASS::get_fault() const NOEXCEPT +template +constexpr size_t CLASS::stride() NOEXCEPT { - return file_.get_fault(); -} + using namespace system; + constexpr auto size = std::get(sizes); -TEMPLATE -size_t CLASS::get_space() const NOEXCEPT -{ - return file_.get_space(); + if constexpr (size == max_size_t) + { + // Slab: link/key incorporated into size (byte-addressed map). + return size; + } + else if constexpr (is_zero(Column) && is_nonzero(key_size)) + { + // Spine of a keyed map: link/key precede the record. + return Link::size + key_size + size; + } + else + { + // Keyless (nomap/arraymap) or non-spine column: pure record. + static_assert(is_nonzero(size)); + return size; + } } +// private TEMPLATE -code CLASS::reload() NOEXCEPT +template +constexpr size_t CLASS::strides(std::index_sequence) NOEXCEPT { - return file_.load(); + return (stride() + ...); } -// static -// ---------------------------------------------------------------------------- - TEMPLATE +template constexpr size_t CLASS::link_to_position(const Link& link) NOEXCEPT { using namespace system; const auto value = possible_narrow_cast(link.value); + constexpr auto element_size = stride(); if constexpr (is_slab) { // Slab implies link/key incorporated into size. return value; } - else if constexpr (is_nonzero(key_size)) + else { - // Record implies link/key independent of Size. - constexpr auto element_size = Link::size + key_size + Size; + // Record (keyed spine or keyless) implies fixed element stride. BC_ASSERT(!is_multiply_overflow(value, element_size)); return value * element_size; } - else - { - // No key implies no linked list (not a hashmap), so record array. - BC_ASSERT(!is_multiply_overflow(value, Size)); - return value * Size; - } } TEMPLATE +template constexpr Link CLASS::position_to_link(size_t position) NOEXCEPT { using namespace system; + constexpr auto element_size = stride(); if constexpr (is_slab) { // Slab implies link/key incorporated into size. return { cast_link(position) }; } - else if constexpr (is_nonzero(key_size)) + else { - // Record implies link/key independent of Size. - constexpr auto element_size = Link::size + key_size + Size; + static_assert(is_nonzero(element_size)); return { cast_link(position / element_size) }; } +} + +// private +TEMPLATE +constexpr size_t CLASS::link_to_elements(const Link& link) NOEXCEPT +{ + using namespace system; + + // Single column: file elements are bytes (width one). Records convert + // by stride (slab passes through, slab links are already byte offsets). + // Aggregate: file rows are the shared record count (no conversion). + if constexpr (is_one(columns)) + return link_to_position(link); else - { - // No key implies no linked list. - static_assert(is_nonzero(Size)); - return { cast_link(position / Size) }; - } + return possible_narrow_cast(link.value); +} + +// private +TEMPLATE +constexpr Link CLASS::elements_to_link(size_t elements) NOEXCEPT +{ + // Inverse of link_to_elements (slab and aggregate pass through). + if constexpr (is_one(columns)) + return position_to_link(elements); + else + return { cast_link(elements) }; } TEMPLATE diff --git a/include/bitcoin/database/impl/primitives/nomap.ipp b/include/bitcoin/database/impl/primitives/nomap.ipp index 77041d5ce..f3b68090c 100644 --- a/include/bitcoin/database/impl/primitives/nomap.ipp +++ b/include/bitcoin/database/impl/primitives/nomap.ipp @@ -91,16 +91,17 @@ size_t CLASS::body_size() const NOEXCEPT } TEMPLATE -Link CLASS::count() const NOEXCEPT +size_t CLASS::capacity() const NOEXCEPT { - return manager_.count(); + return manager_.capacity(); } TEMPLATE -size_t CLASS::capacity() const NOEXCEPT +Link CLASS::count() const NOEXCEPT { - return manager_.capacity(); + return manager_.count(); } + TEMPLATE bool CLASS::truncate(const Link& count) NOEXCEPT { @@ -119,6 +120,19 @@ bool CLASS::drop() NOEXCEPT return manager_.truncate(0) && backup(); } +TEMPLATE +bool CLASS::reserve(const Link& size) NOEXCEPT +{ + // Not writer-writer thread safe (two writers may share reserve). + return manager_.reserve(size); +} + +TEMPLATE +memory_ptr CLASS::get_memory() const NOEXCEPT +{ + return manager_.get(); +} + // error condition // ---------------------------------------------------------------------------- @@ -143,19 +157,6 @@ code CLASS::reload() NOEXCEPT // query interface // ---------------------------------------------------------------------------- -TEMPLATE -inline bool CLASS::reserve(const Link& size) NOEXCEPT -{ - // Reserve not writer-writer thread safe (two writers may share reserve). - return manager_.reserve(size); -} - -TEMPLATE -inline memory_ptr CLASS::get_memory() const NOEXCEPT -{ - return manager_.get(); -} - // static TEMPLATE template > @@ -182,7 +183,11 @@ bool CLASS::get(const memory_ptr& ptr, const Link& link, iostream stream{ offset, size - position }; reader source{ stream }; - if constexpr (!is_slab) { BC_DEBUG_ONLY(source.set_limit(Size * element.count());) } + if constexpr (!is_slab) + { + BC_DEBUG_ONLY(source.set_limit(Size * element.count());) + } + return element.from_data(source); } @@ -221,7 +226,11 @@ bool CLASS::put(const memory_ptr& ptr, const Element& element) NOEXCEPT iostream stream{ *ptr }; flipper sink{ stream }; - if constexpr (!is_slab) { BC_DEBUG_ONLY(sink.set_limit(Size * element.count());) } + if constexpr (!is_slab) + { + BC_DEBUG_ONLY(sink.set_limit(Size * element.count());) + } + return element.to_data(sink); } diff --git a/include/bitcoin/database/impl/primitives/nomaps.ipp b/include/bitcoin/database/impl/primitives/nomaps.ipp new file mode 100644 index 000000000..a79fa6f0b --- /dev/null +++ b/include/bitcoin/database/impl/primitives/nomaps.ipp @@ -0,0 +1,198 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_PRIMITIVES_NOMAPS_IPP +#define LIBBITCOIN_DATABASE_PRIMITIVES_NOMAPS_IPP + +#include +#include + +namespace libbitcoin { +namespace database { + +TEMPLATE +CLASS::nomaps(storage& header, storage& body) NOEXCEPT + : head_(header, 0), + manager_(body) +{ +} + +// not thread safe +// ---------------------------------------------------------------------------- + +TEMPLATE +bool CLASS::create() NOEXCEPT +{ + Link count{}; + return head_.create() && + head_.get_body_count(count) && manager_.truncate(count); +} + +TEMPLATE +bool CLASS::close() NOEXCEPT +{ + return head_.set_body_count(manager_.count()); +} + +TEMPLATE +bool CLASS::backup(bool) NOEXCEPT +{ + return head_.set_body_count(manager_.count()); +} + +TEMPLATE +bool CLASS::restore() NOEXCEPT +{ + Link count{}; + return head_.verify() && + head_.get_body_count(count) && manager_.truncate(count); +} + +TEMPLATE +bool CLASS::verify() const NOEXCEPT +{ + Link count{}; + return head_.verify() && + head_.get_body_count(count) && count == manager_.count(); +} + +// sizing +// ---------------------------------------------------------------------------- + +TEMPLATE +size_t CLASS::body_size() const NOEXCEPT +{ + return manager_.size(); +} + +TEMPLATE +Link CLASS::count() const NOEXCEPT +{ + return manager_.count(); +} + +TEMPLATE +Link CLASS::allocate(const Link& count) NOEXCEPT +{ + return manager_.allocate(count); +} + +TEMPLATE +bool CLASS::truncate(const Link& count) NOEXCEPT +{ + return manager_.truncate(count); +} + +// Faults. +// ---------------------------------------------------------------------------- + +TEMPLATE +code CLASS::get_fault() const NOEXCEPT +{ + return manager_.get_fault(); +} + +TEMPLATE +size_t CLASS::get_space() const NOEXCEPT +{ + return manager_.get_space(); +} + +TEMPLATE +code CLASS::reload() NOEXCEPT +{ + return manager_.reload(); +} + +// query interface +// ---------------------------------------------------------------------------- + +TEMPLATE +template +memory_ptr CLASS::get_memory() const NOEXCEPT +{ + return manager_.template get(); +} + +// static +TEMPLATE +template +bool CLASS::get(const memory_ptr& ptr, const Link& link, + Element& element) NOEXCEPT +{ + static_assert(Element::size == width, "element size != width"); + using namespace system; + if (!ptr || link.is_terminal()) + return false; + + const auto start = body::template link_to_position(link); + if (is_limited(start)) + return false; + + const auto size = ptr->size(); + const auto position = possible_narrow_and_sign_cast(start); + if (position >= size) + return false; + + const auto offset = ptr->offset(start); + if (is_null(offset)) + return false; + + iostream stream{ offset, size - position }; + reader source{ stream }; + + BC_DEBUG_ONLY(source.set_limit(width * element.count());) + return element.from_data(source); +} + +TEMPLATE +template +bool CLASS::get(const Link& link, Element& element) const NOEXCEPT +{ + return get(get_memory(), link, element); +} + +TEMPLATE +template +bool CLASS::put(const Link& link, const Element& element) NOEXCEPT +{ + using namespace system; + const auto ptr = manager_.template get(link); + return put(ptr, element); +} + +TEMPLATE +template +bool CLASS::put(const memory_ptr& ptr, const Element& element) NOEXCEPT +{ + static_assert(Element::size == width, "element size != width"); + using namespace system; + if (!ptr) + return false; + + iostream stream{ *ptr }; + flipper sink{ stream }; + + BC_DEBUG_ONLY(sink.set_limit(width * element.count());) + return element.to_data(sink); +} + +} // namespace database +} // namespace libbitcoin + +#endif diff --git a/include/bitcoin/database/impl/query/batch/ecdsa.ipp b/include/bitcoin/database/impl/query/batch/ecdsa.ipp index 7dbb25c0c..2efbb6b79 100644 --- a/include/bitcoin/database/impl/query/batch/ecdsa.ipp +++ b/include/bitcoin/database/impl/query/batch/ecdsa.ipp @@ -73,31 +73,26 @@ bool CLASS::purge_ecdsa_signatures() NOEXCEPT } TEMPLATE -bool CLASS::set_signature(const hash_digest& digest, const ec_compressed& point, - const ec_signature& signature, uint16_t id, +bool CLASS::set_signature(const hash_digest& digest, + const ec_compressed& point, const ec_signature& signature, uint16_t id, const header_link& link) NOEXCEPT { - using allocate_t = table::ecdsa_correlate::allocate1; using correlate_t = table::ecdsa_correlate::put_ref; using digest_t = table::ecdsa_digest::put_ref; using compressed_t = table::ecdsa_compressed::put_ref; using signature_t = table::ecdsa_signature::put_ref; - using namespace system; - // All values in the table are only valid under write exclusion. - // Table row cannot be assumed equal and tables remap independently. + // Caller must guard reads, this is writing into hot storage. // ======================================================================== const auto scope = store_.get_transactor(); - ecdsa_link fk{}; + using namespace system; const auto row = possible_narrow_cast(one); - // Allocate one correlate row and write terminal to it, gets fk. - // Then expand (as necessary) subordinate tables to same size. - if (!store_.ecdsa.correlate.put_link(fk, allocate_t{}) || - !store_.ecdsa.digest.expand(fk + row) || - !store_.ecdsa.compressed.expand(fk + row) || - !store_.ecdsa.signature.expand(fk + row)) + // Allocate 1 row across all columns. + // TODO: this could provide a single remap lock for all puts below. + const auto fk = store_.ecdsa.allocate(row); + if (fk.is_terminal()) return false; // Write one value to each column in corresponding positions. @@ -114,31 +109,26 @@ bool CLASS::set_signatures(const hash_digest& digest, const ec_compresseds& keys, const ec_signatures& sigs, uint16_t id, const header_link& link) NOEXCEPT { - using allocate_t = table::ecdsa_correlate::allocate; using correlate_t = table::ecdsa_correlate::put_refs; using digest_t = table::ecdsa_digest::put_refs; using compressed_t = table::ecdsa_compressed::put_refs; using signature_t = table::ecdsa_signature::put_refs; - using namespace system; const auto csigs = sigs.size(); const auto ckeys = keys.size(); const auto count = table::ecdsa_count(csigs, ckeys); - // All values in the table are only valid under write exclusion. - // Table row cannot be assumed equal and tables remap independently. + // Caller must guard reads, this is writing into hot storage. // ======================================================================== const auto scope = store_.get_transactor(); - ecdsa_link fk{}; + using namespace system; const auto rows = possible_narrow_cast(count); - // Allocate contiguous correlate rows and write terminal to each, gets fk. - // Then expand (as necessary) subordinate tables to same size. - if (!store_.ecdsa.correlate.put_link(fk, allocate_t{ {}, rows }) || - !store_.ecdsa.digest.expand(fk + rows) || - !store_.ecdsa.compressed.expand(fk + rows) || - !store_.ecdsa.signature.expand(fk + rows)) + // Allocate rows across all columns. + // TODO: this could provide a single remap lock for all puts below. + const auto fk = store_.ecdsa.allocate(rows); + if (fk.is_terminal()) return false; // Write values to each column in corresponding positions. diff --git a/include/bitcoin/database/impl/query/batch/schnorr.ipp b/include/bitcoin/database/impl/query/batch/schnorr.ipp index 147624b07..a62c38b68 100644 --- a/include/bitcoin/database/impl/query/batch/schnorr.ipp +++ b/include/bitcoin/database/impl/query/batch/schnorr.ipp @@ -77,27 +77,23 @@ bool CLASS::set_signature(const hash_digest& digest, const ec_xonly& point, const ec_signature& signature, uint16_t id, const header_link& link) NOEXCEPT { - using allocate_t = table::schnorr_correlate::allocate1; using correlate_t = table::schnorr_correlate::put_ref; using digest_t = table::schnorr_digest::put_ref; using xonly_t = table::schnorr_xonly::put_ref; using signature_t = table::schnorr_signature::put_ref; using namespace system; - // All values in the table are only valid under write exclusion. - // Table row cannot be assumed equal and tables remap independently. + // Caller must guard reads, this is writing into hot storage. // ======================================================================== const auto scope = store_.get_transactor(); - schnorr_link fk{}; + using namespace system; const auto row = possible_narrow_cast(one); - // Allocate one correlate row and write terminal to it, gets fk. - // Then expand (as necessary) subordinate tables to same size. - if (!store_.schnorr.correlate.put_link(fk, allocate_t{}) || - !store_.schnorr.digest.expand(fk + row) || - !store_.schnorr.xonly.expand(fk + row) || - !store_.schnorr.signature.expand(fk + row)) + // Allocate 1 row across all columns. + // TODO: this could provide a single remap lock for all puts below. + const auto fk = store_.schnorr.allocate(row); + if (fk.is_terminal()) return false; // Write one value to each column in corresponding positions. @@ -113,28 +109,24 @@ TEMPLATE bool CLASS::set_signatures(const threshold& batch, uint16_t id, const header_link& link) NOEXCEPT { - using allocate_t = table::schnorr_correlate::allocate; using correlate_t = table::schnorr_correlate::put_refs; using digest_t = table::schnorr_digest::put_refs; using xonly_t = table::schnorr_xonly::put_refs; using signature_t = table::schnorr_signature::put_refs; using namespace system; - // All values in the table are only valid under write exclusion. - // Table row cannot be assumed equal and tables remap independently. + // Caller must guard reads, this is writing into hot storage. // ======================================================================== const auto scope = store_.get_transactor(); - schnorr_link fk{}; + using namespace system; const auto& set = batch.tuples; const auto rows = possible_narrow_cast(set.size()); - // Allocate contiguous correlate rows and write terminal to each, gets fk. - // Then expand (as necessary) subordinate tables to same size. - if (!store_.schnorr.correlate.put_link(fk, allocate_t{ {}, rows }) || - !store_.schnorr.digest.expand(fk + rows) || - !store_.schnorr.xonly.expand(fk + rows) || - !store_.schnorr.signature.expand(fk + rows)) + // Allocate rows across all columns. + // TODO: this could provide a single remap lock for all puts below. + const auto fk = store_.silent.allocate(rows); + if (fk.is_terminal()) return false; // Write one value to each column in corresponding positions. diff --git a/include/bitcoin/database/impl/query/batch/silent.ipp b/include/bitcoin/database/impl/query/batch/silent.ipp index 86b577b2b..b885a63fc 100644 --- a/include/bitcoin/database/impl/query/batch/silent.ipp +++ b/include/bitcoin/database/impl/query/batch/silent.ipp @@ -116,23 +116,21 @@ bool CLASS::set_silent(const tx_link& link, using correlate_t = table::silent_correlate::records; using prefix_t = table::silent_prefix::put_ref; using compressed_t = table::silent_compressed::put_ref; - using namespace system; + // TODO: caller must guard reads, this is writing into hot storage. // ======================================================================== const auto scope = store_.get_transactor(); + using namespace system; auto rows = possible_narrow_cast(prefixes.size()); - constexpr auto term = silent_link::terminal; - silent_link fk{}; - - // Allocate contiguous correlate rows and write terminal to each, gets fk. - // Then expand (as necessary) subordinate tables to same size. - if (!store_.silent.correlate.put_link(fk, correlate_t{ {}, rows, term }) || - !store_.silent.prefix.expand(fk + rows) || - !store_.silent.compressed.expand(fk + rows)) + + // Allocate rows across all columns. + // TODO: this could provide a single remap lock for all puts below. + const auto fk = store_.silent.allocate(rows); + if (fk.is_terminal()) return false; - // Write one value to each column in corresponding positions. + // Write values to each column in corresponding positions. return store_.silent.correlate.put(fk, correlate_t{ {}, rows, link }) && store_.silent.prefix.put(fk, prefix_t{ {}, prefixes }) && diff --git a/include/bitcoin/database/memory/interfaces/storage.hpp b/include/bitcoin/database/memory/interfaces/storage.hpp index cdbcb1cb5..e190c1ebb 100644 --- a/include/bitcoin/database/memory/interfaces/storage.hpp +++ b/include/bitcoin/database/memory/interfaces/storage.hpp @@ -27,10 +27,12 @@ namespace libbitcoin { namespace database { /// Mapped memory interface. +/// A slab has a row width of 1, so "count" implies "bytes" for slabs below. class storage { public: static constexpr auto eof = system::bit_all; + using path = std::filesystem::path; /// Get the fault condition. virtual code get_fault() const NOEXCEPT = 0; @@ -39,7 +41,7 @@ class storage virtual size_t get_space() const NOEXCEPT = 0; /// The filesystem path of the backing storage. - virtual const std::filesystem::path& file() const NOEXCEPT = 0; + virtual const path& file() const NOEXCEPT = 0; /// Create empty file, must not exist. virtual code create() const NOEXCEPT = 0; @@ -66,25 +68,25 @@ class storage virtual code shrink() NOEXCEPT = 0; /// Dump current logical map to a new file in path, must not exist. - virtual code dump(const std::filesystem::path& path) const NOEXCEPT = 0; + virtual code dump(const path& path) const NOEXCEPT = 0; - /// The current logical size of the memory map (zero if closed). + /// Current of rows/bytes in map (zero if closed). virtual size_t size() const NOEXCEPT = 0; - /// The current capacity of the memory map (zero if unmapped). + /// The current count of rows/bytes in map (zero if closed). virtual size_t capacity() const NOEXCEPT = 0; - /// Reduce logical size to specified bytes (false if exceeds logical). - virtual bool truncate(size_t size) NOEXCEPT = 0; + /// Reduce logical size to specified rows/bytes (false if exceeds logical). + virtual bool truncate(size_t count) NOEXCEPT = 0; - /// Increase logical size to specified bytes as required (false if fails). - virtual bool expand(size_t size) NOEXCEPT = 0; + /// Increase logical to specified rows/bytes as required (false if fails). + virtual bool expand(size_t count) NOEXCEPT = 0; - /// Increase capacity by specified bytes (false only if fails). - virtual bool reserve(size_t size) NOEXCEPT = 0; + /// Increase capacity by specified rows/bytes (false only if fails). + virtual bool reserve(size_t count) NOEXCEPT = 0; - /// Increase logical by specified bytes, return offset to first (or eof). - virtual size_t allocate(size_t chunk) NOEXCEPT = 0; + /// Increase logical by specified rows/bytes, return row of first (or eof). + virtual size_t allocate(size_t count) NOEXCEPT = 0; /// Get remap-protected r/w access to start/offset of memory map (or null). /// Pointer is constrained to starting write within full capacity. diff --git a/include/bitcoin/database/memory/maps.hpp b/include/bitcoin/database/memory/maps.hpp deleted file mode 100644 index 998448c27..000000000 --- a/include/bitcoin/database/memory/maps.hpp +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Copyright (c) 2011-2026 libbitcoin developers (see AUTHORS) - * - * This file is part of libbitcoin. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef LIBBITCOIN_DATABASE_MEMORY_MMAPS_HPP -#define LIBBITCOIN_DATABASE_MEMORY_MMAPS_HPP - -#include -#include -#include - -// Variadic forwarding with early termination. -#define DISPATCH_METHOD(method, ...) \ - code method() __VA_ARGS__ NOEXCEPT \ - { \ - code ec{}; \ - std::apply([&](auto&... column) \ - { \ - ((ec = ec ? ec : column.method()), ...); \ - }, files_); \ - return ec; \ - } - -namespace libbitcoin { -namespace database { - -/// Variadic template for defining an SoA aggregate table map. -template -class maps -{ -public: - using path = std::filesystem::path; - static constexpr auto count = sizeof...(Columns); - static constexpr auto stride = (Columns::width + ...); - static constexpr auto widths = std::array{ Columns::width... }; - static constexpr auto sequence = std::make_index_sequence{}; - static constexpr auto suffixes = std::array - { std::string_view{ Columns::suffix.data(), Columns::suffix.size() }... }; - - template = true> - Storage& file() NOEXCEPT { return std::get(files_); } - - template = true> - const Storage& file() const NOEXCEPT { return std::get(files_); } - - maps(const path& base_path, size_t size, size_t rate, - bool random_access) NOEXCEPT - : maps{ base_path, size, rate, random_access, sequence } - { - } - - DISPATCH_METHOD(create, const) - DISPATCH_METHOD(open) - DISPATCH_METHOD(close) - DISPATCH_METHOD(load) - DISPATCH_METHOD(reload) - DISPATCH_METHOD(flush) - DISPATCH_METHOD(unload) - DISPATCH_METHOD(shrink) - DISPATCH_METHOD(get_fault, const) - - /// Macro not used because of accumulator. - size_t get_space() const NOEXCEPT - { - size_t total{}; - std::apply([&](const auto&... column) - { - ((total += column.get_space()), ...); - }, files_); - return total; - } - - /// Inject column filename extensions. - code dump(const path& other_path) const NOEXCEPT - { - code ec{}; - std::apply([&](const auto&... column) - { - size_t index{}; - ((ec = ec ? ec : column.dump(to_subpath(other_path, - suffixes[index++]))), ...); - }, files_); - return ec; - } - -protected: - template - maps(const path& base_path, size_t size, size_t rate, bool random_access, - std::index_sequence) NOEXCEPT - : base_path_{ base_path }, - files_{ Storage - { - to_subpath(base_path, suffixes[Index]), - std::max(system::ceilinged_multiply(size, widths[Index]) / stride, one), - rate, - random_access - }... } - { - } - - path to_subpath(const path& base, const std::string_view& suffix) const - { - auto out = base; - out.replace_extension(); - out += "_"; - out += suffix; - out += base.extension(); - return out; - } - - // These are thread safe. - const path base_path_; - std::array files_; -}; - -} // namespace database -} // namespace libbitcoin - -#undef DISPATCH_METHOD - -#endif diff --git a/include/bitcoin/database/memory/memory.hpp b/include/bitcoin/database/memory/memory.hpp index d5030c843..0d8002f60 100644 --- a/include/bitcoin/database/memory/memory.hpp +++ b/include/bitcoin/database/memory/memory.hpp @@ -23,9 +23,9 @@ #include #include #include -#include #include #include +#include #include #endif diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index a9134a377..c27b3a38b 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -35,6 +35,7 @@ namespace database { /// Thread safe access to a memory-mapped file, or to a set of column files /// sharing one allocation/remap guard set (SoA aggregate). +/// A slab has a row width of 1, so "count" implies "bytes" for slabs below. template class mmap : public storage @@ -45,8 +46,6 @@ class mmap /// Number of backing columns (1 == scalar map). static constexpr size_t columns = sizeof...(Widths); static_assert(!is_zero(columns), "requires at least one column"); - - using path = std::filesystem::path; using paths = std::array; using sizes = std::array; @@ -61,7 +60,7 @@ class mmap /// Scalar construction (columns == 1): unchanged signature and codegen. mmap(const path& filename, size_t minimum=1, size_t expansion=0, - bool random=true) NOEXCEPT requires (columns == one); + bool random=true) NOEXCEPT requires (is_one(columns)); /// Aggregate construction (columns > 1): one file per column, shared guards. mmap(const paths& filenames, size_t minimum=1, size_t expansion=0, @@ -86,7 +85,7 @@ class mmap size_t get_space() const NOEXCEPT override; /// The filesystem path of the (first) backing file. - const std::filesystem::path& file() const NOEXCEPT override; + const path& file() const NOEXCEPT override; /// Create empty file(s), must be closed. code create() const NOEXCEPT override; @@ -113,25 +112,25 @@ class mmap code shrink() NOEXCEPT override; /// Dump current logical map to a new file in path, must not exist. - code dump(const std::filesystem::path& path) const NOEXCEPT override; + code dump(const path& path) const NOEXCEPT override; - /// The current logical size of the memory map (zero if closed). + /// The current count of rows/bytes in map (zero if closed). size_t size() const NOEXCEPT override; - /// The current capacity of the memory map (zero if unloaded). + /// The current row/byte capacity of the memory map (zero if unmapped). size_t capacity() const NOEXCEPT override; - /// Reduce logical size to specified (false if exceeds logical). - bool truncate(size_t size) NOEXCEPT override; + /// Reduce logical size to specified rows/bytes (false if exceeds logical). + bool truncate(size_t count) NOEXCEPT override; - /// Increase logical size to specified as required (false if fails). - bool expand(size_t size) NOEXCEPT override; + /// Increase logical to specified rows/bytes as required (false if fails). + bool expand(size_t count) NOEXCEPT override; - /// Increase capacity by specified (false only if fails). - bool reserve(size_t size) NOEXCEPT override; + /// Increase capacity by specified rows/bytes (false only if fails). + bool reserve(size_t count) NOEXCEPT override; - /// Increase logical by specified, return offset to first (or eof). - size_t allocate(size_t chunk) NOEXCEPT override; + /// Increase logical by specified rows/bytes, return row of first (or eof). + size_t allocate(size_t count) NOEXCEPT override; /// Remap-protected r/w access to start/offset (or null), within capacity. memory_ptr get_capacity(size_t offset=zero) const NOEXCEPT override; @@ -151,24 +150,23 @@ class mmap size_t offset=zero) const NOEXCEPT override; protected: - /// Row<->byte transpose by the constexpr column width (folds for width 1). template static constexpr size_t to_width(size_t offset) NOEXCEPT { return offset * std::get(widths); } - // Capacity in rows: column 0 bytes transposed back to rows. - static constexpr size_t capacity_rows(const sizes& capacity) NOEXCEPT - { - return std::get(capacity) / std::get(widths); - } - - // Logical rows: column 0 bytes transposed back to rows. static constexpr size_t logical_rows(size_t bytes) NOEXCEPT { return bytes / std::get(widths); } + + static constexpr size_t to_rows(size_t bytes) NOEXCEPT + { + // Convert the user's byte minimum to row denomination. + constexpr auto row = (Widths + ...); + return system::ceilinged_divide(bytes, row); + } size_t to_capacity(size_t required) const NOEXCEPT; void set_first_code(const error::error_t& ec) NOEXCEPT; @@ -194,7 +192,7 @@ class mmap template bool flush_all_(std::index_sequence) NOEXCEPT; template - bool remap_all_(size_t logical, std::index_sequence) NOEXCEPT; + bool remap_all_(size_t capacity, std::index_sequence) NOEXCEPT; // mman wrappers, not thread safe. template @@ -219,11 +217,10 @@ class mmap std::atomic error_{ error::success }; // These are protected by field_mutex_. - // Shared allocation state and per-column file/capacity state. // Fields require field_mutex_ exclusive for write, shared for flush/read. - // logical_ is the shared row count across all columns (bytes for width 1). + // logical_ and capacity_ are row counts (byte cound if width is one). std::array opened_; - sizes capacity_{}; + size_t capacity_{}; size_t logical_{}; bool fault_{}; bool loaded_{}; @@ -234,8 +231,7 @@ class mmap mutable std::shared_mutex remap_mutex_{}; }; -// TODO: rename mmap<> to mmaps<> and map to mmap. -using map = mmap<1>; +using map = mmap; } // namespace database } // namespace libbitcoin diff --git a/include/bitcoin/database/memory/mmaps.hpp b/include/bitcoin/database/memory/mmaps.hpp new file mode 100644 index 000000000..0f48f98c5 --- /dev/null +++ b/include/bitcoin/database/memory/mmaps.hpp @@ -0,0 +1,88 @@ +/** + * Copyright (c) 2011-2026 libbitcoin developers (see AUTHORS) + * + * This file is part of libbitcoin. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#ifndef LIBBITCOIN_DATABASE_MEMORY_MMAPS_HPP +#define LIBBITCOIN_DATABASE_MEMORY_MMAPS_HPP + +#include +#include +#include +#include +#include + +namespace libbitcoin { +namespace database { + +/// SoA aggregate body: one substitutable multi-column map (Storage) over N +/// column files, one shared allocation guard. Storage is the map family. +template