From dcdfb714d57fa1ee7f4eed3777a50c0af623fb1b Mon Sep 17 00:00:00 2001 From: Razdoburdin Date: Mon, 1 Dec 2025 04:46:10 -0800 Subject: [PATCH 01/10] make block_size flexible --- .../svs/runtime/dynamic_vamana_index.h | 2 +- bindings/cpp/src/dynamic_vamana_index.cpp | 4 +- bindings/cpp/src/dynamic_vamana_index_impl.h | 20 ++++-- .../src/dynamic_vamana_index_leanvec_impl.h | 10 ++- bindings/cpp/src/svs_runtime_utils.h | 18 ++++-- bindings/cpp/tests/runtime_test.cpp | 61 +++++++++++++++---- 6 files changed, 85 insertions(+), 30 deletions(-) diff --git a/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h b/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h index 2e53b8050..f50f8bb3f 100644 --- a/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h +++ b/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h @@ -29,7 +29,7 @@ namespace v0 { // Abstract interface for Dynamic Vamana-based indexes. struct SVS_RUNTIME_API DynamicVamanaIndex : public VamanaIndex { - virtual Status add(size_t n, const size_t* labels, const float* x) noexcept = 0; + virtual Status add(size_t n, const size_t* labels, const float* x, int blocksize_exp = 30) noexcept = 0; virtual Status remove_selected(size_t* num_removed, const IDFilter& selector) noexcept = 0; virtual Status remove(size_t n, const size_t* labels) noexcept = 0; diff --git a/bindings/cpp/src/dynamic_vamana_index.cpp b/bindings/cpp/src/dynamic_vamana_index.cpp index 10e677da0..219a4bb5a 100644 --- a/bindings/cpp/src/dynamic_vamana_index.cpp +++ b/bindings/cpp/src/dynamic_vamana_index.cpp @@ -55,11 +55,11 @@ struct DynamicVamanaIndexManagerBase : public DynamicVamanaIndex { DynamicVamanaIndexManagerBase& operator=(DynamicVamanaIndexManagerBase&&) = default; ~DynamicVamanaIndexManagerBase() override = default; - Status add(size_t n, const size_t* labels, const float* x) noexcept override { + Status add(size_t n, const size_t* labels, const float* x, int blocksize_exp) noexcept override { return runtime_error_wrapper([&] { svs::data::ConstSimpleDataView data{x, n, impl_->dimensions()}; std::span lbls(labels, n); - impl_->add(data, lbls); + impl_->add(data, lbls, blocksize_exp); }); } diff --git a/bindings/cpp/src/dynamic_vamana_index_impl.h b/bindings/cpp/src/dynamic_vamana_index_impl.h index fd91e5bc8..4debec2c9 100644 --- a/bindings/cpp/src/dynamic_vamana_index_impl.h +++ b/bindings/cpp/src/dynamic_vamana_index_impl.h @@ -69,9 +69,10 @@ class DynamicVamanaIndexImpl { StorageKind get_storage_kind() const { return storage_kind_; } - void add(data::ConstSimpleDataView data, std::span labels) { + void add(data::ConstSimpleDataView data, std::span labels, + int blocksize_exp) { if (!impl_) { - return init_impl(data, labels); + return init_impl(data, labels, blocksize_exp); } impl_->add_points(data, labels); @@ -389,14 +390,17 @@ class DynamicVamanaIndexImpl { const index::vamana::VamanaBuildParameters& parameters, const svs::data::ConstSimpleDataView& data, std::span labels, + int blocksize_exp, StorageArgs&&... storage_args ) { auto threadpool = default_threadpool(); + auto blocksize_bytes = svs::lib::PowerOfTwo(blocksize_exp); auto storage = make_storage( std::forward(tag), data, threadpool, + blocksize_bytes, std::forward(storage_args)... ); @@ -413,13 +417,15 @@ class DynamicVamanaIndexImpl { } virtual void - init_impl(data::ConstSimpleDataView data, std::span labels) { + init_impl(data::ConstSimpleDataView data, std::span labels, + int blocksize_exp) { impl_.reset(storage::dispatch_storage_kind( get_storage_kind(), [this]( auto&& tag, data::ConstSimpleDataView data, - std::span labels + std::span labels, + int blocksize_exp ) { using Tag = std::decay_t; return build_impl( @@ -427,11 +433,13 @@ class DynamicVamanaIndexImpl { this->metric_type_, this->vamana_build_parameters(), data, - labels + labels, + blocksize_exp ); }, data, - labels + labels, + blocksize_exp )); } diff --git a/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h b/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h index de998ae7d..67b36c7d4 100644 --- a/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h +++ b/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h @@ -91,7 +91,8 @@ struct DynamicVamanaIndexLeanVecImpl : public DynamicVamanaIndexImpl { } } - void init_impl(data::ConstSimpleDataView data, std::span labels) + void init_impl(data::ConstSimpleDataView data, std::span labels, + int blocksize_exp) override { assert(storage::is_leanvec_storage(this->storage_kind_)); impl_.reset(dispatch_leanvec_storage_kind( @@ -99,7 +100,8 @@ struct DynamicVamanaIndexLeanVecImpl : public DynamicVamanaIndexImpl { [this]( auto&& tag, data::ConstSimpleDataView data, - std::span labels + std::span labels, + int blocksize_exp ) { using Tag = std::decay_t; return DynamicVamanaIndexImpl::build_impl( @@ -108,12 +110,14 @@ struct DynamicVamanaIndexLeanVecImpl : public DynamicVamanaIndexImpl { this->vamana_build_parameters(), data, labels, + blocksize_exp, this->leanvec_dims_, this->leanvec_matrices_ ); }, data, - labels + labels, + blocksize_exp )); } diff --git a/bindings/cpp/src/svs_runtime_utils.h b/bindings/cpp/src/svs_runtime_utils.h index 73969758e..50d5e18c8 100644 --- a/bindings/cpp/src/svs_runtime_utils.h +++ b/bindings/cpp/src/svs_runtime_utils.h @@ -198,7 +198,8 @@ template <> struct StorageFactory { template static StorageType init( const svs::data::ConstSimpleDataView& SVS_UNUSED(data), - Pool& SVS_UNUSED(pool) + Pool& SVS_UNUSED(pool), + svs::lib::PowerOfTwo SVS_UNUSED(blocksize_bytes) ) { throw StatusException( ErrorCode::NOT_IMPLEMENTED, "Requested storage kind is not supported" @@ -218,8 +219,12 @@ template struct StorageFactory; template - static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool) { - StorageType result(data.size(), data.dimensions()); + static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool, + svs::lib::PowerOfTwo blocksize_bytes = svs::data::BlockingParameters::default_blocksize_bytes) { + auto parameters = svs::data::BlockingParameters{ + .blocksize_bytes = blocksize_bytes}; + typename StorageType::allocator_type alloc(parameters); + StorageType result(data.size(), data.dimensions(), alloc); svs::threads::parallel_for( pool, svs::threads::StaticPartition(result.size()), @@ -244,7 +249,8 @@ struct StorageFactory { using StorageType = SQStorageType; template - static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool) { + static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool, + svs::lib::PowerOfTwo SVS_UNUSED(blocksize_bytes)) { return SQStorageType::compress(data, pool); } @@ -275,7 +281,8 @@ struct StorageFactory { using StorageType = LVQStorageType; template - static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool) { + static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool, + svs::lib::PowerOfTwo SVS_UNUSED(blocksize_bytes)) { return LVQStorageType::compress(data, pool, 0); } @@ -309,6 +316,7 @@ struct StorageFactory { static StorageType init( const svs::data::ConstSimpleDataView& data, Pool& pool, + svs::lib::PowerOfTwo SVS_UNUSED(blocksize_bytes), size_t leanvec_d = 0, std::optional> matrices = std::nullopt ) { diff --git a/bindings/cpp/tests/runtime_test.cpp b/bindings/cpp/tests/runtime_test.cpp index 2e60ab5a1..9abfb7863 100644 --- a/bindings/cpp/tests/runtime_test.cpp +++ b/bindings/cpp/tests/runtime_test.cpp @@ -83,6 +83,7 @@ void write_and_read_index( size_t n, size_t d, svs::runtime::v0::StorageKind storage_kind, + int blocksize_exp2, svs::runtime::v0::MetricType metric = svs::runtime::v0::MetricType::L2 ) { // Build index @@ -99,7 +100,7 @@ void write_and_read_index( std::vector labels(n); std::iota(labels.begin(), labels.end(), 0); - status = index->add(n, labels.data(), xb.data()); + status = index->add(n, labels.data(), xb.data(), blocksize_exp2); CATCH_REQUIRE(status.ok()); svs_test::prepare_temp_directory(); @@ -141,7 +142,7 @@ void write_and_read_index( // Helper that writes and reads and index of requested size // Reports memory usage -UsageInfo run_save_and_load_test(const size_t target_mibytes) { +UsageInfo run_save_and_load_test(const size_t target_mibytes, int blocksize_exp2) { // Generate requested MiB of test data constexpr size_t mem_test_d = 128; const size_t target_bytes = target_mibytes * 1024 * 1024; @@ -171,7 +172,7 @@ UsageInfo run_save_and_load_test(const size_t target_mibytes) { ); CATCH_REQUIRE(status.ok()); CATCH_REQUIRE(index != nullptr); - status = index->add(mem_test_n, labels.data(), large_test_data.data()); + status = index->add(mem_test_n, labels.data(), large_test_data.data(), blocksize_exp2); CATCH_REQUIRE(status.ok()); std::ofstream out(filename, std::ios::binary); @@ -224,7 +225,7 @@ CATCH_TEST_CASE("WriteAndReadIndexSVS", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::FP32 + build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::FP32, 30 ); } @@ -241,7 +242,7 @@ CATCH_TEST_CASE("WriteAndReadIndexSVSFP16", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::FP16 + build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::FP16, 30 ); } @@ -258,7 +259,7 @@ CATCH_TEST_CASE("WriteAndReadIndexSVSSQI8", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::SQI8 + build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::SQI8, 30 ); } @@ -275,7 +276,7 @@ CATCH_TEST_CASE("WriteAndReadIndexSVSLVQ4x4", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::LVQ4x4 + build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::LVQ4x4, 30 ); } @@ -293,7 +294,7 @@ CATCH_TEST_CASE("WriteAndReadIndexSVSVamanaLeanVec4x4", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::LeanVec4x4 + build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::LeanVec4x4, 30 ); } @@ -330,6 +331,40 @@ CATCH_TEST_CASE("LeanVecWithTrainingData", "[runtime]") { svs::runtime::v0::DynamicVamanaIndex::destroy(index); } +CATCH_TEST_CASE("LeanVecWithTrainingDataCustomBlockSize", "[runtime]") { + const auto& test_data = get_test_data(); + // Build LeanVec index with explicit training + svs::runtime::v0::DynamicVamanaIndex* index = nullptr; + svs::runtime::v0::VamanaIndex::BuildParams build_params{64}; + svs::runtime::v0::Status status = svs::runtime::v0::DynamicVamanaIndexLeanVec::build( + &index, + test_d, + svs::runtime::v0::MetricType::L2, + svs::runtime::v0::StorageKind::LeanVec4x4, + 32, + build_params + ); + if (!svs::runtime::v0::DynamicVamanaIndex::check_storage_kind( + svs::runtime::v0::StorageKind::LeanVec4x4 + ) + .ok()) { + CATCH_REQUIRE(!status.ok()); + CATCH_SKIP("Storage kind is not supported, skipping test."); + } + CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index != nullptr); + + // Add data - should work with provided leanvec dims + std::vector labels(test_n); + std::iota(labels.begin(), labels.end(), 0); + + int block_size_exp = 17; // block_size = 2^block_size_exp + status = index->add(test_n, labels.data(), test_data.data(), block_size_exp); + CATCH_REQUIRE(status.ok()); + + svs::runtime::v0::DynamicVamanaIndex::destroy(index); +} + CATCH_TEST_CASE("FlatIndexWriteAndRead", "[runtime]") { const auto& test_data = get_test_data(); svs::runtime::v0::FlatIndex* index = nullptr; @@ -399,7 +434,7 @@ CATCH_TEST_CASE("SearchWithIDFilter", "[runtime]") { // Add data std::vector labels(test_n); std::iota(labels.begin(), labels.end(), 0); - status = index->add(test_n, labels.data(), test_data.data()); + status = index->add(test_n, labels.data(), test_data.data(), 30); CATCH_REQUIRE(status.ok()); const int nq = 8; @@ -445,7 +480,7 @@ CATCH_TEST_CASE("RangeSearchFunctional", "[runtime]") { // Add data std::vector labels(test_n); std::iota(labels.begin(), labels.end(), 0); - status = index->add(test_n, labels.data(), test_data.data()); + status = index->add(test_n, labels.data(), test_data.data(), 30); CATCH_REQUIRE(status.ok()); const int nq = 5; @@ -472,19 +507,19 @@ CATCH_TEST_CASE("MemoryUsageOnLoad", "[runtime][memory]") { }; CATCH_SECTION("SmallIndex") { - auto stats = run_save_and_load_test(10); + auto stats = run_save_and_load_test(10, 30); CATCH_REQUIRE(stats.file_size < 20 * 1024 * 1024); CATCH_REQUIRE(stats.rss_increase < threshold(stats.file_size)); } CATCH_SECTION("MediumIndex") { - auto stats = run_save_and_load_test(50); + auto stats = run_save_and_load_test(50, 30); CATCH_REQUIRE(stats.file_size < 100 * 1024 * 1024); CATCH_REQUIRE(stats.rss_increase < threshold(stats.file_size)); } CATCH_SECTION("LargeIndex") { - auto stats = run_save_and_load_test(200); + auto stats = run_save_and_load_test(200, 30); CATCH_REQUIRE(stats.file_size < 400 * 1024 * 1024); CATCH_REQUIRE(stats.rss_increase < threshold(stats.file_size)); } From a1766d69a3baeabcd4b1c9e78452dcce20033f6a Mon Sep 17 00:00:00 2001 From: Razdoburdin Date: Mon, 1 Dec 2025 04:57:15 -0800 Subject: [PATCH 02/10] linting --- .../svs/runtime/dynamic_vamana_index.h | 4 ++- bindings/cpp/src/dynamic_vamana_index.cpp | 4 ++- bindings/cpp/src/dynamic_vamana_index_impl.h | 14 +++++++---- .../src/dynamic_vamana_index_leanvec_impl.h | 8 +++--- bindings/cpp/src/svs_runtime_utils.h | 25 +++++++++++++------ bindings/cpp/tests/runtime_test.cpp | 3 ++- 6 files changed, 39 insertions(+), 19 deletions(-) diff --git a/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h b/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h index f50f8bb3f..d522e76d2 100644 --- a/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h +++ b/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h @@ -29,7 +29,9 @@ namespace v0 { // Abstract interface for Dynamic Vamana-based indexes. struct SVS_RUNTIME_API DynamicVamanaIndex : public VamanaIndex { - virtual Status add(size_t n, const size_t* labels, const float* x, int blocksize_exp = 30) noexcept = 0; + virtual Status + add(size_t n, const size_t* labels, const float* x, int blocksize_exp = 30 + ) noexcept = 0; virtual Status remove_selected(size_t* num_removed, const IDFilter& selector) noexcept = 0; virtual Status remove(size_t n, const size_t* labels) noexcept = 0; diff --git a/bindings/cpp/src/dynamic_vamana_index.cpp b/bindings/cpp/src/dynamic_vamana_index.cpp index 219a4bb5a..9155c4f18 100644 --- a/bindings/cpp/src/dynamic_vamana_index.cpp +++ b/bindings/cpp/src/dynamic_vamana_index.cpp @@ -55,7 +55,9 @@ struct DynamicVamanaIndexManagerBase : public DynamicVamanaIndex { DynamicVamanaIndexManagerBase& operator=(DynamicVamanaIndexManagerBase&&) = default; ~DynamicVamanaIndexManagerBase() override = default; - Status add(size_t n, const size_t* labels, const float* x, int blocksize_exp) noexcept override { + Status + add(size_t n, const size_t* labels, const float* x, int blocksize_exp + ) noexcept override { return runtime_error_wrapper([&] { svs::data::ConstSimpleDataView data{x, n, impl_->dimensions()}; std::span lbls(labels, n); diff --git a/bindings/cpp/src/dynamic_vamana_index_impl.h b/bindings/cpp/src/dynamic_vamana_index_impl.h index 4debec2c9..697f3fda6 100644 --- a/bindings/cpp/src/dynamic_vamana_index_impl.h +++ b/bindings/cpp/src/dynamic_vamana_index_impl.h @@ -69,8 +69,10 @@ class DynamicVamanaIndexImpl { StorageKind get_storage_kind() const { return storage_kind_; } - void add(data::ConstSimpleDataView data, std::span labels, - int blocksize_exp) { + void + add(data::ConstSimpleDataView data, + std::span labels, + int blocksize_exp) { if (!impl_) { return init_impl(data, labels, blocksize_exp); } @@ -416,9 +418,11 @@ class DynamicVamanaIndexImpl { }); } - virtual void - init_impl(data::ConstSimpleDataView data, std::span labels, - int blocksize_exp) { + virtual void init_impl( + data::ConstSimpleDataView data, + std::span labels, + int blocksize_exp + ) { impl_.reset(storage::dispatch_storage_kind( get_storage_kind(), [this]( diff --git a/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h b/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h index 67b36c7d4..3822c238f 100644 --- a/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h +++ b/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h @@ -91,9 +91,11 @@ struct DynamicVamanaIndexLeanVecImpl : public DynamicVamanaIndexImpl { } } - void init_impl(data::ConstSimpleDataView data, std::span labels, - int blocksize_exp) - override { + void init_impl( + data::ConstSimpleDataView data, + std::span labels, + int blocksize_exp + ) override { assert(storage::is_leanvec_storage(this->storage_kind_)); impl_.reset(dispatch_leanvec_storage_kind( this->storage_kind_, diff --git a/bindings/cpp/src/svs_runtime_utils.h b/bindings/cpp/src/svs_runtime_utils.h index 50d5e18c8..abecfc3c8 100644 --- a/bindings/cpp/src/svs_runtime_utils.h +++ b/bindings/cpp/src/svs_runtime_utils.h @@ -219,10 +219,13 @@ template struct StorageFactory; template - static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool, - svs::lib::PowerOfTwo blocksize_bytes = svs::data::BlockingParameters::default_blocksize_bytes) { - auto parameters = svs::data::BlockingParameters{ - .blocksize_bytes = blocksize_bytes}; + static StorageType init( + const svs::data::ConstSimpleDataView& data, + Pool& pool, + svs::lib::PowerOfTwo blocksize_bytes = + svs::data::BlockingParameters::default_blocksize_bytes + ) { + auto parameters = svs::data::BlockingParameters{.blocksize_bytes = blocksize_bytes}; typename StorageType::allocator_type alloc(parameters); StorageType result(data.size(), data.dimensions(), alloc); svs::threads::parallel_for( @@ -249,8 +252,11 @@ struct StorageFactory { using StorageType = SQStorageType; template - static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool, - svs::lib::PowerOfTwo SVS_UNUSED(blocksize_bytes)) { + static StorageType init( + const svs::data::ConstSimpleDataView& data, + Pool& pool, + svs::lib::PowerOfTwo SVS_UNUSED(blocksize_bytes) + ) { return SQStorageType::compress(data, pool); } @@ -281,8 +287,11 @@ struct StorageFactory { using StorageType = LVQStorageType; template - static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool, - svs::lib::PowerOfTwo SVS_UNUSED(blocksize_bytes)) { + static StorageType init( + const svs::data::ConstSimpleDataView& data, + Pool& pool, + svs::lib::PowerOfTwo SVS_UNUSED(blocksize_bytes) + ) { return LVQStorageType::compress(data, pool, 0); } diff --git a/bindings/cpp/tests/runtime_test.cpp b/bindings/cpp/tests/runtime_test.cpp index 9abfb7863..94e7525a5 100644 --- a/bindings/cpp/tests/runtime_test.cpp +++ b/bindings/cpp/tests/runtime_test.cpp @@ -172,7 +172,8 @@ UsageInfo run_save_and_load_test(const size_t target_mibytes, int blocksize_exp2 ); CATCH_REQUIRE(status.ok()); CATCH_REQUIRE(index != nullptr); - status = index->add(mem_test_n, labels.data(), large_test_data.data(), blocksize_exp2); + status = + index->add(mem_test_n, labels.data(), large_test_data.data(), blocksize_exp2); CATCH_REQUIRE(status.ok()); std::ofstream out(filename, std::ios::binary); From a9f6595dba6c7689f6a82989695f09924ec5fc74 Mon Sep 17 00:00:00 2001 From: Razdoburdin Date: Mon, 8 Dec 2025 05:01:03 -0800 Subject: [PATCH 03/10] add blocksize bound chack; add blocksize_bytes getter --- bindings/cpp/CMakeLists.txt | 1 + .../svs/runtime/dynamic_vamana_index.h | 6 +- .../cpp/include/svs/runtime/index_blocksize.h | 46 +++++++++++ bindings/cpp/src/dynamic_vamana_index.cpp | 14 +++- bindings/cpp/src/dynamic_vamana_index_impl.h | 29 ++++--- .../src/dynamic_vamana_index_leanvec_impl.h | 8 +- bindings/cpp/src/flat_index_impl.h | 1 + bindings/cpp/src/svs_runtime_utils.h | 8 +- bindings/cpp/tests/CMakeLists.txt | 10 +++ bindings/cpp/tests/runtime_test.cpp | 80 ++++++++++++++----- bindings/cpp/tests/utils.h | 3 +- include/svs/index/vamana/dynamic_index.h | 13 +++ include/svs/orchestrators/dynamic_vamana.h | 6 ++ 13 files changed, 184 insertions(+), 41 deletions(-) create mode 100644 bindings/cpp/include/svs/runtime/index_blocksize.h diff --git a/bindings/cpp/CMakeLists.txt b/bindings/cpp/CMakeLists.txt index 80cfe36ba..2e6332070 100644 --- a/bindings/cpp/CMakeLists.txt +++ b/bindings/cpp/CMakeLists.txt @@ -22,6 +22,7 @@ set(SVS_RUNTIME_HEADERS include/svs/runtime/training.h include/svs/runtime/vamana_index.h include/svs/runtime/dynamic_vamana_index.h + include/svs/runtime/index_blocksize.h include/svs/runtime/flat_index.h ) diff --git a/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h b/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h index d522e76d2..ad34b27c0 100644 --- a/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h +++ b/bindings/cpp/include/svs/runtime/dynamic_vamana_index.h @@ -16,6 +16,7 @@ #pragma once #include +#include #include #include @@ -30,8 +31,9 @@ namespace v0 { // Abstract interface for Dynamic Vamana-based indexes. struct SVS_RUNTIME_API DynamicVamanaIndex : public VamanaIndex { virtual Status - add(size_t n, const size_t* labels, const float* x, int blocksize_exp = 30 + add(size_t n, const size_t* labels, const float* x, IndexBlockSize blocksize ) noexcept = 0; + virtual Status add(size_t n, const size_t* labels, const float* x) noexcept = 0; virtual Status remove_selected(size_t* num_removed, const IDFilter& selector) noexcept = 0; virtual Status remove(size_t n, const size_t* labels) noexcept = 0; @@ -60,6 +62,8 @@ struct SVS_RUNTIME_API DynamicVamanaIndex : public VamanaIndex { MetricType metric, StorageKind storage_kind ) noexcept; + + virtual lib::PowerOfTwo blocksize_bytes() const noexcept = 0; }; struct SVS_RUNTIME_API DynamicVamanaIndexLeanVec : public DynamicVamanaIndex { diff --git a/bindings/cpp/include/svs/runtime/index_blocksize.h b/bindings/cpp/include/svs/runtime/index_blocksize.h new file mode 100644 index 000000000..70273f534 --- /dev/null +++ b/bindings/cpp/include/svs/runtime/index_blocksize.h @@ -0,0 +1,46 @@ +/* + * Copyright 2023 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include +#include + +namespace svs::runtime::v0 { + +class IndexBlockSize { + constexpr static size_t kMaxBlockSizeExp = 30; // 1GB + constexpr static size_t kMinBlockSizeExp = 12; // 4KB + + svs::lib::PowerOfTwo blocksize_bytes_; + + public: + explicit IndexBlockSize(size_t blocksize_exp) { + if (blocksize_exp > kMaxBlockSizeExp) { + throw ANNEXCEPTION("Blocksize is too large!"); + } else if (blocksize_exp < kMinBlockSizeExp) { + throw ANNEXCEPTION("Blocksize is too small!"); + } + + blocksize_bytes_ = svs::lib::PowerOfTwo(blocksize_exp); + } + + svs::lib::PowerOfTwo BlockSizeBytes() const { return blocksize_bytes_; } +}; + +} // namespace svs::runtime::v0 diff --git a/bindings/cpp/src/dynamic_vamana_index.cpp b/bindings/cpp/src/dynamic_vamana_index.cpp index 9155c4f18..4ec53f1a0 100644 --- a/bindings/cpp/src/dynamic_vamana_index.cpp +++ b/bindings/cpp/src/dynamic_vamana_index.cpp @@ -56,12 +56,20 @@ struct DynamicVamanaIndexManagerBase : public DynamicVamanaIndex { ~DynamicVamanaIndexManagerBase() override = default; Status - add(size_t n, const size_t* labels, const float* x, int blocksize_exp + add(size_t n, const size_t* labels, const float* x, IndexBlockSize blocksize ) noexcept override { return runtime_error_wrapper([&] { svs::data::ConstSimpleDataView data{x, n, impl_->dimensions()}; std::span lbls(labels, n); - impl_->add(data, lbls, blocksize_exp); + impl_->add(data, lbls, blocksize); + }); + } + + Status add(size_t n, const size_t* labels, const float* x) noexcept override { + return runtime_error_wrapper([&] { + svs::data::ConstSimpleDataView data{x, n, impl_->dimensions()}; + std::span lbls(labels, n); + impl_->add(data, lbls); }); } @@ -79,6 +87,8 @@ struct DynamicVamanaIndexManagerBase : public DynamicVamanaIndex { }); } + lib::PowerOfTwo blocksize_bytes() const noexcept { return impl_->blocksize_bytes(); } + Status search( size_t n, const float* x, diff --git a/bindings/cpp/src/dynamic_vamana_index_impl.h b/bindings/cpp/src/dynamic_vamana_index_impl.h index 697f3fda6..8dc1e5a3c 100644 --- a/bindings/cpp/src/dynamic_vamana_index_impl.h +++ b/bindings/cpp/src/dynamic_vamana_index_impl.h @@ -63,6 +63,8 @@ class DynamicVamanaIndexImpl { size_t size() const { return impl_ ? impl_->size() : 0; } + lib::PowerOfTwo blocksize_bytes() const { return impl_->blocksize_bytes(); } + size_t dimensions() const { return dim_; } MetricType metric_type() const { return metric_type_; } @@ -72,9 +74,19 @@ class DynamicVamanaIndexImpl { void add(data::ConstSimpleDataView data, std::span labels, - int blocksize_exp) { + IndexBlockSize blocksize) { + if (!impl_) { + return init_impl(data, labels, blocksize); + } + + impl_->add_points(data, labels); + } + + void add(data::ConstSimpleDataView data, std::span labels) { if (!impl_) { - return init_impl(data, labels, blocksize_exp); + IndexBlockSize blocksize(data::BlockingParameters::default_blocksize_bytes.raw() + ); + return init_impl(data, labels, blocksize); } impl_->add_points(data, labels); @@ -392,17 +404,16 @@ class DynamicVamanaIndexImpl { const index::vamana::VamanaBuildParameters& parameters, const svs::data::ConstSimpleDataView& data, std::span labels, - int blocksize_exp, + IndexBlockSize blocksize, StorageArgs&&... storage_args ) { auto threadpool = default_threadpool(); - auto blocksize_bytes = svs::lib::PowerOfTwo(blocksize_exp); auto storage = make_storage( std::forward(tag), data, threadpool, - blocksize_bytes, + blocksize.BlockSizeBytes(), std::forward(storage_args)... ); @@ -421,7 +432,7 @@ class DynamicVamanaIndexImpl { virtual void init_impl( data::ConstSimpleDataView data, std::span labels, - int blocksize_exp + IndexBlockSize blocksize ) { impl_.reset(storage::dispatch_storage_kind( get_storage_kind(), @@ -429,7 +440,7 @@ class DynamicVamanaIndexImpl { auto&& tag, data::ConstSimpleDataView data, std::span labels, - int blocksize_exp + IndexBlockSize blocksize ) { using Tag = std::decay_t; return build_impl( @@ -438,12 +449,12 @@ class DynamicVamanaIndexImpl { this->vamana_build_parameters(), data, labels, - blocksize_exp + blocksize ); }, data, labels, - blocksize_exp + blocksize )); } diff --git a/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h b/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h index 3822c238f..a97187f9c 100644 --- a/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h +++ b/bindings/cpp/src/dynamic_vamana_index_leanvec_impl.h @@ -94,7 +94,7 @@ struct DynamicVamanaIndexLeanVecImpl : public DynamicVamanaIndexImpl { void init_impl( data::ConstSimpleDataView data, std::span labels, - int blocksize_exp + IndexBlockSize blocksize ) override { assert(storage::is_leanvec_storage(this->storage_kind_)); impl_.reset(dispatch_leanvec_storage_kind( @@ -103,7 +103,7 @@ struct DynamicVamanaIndexLeanVecImpl : public DynamicVamanaIndexImpl { auto&& tag, data::ConstSimpleDataView data, std::span labels, - int blocksize_exp + IndexBlockSize blocksize ) { using Tag = std::decay_t; return DynamicVamanaIndexImpl::build_impl( @@ -112,14 +112,14 @@ struct DynamicVamanaIndexLeanVecImpl : public DynamicVamanaIndexImpl { this->vamana_build_parameters(), data, labels, - blocksize_exp, + blocksize, this->leanvec_dims_, this->leanvec_matrices_ ); }, data, labels, - blocksize_exp + blocksize )); } diff --git a/bindings/cpp/src/flat_index_impl.h b/bindings/cpp/src/flat_index_impl.h index 1e12dfa4b..2f669597d 100644 --- a/bindings/cpp/src/flat_index_impl.h +++ b/bindings/cpp/src/flat_index_impl.h @@ -19,6 +19,7 @@ #include "svs_runtime_utils.h" #include +#include #include #include diff --git a/bindings/cpp/src/svs_runtime_utils.h b/bindings/cpp/src/svs_runtime_utils.h index abecfc3c8..968e211b0 100644 --- a/bindings/cpp/src/svs_runtime_utils.h +++ b/bindings/cpp/src/svs_runtime_utils.h @@ -222,8 +222,7 @@ template struct StorageFactory& data, Pool& pool, - svs::lib::PowerOfTwo blocksize_bytes = - svs::data::BlockingParameters::default_blocksize_bytes + svs::lib::PowerOfTwo blocksize_bytes ) { auto parameters = svs::data::BlockingParameters{.blocksize_bytes = blocksize_bytes}; typename StorageType::allocator_type alloc(parameters); @@ -240,6 +239,11 @@ template struct StorageFactory + static StorageType init(const svs::data::ConstSimpleDataView& data, Pool& pool) { + return init(data, pool, svs::data::BlockingParameters::default_blocksize_bytes); + } + template static StorageType load(const std::filesystem::path& path, Args&&... args) { return svs::lib::load_from_disk(path, SVS_FWD(args)...); diff --git a/bindings/cpp/tests/CMakeLists.txt b/bindings/cpp/tests/CMakeLists.txt index 1fb81ec5f..2ceab78f1 100644 --- a/bindings/cpp/tests/CMakeLists.txt +++ b/bindings/cpp/tests/CMakeLists.txt @@ -37,6 +37,14 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(Catch2) + +FetchContent_Declare( + fmt + GIT_REPOSITORY https://github.com/fmtlib/fmt + GIT_TAG 11.2.0 +) +FetchContent_MakeAvailable(fmt) + set(CMAKE_CXX_STANDARD ${PRESET_CMAKE_CXX_STANDARD}) # Add test executable @@ -50,6 +58,7 @@ add_executable(svs_runtime_test ${TEST_SOURCES}) target_link_libraries(svs_runtime_test PRIVATE svs_runtime Catch2::Catch2WithMain + fmt::fmt ) # Set C++ standard @@ -64,6 +73,7 @@ set_target_properties(svs_runtime_test PROPERTIES target_include_directories(svs_runtime_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../include + ${CMAKE_CURRENT_SOURCE_DIR}/../../../include ) # Enable testing with CTest diff --git a/bindings/cpp/tests/runtime_test.cpp b/bindings/cpp/tests/runtime_test.cpp index 94e7525a5..52e6ac13a 100644 --- a/bindings/cpp/tests/runtime_test.cpp +++ b/bindings/cpp/tests/runtime_test.cpp @@ -83,7 +83,7 @@ void write_and_read_index( size_t n, size_t d, svs::runtime::v0::StorageKind storage_kind, - int blocksize_exp2, + svs::runtime::v0::IndexBlockSize blocksize, svs::runtime::v0::MetricType metric = svs::runtime::v0::MetricType::L2 ) { // Build index @@ -100,7 +100,7 @@ void write_and_read_index( std::vector labels(n); std::iota(labels.begin(), labels.end(), 0); - status = index->add(n, labels.data(), xb.data(), blocksize_exp2); + status = index->add(n, labels.data(), xb.data(), blocksize); CATCH_REQUIRE(status.ok()); svs_test::prepare_temp_directory(); @@ -142,7 +142,9 @@ void write_and_read_index( // Helper that writes and reads and index of requested size // Reports memory usage -UsageInfo run_save_and_load_test(const size_t target_mibytes, int blocksize_exp2) { +UsageInfo run_save_and_load_test( + const size_t target_mibytes, svs::runtime::v0::IndexBlockSize blocksize +) { // Generate requested MiB of test data constexpr size_t mem_test_d = 128; const size_t target_bytes = target_mibytes * 1024 * 1024; @@ -172,8 +174,7 @@ UsageInfo run_save_and_load_test(const size_t target_mibytes, int blocksize_exp2 ); CATCH_REQUIRE(status.ok()); CATCH_REQUIRE(index != nullptr); - status = - index->add(mem_test_n, labels.data(), large_test_data.data(), blocksize_exp2); + status = index->add(mem_test_n, labels.data(), large_test_data.data(), blocksize); CATCH_REQUIRE(status.ok()); std::ofstream out(filename, std::ios::binary); @@ -226,7 +227,12 @@ CATCH_TEST_CASE("WriteAndReadIndexSVS", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::FP32, 30 + build_func, + test_data, + test_n, + test_d, + svs::runtime::v0::StorageKind::FP32, + svs::runtime::v0::IndexBlockSize(30) ); } @@ -243,7 +249,12 @@ CATCH_TEST_CASE("WriteAndReadIndexSVSFP16", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::FP16, 30 + build_func, + test_data, + test_n, + test_d, + svs::runtime::v0::StorageKind::FP16, + svs::runtime::v0::IndexBlockSize(30) ); } @@ -260,7 +271,12 @@ CATCH_TEST_CASE("WriteAndReadIndexSVSSQI8", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::SQI8, 30 + build_func, + test_data, + test_n, + test_d, + svs::runtime::v0::StorageKind::SQI8, + svs::runtime::v0::IndexBlockSize(30) ); } @@ -277,7 +293,12 @@ CATCH_TEST_CASE("WriteAndReadIndexSVSLVQ4x4", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::LVQ4x4, 30 + build_func, + test_data, + test_n, + test_d, + svs::runtime::v0::StorageKind::LVQ4x4, + svs::runtime::v0::IndexBlockSize(30) ); } @@ -295,7 +316,12 @@ CATCH_TEST_CASE("WriteAndReadIndexSVSVamanaLeanVec4x4", "[runtime]") { ); }; write_and_read_index( - build_func, test_data, test_n, test_d, svs::runtime::v0::StorageKind::LeanVec4x4, 30 + build_func, + test_data, + test_n, + test_d, + svs::runtime::v0::StorageKind::LeanVec4x4, + svs::runtime::v0::IndexBlockSize(30) ); } @@ -332,21 +358,20 @@ CATCH_TEST_CASE("LeanVecWithTrainingData", "[runtime]") { svs::runtime::v0::DynamicVamanaIndex::destroy(index); } -CATCH_TEST_CASE("LeanVecWithTrainingDataCustomBlockSize", "[runtime]") { +CATCH_TEST_CASE("TrainingDataCustomBlockSize", "[runtime]") { const auto& test_data = get_test_data(); // Build LeanVec index with explicit training svs::runtime::v0::DynamicVamanaIndex* index = nullptr; svs::runtime::v0::VamanaIndex::BuildParams build_params{64}; - svs::runtime::v0::Status status = svs::runtime::v0::DynamicVamanaIndexLeanVec::build( + svs::runtime::v0::Status status = svs::runtime::v0::DynamicVamanaIndex::build( &index, test_d, svs::runtime::v0::MetricType::L2, - svs::runtime::v0::StorageKind::LeanVec4x4, - 32, + svs::runtime::v0::StorageKind::FP32, build_params ); if (!svs::runtime::v0::DynamicVamanaIndex::check_storage_kind( - svs::runtime::v0::StorageKind::LeanVec4x4 + svs::runtime::v0::StorageKind::FP32 ) .ok()) { CATCH_REQUIRE(!status.ok()); @@ -359,10 +384,17 @@ CATCH_TEST_CASE("LeanVecWithTrainingDataCustomBlockSize", "[runtime]") { std::vector labels(test_n); std::iota(labels.begin(), labels.end(), 0); - int block_size_exp = 17; // block_size = 2^block_size_exp - status = index->add(test_n, labels.data(), test_data.data(), block_size_exp); + int block_size_exp = 17; // block_size_bytes = 2^block_size_exp + status = index->add( + test_n, + labels.data(), + test_data.data(), + svs::runtime::v0::IndexBlockSize(block_size_exp) + ); CATCH_REQUIRE(status.ok()); + CATCH_REQUIRE(index->blocksize_bytes().raw() == block_size_exp); + svs::runtime::v0::DynamicVamanaIndex::destroy(index); } @@ -435,7 +467,9 @@ CATCH_TEST_CASE("SearchWithIDFilter", "[runtime]") { // Add data std::vector labels(test_n); std::iota(labels.begin(), labels.end(), 0); - status = index->add(test_n, labels.data(), test_data.data(), 30); + status = index->add( + test_n, labels.data(), test_data.data(), svs::runtime::v0::IndexBlockSize(30) + ); CATCH_REQUIRE(status.ok()); const int nq = 8; @@ -481,7 +515,9 @@ CATCH_TEST_CASE("RangeSearchFunctional", "[runtime]") { // Add data std::vector labels(test_n); std::iota(labels.begin(), labels.end(), 0); - status = index->add(test_n, labels.data(), test_data.data(), 30); + status = index->add( + test_n, labels.data(), test_data.data(), svs::runtime::v0::IndexBlockSize(30) + ); CATCH_REQUIRE(status.ok()); const int nq = 5; @@ -508,19 +544,19 @@ CATCH_TEST_CASE("MemoryUsageOnLoad", "[runtime][memory]") { }; CATCH_SECTION("SmallIndex") { - auto stats = run_save_and_load_test(10, 30); + auto stats = run_save_and_load_test(10, svs::runtime::v0::IndexBlockSize(30)); CATCH_REQUIRE(stats.file_size < 20 * 1024 * 1024); CATCH_REQUIRE(stats.rss_increase < threshold(stats.file_size)); } CATCH_SECTION("MediumIndex") { - auto stats = run_save_and_load_test(50, 30); + auto stats = run_save_and_load_test(50, svs::runtime::v0::IndexBlockSize(30)); CATCH_REQUIRE(stats.file_size < 100 * 1024 * 1024); CATCH_REQUIRE(stats.rss_increase < threshold(stats.file_size)); } CATCH_SECTION("LargeIndex") { - auto stats = run_save_and_load_test(200, 30); + auto stats = run_save_and_load_test(200, svs::runtime::v0::IndexBlockSize(30)); CATCH_REQUIRE(stats.file_size < 400 * 1024 * 1024); CATCH_REQUIRE(stats.rss_increase < threshold(stats.file_size)); } diff --git a/bindings/cpp/tests/utils.h b/bindings/cpp/tests/utils.h index 8d1bc89f6..ae9974296 100644 --- a/bindings/cpp/tests/utils.h +++ b/bindings/cpp/tests/utils.h @@ -32,7 +32,8 @@ namespace svs_test { inline std::filesystem::path temp_directory() { // Use /tmp for runtime binding tests - return std::filesystem::path("/tmp/svs_runtime_test"); + // return std::filesystem::path("/tmp/svs_runtime_test"); + return std::filesystem::path("~/svs_runtime_test"); } inline bool cleanup_temp_directory() { diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 169be1995..86ac4b62a 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -722,6 +722,19 @@ class MutableVamanaIndex { return slots; } + lib::PowerOfTwo blocksize_bytes() const { + if constexpr (std::is_same_v< + Data, + svs::data::SimpleData< + typename Data::element_type, + Data::value_type::extent, + typename Data::allocator_type>>) { + return data_.blocksize_bytes(); + } else { + return lib::PowerOfTwo(0); + } + } + /// /// Delete all IDs stored in the random-access container `ids`. /// diff --git a/include/svs/orchestrators/dynamic_vamana.h b/include/svs/orchestrators/dynamic_vamana.h index da1af7b19..3f082e8de 100644 --- a/include/svs/orchestrators/dynamic_vamana.h +++ b/include/svs/orchestrators/dynamic_vamana.h @@ -43,6 +43,8 @@ class DynamicVamanaInterface : public VamanaInterface { bool reuse_empty = false ) = 0; + virtual lib::PowerOfTwo blocksize_bytes() const = 0; + virtual void delete_points(std::span ids) = 0; virtual void consolidate() = 0; virtual void compact(size_t batchsize = 1'000'000) = 0; @@ -80,6 +82,8 @@ class DynamicVamanaImpl : public VamanaImpl ids) override { impl().delete_entries(ids); } void consolidate() override { impl().consolidate(); } void compact(size_t batchsize) override { impl().compact(batchsize); } @@ -185,6 +189,8 @@ class DynamicVamana : public manager::IndexManager { return *this; } + lib::PowerOfTwo blocksize_bytes() const { return impl_->blocksize_bytes(); } + DynamicVamana& delete_points(std::span ids) { impl_->delete_points(ids); return *this; From 321ac45280a40723ebf8698fa830a2326ad894fb Mon Sep 17 00:00:00 2001 From: Dmitry Razdoburdin Date: Mon, 8 Dec 2025 05:31:38 -0800 Subject: [PATCH 04/10] blocksize getter for blocked alloc only --- bindings/cpp/tests/utils.h | 3 +-- include/svs/index/vamana/dynamic_index.h | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/bindings/cpp/tests/utils.h b/bindings/cpp/tests/utils.h index ae9974296..8d1bc89f6 100644 --- a/bindings/cpp/tests/utils.h +++ b/bindings/cpp/tests/utils.h @@ -32,8 +32,7 @@ namespace svs_test { inline std::filesystem::path temp_directory() { // Use /tmp for runtime binding tests - // return std::filesystem::path("/tmp/svs_runtime_test"); - return std::filesystem::path("~/svs_runtime_test"); + return std::filesystem::path("/tmp/svs_runtime_test"); } inline bool cleanup_temp_directory() { diff --git a/include/svs/index/vamana/dynamic_index.h b/include/svs/index/vamana/dynamic_index.h index 86ac4b62a..699487800 100644 --- a/include/svs/index/vamana/dynamic_index.h +++ b/include/svs/index/vamana/dynamic_index.h @@ -260,6 +260,12 @@ class MutableVamanaIndex { ); } + template