From bfb03fd2027a6b0f101c4aedf46dcc83ea23f8a0 Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 18 Dec 2025 10:59:03 -0800 Subject: [PATCH 1/3] delete extra files --- .../example_vamana_with_compression.cpp | 101 ------------------ 1 file changed, 101 deletions(-) delete mode 100644 examples/cpp/shared/example_vamana_with_compression.cpp diff --git a/examples/cpp/shared/example_vamana_with_compression.cpp b/examples/cpp/shared/example_vamana_with_compression.cpp deleted file mode 100644 index 362abee8a..000000000 --- a/examples/cpp/shared/example_vamana_with_compression.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2025 Intel Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// SVS -#include "svs/core/recall.h" -#include "svs/extensions/flat/leanvec.h" -#include "svs/extensions/flat/lvq.h" -#include "svs/extensions/vamana/leanvec.h" -#include "svs/extensions/vamana/lvq.h" -#include "svs/orchestrators/dynamic_vamana.h" -#include "svs/orchestrators/exhaustive.h" -#include "svs/orchestrators/vamana.h" - -int main() { - // STEP 1: Compress Data with LeanVec, reducing dimensionality to leanvec_dim dimensions - // and using 4 and 8 bits for primary and secondary levels respectively. - //! [Compress data] - const size_t num_threads = 4; - size_t padding = 32; - size_t leanvec_dim = 64; - auto threadpool = svs::threads::as_threadpool(num_threads); - auto loaded = - svs::VectorDataLoader(std::filesystem::path(SVS_DATA_DIR) / "data_f32.svs") - .load(); - auto data = svs::leanvec::LeanDataset< - svs::leanvec::UsingLVQ<4>, - svs::leanvec::UsingLVQ<8>, - svs::Dynamic, - svs::Dynamic>:: - reduce( - loaded, - std::nullopt, - threadpool, - padding, - svs::lib::MaybeStatic(leanvec_dim) - ); - //! [Compress data] - - // STEP 2: Build Vamana Index - //! [Index Build] - auto parameters = svs::index::vamana::VamanaBuildParameters{}; - svs::Vamana index = svs::Vamana::build( - parameters, data, svs::distance::DistanceL2(), num_threads - ); - //! [Index Build] - - // STEP 3: Search the Index - //! [Perform Queries] - const size_t search_window_size = 50; - const size_t n_neighbors = 10; - index.set_search_window_size(search_window_size); - - auto queries = - svs::load_data(std::filesystem::path(SVS_DATA_DIR) / "queries_f32.fvecs"); - auto results = index.search(queries, n_neighbors); - //! [Perform Queries] - - //! [Recall] - auto groundtruth = svs::load_data( - std::filesystem::path(SVS_DATA_DIR) / "groundtruth_euclidean.ivecs" - ); - double recall = svs::k_recall_at_n(groundtruth, results, n_neighbors, n_neighbors); - - fmt::print("Recall@{} = {:.4f}\n", n_neighbors, recall); - //! [Recall] - - // STEP 4: Saving and reloading the index - //! [Saving Loading] - index.save("config", "graph", "data"); - index = svs::Vamana::assemble( - "config", - svs::GraphLoader("graph"), - svs::lib::load_from_disk, - svs::leanvec::UsingLVQ<8>, - svs::Dynamic, - svs::Dynamic>>("data", padding), - svs::distance::DistanceL2(), - num_threads - ); - //! [Saving Loading] - index.set_search_window_size(search_window_size); - recall = svs::k_recall_at_n(groundtruth, results, n_neighbors, n_neighbors); - - fmt::print("Recall@{} after saving and reloading = {:.4f}\n", n_neighbors, recall); - - return 0; -} From 8d62fbcac25a28ae6f65097b8472e613e7436f8d Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 18 Dec 2025 11:02:07 -0800 Subject: [PATCH 2/3] restore deleted example --- .../example_vamana_with_compression.cpp | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 examples/cpp/shared/example_vamana_with_compression.cpp diff --git a/examples/cpp/shared/example_vamana_with_compression.cpp b/examples/cpp/shared/example_vamana_with_compression.cpp new file mode 100644 index 000000000..362abee8a --- /dev/null +++ b/examples/cpp/shared/example_vamana_with_compression.cpp @@ -0,0 +1,101 @@ +/* + * Copyright 2025 Intel Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// SVS +#include "svs/core/recall.h" +#include "svs/extensions/flat/leanvec.h" +#include "svs/extensions/flat/lvq.h" +#include "svs/extensions/vamana/leanvec.h" +#include "svs/extensions/vamana/lvq.h" +#include "svs/orchestrators/dynamic_vamana.h" +#include "svs/orchestrators/exhaustive.h" +#include "svs/orchestrators/vamana.h" + +int main() { + // STEP 1: Compress Data with LeanVec, reducing dimensionality to leanvec_dim dimensions + // and using 4 and 8 bits for primary and secondary levels respectively. + //! [Compress data] + const size_t num_threads = 4; + size_t padding = 32; + size_t leanvec_dim = 64; + auto threadpool = svs::threads::as_threadpool(num_threads); + auto loaded = + svs::VectorDataLoader(std::filesystem::path(SVS_DATA_DIR) / "data_f32.svs") + .load(); + auto data = svs::leanvec::LeanDataset< + svs::leanvec::UsingLVQ<4>, + svs::leanvec::UsingLVQ<8>, + svs::Dynamic, + svs::Dynamic>:: + reduce( + loaded, + std::nullopt, + threadpool, + padding, + svs::lib::MaybeStatic(leanvec_dim) + ); + //! [Compress data] + + // STEP 2: Build Vamana Index + //! [Index Build] + auto parameters = svs::index::vamana::VamanaBuildParameters{}; + svs::Vamana index = svs::Vamana::build( + parameters, data, svs::distance::DistanceL2(), num_threads + ); + //! [Index Build] + + // STEP 3: Search the Index + //! [Perform Queries] + const size_t search_window_size = 50; + const size_t n_neighbors = 10; + index.set_search_window_size(search_window_size); + + auto queries = + svs::load_data(std::filesystem::path(SVS_DATA_DIR) / "queries_f32.fvecs"); + auto results = index.search(queries, n_neighbors); + //! [Perform Queries] + + //! [Recall] + auto groundtruth = svs::load_data( + std::filesystem::path(SVS_DATA_DIR) / "groundtruth_euclidean.ivecs" + ); + double recall = svs::k_recall_at_n(groundtruth, results, n_neighbors, n_neighbors); + + fmt::print("Recall@{} = {:.4f}\n", n_neighbors, recall); + //! [Recall] + + // STEP 4: Saving and reloading the index + //! [Saving Loading] + index.save("config", "graph", "data"); + index = svs::Vamana::assemble( + "config", + svs::GraphLoader("graph"), + svs::lib::load_from_disk, + svs::leanvec::UsingLVQ<8>, + svs::Dynamic, + svs::Dynamic>>("data", padding), + svs::distance::DistanceL2(), + num_threads + ); + //! [Saving Loading] + index.set_search_window_size(search_window_size); + recall = svs::k_recall_at_n(groundtruth, results, n_neighbors, n_neighbors); + + fmt::print("Recall@{} after saving and reloading = {:.4f}\n", n_neighbors, recall); + + return 0; +} From fbd43063c52970789bd4e9288be8c22e9f5620ed Mon Sep 17 00:00:00 2001 From: Andreas Huber Date: Thu, 18 Dec 2025 11:32:22 -0800 Subject: [PATCH 3/3] bring back wip file --- include/svs/index/ivf/common.h | 64 +++++++++++++++++----------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/include/svs/index/ivf/common.h b/include/svs/index/ivf/common.h index 28f12151b..6bbd3eaf6 100644 --- a/include/svs/index/ivf/common.h +++ b/include/svs/index/ivf/common.h @@ -242,39 +242,39 @@ void compute_matmul( n // const int ldc ); } else if constexpr (std::is_same_v) { - cblas_gemm_bf16bf16f32( - CblasRowMajor, // CBLAS_LAYOUT layout - CblasNoTrans, // CBLAS_TRANSPOSE TransA - CblasTrans, // CBLAS_TRANSPOSE TransB - m, // const int M - n, // const int N - k, // const int K - 1.0, // float alpha - (const uint16_t*)data, // const *uint16_t A - k, // const int lda - (const uint16_t*)centroids, // const uint16_t* B - k, // const int ldb - 0.0, // const float beta - results, // float* c - n // const int ldc - ); + //cblas_gemm_bf16bf16f32( + //CblasRowMajor, // CBLAS_LAYOUT layout + //CblasNoTrans, // CBLAS_TRANSPOSE TransA + //CblasTrans, // CBLAS_TRANSPOSE TransB + //m, // const int M + //n, // const int N + //k, // const int K + //1.0, // float alpha + //(const uint16_t*)data, // const *uint16_t A + //k, // const int lda + //(const uint16_t*)centroids, // const uint16_t* B + //k, // const int ldb + //0.0, // const float beta + //results, // float* c + //n // const int ldc + //); } else if constexpr (std::is_same_v) { - cblas_gemm_f16f16f32( - CblasRowMajor, // CBLAS_LAYOUT layout - CblasNoTrans, // CBLAS_TRANSPOSE TransA - CblasTrans, // CBLAS_TRANSPOSE TransB - m, // const int M - n, // const int N - k, // const int K - 1.0, // float alpha - (const uint16_t*)data, // const *uint16_t A - k, // const int lda - (const uint16_t*)centroids, // const uint16_t* B - k, // const int ldb - 0.0, // const float beta - results, // float* c - n // const int ldc - ); + //cblas_gemm_f16f16f32( + //CblasRowMajor, // CBLAS_LAYOUT layout + //CblasNoTrans, // CBLAS_TRANSPOSE TransA + //CblasTrans, // CBLAS_TRANSPOSE TransB + //m, // const int M + //n, // const int N + //k, // const int K + //1.0, // float alpha + //(const uint16_t*)data, // const *uint16_t A + //k, // const int lda + //(const uint16_t*)centroids, // const uint16_t* B + //k, // const int ldb + //0.0, // const float beta + //results, // float* c + //n // const int ldc + //); } else { throw ANNEXCEPTION("GEMM type not supported!"); }