diff --git a/src/VecSim/algorithms/hnsw/hnsw.h b/src/VecSim/algorithms/hnsw/hnsw.h index ee2267bbe..996747d80 100644 --- a/src/VecSim/algorithms/hnsw/hnsw.h +++ b/src/VecSim/algorithms/hnsw/hnsw.h @@ -2143,12 +2143,12 @@ VecSimDebugInfoIterator *HNSWIndex::debugInfoIterator() cons .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.entrypoint}}}); infoIterator->addInfoField( - VecSim_InfoField{.fieldName = VecSimCommonStrings::HNSW_EPSILON_STRING, + VecSim_InfoField{.fieldName = VecSimCommonStrings::EPSILON_STRING, .fieldType = INFOFIELD_FLOAT64, .fieldValue = {FieldValue{.floatingPointValue = info.hnswInfo.epsilon}}}); infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::HNSW_NUM_MARKED_DELETED, + .fieldName = VecSimCommonStrings::NUM_MARKED_DELETED, .fieldType = INFOFIELD_UINT64, .fieldValue = {FieldValue{.uintegerValue = info.hnswInfo.numberOfMarkedDeletedNodes}}}); diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index b87ae8154..bea0bf80f 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -1145,6 +1145,9 @@ VecSimIndexDebugInfo TieredHNSWIndex::debugInfo() const { HnswTieredInfo hnswTieredInfo = {.pendingSwapJobsThreshold = this->pendingSwapJobsThreshold}; info.tieredInfo.specificTieredBackendInfo.hnswTieredInfo = hnswTieredInfo; + info.tieredInfo.backgroundIndexing = + info.tieredInfo.frontendCommonInfo.indexSize > 0 ? VecSimBool_TRUE : VecSimBool_FALSE; + return info; } @@ -1154,6 +1157,11 @@ VecSimDebugInfoIterator *TieredHNSWIndex::debugInfoIterator( // Get the base tiered fields. auto *infoIterator = VecSimTieredIndex::debugInfoIterator(); + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING, + .fieldType = INFOFIELD_INT64, + .fieldValue = {FieldValue{.integerValue = info.tieredInfo.backgroundIndexing}}}); + // Tiered HNSW specific param. infoIterator->addInfoField(VecSim_InfoField{ .fieldName = VecSimCommonStrings::TIERED_HNSW_SWAP_JOBS_THRESHOLD_STRING, diff --git a/src/VecSim/algorithms/svs/svs.h b/src/VecSim/algorithms/svs/svs.h index 35997254f..5e0e5615f 100644 --- a/src/VecSim/algorithms/svs/svs.h +++ b/src/VecSim/algorithms/svs/svs.h @@ -374,7 +374,7 @@ class SVSIndex : public VecSimIndexAbstract, fl VecSimDebugInfoIterator *debugInfoIterator() const override { VecSimIndexDebugInfo info = this->debugInfo(); // For readability. Update this number when needed. - size_t numberOfInfoFields = 10; + size_t numberOfInfoFields = 23; VecSimDebugInfoIterator *infoIterator = new VecSimDebugInfoIterator(numberOfInfoFields, this->allocator); @@ -384,10 +384,78 @@ class SVSIndex : public VecSimIndexAbstract, fl .fieldValue = { FieldValue{.stringValue = VecSimAlgo_ToString(info.commonInfo.basicInfo.algo)}}}); this->addCommonInfoToIterator(infoIterator, info.commonInfo); + infoIterator->addInfoField(VecSim_InfoField{ .fieldName = VecSimCommonStrings::BLOCK_SIZE_STRING, .fieldType = INFOFIELD_UINT64, .fieldValue = {FieldValue{.uintegerValue = info.commonInfo.basicInfo.blockSize}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SVS_QUANT_BITS_STRING, + .fieldType = INFOFIELD_STRING, + .fieldValue = { + FieldValue{.stringValue = VecSimQuantBits_ToString(info.svsInfo.quantBits)}}}); + + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::SVS_ALPHA_STRING, + .fieldType = INFOFIELD_FLOAT64, + .fieldValue = {FieldValue{.floatingPointValue = info.svsInfo.alpha}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SVS_GRAPH_MAX_DEGREE_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.graphMaxDegree}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SVS_CONSTRUCTION_WS_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.constructionWindowSize}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SVS_MAX_CANDIDATE_POOL_SIZE_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.maxCandidatePoolSize}}}); + + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::SVS_PRUNE_TO_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.pruneTo}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SVS_USE_SEARCH_HISTORY_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.useSearchHistory}}}); + + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::SVS_NUM_THREADS_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.numThreads}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::NUM_MARKED_DELETED, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.numberOfMarkedDeletedNodes}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SVS_SEARCH_WS_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.searchWindowSize}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::SVS_SEARCH_BC_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.searchBufferCapacity}}}); + + infoIterator->addInfoField( + VecSim_InfoField{.fieldName = VecSimCommonStrings::SVS_LEANVEC_DIM_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{.uintegerValue = info.svsInfo.leanvecDim}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::EPSILON_STRING, + .fieldType = INFOFIELD_FLOAT64, + .fieldValue = {FieldValue{.floatingPointValue = info.svsInfo.epsilon}}}); + return infoIterator; } diff --git a/src/VecSim/algorithms/svs/svs_tiered.h b/src/VecSim/algorithms/svs/svs_tiered.h index 065ce93fb..a6a458e38 100644 --- a/src/VecSim/algorithms/svs/svs_tiered.h +++ b/src/VecSim/algorithms/svs/svs_tiered.h @@ -861,12 +861,17 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimIndexDebugInfo debugInfo() const override { auto info = Base::debugInfo(); + SvsTieredInfo svsTieredInfo = {.trainingTriggerThreshold = this->trainingTriggerThreshold, .updateTriggerThreshold = this->updateTriggerThreshold, .updateJobWaitTime = this->updateJobWaitTime, .indexUpdateScheduled = static_cast(this->indexUpdateScheduled.test())}; info.tieredInfo.specificTieredBackendInfo.svsTieredInfo = svsTieredInfo; + info.tieredInfo.backgroundIndexing = + svsTieredInfo.indexUpdateScheduled && info.tieredInfo.frontendCommonInfo.indexSize > 0 + ? VecSimBool_TRUE + : VecSimBool_FALSE; return info; } @@ -881,7 +886,33 @@ class TieredSVSIndex : public VecSimTieredIndex { VecSimDebugInfoIterator *debugInfoIterator() const override { // Get the base tiered fields. auto *infoIterator = Base::debugInfoIterator(); - // TODO: Add SVS specific info. + VecSimIndexDebugInfo info = this->debugInfo(); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING, + .fieldType = INFOFIELD_INT64, + .fieldValue = {FieldValue{.integerValue = info.tieredInfo.backgroundIndexing}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::TIERED_SVS_TRAINING_THRESHOLD_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = { + FieldValue{.uintegerValue = info.tieredInfo.specificTieredBackendInfo.svsTieredInfo + .trainingTriggerThreshold}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::TIERED_SVS_UPDATE_THRESHOLD_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = { + FieldValue{.uintegerValue = info.tieredInfo.specificTieredBackendInfo.svsTieredInfo + .updateTriggerThreshold}}}); + + infoIterator->addInfoField(VecSim_InfoField{ + .fieldName = VecSimCommonStrings::TIERED_SVS_THREADS_RESERVE_TIMEOUT_STRING, + .fieldType = INFOFIELD_UINT64, + .fieldValue = {FieldValue{ + .uintegerValue = + info.tieredInfo.specificTieredBackendInfo.svsTieredInfo.updateJobWaitTime}}}); return infoIterator; } diff --git a/src/VecSim/utils/vec_utils.cpp b/src/VecSim/utils/vec_utils.cpp index 46babdb8f..65db2e500 100644 --- a/src/VecSim/utils/vec_utils.cpp +++ b/src/VecSim/utils/vec_utils.cpp @@ -49,14 +49,22 @@ const char *VecSimCommonStrings::MEMORY_STRING = "MEMORY"; const char *VecSimCommonStrings::HNSW_EF_RUNTIME_STRING = "EF_RUNTIME"; const char *VecSimCommonStrings::HNSW_M_STRING = "M"; const char *VecSimCommonStrings::HNSW_EF_CONSTRUCTION_STRING = "EF_CONSTRUCTION"; -const char *VecSimCommonStrings::HNSW_EPSILON_STRING = "EPSILON"; +const char *VecSimCommonStrings::EPSILON_STRING = "EPSILON"; const char *VecSimCommonStrings::HNSW_MAX_LEVEL = "MAX_LEVEL"; const char *VecSimCommonStrings::HNSW_ENTRYPOINT = "ENTRYPOINT"; -const char *VecSimCommonStrings::HNSW_NUM_MARKED_DELETED = "NUMBER_OF_MARKED_DELETED"; +const char *VecSimCommonStrings::NUM_MARKED_DELETED = "NUMBER_OF_MARKED_DELETED"; const char *VecSimCommonStrings::SVS_SEARCH_WS_STRING = "SEARCH_WINDOW_SIZE"; +const char *VecSimCommonStrings::SVS_CONSTRUCTION_WS_STRING = "CONSTRUCTION_WINDOW_SIZE"; const char *VecSimCommonStrings::SVS_SEARCH_BC_STRING = "SEARCH_BUFFER_CAPACITY"; const char *VecSimCommonStrings::SVS_USE_SEARCH_HISTORY_STRING = "USE_SEARCH_HISTORY"; +const char *VecSimCommonStrings::SVS_ALPHA_STRING = "ALPHA"; +const char *VecSimCommonStrings::SVS_QUANT_BITS_STRING = "QUANT_BITS"; +const char *VecSimCommonStrings::SVS_GRAPH_MAX_DEGREE_STRING = "GRAPH_MAX_DEGREE"; +const char *VecSimCommonStrings::SVS_MAX_CANDIDATE_POOL_SIZE_STRING = "MAX_CANDIDATE_POOL_SIZE"; +const char *VecSimCommonStrings::SVS_PRUNE_TO_STRING = "PRUNE_TO"; +const char *VecSimCommonStrings::SVS_NUM_THREADS_STRING = "NUM_THREADS"; +const char *VecSimCommonStrings::SVS_LEANVEC_DIM_STRING = "LEANVEC_DIMENSION"; const char *VecSimCommonStrings::BLOCK_SIZE_STRING = "BLOCK_SIZE"; const char *VecSimCommonStrings::SEARCH_MODE_STRING = "LAST_SEARCH_MODE"; @@ -68,8 +76,15 @@ const char *VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING = "BACKGROUND const char *VecSimCommonStrings::TIERED_BUFFER_LIMIT_STRING = "TIERED_BUFFER_LIMIT"; const char *VecSimCommonStrings::FRONTEND_INDEX_STRING = "FRONTEND_INDEX"; const char *VecSimCommonStrings::BACKEND_INDEX_STRING = "BACKEND_INDEX"; +// Tiered HNSW specific const char *VecSimCommonStrings::TIERED_HNSW_SWAP_JOBS_THRESHOLD_STRING = "TIERED_HNSW_SWAP_JOBS_THRESHOLD"; +// Tiered SVS specific +const char *VecSimCommonStrings::TIERED_SVS_TRAINING_THRESHOLD_STRING = + "TIERED_SVS_TRAINING_THRESHOLD"; +const char *VecSimCommonStrings::TIERED_SVS_UPDATE_THRESHOLD_STRING = "TIERED_SVS_UPDATE_THRESHOLD"; +const char *VecSimCommonStrings::TIERED_SVS_THREADS_RESERVE_TIMEOUT_STRING = + "TIERED_SVS_THREADS_RESERVE_TIMEOUT"; // Log levels const char *VecSimCommonStrings::LOG_DEBUG_STRING = "debug"; @@ -216,6 +231,28 @@ const char *VecSimSearchMode_ToString(VecSearchMode vecsimSearchMode) { return NULL; } +const char *VecSimQuantBits_ToString(VecSimSvsQuantBits quantBits) { + switch (quantBits) { + case VecSimSvsQuant_NONE: + return "NONE"; + case VecSimSvsQuant_Scalar: + return "Scalar"; + case VecSimSvsQuant_4: + return "4"; + case VecSimSvsQuant_8: + return "8"; + case VecSimSvsQuant_4x4: + return "4x4"; + case VecSimSvsQuant_4x8: + return "4x8"; + case VecSimSvsQuant_4x8_LeanVec: + return "4x8_LeanVec"; + case VecSimSvsQuant_8x8_LeanVec: + return "8x8_LeanVec"; + } + return NULL; +} + size_t VecSimType_sizeof(VecSimType type) { switch (type) { case VecSimType_FLOAT32: diff --git a/src/VecSim/utils/vec_utils.h b/src/VecSim/utils/vec_utils.h index e8e96be45..744eaef64 100644 --- a/src/VecSim/utils/vec_utils.h +++ b/src/VecSim/utils/vec_utils.h @@ -49,15 +49,23 @@ struct VecSimCommonStrings { static const char *HNSW_EF_RUNTIME_STRING; static const char *HNSW_EF_CONSTRUCTION_STRING; static const char *HNSW_M_STRING; - static const char *HNSW_EPSILON_STRING; + static const char *EPSILON_STRING; static const char *HNSW_MAX_LEVEL; static const char *HNSW_ENTRYPOINT; - static const char *HNSW_NUM_MARKED_DELETED; + static const char *NUM_MARKED_DELETED; // static const char *HNSW_VISITED_NODES_POOL_SIZE_STRING; static const char *SVS_SEARCH_WS_STRING; + static const char *SVS_CONSTRUCTION_WS_STRING; static const char *SVS_SEARCH_BC_STRING; static const char *SVS_USE_SEARCH_HISTORY_STRING; + static const char *SVS_ALPHA_STRING; + static const char *SVS_QUANT_BITS_STRING; + static const char *SVS_GRAPH_MAX_DEGREE_STRING; + static const char *SVS_MAX_CANDIDATE_POOL_SIZE_STRING; + static const char *SVS_PRUNE_TO_STRING; + static const char *SVS_NUM_THREADS_STRING; + static const char *SVS_LEANVEC_DIM_STRING; static const char *BLOCK_SIZE_STRING; static const char *SEARCH_MODE_STRING; @@ -69,7 +77,12 @@ struct VecSimCommonStrings { static const char *TIERED_BUFFER_LIMIT_STRING; static const char *FRONTEND_INDEX_STRING; static const char *BACKEND_INDEX_STRING; + // tiered HNSW specific static const char *TIERED_HNSW_SWAP_JOBS_THRESHOLD_STRING; + // tiered SVS specific + static const char *TIERED_SVS_TRAINING_THRESHOLD_STRING; + static const char *TIERED_SVS_UPDATE_THRESHOLD_STRING; + static const char *TIERED_SVS_THREADS_RESERVE_TIMEOUT_STRING; // Log levels static const char *LOG_DEBUG_STRING; @@ -100,6 +113,8 @@ const char *VecSimMetric_ToString(VecSimMetric vecsimMetric); const char *VecSimSearchMode_ToString(VecSearchMode vecsimSearchMode); +const char *VecSimQuantBits_ToString(VecSimSvsQuantBits quantBits); + size_t VecSimType_sizeof(VecSimType vecsimType); /** Returns the size in bytes of a stored or query vector */ diff --git a/src/VecSim/vec_sim.cpp b/src/VecSim/vec_sim.cpp index 37d4e936d..6238419ae 100644 --- a/src/VecSim/vec_sim.cpp +++ b/src/VecSim/vec_sim.cpp @@ -224,7 +224,7 @@ extern "C" VecSimResolveCode VecSimIndex_ResolveParams(VecSimIndex *index, VecSi VecSimParamResolver_OK) { return res; } - } else if (!strcasecmp(rparams[i].name, VecSimCommonStrings::HNSW_EPSILON_STRING)) { + } else if (!strcasecmp(rparams[i].name, VecSimCommonStrings::EPSILON_STRING)) { if ((res = _ResolveParams_Epsilon(index_type, rparams[i], qparams, query_type)) != VecSimParamResolver_OK) { return res; diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index 4cb086598..643155b80 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -70,6 +70,12 @@ typedef enum { VecSimOption_DISABLE = 2, } VecSimOptionMode; +typedef enum { + VecSimBool_TRUE = 1, + VecSimBool_FALSE = 0, + VecSimBool_UNSET = -1, +} VecSimBool; + // Distance metric typedef enum { VecSimMetric_L2, VecSimMetric_IP, VecSimMetric_Cosine } VecSimMetric; @@ -379,7 +385,7 @@ typedef struct { bfInfoStruct bfInfo; // The brute force index info. uint64_t management_layer_memory; // Memory consumption of the management layer. - bool backgroundIndexing; // Determines if the index is currently being indexed in the + VecSimBool backgroundIndexing; // Determines if the index is currently being indexed in the // background. size_t bufferLimit; // Maximum number of vectors allowed in the flat buffer. } tieredInfoStruct; diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index bb736306c..fddf17135 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -359,6 +359,9 @@ VecSimIndexDebugInfo VecSimTieredIndex::debugInfo() const { }; info.commonInfo.basicInfo = basic_info; + // NOTE: backgroundIndexing needs to be set by the backend index. + info.tieredInfo.backgroundIndexing = VecSimBool_UNSET; + switch (backendInfo.commonInfo.basicInfo.algo) { case VecSimAlgo_HNSWLIB: info.tieredInfo.backendInfo.hnswInfo = backendInfo.hnswInfo; @@ -376,7 +379,6 @@ VecSimIndexDebugInfo VecSimTieredIndex::debugInfo() const { info.tieredInfo.frontendCommonInfo = frontendInfo.commonInfo; info.tieredInfo.bfInfo = frontendInfo.bfInfo; - info.tieredInfo.backgroundIndexing = frontendInfo.commonInfo.indexSize > 0; info.tieredInfo.management_layer_memory = this->allocator->getAllocationSize(); info.tieredInfo.bufferLimit = this->flatBufferLimit; return info; @@ -402,11 +404,6 @@ VecSimDebugInfoIterator *VecSimTieredIndex::debugInfoIterato .fieldType = INFOFIELD_UINT64, .fieldValue = {FieldValue{.uintegerValue = info.tieredInfo.management_layer_memory}}}); - infoIterator->addInfoField(VecSim_InfoField{ - .fieldName = VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING, - .fieldType = INFOFIELD_UINT64, - .fieldValue = {FieldValue{.uintegerValue = info.tieredInfo.backgroundIndexing}}}); - infoIterator->addInfoField( VecSim_InfoField{.fieldName = VecSimCommonStrings::TIERED_BUFFER_LIMIT_STRING, .fieldType = INFOFIELD_UINT64, diff --git a/tests/unit/test_hnsw_tiered.cpp b/tests/unit/test_hnsw_tiered.cpp index 649790941..067330327 100644 --- a/tests/unit/test_hnsw_tiered.cpp +++ b/tests/unit/test_hnsw_tiered.cpp @@ -2855,77 +2855,8 @@ TYPED_TEST(HNSWTieredIndexTest, testInfoIterator) { VecSimIndexDebugInfo backendIndexInfo = tiered_index->backendIndex->debugInfo(); VecSimDebugInfoIterator *infoIterator = tiered_index->debugInfoIterator(); - EXPECT_EQ(infoIterator->numberOfFields(), 15); - - while (infoIterator->hasNext()) { - VecSim_InfoField *infoField = VecSimDebugInfoIterator_NextField(infoIterator); - - if (!strcmp(infoField->fieldName, VecSimCommonStrings::ALGORITHM_STRING)) { - // Algorithm type. - ASSERT_EQ(infoField->fieldType, INFOFIELD_STRING); - ASSERT_STREQ(infoField->fieldValue.stringValue, VecSimCommonStrings::TIERED_STRING); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::TYPE_STRING)) { - // Vector type. - ASSERT_EQ(infoField->fieldType, INFOFIELD_STRING); - ASSERT_STREQ(infoField->fieldValue.stringValue, - VecSimType_ToString(info.commonInfo.basicInfo.type)); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::DIMENSION_STRING)) { - // Vector dimension. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.basicInfo.dim); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::METRIC_STRING)) { - // Metric. - ASSERT_EQ(infoField->fieldType, INFOFIELD_STRING); - ASSERT_STREQ(infoField->fieldValue.stringValue, - VecSimMetric_ToString(info.commonInfo.basicInfo.metric)); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SEARCH_MODE_STRING)) { - // Search mode. - ASSERT_EQ(infoField->fieldType, INFOFIELD_STRING); - ASSERT_STREQ(infoField->fieldValue.stringValue, - VecSimSearchMode_ToString(info.commonInfo.lastMode)); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::INDEX_SIZE_STRING)) { - // Index size. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.indexSize); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::INDEX_LABEL_COUNT_STRING)) { - // Index label count. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.indexLabelCount); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::IS_MULTI_STRING)) { - // Is the index multi value. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.basicInfo.isMulti); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::MEMORY_STRING)) { - // Memory. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.memory); - } else if (!strcmp(infoField->fieldName, - VecSimCommonStrings::TIERED_MANAGEMENT_MEMORY_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.tieredInfo.management_layer_memory); - } else if (!strcmp(infoField->fieldName, - VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.tieredInfo.backgroundIndexing); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::FRONTEND_INDEX_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_ITERATOR); - compareFlatIndexInfoToIterator(frontendIndexInfo, infoField->fieldValue.iteratorValue); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::BACKEND_INDEX_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_ITERATOR); - compareHNSWIndexInfoToIterator(backendIndexInfo, infoField->fieldValue.iteratorValue); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::TIERED_BUFFER_LIMIT_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.tieredInfo.bufferLimit); - } else if (!strcmp(infoField->fieldName, - VecSimCommonStrings::TIERED_HNSW_SWAP_JOBS_THRESHOLD_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ( - infoField->fieldValue.uintegerValue, - info.tieredInfo.specificTieredBackendInfo.hnswTieredInfo.pendingSwapJobsThreshold); - } else { - FAIL(); - } - } + compareTieredIndexInfoToIterator(info, frontendIndexInfo, backendIndexInfo, infoIterator); + VecSimDebugInfoIterator_Free(infoIterator); } diff --git a/tests/unit/test_svs.cpp b/tests/unit/test_svs.cpp index 012f1857c..564bb7ce4 100644 --- a/tests/unit/test_svs.cpp +++ b/tests/unit/test_svs.cpp @@ -51,7 +51,8 @@ class SVSTest : public ::testing::Test { protected: void SetTypeParams(SVSParams ¶ms) { - params.quantBits = index_type_t::get_quant_bits(); + params.quantBits = params.quantBits == VecSimSvsQuant_NONE ? index_type_t::get_quant_bits() + : params.quantBits; params.type = index_type_t::get_index_type(); params.multi = false; } @@ -1064,86 +1065,94 @@ TYPED_TEST(SVSTest, test_basic_svs_info_iterator) { } TYPED_TEST(SVSTest, test_dynamic_svs_info_iterator) { + if (TypeParam::get_quant_bits() == VecSimSvsQuant_8 || + TypeParam::get_quant_bits() == VecSimSvsQuant_8x8_LeanVec) { + GTEST_SKIP() << "Already included in the test loop."; + } size_t d = 128; + for (auto quant_bits : {VecSimSvsQuant_NONE, VecSimSvsQuant_Scalar, VecSimSvsQuant_8, + VecSimSvsQuant_4, VecSimSvsQuant_4x4, VecSimSvsQuant_4x8, + VecSimSvsQuant_4x8_LeanVec, VecSimSvsQuant_8x8_LeanVec}) { - SVSParams params = { - .dim = d, - .metric = VecSimMetric_L2, - .blockSize = 1, - /* SVS-Vamana specifics */ - .alpha = 1.2, - .graph_max_degree = 64, - .construction_window_size = 20, - .max_candidate_pool_size = 1024, - .prune_to = 60, - .use_search_history = VecSimOption_ENABLE, - }; + SVSParams params = { + .dim = d, + .metric = VecSimMetric_L2, + .blockSize = 1, + /* SVS-Vamana specifics */ + .quantBits = quant_bits, + .alpha = 1.2, + .graph_max_degree = 64, + .construction_window_size = 20, + .max_candidate_pool_size = 1024, + .prune_to = 60, + .use_search_history = VecSimOption_ENABLE, + }; + VecSimIndex *index = this->CreateNewIndex(params); + ASSERT_INDEX(index); - VecSimIndex *index = this->CreateNewIndex(params); - ASSERT_INDEX(index); + VecSimIndexDebugInfo info = VecSimIndex_DebugInfo(index); + VecSimDebugInfoIterator *infoIter = VecSimIndex_DebugInfoIterator(index); + ASSERT_EQ(1, info.commonInfo.basicInfo.blockSize); + ASSERT_EQ(0, info.commonInfo.indexSize); + compareSVSIndexInfoToIterator(info, infoIter); + VecSimDebugInfoIterator_Free(infoIter); - VecSimIndexDebugInfo info = VecSimIndex_DebugInfo(index); - VecSimDebugInfoIterator *infoIter = VecSimIndex_DebugInfoIterator(index); - ASSERT_EQ(1, info.commonInfo.basicInfo.blockSize); - ASSERT_EQ(0, info.commonInfo.indexSize); - compareFlatIndexInfoToIterator(info, infoIter); - VecSimDebugInfoIterator_Free(infoIter); - - TEST_DATA_T v[d]; - for (size_t i = 0; i < d; i++) { - v[i] = (TEST_DATA_T)i; - } - // Add vector. - VecSimIndex_AddVector(index, v, 0); - info = VecSimIndex_DebugInfo(index); - infoIter = VecSimIndex_DebugInfoIterator(index); - ASSERT_EQ(1, info.commonInfo.indexSize); - compareFlatIndexInfoToIterator(info, infoIter); - VecSimDebugInfoIterator_Free(infoIter); + TEST_DATA_T v[d]; + for (size_t i = 0; i < d; i++) { + v[i] = (TEST_DATA_T)i; + } + // Add vector. + VecSimIndex_AddVector(index, v, 0); + info = VecSimIndex_DebugInfo(index); + infoIter = VecSimIndex_DebugInfoIterator(index); + ASSERT_EQ(1, info.commonInfo.indexSize); + compareSVSIndexInfoToIterator(info, infoIter); + VecSimDebugInfoIterator_Free(infoIter); - // Delete vector. - VecSimIndex_DeleteVector(index, 0); - info = VecSimIndex_DebugInfo(index); - infoIter = VecSimIndex_DebugInfoIterator(index); - ASSERT_EQ(0, info.commonInfo.indexSize); - compareFlatIndexInfoToIterator(info, infoIter); - VecSimDebugInfoIterator_Free(infoIter); - - // Perform (or simulate) Search in all modes. - VecSimIndex_AddVector(index, v, 0); - auto res = VecSimIndex_TopKQuery(index, v, 1, nullptr, BY_SCORE); - VecSimQueryReply_Free(res); - info = VecSimIndex_DebugInfo(index); - infoIter = VecSimIndex_DebugInfoIterator(index); - ASSERT_EQ(STANDARD_KNN, info.commonInfo.lastMode); - compareFlatIndexInfoToIterator(info, infoIter); - VecSimDebugInfoIterator_Free(infoIter); + // Delete vector. + VecSimIndex_DeleteVector(index, 0); + info = VecSimIndex_DebugInfo(index); + infoIter = VecSimIndex_DebugInfoIterator(index); + ASSERT_EQ(0, info.commonInfo.indexSize); + compareSVSIndexInfoToIterator(info, infoIter); + VecSimDebugInfoIterator_Free(infoIter); - res = VecSimIndex_RangeQuery(index, v, 1, nullptr, BY_SCORE); - VecSimQueryReply_Free(res); - info = VecSimIndex_DebugInfo(index); - infoIter = VecSimIndex_DebugInfoIterator(index); - ASSERT_EQ(RANGE_QUERY, info.commonInfo.lastMode); - compareFlatIndexInfoToIterator(info, infoIter); - VecSimDebugInfoIterator_Free(infoIter); + // Perform (or simulate) Search in all modes. + VecSimIndex_AddVector(index, v, 0); + auto res = VecSimIndex_TopKQuery(index, v, 1, nullptr, BY_SCORE); + VecSimQueryReply_Free(res); + info = VecSimIndex_DebugInfo(index); + infoIter = VecSimIndex_DebugInfoIterator(index); + ASSERT_EQ(STANDARD_KNN, info.commonInfo.lastMode); + compareSVSIndexInfoToIterator(info, infoIter); + VecSimDebugInfoIterator_Free(infoIter); - ASSERT_TRUE(VecSimIndex_PreferAdHocSearch(index, 1, 1, true)); - info = VecSimIndex_DebugInfo(index); - infoIter = VecSimIndex_DebugInfoIterator(index); - ASSERT_EQ(HYBRID_ADHOC_BF, info.commonInfo.lastMode); - compareFlatIndexInfoToIterator(info, infoIter); - VecSimDebugInfoIterator_Free(infoIter); - - // Simulate the case where another call to the heuristics is done after realizing that - // the subset size is smaller, and change the policy as a result. - ASSERT_TRUE(VecSimIndex_PreferAdHocSearch(index, 1, 1, false)); - info = VecSimIndex_DebugInfo(index); - infoIter = VecSimIndex_DebugInfoIterator(index); - ASSERT_EQ(HYBRID_BATCHES_TO_ADHOC_BF, info.commonInfo.lastMode); - compareFlatIndexInfoToIterator(info, infoIter); - VecSimDebugInfoIterator_Free(infoIter); + res = VecSimIndex_RangeQuery(index, v, 1, nullptr, BY_SCORE); + VecSimQueryReply_Free(res); + info = VecSimIndex_DebugInfo(index); + infoIter = VecSimIndex_DebugInfoIterator(index); + ASSERT_EQ(RANGE_QUERY, info.commonInfo.lastMode); + compareSVSIndexInfoToIterator(info, infoIter); + VecSimDebugInfoIterator_Free(infoIter); - VecSimIndex_Free(index); + ASSERT_TRUE(VecSimIndex_PreferAdHocSearch(index, 1, 1, true)); + info = VecSimIndex_DebugInfo(index); + infoIter = VecSimIndex_DebugInfoIterator(index); + ASSERT_EQ(HYBRID_ADHOC_BF, info.commonInfo.lastMode); + compareSVSIndexInfoToIterator(info, infoIter); + VecSimDebugInfoIterator_Free(infoIter); + + // Simulate the case where another call to the heuristics is done after realizing that + // the subset size is smaller, and change the policy as a result. + ASSERT_TRUE(VecSimIndex_PreferAdHocSearch(index, 1, 1, false)); + info = VecSimIndex_DebugInfo(index); + infoIter = VecSimIndex_DebugInfoIterator(index); + ASSERT_EQ(HYBRID_BATCHES_TO_ADHOC_BF, info.commonInfo.lastMode); + compareSVSIndexInfoToIterator(info, infoIter); + VecSimDebugInfoIterator_Free(infoIter); + + VecSimIndex_Free(index); + } } TYPED_TEST(SVSTest, svs_vector_search_test_ip) { diff --git a/tests/unit/test_svs_multi.cpp b/tests/unit/test_svs_multi.cpp index 6d97b5b50..70f2a9fda 100644 --- a/tests/unit/test_svs_multi.cpp +++ b/tests/unit/test_svs_multi.cpp @@ -495,7 +495,7 @@ TYPED_TEST(SVSMultiTest, test_basic_svs_info_iterator) { VecSimIndexDebugInfo info = VecSimIndex_DebugInfo(index); VecSimDebugInfoIterator *infoIter = VecSimIndex_DebugInfoIterator(index); - compareFlatIndexInfoToIterator(info, infoIter); + compareSVSIndexInfoToIterator(info, infoIter); VecSimDebugInfoIterator_Free(infoIter); VecSimIndex_Free(index); } @@ -513,7 +513,7 @@ TYPED_TEST(SVSMultiTest, test_dynamic_svs_info_iterator) { VecSimDebugInfoIterator *infoIter = VecSimIndex_DebugInfoIterator(index); ASSERT_EQ(DEFAULT_BLOCK_SIZE, info.commonInfo.basicInfo.blockSize); ASSERT_EQ(0, info.commonInfo.indexSize); - compareFlatIndexInfoToIterator(info, infoIter); + compareSVSIndexInfoToIterator(info, infoIter); VecSimDebugInfoIterator_Free(infoIter); // Add vectors. @@ -529,7 +529,7 @@ TYPED_TEST(SVSMultiTest, test_dynamic_svs_info_iterator) { infoIter = VecSimIndex_DebugInfoIterator(index); ASSERT_EQ(4, info.commonInfo.indexSize); ASSERT_EQ(2, info.commonInfo.indexLabelCount); - compareFlatIndexInfoToIterator(info, infoIter); + compareSVSIndexInfoToIterator(info, infoIter); VecSimDebugInfoIterator_Free(infoIter); // Delete vectors. @@ -538,7 +538,7 @@ TYPED_TEST(SVSMultiTest, test_dynamic_svs_info_iterator) { infoIter = VecSimIndex_DebugInfoIterator(index); ASSERT_EQ(2, info.commonInfo.indexSize); ASSERT_EQ(1, info.commonInfo.indexLabelCount); - compareFlatIndexInfoToIterator(info, infoIter); + compareSVSIndexInfoToIterator(info, infoIter); VecSimDebugInfoIterator_Free(infoIter); // Perform (or simulate) Search in all modes. @@ -548,7 +548,7 @@ TYPED_TEST(SVSMultiTest, test_dynamic_svs_info_iterator) { info = VecSimIndex_DebugInfo(index); infoIter = VecSimIndex_DebugInfoIterator(index); ASSERT_EQ(STANDARD_KNN, info.commonInfo.lastMode); - compareFlatIndexInfoToIterator(info, infoIter); + compareSVSIndexInfoToIterator(info, infoIter); VecSimDebugInfoIterator_Free(infoIter); res = VecSimIndex_RangeQuery(index, v, 1, nullptr, BY_SCORE); @@ -556,14 +556,14 @@ TYPED_TEST(SVSMultiTest, test_dynamic_svs_info_iterator) { info = VecSimIndex_DebugInfo(index); infoIter = VecSimIndex_DebugInfoIterator(index); ASSERT_EQ(RANGE_QUERY, info.commonInfo.lastMode); - compareFlatIndexInfoToIterator(info, infoIter); + compareSVSIndexInfoToIterator(info, infoIter); VecSimDebugInfoIterator_Free(infoIter); ASSERT_TRUE(VecSimIndex_PreferAdHocSearch(index, 1, 1, true)); info = VecSimIndex_DebugInfo(index); infoIter = VecSimIndex_DebugInfoIterator(index); ASSERT_EQ(HYBRID_ADHOC_BF, info.commonInfo.lastMode); - compareFlatIndexInfoToIterator(info, infoIter); + compareSVSIndexInfoToIterator(info, infoIter); VecSimDebugInfoIterator_Free(infoIter); // Simulate the case where another call to the heuristics is done after realizing that @@ -572,7 +572,7 @@ TYPED_TEST(SVSMultiTest, test_dynamic_svs_info_iterator) { info = VecSimIndex_DebugInfo(index); infoIter = VecSimIndex_DebugInfoIterator(index); ASSERT_EQ(HYBRID_BATCHES_TO_ADHOC_BF, info.commonInfo.lastMode); - compareFlatIndexInfoToIterator(info, infoIter); + compareSVSIndexInfoToIterator(info, infoIter); VecSimDebugInfoIterator_Free(infoIter); VecSimIndex_Free(index); diff --git a/tests/unit/test_svs_tiered.cpp b/tests/unit/test_svs_tiered.cpp index 073641bef..3bca3feff 100644 --- a/tests/unit/test_svs_tiered.cpp +++ b/tests/unit/test_svs_tiered.cpp @@ -2484,71 +2484,8 @@ TYPED_TEST(SVSTieredIndexTest, testInfoIterator) { VecSimIndexDebugInfo backendIndexInfo = tiered_index->GetBackendIndex()->debugInfo(); VecSimDebugInfoIterator *infoIterator = tiered_index->debugInfoIterator(); - EXPECT_EQ(infoIterator->numberOfFields(), 14); - - while (infoIterator->hasNext()) { - VecSim_InfoField *infoField = VecSimDebugInfoIterator_NextField(infoIterator); - - if (!strcmp(infoField->fieldName, VecSimCommonStrings::ALGORITHM_STRING)) { - // Algorithm type. - ASSERT_EQ(infoField->fieldType, INFOFIELD_STRING); - ASSERT_STREQ(infoField->fieldValue.stringValue, VecSimCommonStrings::TIERED_STRING); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::TYPE_STRING)) { - // Vector type. - ASSERT_EQ(infoField->fieldType, INFOFIELD_STRING); - ASSERT_STREQ(infoField->fieldValue.stringValue, - VecSimType_ToString(info.commonInfo.basicInfo.type)); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::DIMENSION_STRING)) { - // Vector dimension. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.basicInfo.dim); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::METRIC_STRING)) { - // Metric. - ASSERT_EQ(infoField->fieldType, INFOFIELD_STRING); - ASSERT_STREQ(infoField->fieldValue.stringValue, - VecSimMetric_ToString(info.commonInfo.basicInfo.metric)); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SEARCH_MODE_STRING)) { - // Search mode. - ASSERT_EQ(infoField->fieldType, INFOFIELD_STRING); - ASSERT_STREQ(infoField->fieldValue.stringValue, - VecSimSearchMode_ToString(info.commonInfo.lastMode)); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::INDEX_SIZE_STRING)) { - // Index size. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.indexSize); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::INDEX_LABEL_COUNT_STRING)) { - // Index label count. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.indexLabelCount); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::IS_MULTI_STRING)) { - // Is the index multi value. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.basicInfo.isMulti); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::MEMORY_STRING)) { - // Memory. - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.memory); - } else if (!strcmp(infoField->fieldName, - VecSimCommonStrings::TIERED_MANAGEMENT_MEMORY_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.tieredInfo.management_layer_memory); - } else if (!strcmp(infoField->fieldName, - VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.tieredInfo.backgroundIndexing); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::FRONTEND_INDEX_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_ITERATOR); - compareFlatIndexInfoToIterator(frontendIndexInfo, infoField->fieldValue.iteratorValue); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::BACKEND_INDEX_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_ITERATOR); - compareSVSIndexInfoToIterator(backendIndexInfo, infoField->fieldValue.iteratorValue); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::TIERED_BUFFER_LIMIT_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.tieredInfo.bufferLimit); - } else { - FAIL(); - } - } + compareTieredIndexInfoToIterator(info, frontendIndexInfo, backendIndexInfo, infoIterator); + VecSimDebugInfoIterator_Free(infoIterator); } diff --git a/tests/unit/unit_test_utils.cpp b/tests/unit/unit_test_utils.cpp index c4c3d89dd..42b465936 100644 --- a/tests/unit/unit_test_utils.cpp +++ b/tests/unit/unit_test_utils.cpp @@ -19,6 +19,18 @@ using bfloat16 = vecsim_types::bfloat16; using float16 = vecsim_types::float16; +// Map index types to their expected number of debug iterator fields +namespace DebugInfoIteratorFieldCount { +constexpr size_t FLAT = 10; +constexpr size_t HNSW = 17; +constexpr size_t SVS = 23; +constexpr size_t TIERED_HNSW = 15; +constexpr size_t TIERED_SVS = 17; +} // namespace DebugInfoIteratorFieldCount + +static void chooseCompareIndexInfoToIterator(VecSimIndexDebugInfo info, + VecSimDebugInfoIterator *infoIter); + VecsimQueryType test_utils::query_types[4] = {QUERY_TYPE_NONE, QUERY_TYPE_KNN, QUERY_TYPE_HYBRID, QUERY_TYPE_RANGE}; @@ -228,7 +240,7 @@ template void runRangeTieredIndexSearchTest( VecSimQueryParams *); void compareFlatIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIterator *infoIter) { - ASSERT_EQ(10, VecSimDebugInfoIterator_NumberOfFields(infoIter)); + ASSERT_EQ(DebugInfoIteratorFieldCount::FLAT, VecSimDebugInfoIterator_NumberOfFields(infoIter)); while (VecSimDebugInfoIterator_HasNextField(infoIter)) { VecSim_InfoField *infoField = VecSimDebugInfoIterator_NextField(infoIter); if (!strcmp(infoField->fieldName, VecSimCommonStrings::ALGORITHM_STRING)) { @@ -282,7 +294,7 @@ void compareFlatIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIt } void compareHNSWIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIterator *infoIter) { - ASSERT_EQ(17, VecSimDebugInfoIterator_NumberOfFields(infoIter)); + ASSERT_EQ(DebugInfoIteratorFieldCount::HNSW, VecSimDebugInfoIterator_NumberOfFields(infoIter)); while (VecSimDebugInfoIterator_HasNextField(infoIter)) { VecSim_InfoField *infoField = VecSimDebugInfoIterator_NextField(infoIter); if (!strcmp(infoField->fieldName, VecSimCommonStrings::ALGORITHM_STRING)) { @@ -330,7 +342,7 @@ void compareHNSWIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIt // EF runtime. ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); ASSERT_EQ(infoField->fieldValue.uintegerValue, info.hnswInfo.efRuntime); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::HNSW_EPSILON_STRING)) { + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::EPSILON_STRING)) { // Epsilon. ASSERT_EQ(infoField->fieldType, INFOFIELD_FLOAT64); ASSERT_EQ(infoField->fieldValue.floatingPointValue, info.hnswInfo.epsilon); @@ -354,7 +366,7 @@ void compareHNSWIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIt // Block size. ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.basicInfo.blockSize); - } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::HNSW_NUM_MARKED_DELETED)) { + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::NUM_MARKED_DELETED)) { // Number of marked deleted. ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); ASSERT_EQ(infoField->fieldValue.uintegerValue, @@ -365,10 +377,20 @@ void compareHNSWIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIt } } -void compareTieredHNSWIndexInfoToIterator(VecSimIndexDebugInfo info, - VecSimIndexDebugInfo frontendIndexInfo, - VecSimIndexDebugInfo backendIndexInfo, - VecSimDebugInfoIterator *infoIterator) { +void compareTieredIndexInfoToIterator(VecSimIndexDebugInfo info, + VecSimIndexDebugInfo frontendIndexInfo, + VecSimIndexDebugInfo backendIndexInfo, + VecSimDebugInfoIterator *infoIterator) { + VecSimAlgo backendAlgo = backendIndexInfo.commonInfo.basicInfo.algo; + if (backendAlgo == VecSimAlgo_HNSWLIB) { + ASSERT_EQ(DebugInfoIteratorFieldCount::TIERED_HNSW, + VecSimDebugInfoIterator_NumberOfFields(infoIterator)); + } else if (backendAlgo == VecSimAlgo_SVS) { + ASSERT_EQ(DebugInfoIteratorFieldCount::TIERED_SVS, + VecSimDebugInfoIterator_NumberOfFields(infoIterator)); + } else { + FAIL() << "Unsupported backend algorithm"; + } while (infoIterator->hasNext()) { VecSim_InfoField *infoField = VecSimDebugInfoIterator_NextField(infoIterator); @@ -417,31 +439,68 @@ void compareTieredHNSWIndexInfoToIterator(VecSimIndexDebugInfo info, ASSERT_EQ(infoField->fieldValue.uintegerValue, info.tieredInfo.management_layer_memory); } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::TIERED_BACKGROUND_INDEXING_STRING)) { - ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); - ASSERT_EQ(infoField->fieldValue.uintegerValue, info.tieredInfo.backgroundIndexing); + ASSERT_EQ(infoField->fieldType, INFOFIELD_INT64); + ASSERT_EQ(infoField->fieldValue.integerValue, info.tieredInfo.backgroundIndexing); } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::FRONTEND_INDEX_STRING)) { ASSERT_EQ(infoField->fieldType, INFOFIELD_ITERATOR); compareFlatIndexInfoToIterator(frontendIndexInfo, infoField->fieldValue.iteratorValue); } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::BACKEND_INDEX_STRING)) { ASSERT_EQ(infoField->fieldType, INFOFIELD_ITERATOR); - compareHNSWIndexInfoToIterator(backendIndexInfo, infoField->fieldValue.iteratorValue); + chooseCompareIndexInfoToIterator(backendIndexInfo, infoField->fieldValue.iteratorValue); } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::TIERED_BUFFER_LIMIT_STRING)) { ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); ASSERT_EQ(infoField->fieldValue.uintegerValue, info.tieredInfo.bufferLimit); + // HNSW specific fields } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::TIERED_HNSW_SWAP_JOBS_THRESHOLD_STRING)) { + ASSERT_EQ(backendAlgo, VecSimAlgo_HNSWLIB); ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); ASSERT_EQ( infoField->fieldValue.uintegerValue, info.tieredInfo.specificTieredBackendInfo.hnswTieredInfo.pendingSwapJobsThreshold); + // SVS specific fields + } else if (!strcmp(infoField->fieldName, + VecSimCommonStrings::TIERED_SVS_TRAINING_THRESHOLD_STRING)) { + ASSERT_EQ(backendAlgo, VecSimAlgo_SVS); + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ( + infoField->fieldValue.uintegerValue, + info.tieredInfo.specificTieredBackendInfo.svsTieredInfo.trainingTriggerThreshold); + } else if (!strcmp(infoField->fieldName, + VecSimCommonStrings::TIERED_SVS_UPDATE_THRESHOLD_STRING)) { + ASSERT_EQ(backendAlgo, VecSimAlgo_SVS); + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ( + infoField->fieldValue.uintegerValue, + info.tieredInfo.specificTieredBackendInfo.svsTieredInfo.updateTriggerThreshold); + } else if (!strcmp(infoField->fieldName, + VecSimCommonStrings::TIERED_SVS_THREADS_RESERVE_TIMEOUT_STRING)) { + ASSERT_EQ(backendAlgo, VecSimAlgo_SVS); + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, + info.tieredInfo.specificTieredBackendInfo.svsTieredInfo.updateJobWaitTime); } else { FAIL(); } } } +static void chooseCompareIndexInfoToIterator(VecSimIndexDebugInfo info, + VecSimDebugInfoIterator *infoIter) { + switch (info.commonInfo.basicInfo.algo) { + case VecSimAlgo_HNSWLIB: + compareHNSWIndexInfoToIterator(info, infoIter); + break; + case VecSimAlgo_SVS: + compareSVSIndexInfoToIterator(info, infoIter); + break; + default: + FAIL(); + } +} + void compareSVSIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIterator *infoIter) { - ASSERT_EQ(10, VecSimDebugInfoIterator_NumberOfFields(infoIter)); + ASSERT_EQ(DebugInfoIteratorFieldCount::SVS, VecSimDebugInfoIterator_NumberOfFields(infoIter)); while (VecSimDebugInfoIterator_HasNextField(infoIter)) { VecSim_InfoField *infoField = VecSimDebugInfoIterator_NextField(infoIter); if (!strcmp(infoField->fieldName, VecSimCommonStrings::ALGORITHM_STRING)) { @@ -488,6 +547,62 @@ void compareSVSIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIte // Block size. ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); ASSERT_EQ(infoField->fieldValue.uintegerValue, info.commonInfo.basicInfo.blockSize); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SVS_QUANT_BITS_STRING)) { + // SVS quantization bits. + ASSERT_EQ(infoField->fieldType, INFOFIELD_STRING); + ASSERT_STREQ(infoField->fieldValue.stringValue, + VecSimQuantBits_ToString(info.svsInfo.quantBits)); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SVS_ALPHA_STRING)) { + // SVS alpha parameter. + ASSERT_EQ(infoField->fieldType, INFOFIELD_FLOAT64); + ASSERT_EQ(infoField->fieldValue.floatingPointValue, info.svsInfo.alpha); + } else if (!strcmp(infoField->fieldName, + VecSimCommonStrings::SVS_GRAPH_MAX_DEGREE_STRING)) { + // SVS graph max degree. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.graphMaxDegree); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SVS_CONSTRUCTION_WS_STRING)) { + // SVS construction window size. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.constructionWindowSize); + } else if (!strcmp(infoField->fieldName, + VecSimCommonStrings::SVS_MAX_CANDIDATE_POOL_SIZE_STRING)) { + // SVS max candidate pool size. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.maxCandidatePoolSize); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SVS_PRUNE_TO_STRING)) { + // SVS prune to parameter. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.pruneTo); + } else if (!strcmp(infoField->fieldName, + VecSimCommonStrings::SVS_USE_SEARCH_HISTORY_STRING)) { + // SVS use search history. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.useSearchHistory); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SVS_NUM_THREADS_STRING)) { + // SVS number of threads. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.numThreads); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::NUM_MARKED_DELETED)) { + // SVS number of marked deleted nodes. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.numberOfMarkedDeletedNodes); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SVS_SEARCH_WS_STRING)) { + // SVS search window size. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.searchWindowSize); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SVS_SEARCH_BC_STRING)) { + // SVS search buffer capacity. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.searchBufferCapacity); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::SVS_LEANVEC_DIM_STRING)) { + // SVS leanvec dimension. + ASSERT_EQ(infoField->fieldType, INFOFIELD_UINT64); + ASSERT_EQ(infoField->fieldValue.uintegerValue, info.svsInfo.leanvecDim); + } else if (!strcmp(infoField->fieldName, VecSimCommonStrings::EPSILON_STRING)) { + // SVS epsilon parameter. + ASSERT_EQ(infoField->fieldType, INFOFIELD_FLOAT64); + ASSERT_EQ(infoField->fieldValue.floatingPointValue, info.svsInfo.epsilon); } else { FAIL(); } diff --git a/tests/unit/unit_test_utils.h b/tests/unit/unit_test_utils.h index f15e6523d..322964d76 100644 --- a/tests/unit/unit_test_utils.h +++ b/tests/unit/unit_test_utils.h @@ -160,10 +160,10 @@ void compareFlatIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIt void compareHNSWIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIterator *infoIter); -void compareTieredHNSWIndexInfoToIterator(VecSimIndexDebugInfo info, - VecSimIndexDebugInfo frontendIndexInfo, - VecSimIndexDebugInfo backendIndexInfo, - VecSimDebugInfoIterator *infoIterator); +void compareTieredIndexInfoToIterator(VecSimIndexDebugInfo info, + VecSimIndexDebugInfo frontendIndexInfo, + VecSimIndexDebugInfo backendIndexInfo, + VecSimDebugInfoIterator *infoIterator); void compareSVSIndexInfoToIterator(VecSimIndexDebugInfo info, VecSimDebugInfoIterator *infoIter);