Skip to content

Commit aa1d6d4

Browse files
normen662normen662
committed
adding vector index scan options and more tests
1 parent 7f1d54e commit aa1d6d4

File tree

19 files changed

+1310
-331
lines changed

19 files changed

+1310
-331
lines changed

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Index.java‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -190,25 +190,6 @@ public Index(@Nonnull Index orig, @Nullable final IndexPredicate predicate) {
190190
this.lastModifiedVersion = orig.lastModifiedVersion;
191191
}
192192

193-
/**
194-
* Copy constructor. This will create an index that is identical to the current <code>Index</code> with a given
195-
* set of index options.
196-
* @param orig original index to copy
197-
* @param indexOptions the index options.
198-
*/
199-
public Index(@Nonnull Index orig, @Nonnull final Map<String, String> indexOptions) {
200-
this(orig.name, orig.rootExpression, orig.type, ImmutableMap.copyOf(indexOptions), orig.predicate);
201-
if (orig.primaryKeyComponentPositions != null) {
202-
this.primaryKeyComponentPositions = Arrays.copyOf(orig.primaryKeyComponentPositions, orig.primaryKeyComponentPositions.length);
203-
} else {
204-
this.primaryKeyComponentPositions = null;
205-
}
206-
this.subspaceKey = normalizeSubspaceKey(name, orig.subspaceKey);
207-
this.useExplicitSubspaceKey = orig.useExplicitSubspaceKey;
208-
this.addedVersion = orig.addedVersion;
209-
this.lastModifiedVersion = orig.lastModifiedVersion;
210-
}
211-
212193
@SuppressWarnings({"deprecation", "squid:CallToDeprecatedMethod", "java:S3776"}) // Old (deprecated) index type needs grouping compatibility
213194
@SpotBugsSuppressWarnings("NP_NONNULL_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR")
214195
public Index(@Nonnull RecordMetaDataProto.Index proto) throws KeyExpression.DeserializationException {

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/VectorIndexScanBounds.java‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -43,20 +43,18 @@ public class VectorIndexScanBounds implements IndexScanBounds {
4343
@Nullable
4444
private final RealVector queryVector;
4545
private final int limit;
46-
47-
@Nonnull
48-
private final TupleRange suffixRange;
46+
@Nonnull final VectorIndexScanOptions vectorIndexScanOptions;
4947

5048
public VectorIndexScanBounds(@Nonnull final TupleRange prefixRange,
5149
@Nonnull final Comparisons.Type comparisonType,
5250
@Nullable final RealVector queryVector,
5351
final int limit,
54-
@Nonnull final TupleRange suffixRange) {
52+
@Nonnull final VectorIndexScanOptions vectorIndexScanOptions) {
5553
this.prefixRange = prefixRange;
5654
this.comparisonType = comparisonType;
5755
this.queryVector = queryVector;
5856
this.limit = limit;
59-
this.suffixRange = suffixRange;
57+
this.vectorIndexScanOptions = vectorIndexScanOptions;
6058
}
6159

6260
@Nonnull
@@ -84,6 +82,11 @@ public int getLimit() {
8482
return limit;
8583
}
8684

85+
@Nonnull
86+
public VectorIndexScanOptions getVectorIndexScanOptions() {
87+
return vectorIndexScanOptions;
88+
}
89+
8790
public int getAdjustedLimit() {
8891
switch (getComparisonType()) {
8992
case DISTANCE_RANK_LESS_THAN:
@@ -95,11 +98,6 @@ public int getAdjustedLimit() {
9598
}
9699
}
97100

98-
@Nonnull
99-
public TupleRange getSuffixRange() {
100-
return suffixRange;
101-
}
102-
103101
public boolean isWithinLimit(int rank) {
104102
switch (getComparisonType()) {
105103
case DISTANCE_RANK_EQUALS:

‎fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/VectorIndexScanComparisons.java‎

Lines changed: 35 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,14 @@ public class VectorIndexScanComparisons implements IndexScanParameters {
6060
@Nonnull
6161
private final DistanceRankValueComparison distanceRankValueComparison;
6262
@Nonnull
63-
private final ScanComparisons suffixScanComparisons;
63+
private final VectorIndexScanOptions vectorIndexScanOptions;
6464

6565
public VectorIndexScanComparisons(@Nonnull final ScanComparisons prefixScanComparisons,
6666
@Nonnull final DistanceRankValueComparison distanceRankValueComparison,
67-
@Nonnull final ScanComparisons suffixKeyComparisonRanges) {
67+
@Nonnull final VectorIndexScanOptions vectorIndexScanOptions) {
6868
this.prefixScanComparisons = prefixScanComparisons;
6969
this.distanceRankValueComparison = distanceRankValueComparison;
70-
this.suffixScanComparisons = suffixKeyComparisonRanges;
70+
this.vectorIndexScanOptions = vectorIndexScanOptions;
7171
}
7272

7373
@Nonnull
@@ -87,8 +87,8 @@ public DistanceRankValueComparison getDistanceRankValueComparison() {
8787
}
8888

8989
@Nonnull
90-
public ScanComparisons getSuffixScanComparisons() {
91-
return suffixScanComparisons;
90+
public VectorIndexScanOptions getVectorIndexScanOptions() {
91+
return vectorIndexScanOptions;
9292
}
9393

9494
@Nonnull
@@ -97,18 +97,25 @@ public VectorIndexScanBounds bind(@Nonnull final FDBRecordStoreBase<?> store, @N
9797
@Nonnull final EvaluationContext context) {
9898
return new VectorIndexScanBounds(prefixScanComparisons.toTupleRange(store, context),
9999
distanceRankValueComparison.getType(), distanceRankValueComparison.getVector(store, context),
100-
distanceRankValueComparison.getLimit(store, context), suffixScanComparisons.toTupleRange(store, context));
100+
distanceRankValueComparison.getLimit(store, context), vectorIndexScanOptions);
101101
}
102102

103103
@Override
104104
public int planHash(@Nonnull PlanHashMode mode) {
105105
return PlanHashable.objectsPlanHash(mode, prefixScanComparisons, distanceRankValueComparison,
106-
suffixScanComparisons);
106+
vectorIndexScanOptions);
107107
}
108108

109109
@Override
110110
public boolean isUnique(@Nonnull Index index) {
111-
return prefixScanComparisons.isEquality() && prefixScanComparisons.size() == index.getColumnSize();
111+
//
112+
// This is currently never true as we would need an equality-bound scan comparison that includes the primary
113+
// key which we currently cannot express. We can only express equality-bound constraints on the prefix, thus
114+
// the only case where this is true, and we can detect it would currently occur if the prefix contained the
115+
// primary key which in turn would make for a very uninteresting scenario, as each partition would contain
116+
// exactly one vector.
117+
//
118+
return false;
112119
}
113120

114121
@Nonnull
@@ -133,13 +140,9 @@ public ExplainTokensWithPrecedence explain() {
133140
new ExplainTokens().addNested(distanceRankValueComparison.explain().getExplainTokens());
134141
}
135142

136-
tupleRange = suffixScanComparisons.toTupleRangeWithoutContext();
137-
final var suffix = tupleRange == null
138-
? suffixScanComparisons.explain().getExplainTokens()
139-
: new ExplainTokens().addToString(tupleRange);
140-
141143
return ExplainTokensWithPrecedence.of(prefix.addOptionalWhitespace().addToString(":{").addOptionalWhitespace()
142-
.addNested(distanceRank).addOptionalWhitespace().addToString("}:").addOptionalWhitespace().addNested(suffix));
144+
.addNested(distanceRank).addOptionalWhitespace().addToString("}:")
145+
.addOptionalWhitespace().addNested(vectorIndexScanOptions.explain().getExplainTokens()));
143146
}
144147

145148
@SuppressWarnings("checkstyle:VariableDeclarationUsageDistance")
@@ -167,15 +170,8 @@ public void getPlannerGraphDetails(@Nonnull ImmutableList.Builder<String> detail
167170
attributeMapBuilder.put("comparison", Attribute.gml(distanceRankValueComparison));
168171
}
169172

170-
tupleRange = suffixScanComparisons.toTupleRangeWithoutContext();
171-
if (tupleRange != null) {
172-
detailsBuilder.add("suffix: " + tupleRange.getLowEndpoint().toString(false) + "{{slow}}, {{shigh}}" + tupleRange.getHighEndpoint().toString(true));
173-
attributeMapBuilder.put("slow", Attribute.gml(tupleRange.getLow() == null ? "-∞" : tupleRange.getLow().toString()));
174-
attributeMapBuilder.put("shigh", Attribute.gml(tupleRange.getHigh() == null ? "∞" : tupleRange.getHigh().toString()));
175-
} else {
176-
detailsBuilder.add("suffix comparisons: {{scomparisons}}");
177-
attributeMapBuilder.put("scomparisons", Attribute.gml(suffixScanComparisons.toString()));
178-
}
173+
detailsBuilder.add("scan options: {{scanoptions}}");
174+
attributeMapBuilder.put("scanoptions", Attribute.gml(vectorIndexScanOptions.toString()));
179175
}
180176

181177
@Nonnull
@@ -184,7 +180,6 @@ public Set<CorrelationIdentifier> getCorrelatedTo() {
184180
final ImmutableSet.Builder<CorrelationIdentifier> correlatedToBuilder = ImmutableSet.builder();
185181
correlatedToBuilder.addAll(prefixScanComparisons.getCorrelatedTo());
186182
correlatedToBuilder.addAll(distanceRankValueComparison.getCorrelatedTo());
187-
correlatedToBuilder.addAll(suffixScanComparisons.getCorrelatedTo());
188183
return correlatedToBuilder.build();
189184
}
190185

@@ -213,14 +208,14 @@ public boolean semanticEquals(@Nullable final Object other, @Nonnull final Alias
213208
if (!distanceRankValueComparison.semanticEquals(that.distanceRankValueComparison, aliasMap)) {
214209
return false;
215210
}
216-
return suffixScanComparisons.semanticEquals(that.suffixScanComparisons, aliasMap);
211+
return vectorIndexScanOptions.equals(that.vectorIndexScanOptions);
217212
}
218213

219214
@Override
220215
public int semanticHashCode() {
221216
int hashCode = prefixScanComparisons.semanticHashCode();
222217
hashCode = 31 * hashCode + distanceRankValueComparison.semanticHashCode();
223-
return 31 * hashCode + suffixScanComparisons.semanticHashCode();
218+
return 31 * hashCode + vectorIndexScanOptions.hashCode();
224219
}
225220

226221
@Nonnull
@@ -234,29 +229,25 @@ public IndexScanParameters translateCorrelations(@Nonnull final TranslationMap t
234229
final DistanceRankValueComparison translatedDistanceRankValueComparison =
235230
distanceRankValueComparison.translateCorrelations(translationMap, shouldSimplifyValues);
236231

237-
final ScanComparisons translatedSuffixKeyScanComparisons =
238-
suffixScanComparisons.translateCorrelations(translationMap, shouldSimplifyValues);
239-
240232
if (translatedPrefixScanComparisons != prefixScanComparisons ||
241-
translatedDistanceRankValueComparison != distanceRankValueComparison ||
242-
translatedSuffixKeyScanComparisons != suffixScanComparisons) {
243-
return withComparisons(translatedPrefixScanComparisons, translatedDistanceRankValueComparison,
244-
translatedSuffixKeyScanComparisons);
233+
translatedDistanceRankValueComparison != distanceRankValueComparison) {
234+
return withComparisonsAndOptions(translatedPrefixScanComparisons, translatedDistanceRankValueComparison,
235+
vectorIndexScanOptions);
245236
}
246237
return this;
247238
}
248239

249240
@Nonnull
250-
protected VectorIndexScanComparisons withComparisons(@Nonnull final ScanComparisons prefixScanComparisons,
251-
@Nonnull final DistanceRankValueComparison distanceRankValueComparison,
252-
@Nonnull final ScanComparisons suffixKeyScanComparisons) {
241+
protected VectorIndexScanComparisons withComparisonsAndOptions(@Nonnull final ScanComparisons prefixScanComparisons,
242+
@Nonnull final DistanceRankValueComparison distanceRankValueComparison,
243+
@Nonnull final VectorIndexScanOptions vectorIndexScanOptions) {
253244
return new VectorIndexScanComparisons(prefixScanComparisons, distanceRankValueComparison,
254-
suffixKeyScanComparisons);
245+
vectorIndexScanOptions);
255246
}
256247

257248
@Override
258249
public String toString() {
259-
return "BY_VALUE(VECTOR):" + prefixScanComparisons + ":" + distanceRankValueComparison + ":" + suffixScanComparisons;
250+
return "BY_VALUE(VECTOR):" + prefixScanComparisons + ":" + distanceRankValueComparison + ":" + vectorIndexScanOptions;
260251
}
261252

262253
@Override
@@ -277,7 +268,7 @@ public PVectorIndexScanComparisons toProto(@Nonnull final PlanSerializationConte
277268
final PVectorIndexScanComparisons.Builder builder = PVectorIndexScanComparisons.newBuilder();
278269
builder.setPrefixScanComparisons(prefixScanComparisons.toProto(serializationContext));
279270
builder.setDistanceRankValueComparison(distanceRankValueComparison.toProto(serializationContext));
280-
builder.setSuffixScanComparisons(suffixScanComparisons.toProto(serializationContext));
271+
builder.setVectorIndexScanOptions(vectorIndexScanOptions.toProto(serializationContext));
281272
return builder.build();
282273
}
283274

@@ -292,23 +283,20 @@ public static VectorIndexScanComparisons fromProto(@Nonnull final PlanSerializat
292283
@Nonnull final PVectorIndexScanComparisons vectorIndexScanComparisonsProto) {
293284
return new VectorIndexScanComparisons(ScanComparisons.fromProto(serializationContext,
294285
Objects.requireNonNull(vectorIndexScanComparisonsProto.getPrefixScanComparisons())),
295-
Objects.requireNonNull(DistanceRankValueComparison.fromProto(serializationContext, vectorIndexScanComparisonsProto.getDistanceRankValueComparison())),
296-
ScanComparisons.fromProto(serializationContext, Objects.requireNonNull(vectorIndexScanComparisonsProto.getSuffixScanComparisons())));
286+
DistanceRankValueComparison.fromProto(serializationContext, Objects.requireNonNull(vectorIndexScanComparisonsProto.getDistanceRankValueComparison())),
287+
VectorIndexScanOptions.fromProto(Objects.requireNonNull(vectorIndexScanComparisonsProto.getVectorIndexScanOptions())));
297288
}
298289

299290
@Nonnull
300291
public static VectorIndexScanComparisons byValue(@Nullable ScanComparisons prefixScanComparisons,
301292
@Nonnull final DistanceRankValueComparison distanceRankValueComparison,
302-
@Nullable ScanComparisons suffixKeyScanComparisons) {
293+
@Nonnull VectorIndexScanOptions vectorIndexScanOptions) {
303294
if (prefixScanComparisons == null) {
304295
prefixScanComparisons = ScanComparisons.EMPTY;
305296
}
306297

307-
if (suffixKeyScanComparisons == null) {
308-
suffixKeyScanComparisons = ScanComparisons.EMPTY;
309-
}
310-
311-
return new VectorIndexScanComparisons(prefixScanComparisons, distanceRankValueComparison, suffixKeyScanComparisons);
298+
return new VectorIndexScanComparisons(prefixScanComparisons, distanceRankValueComparison,
299+
vectorIndexScanOptions);
312300
}
313301

314302
/**

0 commit comments

Comments
 (0)