diff --git a/changelog/unreleased/SOLR-18217-semver-field.yml b/changelog/unreleased/SOLR-18217-semver-field.yml new file mode 100644 index 000000000000..b20c2dea5f84 --- /dev/null +++ b/changelog/unreleased/SOLR-18217-semver-field.yml @@ -0,0 +1,8 @@ +title: Add a SemVer field type to easily sort and compute range queries for multi-part versions +type: added +authors: + - name: Houston Putman + nick: HoustonPutman +links: + - name: SOLR-18217 + url: https://issues.apache.org/jira/browse/SOLR-18217 diff --git a/solr/core/src/java/org/apache/solr/schema/SemVerField.java b/solr/core/src/java/org/apache/solr/schema/SemVerField.java new file mode 100644 index 000000000000..21e753608587 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/SemVerField.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.util.Collection; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.LongFieldSource; +import org.apache.lucene.queries.function.valuesource.MultiValuedLongFieldSource; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.common.SolrException; +import org.apache.solr.search.QParser; + +/** + * A field type for semantic versioning strings with up to 6 dot-separated parts (e.g. "2.3.1"). + * Each part must be in the range [0, 999]. + * + *

Internally stored as a {@code long} by encoding each part as: {@code part[i] * 1000^(5-i)}. + * For example, "2.3.1" encodes as {@code 2*1000^5 + 3*1000^4 + 1*1000^3 = 2_003_001_000_000}. + * + *

This encoding preserves version ordering, so range queries, sorting, and comparisons all work + * naturally. Missing trailing parts are treated as zero, so "2.3" and "2.3.0" are equivalent. + */ +public class SemVerField extends LongPointField { + + static final int MAX_PARTS = 6; + static final long PART_MULTIPLIER = 1000L; + static final int MAX_PART_VALUE = 999; + + static final long[] POWERS = new long[MAX_PARTS]; + + static { + POWERS[MAX_PARTS - 1] = 1L; + for (int i = MAX_PARTS - 2; i >= 0; i--) { + POWERS[i] = POWERS[i + 1] * PART_MULTIPLIER; + } + } + + public static String decodeDocValue(long value) { + return longToSemVer(value); + } + + static long parseSemVer(String semver) { + if (semver == null || semver.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Empty semver value"); + } + String[] parts = semver.split("\\.", -1); + if (parts.length < 1 || parts.length > MAX_PARTS) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Invalid semver '" + semver + "': must have 1 to " + MAX_PARTS + " dot-separated parts"); + } + long result = 0; + for (int i = 0; i < parts.length; i++) { + int val; + try { + val = Integer.parseInt(parts[i]); + } catch (NumberFormatException e) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Invalid semver part '" + parts[i] + "' in '" + semver + "'"); + } + if (val < 0 || val > MAX_PART_VALUE) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Semver part " + val + " out of range [0, " + MAX_PART_VALUE + "] in '" + semver + "'"); + } + result += val * POWERS[i]; + } + return result; + } + + static String longToSemVer(long value) { + if (value < 0) { + return Long.toString(value); + } + int[] parts = new int[MAX_PARTS]; + long remaining = value; + for (int i = 0; i < MAX_PARTS; i++) { + parts[i] = (int) (remaining / POWERS[i]); + remaining %= POWERS[i]; + } + int lastNonZero = 0; + for (int i = MAX_PARTS - 1; i > 0; i--) { + if (parts[i] != 0) { + lastNonZero = i; + break; + } + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i <= lastNonZero; i++) { + if (i > 0) sb.append('.'); + sb.append(parts[i]); + } + return sb.toString(); + } + + @Override + public Object toNativeType(Object val) { + if (val == null) return null; + if (val instanceof Number) return ((Number) val).longValue(); + if (val instanceof CharSequence) return parseSemVer(val.toString()); + return super.toNativeType(val); + } + + @Override + protected Query getDocValuesRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + return numericDocValuesRangeQuery( + field.getName(), + min == null ? null : parseSemVer(min), + max == null ? null : parseSemVer(max), + minInclusive, + maxInclusive, + field.multiValued()); + } + + @Override + public Query getPointRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + long actualMin, actualMax; + if (min == null) { + actualMin = Long.MIN_VALUE; + } else { + actualMin = parseSemVer(min); + if (!minInclusive) { + if (actualMin == Long.MAX_VALUE) return new MatchNoDocsQuery(); + actualMin++; + } + } + if (max == null) { + actualMax = Long.MAX_VALUE; + } else { + actualMax = parseSemVer(max); + if (!maxInclusive) { + if (actualMax == Long.MIN_VALUE) return new MatchNoDocsQuery(); + actualMax--; + } + } + return LongPoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return longToSemVer(LongPoint.decodeDimension(term.bytes, term.offset)); + } + + @Override + public Object toObject(IndexableField f) { + final Number val = f.numericValue(); + if (val != null) { + return longToSemVer(val.longValue()); + } + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + + @Override + protected Query getExactQuery(SchemaField field, String externalVal) { + return LongPoint.newExactQuery(field.getName(), parseSemVer(externalVal)); + } + + @Override + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVal) { + assert externalVal.size() > 0; + if (!field.indexed()) { + return super.getSetQuery(parser, field, externalVal); + } + long[] values = new long[externalVal.size()]; + int i = 0; + for (String val : externalVal) { + values[i++] = parseSemVer(val); + } + if (field.hasDocValues()) { + return LongField.newSetQuery(field.getName(), values); + } else { + return LongPoint.newSetQuery(field.getName(), values); + } + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return longToSemVer(LongPoint.decodeDimension(indexedForm.bytes, indexedForm.offset)); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + result.grow(Long.BYTES); + result.setLength(Long.BYTES); + LongPoint.encodeDimension(parseSemVer(val.toString()), result.bytes(), 0); + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new SemVerFieldSource(field.getName()); + } + + @Override + protected ValueSource getSingleValueSource(SortedNumericSelector.Type choice, SchemaField field) { + return new MultiValuedLongFieldSource(field.getName(), choice); + } + + @Override + public IndexableField createField(SchemaField field, Object value) { + long longValue = + (value instanceof Number) ? ((Number) value).longValue() : parseSemVer(value.toString()); + return new LongPoint(field.getName(), longValue); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), (Long) this.toNativeType(value)); + } + + private static class SemVerFieldSource extends LongFieldSource { + + public SemVerFieldSource(String field) { + super(field); + } + + @Override + public String description() { + return "semver(" + field + ')'; + } + + @Override + public String longToString(long val) { + return longToSemVer(val); + } + + @Override + public long externalToLong(String extVal) { + return parseSemVer(extVal); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java index 3366d5241ef5..a1fdeabd1fbb 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java @@ -72,6 +72,7 @@ import org.apache.solr.schema.LatLonPointSpatialField; import org.apache.solr.schema.NumberType; import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.SemVerField; import org.apache.solr.schema.StrField; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -699,6 +700,9 @@ private Object decodeNumberFromDV(SchemaField schemaField, long value, boolean s if (schemaField.getType() instanceof LatLonPointSpatialField) { return LatLonPointSpatialField.decodeDocValueToString(value); } + if (schemaField.getType() instanceof SemVerField) { + return SemVerField.decodeDocValue(value); + } if (schemaField.getType().getNumberType() == null) { log.warn( diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-semver.xml b/solr/core/src/test-files/solr/collection1/conf/schema-semver.xml new file mode 100644 index 000000000000..a56494caca01 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-semver.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + id + diff --git a/solr/core/src/test/org/apache/solr/schema/SemVerFieldTest.java b/solr/core/src/test/org/apache/solr/schema/SemVerFieldTest.java new file mode 100644 index 000000000000..5c9750945bad --- /dev/null +++ b/solr/core/src/test/org/apache/solr/schema/SemVerFieldTest.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.junit.BeforeClass; +import org.junit.Test; + +public class SemVerFieldTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-basic.xml", "schema-semver.xml"); + } + + @Test + public void testParseSemVer() { + assertEquals(0L, SemVerField.parseSemVer("0")); + assertEquals(1_000_000_000_000_000L, SemVerField.parseSemVer("1")); + assertEquals(2_003_001_000_000_000L, SemVerField.parseSemVer("2.3.1")); + assertEquals(1_002_003_004_005_006L, SemVerField.parseSemVer("1.2.3.4.5.6")); + assertEquals(999_999_999_999_999_999L, SemVerField.parseSemVer("999.999.999.999.999.999")); + assertEquals(SemVerField.parseSemVer("2.3"), SemVerField.parseSemVer("2.3.0")); + assertEquals(SemVerField.parseSemVer("2.3"), SemVerField.parseSemVer("2.3.0.0.0.0")); + } + + @Test + public void testLongToSemVer() { + assertEquals("0", SemVerField.longToSemVer(0L)); + assertEquals("1", SemVerField.longToSemVer(1_000_000_000_000_000L)); + assertEquals("2.3.1", SemVerField.longToSemVer(2_003_001_000_000_000L)); + assertEquals("1.2.3.4.5.6", SemVerField.longToSemVer(1_002_003_004_005_006L)); + assertEquals("0.0.0.0.0.1", SemVerField.longToSemVer(1L)); + } + + @Test + public void testRoundTrip() { + String[] versions = {"0", "1", "1.2.3", "999.999.999.999.999.999", "0.0.0.0.0.1"}; + for (String v : versions) { + long parsed = SemVerField.parseSemVer(v); + String formatted = SemVerField.longToSemVer(parsed); + assertEquals("Round trip for " + v, parsed, SemVerField.parseSemVer(formatted)); + } + } + + @Test + public void testParseSemVerErrors() { + assertThrows(SolrException.class, () -> SemVerField.parseSemVer("1.2.3.4.5.6.7")); + assertThrows(SolrException.class, () -> SemVerField.parseSemVer("")); + assertThrows(SolrException.class, () -> SemVerField.parseSemVer("abc")); + assertThrows(SolrException.class, () -> SemVerField.parseSemVer("1.2.abc")); + assertThrows(SolrException.class, () -> SemVerField.parseSemVer("1000.0.0")); + assertThrows(SolrException.class, () -> SemVerField.parseSemVer("-1.0.0")); + assertThrows(SolrException.class, () -> SemVerField.parseSemVer("1.2.")); + } + + @Test + public void testIndexAndExactQuery() { + clearIndex(); + assertU(adoc("id", "1", "version", "1.2.3")); + assertU(adoc("id", "2", "version", "2.0.0")); + assertU(adoc("id", "3", "version", "1.2.3")); + assertU(commit()); + + assertQ("Exact match should find 2 docs", req("q", "version:1.2.3"), "//*[@numFound='2']"); + + assertQ( + "Exact match single doc", + req("q", "version:2.0.0"), + "//*[@numFound='1']", + "//result/doc[1]/str[@name='id'][.='2']"); + + assertQ("Trailing zeros are equivalent", req("q", "version:2.0"), "//*[@numFound='1']"); + } + + @Test + public void testRangeQuery() { + clearIndex(); + assertU(adoc("id", "1", "version", "1.0.0")); + assertU(adoc("id", "2", "version", "1.5.0")); + assertU(adoc("id", "3", "version", "2.0.0")); + assertU(adoc("id", "4", "version", "3.0.0")); + assertU(commit()); + + assertQ("Inclusive range", req("q", "version:[1.0.0 TO 2.0.0]"), "//*[@numFound='3']"); + + assertQ("Exclusive upper bound", req("q", "version:[1.0.0 TO 2.0.0}"), "//*[@numFound='2']"); + + assertQ("Open-ended upper", req("q", "version:[2.0.0 TO *]"), "//*[@numFound='2']"); + + assertQ("Open-ended lower", req("q", "version:[* TO 1.5.0]"), "//*[@numFound='2']"); + } + + @Test + public void testSorting() { + clearIndex(); + assertU(adoc("id", "1", "version", "3.1.0")); + assertU(adoc("id", "2", "version", "1.0.0")); + assertU(adoc("id", "3", "version", "2.5.0")); + assertU(commit()); + + assertQ( + "Sort ascending", + req("fl", "id", "q", "*:*", "sort", "version asc"), + "//*[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='3']", + "//result/doc[3]/str[@name='id'][.='1']"); + + assertQ( + "Sort descending", + req("fl", "id", "q", "*:*", "sort", "version desc"), + "//*[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='3']", + "//result/doc[3]/str[@name='id'][.='2']"); + } + + @Test + public void testStoredValue() { + clearIndex(); + assertU(adoc("id", "1", "version", "2.3.1")); + assertU(commit()); + + assertQ( + "Stored value should be semver string", + req("q", "id:1", "fl", "version"), + "//result/doc[1]/str[@name='version'][.='2.3.1']"); + } + + @Test + public void testSortMissingLast() { + clearIndex(); + assertU(adoc("id", "1")); + assertU(adoc("id", "2", "version_last", "2.0.0")); + assertU(adoc("id", "3", "version_last", "1.0.0")); + assertU(commit()); + + assertQ( + "Sort asc, missing last", + req("fl", "id", "q", "*:*", "sort", "version_last asc"), + "//*[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='3']", + "//result/doc[2]/str[@name='id'][.='2']", + "//result/doc[3]/str[@name='id'][.='1']"); + + assertQ( + "Sort desc, missing last", + req("fl", "id", "q", "*:*", "sort", "version_last desc"), + "//*[@numFound='3']", + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='3']", + "//result/doc[3]/str[@name='id'][.='1']"); + } +} diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc index 7a9508e083f8..fe8e3fe5416d 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/field-types-included-with-solr.adoc @@ -73,6 +73,8 @@ The {solr-javadocs}/core/org/apache/solr/schema/package-summary.html[`org.apache |RptWithGeometrySpatialField |A derivative of `SpatialRecursivePrefixTreeFieldType` that also stores the original geometry. See xref:query-guide:spatial-search.adoc[] for more information and usage with geospatial results transformer. +|SemVerField |A field type for semantic versioning strings with up to 6 dot-separated numeric parts (e.g., "2.3.1" or "1.0.4.2"). Each part must be in the range [0, 999]. Internally encoded as a long, preserving version ordering so that range queries, sorting, and comparisons work naturally. Missing trailing parts are treated as zero ("2.3" equals "2.3.0"). For single valued fields, `docValues="true"` must be used to enable sorting. + |SortableTextField |A specialized version of TextField that allows (and defaults to) `docValues="true"` for sorting on the first 1024 characters of the original string prior to analysis. The number of characters used for sorting can be overridden with the `maxCharsForDocValues` attribute. See xref:query-guide:common-query-parameters.adoc#sort-parameter[sort parameter discussion] for details. |SpatialRecursivePrefixTreeFieldType |(RPT for short) Accepts latitude comma longitude strings or other shapes in WKT format. See xref:query-guide:spatial-search.adoc[] for more information.