WIP - new index DDL

fdb-java-annotations/src/main/java/com/apple/foundationdb/annotation/GenerateVisitorAnnotationHelper.java

@@ -20,6 +20,7 @@
 
 package com.apple.foundationdb.annotation;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.squareup.javapoet.AnnotationSpec;
 import com.squareup.javapoet.ClassName;
 import com.squareup.javapoet.CodeBlock;
@@ -43,18 +44,21 @@
 import javax.lang.model.element.Modifier;
 import javax.lang.model.element.PackageElement;
 import javax.lang.model.element.TypeElement;
+import javax.lang.model.type.DeclaredType;
 import javax.lang.model.type.TypeKind;
 import javax.lang.model.type.TypeMirror;
 import javax.lang.model.util.Types;
 import javax.tools.Diagnostic;
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * A separate class to support (@link GenerateVisitorAnnotationProcessor) so that dependency on javapoet does not leak to anyone
@@ -102,8 +106,8 @@
                     .getEnclosedElements()
                     .stream()
                     .flatMap(packageElement -> packageElement.getEnclosedElements().stream())
-                    .filter(element -> element.getKind() == ElementKind.CLASS &&
-                            !element.getModifiers().contains(Modifier.ABSTRACT))
+                    .flatMap(element -> element.getKind() == ElementKind.CLASS && element.getModifiers().contains(Modifier.ABSTRACT) ? element.getEnclosedElements().stream() :  Stream.of(element) )
+                    .filter(element -> element.getKind() == ElementKind.CLASS && !element.getModifiers().contains(Modifier.ABSTRACT))
                     .map(Element::asType)
                     .filter(mirror -> mirror.getKind() == TypeKind.DECLARED)
                     .filter(mirror -> typeUtils.isSubtype(mirror, rootTypeMirror))
@@ -152,18 +156,19 @@
                         .addModifiers(Modifier.PUBLIC)
                         .addTypeVariable(typeVariableName);
 
+        final var packageName = packageElement.getQualifiedName().toString();
         final var jumpMapBuilder = FieldSpec.builder(ParameterizedTypeName.get(ClassName.get(Map.class),
                         ParameterizedTypeName.get(ClassName.get(Class.class), WildcardTypeName.subtypeOf(Object.class)),
                         ParameterizedTypeName.get(ClassName.get(BiFunction.class),
-                                ParameterizedTypeName.get(ClassName.get(packageElement.getQualifiedName().toString(), interfaceName), WildcardTypeName.subtypeOf(Object.class)),
+                                ParameterizedTypeName.get(ClassName.get(packageName, interfaceName), WildcardTypeName.subtypeOf(Object.class)),
                                 TypeName.get(rootTypeMirror),
                                 WildcardTypeName.subtypeOf(Object.class))),
                 "jumpMap", Modifier.PUBLIC, Modifier.STATIC, Modifier.FINAL);
 
         final var initializerStrings = subClassTypeMirrors.stream()
                 .map(typeMirror -> {
                     final var typeElement = (TypeElement)typeUtils.asElement(typeMirror);
-                    return "Map.entry(" + typeElement.getSimpleName() + ".class, (visitor, element) -> visitor." + methodNameOfVisitMethod(generateVisitor, typeElement) + "((" + typeElement.getSimpleName() + ")element))";
+                    return "Map.entry(" + getRawTypeName(typeMirror, packageName) + ".class, (visitor, element) -> visitor." + methodNameOfVisitMethod(generateVisitor, typeElement) + "((" + getWildcardTypeName(typeMirror, packageName) + ")element))";
                 })
                 .collect(Collectors.joining(", \n"));
 
@@ -172,6 +177,7 @@
                 .build();
 
         typeBuilder.addField(jumpMapBuilder
+                .addAnnotation(AnnotationSpec.builder(SuppressWarnings.class).addMember("value", "$S", "unchecked").build())
                 .initializer(initializerBlock)
                 .build());
 
@@ -183,7 +189,7 @@
                             .methodBuilder(methodName)
                             .addModifiers(Modifier.PUBLIC, Modifier.ABSTRACT)
                             .addAnnotation(Nonnull.class)
-                            .addParameter(ParameterSpec.builder(TypeName.get(typeMirror), parameterName).addAnnotation(Nonnull.class).build())
+                            .addParameter(ParameterSpec.builder(getWildcardTypeName(typeMirror, packageName), parameterName).addAnnotation(Nonnull.class).build())
                             .returns(typeVariableName);
             typeBuilder.addMethod(specificVisitMethodBuilder.build());
         }
@@ -193,7 +199,7 @@
                         .methodBuilder(defaultMethodName)
                         .addModifiers(Modifier.PUBLIC, Modifier.ABSTRACT)
                         .addAnnotation(Nonnull.class)
-                        .addParameter(ParameterSpec.builder(TypeName.get(rootTypeMirror), parameterName).addAnnotation(Nonnull.class).build())
+                        .addParameter(ParameterSpec.builder(getWildcardTypeName(rootTypeMirror, packageName), parameterName).addAnnotation(Nonnull.class).build())
                         .returns(typeVariableName);
         typeBuilder.addMethod(visitDefaultMethodBuilder.build());
 
@@ -216,6 +222,105 @@
                 .writeTo(Objects.requireNonNull(filer));
     }
 
+    /**
+     * Converts a type mirror to a raw TypeName without type parameters.
+     * <p>
+     * For generic types, this method returns the raw type without any type arguments
+     * (e.g., {@code List<String>} becomes {@code List}). For non-generic types, the type
+     * is returned as-is. If the type belongs to the same package as {@code currentPackage},
+     * the package prefix is omitted from the generated type name.
+     * <p>
+     * This is particularly useful when generating code that needs to reference the
+     * {@code .class} literal of a generic type, as class literals must use raw types.
+     *
+     * @param typeMirror the type mirror to convert
+     * @param currentPackage the current package name, used to determine whether to omit
+     *                       package prefixes for types in the same package
+     * @return a TypeName representing the raw type (without type parameters) if the type
+     *         is generic, or the original type name if not generic
+     */
+    @Nonnull
+    private static TypeName getRawTypeName(@Nonnull TypeMirror typeMirror, @Nonnull String currentPackage) {
+        if (typeMirror.getKind() == TypeKind.DECLARED) {
+            final var declaredType = (DeclaredType) typeMirror;
+            final var typeElement = (TypeElement) declaredType.asElement();
+            final boolean isGeneric = !typeElement.getTypeParameters().isEmpty();
+
+            if (isGeneric) {
+                final ClassName className = ClassName.get(typeElement);
+                return removePackagePrefix(className, currentPackage);
+            }
+        }
+
+        // return as-is, remove the package if it is the same as the currentPackage.
+        final TypeName typeName = TypeName.get(typeMirror);
+        if (typeName instanceof ClassName) {
+            return removePackagePrefix((ClassName) typeName, currentPackage);
+        }
+
+        return typeName;
+    }
+
+    /**
+     * Converts a type mirror to a TypeName with wildcard type arguments for generic types.
+     * <p>
+     * For generic types, this method creates a parameterized type with wildcard bounds
+     * (e.g., {@code List<String>} becomes {@code List<?>}). For non-generic types, the type
+     * is returned as-is. If the type belongs to the same package as {@code currentPackage},
+     * the package prefix is omitted from the generated type name.
+     *
+     * @param typeMirror the type mirror to convert
+     * @param currentPackage the current package name, used to determine whether to omit
+     *                       package prefixes for types in the same package
+     * @return a TypeName representing the type with wildcard type arguments if the type
+     *         is generic, or the original type name if not generic
+     */
+    @Nonnull
+    private static TypeName getWildcardTypeName(@Nonnull final TypeMirror typeMirror, @Nonnull final String currentPackage) {
+        if (typeMirror.getKind() == TypeKind.DECLARED) {
+            final var declaredType = (DeclaredType) typeMirror;
+            final var typeElement = (TypeElement) declaredType.asElement();
+            final boolean isGeneric = !typeElement.getTypeParameters().isEmpty();
+
+            if (isGeneric) {
+                ClassName rawType = ClassName.get(typeElement);
+                rawType = removePackagePrefix(rawType, currentPackage);
+
+                final WildcardTypeName[] wildcards = new WildcardTypeName[typeElement.getTypeParameters().size()];
+                Arrays.fill(wildcards, WildcardTypeName.subtypeOf(Object.class));
+                return ParameterizedTypeName.get(rawType, wildcards);
+            }
+        }
+
+        // return as-is, remove the package if it is the same as the currentPackage.
+        final TypeName typeName = TypeName.get(typeMirror);
+        if (typeName instanceof ClassName) {
+            return removePackagePrefix((ClassName) typeName, currentPackage);
+        }
+
+        return typeName;
+    }
+
+    /**
+     * Removes the package prefix from a ClassName if it belongs to the same package as currentPackage.
+     * <p>
+     * This is useful when generating code references to types that are in the same package,
+     * as the package prefix can be omitted for brevity.
+     *
+     * @param className the ClassName to potentially strip the package prefix from
+     * @param currentPackage the current package name to compare against
+     * @return a ClassName without the package prefix if it's in the same package,
+     *         otherwise returns the original ClassName unchanged
+     */
+    @Nonnull
+    private static ClassName removePackagePrefix(@Nonnull final ClassName className, @Nonnull final String currentPackage) {
+        if (className.packageName().equals(currentPackage)) {
+            return ClassName.get("", className.topLevelClassName().simpleName(),
+                    className.simpleNames().subList(1, className.simpleNames().size()).toArray(new String[0]));
+        }
+        return className;
+    }
+
     private static void generateImplementationWithDefaults(@Nonnull final Types typeUtils,
                                                            @Nonnull final Filer filer,
                                                            @Nonnull final GenerateVisitor generateVisitor,
@@ -240,7 +345,7 @@
                             .addModifiers(Modifier.PUBLIC, Modifier.DEFAULT)
                             .addAnnotation(Nonnull.class)
                             .addAnnotation(Override.class)
-                            .addParameter(ParameterSpec.builder(TypeName.get(typeMirror), parameterName).addAnnotation(Nonnull.class).build())
+                            .addParameter(ParameterSpec.builder(getWildcardTypeName(typeMirror, packageElement.getQualifiedName().toString()), parameterName).addAnnotation(Nonnull.class).build())
                             .returns(typeVariableName)
                             .addCode(CodeBlock.builder()
                                     .addStatement("return " + defaultMethodName + "(" + parameterName + ")")

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/IndexTypes.java

@@ -164,6 +164,11 @@ public class IndexTypes {
      */
     public static final String MULTIDIMENSIONAL = "multidimensional";
 
+    /**
+     * An index using an HNSW structure.
+     */
+    public static final String VECTOR = "vector";
+
     private IndexTypes() {
     }
 }

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java

@@ -463,6 +463,9 @@ public boolean copy(@Nonnull Descriptors.Descriptor recordDescriptor, @Nonnull M
                 return !fieldDescriptor.isRequired();
             }
             switch (fieldDescriptor.getType()) {
+                case INT32:
+                    value = ((Number)value).intValue();
+                    break;
                 case MESSAGE:
                     value = TupleFieldsHelper.toProto(value, fieldDescriptor.getMessageType());
                     break;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/Value.java

@@ -21,6 +21,7 @@
 package com.apple.foundationdb.record.query.plan.cascades.values;
 
 import com.apple.foundationdb.annotation.API;
+import com.apple.foundationdb.annotation.GenerateVisitor;
 import com.apple.foundationdb.record.EvaluationContext;
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.PlanSerializable;
@@ -91,6 +92,7 @@
  * A scalar value type.
  */
 @API(API.Status.EXPERIMENTAL)
+@GenerateVisitor
 public interface Value extends Correlated<Value>, TreeLike<Value>, UsesValueEquivalence<Value>, PlanHashable, Typed, Narrowable<Value>, PlanSerializable {
 
     @Nonnull

fdb-relational-core/src/jmh/java/com/apple/foundationdb/relational/recordlayer/EmbeddedRelationalBenchmark.java

@@ -63,8 +63,8 @@ public abstract class EmbeddedRelationalBenchmark {
                     "CREATE TABLE \"RestaurantRecord\" (\"rest_no\" bigint, \"name\" string, \"location\" \"Location\", \"reviews\" \"RestaurantReview\" ARRAY, \"tags\" \"RestaurantTag\" ARRAY, \"customer\" string ARRAY, PRIMARY KEY(\"rest_no\")) " +
                     "CREATE TABLE \"RestaurantReviewer\" (\"id\" bigint, \"name\" string, \"email\" string, \"stats\" \"ReviewerStats\", PRIMARY KEY(\"id\")) " +
 
-                    "CREATE INDEX \"record_name_idx\" as select \"name\" from \"RestaurantRecord\" " +
-                    "CREATE INDEX \"reviewer_name_idx\" as select \"name\" from \"RestaurantReviewer\" ";
+                    "CREATE INDEX \"record_name_idx\" ON \"RestaurantRecord\"(\"name\") " +
+                    "CREATE INDEX \"reviewer_name_idx\" ON \"RestaurantReviewer\"(\"name\") ";
 
     static final String restaurantRecordTable = "RestaurantRecord";
 

fdb-relational-core/src/main/antlr/RelationalLexer.g4

@@ -999,6 +999,15 @@ GREATEST:                            'GREATEST';
 GTID_SUBSET:                         'GTID_SUBSET';
 GTID_SUBTRACT:                       'GTID_SUBTRACT';
 HEX:                                 'HEX';
+HNSW_EF_CONSTRUCTION:                'HNSW_EF_CONSTRUCTION';
+HNSW_M_MAX:                          'HNSW_M_MAX';
+HNSW_M:                              'HNSW_M';
+HNSW_MAINTAIN_STATS_PROBABILITY:     'HNSW_MAINTAIN_STATS_PROBABILITY';
+HNSW_METRIC:                         'HNSW_METRIC';
+HNSW_RABITQ_NUM_EX_BITS:             'HNSW_RABITQ_NUM_EX_BITS';
+HNSW_SAMPLE_VECTOR_STATS_PROBABILITY:'HNSW_SAMPLE_VECTOR_STATS_PROBABILITY';
+HNSW_STATS_THRESHOLD:                'HNSW_STATS_THRESHOLD';
+HNSW_USE_RABITQ:                     'HNSW_USE_RABITQ';
 IFNULL:                              'IFNULL';
 INET6_ATON:                          'INET6_ATON';
 INET6_NTOA:                          'INET6_NTOA';

fdb-relational-core/src/main/antlr/RelationalParser.g4

@@ -168,7 +168,49 @@ enumDefinition
     ;
 
 indexDefinition
-    : (UNIQUE)? INDEX indexName=uid AS queryTerm indexAttributes?
+    : (UNIQUE)? INDEX indexName=uid AS queryTerm indexAttributes?                                        #indexAsSelectDefinition
+    | (UNIQUE)? INDEX indexName=uid ON source=fullId indexColumnList includeClause? indexOptions?        #indexOnSourceDefinition
+    | VECTOR INDEX indexName=uid ON source=fullId indexColumnList partitionClause? vectorIndexOptions?   #vectorIndexDefinition
+    ;
+
+indexColumnList
+    : '(' indexColumnSpec (',' indexColumnSpec)* ')'
+    ;
+
+indexColumnSpec
+    : columnName=uid orderClause?
+    ;
+
+includeClause
+    : INCLUDE '(' uidList ')'
+    ;
+
+indexType
+    : UNIQUE | VECTOR
+    ;
+
+indexOptions
+    : OPTIONS '(' indexOption (COMMA indexOption)* ')'
+    ;
+
+indexOption
+    : LEGACY_EXTREMUM_EVER
+    ;
+
+vectorIndexOptions
+    : OPTIONS '(' vectorIndexOption (COMMA vectorIndexOptions)* ')'
+    ;
+
+vectorIndexOption
+    : HNSW_EF_CONSTRUCTION '=' mValue=DECIMAL_LITERAL
+    | HNSW_M '=' mValue=DECIMAL_LITERAL
+    | HNSW_M_MAX '=' mValue=DECIMAL_LITERAL
+    | HNSW_MAINTAIN_STATS_PROBABILITY '=' mValue=DECIMAL_LITERAL
+    | HNSW_METRIC '=' mValue=DECIMAL_LITERAL // change
+    | HNSW_RABITQ_NUM_EX_BITS '=' mValue=DECIMAL_LITERAL
+    | HNSW_SAMPLE_VECTOR_STATS_PROBABILITY '=' mValue=DECIMAL_LITERAL
+    | HNSW_STATS_THRESHOLD '=' mValue=DECIMAL_LITERAL
+    | HNSW_USE_RABITQ '=' mValue=DECIMAL_LITERAL // change
     ;
 
 indexAttributes
@@ -413,7 +455,12 @@ orderByClause
     ;
 
 orderByExpression
-    : expression order=(ASC | DESC)? (NULLS nulls=(FIRST | LAST))?
+    : expression orderClause?
+    ;
+
+orderClause
+    : order=(ASC | DESC) (NULLS nulls=(FIRST | LAST))?
+    | NULLS nulls=(FIRST | LAST)
     ;
 
 tableSources // done
@@ -1104,10 +1151,11 @@ frameRange
     | expression (PRECEDING | FOLLOWING)
     ;
 
+*/
+
 partitionClause
-    : PARTITION BY expression (',' expression)*
+    : PARTITION BY uid (',' uid)*
     ;
-*/
 
 scalarFunctionName
     : functionNameBase

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/AbstractEmbeddedStatement.java

@@ -215,10 +215,15 @@ private int countUpdates(@Nonnull ResultSet resultSet) throws SQLException {
             }
             return count;
         } catch (SQLException | RuntimeException ex) {
-            if (conn.canCommit()) {
-                conn.rollbackInternal();
+            SQLException finalException = ExceptionUtil.toRelationalException(ex).toSqlException();
+            try {
+                if (conn.canCommit()) {
+                    conn.rollbackInternal();
+                }
+            } catch (SQLException | RuntimeException rollbackError) {
+                finalException.addSuppressed(rollbackError);
             }
-            throw ExceptionUtil.toRelationalException(ex).toSqlException();
+            throw finalException;
         }
     }
 

fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerIndex.java

@@ -200,6 +200,15 @@ public Builder setOptions(@Nonnull final Map<String, String> options) {
             return this;
         }
 
+        @Nonnull
+        public Builder addAllOptions(@Nonnull final Map<String, String> options) {
+            if (optionsBuilder == null) {
+                optionsBuilder = ImmutableMap.builder();
+            }
+            optionsBuilder.putAll(options);
+            return this;
+        }
+
         @Nonnull
         public Builder setOption(@Nonnull final String optionKey, @Nonnull final String optionValue) {
             if (optionsBuilder == null) {