diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java index 83cb06997..68600a73a 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java @@ -176,6 +176,9 @@ *

### Query Generation *

Options for query generation. *

{@docTable:queryGeneration} + *

### Query Processing + *

Options for query processing. + *

{@docTable:queryProcessing} *

### Source Path Defaults *

Defaults for the path expressions in `sourcePath`, also see [Source Path * Syntax](#path-syntax). @@ -266,6 +269,9 @@ *

### Query-Generierung *

Optionen für die Query-Generierung in `queryGeneration`. *

{@docTable:queryGeneration} + *

### Query-Verarbeitung + *

Optionen für die Query-Verarbeitung in `queryProcessing`. + *

{@docTable:queryProcessing} *

### SQL-Pfad-Defaults *

Defaults für die Pfad-Ausdrücke in `sourcePath`, siehe auch * [SQL-Pfad-Syntax](#path-syntax). @@ -363,6 +369,8 @@ * @ref:sourcePathDefaults {@link de.ii.xtraplatform.features.sql.domain.ImmutableSqlPathDefaults} * @ref:queryGeneration {@link * de.ii.xtraplatform.features.sql.domain.ImmutableQueryGeneratorSettings} + * @ref:queryProcessing {@link + * de.ii.xtraplatform.features.sql.domain.ImmutableQueryProcessorSettings} * @ref:datasetChanges2 {@link * de.ii.xtraplatform.features.sql.domain.FeatureProviderSqlData.DatasetChangeSettings} */ @@ -403,6 +411,13 @@ @DocStep(type = Step.JSON_PROPERTIES) }, columnSet = ColumnSet.JSON_PROPERTIES), + @DocTable( + name = "queryProcessing", + rows = { + @DocStep(type = Step.TAG_REFS, params = "{@ref:queryProcessing}"), + @DocStep(type = Step.JSON_PROPERTIES) + }, + columnSet = ColumnSet.JSON_PROPERTIES), }, vars = { @DocVar( @@ -1467,6 +1482,14 @@ public boolean supportsIsNull() { return true; } + @Override + public boolean skipUnusedPipelineSteps() { + if (Objects.nonNull(getData().getQueryProcessing())) { + return getData().getQueryProcessing().getSkipUnusedPipelineSteps(); + } + return false; + } + @Override public FeatureSchema getQueryablesSchema( FeatureSchema schema, diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java index 30657df63..36d3702c6 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java @@ -75,6 +75,16 @@ public interface FeatureProviderSqlData @Nullable QueryGeneratorSettings getQueryGeneration(); + /** + * @langEn Options for query processing, for details see [Query + * Processing](10-sql.md#query-processing) below. + * @langDe Einstellungen für die Query-Verarbeitung, für Details siehe + * [Query-Verarbeitung](10-sql.md#query-processing). + */ + @DocMarker("specific") + @Nullable + QueryProcessorSettings getQueryProcessing(); + // for json ordering @Override BuildableMap getTypes(); @@ -211,6 +221,33 @@ default boolean getGeometryAsWkb() { } } + @Value.Immutable + @JsonDeserialize(builder = ImmutableQueryProcessorSettings.Builder.class) + interface QueryProcessorSettings { + + /** + * @langEn Skip unused pipeline steps in the feature stream processing. If set to true, steps + * that are not required to fulfil the request (e.g. coordinate processing, if no coordinate + * transformation or specific coordinate precision is needed) are skipped. This can improve + * performance depending on the query and the capabilities used in the feature provider. For + * now the default is `false`, but the default may change to `true`, if experience shows + * that the option does not have side effects. + * @langDe Überspringen Sie nicht verwendete Pipeline-Schritte in der + * Feature-Stream-Verarbeitung. Wenn diese Option auf `true` gesetzt ist, werden Schritte + * übersprungen, die zur Erfüllung der Query nicht erforderlich sind (z. B. + * Koordinatenverarbeitung, wenn keine Koordinatentransformation oder bestimmte + * Koordinatengenauigkeit erforderlich ist). Dies kann die Leistung je nach Query und den im + * Feature-Provider verwendeten Möglichkeiten verbessern. Derzeit ist die + * Standardeinstellung `false`, aber die Standardeinstellung kann sich zu `true` ändern, + * wenn die Erfahrung zeigt, dass die Option keine Nebenwirkungen hat. + * @default false + */ + @Value.Default + default boolean getSkipUnusedPipelineSteps() { + return false; + } + } + @Value.Check default FeatureProviderSqlData migrateAssumeExternalChanges() { if (Objects.isNull(getDatasetChanges()) diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java new file mode 100644 index 000000000..3bcf67076 --- /dev/null +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java @@ -0,0 +1,171 @@ +/* + * Copyright 2025 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.domain; + +import com.google.common.collect.ImmutableSet; +import de.ii.xtraplatform.crs.domain.EpsgCrs; +import de.ii.xtraplatform.crs.domain.OgcCrs; +import de.ii.xtraplatform.features.domain.FeatureStream.PipelineSteps; +import de.ii.xtraplatform.features.domain.SchemaBase.Type; +import de.ii.xtraplatform.features.domain.transform.PropertyTransformation; +import de.ii.xtraplatform.features.domain.transform.PropertyTransformations; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; + +public class DeterminePipelineStepsThatCannotBeSkipped + implements SchemaVisitorTopDown> { + + private final EpsgCrs nativeCrs; + private final EpsgCrs targetCrs; + private final TypeQuery query; + private final Optional propertyTransformations; + private final boolean simplifyGeometries; + private final boolean deriveMetadataFromContent; + private final boolean requiresPropertiesInSequence; + private final boolean supportSecondaryGeometry; + private final boolean distinguishNullAndMissing; + private final String featureType; + + public DeterminePipelineStepsThatCannotBeSkipped( + TypeQuery query, + String featureType, + Optional propertyTransformations, + EpsgCrs nativeCrs, + EpsgCrs targetCrs, + boolean deriveMetadataFromContent, + boolean requiresPropertiesInSequence, + boolean supportSecondaryGeometry, + boolean distinguishNullAndMissing, + boolean simplifyGeometries) { + this.query = query; + this.propertyTransformations = propertyTransformations; + this.nativeCrs = nativeCrs; + this.targetCrs = targetCrs; + this.deriveMetadataFromContent = deriveMetadataFromContent; + this.requiresPropertiesInSequence = requiresPropertiesInSequence; + this.supportSecondaryGeometry = supportSecondaryGeometry; + this.distinguishNullAndMissing = distinguishNullAndMissing; + this.featureType = featureType; + this.simplifyGeometries = simplifyGeometries; + } + + @Override + public Set visit( + FeatureSchema schema, + List parents, + List> visitedProperties) { + ImmutableSet.Builder steps = ImmutableSet.builder(); + + if (parents.isEmpty()) { + // at the root level: aggregate information from properties and test global settings + + // coordinate processing is needed if a target CRS differs from the native CRS or geometries + // are simplified + if (!targetCrs.equals(nativeCrs) + || (simplifyGeometries) + || (!(OgcCrs.CRS84.equals(nativeCrs) || OgcCrs.CRS84h.equals(nativeCrs)) + && supportSecondaryGeometry + && schema.isSecondaryGeometry())) { + steps.add(PipelineSteps.COORDINATES); + } + + // metadata processing (extents, etag) is needed only if the response is not sent as a stream + if (deriveMetadataFromContent) { + steps.add(PipelineSteps.METADATA, PipelineSteps.ETAG); + } + + // aggregate information from visited properties + visitedProperties.forEach(steps::addAll); + + // post-process special cases + Set intermediateResult = steps.build(); + + // include transformations from the feature provider as in the feature stream + PropertyTransformations mergedTransformations = + FeatureStreamImpl.getPropertyTransformations( + Map.of(featureType, schema), query, propertyTransformations); + + // if null values are not removed, cleaning is not needed + if (intermediateResult.contains(PipelineSteps.CLEAN) + && (mergedTransformations.hasTransformation( + PropertyTransformations.WILDCARD, pt -> !pt.getRemoveNullValues().orElse(true)) + || !distinguishNullAndMissing)) { + steps = ImmutableSet.builder(); + intermediateResult.stream().filter(s -> s != PipelineSteps.CLEAN).forEach(steps::add); + } + + // mapping is also needed, if specific property transformations are applied (the ones with a + // wildcard are handled otherwise: nulls are removed in the CLEAN step and flattening is + // already handled by including MAPPING for any objects or arrays); + // if only value transformations are applied, and no other mapping is needed, just execute + // the value transformations, but skip schema transformations and token slice transformers + if (!intermediateResult.contains(PipelineSteps.MAPPING_SCHEMA)) { + if (requiresPropertiesInSequence) { + steps.add(PipelineSteps.MAPPING_SCHEMA); + steps.add(PipelineSteps.MAPPING_VALUES); + } else { + List transformations = + mergedTransformations.getTransformations().entrySet().stream() + .filter(entry -> !PropertyTransformations.WILDCARD.equals(entry.getKey())) + .map(Entry::getValue) + .flatMap(Collection::stream) + .toList(); + if (!transformations.isEmpty()) { + if (transformations.stream() + .allMatch(PropertyTransformation::onlyValueTransformations)) { + steps.add(PipelineSteps.MAPPING_VALUES); + } else { + steps.add(PipelineSteps.MAPPING_SCHEMA); + steps.add(PipelineSteps.MAPPING_VALUES); + } + } + } + } else { + steps.add(PipelineSteps.MAPPING_VALUES); + } + + } else { + // at property level: determine needed steps based on schema information + + // mapping is needed for any complex schema: concat/coalesce/merge, an array/object, or use of + // a sub-decoder + if (!schema.getConcat().isEmpty() + || !schema.getCoalesce().isEmpty() + || !schema.getMerge().isEmpty() + || schema.isArray() + || schema.isObject() + || schema + .getSourcePath() + .filter(sourcePath -> sourcePath.matches(".+?\\[[^=\\]]+].+")) + .isPresent()) { + steps.add(PipelineSteps.MAPPING_SCHEMA); + } + + // geometry processing is needed for geometries with constraints that require special handling + // to upgrade the geometry type + if (schema.getType() == Type.GEOMETRY + && schema + .getConstraints() + .filter(constraints -> constraints.isClosed() || constraints.isComposite()) + .isPresent()) { + steps.add(PipelineSteps.GEOMETRY); + } + + // unless all properties are required, cleaning maybe needed to remove null values + if (schema.getConstraints().filter(SchemaConstraints::isRequired).isEmpty()) { + steps.add(PipelineSteps.CLEAN); + } + } + + return steps.build(); + } +} diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureQueries.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureQueries.java index 940f3419f..15662ede6 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureQueries.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureQueries.java @@ -32,6 +32,10 @@ default boolean supportsIsNull() { return false; } + default boolean skipUnusedPipelineSteps() { + return false; + } + default FeatureStream getFeatureStream(FeatureQuery query) { throw new UnsupportedOperationException(); } diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStream.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStream.java index bb7ddb05f..8746b42fd 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStream.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStream.java @@ -30,7 +30,8 @@ public interface FeatureStream { enum PipelineSteps { - MAPPING, + MAPPING_SCHEMA, + MAPPING_VALUES, GEOMETRY, COORDINATES, CLEAN, diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java index 77f9a411d..65400a1a9 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java @@ -8,7 +8,6 @@ package de.ii.xtraplatform.features.domain; import static de.ii.xtraplatform.features.domain.transform.FeaturePropertyTransformerDateFormat.DATETIME_FORMAT; -import static de.ii.xtraplatform.features.domain.transform.FeaturePropertyTransformerDateFormat.DATE_FORMAT; import static de.ii.xtraplatform.features.domain.transform.PropertyTransformations.WILDCARD; import com.google.common.collect.ImmutableMap; @@ -44,7 +43,8 @@ public class FeatureStreamImpl implements FeatureStream { private final Map codelists; private final QueryRunner runner; private final boolean doTransform; - private final boolean stepMapping; + private final boolean stepMappingSchema; + private final boolean stepMappingValues; private final boolean stepGeometry; private final boolean stepCoordinates; private final boolean stepClean; @@ -66,24 +66,27 @@ public FeatureStreamImpl( this.codelists = codelists; this.runner = runner; this.doTransform = doTransform; - this.stepMapping = - !query.debugSkipPipelineSteps().contains(PipelineSteps.MAPPING) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + + this.stepMappingSchema = + !query.skipPipelineSteps().contains(PipelineSteps.MAPPING_SCHEMA) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); + this.stepMappingValues = + stepMappingSchema || !query.skipPipelineSteps().contains(PipelineSteps.MAPPING_VALUES); this.stepGeometry = - !query.debugSkipPipelineSteps().contains(PipelineSteps.GEOMETRY) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.GEOMETRY) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.stepCoordinates = - !query.debugSkipPipelineSteps().contains(PipelineSteps.COORDINATES) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.COORDINATES) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.stepClean = - !query.debugSkipPipelineSteps().contains(PipelineSteps.CLEAN) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.CLEAN) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.stepEtag = - !query.debugSkipPipelineSteps().contains(PipelineSteps.ETAG) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.ETAG) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.stepMetadata = - !query.debugSkipPipelineSteps().contains(PipelineSteps.METADATA) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.METADATA) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); } @Override @@ -93,7 +96,7 @@ public CompletionStage runWith( CompletableFuture onCollectionMetadata) { Map mergedTransformations = - getMergedTransformations(propertyTransformations); + getMergedTransformations(data.getTypes(), query, propertyTransformations); BiFunction, Stream> stream = (tokenSource, virtualTables) -> { @@ -160,7 +163,7 @@ public CompletionStage> runWith( CompletableFuture onCollectionMetadata) { Map mergedTransformations = - getMergedTransformations(propertyTransformations); + getMergedTransformations(data.getTypes(), query, propertyTransformations); BiFunction, Reactive.Stream>> stream = (tokenSource, virtualTables) -> { @@ -223,46 +226,50 @@ public CompletionStage> runWith( private FeatureTokenSource getFeatureTokenSourceTransformed( FeatureTokenSource featureTokenSource, Map propertyTransformations) { - FeatureTokenTransformerMappings schemaMapper = - new FeatureTokenTransformerMappings( - propertyTransformations, codelists, data.getNativeTimeZone().orElse(ZoneId.of("UTC"))); - - Optional crsTransformer = - query - .getCrs() - .flatMap( - targetCrs -> - crsTransformerFactory.getTransformer( - data.getNativeCrs().orElse(OgcCrs.CRS84), targetCrs)); - - Optional crsTransformerWgs84 = - query - .getCrs() - .flatMap( - targetCrs -> - crsTransformerFactory.getTransformer( - data.getNativeCrs().orElse(OgcCrs.CRS84), - nativeCrsIs3d ? OgcCrs.CRS84h : OgcCrs.CRS84)); - FeatureTokenTransformerGeometry geometryMapper = new FeatureTokenTransformerGeometry(); - - FeatureTokenTransformerCoordinates coordinatesMapper = - new FeatureTokenTransformerCoordinates(crsTransformer, crsTransformerWgs84); - - FeatureTokenTransformerRemoveEmptyOptionals cleaner = - new FeatureTokenTransformerRemoveEmptyOptionals(propertyTransformations); - FeatureTokenSource tokenSourceTransformed = featureTokenSource; - if (stepMapping) { + if (stepMappingSchema) { + FeatureTokenTransformerMappings schemaMapper = + new FeatureTokenTransformerMappings( + propertyTransformations, + codelists, + data.getNativeTimeZone().orElse(ZoneId.of("UTC"))); tokenSourceTransformed = tokenSourceTransformed.via(schemaMapper); + } else if (stepMappingValues) { + FeatureTokenTransformerMappingValuesOnly valueMapper = + new FeatureTokenTransformerMappingValuesOnly( + propertyTransformations, + codelists, + data.getNativeTimeZone().orElse(ZoneId.of("UTC"))); + tokenSourceTransformed = tokenSourceTransformed.via(valueMapper); } if (stepGeometry) { + FeatureTokenTransformerGeometry geometryMapper = new FeatureTokenTransformerGeometry(); tokenSourceTransformed = tokenSourceTransformed.via(geometryMapper); } if (stepCoordinates) { + Optional crsTransformer = + query + .getCrs() + .flatMap( + targetCrs -> + crsTransformerFactory.getTransformer( + data.getNativeCrs().orElse(OgcCrs.CRS84), targetCrs)); + Optional crsTransformerWgs84 = + query + .getCrs() + .flatMap( + targetCrs -> + crsTransformerFactory.getTransformer( + data.getNativeCrs().orElse(OgcCrs.CRS84), + nativeCrsIs3d ? OgcCrs.CRS84h : OgcCrs.CRS84)); + FeatureTokenTransformerCoordinates coordinatesMapper = + new FeatureTokenTransformerCoordinates(crsTransformer, crsTransformerWgs84); tokenSourceTransformed = tokenSourceTransformed.via(coordinatesMapper); } if (stepClean) { + FeatureTokenTransformerRemoveEmptyOptionals cleaner = + new FeatureTokenTransformerRemoveEmptyOptionals(propertyTransformations); tokenSourceTransformed = tokenSourceTransformed.via(cleaner); } if (FeatureTokenValidator.LOGGER.isTraceEnabled()) { @@ -272,27 +279,27 @@ private FeatureTokenSource getFeatureTokenSourceTransformed( return tokenSourceTransformed; } - private Map getMergedTransformations( + static Map getMergedTransformations( + Map featureSchemas, + Query query, Map propertyTransformations) { - if (query instanceof FeatureQuery) { - FeatureQuery featureQuery = (FeatureQuery) query; - + if (query instanceof FeatureQuery featureQuery) { return ImmutableMap.of( featureQuery.getType(), getPropertyTransformations( - (FeatureQuery) query, + featureSchemas, + featureQuery, Optional.ofNullable(propertyTransformations.get(featureQuery.getType())))); } - if (query instanceof MultiFeatureQuery) { - MultiFeatureQuery multiFeatureQuery = (MultiFeatureQuery) query; - + if (query instanceof MultiFeatureQuery multiFeatureQuery) { return multiFeatureQuery.getQueries().stream() .map( typeQuery -> new SimpleImmutableEntry<>( typeQuery.getType(), getPropertyTransformations( + featureSchemas, typeQuery, Optional.ofNullable(propertyTransformations.get(typeQuery.getType()))))) .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -301,17 +308,21 @@ private Map getMergedTransformations( return ImmutableMap.of(); } - private PropertyTransformations getPropertyTransformations( - TypeQuery typeQuery, Optional propertyTransformations) { - FeatureSchema featureSchema = data.getTypes().get(typeQuery.getType()); - + static PropertyTransformations getPropertyTransformations( + Map featureSchemas, + TypeQuery typeQuery, + Optional propertyTransformations) { if (typeQuery instanceof FeatureQuery && ((FeatureQuery) typeQuery).getSchemaScope() == SchemaBase.Scope.RECEIVABLE) { - return () -> getProviderTransformations(featureSchema, SchemaBase.Scope.RECEIVABLE); + return () -> + getProviderTransformations( + featureSchemas.get(typeQuery.getType()), SchemaBase.Scope.RECEIVABLE); } PropertyTransformations providerTransformations = - () -> getProviderTransformations(featureSchema, SchemaBase.Scope.RETURNABLE); + () -> + getProviderTransformations( + featureSchemas.get(typeQuery.getType()), SchemaBase.Scope.RETURNABLE); PropertyTransformations merged = propertyTransformations @@ -321,7 +332,8 @@ private PropertyTransformations getPropertyTransformations( return applyRename(merged); } - private PropertyTransformations applyRename(PropertyTransformations propertyTransformations) { + private static PropertyTransformations applyRename( + PropertyTransformations propertyTransformations) { if (propertyTransformations.getTransformations().values().stream() .flatMap(Collection::stream) .anyMatch(propertyTransformation -> propertyTransformation.getRename().isPresent())) { @@ -361,7 +373,7 @@ private PropertyTransformations applyRename(PropertyTransformations propertyTran return propertyTransformations; } - private Map> getProviderTransformations( + private static Map> getProviderTransformations( FeatureSchema featureSchema, SchemaBase.Scope scope) { return featureSchema .accept( @@ -376,19 +388,16 @@ private Map> getProviderTransformations( .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))); } - private java.util.stream.Stream>> + private static java.util.stream.Stream>> getProviderTransformationsForProperty(FeatureSchema schema, SchemaBase.Scope scope) { if (schema.getTransformations().isEmpty()) { - if (schema.isTemporal()) { + if (schema.isTemporal() && schema.getType() == SchemaBase.Type.DATETIME) { return java.util.stream.Stream.of( Map.entry( schema.getFullPathAsString(), List.of( new ImmutablePropertyTransformation.Builder() - .dateFormat( - schema.getType() == SchemaBase.Type.DATETIME - ? DATETIME_FORMAT - : DATE_FORMAT) + .dateFormat(DATETIME_FORMAT) .build()))); } return java.util.stream.Stream.empty(); diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerMappingValuesOnly.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerMappingValuesOnly.java new file mode 100644 index 000000000..19bfa629e --- /dev/null +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerMappingValuesOnly.java @@ -0,0 +1,93 @@ +/* + * Copyright 2022 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.domain; + +import com.google.common.collect.ImmutableMap; +import de.ii.xtraplatform.codelists.domain.Codelist; +import de.ii.xtraplatform.features.domain.SchemaBase.Type; +import de.ii.xtraplatform.features.domain.transform.FeaturePropertyValueTransformer; +import de.ii.xtraplatform.features.domain.transform.PropertyTransformations; +import de.ii.xtraplatform.features.domain.transform.TransformerChain; +import java.time.ZoneId; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FeatureTokenTransformerMappingValuesOnly extends FeatureTokenTransformer { + + private static final Logger LOGGER = + LoggerFactory.getLogger(FeatureTokenTransformerMappingValuesOnly.class); + + private final Map propertyTransformations; + private final Map codelists; + private final ZoneId nativeTimeZone; + private Map> + valueTransformerChains; + private TransformerChain currentValueTransformerChain; + + public FeatureTokenTransformerMappingValuesOnly( + Map propertyTransformations, + Map codelists, + ZoneId nativeTimeZone) { + this.propertyTransformations = propertyTransformations; + this.codelists = codelists; + this.nativeTimeZone = nativeTimeZone; + } + + @Override + protected void init() { + super.init(); + } + + @Override + public void onStart(ModifiableContext context) { + this.valueTransformerChains = + context.mappings().entrySet().stream() + .map( + entry -> + new SimpleImmutableEntry<>( + entry.getKey(), + propertyTransformations + .get(entry.getKey()) + .getValueTransformations(entry.getValue(), codelists, nativeTimeZone))) + .collect(ImmutableMap.toImmutableMap(Entry::getKey, Entry::getValue)); + + getDownstream().onStart(context); + } + + @Override + public void onFeatureStart(ModifiableContext context) { + this.currentValueTransformerChain = valueTransformerChains.get(context.type()); + + getDownstream().onFeatureStart(context); + } + + @Override + public void onValue(ModifiableContext context) { + if (context.schema().filter(FeatureSchema::isValue).isPresent()) { + FeatureSchema schema = context.schema().get(); + String value = context.value(); + + if (Objects.nonNull(value)) { + value = currentValueTransformerChain.transform(schema.getFullPathAsString(), value); + context.setValue(value); + + Type valueType = + schema.isSpatial() + ? context.valueType() + : schema.getValueType().orElse(schema.getType()); + context.setValueType(valueType); + } + + getDownstream().onValue(context); + } + } +} diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java index cb354348d..48f7085ad 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java @@ -14,7 +14,9 @@ public interface MultiFeatureQuery extends Query { @Value.Immutable - interface SubQuery extends TypeQuery {} + interface SubQuery extends TypeQuery { + String getCollectionId(); + } List getQueries(); } diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/Query.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/Query.java index 1752795e3..4332028c6 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/Query.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/Query.java @@ -49,7 +49,7 @@ default boolean hitsOnly() { } @Value.Default - default List debugSkipPipelineSteps() { + default List skipPipelineSteps() { return List.of(); } } diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformation.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformation.java index 536f74521..9c211be33 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformation.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformation.java @@ -106,6 +106,26 @@ default Builder getBuilder() { return new ImmutablePropertyTransformation.Builder().from(this); } + @JsonIgnore + @Value.Derived + @Value.Auxiliary + default boolean onlyValueTransformations() { + return getRename().isEmpty() + && getRenamePathOnly().isEmpty() + && getRemove().isEmpty() + && getFlatten().isEmpty() + && getObjectReduceFormat().isEmpty() + && getObjectReduceSelect().isEmpty() + && getObjectRemoveSelect().isEmpty() + && getObjectMapFormat().isEmpty() + && getObjectMapDuplicate().isEmpty() + && getObjectAddConstants().isEmpty() + && getArrayReduceFormat().isEmpty() + && getCoalesce().isEmpty() + && getConcat().isEmpty() + && getWrap().isEmpty(); + } + /** * @langEn Rename a property. * @langDe Benennt die Eigenschaft auf den angegebenen Namen um. diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformations.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformations.java index fecb4da69..9154eacbf 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformations.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformations.java @@ -55,6 +55,12 @@ default boolean hasTransformation(String key, Predicate && getTransformations().get(key).stream().anyMatch(predicate); } + default boolean onlyValueTransformations() { + return getTransformations().values().stream() + .flatMap(List::stream) + .allMatch(PropertyTransformation::onlyValueTransformations); + } + default Map> withTransformation( String key, PropertyTransformation transformation) { Map> transformations = diff --git a/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java b/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java index 8a2c6d9cf..6e2c3584e 100644 --- a/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java +++ b/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java @@ -11,6 +11,7 @@ import com.codahale.metrics.Timer; import com.codahale.metrics.Timer.Context; import com.github.azahnen.dagger.annotations.AutoBind; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import de.ii.xtraplatform.base.domain.AppConfiguration; import de.ii.xtraplatform.cql.domain.Bbox; @@ -23,11 +24,13 @@ import de.ii.xtraplatform.crs.domain.BoundingBox; import de.ii.xtraplatform.crs.domain.CrsInfo; import de.ii.xtraplatform.crs.domain.EpsgCrs; +import de.ii.xtraplatform.features.domain.DeterminePipelineStepsThatCannotBeSkipped; import de.ii.xtraplatform.features.domain.FeatureProvider; import de.ii.xtraplatform.features.domain.FeatureQueries; import de.ii.xtraplatform.features.domain.FeatureQuery; import de.ii.xtraplatform.features.domain.FeatureSchema; import de.ii.xtraplatform.features.domain.FeatureStream; +import de.ii.xtraplatform.features.domain.FeatureStream.PipelineSteps; import de.ii.xtraplatform.features.domain.FeatureStream.ResultReduced; import de.ii.xtraplatform.features.domain.FeatureTokenEncoder; import de.ii.xtraplatform.features.domain.ImmutableFeatureQuery; @@ -120,6 +123,21 @@ public byte[] getMvtData( metricRegistry.timer(String.format("tiles.%s.generated.mvt", featureProvider.getId()))); } + String featureType = tileset.getFeatureType().orElse(tileset.getId()); + FeatureSchema schema = featureProvider.info().getSchema(featureType).orElse(null); + + if (Objects.isNull(schema)) { + throw new IllegalArgumentException( + String.format("Unknown feature type '%s' in tileset '%s'", featureType, tileset.getId())); + } + + PropertyTransformations propertyTransformations = + tileQuery + .getGenerationParameters() + .flatMap(TileGenerationParameters::getPropertyTransformations) + .map(pt -> pt.mergeInto(baseTransformations)) + .orElse(baseTransformations); + try (Context timed = timers.get(featureProvider.getId()).time()) { FeatureQuery featureQuery = getFeatureQuery( @@ -131,6 +149,41 @@ public byte[] getMvtData( tileQuery.getGenerationParametersTransient(), featureProvider.queries().get()); + if (featureProvider.queries().isAvailable() + && featureProvider.queries().get().skipUnusedPipelineSteps() + && !featureQuery.skipPipelineSteps().contains(PipelineSteps.ALL)) { + Set keepSteps = + schema.accept( + new DeterminePipelineStepsThatCannotBeSkipped( + featureQuery, + featureType, + Optional.of(propertyTransformations), + featureProvider.crs().get().getNativeCrs(), + featureQuery.getCrs().orElse(tileQuery.getTileMatrixSet().getCrs()), + false, + false, + false, + false, + true)); + + ImmutableList.Builder skipSteps = ImmutableList.builder(); + skipSteps.addAll(featureQuery.skipPipelineSteps()); + for (PipelineSteps step : PipelineSteps.values()) { + if (step != PipelineSteps.ALL && !keepSteps.contains(step)) { + skipSteps.add(step); + } + } + featureQuery = + ImmutableFeatureQuery.builder() + .from(featureQuery) + .skipPipelineSteps(skipSteps.build()) + .build(); + } + + if (LOGGER.isTraceEnabled() && !featureQuery.skipPipelineSteps().isEmpty()) { + LOGGER.trace("Skipping pipeline steps: {}", featureQuery.skipPipelineSteps().toString()); + } + FeatureStream tileSource = featureProvider.queries().get().getFeatureStream(featureQuery); TileGenerationContext tileGenerationContext = @@ -143,22 +196,6 @@ public byte[] getMvtData( FeatureTokenEncoder encoder = ENCODERS.get(tileQuery.getMediaType()).apply(tileGenerationContext); - String featureType = tileset.getFeatureType().orElse(tileset.getId()); - FeatureSchema schema = featureProvider.info().getSchema(featureType).orElse(null); - - if (Objects.isNull(schema)) { - throw new IllegalArgumentException( - String.format( - "Unknown feature type '%s' in tileset '%s'", featureType, tileset.getId())); - } - - PropertyTransformations propertyTransformations = - tileQuery - .getGenerationParameters() - .flatMap(TileGenerationParameters::getPropertyTransformations) - .map(pt -> pt.mergeInto(baseTransformations)) - .orElse(baseTransformations); - ResultReduced resultReduced = generateTile(tileSource, encoder, Map.of(featureType, propertyTransformations));