From 2a6dcac33c5213344f3ed0b8db10a7d62bd2618e Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Wed, 12 Nov 2025 11:55:55 +0100 Subject: [PATCH 01/10] unnecessary pipeline steps may be skipped rename option as it is no longer only for debugging --- .../features/domain/FeatureStreamImpl.java | 25 ++++++++++--------- .../xtraplatform/features/domain/Query.java | 2 +- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java index 77f9a411d..c51bd76c0 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java @@ -66,24 +66,25 @@ public FeatureStreamImpl( this.codelists = codelists; this.runner = runner; this.doTransform = doTransform; + this.stepMapping = - !query.debugSkipPipelineSteps().contains(PipelineSteps.MAPPING) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.MAPPING) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.stepGeometry = - !query.debugSkipPipelineSteps().contains(PipelineSteps.GEOMETRY) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.GEOMETRY) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.stepCoordinates = - !query.debugSkipPipelineSteps().contains(PipelineSteps.COORDINATES) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.COORDINATES) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.stepClean = - !query.debugSkipPipelineSteps().contains(PipelineSteps.CLEAN) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.CLEAN) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.stepEtag = - !query.debugSkipPipelineSteps().contains(PipelineSteps.ETAG) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.ETAG) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); this.stepMetadata = - !query.debugSkipPipelineSteps().contains(PipelineSteps.METADATA) - && !query.debugSkipPipelineSteps().contains(PipelineSteps.ALL); + !query.skipPipelineSteps().contains(PipelineSteps.METADATA) + && !query.skipPipelineSteps().contains(PipelineSteps.ALL); } @Override diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/Query.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/Query.java index 1752795e3..4332028c6 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/Query.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/Query.java @@ -49,7 +49,7 @@ default boolean hitsOnly() { } @Value.Default - default List debugSkipPipelineSteps() { + default List skipPipelineSteps() { return List.of(); } } From 85a44726bdf57e112e793047818995410d4a5349 Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Wed, 12 Nov 2025 12:52:34 +0100 Subject: [PATCH 02/10] add collectionId to sub-queries --- .../de/ii/xtraplatform/features/domain/MultiFeatureQuery.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java index cb354348d..48f7085ad 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/MultiFeatureQuery.java @@ -14,7 +14,9 @@ public interface MultiFeatureQuery extends Query { @Value.Immutable - interface SubQuery extends TypeQuery {} + interface SubQuery extends TypeQuery { + String getCollectionId(); + } List getQueries(); } From 22fa1a50aea196acfbdd01738d5543a1bf26a742 Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Wed, 12 Nov 2025 19:49:12 +0100 Subject: [PATCH 03/10] refactor option to skip pipeline steps --- .../sql/domain/FeatureProviderSql.java | 17 ++ .../sql/domain/FeatureProviderSqlData.java | 37 +++++ ...rminePipelineStepsThatCannotBeSkipped.java | 149 ++++++++++++++++++ .../features/domain/FeatureQueries.java | 4 + .../features/domain/FeatureStreamImpl.java | 43 ++--- .../tiles/app/TileBuilderDefault.java | 68 ++++++-- 6 files changed, 283 insertions(+), 35 deletions(-) create mode 100644 xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java index 43628efde..326caca5c 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java @@ -362,6 +362,8 @@ * @ref:sourcePathDefaults {@link de.ii.xtraplatform.features.sql.domain.ImmutableSqlPathDefaults} * @ref:queryGeneration {@link * de.ii.xtraplatform.features.sql.domain.ImmutableQueryGeneratorSettings} + * @ref:queryProcessing {@link + * de.ii.xtraplatform.features.sql.domain.ImmutableQueryProcessorSettings} * @ref:datasetChanges2 {@link * de.ii.xtraplatform.features.sql.domain.FeatureProviderSqlData.DatasetChangeSettings} */ @@ -402,6 +404,13 @@ @DocStep(type = Step.JSON_PROPERTIES) }, columnSet = ColumnSet.JSON_PROPERTIES), + @DocTable( + name = "queryProcessing", + rows = { + @DocStep(type = Step.TAG_REFS, params = "{@ref:queryProcessing}"), + @DocStep(type = Step.JSON_PROPERTIES) + }, + columnSet = ColumnSet.JSON_PROPERTIES), }, vars = { @DocVar( @@ -1452,6 +1461,14 @@ public boolean supportsIsNull() { return true; } + @Override + public boolean skipUnusedPipelineSteps() { + if (Objects.nonNull(getData().getQueryProcessing())) { + return getData().getQueryProcessing().getSkipUnusedPipelineSteps(); + } + return false; + } + @Override public FeatureSchema getQueryablesSchema( FeatureSchema schema, diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java index 30657df63..36d3702c6 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSqlData.java @@ -75,6 +75,16 @@ public interface FeatureProviderSqlData @Nullable QueryGeneratorSettings getQueryGeneration(); + /** + * @langEn Options for query processing, for details see [Query + * Processing](10-sql.md#query-processing) below. + * @langDe Einstellungen für die Query-Verarbeitung, für Details siehe + * [Query-Verarbeitung](10-sql.md#query-processing). + */ + @DocMarker("specific") + @Nullable + QueryProcessorSettings getQueryProcessing(); + // for json ordering @Override BuildableMap getTypes(); @@ -211,6 +221,33 @@ default boolean getGeometryAsWkb() { } } + @Value.Immutable + @JsonDeserialize(builder = ImmutableQueryProcessorSettings.Builder.class) + interface QueryProcessorSettings { + + /** + * @langEn Skip unused pipeline steps in the feature stream processing. If set to true, steps + * that are not required to fulfil the request (e.g. coordinate processing, if no coordinate + * transformation or specific coordinate precision is needed) are skipped. This can improve + * performance depending on the query and the capabilities used in the feature provider. For + * now the default is `false`, but the default may change to `true`, if experience shows + * that the option does not have side effects. + * @langDe Überspringen Sie nicht verwendete Pipeline-Schritte in der + * Feature-Stream-Verarbeitung. Wenn diese Option auf `true` gesetzt ist, werden Schritte + * übersprungen, die zur Erfüllung der Query nicht erforderlich sind (z. B. + * Koordinatenverarbeitung, wenn keine Koordinatentransformation oder bestimmte + * Koordinatengenauigkeit erforderlich ist). Dies kann die Leistung je nach Query und den im + * Feature-Provider verwendeten Möglichkeiten verbessern. Derzeit ist die + * Standardeinstellung `false`, aber die Standardeinstellung kann sich zu `true` ändern, + * wenn die Erfahrung zeigt, dass die Option keine Nebenwirkungen hat. + * @default false + */ + @Value.Default + default boolean getSkipUnusedPipelineSteps() { + return false; + } + } + @Value.Check default FeatureProviderSqlData migrateAssumeExternalChanges() { if (Objects.isNull(getDatasetChanges()) diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java new file mode 100644 index 000000000..845d3e770 --- /dev/null +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java @@ -0,0 +1,149 @@ +/* + * Copyright 2025 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.domain; + +import com.google.common.collect.ImmutableSet; +import de.ii.xtraplatform.crs.domain.EpsgCrs; +import de.ii.xtraplatform.crs.domain.OgcCrs; +import de.ii.xtraplatform.features.domain.FeatureStream.PipelineSteps; +import de.ii.xtraplatform.features.domain.SchemaBase.Type; +import de.ii.xtraplatform.features.domain.transform.PropertyTransformations; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class DeterminePipelineStepsThatCannotBeSkipped + implements SchemaVisitorTopDown> { + + private final EpsgCrs nativeCrs; + private final EpsgCrs targetCrs; + private final Query query; + private final Optional propertyTransformations; + private final boolean deriveMetadataFromContent; + private final boolean requiresPropertiesInSequence; + private final boolean supportSecondaryGeometry; + private final boolean distinguishNullAndMissing; + private final String featureType; + + public DeterminePipelineStepsThatCannotBeSkipped( + Query query, + String featureType, + Optional propertyTransformations, + EpsgCrs nativeCrs, + EpsgCrs targetCrs, + boolean deriveMetadataFromContent, + boolean requiresPropertiesInSequence, + boolean supportSecondaryGeometry, + boolean distinguishNullAndMissing) { + this.query = query; + this.propertyTransformations = propertyTransformations; + this.nativeCrs = nativeCrs; + this.targetCrs = targetCrs; + this.deriveMetadataFromContent = deriveMetadataFromContent; + this.requiresPropertiesInSequence = requiresPropertiesInSequence; + this.supportSecondaryGeometry = supportSecondaryGeometry; + this.distinguishNullAndMissing = distinguishNullAndMissing; + this.featureType = featureType; + } + + @Override + public Set visit( + FeatureSchema schema, + List parents, + List> visitedProperties) { + ImmutableSet.Builder steps = ImmutableSet.builder(); + + if (parents.isEmpty()) { + // at the root level: aggregate information from properties and test global settings + + // coordinate processing is needed if a target CRS differs from the native CRS or geometries + // are simplified + if (!targetCrs.equals(nativeCrs) + || (query.getMaxAllowableOffset() > 0) + || (!(OgcCrs.CRS84.equals(nativeCrs) || OgcCrs.CRS84h.equals(nativeCrs)) + && supportSecondaryGeometry + && schema.isSecondaryGeometry())) { + steps.add(PipelineSteps.COORDINATES); + } + + // metadata processing (extents, etag) is needed only if the response is not sent as a stream + if (deriveMetadataFromContent) { + steps.add(PipelineSteps.METADATA, PipelineSteps.ETAG); + } + + // aggregate information from visited properties + visitedProperties.forEach(steps::addAll); + + // post-process special cases + Set intermediateResult = steps.build(); + + // include transformations from the feature provider as in the feature stream + PropertyTransformations mergedTransformations = + FeatureStreamImpl.getMergedTransformations( + Map.of(featureType, schema), + query, + Map.of(featureType, propertyTransformations.orElse(Map::of))) + .get(featureType); + + // if null values are not removed, cleaning is not needed + if (intermediateResult.contains(PipelineSteps.CLEAN) + && (mergedTransformations.hasTransformation( + PropertyTransformations.WILDCARD, pt -> !pt.getRemoveNullValues().orElse(true)) + || !distinguishNullAndMissing)) { + steps = ImmutableSet.builder(); + intermediateResult.stream().filter(s -> s != PipelineSteps.CLEAN).forEach(steps::add); + } + + // mapping is also needed, if specific property transformations are applied (the ones with a + // wildcard are handled otherwise: nulls are removed in the CLEAN step and flattening is + // already handled by including MAPPING for any objects or arrays) + if (!intermediateResult.contains(PipelineSteps.MAPPING)) { + if (mergedTransformations.getTransformations().entrySet().stream() + .anyMatch(entry -> !PropertyTransformations.WILDCARD.equals(entry.getKey())) + || requiresPropertiesInSequence) { + steps.add(PipelineSteps.MAPPING); + } + } + + } else { + // at property level: determine needed steps based on schema information + + // mapping is needed for any complex schema: concat/coalesce/merge, an array/object, or use of + // a sub-decoder + if (!schema.getConcat().isEmpty() + || !schema.getCoalesce().isEmpty() + || !schema.getMerge().isEmpty() + || schema.isArray() + || schema.isObject() + || schema + .getSourcePath() + .filter(sourcePath -> sourcePath.matches(".+?\\[[^=\\]]+].+")) + .isPresent()) { + steps.add(PipelineSteps.MAPPING); + } + + // geometry processing is needed for geometries with constraints that require special handling + // to upgrade the geometry type + if (schema.getType() == Type.GEOMETRY + && schema + .getConstraints() + .filter(constraints -> constraints.isClosed() || constraints.isComposite()) + .isPresent()) { + steps.add(PipelineSteps.GEOMETRY); + } + + // unless all properties are required, cleaning maybe needed to remove null values + if (schema.getConstraints().filter(SchemaConstraints::isRequired).isEmpty()) { + steps.add(PipelineSteps.CLEAN); + } + } + + return steps.build(); + } +} diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureQueries.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureQueries.java index 940f3419f..15662ede6 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureQueries.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureQueries.java @@ -32,6 +32,10 @@ default boolean supportsIsNull() { return false; } + default boolean skipUnusedPipelineSteps() { + return false; + } + default FeatureStream getFeatureStream(FeatureQuery query) { throw new UnsupportedOperationException(); } diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java index c51bd76c0..7338de18c 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java @@ -94,7 +94,7 @@ public CompletionStage runWith( CompletableFuture onCollectionMetadata) { Map mergedTransformations = - getMergedTransformations(propertyTransformations); + getMergedTransformations(data.getTypes(), query, propertyTransformations); BiFunction, Stream> stream = (tokenSource, virtualTables) -> { @@ -161,7 +161,7 @@ public CompletionStage> runWith( CompletableFuture onCollectionMetadata) { Map mergedTransformations = - getMergedTransformations(propertyTransformations); + getMergedTransformations(data.getTypes(), query, propertyTransformations); BiFunction, Reactive.Stream>> stream = (tokenSource, virtualTables) -> { @@ -273,27 +273,27 @@ private FeatureTokenSource getFeatureTokenSourceTransformed( return tokenSourceTransformed; } - private Map getMergedTransformations( + static Map getMergedTransformations( + Map featureSchemas, + Query query, Map propertyTransformations) { - if (query instanceof FeatureQuery) { - FeatureQuery featureQuery = (FeatureQuery) query; - + if (query instanceof FeatureQuery featureQuery) { return ImmutableMap.of( featureQuery.getType(), getPropertyTransformations( - (FeatureQuery) query, + featureSchemas, + featureQuery, Optional.ofNullable(propertyTransformations.get(featureQuery.getType())))); } - if (query instanceof MultiFeatureQuery) { - MultiFeatureQuery multiFeatureQuery = (MultiFeatureQuery) query; - + if (query instanceof MultiFeatureQuery multiFeatureQuery) { return multiFeatureQuery.getQueries().stream() .map( typeQuery -> new SimpleImmutableEntry<>( typeQuery.getType(), getPropertyTransformations( + featureSchemas, typeQuery, Optional.ofNullable(propertyTransformations.get(typeQuery.getType()))))) .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue)); @@ -302,17 +302,21 @@ private Map getMergedTransformations( return ImmutableMap.of(); } - private PropertyTransformations getPropertyTransformations( - TypeQuery typeQuery, Optional propertyTransformations) { - FeatureSchema featureSchema = data.getTypes().get(typeQuery.getType()); - + private static PropertyTransformations getPropertyTransformations( + Map featureSchemas, + TypeQuery typeQuery, + Optional propertyTransformations) { if (typeQuery instanceof FeatureQuery && ((FeatureQuery) typeQuery).getSchemaScope() == SchemaBase.Scope.RECEIVABLE) { - return () -> getProviderTransformations(featureSchema, SchemaBase.Scope.RECEIVABLE); + return () -> + getProviderTransformations( + featureSchemas.get(typeQuery.getType()), SchemaBase.Scope.RECEIVABLE); } PropertyTransformations providerTransformations = - () -> getProviderTransformations(featureSchema, SchemaBase.Scope.RETURNABLE); + () -> + getProviderTransformations( + featureSchemas.get(typeQuery.getType()), SchemaBase.Scope.RETURNABLE); PropertyTransformations merged = propertyTransformations @@ -322,7 +326,8 @@ private PropertyTransformations getPropertyTransformations( return applyRename(merged); } - private PropertyTransformations applyRename(PropertyTransformations propertyTransformations) { + private static PropertyTransformations applyRename( + PropertyTransformations propertyTransformations) { if (propertyTransformations.getTransformations().values().stream() .flatMap(Collection::stream) .anyMatch(propertyTransformation -> propertyTransformation.getRename().isPresent())) { @@ -362,7 +367,7 @@ private PropertyTransformations applyRename(PropertyTransformations propertyTran return propertyTransformations; } - private Map> getProviderTransformations( + private static Map> getProviderTransformations( FeatureSchema featureSchema, SchemaBase.Scope scope) { return featureSchema .accept( @@ -377,7 +382,7 @@ private Map> getProviderTransformations( .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))); } - private java.util.stream.Stream>> + private static java.util.stream.Stream>> getProviderTransformationsForProperty(FeatureSchema schema, SchemaBase.Scope scope) { if (schema.getTransformations().isEmpty()) { if (schema.isTemporal()) { diff --git a/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java b/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java index 8a2c6d9cf..8bf6e0dbe 100644 --- a/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java +++ b/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java @@ -11,6 +11,7 @@ import com.codahale.metrics.Timer; import com.codahale.metrics.Timer.Context; import com.github.azahnen.dagger.annotations.AutoBind; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import de.ii.xtraplatform.base.domain.AppConfiguration; import de.ii.xtraplatform.cql.domain.Bbox; @@ -23,11 +24,13 @@ import de.ii.xtraplatform.crs.domain.BoundingBox; import de.ii.xtraplatform.crs.domain.CrsInfo; import de.ii.xtraplatform.crs.domain.EpsgCrs; +import de.ii.xtraplatform.features.domain.DeterminePipelineStepsThatCannotBeSkipped; import de.ii.xtraplatform.features.domain.FeatureProvider; import de.ii.xtraplatform.features.domain.FeatureQueries; import de.ii.xtraplatform.features.domain.FeatureQuery; import de.ii.xtraplatform.features.domain.FeatureSchema; import de.ii.xtraplatform.features.domain.FeatureStream; +import de.ii.xtraplatform.features.domain.FeatureStream.PipelineSteps; import de.ii.xtraplatform.features.domain.FeatureStream.ResultReduced; import de.ii.xtraplatform.features.domain.FeatureTokenEncoder; import de.ii.xtraplatform.features.domain.ImmutableFeatureQuery; @@ -120,6 +123,21 @@ public byte[] getMvtData( metricRegistry.timer(String.format("tiles.%s.generated.mvt", featureProvider.getId()))); } + String featureType = tileset.getFeatureType().orElse(tileset.getId()); + FeatureSchema schema = featureProvider.info().getSchema(featureType).orElse(null); + + if (Objects.isNull(schema)) { + throw new IllegalArgumentException( + String.format("Unknown feature type '%s' in tileset '%s'", featureType, tileset.getId())); + } + + PropertyTransformations propertyTransformations = + tileQuery + .getGenerationParameters() + .flatMap(TileGenerationParameters::getPropertyTransformations) + .map(pt -> pt.mergeInto(baseTransformations)) + .orElse(baseTransformations); + try (Context timed = timers.get(featureProvider.getId()).time()) { FeatureQuery featureQuery = getFeatureQuery( @@ -131,6 +149,40 @@ public byte[] getMvtData( tileQuery.getGenerationParametersTransient(), featureProvider.queries().get()); + if (featureProvider.queries().isAvailable() + && featureProvider.queries().get().skipUnusedPipelineSteps() + && !featureQuery.skipPipelineSteps().contains(PipelineSteps.ALL)) { + Set keepSteps = + schema.accept( + new DeterminePipelineStepsThatCannotBeSkipped( + featureQuery, + featureType, + Optional.of(propertyTransformations), + featureProvider.crs().get().getNativeCrs(), + featureQuery.getCrs().orElse(tileQuery.getTileMatrixSet().getCrs()), + false, + false, + false, + false)); + + ImmutableList.Builder skipSteps = ImmutableList.builder(); + skipSteps.addAll(featureQuery.skipPipelineSteps()); + for (PipelineSteps step : PipelineSteps.values()) { + if (step != PipelineSteps.ALL && !keepSteps.contains(step)) { + skipSteps.add(step); + } + } + featureQuery = + ImmutableFeatureQuery.builder() + .from(featureQuery) + .skipPipelineSteps(skipSteps.build()) + .build(); + } + + if (LOGGER.isDebugEnabled() && !featureQuery.skipPipelineSteps().isEmpty()) { + LOGGER.debug("Skipping pipeline steps: {}", featureQuery.skipPipelineSteps().toString()); + } + FeatureStream tileSource = featureProvider.queries().get().getFeatureStream(featureQuery); TileGenerationContext tileGenerationContext = @@ -143,22 +195,6 @@ public byte[] getMvtData( FeatureTokenEncoder encoder = ENCODERS.get(tileQuery.getMediaType()).apply(tileGenerationContext); - String featureType = tileset.getFeatureType().orElse(tileset.getId()); - FeatureSchema schema = featureProvider.info().getSchema(featureType).orElse(null); - - if (Objects.isNull(schema)) { - throw new IllegalArgumentException( - String.format( - "Unknown feature type '%s' in tileset '%s'", featureType, tileset.getId())); - } - - PropertyTransformations propertyTransformations = - tileQuery - .getGenerationParameters() - .flatMap(TileGenerationParameters::getPropertyTransformations) - .map(pt -> pt.mergeInto(baseTransformations)) - .orElse(baseTransformations); - ResultReduced resultReduced = generateTile(tileSource, encoder, Map.of(featureType, propertyTransformations)); From 1d666672315c61ded9d0133ea44e319224a309d4 Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Thu, 13 Nov 2025 09:08:53 +0100 Subject: [PATCH 04/10] change log level --- .../java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java b/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java index 8bf6e0dbe..93a9be5cb 100644 --- a/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java +++ b/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java @@ -179,8 +179,8 @@ public byte[] getMvtData( .build(); } - if (LOGGER.isDebugEnabled() && !featureQuery.skipPipelineSteps().isEmpty()) { - LOGGER.debug("Skipping pipeline steps: {}", featureQuery.skipPipelineSteps().toString()); + if (LOGGER.isTraceEnabled() && !featureQuery.skipPipelineSteps().isEmpty()) { + LOGGER.trace("Skipping pipeline steps: {}", featureQuery.skipPipelineSteps().toString()); } FeatureStream tileSource = featureProvider.queries().get().getFeatureStream(featureQuery); From 02545cdfe6b8c77d3016b20125cf40fa64b6455c Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Thu, 13 Nov 2025 09:25:21 +0100 Subject: [PATCH 05/10] fix doc --- .../features/sql/domain/FeatureProviderSql.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java index 326caca5c..ac86bc7fe 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java @@ -175,6 +175,9 @@ *

### Query Generation *

Options for query generation. *

{@docTable:queryGeneration} + *

### Query Processing + *

Options for query processing. + *

{@docTable:queryProcessing} *

### Source Path Defaults *

Defaults for the path expressions in `sourcePath`, also see [Source Path * Syntax](#path-syntax). @@ -265,6 +268,9 @@ *

### Query-Generierung *

Optionen für die Query-Generierung in `queryGeneration`. *

{@docTable:queryGeneration} + *

### Query-Verarbeitung + *

Optionen für die Query-Verarbeitung in `queryProcessing`. + *

{@docTable:queryProcessing} *

### SQL-Pfad-Defaults *

Defaults für die Pfad-Ausdrücke in `sourcePath`, siehe auch * [SQL-Pfad-Syntax](#path-syntax). From c8f83efdbfedeac5cac5aae8548784c517ca81ce Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Sat, 15 Nov 2025 12:30:00 +0100 Subject: [PATCH 06/10] remove unnecessary dateFormat transformation dates are already formatted as "yyyy-MM-dd" --- .../xtraplatform/features/domain/FeatureStreamImpl.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java index 7338de18c..1d9727307 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java @@ -8,7 +8,6 @@ package de.ii.xtraplatform.features.domain; import static de.ii.xtraplatform.features.domain.transform.FeaturePropertyTransformerDateFormat.DATETIME_FORMAT; -import static de.ii.xtraplatform.features.domain.transform.FeaturePropertyTransformerDateFormat.DATE_FORMAT; import static de.ii.xtraplatform.features.domain.transform.PropertyTransformations.WILDCARD; import com.google.common.collect.ImmutableMap; @@ -385,16 +384,13 @@ private static Map> getProviderTransformati private static java.util.stream.Stream>> getProviderTransformationsForProperty(FeatureSchema schema, SchemaBase.Scope scope) { if (schema.getTransformations().isEmpty()) { - if (schema.isTemporal()) { + if (schema.isTemporal() && schema.getType() == SchemaBase.Type.DATETIME) { return java.util.stream.Stream.of( Map.entry( schema.getFullPathAsString(), List.of( new ImmutablePropertyTransformation.Builder() - .dateFormat( - schema.getType() == SchemaBase.Type.DATETIME - ? DATETIME_FORMAT - : DATE_FORMAT) + .dateFormat(DATETIME_FORMAT) .build()))); } return java.util.stream.Stream.empty(); From c7f1acec0577dee63465d594165acf831e246e51 Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Sat, 15 Nov 2025 14:28:20 +0100 Subject: [PATCH 07/10] support MAPPING step with just value transformations --- ...rminePipelineStepsThatCannotBeSkipped.java | 30 ++++-- .../features/domain/FeatureStream.java | 3 +- .../features/domain/FeatureStreamImpl.java | 71 +++++++------- ...tureTokenTransformerMappingValuesOnly.java | 93 +++++++++++++++++++ .../transform/PropertyTransformation.java | 20 ++++ .../transform/PropertyTransformations.java | 6 ++ 6 files changed, 183 insertions(+), 40 deletions(-) create mode 100644 xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerMappingValuesOnly.java diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java index 845d3e770..267d4db1e 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java @@ -12,9 +12,12 @@ import de.ii.xtraplatform.crs.domain.OgcCrs; import de.ii.xtraplatform.features.domain.FeatureStream.PipelineSteps; import de.ii.xtraplatform.features.domain.SchemaBase.Type; +import de.ii.xtraplatform.features.domain.transform.PropertyTransformation; import de.ii.xtraplatform.features.domain.transform.PropertyTransformations; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Optional; import java.util.Set; @@ -102,13 +105,26 @@ public Set visit( // mapping is also needed, if specific property transformations are applied (the ones with a // wildcard are handled otherwise: nulls are removed in the CLEAN step and flattening is - // already handled by including MAPPING for any objects or arrays) - if (!intermediateResult.contains(PipelineSteps.MAPPING)) { - if (mergedTransformations.getTransformations().entrySet().stream() - .anyMatch(entry -> !PropertyTransformations.WILDCARD.equals(entry.getKey())) - || requiresPropertiesInSequence) { - steps.add(PipelineSteps.MAPPING); + // already handled by including MAPPING for any objects or arrays); + // if only value transformations are applied, and no other mapping is needed, just execute + // the value transformations, but skip schema transformations and token slice transformers + if (!intermediateResult.contains(PipelineSteps.MAPPING_SCHEMA)) { + if (requiresPropertiesInSequence) { + steps.add(PipelineSteps.MAPPING_SCHEMA); + steps.add(PipelineSteps.MAPPING_VALUES); + } else if (mergedTransformations.getTransformations().entrySet().stream() + .filter(entry -> !PropertyTransformations.WILDCARD.equals(entry.getKey())) + .map(Entry::getValue) + .flatMap(Collection::stream) + .allMatch(PropertyTransformation::onlyValueTransformations)) { + steps.add(PipelineSteps.MAPPING_VALUES); + } else if (mergedTransformations.getTransformations().entrySet().stream() + .anyMatch(entry -> !PropertyTransformations.WILDCARD.equals(entry.getKey()))) { + steps.add(PipelineSteps.MAPPING_SCHEMA); + steps.add(PipelineSteps.MAPPING_VALUES); } + } else { + steps.add(PipelineSteps.MAPPING_VALUES); } } else { @@ -125,7 +141,7 @@ public Set visit( .getSourcePath() .filter(sourcePath -> sourcePath.matches(".+?\\[[^=\\]]+].+")) .isPresent()) { - steps.add(PipelineSteps.MAPPING); + steps.add(PipelineSteps.MAPPING_SCHEMA); } // geometry processing is needed for geometries with constraints that require special handling diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStream.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStream.java index bb7ddb05f..8746b42fd 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStream.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStream.java @@ -30,7 +30,8 @@ public interface FeatureStream { enum PipelineSteps { - MAPPING, + MAPPING_SCHEMA, + MAPPING_VALUES, GEOMETRY, COORDINATES, CLEAN, diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java index 1d9727307..b4c27e708 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java @@ -43,7 +43,8 @@ public class FeatureStreamImpl implements FeatureStream { private final Map codelists; private final QueryRunner runner; private final boolean doTransform; - private final boolean stepMapping; + private final boolean stepMappingSchema; + private final boolean stepMappingValues; private final boolean stepGeometry; private final boolean stepCoordinates; private final boolean stepClean; @@ -66,9 +67,11 @@ public FeatureStreamImpl( this.runner = runner; this.doTransform = doTransform; - this.stepMapping = - !query.skipPipelineSteps().contains(PipelineSteps.MAPPING) + this.stepMappingSchema = + !query.skipPipelineSteps().contains(PipelineSteps.MAPPING_SCHEMA) && !query.skipPipelineSteps().contains(PipelineSteps.ALL); + this.stepMappingValues = + stepMappingSchema || !query.skipPipelineSteps().contains(PipelineSteps.MAPPING_VALUES); this.stepGeometry = !query.skipPipelineSteps().contains(PipelineSteps.GEOMETRY) && !query.skipPipelineSteps().contains(PipelineSteps.ALL); @@ -223,46 +226,50 @@ public CompletionStage> runWith( private FeatureTokenSource getFeatureTokenSourceTransformed( FeatureTokenSource featureTokenSource, Map propertyTransformations) { - FeatureTokenTransformerMappings schemaMapper = - new FeatureTokenTransformerMappings( - propertyTransformations, codelists, data.getNativeTimeZone().orElse(ZoneId.of("UTC"))); - - Optional crsTransformer = - query - .getCrs() - .flatMap( - targetCrs -> - crsTransformerFactory.getTransformer( - data.getNativeCrs().orElse(OgcCrs.CRS84), targetCrs)); - - Optional crsTransformerWgs84 = - query - .getCrs() - .flatMap( - targetCrs -> - crsTransformerFactory.getTransformer( - data.getNativeCrs().orElse(OgcCrs.CRS84), - nativeCrsIs3d ? OgcCrs.CRS84h : OgcCrs.CRS84)); - FeatureTokenTransformerGeometry geometryMapper = new FeatureTokenTransformerGeometry(); - - FeatureTokenTransformerCoordinates coordinatesMapper = - new FeatureTokenTransformerCoordinates(crsTransformer, crsTransformerWgs84); - - FeatureTokenTransformerRemoveEmptyOptionals cleaner = - new FeatureTokenTransformerRemoveEmptyOptionals(propertyTransformations); - FeatureTokenSource tokenSourceTransformed = featureTokenSource; - if (stepMapping) { + if (stepMappingSchema) { + FeatureTokenTransformerMappings schemaMapper = + new FeatureTokenTransformerMappings( + propertyTransformations, + codelists, + data.getNativeTimeZone().orElse(ZoneId.of("UTC"))); tokenSourceTransformed = tokenSourceTransformed.via(schemaMapper); + } else if (stepMappingValues) { + FeatureTokenTransformerMappingValuesOnly valueMapper = + new FeatureTokenTransformerMappingValuesOnly( + propertyTransformations, + codelists, + data.getNativeTimeZone().orElse(ZoneId.of("UTC"))); + tokenSourceTransformed = tokenSourceTransformed.via(valueMapper); } if (stepGeometry) { + FeatureTokenTransformerGeometry geometryMapper = new FeatureTokenTransformerGeometry(); tokenSourceTransformed = tokenSourceTransformed.via(geometryMapper); } if (stepCoordinates) { + Optional crsTransformer = + query + .getCrs() + .flatMap( + targetCrs -> + crsTransformerFactory.getTransformer( + data.getNativeCrs().orElse(OgcCrs.CRS84), targetCrs)); + Optional crsTransformerWgs84 = + query + .getCrs() + .flatMap( + targetCrs -> + crsTransformerFactory.getTransformer( + data.getNativeCrs().orElse(OgcCrs.CRS84), + nativeCrsIs3d ? OgcCrs.CRS84h : OgcCrs.CRS84)); + FeatureTokenTransformerCoordinates coordinatesMapper = + new FeatureTokenTransformerCoordinates(crsTransformer, crsTransformerWgs84); tokenSourceTransformed = tokenSourceTransformed.via(coordinatesMapper); } if (stepClean) { + FeatureTokenTransformerRemoveEmptyOptionals cleaner = + new FeatureTokenTransformerRemoveEmptyOptionals(propertyTransformations); tokenSourceTransformed = tokenSourceTransformed.via(cleaner); } if (FeatureTokenValidator.LOGGER.isTraceEnabled()) { diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerMappingValuesOnly.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerMappingValuesOnly.java new file mode 100644 index 000000000..19bfa629e --- /dev/null +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureTokenTransformerMappingValuesOnly.java @@ -0,0 +1,93 @@ +/* + * Copyright 2022 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.domain; + +import com.google.common.collect.ImmutableMap; +import de.ii.xtraplatform.codelists.domain.Codelist; +import de.ii.xtraplatform.features.domain.SchemaBase.Type; +import de.ii.xtraplatform.features.domain.transform.FeaturePropertyValueTransformer; +import de.ii.xtraplatform.features.domain.transform.PropertyTransformations; +import de.ii.xtraplatform.features.domain.transform.TransformerChain; +import java.time.ZoneId; +import java.util.AbstractMap.SimpleImmutableEntry; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FeatureTokenTransformerMappingValuesOnly extends FeatureTokenTransformer { + + private static final Logger LOGGER = + LoggerFactory.getLogger(FeatureTokenTransformerMappingValuesOnly.class); + + private final Map propertyTransformations; + private final Map codelists; + private final ZoneId nativeTimeZone; + private Map> + valueTransformerChains; + private TransformerChain currentValueTransformerChain; + + public FeatureTokenTransformerMappingValuesOnly( + Map propertyTransformations, + Map codelists, + ZoneId nativeTimeZone) { + this.propertyTransformations = propertyTransformations; + this.codelists = codelists; + this.nativeTimeZone = nativeTimeZone; + } + + @Override + protected void init() { + super.init(); + } + + @Override + public void onStart(ModifiableContext context) { + this.valueTransformerChains = + context.mappings().entrySet().stream() + .map( + entry -> + new SimpleImmutableEntry<>( + entry.getKey(), + propertyTransformations + .get(entry.getKey()) + .getValueTransformations(entry.getValue(), codelists, nativeTimeZone))) + .collect(ImmutableMap.toImmutableMap(Entry::getKey, Entry::getValue)); + + getDownstream().onStart(context); + } + + @Override + public void onFeatureStart(ModifiableContext context) { + this.currentValueTransformerChain = valueTransformerChains.get(context.type()); + + getDownstream().onFeatureStart(context); + } + + @Override + public void onValue(ModifiableContext context) { + if (context.schema().filter(FeatureSchema::isValue).isPresent()) { + FeatureSchema schema = context.schema().get(); + String value = context.value(); + + if (Objects.nonNull(value)) { + value = currentValueTransformerChain.transform(schema.getFullPathAsString(), value); + context.setValue(value); + + Type valueType = + schema.isSpatial() + ? context.valueType() + : schema.getValueType().orElse(schema.getType()); + context.setValueType(valueType); + } + + getDownstream().onValue(context); + } + } +} diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformation.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformation.java index 536f74521..9c211be33 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformation.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformation.java @@ -106,6 +106,26 @@ default Builder getBuilder() { return new ImmutablePropertyTransformation.Builder().from(this); } + @JsonIgnore + @Value.Derived + @Value.Auxiliary + default boolean onlyValueTransformations() { + return getRename().isEmpty() + && getRenamePathOnly().isEmpty() + && getRemove().isEmpty() + && getFlatten().isEmpty() + && getObjectReduceFormat().isEmpty() + && getObjectReduceSelect().isEmpty() + && getObjectRemoveSelect().isEmpty() + && getObjectMapFormat().isEmpty() + && getObjectMapDuplicate().isEmpty() + && getObjectAddConstants().isEmpty() + && getArrayReduceFormat().isEmpty() + && getCoalesce().isEmpty() + && getConcat().isEmpty() + && getWrap().isEmpty(); + } + /** * @langEn Rename a property. * @langDe Benennt die Eigenschaft auf den angegebenen Namen um. diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformations.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformations.java index fecb4da69..9154eacbf 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformations.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/transform/PropertyTransformations.java @@ -55,6 +55,12 @@ default boolean hasTransformation(String key, Predicate && getTransformations().get(key).stream().anyMatch(predicate); } + default boolean onlyValueTransformations() { + return getTransformations().values().stream() + .flatMap(List::stream) + .allMatch(PropertyTransformation::onlyValueTransformations); + } + default Map> withTransformation( String key, PropertyTransformation transformation) { Map> transformations = From 92ac5c38a372e9db856aea06a1f7f0cf774f389d Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Sat, 15 Nov 2025 16:28:38 +0100 Subject: [PATCH 08/10] fix multi-query --- ...erminePipelineStepsThatCannotBeSkipped.java | 18 +++++++++--------- .../features/domain/FeatureStreamImpl.java | 2 +- .../tiles/app/TileBuilderDefault.java | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java index 267d4db1e..1fe18a9ca 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java @@ -26,8 +26,9 @@ public class DeterminePipelineStepsThatCannotBeSkipped private final EpsgCrs nativeCrs; private final EpsgCrs targetCrs; - private final Query query; + private final TypeQuery query; private final Optional propertyTransformations; + private final boolean simplifyGeometries; private final boolean deriveMetadataFromContent; private final boolean requiresPropertiesInSequence; private final boolean supportSecondaryGeometry; @@ -35,7 +36,7 @@ public class DeterminePipelineStepsThatCannotBeSkipped private final String featureType; public DeterminePipelineStepsThatCannotBeSkipped( - Query query, + TypeQuery query, String featureType, Optional propertyTransformations, EpsgCrs nativeCrs, @@ -43,7 +44,8 @@ public DeterminePipelineStepsThatCannotBeSkipped( boolean deriveMetadataFromContent, boolean requiresPropertiesInSequence, boolean supportSecondaryGeometry, - boolean distinguishNullAndMissing) { + boolean distinguishNullAndMissing, + boolean simplifyGeometries) { this.query = query; this.propertyTransformations = propertyTransformations; this.nativeCrs = nativeCrs; @@ -53,6 +55,7 @@ public DeterminePipelineStepsThatCannotBeSkipped( this.supportSecondaryGeometry = supportSecondaryGeometry; this.distinguishNullAndMissing = distinguishNullAndMissing; this.featureType = featureType; + this.simplifyGeometries = simplifyGeometries; } @Override @@ -68,7 +71,7 @@ public Set visit( // coordinate processing is needed if a target CRS differs from the native CRS or geometries // are simplified if (!targetCrs.equals(nativeCrs) - || (query.getMaxAllowableOffset() > 0) + || (simplifyGeometries) || (!(OgcCrs.CRS84.equals(nativeCrs) || OgcCrs.CRS84h.equals(nativeCrs)) && supportSecondaryGeometry && schema.isSecondaryGeometry())) { @@ -88,11 +91,8 @@ public Set visit( // include transformations from the feature provider as in the feature stream PropertyTransformations mergedTransformations = - FeatureStreamImpl.getMergedTransformations( - Map.of(featureType, schema), - query, - Map.of(featureType, propertyTransformations.orElse(Map::of))) - .get(featureType); + FeatureStreamImpl.getPropertyTransformations( + Map.of(featureType, schema), query, propertyTransformations); // if null values are not removed, cleaning is not needed if (intermediateResult.contains(PipelineSteps.CLEAN) diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java index b4c27e708..65400a1a9 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/FeatureStreamImpl.java @@ -308,7 +308,7 @@ static Map getMergedTransformations( return ImmutableMap.of(); } - private static PropertyTransformations getPropertyTransformations( + static PropertyTransformations getPropertyTransformations( Map featureSchemas, TypeQuery typeQuery, Optional propertyTransformations) { diff --git a/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java b/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java index 93a9be5cb..6e2c3584e 100644 --- a/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java +++ b/xtraplatform-tiles/src/main/java/de/ii/xtraplatform/tiles/app/TileBuilderDefault.java @@ -163,7 +163,8 @@ public byte[] getMvtData( false, false, false, - false)); + false, + true)); ImmutableList.Builder skipSteps = ImmutableList.builder(); skipSteps.addAll(featureQuery.skipPipelineSteps()); From 49aacbffb4b4fb4523456a2efce856430881bbbf Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Sun, 16 Nov 2025 15:54:40 +0100 Subject: [PATCH 09/10] tweak analysis --- ...rminePipelineStepsThatCannotBeSkipped.java | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java index 1fe18a9ca..3bcf67076 100644 --- a/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java +++ b/xtraplatform-features/src/main/java/de/ii/xtraplatform/features/domain/DeterminePipelineStepsThatCannotBeSkipped.java @@ -112,16 +112,22 @@ public Set visit( if (requiresPropertiesInSequence) { steps.add(PipelineSteps.MAPPING_SCHEMA); steps.add(PipelineSteps.MAPPING_VALUES); - } else if (mergedTransformations.getTransformations().entrySet().stream() - .filter(entry -> !PropertyTransformations.WILDCARD.equals(entry.getKey())) - .map(Entry::getValue) - .flatMap(Collection::stream) - .allMatch(PropertyTransformation::onlyValueTransformations)) { - steps.add(PipelineSteps.MAPPING_VALUES); - } else if (mergedTransformations.getTransformations().entrySet().stream() - .anyMatch(entry -> !PropertyTransformations.WILDCARD.equals(entry.getKey()))) { - steps.add(PipelineSteps.MAPPING_SCHEMA); - steps.add(PipelineSteps.MAPPING_VALUES); + } else { + List transformations = + mergedTransformations.getTransformations().entrySet().stream() + .filter(entry -> !PropertyTransformations.WILDCARD.equals(entry.getKey())) + .map(Entry::getValue) + .flatMap(Collection::stream) + .toList(); + if (!transformations.isEmpty()) { + if (transformations.stream() + .allMatch(PropertyTransformation::onlyValueTransformations)) { + steps.add(PipelineSteps.MAPPING_VALUES); + } else { + steps.add(PipelineSteps.MAPPING_SCHEMA); + steps.add(PipelineSteps.MAPPING_VALUES); + } + } } } else { steps.add(PipelineSteps.MAPPING_VALUES); From de48d5b480b727f7d48fae2930884af12fa33202 Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Fri, 28 Nov 2025 14:52:42 +0100 Subject: [PATCH 10/10] Revert "Update dependency org.xerial:sqlite-jdbc to v3.51.0.0 (master) (#415)" This reverts commit 0faaa1ea67a7093e1398cec595f2fa8e3ae6f6bb. --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 37c0e0f32..c30efd3ee 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -6,7 +6,7 @@ staxmate = '2.4.1' rxjava-jdbc = '0.1.4-ii.1' hikaricp = '7.0.2' postgresql = '42.7.8' -sqlite = '3.51.0.0' +sqlite = '3.50.3.0' schemacrawler = '16.29.1' jsqlparser = '5.3' jts = '1.20.0'