From 95daef316f7d7d618e258a9a00a6c45c6581c5d0 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Mon, 1 Dec 2025 13:36:09 +0100 Subject: [PATCH 1/9] SED-4415 optimize-reporting-request-and-query-performance --- pom.xml | 2 +- .../step/plugins/timeseries/TimeSeriesControllerPlugin.java | 3 ++- .../artefacts/reports/aggregated/ReportNodeTimeSeries.java | 2 +- .../step/core/timeseries/TimeSeriesCollectionsSettings.java | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index 0269fb995..9ac3b70fa 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2.5.0 + 2025.12.1-692d8a3d11ea7b7be59f1ad4 3.0.23 diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java index abcd97250..de39c5f3a 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java @@ -52,6 +52,7 @@ public class TimeSeriesControllerPlugin extends AbstractControllerPlugin { public static final String PARAM_KEY_ANALYTICS_DASHBOARD_ID = "plugins.timeseries.analytics.dashboard.id"; public static final String PARAM_KEY_RESPONSE_IDEAL_INTERVALS = "timeseries.response.intervals.ideal"; public static final String PARAM_KEY_RESPONSE_MAX_INTERVALS = "timeseries.response.intervals.max"; + public static final String RESOLUTION_PERIOD_PROPERTY = "plugins.timeseries.resolution.period"; public static final String EXECUTION_DASHBOARD_PREPOPULATED_NAME = "Execution Dashboard"; public static final String ANALYTICS_DASHBOARD_PREPOPULATED_NAME = "Analytics Dashboard"; @@ -111,7 +112,7 @@ public void serverStart(GlobalContext context) { WebApplicationConfigurationManager configurationManager = context.require(WebApplicationConfigurationManager.class); // Following property is used by the UI. We could align its name with the configuration property in the future - configurationManager.registerHook(s -> Map.of("plugins.timeseries.resolution.period", String.valueOf(timeSeries.getDefaultCollection().getResolution()))); + configurationManager.registerHook(s -> Map.of(RESOLUTION_PERIOD_PROPERTY, String.valueOf(timeSeries.getDefaultCollection().getResolution()))); } @Override diff --git a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java index 095f33229..11358d3ee 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java +++ b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java @@ -99,7 +99,7 @@ public Map> queryByExecutionIdAndGroupBy(String exec Filter filter = Filters.equals("attributes." + EXECUTION_ID, executionId); Set groupBy = Set.of(groupLevel1, groupLevel2); TimeSeriesAggregationQueryBuilder queryBuilder = new TimeSeriesAggregationQueryBuilder() - .withOptimizationType(TimeSeriesOptimizationType.MOST_ACCURATE) + .withOptimizationType(TimeSeriesOptimizationType.MOST_EFFICIENT) .withFilter(filter) .withGroupDimensions(groupBy) .split(1); diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java index cab6c2c05..de8d2cf70 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java @@ -167,7 +167,7 @@ public TimeSeriesCollectionsSettings setWeeklyFlushInterval(long weeklyFlushInte } public static TimeSeriesCollectionsSettings readSettings(Configuration configuration, String collectionName) { - long mainResolution = getPropertyAsLong(configuration, TIME_SERIES_MAIN_RESOLUTION, collectionName, 1000L); + long mainResolution = getPropertyAsLong(configuration, TIME_SERIES_MAIN_RESOLUTION, collectionName, 5000L); validateMainResolutionParam(mainResolution); return new TimeSeriesCollectionsSettings() .setMainResolution(mainResolution) From fcc5eb302df6fa1b13207d3b91892c7cc3b2b3c2 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Tue, 2 Dec 2025 09:27:32 +0100 Subject: [PATCH 2/9] SED-4415 optimize-reporting-request-and-query-performance --- .../java/step/core/timeseries/TimeSeriesCollectionsBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java index 9db714c8a..72fc0c84e 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java @@ -44,7 +44,7 @@ public List getTimeSeriesCollections(String mainCollection List enabledCollections = new ArrayList<>(); int flushSeriesQueueSize = collectionsSettings.getFlushSeriesQueueSize(); int flushAsyncQueueSize = collectionsSettings.getFlushAsyncQueueSize(); - addIfEnabled(enabledCollections, mainCollectionName, Duration.ofSeconds(1), collectionsSettings.getMainFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, true); + addIfEnabled(enabledCollections, mainCollectionName, Duration.ofMillis(collectionsSettings.getMainResolution()), collectionsSettings.getMainFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, true); addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_PER_MINUTE, Duration.ofMinutes(1), collectionsSettings.getPerMinuteFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, collectionsSettings.isPerMinuteEnabled()); addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_HOURLY, Duration.ofHours(1), collectionsSettings.getHourlyFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize, ignoredAttributesForHighResolution, collectionsSettings.isHourlyEnabled()); addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_DAILY, Duration.ofDays(1), collectionsSettings.getDailyFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize, ignoredAttributesForHighResolution, collectionsSettings.isDailyEnabled()); From eb71b1fde4eacd6c3f566bc372296c102f80e556 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Tue, 2 Dec 2025 17:04:22 +0100 Subject: [PATCH 3/9] SED-4415 changing ideal resolution selection --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9ac3b70fa..b9a16222c 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2025.12.1-692d8a3d11ea7b7be59f1ad4 + 2025.12.2-692f0a5d11ea7b7be5cac669 3.0.23 From 1394e0807a132445c8f11cc49b4ab9fe224aeeb8 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Wed, 3 Dec 2025 12:04:45 +0100 Subject: [PATCH 4/9] Revert "SED-4415 changing ideal resolution selection" This reverts commit eb71b1fde4eacd6c3f566bc372296c102f80e556. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b9a16222c..9ac3b70fa 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2025.12.2-692f0a5d11ea7b7be5cac669 + 2025.12.1-692d8a3d11ea7b7be59f1ad4 3.0.23 From f77d472eb59ffbf9c01a0b9ad1607a0cfa378ac3 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Wed, 3 Dec 2025 16:13:02 +0100 Subject: [PATCH 5/9] SED-4415 introducing new resolutions --- .../core/controller/StepControllerPlugin.java | 13 ++ .../TimeSeriesCollectionsBuilder.java | 42 +++++- .../TimeSeriesCollectionsSettings.java | 142 +++++------------- .../timeseries/ReportNodeTimeSeriesTest.java | 13 +- 4 files changed, 90 insertions(+), 120 deletions(-) diff --git a/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java b/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java index ed5708faf..2b2dbaf8c 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java +++ b/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java @@ -3,8 +3,10 @@ import ch.exense.commons.app.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import step.controller.services.async.AsyncTaskManager; import step.core.Controller; import step.core.GlobalContext; +import step.core.artefacts.reports.aggregated.ReportNodeTimeSeries; import step.core.controller.errorhandling.ErrorFilter; import step.core.deployment.ControllerServices; import step.core.execution.model.Execution; @@ -124,6 +126,17 @@ public void recover(GlobalContext context) throws Exception { @Override public void finalizeStart(GlobalContext context) throws Exception { context.require(ExecutionScheduler.class).start(); + //Initialize new empty resolutions after start (require the async task manager) + //Because the ReportNodeTimeSeries is created in ControllerServer.init directly and not in a plugin, this the right place to do it + ReportNodeTimeSeries reportNodeTimeSeries = context.require(ReportNodeTimeSeries.class); + AsyncTaskManager asyncTaskManager = context.require(AsyncTaskManager.class); + asyncTaskManager.scheduleAsyncTask((empty) -> { + logger.info("ReportNode timeSeries ingestion for empty resolutions has started"); + reportNodeTimeSeries.getTimeSeries().ingestDataForEmptyCollections(); + logger.info("TimeSeries ingestion for empty resolutions has finished"); + return null; + }); + } @Override diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java index 72fc0c84e..aca404994 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java @@ -29,10 +29,29 @@ public class TimeSeriesCollectionsBuilder { - public static final String TIME_SERIES_SUFFIX_PER_MINUTE = "_minute"; - public static final String TIME_SERIES_SUFFIX_HOURLY = "_hour"; - public static final String TIME_SERIES_SUFFIX_DAILY = "_day"; - public static final String TIME_SERIES_SUFFIX_WEEKLY = "_week"; + public static final String COLLECTION_NAME_SEPARATOR = "_"; + + public enum Resolution { + FIFTEEN_SECONDS("15_seconds", Duration.ofSeconds(15), Duration.ofSeconds(5), false), + ONE_MINUTE("minute", Duration.ofMinutes(1), Duration.ofSeconds(10), false), + FIFTEEN_MINUTES("15_minutes", Duration.ofMinutes(15), Duration.ofMinutes(1), false), + ONE_HOUR("hour", Duration.ofHours(1), Duration.ofMinutes(5), true), + SIX_HOURS("6_hours", Duration.ofHours(6), Duration.ofMinutes(30), true), + ONE_DAY("day", Duration.ofDays(1), Duration.ofHours(1), true), + ONE_WEEK("week", Duration.ofDays(7), Duration.ofHours(2), true); + + public final String name; + public final Duration resolution; + public final Duration defaultFlushPeriod; + public final boolean coarseResolution; + + Resolution(String name, Duration resolution, Duration defaultFlushPeriod, boolean coarseResolution) { + this.name = name; + this.resolution = resolution; + this.defaultFlushPeriod = defaultFlushPeriod; + this.coarseResolution = coarseResolution; + } + } private final CollectionFactory collectionFactory; @@ -44,11 +63,18 @@ public List getTimeSeriesCollections(String mainCollection List enabledCollections = new ArrayList<>(); int flushSeriesQueueSize = collectionsSettings.getFlushSeriesQueueSize(); int flushAsyncQueueSize = collectionsSettings.getFlushAsyncQueueSize(); + //Add main resolution addIfEnabled(enabledCollections, mainCollectionName, Duration.ofMillis(collectionsSettings.getMainResolution()), collectionsSettings.getMainFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, true); - addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_PER_MINUTE, Duration.ofMinutes(1), collectionsSettings.getPerMinuteFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, collectionsSettings.isPerMinuteEnabled()); - addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_HOURLY, Duration.ofHours(1), collectionsSettings.getHourlyFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize, ignoredAttributesForHighResolution, collectionsSettings.isHourlyEnabled()); - addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_DAILY, Duration.ofDays(1), collectionsSettings.getDailyFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize, ignoredAttributesForHighResolution, collectionsSettings.isDailyEnabled()); - addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_WEEKLY, Duration.ofDays(7), collectionsSettings.getWeeklyFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize, ignoredAttributesForHighResolution, collectionsSettings.isWeeklyEnabled()); + //Add additional resolutions + for (Resolution resolution: Resolution.values()) { + TimeSeriesCollectionsSettings.ResolutionSettings resolutionSettings = collectionsSettings.getResolutionSettings(resolution); + if (resolutionSettings != null) { + addIfEnabled(enabledCollections, mainCollectionName + COLLECTION_NAME_SEPARATOR + resolution.name, + resolution.resolution, resolutionSettings.flushInterval, flushSeriesQueueSize, flushAsyncQueueSize, + (resolution.coarseResolution ? ignoredAttributesForHighResolution : null), resolutionSettings.enabled); + + } + } return enabledCollections; } diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java index de8d2cf70..42d86bd32 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java @@ -22,6 +22,8 @@ import ch.exense.commons.app.Configuration; import java.time.Duration; +import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.TimeUnit; public class TimeSeriesCollectionsSettings { @@ -30,14 +32,9 @@ public class TimeSeriesCollectionsSettings { public static final String TIME_SERIES_COLLECTION_FLUSH_ASYNC_QUEUE_SIZE = "{collectionName}.flush.async.queue.size"; public static final String TIME_SERIES_COLLECTION_FLUSH_SERIES_QUEUE_SIZE = "{collectionName}.flush.series.queue.size"; public static final String TIME_SERIES_MAIN_RESOLUTION = "{collectionName}.resolution"; - public static final String TIME_SERIES_MINUTE_COLLECTION_ENABLED = "{collectionName}.collections.minute.enabled"; - public static final String TIME_SERIES_MINUTE_COLLECTION_FLUSH_PERIOD = "{collectionName}.collections.minute.flush.period"; - public static final String TIME_SERIES_HOUR_COLLECTION_ENABLED = "{collectionName}.collections.hour.enabled"; - public static final String TIME_SERIES_HOUR_COLLECTION_FLUSH_PERIOD = "{collectionName}.collections.hour.flush.period"; - public static final String TIME_SERIES_DAY_COLLECTION_ENABLED = "{collectionName}.collections.day.enabled"; - public static final String TIME_SERIES_DAY_COLLECTION_FLUSH_PERIOD = "{collectionName}.collections.day.flush.period"; - public static final String TIME_SERIES_WEEK_COLLECTION_ENABLED = "{collectionName}.collections.week.enabled"; - public static final String TIME_SERIES_WEEK_COLLECTION_FLUSH_PERIOD = "{collectionName}.collections.week.flush.period"; + public static final String RESOLUTION_PROPERTY_PREFIX = "{collectionName}.collections."; + public static final String TIME_SERIES_RESOLUTION_ENABLED_SUFFIX = ".enabled"; + public static final String TIME_SERIES_RESOLUTION_FLUSH_PERIOD_SUFFIX = ".flush.period"; private long mainResolution; //Define the interval of the flushing job for the main ingestion pipeline (highest resolution) @@ -49,139 +46,74 @@ public class TimeSeriesCollectionsSettings { //flushing do not write to DB directly but to a linked blocking queue in memory which is processed by an asynchronous processor, the queue size is limited to prevent excessive memory usage //While the queue is full, ingesting new buckets is blocked private int flushAsyncQueueSize; - private boolean perMinuteEnabled; - private long perMinuteFlushInterval; - private boolean hourlyEnabled; - private long hourlyFlushInterval; - private boolean dailyEnabled; - private long dailyFlushInterval; - private boolean weeklyEnabled; - private long weeklyFlushInterval; + private final Map additionalResolutionSettings = new TreeMap<>(); - public boolean isDailyEnabled() { - return dailyEnabled; - } - - public boolean isPerMinuteEnabled() { - return perMinuteEnabled; - } + public static class ResolutionSettings { + public final long flushInterval; + public final boolean enabled; - public boolean isHourlyEnabled() { - return hourlyEnabled; - } - - public boolean isWeeklyEnabled() { - return weeklyEnabled; + public ResolutionSettings(long flushInterval, boolean enabled) { + this.flushInterval = flushInterval; + this.enabled = enabled; + } } public long getMainResolution() { return mainResolution; } - public TimeSeriesCollectionsSettings setMainResolution(long mainResolution) { + public void setMainResolution(long mainResolution) { this.mainResolution = mainResolution; - return this; - } - - public TimeSeriesCollectionsSettings setPerMinuteEnabled(boolean perMinuteEnabled) { - this.perMinuteEnabled = perMinuteEnabled; - return this; - } - - public TimeSeriesCollectionsSettings setHourlyEnabled(boolean hourlyEnabled) { - this.hourlyEnabled = hourlyEnabled; - return this; - } - - public TimeSeriesCollectionsSettings setDailyEnabled(boolean dailyEnabled) { - this.dailyEnabled = dailyEnabled; - return this; - } - - public TimeSeriesCollectionsSettings setWeeklyEnabled(boolean weeklyEnabled) { - this.weeklyEnabled = weeklyEnabled; - return this; } public long getMainFlushInterval() { return mainFlushInterval; } - public TimeSeriesCollectionsSettings setMainFlushInterval(long mainFlushInterval) { + public void setMainFlushInterval(long mainFlushInterval) { this.mainFlushInterval = mainFlushInterval; - return this; - } - - public long getPerMinuteFlushInterval() { - return perMinuteFlushInterval; } public int getFlushSeriesQueueSize() { return flushSeriesQueueSize; } - public TimeSeriesCollectionsSettings setFlushSeriesQueueSize(int flushSeriesQueueSize) { + public void setFlushSeriesQueueSize(int flushSeriesQueueSize) { this.flushSeriesQueueSize = flushSeriesQueueSize; - return this; } - private TimeSeriesCollectionsSettings setFlushAsyncQueueSize(int flushAsyncQueueSize) { + private void setFlushAsyncQueueSize(int flushAsyncQueueSize) { this.flushAsyncQueueSize = flushAsyncQueueSize; - return this; } public int getFlushAsyncQueueSize() { return flushAsyncQueueSize; } - public TimeSeriesCollectionsSettings setPerMinuteFlushInterval(long perMinuteFlushInterval) { - this.perMinuteFlushInterval = perMinuteFlushInterval; - return this; - } - - public long getHourlyFlushInterval() { - return hourlyFlushInterval; - } - - public TimeSeriesCollectionsSettings setHourlyFlushInterval(long hourlyFlushInterval) { - this.hourlyFlushInterval = hourlyFlushInterval; - return this; - } - - public long getDailyFlushInterval() { - return dailyFlushInterval; - } - - public TimeSeriesCollectionsSettings setDailyFlushInterval(long dailyFlushInterval) { - this.dailyFlushInterval = dailyFlushInterval; - return this; + private void addAdditionalResolutionSettings(TimeSeriesCollectionsBuilder.Resolution resolution, ResolutionSettings resolutionSettings) { + additionalResolutionSettings.put(resolution, resolutionSettings); } - - public long getWeeklyFlushInterval() { - return weeklyFlushInterval; - } - - public TimeSeriesCollectionsSettings setWeeklyFlushInterval(long weeklyFlushInterval) { - this.weeklyFlushInterval = weeklyFlushInterval; - return this; + public ResolutionSettings getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution resolution) { + return additionalResolutionSettings.get(resolution); } public static TimeSeriesCollectionsSettings readSettings(Configuration configuration, String collectionName) { + // Validate and read main resolution settings long mainResolution = getPropertyAsLong(configuration, TIME_SERIES_MAIN_RESOLUTION, collectionName, 5000L); validateMainResolutionParam(mainResolution); - return new TimeSeriesCollectionsSettings() - .setMainResolution(mainResolution) - .setMainFlushInterval(getPropertyAsLong(configuration, TIME_SERIES_MAIN_COLLECTION_FLUSH_PERIOD, collectionName, Duration.ofSeconds(1).toMillis())) - .setFlushSeriesQueueSize(getPropertyAsInteger(configuration, TIME_SERIES_COLLECTION_FLUSH_SERIES_QUEUE_SIZE, collectionName, 20000)) - .setFlushAsyncQueueSize(getPropertyAsInteger(configuration, TIME_SERIES_COLLECTION_FLUSH_ASYNC_QUEUE_SIZE, collectionName, 5000)) - .setPerMinuteEnabled(getPropertyAsBoolean(configuration, TIME_SERIES_MINUTE_COLLECTION_ENABLED, collectionName, true)) - .setPerMinuteFlushInterval(getPropertyAsLong(configuration, TIME_SERIES_MINUTE_COLLECTION_FLUSH_PERIOD, collectionName, Duration.ofMinutes(1).toMillis())) - .setHourlyEnabled(getPropertyAsBoolean(configuration, TIME_SERIES_HOUR_COLLECTION_ENABLED, collectionName, true)) - .setHourlyFlushInterval(getPropertyAsLong(configuration, TIME_SERIES_HOUR_COLLECTION_FLUSH_PERIOD, collectionName, Duration.ofMinutes(5).toMillis())) - .setDailyEnabled(getPropertyAsBoolean(configuration, TIME_SERIES_DAY_COLLECTION_ENABLED, collectionName, true)) - .setDailyFlushInterval(getPropertyAsLong(configuration, TIME_SERIES_DAY_COLLECTION_FLUSH_PERIOD, collectionName, Duration.ofHours(1).toMillis())) - .setWeeklyEnabled(getPropertyAsBoolean(configuration, TIME_SERIES_WEEK_COLLECTION_ENABLED, collectionName, true)) - .setWeeklyFlushInterval(getPropertyAsLong(configuration, TIME_SERIES_WEEK_COLLECTION_FLUSH_PERIOD, collectionName, Duration.ofHours(2).toMillis())); + TimeSeriesCollectionsSettings settings = new TimeSeriesCollectionsSettings(); + Long mainResolutionFlushInterval = getPropertyAsLong(configuration, TIME_SERIES_MAIN_COLLECTION_FLUSH_PERIOD, collectionName, Duration.ofSeconds(1).toMillis()); + settings.setMainResolution(mainResolution); + settings.setMainFlushInterval(mainResolutionFlushInterval); + settings.setFlushSeriesQueueSize(getPropertyAsInteger(configuration, TIME_SERIES_COLLECTION_FLUSH_SERIES_QUEUE_SIZE, collectionName, 20000)); + settings.setFlushAsyncQueueSize(getPropertyAsInteger(configuration, TIME_SERIES_COLLECTION_FLUSH_ASYNC_QUEUE_SIZE, collectionName, 5000)); + //Read settings for additional resolutions + for (TimeSeriesCollectionsBuilder.Resolution resolution: TimeSeriesCollectionsBuilder.Resolution.values()){ + boolean resolutionEnabled = getPropertyAsBoolean(configuration, RESOLUTION_PROPERTY_PREFIX + resolution.name + TIME_SERIES_RESOLUTION_ENABLED_SUFFIX, collectionName, true); + Long resolutionFlushInterval = getPropertyAsLong(configuration, RESOLUTION_PROPERTY_PREFIX + resolution.name + TIME_SERIES_RESOLUTION_FLUSH_PERIOD_SUFFIX, collectionName, resolution.defaultFlushPeriod.toMillis()); + settings.addAdditionalResolutionSettings(resolution, new ResolutionSettings(resolutionFlushInterval, resolutionEnabled)); + } + return settings; } private static Long getPropertyAsLong(Configuration configuration, String property, String collectionName, long defaultValue) { @@ -209,10 +141,6 @@ private static void validateMainResolutionParam(long resolution) { public static TimeSeriesCollectionsSettings buildSingleResolutionSettings(long mainResolution, long mainFlushInterval) { TimeSeriesCollectionsSettings timeSeriesCollectionsSettings = new TimeSeriesCollectionsSettings(); - timeSeriesCollectionsSettings.setDailyEnabled(false); - timeSeriesCollectionsSettings.setHourlyEnabled(false); - timeSeriesCollectionsSettings.setPerMinuteEnabled(false); - timeSeriesCollectionsSettings.setWeeklyEnabled(false); timeSeriesCollectionsSettings.setMainResolution(mainResolution); timeSeriesCollectionsSettings.setMainFlushInterval(mainFlushInterval); return timeSeriesCollectionsSettings; diff --git a/step-plans/step-plans-core/src/test/java/step/core/timeseries/ReportNodeTimeSeriesTest.java b/step-plans/step-plans-core/src/test/java/step/core/timeseries/ReportNodeTimeSeriesTest.java index 39cf2cdfe..673442d8c 100644 --- a/step-plans/step-plans-core/src/test/java/step/core/timeseries/ReportNodeTimeSeriesTest.java +++ b/step-plans/step-plans-core/src/test/java/step/core/timeseries/ReportNodeTimeSeriesTest.java @@ -45,7 +45,7 @@ public void reportNodeTimeSeries() { try (ReportNodeTimeSeries reportNodeTimeSeries = new ReportNodeTimeSeries(collectionFactory, configuration)) { TimeSeries timeSeries = reportNodeTimeSeries.getTimeSeries(); List collections = timeSeries.getCollections(); - assertEquals(5, collections.size()); + assertEquals(8, collections.size()); ReportNode reportNode = new ReportNode(); reportNode.setStatus(ReportNodeStatus.PASSED); reportNode.setExecutionID("executionId"); @@ -66,8 +66,11 @@ public void reportNodeTimeSeries() { @Test public void reportNodeTimeSeriesDisabled() { Configuration configuration = new Configuration(); + configuration.putProperty("reportNodeTimeSeries.collections.15_seconds.enabled", "false"); configuration.putProperty("reportNodeTimeSeries.collections.minute.enabled", "false"); + configuration.putProperty("reportNodeTimeSeries.collections.15_minutes.enabled", "false"); configuration.putProperty("reportNodeTimeSeries.collections.hour.enabled", "false"); + configuration.putProperty("reportNodeTimeSeries.collections.6_hours.enabled", "false"); configuration.putProperty("reportNodeTimeSeries.collections.day.enabled", "false"); configuration.putProperty("reportNodeTimeSeries.collections.week.enabled", "false"); CollectionFactory collectionFactory = new InMemoryCollectionFactory(null); @@ -86,10 +89,10 @@ public void reportNodeTimeSeriesFlushPeriods() { configuration.putProperty("reportNodeTimeSeries.collections.week.flush.period", "14"); TimeSeriesCollectionsSettings settings = TimeSeriesCollectionsSettings.readSettings(configuration, "reportNodeTimeSeries"); - assertEquals(11, settings.getPerMinuteFlushInterval()); - assertEquals(12, settings.getHourlyFlushInterval()); - assertEquals(13, settings.getDailyFlushInterval()); - assertEquals(14, settings.getWeeklyFlushInterval()); + assertEquals(11, settings.getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution.ONE_MINUTE).flushInterval); + assertEquals(12, settings.getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution.ONE_HOUR).flushInterval); + assertEquals(13, settings.getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution.ONE_DAY).flushInterval); + assertEquals(14, settings.getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution.ONE_WEEK).flushInterval); } } From e91414a59ee2a6bd83e34b06e46fed7285009c78 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 5 Dec 2025 16:41:13 +0100 Subject: [PATCH 6/9] SED-4415 migration, housekeeping, parallel aggregations --- pom.xml | 2 +- .../controller/ControllerSettingPlugin.java | 3 +- .../core/controller/StepControllerPlugin.java | 2 +- .../tasks/MigrationManagerTasksPlugin.java | 1 + ...pdateTimeSeriesCollectionsAndSettings.java | 98 +++++++++++++++++++ .../src/main/java/step/core/Constants.java | 2 +- .../AggregatedReportViewBuilder.java | 18 +++- .../java/step/core/timeseries/Resolution.java | 49 ++++++++++ .../TimeSeriesCollectionsBuilder.java | 24 ----- .../TimeSeriesCollectionsSettings.java | 59 ++++------- .../timeseries/ReportNodeTimeSeriesTest.java | 17 +++- 11 files changed, 197 insertions(+), 78 deletions(-) create mode 100644 step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_1_UpdateTimeSeriesCollectionsAndSettings.java create mode 100644 step-plans/step-plans-core/src/main/java/step/core/timeseries/Resolution.java diff --git a/pom.xml b/pom.xml index 9ac3b70fa..1ac34a9ba 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2025.12.1-692d8a3d11ea7b7be59f1ad4 + 2025.12.5-6932fb5414e3a93583594624 3.0.23 diff --git a/step-controller/step-controller-server/src/main/java/step/core/controller/ControllerSettingPlugin.java b/step-controller/step-controller-server/src/main/java/step/core/controller/ControllerSettingPlugin.java index 30fb2e4cc..39bfb4aed 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/controller/ControllerSettingPlugin.java +++ b/step-controller/step-controller-server/src/main/java/step/core/controller/ControllerSettingPlugin.java @@ -29,12 +29,13 @@ @Plugin public class ControllerSettingPlugin extends AbstractControllerPlugin { + public static final String SETTINGS = "settings"; private ControllerSettingAccessor controllerSettingAccessor; @Override public void serverStart(GlobalContext context) throws Exception { controllerSettingAccessor = new ControllerSettingAccessorImpl( - context.getCollectionFactory().getCollection("settings", ControllerSetting.class)); + context.getCollectionFactory().getCollection(SETTINGS, ControllerSetting.class)); context.put(ControllerSettingAccessor.class, controllerSettingAccessor); } diff --git a/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java b/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java index 2b2dbaf8c..11de5756d 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java +++ b/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java @@ -133,7 +133,7 @@ public void finalizeStart(GlobalContext context) throws Exception { asyncTaskManager.scheduleAsyncTask((empty) -> { logger.info("ReportNode timeSeries ingestion for empty resolutions has started"); reportNodeTimeSeries.getTimeSeries().ingestDataForEmptyCollections(); - logger.info("TimeSeries ingestion for empty resolutions has finished"); + logger.info("ReportNode timeSeries ingestion for empty resolutions has finished"); return null; }); diff --git a/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java b/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java index 1a591be5a..5478239ce 100644 --- a/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java +++ b/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java @@ -54,6 +54,7 @@ public void serverStart(GlobalContext context) throws Exception { migrationManager.register(V27_4_DropResolvedPlanNodesIndexForPSQLMigrationTask.class); migrationManager.register(V28_0_FixEmptyDefaultMavenSettingsMigrationTask.class); migrationManager.register(V29_0_UpdateAutomationPackageModel.class); + migrationManager.register(V29_1_UpdateTimeSeriesCollectionsAndSettings.class); } @Override diff --git a/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_1_UpdateTimeSeriesCollectionsAndSettings.java b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_1_UpdateTimeSeriesCollectionsAndSettings.java new file mode 100644 index 000000000..aebe87e11 --- /dev/null +++ b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_1_UpdateTimeSeriesCollectionsAndSettings.java @@ -0,0 +1,98 @@ +/* + * ****************************************************************************** + * * Copyright (C) 2020, exense GmbH + * * + * * This file is part of STEP + * * + * * STEP is free software: you can redistribute it and/or modify + * * it under the terms of the GNU Affero General Public License as published by + * * the Free Software Foundation, either version 3 of the License, or + * * (at your option) any later version. + * * + * * STEP is distributed in the hope that it will be useful, + * * but WITHOUT ANY WARRANTY; without even the implied warranty of + * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * * GNU Affero General Public License for more details. + * * + * * You should have received a copy of the GNU Affero General Public License + * * along with STEP. If not, see . + * ***************************************************************************** + */ +package step.migration.tasks; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import step.core.Version; +import step.core.collections.Collection; +import step.core.collections.CollectionFactory; +import step.core.collections.Document; +import step.core.collections.Filters; +import step.core.timeseries.Resolution; +import step.migration.MigrationContext; +import step.migration.MigrationTask; + +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; + +import static step.core.controller.ControllerSettingPlugin.SETTINGS; + +public class V29_1_UpdateTimeSeriesCollectionsAndSettings extends MigrationTask { + + public static final String TIME_SERIES_MAIN_COLLECTION = "timeseries"; + public static final String TIME_SERIES_MAIN_COLLECTION_NEW_NAME = "timeseries_5_seconds"; + public static final String TIME_SERIES_MAIN_COLLECTION_REPORTS = "reportNodeTimeSeries"; + public static final String TIME_SERIES_MAIN_COLLECTION_REPORTS_NEW_NAME = "reportNodeTimeSeries_5_seconds"; + public static final String HOUSEKEEPING_TIME_SERIES_DEFAULT_TTL = "housekeeping_time_series_default_ttl"; + public static final String HOUSEKEEPING_TIME_SERIES_PER_MINUTE_TTL = "housekeeping_time_series_per_minute_ttl"; + public static final String HOUSEKEEPING_TIME_SERIES_HOURLY_TTL = "housekeeping_time_series_hourly_ttl"; + public static final String HOUSEKEEPING_TIME_SERIES_DAILY_TTL = "housekeeping_time_series_daily_ttl"; + public static final String HOUSEKEEPING_TIME_SERIES_WEEKLY_TTL = "housekeeping_time_series_weekly_ttl"; + public static final String HOUSEKEEPING_TIME_SERIES_TTL_PREFIX = "housekeeping_time_series_"; + public static final String HOUSEKEEPING_TIME_SERIES_TTL_SUFFIX = "_ttl"; + + private static final Logger log = LoggerFactory.getLogger(V29_1_UpdateTimeSeriesCollectionsAndSettings.class); + private final Collection timeseriesCollection; + private final Collection settings; + private final Collection reportNodeTimeseriesCollection; + protected AtomicInteger successCount; + + public V29_1_UpdateTimeSeriesCollectionsAndSettings(CollectionFactory collectionFactory, MigrationContext migrationContext) { + super(new Version(3,29,1), collectionFactory, migrationContext); + timeseriesCollection = collectionFactory.getCollection(TIME_SERIES_MAIN_COLLECTION, Document.class); + reportNodeTimeseriesCollection = collectionFactory.getCollection(TIME_SERIES_MAIN_COLLECTION_REPORTS, Document.class); + settings = collectionFactory.getCollection(SETTINGS, Document.class); + } + + @Override + public void runUpgradeScript() { + log.info("Renaming time-series 'main' collections to match their resolutions"); + + timeseriesCollection.rename(TIME_SERIES_MAIN_COLLECTION_NEW_NAME); + + reportNodeTimeseriesCollection.rename(TIME_SERIES_MAIN_COLLECTION_REPORTS_NEW_NAME); + log.info("Time-series 'main' collections renamed to match their resolutions"); + + log.info("Renaming time-series housekeeping setting keys"); + //use names from enum which will then be aligned with the collection names + updateSettingKeyIfPresent(HOUSEKEEPING_TIME_SERIES_DEFAULT_TTL, HOUSEKEEPING_TIME_SERIES_TTL_PREFIX + Resolution.FIVE_SECONDS.name + HOUSEKEEPING_TIME_SERIES_TTL_SUFFIX); + updateSettingKeyIfPresent(HOUSEKEEPING_TIME_SERIES_PER_MINUTE_TTL, HOUSEKEEPING_TIME_SERIES_TTL_PREFIX + Resolution.ONE_MINUTE.name + HOUSEKEEPING_TIME_SERIES_TTL_SUFFIX); + updateSettingKeyIfPresent(HOUSEKEEPING_TIME_SERIES_HOURLY_TTL, HOUSEKEEPING_TIME_SERIES_TTL_PREFIX + Resolution.ONE_HOUR.name + HOUSEKEEPING_TIME_SERIES_TTL_SUFFIX); + updateSettingKeyIfPresent(HOUSEKEEPING_TIME_SERIES_DAILY_TTL, HOUSEKEEPING_TIME_SERIES_TTL_PREFIX + Resolution.ONE_DAY.name + HOUSEKEEPING_TIME_SERIES_TTL_SUFFIX); + updateSettingKeyIfPresent(HOUSEKEEPING_TIME_SERIES_WEEKLY_TTL, HOUSEKEEPING_TIME_SERIES_TTL_PREFIX + Resolution.ONE_WEEK.name + HOUSEKEEPING_TIME_SERIES_TTL_SUFFIX); + log.info("Time-series housekeeping setting keys renamed"); + } + + private void updateSettingKeyIfPresent(String oldKey, String newKey) { + Optional setting = settings.find(Filters.equals("key", oldKey), null, null, null, 0).findFirst(); + setting.ifPresent(s -> { + s.put("key", newKey); + settings.save(s); + }); + + } + + @Override + public void runDowngradeScript() { + + } +} diff --git a/step-core/src/main/java/step/core/Constants.java b/step-core/src/main/java/step/core/Constants.java index 1da256f2a..3762b32ae 100644 --- a/step-core/src/main/java/step/core/Constants.java +++ b/step-core/src/main/java/step/core/Constants.java @@ -19,7 +19,7 @@ package step.core; public interface Constants { - String STEP_API_VERSION_STRING = "3.29.0"; + String STEP_API_VERSION_STRING = "3.29.1"; Version STEP_API_VERSION = new Version(STEP_API_VERSION_STRING); String STEP_YAML_SCHEMA_VERSION_STRING = "1.2.0"; diff --git a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java index 41db30a91..c22360ec6 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java @@ -20,11 +20,13 @@ import step.core.execution.model.ExecutionAccessor; import step.core.execution.model.ExecutionStatus; import step.core.plugins.threadmanager.ThreadManager; +import step.core.timeseries.Resolution; import step.core.timeseries.TimeSeriesCollectionsSettings; import step.core.timeseries.bucket.Bucket; import step.core.timeseries.bucket.BucketBuilder; import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -112,9 +114,17 @@ public AggregatedReport buildAggregatedReport(AggregatedReportViewRequest reques } } // Generate complete aggregated report tree - //First aggregate time series data for the given execution context grouped by artefactHash - Map> countByHashAndStatus = mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, STATUS); - Map> countByHashAndErrorMessage = mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, ERROR_MESSAGE); + //First aggregate time series data for the given execution context grouped by artefactHash and status as wells as artefactHash and error message + CompletableFuture>> countByHashAndStatusFuture = + CompletableFuture.supplyAsync(() -> + mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, STATUS) + ); + CompletableFuture>> countByHashAndErrorMessageFuture = + CompletableFuture.supplyAsync(() -> + mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, ERROR_MESSAGE) + ); + Map> countByHashAndStatus = countByHashAndStatusFuture.join(); + Map> countByHashAndErrorMessage = countByHashAndErrorMessageFuture.join(); //Because the time series time range extends to the time series resolution we need to use the same range when querying the report nodes Range resolvedRange = getResolvedRange(request, countByHashAndStatus); return new AggregatedReport(recursivelyBuildAggregatedReportTree(rootResolvedPlanNode, request, countByHashAndStatus, countByHashAndErrorMessage, mainReportNodeAccessor, null, runningCountByArtefactHash, operationsByArtefactHash, resolvedRange)); @@ -157,7 +167,7 @@ private ReportNodeTimeSeries getInMemoryReportNodeTimeSeries() { //Need to create a configuration with all time series details return new ReportNodeTimeSeries(new InMemoryCollectionFactory(new Properties()), // to build the report we only need a single time bucket and can flush only once all reports are ingested - TimeSeriesCollectionsSettings.buildSingleResolutionSettings(Long.MAX_VALUE, 0), true); + TimeSeriesCollectionsSettings.buildSingleResolutionSettings(Resolution.ONE_WEEK, 0), true); } /** diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/Resolution.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/Resolution.java new file mode 100644 index 000000000..0fa42cdab --- /dev/null +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/Resolution.java @@ -0,0 +1,49 @@ +package step.core.timeseries; + +import java.time.Duration; +import java.util.Arrays; + +/** + * This enum define the time series resolutions supported in Step. It is critical to keep them ordered by resolution from the finest to the coarsest + */ +public enum Resolution { + FIVE_SECONDS("5_seconds", Duration.ofSeconds(5), Duration.ofSeconds(1), false), + FIFTEEN_SECONDS("15_seconds", Duration.ofSeconds(15), Duration.ofSeconds(5), false), + ONE_MINUTE("minute", Duration.ofMinutes(1), Duration.ofSeconds(10), false), + FIFTEEN_MINUTES("15_minutes", Duration.ofMinutes(15), Duration.ofMinutes(1), false), + ONE_HOUR("hour", Duration.ofHours(1), Duration.ofMinutes(5), true), + SIX_HOURS("6_hours", Duration.ofHours(6), Duration.ofMinutes(30), true), + ONE_DAY("day", Duration.ofDays(1), Duration.ofHours(1), true), + ONE_WEEK("week", Duration.ofDays(7), Duration.ofHours(2), true); + + /** + * The name of the resolutions which is used to define the collections names as well as the Step properties and setting keys + */ + public final String name; + /** + * The resolution duration + */ + public final Duration resolution; + /** + * The default flush period if not configured in step.properties + */ + public final Duration defaultFlushPeriod; + /** + * Whether this resolution is coarse, coarse resolution exclude specified attributes by the time-series creator (i.e. the execution id) + */ + public final boolean coarseResolution; + + Resolution(String name, Duration resolution, Duration defaultFlushPeriod, boolean coarseResolution) { + this.name = name; + this.resolution = resolution; + this.defaultFlushPeriod = defaultFlushPeriod; + this.coarseResolution = coarseResolution; + } + + public static Resolution fromName(String name) { + return Arrays.stream(values()) + .filter(r -> r.name.equals(name)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("Unknown resolution: " + name)); + } +} diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java index aca404994..6ca35521a 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java @@ -31,28 +31,6 @@ public class TimeSeriesCollectionsBuilder { public static final String COLLECTION_NAME_SEPARATOR = "_"; - public enum Resolution { - FIFTEEN_SECONDS("15_seconds", Duration.ofSeconds(15), Duration.ofSeconds(5), false), - ONE_MINUTE("minute", Duration.ofMinutes(1), Duration.ofSeconds(10), false), - FIFTEEN_MINUTES("15_minutes", Duration.ofMinutes(15), Duration.ofMinutes(1), false), - ONE_HOUR("hour", Duration.ofHours(1), Duration.ofMinutes(5), true), - SIX_HOURS("6_hours", Duration.ofHours(6), Duration.ofMinutes(30), true), - ONE_DAY("day", Duration.ofDays(1), Duration.ofHours(1), true), - ONE_WEEK("week", Duration.ofDays(7), Duration.ofHours(2), true); - - public final String name; - public final Duration resolution; - public final Duration defaultFlushPeriod; - public final boolean coarseResolution; - - Resolution(String name, Duration resolution, Duration defaultFlushPeriod, boolean coarseResolution) { - this.name = name; - this.resolution = resolution; - this.defaultFlushPeriod = defaultFlushPeriod; - this.coarseResolution = coarseResolution; - } - } - private final CollectionFactory collectionFactory; public TimeSeriesCollectionsBuilder(CollectionFactory collectionFactory) { @@ -63,8 +41,6 @@ public List getTimeSeriesCollections(String mainCollection List enabledCollections = new ArrayList<>(); int flushSeriesQueueSize = collectionsSettings.getFlushSeriesQueueSize(); int flushAsyncQueueSize = collectionsSettings.getFlushAsyncQueueSize(); - //Add main resolution - addIfEnabled(enabledCollections, mainCollectionName, Duration.ofMillis(collectionsSettings.getMainResolution()), collectionsSettings.getMainFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, true); //Add additional resolutions for (Resolution resolution: Resolution.values()) { TimeSeriesCollectionsSettings.ResolutionSettings resolutionSettings = collectionsSettings.getResolutionSettings(resolution); diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java index 42d86bd32..2103da77b 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java @@ -28,28 +28,26 @@ public class TimeSeriesCollectionsSettings { - public static final String TIME_SERIES_MAIN_COLLECTION_FLUSH_PERIOD = "{collectionName}.flush.period"; public static final String TIME_SERIES_COLLECTION_FLUSH_ASYNC_QUEUE_SIZE = "{collectionName}.flush.async.queue.size"; public static final String TIME_SERIES_COLLECTION_FLUSH_SERIES_QUEUE_SIZE = "{collectionName}.flush.series.queue.size"; - public static final String TIME_SERIES_MAIN_RESOLUTION = "{collectionName}.resolution"; public static final String RESOLUTION_PROPERTY_PREFIX = "{collectionName}.collections."; public static final String TIME_SERIES_RESOLUTION_ENABLED_SUFFIX = ".enabled"; public static final String TIME_SERIES_RESOLUTION_FLUSH_PERIOD_SUFFIX = ".flush.period"; - private long mainResolution; - //Define the interval of the flushing job for the main ingestion pipeline (highest resolution) - //Note that flush is only actually performed by the job if the bucket time interval is complete (i.e. full resolution interval) or - //if the max series queue size is reached (to limit and control memory usage) - private long mainFlushInterval; //Define the max queue size for series, if the usage is over the limit flush is performed even for partial time interval private int flushSeriesQueueSize; //flushing do not write to DB directly but to a linked blocking queue in memory which is processed by an asynchronous processor, the queue size is limited to prevent excessive memory usage //While the queue is full, ingesting new buckets is blocked private int flushAsyncQueueSize; - private final Map additionalResolutionSettings = new TreeMap<>(); + //Define the Map of supported Resolutions associated to their settings read from the configuration. + private final Map additionalResolutionSettings = new TreeMap<>(); public static class ResolutionSettings { + //Define the interval of the flushing job for the ingestion pipeline + //Note that flush is only actually performed by the job if the bucket time interval is complete (i.e. full resolution interval) or + //if the max series queue size is reached (to limit and control memory usage) public final long flushInterval; + //Flag to completely disable or enable the collection public final boolean enabled; public ResolutionSettings(long flushInterval, boolean enabled) { @@ -58,22 +56,6 @@ public ResolutionSettings(long flushInterval, boolean enabled) { } } - public long getMainResolution() { - return mainResolution; - } - - public void setMainResolution(long mainResolution) { - this.mainResolution = mainResolution; - } - - public long getMainFlushInterval() { - return mainFlushInterval; - } - - public void setMainFlushInterval(long mainFlushInterval) { - this.mainFlushInterval = mainFlushInterval; - } - public int getFlushSeriesQueueSize() { return flushSeriesQueueSize; } @@ -90,25 +72,19 @@ public int getFlushAsyncQueueSize() { return flushAsyncQueueSize; } - private void addAdditionalResolutionSettings(TimeSeriesCollectionsBuilder.Resolution resolution, ResolutionSettings resolutionSettings) { + private void addAdditionalResolutionSettings(Resolution resolution, ResolutionSettings resolutionSettings) { additionalResolutionSettings.put(resolution, resolutionSettings); } - public ResolutionSettings getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution resolution) { + public ResolutionSettings getResolutionSettings(Resolution resolution) { return additionalResolutionSettings.get(resolution); } public static TimeSeriesCollectionsSettings readSettings(Configuration configuration, String collectionName) { - // Validate and read main resolution settings - long mainResolution = getPropertyAsLong(configuration, TIME_SERIES_MAIN_RESOLUTION, collectionName, 5000L); - validateMainResolutionParam(mainResolution); TimeSeriesCollectionsSettings settings = new TimeSeriesCollectionsSettings(); - Long mainResolutionFlushInterval = getPropertyAsLong(configuration, TIME_SERIES_MAIN_COLLECTION_FLUSH_PERIOD, collectionName, Duration.ofSeconds(1).toMillis()); - settings.setMainResolution(mainResolution); - settings.setMainFlushInterval(mainResolutionFlushInterval); settings.setFlushSeriesQueueSize(getPropertyAsInteger(configuration, TIME_SERIES_COLLECTION_FLUSH_SERIES_QUEUE_SIZE, collectionName, 20000)); settings.setFlushAsyncQueueSize(getPropertyAsInteger(configuration, TIME_SERIES_COLLECTION_FLUSH_ASYNC_QUEUE_SIZE, collectionName, 5000)); //Read settings for additional resolutions - for (TimeSeriesCollectionsBuilder.Resolution resolution: TimeSeriesCollectionsBuilder.Resolution.values()){ + for (Resolution resolution: Resolution.values()) { boolean resolutionEnabled = getPropertyAsBoolean(configuration, RESOLUTION_PROPERTY_PREFIX + resolution.name + TIME_SERIES_RESOLUTION_ENABLED_SUFFIX, collectionName, true); Long resolutionFlushInterval = getPropertyAsLong(configuration, RESOLUTION_PROPERTY_PREFIX + resolution.name + TIME_SERIES_RESOLUTION_FLUSH_PERIOD_SUFFIX, collectionName, resolution.defaultFlushPeriod.toMillis()); settings.addAdditionalResolutionSettings(resolution, new ResolutionSettings(resolutionFlushInterval, resolutionEnabled)); @@ -132,17 +108,16 @@ private static String property(String propertyValue, String collectionName) { return propertyValue.replaceAll("\\{collectionName\\}", collectionName); } - private static void validateMainResolutionParam(long resolution) { - double msInMinute = TimeUnit.MINUTES.toMillis(1); - if (msInMinute % resolution != 0) { - throw new IllegalArgumentException("Invalid interval: " + resolution + " seconds. The interval must be a divisor of one minute (60 seconds)."); - } - } - public static TimeSeriesCollectionsSettings buildSingleResolutionSettings(long mainResolution, long mainFlushInterval) { + /** + * this method is used to generate settings for a timeSeries with a single resolution + * @param resolution the resolution to use + * @param flushInterval the custom flush interval as setting + * @return the TimeSeriesCollectionsSettings to create the time series + */ + public static TimeSeriesCollectionsSettings buildSingleResolutionSettings(Resolution resolution, long flushInterval) { TimeSeriesCollectionsSettings timeSeriesCollectionsSettings = new TimeSeriesCollectionsSettings(); - timeSeriesCollectionsSettings.setMainResolution(mainResolution); - timeSeriesCollectionsSettings.setMainFlushInterval(mainFlushInterval); + timeSeriesCollectionsSettings.addAdditionalResolutionSettings(resolution, new ResolutionSettings(flushInterval, true)); return timeSeriesCollectionsSettings; } diff --git a/step-plans/step-plans-core/src/test/java/step/core/timeseries/ReportNodeTimeSeriesTest.java b/step-plans/step-plans-core/src/test/java/step/core/timeseries/ReportNodeTimeSeriesTest.java index 673442d8c..556433449 100644 --- a/step-plans/step-plans-core/src/test/java/step/core/timeseries/ReportNodeTimeSeriesTest.java +++ b/step-plans/step-plans-core/src/test/java/step/core/timeseries/ReportNodeTimeSeriesTest.java @@ -31,6 +31,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class ReportNodeTimeSeriesTest { @@ -78,6 +79,14 @@ public void reportNodeTimeSeriesDisabled() { TimeSeries timeSeries = reportNodeTimeSeries.getTimeSeries(); List collections = timeSeries.getCollections(); assertEquals(1, collections.size()); + + configuration.putProperty("reportNodeTimeSeries.collections.5_seconds.enabled", "false"); + try { + new ReportNodeTimeSeries(collectionFactory, configuration); + fail("Disabling all resolutions is not allowed"); + } catch (Throwable e) { + assertEquals("At least one time series collection must be registered.", e.getMessage()); + } } @Test @@ -89,10 +98,10 @@ public void reportNodeTimeSeriesFlushPeriods() { configuration.putProperty("reportNodeTimeSeries.collections.week.flush.period", "14"); TimeSeriesCollectionsSettings settings = TimeSeriesCollectionsSettings.readSettings(configuration, "reportNodeTimeSeries"); - assertEquals(11, settings.getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution.ONE_MINUTE).flushInterval); - assertEquals(12, settings.getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution.ONE_HOUR).flushInterval); - assertEquals(13, settings.getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution.ONE_DAY).flushInterval); - assertEquals(14, settings.getResolutionSettings(TimeSeriesCollectionsBuilder.Resolution.ONE_WEEK).flushInterval); + assertEquals(11, settings.getResolutionSettings(Resolution.ONE_MINUTE).flushInterval); + assertEquals(12, settings.getResolutionSettings(Resolution.ONE_HOUR).flushInterval); + assertEquals(13, settings.getResolutionSettings(Resolution.ONE_DAY).flushInterval); + assertEquals(14, settings.getResolutionSettings(Resolution.ONE_WEEK).flushInterval); } } From 8b99727f47225686888073f170b7c10ad739d172 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Mon, 8 Dec 2025 10:25:29 +0100 Subject: [PATCH 7/9] SED-4415 bug fixing --- .../AggregatedReportViewBuilder.java | 23 +++++++-------- .../aggregated/ReportNodeTimeSeries.java | 28 +++++++++++++++++++ .../TimeSeriesCollectionsBuilder.java | 12 ++++++++ .../TimeSeriesCollectionsSettings.java | 26 ++++------------- 4 files changed, 57 insertions(+), 32 deletions(-) diff --git a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java index c22360ec6..0ec5e2566 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java @@ -130,7 +130,7 @@ public AggregatedReport buildAggregatedReport(AggregatedReportViewRequest reques return new AggregatedReport(recursivelyBuildAggregatedReportTree(rootResolvedPlanNode, request, countByHashAndStatus, countByHashAndErrorMessage, mainReportNodeAccessor, null, runningCountByArtefactHash, operationsByArtefactHash, resolvedRange)); } else { // a node is selected to generate a partial aggregated report - try (ReportNodeTimeSeries partialReportNodesTimeSeries = getInMemoryReportNodeTimeSeries()) { + try (ReportNodeTimeSeries partialReportNodesTimeSeries = getPartialTreeReportNodeTimeSeries()) { InMemoryReportNodeAccessor inMemoryReportNodeAccessor = new InMemoryReportNodeAccessor(); AggregatedReport aggregatedReport = new AggregatedReport(); //We now (SED-3882) also wand to get the count for RUNNING artefacts which can only be retrieved from report nodes RAW data @@ -140,9 +140,17 @@ public AggregatedReport buildAggregatedReport(AggregatedReportViewRequest reques Set reportArtefactHashSet = buildPartialReportNodeTimeSeries(aggregatedReport, request.selectedReportNodeId, partialReportNodesTimeSeries, inMemoryReportNodeAccessor, runningCountByArtefactHash, operationsByArtefactHash, request.fetchCurrentOperations); // Only pass the reportArtefactHashSet if aggregate view filtering is enabled reportArtefactHashSet = (request.filterResolvedPlanNodes) ? reportArtefactHashSet : null; - //Aggregate time series data for the given execution reporting context grouped by artefactHash - Map> countByHashAndStatus = partialReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, STATUS); - Map> countByHashAndErrorMessage = mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, ERROR_MESSAGE); + //Aggregate time series data for the given execution reporting context grouped by artefactHash and status ans artefactHash and error messages + CompletableFuture>> countByHashAndStatusFuture = + CompletableFuture.supplyAsync(() -> + partialReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, STATUS) + ); + CompletableFuture>> countByHashAndErrorMessageFuture = + CompletableFuture.supplyAsync(() -> + mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, ERROR_MESSAGE) + ); + Map> countByHashAndStatus = countByHashAndStatusFuture.join(); + Map> countByHashAndErrorMessage = countByHashAndErrorMessageFuture.join(); //Because the time series time range extends to the time series resolution we need to use the same range when querying the report nodes Range resolvedRange = getResolvedRange(request, countByHashAndStatus); aggregatedReport.aggregatedReportView = recursivelyBuildAggregatedReportTree(rootResolvedPlanNode, request, countByHashAndStatus, countByHashAndErrorMessage, inMemoryReportNodeAccessor, reportArtefactHashSet, runningCountByArtefactHash, operationsByArtefactHash, resolvedRange); @@ -163,13 +171,6 @@ private ReportNodeTimeSeries.Range getResolvedRange(AggregatedReportViewRequest return result; } - private ReportNodeTimeSeries getInMemoryReportNodeTimeSeries() { - //Need to create a configuration with all time series details - return new ReportNodeTimeSeries(new InMemoryCollectionFactory(new Properties()), - // to build the report we only need a single time bucket and can flush only once all reports are ingested - TimeSeriesCollectionsSettings.buildSingleResolutionSettings(Resolution.ONE_WEEK, 0), true); - } - /** * Populate an in memory timeseries and report node accessor with all data required to build a partial aggregated report tree * This aggregated tree will be filtered for the execution path of this single report node. If available we filter on the wrapping (nested) iteration diff --git a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java index 11358d3ee..fb7578815 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java +++ b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java @@ -4,6 +4,7 @@ import step.core.artefacts.reports.ReportNode; import step.core.artefacts.reports.ReportNodeStatus; import step.core.collections.*; +import step.core.collections.inmemory.InMemoryCollectionFactory; import step.core.timeseries.*; import step.core.timeseries.aggregation.TimeSeriesAggregationQueryBuilder; import step.core.timeseries.aggregation.TimeSeriesOptimizationType; @@ -11,8 +12,10 @@ import step.core.timeseries.bucket.BucketAttributes; import step.core.timeseries.ingestion.TimeSeriesIngestionPipeline; +import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; @@ -48,6 +51,31 @@ private List getTimeSeriesCollections(TimeSeriesCollection return timeSeriesCollectionsBuilder.getTimeSeriesCollections(TIME_SERIES_MAIN_COLLECTION, collectionsSettings, Set.of(EXECUTION_ID)); } + /** + * Create an in memory report tree time-series, with a single resolution + * The resolution is coarse to ingest only one time bucket + * The flush interval is 0 as we only need to flush once after the ingestion + */ + protected ReportNodeTimeSeries(Duration resolution, long flushInterval) { + InMemoryCollectionFactory inMemoryCollectionFactory = new InMemoryCollectionFactory(new Properties()); + TimeSeriesCollectionsBuilder timeSeriesCollectionsBuilder = new TimeSeriesCollectionsBuilder(inMemoryCollectionFactory); + List timeSeriesCollections = timeSeriesCollectionsBuilder.getSingleTimeSeriesCollections(TIME_SERIES_MAIN_COLLECTION, new TimeSeriesCollectionsSettings(), resolution, flushInterval); + timeSeries = new TimeSeriesBuilder().registerCollections(timeSeriesCollections).build(); + ingestionPipeline = timeSeries.getIngestionPipeline(); + timeSeries.createIndexes(Set.of(new IndexField(EXECUTION_ID, Order.ASC, String.class))); + this.ingestionEnabled = true; + } + + /** + * Get an in memory report tree time-series, with a single resolution + * The resolution is coarse to ingest only one time bucket + * The flush interval is 0 as we only need to flush once after the ingestion + * @return the in memory report node time-series + */ + public static ReportNodeTimeSeries getPartialTreeReportNodeTimeSeries() { + return new ReportNodeTimeSeries(Duration.ofMillis(Long.MAX_VALUE), 0); + } + public TimeSeries getTimeSeries() { return timeSeries; } diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java index 6ca35521a..e54ea1e59 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java @@ -54,6 +54,18 @@ public List getTimeSeriesCollections(String mainCollection return enabledCollections; } + public List getSingleTimeSeriesCollections(String mainCollectionName, TimeSeriesCollectionsSettings collectionsSettings, Duration resolution, Long flushInterval) { + List enabledCollections = new ArrayList<>(); + int flushSeriesQueueSize = collectionsSettings.getFlushSeriesQueueSize(); + int flushAsyncQueueSize = collectionsSettings.getFlushAsyncQueueSize(); + addIfEnabled(enabledCollections, mainCollectionName, + resolution, flushInterval, flushSeriesQueueSize, flushAsyncQueueSize, + null, true); + + + return enabledCollections; + } + private void addIfEnabled(List enabledCollections, String collectionName, Duration resolution, long flushInterval, int flushSeriesQueueSizeThreshold, int flushAsyncQueueSize, Set ignoredAttributes, boolean enabled) { TimeSeriesCollectionSettings settings = new TimeSeriesCollectionSettings() .setResolution(resolution.toMillis()) diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java index 2103da77b..652279237 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java @@ -21,10 +21,8 @@ import ch.exense.commons.app.Configuration; -import java.time.Duration; import java.util.Map; import java.util.TreeMap; -import java.util.concurrent.TimeUnit; public class TimeSeriesCollectionsSettings { @@ -40,7 +38,7 @@ public class TimeSeriesCollectionsSettings { //While the queue is full, ingesting new buckets is blocked private int flushAsyncQueueSize; //Define the Map of supported Resolutions associated to their settings read from the configuration. - private final Map additionalResolutionSettings = new TreeMap<>(); + private final Map resolutionSettings = new TreeMap<>(); public static class ResolutionSettings { //Define the interval of the flushing job for the ingestion pipeline @@ -72,11 +70,11 @@ public int getFlushAsyncQueueSize() { return flushAsyncQueueSize; } - private void addAdditionalResolutionSettings(Resolution resolution, ResolutionSettings resolutionSettings) { - additionalResolutionSettings.put(resolution, resolutionSettings); + private void addResolutionSettings(Resolution resolution, ResolutionSettings resolutionSettings) { + this.resolutionSettings.put(resolution, resolutionSettings); } public ResolutionSettings getResolutionSettings(Resolution resolution) { - return additionalResolutionSettings.get(resolution); + return resolutionSettings.get(resolution); } public static TimeSeriesCollectionsSettings readSettings(Configuration configuration, String collectionName) { @@ -87,7 +85,7 @@ public static TimeSeriesCollectionsSettings readSettings(Configuration configura for (Resolution resolution: Resolution.values()) { boolean resolutionEnabled = getPropertyAsBoolean(configuration, RESOLUTION_PROPERTY_PREFIX + resolution.name + TIME_SERIES_RESOLUTION_ENABLED_SUFFIX, collectionName, true); Long resolutionFlushInterval = getPropertyAsLong(configuration, RESOLUTION_PROPERTY_PREFIX + resolution.name + TIME_SERIES_RESOLUTION_FLUSH_PERIOD_SUFFIX, collectionName, resolution.defaultFlushPeriod.toMillis()); - settings.addAdditionalResolutionSettings(resolution, new ResolutionSettings(resolutionFlushInterval, resolutionEnabled)); + settings.addResolutionSettings(resolution, new ResolutionSettings(resolutionFlushInterval, resolutionEnabled)); } return settings; } @@ -107,18 +105,4 @@ private static boolean getPropertyAsBoolean(Configuration configuration, String private static String property(String propertyValue, String collectionName) { return propertyValue.replaceAll("\\{collectionName\\}", collectionName); } - - - /** - * this method is used to generate settings for a timeSeries with a single resolution - * @param resolution the resolution to use - * @param flushInterval the custom flush interval as setting - * @return the TimeSeriesCollectionsSettings to create the time series - */ - public static TimeSeriesCollectionsSettings buildSingleResolutionSettings(Resolution resolution, long flushInterval) { - TimeSeriesCollectionsSettings timeSeriesCollectionsSettings = new TimeSeriesCollectionsSettings(); - timeSeriesCollectionsSettings.addAdditionalResolutionSettings(resolution, new ResolutionSettings(flushInterval, true)); - return timeSeriesCollectionsSettings; - } - } From 3074e70ba23a9457ee49c4d204e57e9aa72b31ef Mon Sep 17 00:00:00 2001 From: David Stephan Date: Tue, 9 Dec 2025 11:06:06 +0100 Subject: [PATCH 8/9] SED-4415 PR feedbacks --- pom.xml | 2 +- .../V29_1_UpdateTimeSeriesCollectionsAndSettings.java | 7 ++++--- .../core/timeseries/TimeSeriesCollectionsBuilder.java | 8 ++++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 1ac34a9ba..ebebed2f0 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2025.12.5-6932fb5414e3a93583594624 + 2025.12.9-6937f222a52e5331629ad7de 3.0.23 diff --git a/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_1_UpdateTimeSeriesCollectionsAndSettings.java b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_1_UpdateTimeSeriesCollectionsAndSettings.java index aebe87e11..207f56ab1 100644 --- a/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_1_UpdateTimeSeriesCollectionsAndSettings.java +++ b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_1_UpdateTimeSeriesCollectionsAndSettings.java @@ -65,12 +65,12 @@ public V29_1_UpdateTimeSeriesCollectionsAndSettings(CollectionFactory collection @Override public void runUpgradeScript() { - log.info("Renaming time-series 'main' collections to match their resolutions"); - + log.info("Renaming the 'main' collection of the response times time-series to include its resolution"); timeseriesCollection.rename(TIME_SERIES_MAIN_COLLECTION_NEW_NAME); + log.info("Renaming the 'main' collection of the report nodes time-series to include its resolution"); reportNodeTimeseriesCollection.rename(TIME_SERIES_MAIN_COLLECTION_REPORTS_NEW_NAME); - log.info("Time-series 'main' collections renamed to match their resolutions"); + log.info("Renaming time-series housekeeping setting keys"); //use names from enum which will then be aligned with the collection names @@ -87,6 +87,7 @@ private void updateSettingKeyIfPresent(String oldKey, String newKey) { setting.ifPresent(s -> { s.put("key", newKey); settings.save(s); + logger.info("Time-series housekeeping setting key {} renamed to {}", oldKey, newKey); }); } diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java index e54ea1e59..601a50a29 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java @@ -19,6 +19,8 @@ package step.core.timeseries; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import step.core.collections.CollectionFactory; import step.core.timeseries.bucket.Bucket; @@ -29,6 +31,8 @@ public class TimeSeriesCollectionsBuilder { + private static final Logger logger = LoggerFactory.getLogger(TimeSeriesCollectionsBuilder.class); + public static final String COLLECTION_NAME_SEPARATOR = "_"; private final CollectionFactory collectionFactory; @@ -77,8 +81,8 @@ private void addIfEnabled(List enabledCollections, String if (enabled) { enabledCollections.add(collection); } else { - // disabled resolutions will be completely dropped from db - collection.drop(); + // disabled resolutions are not dropped automatically + logger.warn("The time-series resolution with name '{}' is disabled. To reclaim space you can delete the corresponding DB table.", collectionName); } } } From 993b011bfb51c2ee517146f501aff6f6183c3978 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Wed, 10 Dec 2025 12:33:24 +0100 Subject: [PATCH 9/9] SED-4415 bumping framework --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ebebed2f0..e78b273dc 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2025.12.9-6937f222a52e5331629ad7de + 2025.12.10-693957f1a52e5331621ac16a 3.0.23