diff --git a/services/tables/src/main/resources/application.properties b/services/tables/src/main/resources/application.properties index a50b219d2..4a6c82a89 100644 --- a/services/tables/src/main/resources/application.properties +++ b/services/tables/src/main/resources/application.properties @@ -20,5 +20,7 @@ management.endpoint.prometheus.enabled=true management.endpoint.beans.enabled=true management.metrics.distribution.percentiles-histogram.all=true management.metrics.distribution.maximum-expected-value.catalog_metadata_retrieval_latency=600s +management.metrics.distribution.maximum-expected-value.catalog_metadata_update_latency=600s +management.metrics.distribution.maximum-expected-value.http.server.requests=600s server.shutdown=graceful spring.lifecycle.timeout-per-shutdown-phase=60s \ No newline at end of file diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/MetricsHistogramConfigurationTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/MetricsHistogramConfigurationTest.java index 90106ba67..0eb8790ca 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/MetricsHistogramConfigurationTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/e2e/h2/MetricsHistogramConfigurationTest.java @@ -53,6 +53,13 @@ public class MetricsHistogramConfigurationTest { "${management.metrics.distribution.maximum-expected-value.catalog_metadata_retrieval_latency:}") private String maxExpectedValueConfig; + @Value( + "${management.metrics.distribution.maximum-expected-value.catalog_metadata_update_latency:}") + private String maxExpectedValueUpdateConfig; + + @Value("${management.metrics.distribution.maximum-expected-value.http.server.requests:}") + private String maxExpectedValueHttpConfig; + @Value("${management.metrics.distribution.percentiles-histogram.all:false}") private boolean percentilesHistogramEnabled; @@ -80,6 +87,23 @@ void testMaxExpectedValueConfigurationIsSet() { "maximum-expected-value.catalog_metadata_retrieval_latency should be set to 600s"); } + @Test + void testMaxExpectedValueConfigurationIsSetForUpdateLatency() { + assertEquals( + "600s", + maxExpectedValueUpdateConfig, + "maximum-expected-value.catalog_metadata_update_latency should be set to 600s"); + } + + @Test + void testMaxExpectedValueConfigurationIsSetForHttpServerRequests() { + assertEquals( + "600s", + maxExpectedValueHttpConfig, + "maximum-expected-value.http.server.requests should be set to 600s so that " + + "http_server_requests_seconds histogram buckets do not saturate at the default 30s cap"); + } + /** Tests that percentiles histogram is enabled for all metrics. */ @Test void testPercentilesHistogramIsEnabled() { @@ -160,6 +184,60 @@ void testHistogramBucketsExtendTo600Seconds() { maxBucketSeconds, bucketList)); } + @Test + void testHistogramBucketsExtendTo600SecondsForUpdateLatency() { + assertTimerHistogramExtendsTo600s( + "catalog_metadata_update_latency", + "maximum-expected-value.catalog_metadata_update_latency=600s"); + } + + @Test + void testHistogramBucketsExtendTo600SecondsForHttpServerRequests() { + // Spring Boot's auto-instrumented timer for HTTP requests is registered as + // "http.server.requests" — it's exported to Prometheus as http_server_requests_seconds. + assertTimerHistogramExtendsTo600s( + "http.server.requests", "maximum-expected-value.http.server.requests=600s"); + } + + private void assertTimerHistogramExtendsTo600s(String timerName, String configDescription) { + Timer timer = meterRegistry.timer(timerName); + + timer.record(100, TimeUnit.MILLISECONDS); + timer.record(1, TimeUnit.SECONDS); + timer.record(60, TimeUnit.SECONDS); + timer.record(600, TimeUnit.SECONDS); + timer.record(700, TimeUnit.SECONDS); + + HistogramSnapshot snapshot = timer.takeSnapshot(); + CountAtBucket[] buckets = snapshot.histogramCounts(); + + assertTrue( + buckets.length > 0, + String.format("Histogram for %s should have bucket entries", timerName)); + + double maxBucketSeconds = + Arrays.stream(buckets) + .mapToDouble(b -> b.bucket(TimeUnit.SECONDS)) + .filter(b -> b != Double.POSITIVE_INFINITY) + .max() + .orElse(0); + + String bucketList = + Arrays.stream(buckets) + .map(b -> String.valueOf(b.bucket(TimeUnit.SECONDS))) + .reduce((a, b) -> a + ", " + b) + .orElse("none"); + + assertTrue( + maxBucketSeconds >= 600.0, + String.format( + "Histogram for %s should have buckets extending to at least 600s. " + + "This validates the %s configuration. " + + "Without this override the default 30s cap would saturate p99/max around 31s. " + + "Found max bucket: %.1fs, all buckets: [%s]", + timerName, configDescription, maxBucketSeconds, bucketList)); + } + /** * Tests that the 600s configuration value can be parsed as a Duration. This validates the format * used in application.properties is correct.