diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java index 384aded25b59..0fabba6df3bf 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java @@ -413,6 +413,8 @@ private static void addDeprecatedKeys() { HddsConfigKeys.HDDS_METRICS_PERCENTILES_INTERVALS_KEY), new DeprecationDelta("hdds.recon.heartbeat.interval", HddsConfigKeys.HDDS_RECON_HEARTBEAT_INTERVAL), + new DeprecationDelta("hdds.tracing.enabled", + "ozone.tracing.enabled"), }); } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 69a4d0411f2b..2c44fa881c41 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -469,9 +469,6 @@ public final class ScmConfigKeys { public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_DEFAULT = "network-topology-default.xml"; - public static final String HDDS_TRACING_ENABLED = "hdds.tracing.enabled"; - public static final boolean HDDS_TRACING_ENABLED_DEFAULT = false; - public static final String OZONE_SCM_RATIS_PORT_KEY = "ozone.scm.ratis.port"; public static final int OZONE_SCM_RATIS_PORT_DEFAULT diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/SpanSampler.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/SpanSampler.java index 317573a5e19f..8f40ba5b5ed1 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/SpanSampler.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/SpanSampler.java @@ -59,7 +59,7 @@ public SamplingResult shouldSample( // check if we have a valid parent span if (!parentSpan.getSpanContext().isValid()) { // Root span: always delegate to trace-level sampler - // This ensures OTEL_TRACES_SAMPLER_ARG is respected + // This ensures ozone.tracing.sampler is respected return rootSampler.shouldSample(parentContext, traceId, spanName, spanKind, attributes, parentLinks); } diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java new file mode 100644 index 000000000000..451c80b5e7fa --- /dev/null +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hdds.tracing; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hdds.conf.Config; +import org.apache.hadoop.hdds.conf.ConfigGroup; +import org.apache.hadoop.hdds.conf.ConfigTag; +import org.apache.hadoop.hdds.conf.ConfigType; +import org.apache.hadoop.hdds.conf.PostConstruct; +import org.apache.hadoop.hdds.conf.ReconfigurableConfig; + +/** + * OpenTelemetry tracing configuration for Ozone services. + * Priority is given as such: + * 1. Explicit configuration keys + * 2. Environment variables + * 3. Default values defined in this class + */ +@ConfigGroup(prefix = "ozone.tracing") +public class TracingConfig extends ReconfigurableConfig { + + private static final String OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT"; + private static final String OTEL_TRACES_SAMPLER_ARG = "OTEL_TRACES_SAMPLER_ARG"; + private static final String OTEL_SPAN_SAMPLING_ARG = "OTEL_SPAN_SAMPLING_ARG"; + + @Config( + key = "ozone.tracing.enabled", + defaultValue = "false", + type = ConfigType.BOOLEAN, + tags = { ConfigTag.OZONE, ConfigTag.HDDS }, + description = "If true, tracing is initialized and spans may be exported (subject to sampling)." + ) + private boolean tracingEnabled; + + @Config( + key = "ozone.tracing.endpoint", + defaultValue = "", + type = ConfigType.STRING, + tags = { ConfigTag.OZONE, ConfigTag.HDDS }, + description = "OTLP gRPC receiver endpoint URL." + ) + private String tracingEndpoint; + + @Config( + key = "ozone.tracing.sampler", + defaultValue = "-1", + type = ConfigType.DOUBLE, + tags = { ConfigTag.OZONE, ConfigTag.HDDS }, + description = "Root trace sampling ratio (0.0 to 1.0)." + ) + private double traceSamplerRatio; + + @Config( + key = "ozone.tracing.span.sampling", + defaultValue = "", + type = ConfigType.STRING, + tags = { ConfigTag.OZONE, ConfigTag.HDDS }, + description = "Optional per-span sampling: comma-separated spanName:rate entries." + ) + private String spanSampling; + + public boolean isTracingEnabled() { + return tracingEnabled; + } + + @PostConstruct + public void validate() { + if (tracingEndpoint.isEmpty()) { + tracingEndpoint = System.getenv(OTEL_EXPORTER_OTLP_ENDPOINT); + } + if (StringUtils.isBlank(tracingEndpoint)) { + tracingEndpoint = "http://localhost:4317"; + } + + if (traceSamplerRatio < 0) { + String envTraceRatio = System.getenv(OTEL_TRACES_SAMPLER_ARG); + if (envTraceRatio != null) { + try { + traceSamplerRatio = Double.parseDouble(envTraceRatio.trim()); + } catch (NumberFormatException ignored) { + } + } + } + if (traceSamplerRatio < 0 || traceSamplerRatio > 1) { + traceSamplerRatio = 1.0; + } + + if (spanSampling.isEmpty()) { + String envSampling = System.getenv(OTEL_SPAN_SAMPLING_ARG); + if (!StringUtils.isBlank(envSampling)) { + spanSampling = envSampling; + } + } + } + + public String getTracingEndpoint() { + return tracingEndpoint; + } + + public double getTraceSamplerRatio() { + return traceSamplerRatio; + } + + public String getSpanSampling() { + return spanSampling; + } + +} diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java index 0f8aa4561f12..7fc421c80a9b 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java @@ -37,7 +37,6 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.hdds.conf.ConfigurationSource; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.ratis.util.function.CheckedRunnable; import org.apache.ratis.util.function.CheckedSupplier; import org.slf4j.Logger; @@ -49,12 +48,6 @@ public final class TracingUtil { private static final Logger LOG = LoggerFactory.getLogger(TracingUtil.class); private static final String NULL_SPAN_AS_STRING = ""; - private static final String OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT"; - private static final String OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT = "http://localhost:4317"; - private static final String OTEL_TRACES_SAMPLER_ARG = "OTEL_TRACES_SAMPLER_ARG"; - private static final double OTEL_TRACES_SAMPLER_RATIO_DEFAULT = 1.0; - private static final String OTEL_SPAN_SAMPLING_ARG = "OTEL_SPAN_SAMPLING_ARG"; - private static final String OTEL_TRACES_SAMPLER_CONFIG_DEFAULT = ""; private static volatile boolean isInit = false; private static Tracer tracer = OpenTelemetry.noop().getTracer("noop"); @@ -72,7 +65,7 @@ public static void initTracing( } try { - initialize(serviceName); + initialize(serviceName, conf); isInit = true; LOG.info("Initialized tracing service: {}", serviceName); } catch (Exception e) { @@ -80,38 +73,15 @@ public static void initTracing( } } - private static void initialize(String serviceName) { - String otelEndPoint = System.getenv(OTEL_EXPORTER_OTLP_ENDPOINT); - if (otelEndPoint == null || otelEndPoint.isEmpty()) { - otelEndPoint = OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT; - } - - double samplerRatio = OTEL_TRACES_SAMPLER_RATIO_DEFAULT; - try { - String sampleStrRatio = System.getenv(OTEL_TRACES_SAMPLER_ARG); - if (sampleStrRatio != null && !sampleStrRatio.isEmpty()) { - samplerRatio = Double.parseDouble(System.getenv(OTEL_TRACES_SAMPLER_ARG)); - LOG.info("Sampling Trace Config = '{}'", samplerRatio); - } - } catch (NumberFormatException ex) { - // log and use the default value. - LOG.warn("Invalid value for {}: '{}'. Falling back to default: {}", - OTEL_TRACES_SAMPLER_ARG, System.getenv(OTEL_TRACES_SAMPLER_ARG), OTEL_TRACES_SAMPLER_RATIO_DEFAULT, ex); - } + private static void initialize(String serviceName, ConfigurationSource conf) { + //Fetch and log the right tracing parameters based on config, environment variable and default value priority. + TracingConfig tracingConfig = conf.getObject(TracingConfig.class); + String otelEndPoint = tracingConfig.getTracingEndpoint(); + double samplerRatio = tracingConfig.getTraceSamplerRatio(); + LOG.info("Sampling Trace Config = '{}'", samplerRatio); + String spanSamplingConfig = tracingConfig.getSpanSampling(); + LOG.info("Sampling Span Config = '{}'", spanSamplingConfig); - String spanSamplingConfig = OTEL_TRACES_SAMPLER_CONFIG_DEFAULT; - try { - String spanStrConfig = System.getenv(OTEL_SPAN_SAMPLING_ARG); - if (spanStrConfig != null && !spanStrConfig.isEmpty()) { - spanSamplingConfig = spanStrConfig; - } - LOG.info("Sampling Span Config = '{}'", spanSamplingConfig); - } catch (Exception ex) { - // Log and use the default value. - LOG.warn("Failed to process {}. Falling back to default configuration: {}", - OTEL_SPAN_SAMPLING_ARG, OTEL_TRACES_SAMPLER_CONFIG_DEFAULT, ex); - } - // Pass the config to parseSpanSamplingConfig to get spans to be sampled. Map spanMap = parseSpanSamplingConfig(spanSamplingConfig); Resource resource = Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), serviceName)); @@ -201,11 +171,8 @@ public static T createProxy( new TraceAllMethod<>(delegate, itf.getSimpleName()))); } - public static boolean isTracingEnabled( - ConfigurationSource conf) { - return conf.getBoolean( - ScmConfigKeys.HDDS_TRACING_ENABLED, - ScmConfigKeys.HDDS_TRACING_ENABLED_DEFAULT); + public static boolean isTracingEnabled(ConfigurationSource conf) { + return conf.getObject(TracingConfig.class).isTracingEnabled(); } /** diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index 6c4bd26ebe88..3ef2f2914e98 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -3573,14 +3573,6 @@ If enabled, SCM DB Snapshot is taken by Recon. - - hdds.tracing.enabled - false - OZONE, HDDS - - If enabled, tracing information is sent to tracing server. - - ozone.recon.task.thread.count 1 diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java index 7b73b30ab7f1..d0e58d76665c 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java @@ -28,7 +28,6 @@ import java.io.IOException; import org.apache.hadoop.hdds.conf.InMemoryConfigurationForTesting; import org.apache.hadoop.hdds.conf.MutableConfigurationSource; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.tracing.TestTraceAllMethod.Service; import org.apache.hadoop.hdds.tracing.TestTraceAllMethod.ServiceImpl; import org.junit.jupiter.api.Test; @@ -58,7 +57,7 @@ public void testInitTracing() { private static MutableConfigurationSource tracingEnabled() { MutableConfigurationSource config = new InMemoryConfigurationForTesting(); - config.setBoolean(ScmConfigKeys.HDDS_TRACING_ENABLED, true); + config.setBoolean("ozone.tracing.enabled", true); return config; }