HDDS-14961. Introduce configuration for tracing

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/conf/OzoneConfiguration.java

@@ -413,6 +413,8 @@ private static void addDeprecatedKeys() {
             HddsConfigKeys.HDDS_METRICS_PERCENTILES_INTERVALS_KEY),
         new DeprecationDelta("hdds.recon.heartbeat.interval",
             HddsConfigKeys.HDDS_RECON_HEARTBEAT_INTERVAL),
+        new DeprecationDelta("hdds.tracing.enabled",
+            "ozone.tracing.enabled"),
     });
   }
 

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java

@@ -469,9 +469,6 @@ public final class ScmConfigKeys {
   public static final String OZONE_SCM_NETWORK_TOPOLOGY_SCHEMA_FILE_DEFAULT =
       "network-topology-default.xml";
 
-  public static final String HDDS_TRACING_ENABLED = "hdds.tracing.enabled";
-  public static final boolean HDDS_TRACING_ENABLED_DEFAULT = false;
-
   public static final String OZONE_SCM_RATIS_PORT_KEY
       = "ozone.scm.ratis.port";
   public static final int OZONE_SCM_RATIS_PORT_DEFAULT

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/SpanSampler.java

@@ -59,7 +59,7 @@ public SamplingResult shouldSample(
     // check if we have a valid parent span
     if (!parentSpan.getSpanContext().isValid()) {
       // Root span: always delegate to trace-level sampler
-      // This ensures OTEL_TRACES_SAMPLER_ARG is respected
+      // This ensures ozone.tracing.sampler is respected
       return rootSampler.shouldSample(parentContext, traceId, spanName,
           spanKind, attributes, parentLinks);
     }

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingConfig.java

@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.tracing;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigTag;
+import org.apache.hadoop.hdds.conf.ConfigType;
+import org.apache.hadoop.hdds.conf.PostConstruct;
+import org.apache.hadoop.hdds.conf.ReconfigurableConfig;
+
+/**
+ * OpenTelemetry tracing configuration for Ozone services.
+ * Priority is given as such:
+ * 1. Explicit configuration keys
+ * 2. Environment variables
+ * 3. Default values defined in this class
+ */
+@ConfigGroup(prefix = "ozone.tracing")
+public class TracingConfig extends ReconfigurableConfig {
+
+  private static final String OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT";
+  private static final String OTEL_TRACES_SAMPLER_ARG = "OTEL_TRACES_SAMPLER_ARG";
+  private static final String OTEL_SPAN_SAMPLING_ARG = "OTEL_SPAN_SAMPLING_ARG";
+
+  @Config(
+      key = "ozone.tracing.enabled",
+      defaultValue = "false",
+      type = ConfigType.BOOLEAN,
+      tags = { ConfigTag.OZONE, ConfigTag.HDDS },
+      description = "If true, tracing is initialized and spans may be exported (subject to sampling)."
+  )
+  private boolean tracingEnabled;
+
+  @Config(
+      key = "ozone.tracing.endpoint",
+      defaultValue = "",
+      type = ConfigType.STRING,
+      tags = { ConfigTag.OZONE, ConfigTag.HDDS },
+      description = "OTLP gRPC receiver endpoint URL."
+  )
+  private String tracingEndpoint;
+
+  @Config(
+      key = "ozone.tracing.sampler",
+      defaultValue = "-1",
+      type = ConfigType.DOUBLE,
+      tags = { ConfigTag.OZONE, ConfigTag.HDDS },
+      description = "Root trace sampling ratio (0.0 to 1.0)."
+  )
+  private double traceSamplerRatio;
+
+  @Config(
+      key = "ozone.tracing.span.sampling",
+      defaultValue = "",
+      type = ConfigType.STRING,
+      tags = { ConfigTag.OZONE, ConfigTag.HDDS },
+      description = "Optional per-span sampling: comma-separated spanName:rate entries."
+  )
+  private String spanSampling;
+
+  public boolean isTracingEnabled() {
+    return tracingEnabled;
+  }
+
+  @PostConstruct
+  public void validate() {
+    if (tracingEndpoint.isEmpty()) {
+      tracingEndpoint = System.getenv(OTEL_EXPORTER_OTLP_ENDPOINT);
+    }
+    if (StringUtils.isBlank(tracingEndpoint)) {
+      tracingEndpoint = "http://localhost:4317";
+    }
+
+    if (traceSamplerRatio < 0) {
+      String envTraceRatio = System.getenv(OTEL_TRACES_SAMPLER_ARG);
+      if (envTraceRatio != null) {
+        try {
+          traceSamplerRatio = Double.parseDouble(envTraceRatio.trim());
+        } catch (NumberFormatException ignored) {
+        }
+      }
+    }
+    if (traceSamplerRatio < 0 || traceSamplerRatio > 1) {
+      traceSamplerRatio = 1.0;
+    }
+
+    if (spanSampling.isEmpty()) {
+      String envSampling = System.getenv(OTEL_SPAN_SAMPLING_ARG);
+      if (!StringUtils.isBlank(envSampling)) {
+        spanSampling = envSampling;
+      }
+    }
+  }
+
+  public String getTracingEndpoint() {
+    return tracingEndpoint;
+  }
+
+  public double getTraceSamplerRatio() {
+    return traceSamplerRatio;
+  }
+
+  public String getSpanSampling() {
+    return spanSampling;
+  }
+
+}

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/tracing/TracingUtil.java

@@ -37,7 +37,6 @@
 import java.util.HashMap;
 import java.util.Map;
 import org.apache.hadoop.hdds.conf.ConfigurationSource;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.ratis.util.function.CheckedRunnable;
 import org.apache.ratis.util.function.CheckedSupplier;
 import org.slf4j.Logger;
@@ -49,12 +48,6 @@
 public final class TracingUtil {
   private static final Logger LOG = LoggerFactory.getLogger(TracingUtil.class);
   private static final String NULL_SPAN_AS_STRING = "";
-  private static final String OTEL_EXPORTER_OTLP_ENDPOINT = "OTEL_EXPORTER_OTLP_ENDPOINT";
-  private static final String OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT = "http://localhost:4317";
-  private static final String OTEL_TRACES_SAMPLER_ARG = "OTEL_TRACES_SAMPLER_ARG";
-  private static final double OTEL_TRACES_SAMPLER_RATIO_DEFAULT = 1.0;
-  private static final String OTEL_SPAN_SAMPLING_ARG = "OTEL_SPAN_SAMPLING_ARG";
-  private static final String OTEL_TRACES_SAMPLER_CONFIG_DEFAULT = "";
 
   private static volatile boolean isInit = false;
   private static Tracer tracer = OpenTelemetry.noop().getTracer("noop");
@@ -72,46 +65,23 @@ public static void initTracing(
     }
 
     try {
-      initialize(serviceName);
+      initialize(serviceName, conf);
       isInit = true;
       LOG.info("Initialized tracing service: {}", serviceName);
     } catch (Exception e) {
       LOG.error("Failed to initialize tracing", e);
     }
   }
 
-  private static void initialize(String serviceName) {
-    String otelEndPoint = System.getenv(OTEL_EXPORTER_OTLP_ENDPOINT);
-    if (otelEndPoint == null || otelEndPoint.isEmpty()) {
-      otelEndPoint = OTEL_EXPORTER_OTLP_ENDPOINT_DEFAULT;
-    }
-
-    double samplerRatio = OTEL_TRACES_SAMPLER_RATIO_DEFAULT;
-    try {
-      String sampleStrRatio = System.getenv(OTEL_TRACES_SAMPLER_ARG);
-      if (sampleStrRatio != null && !sampleStrRatio.isEmpty()) {
-        samplerRatio = Double.parseDouble(System.getenv(OTEL_TRACES_SAMPLER_ARG));
-        LOG.info("Sampling Trace Config = '{}'", samplerRatio);
-      }
-    } catch (NumberFormatException ex) {
-      // log and use the default value.
-      LOG.warn("Invalid value for {}: '{}'. Falling back to default: {}",
-          OTEL_TRACES_SAMPLER_ARG, System.getenv(OTEL_TRACES_SAMPLER_ARG), OTEL_TRACES_SAMPLER_RATIO_DEFAULT, ex);
-    }
+  private static void initialize(String serviceName, ConfigurationSource conf) {
+    //Fetch and log the right tracing parameters based on config, environment variable and default value priority.
+    TracingConfig tracingConfig = conf.getObject(TracingConfig.class);
+    String otelEndPoint = tracingConfig.getTracingEndpoint();
+    double samplerRatio = tracingConfig.getTraceSamplerRatio();
+    LOG.info("Sampling Trace Config = '{}'", samplerRatio);
+    String spanSamplingConfig = tracingConfig.getSpanSampling();
+    LOG.info("Sampling Span Config = '{}'", spanSamplingConfig);
 
-    String spanSamplingConfig = OTEL_TRACES_SAMPLER_CONFIG_DEFAULT;
-    try {
-      String spanStrConfig = System.getenv(OTEL_SPAN_SAMPLING_ARG);
-      if (spanStrConfig != null && !spanStrConfig.isEmpty()) {
-        spanSamplingConfig = spanStrConfig;
-      }
-      LOG.info("Sampling Span Config = '{}'", spanSamplingConfig);
-    } catch (Exception ex) {
-      // Log and use the default value.
-      LOG.warn("Failed to process {}. Falling back to default configuration: {}",
-          OTEL_SPAN_SAMPLING_ARG, OTEL_TRACES_SAMPLER_CONFIG_DEFAULT, ex);
-    }
-    // Pass the config to parseSpanSamplingConfig to get spans to be sampled.
     Map<String, LoopSampler> spanMap = parseSpanSamplingConfig(spanSamplingConfig);
 
     Resource resource = Resource.create(Attributes.of(AttributeKey.stringKey("service.name"), serviceName));
@@ -201,11 +171,8 @@ public static <T> T createProxy(
         new TraceAllMethod<>(delegate, itf.getSimpleName())));
   }
 
-  public static boolean isTracingEnabled(
-      ConfigurationSource conf) {
-    return conf.getBoolean(
-        ScmConfigKeys.HDDS_TRACING_ENABLED,
-        ScmConfigKeys.HDDS_TRACING_ENABLED_DEFAULT);
+  public static boolean isTracingEnabled(ConfigurationSource conf) {
+    return conf.getObject(TracingConfig.class).isTracingEnabled();
   }
 
   /**

hadoop-hdds/common/src/main/resources/ozone-default.xml

@@ -3573,14 +3573,6 @@
       If enabled, SCM DB Snapshot is taken by Recon.
     </description>
   </property>
-  <property>
-    <name>hdds.tracing.enabled</name>
-    <value>false</value>
-    <tag>OZONE, HDDS</tag>
-    <description>
-      If enabled, tracing information is sent to tracing server.
-    </description>
-  </property>
   <property>
     <name>ozone.recon.task.thread.count</name>
     <value>1</value>

hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/tracing/TestTracingUtil.java

@@ -28,7 +28,6 @@
 import java.io.IOException;
 import org.apache.hadoop.hdds.conf.InMemoryConfigurationForTesting;
 import org.apache.hadoop.hdds.conf.MutableConfigurationSource;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
 import org.apache.hadoop.hdds.tracing.TestTraceAllMethod.Service;
 import org.apache.hadoop.hdds.tracing.TestTraceAllMethod.ServiceImpl;
 import org.junit.jupiter.api.Test;
@@ -58,7 +57,7 @@ public void testInitTracing() {
 
   private static MutableConfigurationSource tracingEnabled() {
     MutableConfigurationSource config = new InMemoryConfigurationForTesting();
-    config.setBoolean(ScmConfigKeys.HDDS_TRACING_ENABLED, true);
+    config.setBoolean("ozone.tracing.enabled", true);
     return config;
   }