From b91a3711749fbe9dbf8924dbd29e97f687e0c75d Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Wed, 15 Oct 2025 12:46:09 -0600 Subject: [PATCH 01/11] Add dynamic tags support to JMX Fetch --- DOCKER_USAGE_GUIDE.md | 215 +++++++++++++ DYNAMIC_TAGS_FEATURE.md | 174 ++++++++++ kafka_dynamic_tags_example.yaml | 42 +++ .../java/org/datadog/jmxfetch/DynamicTag.java | 227 +++++++++++++ .../java/org/datadog/jmxfetch/Instance.java | 53 +++- .../datadog/jmxfetch/DynamicTagTestApp.java | 49 +++ .../jmxfetch/DynamicTagTestAppMBean.java | 13 + .../org/datadog/jmxfetch/TestDynamicTags.java | 297 ++++++++++++++++++ src/test/resources/jmx_dynamic_tags.yaml | 19 ++ .../resources/jmx_dynamic_tags_map_style.yaml | 19 ++ .../jmx_dynamic_tags_nonexistent.yaml | 18 ++ ...mx_dynamic_tags_with_attribute_prefix.yaml | 18 ++ 12 files changed, 1143 insertions(+), 1 deletion(-) create mode 100644 DOCKER_USAGE_GUIDE.md create mode 100644 DYNAMIC_TAGS_FEATURE.md create mode 100644 kafka_dynamic_tags_example.yaml create mode 100644 src/main/java/org/datadog/jmxfetch/DynamicTag.java create mode 100644 src/test/java/org/datadog/jmxfetch/DynamicTagTestApp.java create mode 100644 src/test/java/org/datadog/jmxfetch/DynamicTagTestAppMBean.java create mode 100644 src/test/java/org/datadog/jmxfetch/TestDynamicTags.java create mode 100644 src/test/resources/jmx_dynamic_tags.yaml create mode 100644 src/test/resources/jmx_dynamic_tags_map_style.yaml create mode 100644 src/test/resources/jmx_dynamic_tags_nonexistent.yaml create mode 100644 src/test/resources/jmx_dynamic_tags_with_attribute_prefix.yaml diff --git a/DOCKER_USAGE_GUIDE.md b/DOCKER_USAGE_GUIDE.md new file mode 100644 index 00000000..e1ea6296 --- /dev/null +++ b/DOCKER_USAGE_GUIDE.md @@ -0,0 +1,215 @@ +# Using Custom JMXFetch with Docker Agent + +This guide shows how to use your custom JMXFetch build (with dynamic tags feature) in a Docker-based Datadog Agent **without rebuilding anything**. + +## Quick Start + +### Method 1: Volume Mount (Recommended) + +This is the easiest approach - just mount the custom JAR into your container. + +**1. Your custom JAR is here:** +```bash +/Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar +``` + +**2. Add a volume mount to your docker-compose.yaml or docker run command:** + +#### Docker Compose: +```yaml +version: '3' +services: + datadog: + image: gcr.io/datadoghq/agent:latest + environment: + - DD_API_KEY=your_api_key + - DD_SITE=datadoghq.com + # ... other env vars + volumes: + # Mount custom JMXFetch + - /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar:ro + + # Your other volumes + - /var/run/docker.sock:/var/run/docker.sock:ro + - ./conf.d:/etc/datadog-agent/conf.d:ro + # ... other volumes +``` + +#### Docker Run: +```bash +docker run -d \ + --name datadog-agent \ + -e DD_API_KEY=your_api_key \ + -e DD_SITE=datadoghq.com \ + -v /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar:ro \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ + -v /opt/datadog-agent/run:/opt/datadog-agent/run:rw \ + gcr.io/datadoghq/agent:latest +``` + +**3. Restart your agent:** +```bash +docker-compose restart datadog +# or +docker restart datadog-agent +``` + +**4. Verify it's working:** +```bash +# Check agent logs +docker logs datadog-agent | grep -i jmx + +# You should see logs about dynamic tags being resolved +docker logs datadog-agent | grep -i "Resolved dynamic tag" +``` + +--- + +### Method 2: Copy into Running Container + +If you already have a running container and don't want to restart with new volumes: + +```bash +# Copy the JAR into the running container +docker cp \ + /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar \ + datadog-agent:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar + +# Restart the agent process inside the container +docker exec datadog-agent agent restart +``` + +--- + +## Example Configuration + +Now you can use dynamic tags in your JMX configuration. Create or update your Kafka integration config: + +**File: `conf.d/kafka.yaml`** (or wherever your JMX configs are) + +```yaml +init_config: + is_jmx: true + +instances: + - host: kafka-broker + port: 9999 + tags: + - env:production + - service:kafka + # 🎉 NEW: Dynamic tags that pull values from JMX beans! + - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value + + conf: + - include: + domain: kafka.server + bean: kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec + attribute: + Count: + metric_type: rate + alias: kafka.messages_in +``` + +--- + +## Verification + +**1. Check JMXFetch version in logs:** +```bash +docker logs datadog-agent 2>&1 | grep -i "jmxfetch version" +``` + +**2. Check for dynamic tag resolution:** +```bash +docker logs datadog-agent 2>&1 | grep -i "Resolving.*dynamic tag" +``` + +You should see logs like: +``` +INFO: Resolving 1 dynamic tag(s) for instance kafka_instance +INFO: Resolved dynamic tag 'cluster_id' to value 'prod-cluster-123' from bean 'kafka.server:type=KafkaServer,name=ClusterId' attribute 'Value' +``` + +**3. Verify metrics have the dynamic tags:** +```bash +# In Datadog UI, check that your Kafka metrics now have the cluster_id tag +# with the actual value from your Kafka cluster +``` + +--- + +## Troubleshooting + +### JAR not found error +**Error:** `FileNotFoundException: jmxfetch.jar` + +**Solution:** The default JMXFetch location varies by agent version. Try these paths: +- `/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar` (newer versions) +- `/opt/datadog-agent/embedded/bin/jmxfetch.jar` (older versions) + +Find the correct path: +```bash +docker exec datadog-agent find /opt/datadog-agent -name "jmxfetch.jar" +``` + +### Dynamic tags not resolving +**Check:** +1. JMX connection is successful: `docker logs datadog-agent | grep "Connected to JMX"` +2. Bean name is correct: `docker exec datadog-agent agent jmx list matching` +3. Attribute name matches exactly (case-sensitive!) + +### Agent won't start +**Error:** Permission denied + +**Solution:** Make sure the JAR file has read permissions: +```bash +chmod 644 /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar +``` + +--- + +## Rolling Back + +To go back to the standard JMXFetch: + +**If using volume mount:** +1. Remove the volume mount from docker-compose.yaml +2. Restart: `docker-compose restart datadog` + +**If you copied the file:** +```bash +# The agent will automatically download the standard version on restart +docker restart datadog-agent +``` + +--- + +## Development Workflow + +When you make changes to JMXFetch: + +```bash +# 1. Rebuild the JAR +cd /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch +mvn clean package -DskipTests + +# 2. If using volume mount: just restart +docker-compose restart datadog + +# 3. If using copy method: copy and restart +docker cp target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar \ + datadog-agent:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar +docker exec datadog-agent agent restart +``` + +--- + +## Notes + +- ✅ **No rebuild needed** - Agent uses the JAR you provide +- ✅ **Hot reload** - Just restart the agent container +- ✅ **Preserves config** - All your existing configs work unchanged +- ⚠️ **Version mismatch** - If the agent version is very different from JMXFetch, you might see compatibility issues (rare) +- 💡 **Tip:** Use `:ro` (read-only) mount to prevent accidental modifications + + diff --git a/DYNAMIC_TAGS_FEATURE.md b/DYNAMIC_TAGS_FEATURE.md new file mode 100644 index 00000000..0a56f0fc --- /dev/null +++ b/DYNAMIC_TAGS_FEATURE.md @@ -0,0 +1,174 @@ +# Dynamic Tags Feature for JMXFetch + +## Overview + +JMXFetch now supports **dynamic tags** - the ability to extract values from JMX bean attributes and use them as tags on all emitted metrics. This is useful for adding contextual information like cluster IDs, version numbers, broker IDs, or other dynamic configuration values that are exposed via JMX. + +## Syntax + +Dynamic tags use a special syntax in the `tags` configuration: + +```yaml +tag_name:$bean_name#AttributeName +``` + +OR with the `attribute.` prefix for clarity: + +```yaml +tag_name:$bean_name#attribute.AttributeName +``` + +### Components: +- `tag_name`: The name you want for the tag (e.g., `cluster_id`, `kafka_version`) +- `$`: Prefix indicating this is a dynamic tag reference +- `bean_name`: The full JMX ObjectName (e.g., `kafka.server:type=KafkaServer,name=ClusterId`) +- `#`: Separator between bean name and attribute +- `AttributeName`: The name of the MBean attribute to fetch (e.g., `Value`, `Version`) + +## Examples + +### Kafka Cluster ID + +```yaml +instances: + - host: kafka + port: 9101 + tags: + - env:production + - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value + conf: + - include: + domain: kafka.server +``` + +This will: +1. Connect to the Kafka JMX server +2. Fetch the value of `kafka.server:type=KafkaServer,name=ClusterId` → `Value` attribute +3. Add a tag like `cluster_id:abc-xyz-123` to **all** metrics emitted from this instance + +### Multiple Dynamic Tags + +```yaml +instances: + - host: application-server + port: 9999 + tags: + - env:local + - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value + - broker_id:$kafka.server:type=KafkaServer,name=BrokerId#Value + - version:$kafka.server:type=app-info,id=0#Version +``` + +### Map-style Tags + +You can also use map-style tag configuration: + +```yaml +instances: + - host: kafka + port: 9101 + tags: + env: production + service: kafka + cluster_id: $kafka.server:type=KafkaServer,name=ClusterId#Value +``` + +## How It Works + +1. **Parse Phase**: When the instance configuration is loaded, JMXFetch identifies dynamic tags (those starting with `$`) +2. **Connection Phase**: After establishing the JMX connection, JMXFetch resolves the dynamic tags by fetching the referenced attribute values +3. **Collection Phase**: The resolved tag values are applied to all metrics collected from that instance + +## Features + +- **Type Conversion**: Attribute values are automatically converted to strings (works with String, Integer, Long, Double, etc.) +- **Graceful Degradation**: If a dynamic tag cannot be resolved (e.g., bean doesn't exist), the metric collection continues without that tag +- **Logging**: Resolution success/failure is logged for debugging +- **Performance**: Dynamic tags are resolved once at connection time, not on every metric collection + +## Error Handling + +If a dynamic tag fails to resolve: +- A warning is logged with the reason +- Metrics are still collected and emitted +- Other dynamic tags continue to be resolved +- Only the failed tag is omitted + +Example log output: +``` +INFO: Resolved dynamic tag 'cluster_id' to value 'prod-cluster-01' from bean 'kafka.server:type=KafkaServer,name=ClusterId' attribute 'Value' +WARN: Failed to resolve dynamic tag 'broker_id' from bean 'kafka.server:type=NonExistent,name=BrokerId' attribute 'Value': InstanceNotFoundException +``` + +## Testing + +The feature includes comprehensive tests covering: +- Basic dynamic tag resolution (list-style tags) +- Map-style tag configuration +- Multiple dynamic tags from different beans +- Integer/numeric attribute values +- Non-existent beans (error handling) +- Mixed static and dynamic tags + +Run tests with: +```bash +mvn test -Dtest=TestDynamicTags +``` + +## Implementation Details + +### New Classes +- `DynamicTag.java`: Handles parsing and resolution of dynamic tag references + - `parse()`: Parses tag syntax + - `resolve()`: Fetches attribute value from JMX + - `resolveAll()`: Batch resolution of multiple dynamic tags + +### Modified Classes +- `Instance.java`: + - Added `dynamicTags` field to store parsed dynamic tags + - Modified `getTagsMap()` to skip dynamic tags during initial parsing + - Added `resolveDynamicTags()` called after JMX connection is established + - Dynamic tags are resolved and added to the instance tags map + +## Limitations + +- Dynamic tags are resolved once at connection time (not per collection cycle) +- If a bean attribute value changes, you need to restart/reconnect to pick up the new value +- Currently only supports simple attribute types (String, Integer, Long, Double, Boolean) +- Complex types (CompositeData, TabularData) are not supported as tag values + +## Future Enhancements + +Potential improvements: +- Periodic re-resolution of dynamic tags (e.g., every N minutes) +- Support for nested attributes in CompositeData (e.g., `#Memory.used`) +- Template syntax for combining multiple attributes (e.g., `{#Attr1}-{#Attr2}`) +- Per-metric dynamic tags (in addition to instance-level) + +## Migration Guide + +If you're currently hardcoding values that could be dynamic: + +**Before:** +```yaml +instances: + - host: kafka-1 + port: 9101 + tags: + - cluster_id:prod-cluster-01 # Hardcoded! +``` + +**After:** +```yaml +instances: + - host: kafka-1 + port: 9101 + tags: + - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value # Dynamic! +``` + +## Support + +The feature is backward compatible - existing configurations without dynamic tags continue to work unchanged. + + diff --git a/kafka_dynamic_tags_example.yaml b/kafka_dynamic_tags_example.yaml new file mode 100644 index 00000000..bc22d334 --- /dev/null +++ b/kafka_dynamic_tags_example.yaml @@ -0,0 +1,42 @@ +# Example Kafka JMX configuration with dynamic tags +# This demonstrates how to automatically tag all metrics with values from JMX bean attributes + +instances: + - host: kafka-broker-1 + port: 9999 + tags: + # Static tags + - env:production + - service:kafka + + # Dynamic tag: Extract the Kafka cluster ID from the ClusterId MBean + # Syntax: tag_name:$bean_name#AttributeName + - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value + + # You can also use the 'attribute.' prefix for clarity + - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#attribute.Value + + # More examples of dynamic tags: + # - kafka_version:$kafka.server:type=app-info,id=1#Version + # - broker_id:$kafka.server:type=KafkaServer,name=BrokerId#Value + + conf: + # Your normal JMX metric configuration + - include: + domain: kafka.server + bean_regex: + - kafka.server:type=BrokerTopicMetrics,name=.* + attribute: + Count: + metric_type: rate + alias: kafka.broker.topic.count + + - include: + domain: kafka.network + bean: kafka.network:type=RequestMetrics,name=RequestsPerSec,request=Produce + attribute: + Count: + metric_type: rate + alias: kafka.network.produce.requests + + diff --git a/src/main/java/org/datadog/jmxfetch/DynamicTag.java b/src/main/java/org/datadog/jmxfetch/DynamicTag.java new file mode 100644 index 00000000..ad88872b --- /dev/null +++ b/src/main/java/org/datadog/jmxfetch/DynamicTag.java @@ -0,0 +1,227 @@ +package org.datadog.jmxfetch; + +import lombok.extern.slf4j.Slf4j; + +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Handles parsing and resolution of dynamic tags that reference JMX bean attribute values. + * + *

Dynamic tags allow you to extract values from JMX bean attributes and use them as tags + * on all metrics. This is useful for adding contextual information like cluster IDs, version + * numbers, or other dynamic configuration values. + * + *

Supported formats: + *

+ * + *

Example configuration: + * 

+ * instances:
+ *   - host: kafka
+ *     port: 9101
+ *     tags:
+ *       - env:local
+ *       - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value
+ *
+ */ +@Slf4j +public class DynamicTag { + // Pattern to match dynamic tag references like: + // $domain:type=Something,name=Something#AttributeName + // $domain:type=Something,name=Something#attribute.AttributeName + private static final Pattern DYNAMIC_TAG_PATTERN = + Pattern.compile("^\\$([^#]+)#(?:attribute\\.)?(.+)$"); + + private final String tagName; + private final String beanName; + private final String attributeName; + + /** + * Creates a DynamicTag from a tag key and value. + * + * @param tagKey the tag name (e.g., "cluster_id") + * @param tagValue the tag value with dynamic reference (e.g., "$kafka.server:type=KafkaServer#Value") + * @return a DynamicTag instance, or null if the tag is not a dynamic tag + */ + public static DynamicTag parse(String tagKey, String tagValue) { + if (tagValue == null || !tagValue.startsWith("$")) { + return null; + } + + Matcher matcher = DYNAMIC_TAG_PATTERN.matcher(tagValue); + if (!matcher.matches()) { + log.warn("Invalid dynamic tag format: {}. Expected format: $domain:bean_params#AttributeName", + tagValue); + return null; + } + + String beanName = matcher.group(1); + String attributeName = matcher.group(2); + + return new DynamicTag(tagKey, beanName, attributeName); + } + + /** + * Parses dynamic tags from a list of tag strings. + * + * @param tags list of tag strings (e.g., ["env:prod", "cluster_id:$kafka.server:type=KafkaServer#Value"]) + * @return list of DynamicTag instances + */ + public static List parseFromList(List tags) { + List dynamicTags = new ArrayList<>(); + if (tags == null) { + return dynamicTags; + } + + for (String tag : tags) { + int colonIndex = tag.indexOf(':'); + if (colonIndex > 0) { + String tagKey = tag.substring(0, colonIndex); + String tagValue = tag.substring(colonIndex + 1); + DynamicTag dynamicTag = parse(tagKey, tagValue); + if (dynamicTag != null) { + dynamicTags.add(dynamicTag); + } + } + } + + return dynamicTags; + } + + /** + * Parses dynamic tags from a map of tag key-value pairs. + * + * @param tags map of tag key-value pairs + * @return list of DynamicTag instances + */ + public static List parseFromMap(Map tags) { + List dynamicTags = new ArrayList<>(); + if (tags == null) { + return dynamicTags; + } + + for (Map.Entry entry : tags.entrySet()) { + DynamicTag dynamicTag = parse(entry.getKey(), entry.getValue()); + if (dynamicTag != null) { + dynamicTags.add(dynamicTag); + } + } + + return dynamicTags; + } + + /** + * Parses dynamic tags from the tag object which could be a List or Map. + * + * @param tagsObj the tags object from YAML configuration + * @return list of DynamicTag instances + */ + @SuppressWarnings("unchecked") + public static List parseFromObject(Object tagsObj) { + if (tagsObj == null) { + return new ArrayList<>(); + } + + if (tagsObj instanceof Map) { + return parseFromMap((Map) tagsObj); + } else if (tagsObj instanceof List) { + return parseFromList((List) tagsObj); + } + + return new ArrayList<>(); + } + + private DynamicTag(String tagName, String beanName, String attributeName) { + this.tagName = tagName; + this.beanName = beanName; + this.attributeName = attributeName; + } + + public String getTagName() { + return tagName; + } + + public String getBeanName() { + return beanName; + } + + public String getAttributeName() { + return attributeName; + } + + /** + * Resolves the dynamic tag by fetching the actual value from the JMX server. + * + * @param connection the JMX connection to use + * @return a map entry with the tag name and resolved value, or null if resolution failed + */ + public Map.Entry resolve(Connection connection) { + try { + ObjectName objectName = new ObjectName(beanName); + Object value = connection.getAttribute(objectName, attributeName); + + if (value == null) { + log.warn("Dynamic tag '{}' resolved to null for bean '{}' attribute '{}'", + tagName, beanName, attributeName); + return null; + } + + String stringValue = value.toString(); + log.info("Resolved dynamic tag '{}' to value '{}' from bean '{}' attribute '{}'", + tagName, stringValue, beanName, attributeName); + + return new HashMap.SimpleEntry<>(tagName, stringValue); + + } catch (MalformedObjectNameException e) { + log.error("Invalid bean name '{}' for dynamic tag '{}': {}", + beanName, tagName, e.getMessage()); + return null; + } catch (Exception e) { + log.warn("Failed to resolve dynamic tag '{}' from bean '{}' attribute '{}': {}", + tagName, beanName, attributeName, e.getMessage()); + log.debug("Dynamic tag resolution error details", e); + return null; + } + } + + /** + * Resolves multiple dynamic tags using the provided connection. + * + * @param dynamicTags list of dynamic tags to resolve + * @param connection the JMX connection to use + * @return map of resolved tag names to values + */ + public static Map resolveAll(List dynamicTags, Connection connection) { + Map resolvedTags = new HashMap<>(); + + if (dynamicTags == null || dynamicTags.isEmpty()) { + return resolvedTags; + } + + for (DynamicTag dynamicTag : dynamicTags) { + Map.Entry resolved = dynamicTag.resolve(connection); + if (resolved != null) { + resolvedTags.put(resolved.getKey(), resolved.getValue()); + } + } + + return resolvedTags; + } + + @Override + public String toString() { + return String.format("DynamicTag{name='%s', bean='%s', attribute='%s'}", + tagName, beanName, attributeName); + } +} + diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index 9dd22fbe..6f0fbf0b 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -79,6 +79,7 @@ public Yaml initialValue() { private ObjectName instanceTelemetryBeanName; private MBeanServer mbs; private Boolean normalizeBeanParamTags; + private List dynamicTags; /** Constructor, instantiates Instance based of a previous instance and appConfig. */ public Instance(Instance instance, AppConfig appConfig) { @@ -107,6 +108,10 @@ public Instance( instanceMap != null ? new HashMap(instanceMap) : null; this.initConfig = initConfig != null ? new HashMap(initConfig) : null; this.instanceName = (String) instanceMap.get("name"); + + // Parse dynamic tags before processing regular tags + this.dynamicTags = DynamicTag.parseFromObject(instanceMap.get("tags")); + this.tags = getTagsMap(instanceMap.get("tags"), appConfig); this.checkName = checkName; this.matchingAttributes = new ArrayList(); @@ -387,6 +392,7 @@ static void loadMetricConfigResources( /** * Format the instance tags defined in the YAML configuration file to a `HashMap`. * Supported inputs: `List`, `Map`. + * Dynamic tags (starting with $) are skipped here and will be resolved after connection. */ private static Map getTagsMap(Object tagsMap, AppConfig appConfig) { Map tags = new HashMap(); @@ -395,9 +401,27 @@ private static Map getTagsMap(Object tagsMap, AppConfig appConfi } if (tagsMap != null) { if (tagsMap instanceof Map) { - tags.putAll((Map) tagsMap); + // Add non-dynamic tags, skip dynamic tags (they'll be resolved later) + for (Map.Entry entry : ((Map) tagsMap).entrySet()) { + if (entry.getValue() == null || !entry.getValue().startsWith("$")) { + tags.put(entry.getKey(), entry.getValue()); + } else { + log.debug("Skipping dynamic tag '{}' in initial tag parsing, " + + "will be resolved after connection", entry.getKey()); + } + } } else if (tagsMap instanceof List) { for (String tag : (List) tagsMap) { + // Check if this is a dynamic tag (contains : and value starts with $) + int colonIndex = tag.indexOf(':'); + if (colonIndex > 0) { + String tagValue = tag.substring(colonIndex + 1); + if (tagValue.startsWith("$")) { + log.debug("Skipping dynamic tag '{}' in initial tag parsing, " + + "will be resolved after connection", tag); + continue; + } + } tags.put(tag, null); } } else { @@ -447,6 +471,10 @@ public void init(boolean forceNewConnection) throws IOException, FailedLoginException, SecurityException { log.info("Trying to connect to JMX Server at " + this.toString()); connection = getConnection(instanceMap, forceNewConnection); + + // Resolve dynamic tags after connection is established + resolveDynamicTags(); + log.info( "Trying to collect bean list for the first time for JMX Server at {}", this); this.refreshBeansList(); @@ -455,6 +483,29 @@ public void init(boolean forceNewConnection) this.getMatchingAttributes(); log.info("Done initializing JMX Server at {}", this); } + + /** + * Resolves dynamic tags by fetching their values from JMX beans. + * This method is called after the JMX connection is established. + */ + private void resolveDynamicTags() { + if (dynamicTags == null || dynamicTags.isEmpty()) { + return; + } + + log.info("Resolving {} dynamic tag(s) for instance {}", dynamicTags.size(), instanceName); + + Map resolvedTags = DynamicTag.resolveAll(dynamicTags, connection); + + if (!resolvedTags.isEmpty()) { + // Add resolved tags to the instance tags + this.tags.putAll(resolvedTags); + log.info("Successfully resolved {} dynamic tag(s) for instance {}: {}", + resolvedTags.size(), instanceName, resolvedTags); + } else { + log.warn("No dynamic tags could be resolved for instance {}", instanceName); + } + } /** Returns a string representation for the instance. */ @Override diff --git a/src/test/java/org/datadog/jmxfetch/DynamicTagTestApp.java b/src/test/java/org/datadog/jmxfetch/DynamicTagTestApp.java new file mode 100644 index 00000000..8cb21dac --- /dev/null +++ b/src/test/java/org/datadog/jmxfetch/DynamicTagTestApp.java @@ -0,0 +1,49 @@ +package org.datadog.jmxfetch; + +/** + * Test application for dynamic tag resolution. + * Exposes various attributes that can be used as dynamic tags. + */ +public class DynamicTagTestApp implements DynamicTagTestAppMBean { + private final String clusterId; + private final String version; + private final int port; + private double metric; + + public DynamicTagTestApp() { + this("local-kafka-cluster", "2.8.0", 9092); + } + + public DynamicTagTestApp(String clusterId, String version, int port) { + this.clusterId = clusterId; + this.version = version; + this.port = port; + this.metric = 100.0; + } + + @Override + public String getClusterId() { + return clusterId; + } + + @Override + public String getVersion() { + return version; + } + + @Override + public int getPort() { + return port; + } + + @Override + public Double getMetric() { + return metric; + } + + public void setMetric(double metric) { + this.metric = metric; + } +} + + diff --git a/src/test/java/org/datadog/jmxfetch/DynamicTagTestAppMBean.java b/src/test/java/org/datadog/jmxfetch/DynamicTagTestAppMBean.java new file mode 100644 index 00000000..db5df189 --- /dev/null +++ b/src/test/java/org/datadog/jmxfetch/DynamicTagTestAppMBean.java @@ -0,0 +1,13 @@ +package org.datadog.jmxfetch; + +/** + * MBean interface for testing dynamic tag resolution. + */ +public interface DynamicTagTestAppMBean { + String getClusterId(); + String getVersion(); + int getPort(); + Double getMetric(); +} + + diff --git a/src/test/java/org/datadog/jmxfetch/TestDynamicTags.java b/src/test/java/org/datadog/jmxfetch/TestDynamicTags.java new file mode 100644 index 00000000..dff2041d --- /dev/null +++ b/src/test/java/org/datadog/jmxfetch/TestDynamicTags.java @@ -0,0 +1,297 @@ +package org.datadog.jmxfetch; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * Tests for dynamic tag resolution from JMX bean attributes. + */ +public class TestDynamicTags extends TestCommon { + + /** + * Test basic dynamic tag resolution with list-style tags. + */ + @Test + public void testDynamicTagsBasic() throws Exception { + // Register a test MBean with attributes we'll use as tags + registerMBean( + new DynamicTagTestApp("prod-kafka-cluster", "3.0.0", 9092), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); + + // Also register the SimpleTestJavaApp for default JVM metrics + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); + + initApplication("jmx_dynamic_tags.yaml"); + + // Run the collection + run(); + + List> metrics = getMetrics(); + assertNotNull(metrics); + assertTrue("Should have collected metrics", metrics.size() > 0); + + // Verify that our test metric has the dynamic tags + List expectedTags = Arrays.asList( + "env:test", + "cluster_id:prod-kafka-cluster", + "kafka_version:3.0.0", + "instance:jmx_dynamic_tags_test", + "jmx_domain:org.datadog.jmxfetch.test", + "dd.internal.jmx_check_name:jmx_dynamic_tags", + "type:DynamicTagTestApp" + ); + + assertMetric("test.dynamic.tags.metric", 100.0, expectedTags, 7); + } + + /** + * Test dynamic tag resolution with "attribute." prefix in the reference. + */ + @Test + public void testDynamicTagsWithAttributePrefix() throws Exception { + registerMBean( + new DynamicTagTestApp("dev-kafka-cluster", "2.8.0", 9093), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); + + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); + + initApplication("jmx_dynamic_tags_with_attribute_prefix.yaml"); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + // Verify the dynamic tag was resolved correctly + List expectedTags = Arrays.asList( + "env:test", + "cluster_id:dev-kafka-cluster", + "instance:jmx_dynamic_tags_attribute_test", + "jmx_domain:org.datadog.jmxfetch.test", + "dd.internal.jmx_check_name:jmx_dynamic_tags_with_attribute_prefix", + "type:DynamicTagTestApp" + ); + + assertMetric("test.dynamic.tags.metric", 100.0, expectedTags, 6); + } + + /** + * Test dynamic tag resolution with map-style tags. + */ + @Test + public void testDynamicTagsMapStyle() throws Exception { + registerMBean( + new DynamicTagTestApp("staging-cluster", "3.1.0", 9094), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); + + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); + + initApplication("jmx_dynamic_tags_map_style.yaml"); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + // Verify both static and dynamic tags are present + List expectedTags = Arrays.asList( + "env:test", + "cluster_id:staging-cluster", + "static_tag:static_value", + "instance:jmx_dynamic_tags_map_test", + "jmx_domain:org.datadog.jmxfetch.test", + "dd.internal.jmx_check_name:jmx_dynamic_tags_map_style", + "type:DynamicTagTestApp" + ); + + assertMetric("test.dynamic.tags.metric", 100.0, expectedTags, 7); + } + + /** + * Test that metrics are still collected when a dynamic tag reference fails to resolve. + * The tag should simply be absent rather than causing the entire instance to fail. + */ + @Test + public void testDynamicTagsNonExistentBean() throws Exception { + // Only register the app bean, not the bean referenced in the dynamic tag + registerMBean( + new DynamicTagTestApp("test-cluster", "1.0.0", 9095), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); + + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); + + initApplication("jmx_dynamic_tags_nonexistent.yaml"); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics despite failed dynamic tag resolution", + metrics.size() > 0); + + // The metric should be collected, but without the failed dynamic tag + List expectedTags = Arrays.asList( + "env:test", + "instance:jmx_dynamic_tags_nonexistent_test", + "jmx_domain:org.datadog.jmxfetch.test", + "dd.internal.jmx_check_name:jmx_dynamic_tags_nonexistent", + "type:DynamicTagTestApp" + ); + + // Note: cluster_id tag should NOT be present since the bean doesn't exist + assertMetric("test.dynamic.tags.metric", 100.0, expectedTags, 5); + } + + /** + * Test that dynamic tags work with different attribute types (integer). + */ + @Test + public void testDynamicTagsWithIntegerAttribute() throws Exception { + registerMBean( + new DynamicTagTestApp("int-test-cluster", "1.0.0", 8888), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); + + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); + + initApplicationWithYamlLines( + "init_config:", + "", + "instances:", + " - process_name_regex: .*surefire.*", + " name: jmx_dynamic_tags_int_test", + " tags:", + " - env:test", + " - port:$org.datadog.jmxfetch.test:type=DynamicTagTestApp#Port", + " conf:", + " - include:", + " domain: org.datadog.jmxfetch.test", + " type: DynamicTagTestApp", + " attribute:", + " Metric:", + " metric_type: gauge", + " alias: test.dynamic.tags.metric" + ); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + // Verify the port (integer) was converted to a string tag value + // Note: dd.internal.jmx_check_name will be a generated config name since we're using initApplicationWithYamlLines + + // Find the test metric + boolean foundMetric = false; + for (Map metric : metrics) { + if ("test.dynamic.tags.metric".equals(metric.get("name"))) { + String[] tags = (String[]) metric.get("tags"); + List tagList = Arrays.asList(tags); + + // Check that required tags are present + assertTrue("Should have env:test tag", tagList.contains("env:test")); + assertTrue("Should have port:8888 tag", tagList.contains("port:8888")); + + // Check for instance tag + boolean hasInstanceTag = false; + for (String tag : tagList) { + if (tag.startsWith("instance:jmx_dynamic_tags_int_test")) { + hasInstanceTag = true; + break; + } + } + assertTrue("Should have instance tag", hasInstanceTag); + assertTrue("Should have type tag", tagList.contains("type:DynamicTagTestApp")); + foundMetric = true; + break; + } + } + assertTrue("Should have found test.dynamic.tags.metric", foundMetric); + } + + /** + * Test that multiple instances can have different dynamic tag values. + */ + @Test + public void testDynamicTagsMultipleInstances() throws Exception { + // Register two different instances with different cluster IDs + registerMBean( + new DynamicTagTestApp("cluster-1", "1.0.0", 9001), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1"); + + registerMBean( + new DynamicTagTestApp("cluster-2", "2.0.0", 9002), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2"); + + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); + + initApplicationWithYamlLines( + "init_config:", + "", + "instances:", + " - process_name_regex: .*surefire.*", + " name: jmx_dynamic_tags_multi_test", + " tags:", + " - cluster_id_1:$org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1#ClusterId", + " - cluster_id_2:$org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2#ClusterId", + " conf:", + " - include:", + " domain: org.datadog.jmxfetch.test", + " bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1", + " attribute:", + " Metric:", + " metric_type: gauge", + " alias: test.instance1.metric" + ); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + // Find the test metric and verify both cluster IDs are present + boolean foundMetric = false; + for (Map metric : metrics) { + if ("test.instance1.metric".equals(metric.get("name"))) { + String[] tags = (String[]) metric.get("tags"); + List tagList = Arrays.asList(tags); + + // Check that both dynamic tags are present with correct values + assertTrue("Should have cluster_id_1 tag", tagList.contains("cluster_id_1:cluster-1")); + assertTrue("Should have cluster_id_2 tag", tagList.contains("cluster_id_2:cluster-2")); + + // Check for instance tag + boolean hasInstanceTag = false; + for (String tag : tagList) { + if (tag.startsWith("instance:jmx_dynamic_tags_multi_test")) { + hasInstanceTag = true; + break; + } + } + assertTrue("Should have instance tag", hasInstanceTag); + assertTrue("Should have type tag", tagList.contains("type:DynamicTagTestApp")); + assertTrue("Should have name tag", tagList.contains("name:Instance1")); + foundMetric = true; + break; + } + } + assertTrue("Should have found test.instance1.metric", foundMetric); + } +} + diff --git a/src/test/resources/jmx_dynamic_tags.yaml b/src/test/resources/jmx_dynamic_tags.yaml new file mode 100644 index 00000000..5e7bd360 --- /dev/null +++ b/src/test/resources/jmx_dynamic_tags.yaml @@ -0,0 +1,19 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_dynamic_tags_test + tags: + - env:test + - cluster_id:$org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId + - kafka_version:$org.datadog.jmxfetch.test:type=DynamicTagTestApp#Version + conf: + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + attribute: + Metric: + metric_type: gauge + alias: test.dynamic.tags.metric + + diff --git a/src/test/resources/jmx_dynamic_tags_map_style.yaml b/src/test/resources/jmx_dynamic_tags_map_style.yaml new file mode 100644 index 00000000..f41c1e12 --- /dev/null +++ b/src/test/resources/jmx_dynamic_tags_map_style.yaml @@ -0,0 +1,19 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_dynamic_tags_map_test + tags: + env: test + cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId + static_tag: static_value + conf: + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + attribute: + Metric: + metric_type: gauge + alias: test.dynamic.tags.metric + + diff --git a/src/test/resources/jmx_dynamic_tags_nonexistent.yaml b/src/test/resources/jmx_dynamic_tags_nonexistent.yaml new file mode 100644 index 00000000..fd46b409 --- /dev/null +++ b/src/test/resources/jmx_dynamic_tags_nonexistent.yaml @@ -0,0 +1,18 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_dynamic_tags_nonexistent_test + tags: + - env:test + - cluster_id:$org.datadog.jmxfetch.test:type=NonExistentBean#Value + conf: + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + attribute: + Metric: + metric_type: gauge + alias: test.dynamic.tags.metric + + diff --git a/src/test/resources/jmx_dynamic_tags_with_attribute_prefix.yaml b/src/test/resources/jmx_dynamic_tags_with_attribute_prefix.yaml new file mode 100644 index 00000000..fa1f2bbe --- /dev/null +++ b/src/test/resources/jmx_dynamic_tags_with_attribute_prefix.yaml @@ -0,0 +1,18 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_dynamic_tags_attribute_test + tags: + - env:test + - cluster_id:$org.datadog.jmxfetch.test:type=DynamicTagTestApp#attribute.ClusterId + conf: + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + attribute: + Metric: + metric_type: gauge + alias: test.dynamic.tags.metric + + From 9961226cef876b3fb72055331c573e055b51b062 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Wed, 15 Oct 2025 15:04:01 -0600 Subject: [PATCH 02/11] Use tags from JMX sections, not instance tags --- DOCKER_USAGE_GUIDE.md | 215 ------------- DYNAMIC_TAGS_FEATURE.md | 174 ---------- kafka_dynamic_tags_example.yaml | 42 --- .../org/datadog/jmxfetch/Configuration.java | 43 +++ .../java/org/datadog/jmxfetch/Connection.java | 5 + .../java/org/datadog/jmxfetch/DynamicTag.java | 141 ++------- .../java/org/datadog/jmxfetch/Filter.java | 51 ++- .../java/org/datadog/jmxfetch/Instance.java | 78 +++-- .../org/datadog/jmxfetch/JmxAttribute.java | 7 + .../datadog/jmxfetch/DynamicTagTestApp.java | 4 - .../jmxfetch/DynamicTagTestAppMBean.java | 3 - .../jmxfetch/TestConfigDynamicTags.java | 297 ++++++++++++++++++ .../org/datadog/jmxfetch/TestDynamicTags.java | 297 ------------------ .../resources/jmx_config_dynamic_tags.yaml | 18 ++ .../jmx_config_dynamic_tags_multi.yaml | 30 ++ src/test/resources/jmx_dynamic_tags.yaml | 19 -- .../resources/jmx_dynamic_tags_map_style.yaml | 19 -- .../jmx_dynamic_tags_nonexistent.yaml | 18 -- ...mx_dynamic_tags_with_attribute_prefix.yaml | 18 -- 19 files changed, 498 insertions(+), 981 deletions(-) delete mode 100644 DOCKER_USAGE_GUIDE.md delete mode 100644 DYNAMIC_TAGS_FEATURE.md delete mode 100644 kafka_dynamic_tags_example.yaml create mode 100644 src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java delete mode 100644 src/test/java/org/datadog/jmxfetch/TestDynamicTags.java create mode 100644 src/test/resources/jmx_config_dynamic_tags.yaml create mode 100644 src/test/resources/jmx_config_dynamic_tags_multi.yaml delete mode 100644 src/test/resources/jmx_dynamic_tags.yaml delete mode 100644 src/test/resources/jmx_dynamic_tags_map_style.yaml delete mode 100644 src/test/resources/jmx_dynamic_tags_nonexistent.yaml delete mode 100644 src/test/resources/jmx_dynamic_tags_with_attribute_prefix.yaml diff --git a/DOCKER_USAGE_GUIDE.md b/DOCKER_USAGE_GUIDE.md deleted file mode 100644 index e1ea6296..00000000 --- a/DOCKER_USAGE_GUIDE.md +++ /dev/null @@ -1,215 +0,0 @@ -# Using Custom JMXFetch with Docker Agent - -This guide shows how to use your custom JMXFetch build (with dynamic tags feature) in a Docker-based Datadog Agent **without rebuilding anything**. - -## Quick Start - -### Method 1: Volume Mount (Recommended) - -This is the easiest approach - just mount the custom JAR into your container. - -**1. Your custom JAR is here:** -```bash -/Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar -``` - -**2. Add a volume mount to your docker-compose.yaml or docker run command:** - -#### Docker Compose: -```yaml -version: '3' -services: - datadog: - image: gcr.io/datadoghq/agent:latest - environment: - - DD_API_KEY=your_api_key - - DD_SITE=datadoghq.com - # ... other env vars - volumes: - # Mount custom JMXFetch - - /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar:ro - - # Your other volumes - - /var/run/docker.sock:/var/run/docker.sock:ro - - ./conf.d:/etc/datadog-agent/conf.d:ro - # ... other volumes -``` - -#### Docker Run: -```bash -docker run -d \ - --name datadog-agent \ - -e DD_API_KEY=your_api_key \ - -e DD_SITE=datadoghq.com \ - -v /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar:ro \ - -v /var/run/docker.sock:/var/run/docker.sock:ro \ - -v /opt/datadog-agent/run:/opt/datadog-agent/run:rw \ - gcr.io/datadoghq/agent:latest -``` - -**3. Restart your agent:** -```bash -docker-compose restart datadog -# or -docker restart datadog-agent -``` - -**4. Verify it's working:** -```bash -# Check agent logs -docker logs datadog-agent | grep -i jmx - -# You should see logs about dynamic tags being resolved -docker logs datadog-agent | grep -i "Resolved dynamic tag" -``` - ---- - -### Method 2: Copy into Running Container - -If you already have a running container and don't want to restart with new volumes: - -```bash -# Copy the JAR into the running container -docker cp \ - /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar \ - datadog-agent:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar - -# Restart the agent process inside the container -docker exec datadog-agent agent restart -``` - ---- - -## Example Configuration - -Now you can use dynamic tags in your JMX configuration. Create or update your Kafka integration config: - -**File: `conf.d/kafka.yaml`** (or wherever your JMX configs are) - -```yaml -init_config: - is_jmx: true - -instances: - - host: kafka-broker - port: 9999 - tags: - - env:production - - service:kafka - # 🎉 NEW: Dynamic tags that pull values from JMX beans! - - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value - - conf: - - include: - domain: kafka.server - bean: kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec - attribute: - Count: - metric_type: rate - alias: kafka.messages_in -``` - ---- - -## Verification - -**1. Check JMXFetch version in logs:** -```bash -docker logs datadog-agent 2>&1 | grep -i "jmxfetch version" -``` - -**2. Check for dynamic tag resolution:** -```bash -docker logs datadog-agent 2>&1 | grep -i "Resolving.*dynamic tag" -``` - -You should see logs like: -``` -INFO: Resolving 1 dynamic tag(s) for instance kafka_instance -INFO: Resolved dynamic tag 'cluster_id' to value 'prod-cluster-123' from bean 'kafka.server:type=KafkaServer,name=ClusterId' attribute 'Value' -``` - -**3. Verify metrics have the dynamic tags:** -```bash -# In Datadog UI, check that your Kafka metrics now have the cluster_id tag -# with the actual value from your Kafka cluster -``` - ---- - -## Troubleshooting - -### JAR not found error -**Error:** `FileNotFoundException: jmxfetch.jar` - -**Solution:** The default JMXFetch location varies by agent version. Try these paths: -- `/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar` (newer versions) -- `/opt/datadog-agent/embedded/bin/jmxfetch.jar` (older versions) - -Find the correct path: -```bash -docker exec datadog-agent find /opt/datadog-agent -name "jmxfetch.jar" -``` - -### Dynamic tags not resolving -**Check:** -1. JMX connection is successful: `docker logs datadog-agent | grep "Connected to JMX"` -2. Bean name is correct: `docker exec datadog-agent agent jmx list matching` -3. Attribute name matches exactly (case-sensitive!) - -### Agent won't start -**Error:** Permission denied - -**Solution:** Make sure the JAR file has read permissions: -```bash -chmod 644 /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch/target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar -``` - ---- - -## Rolling Back - -To go back to the standard JMXFetch: - -**If using volume mount:** -1. Remove the volume mount from docker-compose.yaml -2. Restart: `docker-compose restart datadog` - -**If you copied the file:** -```bash -# The agent will automatically download the standard version on restart -docker restart datadog-agent -``` - ---- - -## Development Workflow - -When you make changes to JMXFetch: - -```bash -# 1. Rebuild the JAR -cd /Users/piotr.wolski/go/src/github.com/DataDog/jmxfetch -mvn clean package -DskipTests - -# 2. If using volume mount: just restart -docker-compose restart datadog - -# 3. If using copy method: copy and restart -docker cp target/jmxfetch-0.50.1-SNAPSHOT-jar-with-dependencies.jar \ - datadog-agent:/opt/datadog-agent/bin/agent/dist/jmx/jmxfetch.jar -docker exec datadog-agent agent restart -``` - ---- - -## Notes - -- ✅ **No rebuild needed** - Agent uses the JAR you provide -- ✅ **Hot reload** - Just restart the agent container -- ✅ **Preserves config** - All your existing configs work unchanged -- ⚠️ **Version mismatch** - If the agent version is very different from JMXFetch, you might see compatibility issues (rare) -- 💡 **Tip:** Use `:ro` (read-only) mount to prevent accidental modifications - - diff --git a/DYNAMIC_TAGS_FEATURE.md b/DYNAMIC_TAGS_FEATURE.md deleted file mode 100644 index 0a56f0fc..00000000 --- a/DYNAMIC_TAGS_FEATURE.md +++ /dev/null @@ -1,174 +0,0 @@ -# Dynamic Tags Feature for JMXFetch - -## Overview - -JMXFetch now supports **dynamic tags** - the ability to extract values from JMX bean attributes and use them as tags on all emitted metrics. This is useful for adding contextual information like cluster IDs, version numbers, broker IDs, or other dynamic configuration values that are exposed via JMX. - -## Syntax - -Dynamic tags use a special syntax in the `tags` configuration: - -```yaml -tag_name:$bean_name#AttributeName -``` - -OR with the `attribute.` prefix for clarity: - -```yaml -tag_name:$bean_name#attribute.AttributeName -``` - -### Components: -- `tag_name`: The name you want for the tag (e.g., `cluster_id`, `kafka_version`) -- `$`: Prefix indicating this is a dynamic tag reference -- `bean_name`: The full JMX ObjectName (e.g., `kafka.server:type=KafkaServer,name=ClusterId`) -- `#`: Separator between bean name and attribute -- `AttributeName`: The name of the MBean attribute to fetch (e.g., `Value`, `Version`) - -## Examples - -### Kafka Cluster ID - -```yaml -instances: - - host: kafka - port: 9101 - tags: - - env:production - - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value - conf: - - include: - domain: kafka.server -``` - -This will: -1. Connect to the Kafka JMX server -2. Fetch the value of `kafka.server:type=KafkaServer,name=ClusterId` → `Value` attribute -3. Add a tag like `cluster_id:abc-xyz-123` to **all** metrics emitted from this instance - -### Multiple Dynamic Tags - -```yaml -instances: - - host: application-server - port: 9999 - tags: - - env:local - - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value - - broker_id:$kafka.server:type=KafkaServer,name=BrokerId#Value - - version:$kafka.server:type=app-info,id=0#Version -``` - -### Map-style Tags - -You can also use map-style tag configuration: - -```yaml -instances: - - host: kafka - port: 9101 - tags: - env: production - service: kafka - cluster_id: $kafka.server:type=KafkaServer,name=ClusterId#Value -``` - -## How It Works - -1. **Parse Phase**: When the instance configuration is loaded, JMXFetch identifies dynamic tags (those starting with `$`) -2. **Connection Phase**: After establishing the JMX connection, JMXFetch resolves the dynamic tags by fetching the referenced attribute values -3. **Collection Phase**: The resolved tag values are applied to all metrics collected from that instance - -## Features - -- **Type Conversion**: Attribute values are automatically converted to strings (works with String, Integer, Long, Double, etc.) -- **Graceful Degradation**: If a dynamic tag cannot be resolved (e.g., bean doesn't exist), the metric collection continues without that tag -- **Logging**: Resolution success/failure is logged for debugging -- **Performance**: Dynamic tags are resolved once at connection time, not on every metric collection - -## Error Handling - -If a dynamic tag fails to resolve: -- A warning is logged with the reason -- Metrics are still collected and emitted -- Other dynamic tags continue to be resolved -- Only the failed tag is omitted - -Example log output: -``` -INFO: Resolved dynamic tag 'cluster_id' to value 'prod-cluster-01' from bean 'kafka.server:type=KafkaServer,name=ClusterId' attribute 'Value' -WARN: Failed to resolve dynamic tag 'broker_id' from bean 'kafka.server:type=NonExistent,name=BrokerId' attribute 'Value': InstanceNotFoundException -``` - -## Testing - -The feature includes comprehensive tests covering: -- Basic dynamic tag resolution (list-style tags) -- Map-style tag configuration -- Multiple dynamic tags from different beans -- Integer/numeric attribute values -- Non-existent beans (error handling) -- Mixed static and dynamic tags - -Run tests with: -```bash -mvn test -Dtest=TestDynamicTags -``` - -## Implementation Details - -### New Classes -- `DynamicTag.java`: Handles parsing and resolution of dynamic tag references - - `parse()`: Parses tag syntax - - `resolve()`: Fetches attribute value from JMX - - `resolveAll()`: Batch resolution of multiple dynamic tags - -### Modified Classes -- `Instance.java`: - - Added `dynamicTags` field to store parsed dynamic tags - - Modified `getTagsMap()` to skip dynamic tags during initial parsing - - Added `resolveDynamicTags()` called after JMX connection is established - - Dynamic tags are resolved and added to the instance tags map - -## Limitations - -- Dynamic tags are resolved once at connection time (not per collection cycle) -- If a bean attribute value changes, you need to restart/reconnect to pick up the new value -- Currently only supports simple attribute types (String, Integer, Long, Double, Boolean) -- Complex types (CompositeData, TabularData) are not supported as tag values - -## Future Enhancements - -Potential improvements: -- Periodic re-resolution of dynamic tags (e.g., every N minutes) -- Support for nested attributes in CompositeData (e.g., `#Memory.used`) -- Template syntax for combining multiple attributes (e.g., `{#Attr1}-{#Attr2}`) -- Per-metric dynamic tags (in addition to instance-level) - -## Migration Guide - -If you're currently hardcoding values that could be dynamic: - -**Before:** -```yaml -instances: - - host: kafka-1 - port: 9101 - tags: - - cluster_id:prod-cluster-01 # Hardcoded! -``` - -**After:** -```yaml -instances: - - host: kafka-1 - port: 9101 - tags: - - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value # Dynamic! -``` - -## Support - -The feature is backward compatible - existing configurations without dynamic tags continue to work unchanged. - - diff --git a/kafka_dynamic_tags_example.yaml b/kafka_dynamic_tags_example.yaml deleted file mode 100644 index bc22d334..00000000 --- a/kafka_dynamic_tags_example.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Example Kafka JMX configuration with dynamic tags -# This demonstrates how to automatically tag all metrics with values from JMX bean attributes - -instances: - - host: kafka-broker-1 - port: 9999 - tags: - # Static tags - - env:production - - service:kafka - - # Dynamic tag: Extract the Kafka cluster ID from the ClusterId MBean - # Syntax: tag_name:$bean_name#AttributeName - - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value - - # You can also use the 'attribute.' prefix for clarity - - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#attribute.Value - - # More examples of dynamic tags: - # - kafka_version:$kafka.server:type=app-info,id=1#Version - # - broker_id:$kafka.server:type=KafkaServer,name=BrokerId#Value - - conf: - # Your normal JMX metric configuration - - include: - domain: kafka.server - bean_regex: - - kafka.server:type=BrokerTopicMetrics,name=.* - attribute: - Count: - metric_type: rate - alias: kafka.broker.topic.count - - - include: - domain: kafka.network - bean: kafka.network:type=RequestMetrics,name=RequestsPerSec,request=Produce - attribute: - Count: - metric_type: rate - alias: kafka.network.produce.requests - - diff --git a/src/main/java/org/datadog/jmxfetch/Configuration.java b/src/main/java/org/datadog/jmxfetch/Configuration.java index 0960c98f..0f5341f4 100644 --- a/src/main/java/org/datadog/jmxfetch/Configuration.java +++ b/src/main/java/org/datadog/jmxfetch/Configuration.java @@ -1,5 +1,7 @@ package org.datadog.jmxfetch; +import lombok.extern.slf4j.Slf4j; + import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -8,12 +10,15 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import javax.management.MBeanServerConnection; +@Slf4j public class Configuration { private Map conf; private Filter include; private Filter exclude; + private Map resolvedDynamicTags = null; /** * Access configuration elements more easily @@ -45,6 +50,44 @@ public String toString() { private Boolean hasInclude() { return getInclude() != null && !getInclude().isEmptyFilter(); } + + /** Resolve dynamic tags for this configuration using cached values. */ + public void resolveDynamicTags( + MBeanServerConnection connection, Map> cache) { + if (resolvedDynamicTags != null) { + return; + } + + resolvedDynamicTags = new HashMap(); + + if (include == null) { + return; + } + + List dynamicTags = include.getDynamicTags(); + if (dynamicTags == null || dynamicTags.isEmpty()) { + return; + } + + for (DynamicTag dynamicTag : dynamicTags) { + String cacheKey = dynamicTag.getBeanName() + "#" + dynamicTag.getAttributeName(); + Map.Entry cached = cache.get(cacheKey); + if (cached != null) { + resolvedDynamicTags.put(cached.getKey(), cached.getValue()); + } + } + + log.debug("Applied {} dynamic tag(s) to configuration from cache", + resolvedDynamicTags.size()); + } + + /** Get all resolved dynamic tags for this configuration. */ + public Map getResolvedDynamicTags() { + if (resolvedDynamicTags == null) { + return new HashMap(); + } + return resolvedDynamicTags; + } /** * Filter a configuration list to keep the ones with `include` filters. diff --git a/src/main/java/org/datadog/jmxfetch/Connection.java b/src/main/java/org/datadog/jmxfetch/Connection.java index 803cd817..84c11e53 100644 --- a/src/main/java/org/datadog/jmxfetch/Connection.java +++ b/src/main/java/org/datadog/jmxfetch/Connection.java @@ -68,6 +68,11 @@ public Object getAttribute(ObjectName objectName, String attributeName) } return attr; } + + /** Gets the underlying MBeanServerConnection. */ + public MBeanServerConnection getMBeanServerConnection() { + return mbs; + } /** Closes the connector. */ public void closeConnector() { diff --git a/src/main/java/org/datadog/jmxfetch/DynamicTag.java b/src/main/java/org/datadog/jmxfetch/DynamicTag.java index ad88872b..b83f354c 100644 --- a/src/main/java/org/datadog/jmxfetch/DynamicTag.java +++ b/src/main/java/org/datadog/jmxfetch/DynamicTag.java @@ -2,43 +2,23 @@ import lombok.extern.slf4j.Slf4j; -import javax.management.MalformedObjectNameException; -import javax.management.ObjectName; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.management.AttributeNotFoundException; +import javax.management.InstanceNotFoundException; +import javax.management.MBeanException; +import javax.management.MBeanServerConnection; +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; +import javax.management.ReflectionException; -/** - * Handles parsing and resolution of dynamic tags that reference JMX bean attribute values. - * - *

Dynamic tags allow you to extract values from JMX bean attributes and use them as tags - * on all metrics. This is useful for adding contextual information like cluster IDs, version - * numbers, or other dynamic configuration values. - * - *

Supported formats: - *

    - *
  • Simple attribute: {@code tag_name:$domain:bean_params#AttributeName}
    • - *
  • With "attribute" prefix: {@code tag_name:$domain:bean_params#attribute.AttributeName}
    • - *
- * - *

Example configuration: - * 

- * instances:
- *   - host: kafka
- *     port: 9101
- *     tags:
- *       - env:local
- *       - cluster_id:$kafka.server:type=KafkaServer,name=ClusterId#Value
- *
- */ @Slf4j public class DynamicTag { - // Pattern to match dynamic tag references like: - // $domain:type=Something,name=Something#AttributeName - // $domain:type=Something,name=Something#attribute.AttributeName private static final Pattern DYNAMIC_TAG_PATTERN = Pattern.compile("^\\$([^#]+)#(?:attribute\\.)?(.+)$"); @@ -46,13 +26,7 @@ public class DynamicTag { private final String beanName; private final String attributeName; - /** - * Creates a DynamicTag from a tag key and value. - * - * @param tagKey the tag name (e.g., "cluster_id") - * @param tagValue the tag value with dynamic reference (e.g., "$kafka.server:type=KafkaServer#Value") - * @return a DynamicTag instance, or null if the tag is not a dynamic tag - */ + /** Parse a dynamic tag from a tag key and value. */ public static DynamicTag parse(String tagKey, String tagValue) { if (tagValue == null || !tagValue.startsWith("$")) { return null; @@ -60,8 +34,8 @@ public static DynamicTag parse(String tagKey, String tagValue) { Matcher matcher = DYNAMIC_TAG_PATTERN.matcher(tagValue); if (!matcher.matches()) { - log.warn("Invalid dynamic tag format: {}. Expected format: $domain:bean_params#AttributeName", - tagValue); + log.warn("Invalid dynamic tag format: {}. " + + "Expected format: $domain:bean_params#AttributeName", tagValue); return null; } @@ -71,76 +45,6 @@ public static DynamicTag parse(String tagKey, String tagValue) { return new DynamicTag(tagKey, beanName, attributeName); } - /** - * Parses dynamic tags from a list of tag strings. - * - * @param tags list of tag strings (e.g., ["env:prod", "cluster_id:$kafka.server:type=KafkaServer#Value"]) - * @return list of DynamicTag instances - */ - public static List parseFromList(List tags) { - List dynamicTags = new ArrayList<>(); - if (tags == null) { - return dynamicTags; - } - - for (String tag : tags) { - int colonIndex = tag.indexOf(':'); - if (colonIndex > 0) { - String tagKey = tag.substring(0, colonIndex); - String tagValue = tag.substring(colonIndex + 1); - DynamicTag dynamicTag = parse(tagKey, tagValue); - if (dynamicTag != null) { - dynamicTags.add(dynamicTag); - } - } - } - - return dynamicTags; - } - - /** - * Parses dynamic tags from a map of tag key-value pairs. - * - * @param tags map of tag key-value pairs - * @return list of DynamicTag instances - */ - public static List parseFromMap(Map tags) { - List dynamicTags = new ArrayList<>(); - if (tags == null) { - return dynamicTags; - } - - for (Map.Entry entry : tags.entrySet()) { - DynamicTag dynamicTag = parse(entry.getKey(), entry.getValue()); - if (dynamicTag != null) { - dynamicTags.add(dynamicTag); - } - } - - return dynamicTags; - } - - /** - * Parses dynamic tags from the tag object which could be a List or Map. - * - * @param tagsObj the tags object from YAML configuration - * @return list of DynamicTag instances - */ - @SuppressWarnings("unchecked") - public static List parseFromObject(Object tagsObj) { - if (tagsObj == null) { - return new ArrayList<>(); - } - - if (tagsObj instanceof Map) { - return parseFromMap((Map) tagsObj); - } else if (tagsObj instanceof List) { - return parseFromList((List) tagsObj); - } - - return new ArrayList<>(); - } - private DynamicTag(String tagName, String beanName, String attributeName) { this.tagName = tagName; this.beanName = beanName; @@ -159,13 +63,8 @@ public String getAttributeName() { return attributeName; } - /** - * Resolves the dynamic tag by fetching the actual value from the JMX server. - * - * @param connection the JMX connection to use - * @return a map entry with the tag name and resolved value, or null if resolution failed - */ - public Map.Entry resolve(Connection connection) { + /** Resolve the dynamic tag by fetching the attribute value from JMX. */ + public Map.Entry resolve(MBeanServerConnection connection) { try { ObjectName objectName = new ObjectName(beanName); Object value = connection.getAttribute(objectName, attributeName); @@ -186,7 +85,8 @@ public Map.Entry resolve(Connection connection) { log.error("Invalid bean name '{}' for dynamic tag '{}': {}", beanName, tagName, e.getMessage()); return null; - } catch (Exception e) { + } catch (AttributeNotFoundException | InstanceNotFoundException + | MBeanException | ReflectionException | IOException e) { log.warn("Failed to resolve dynamic tag '{}' from bean '{}' attribute '{}': {}", tagName, beanName, attributeName, e.getMessage()); log.debug("Dynamic tag resolution error details", e); @@ -194,14 +94,9 @@ public Map.Entry resolve(Connection connection) { } } - /** - * Resolves multiple dynamic tags using the provided connection. - * - * @param dynamicTags list of dynamic tags to resolve - * @param connection the JMX connection to use - * @return map of resolved tag names to values - */ - public static Map resolveAll(List dynamicTags, Connection connection) { + /** Resolve multiple dynamic tags at once. */ + public static Map resolveAll( + List dynamicTags, MBeanServerConnection connection) { Map resolvedTags = new HashMap<>(); if (dynamicTags == null || dynamicTags.isEmpty()) { diff --git a/src/main/java/org/datadog/jmxfetch/Filter.java b/src/main/java/org/datadog/jmxfetch/Filter.java index eb886ed9..2039b80d 100644 --- a/src/main/java/org/datadog/jmxfetch/Filter.java +++ b/src/main/java/org/datadog/jmxfetch/Filter.java @@ -16,6 +16,8 @@ class Filter { List beanRegexes = null; List excludeTags = null; Map additionalTags = null; + List dynamicTags = null; + boolean tagsParsed = false; /** * A simple class to manipulate include/exclude filter elements more easily A filter may @@ -120,18 +122,53 @@ public List getExcludeTags() { return this.excludeTags; } - public Map getAdditionalTags() { - // Return additional tags - if (this.additionalTags == null) { - if (filter.get("tags") == null) { - this.additionalTags = new HashMap(); + private void parseTags() { + if (tagsParsed) { + return; + } + + tagsParsed = true; + this.additionalTags = new HashMap(); + this.dynamicTags = new ArrayList(); + + if (filter.get("tags") == null) { + return; + } + + Map allTags = (Map) filter.get("tags"); + + for (Map.Entry entry : allTags.entrySet()) { + String tagName = entry.getKey(); + String tagValue = entry.getValue(); + + if (tagValue != null && tagValue.contains("#") && tagValue.startsWith("$")) { + try { + DynamicTag dynamicTag = DynamicTag.parse(tagName, tagValue); + this.dynamicTags.add(dynamicTag); + } catch (Exception e) { + this.additionalTags.put(tagName, tagValue); + } } else { - this.additionalTags = (Map) filter.get("tags"); + this.additionalTags.put(tagName, tagValue); } } - + } + + public Map getAdditionalTags() { + if (this.additionalTags == null) { + parseTags(); + } + return this.additionalTags; } + + public List getDynamicTags() { + if (this.dynamicTags == null) { + parseTags(); + } + + return this.dynamicTags; + } public String getDomain() { return (String) filter.get("domain"); diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index 6f0fbf0b..3c25cf27 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -79,7 +79,6 @@ public Yaml initialValue() { private ObjectName instanceTelemetryBeanName; private MBeanServer mbs; private Boolean normalizeBeanParamTags; - private List dynamicTags; /** Constructor, instantiates Instance based of a previous instance and appConfig. */ public Instance(Instance instance, AppConfig appConfig) { @@ -108,10 +107,6 @@ public Instance( instanceMap != null ? new HashMap(instanceMap) : null; this.initConfig = initConfig != null ? new HashMap(initConfig) : null; this.instanceName = (String) instanceMap.get("name"); - - // Parse dynamic tags before processing regular tags - this.dynamicTags = DynamicTag.parseFromObject(instanceMap.get("tags")); - this.tags = getTagsMap(instanceMap.get("tags"), appConfig); this.checkName = checkName; this.matchingAttributes = new ArrayList(); @@ -392,7 +387,6 @@ static void loadMetricConfigResources( /** * Format the instance tags defined in the YAML configuration file to a `HashMap`. * Supported inputs: `List`, `Map`. - * Dynamic tags (starting with $) are skipped here and will be resolved after connection. */ private static Map getTagsMap(Object tagsMap, AppConfig appConfig) { Map tags = new HashMap(); @@ -401,27 +395,9 @@ private static Map getTagsMap(Object tagsMap, AppConfig appConfi } if (tagsMap != null) { if (tagsMap instanceof Map) { - // Add non-dynamic tags, skip dynamic tags (they'll be resolved later) - for (Map.Entry entry : ((Map) tagsMap).entrySet()) { - if (entry.getValue() == null || !entry.getValue().startsWith("$")) { - tags.put(entry.getKey(), entry.getValue()); - } else { - log.debug("Skipping dynamic tag '{}' in initial tag parsing, " - + "will be resolved after connection", entry.getKey()); - } - } + tags.putAll((Map) tagsMap); } else if (tagsMap instanceof List) { for (String tag : (List) tagsMap) { - // Check if this is a dynamic tag (contains : and value starts with $) - int colonIndex = tag.indexOf(':'); - if (colonIndex > 0) { - String tagValue = tag.substring(colonIndex + 1); - if (tagValue.startsWith("$")) { - log.debug("Skipping dynamic tag '{}' in initial tag parsing, " - + "will be resolved after connection", tag); - continue; - } - } tags.put(tag, null); } } else { @@ -472,8 +448,8 @@ public void init(boolean forceNewConnection) log.info("Trying to connect to JMX Server at " + this.toString()); connection = getConnection(instanceMap, forceNewConnection); - // Resolve dynamic tags after connection is established - resolveDynamicTags(); + // Resolve configuration-level dynamic tags for all configurations + resolveConfigurationDynamicTags(); log.info( "Trying to collect bean list for the first time for JMX Server at {}", this); @@ -484,26 +460,44 @@ public void init(boolean forceNewConnection) log.info("Done initializing JMX Server at {}", this); } - /** - * Resolves dynamic tags by fetching their values from JMX beans. - * This method is called after the JMX connection is established. - */ - private void resolveDynamicTags() { - if (dynamicTags == null || dynamicTags.isEmpty()) { + private void resolveConfigurationDynamicTags() { + if (configurationList == null || configurationList.isEmpty()) { + return; + } + + Map> cache = new HashMap<>(); + List allDynamicTags = new ArrayList<>(); + + for (Configuration config : configurationList) { + Filter include = config.getInclude(); + if (include != null) { + List dynamicTags = include.getDynamicTags(); + if (dynamicTags != null && !dynamicTags.isEmpty()) { + allDynamicTags.addAll(dynamicTags); + } + } + } + + if (allDynamicTags.isEmpty()) { return; } - log.info("Resolving {} dynamic tag(s) for instance {}", dynamicTags.size(), instanceName); + for (DynamicTag dynamicTag : allDynamicTags) { + String cacheKey = dynamicTag.getBeanName() + "#" + dynamicTag.getAttributeName(); + if (!cache.containsKey(cacheKey)) { + Map.Entry resolved = + dynamicTag.resolve(connection.getMBeanServerConnection()); + if (resolved != null) { + cache.put(cacheKey, resolved); + } + } + } - Map resolvedTags = DynamicTag.resolveAll(dynamicTags, connection); + log.info("Resolved {} unique dynamic tag(s) from {} total references for instance {}", + cache.size(), allDynamicTags.size(), instanceName); - if (!resolvedTags.isEmpty()) { - // Add resolved tags to the instance tags - this.tags.putAll(resolvedTags); - log.info("Successfully resolved {} dynamic tag(s) for instance {}: {}", - resolvedTags.size(), instanceName, resolvedTags); - } else { - log.warn("No dynamic tags could be resolved for instance {}", instanceName); + for (Configuration config : configurationList) { + config.resolveDynamicTags(connection.getMBeanServerConnection(), cache); } } diff --git a/src/main/java/org/datadog/jmxfetch/JmxAttribute.java b/src/main/java/org/datadog/jmxfetch/JmxAttribute.java index 66755ea2..3a762fac 100644 --- a/src/main/java/org/datadog/jmxfetch/JmxAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JmxAttribute.java @@ -138,6 +138,13 @@ private void addAdditionalTags() { } } } + + Map resolvedDynamicTags = this.matchingConf.getResolvedDynamicTags(); + if (resolvedDynamicTags != null && !resolvedDynamicTags.isEmpty()) { + for (Map.Entry tag : resolvedDynamicTags.entrySet()) { + this.defaultTagsList.add(tag.getKey() + ":" + tag.getValue()); + } + } } private void addServiceTags() { diff --git a/src/test/java/org/datadog/jmxfetch/DynamicTagTestApp.java b/src/test/java/org/datadog/jmxfetch/DynamicTagTestApp.java index 8cb21dac..95b6fc1f 100644 --- a/src/test/java/org/datadog/jmxfetch/DynamicTagTestApp.java +++ b/src/test/java/org/datadog/jmxfetch/DynamicTagTestApp.java @@ -1,9 +1,5 @@ package org.datadog.jmxfetch; -/** - * Test application for dynamic tag resolution. - * Exposes various attributes that can be used as dynamic tags. - */ public class DynamicTagTestApp implements DynamicTagTestAppMBean { private final String clusterId; private final String version; diff --git a/src/test/java/org/datadog/jmxfetch/DynamicTagTestAppMBean.java b/src/test/java/org/datadog/jmxfetch/DynamicTagTestAppMBean.java index db5df189..34483152 100644 --- a/src/test/java/org/datadog/jmxfetch/DynamicTagTestAppMBean.java +++ b/src/test/java/org/datadog/jmxfetch/DynamicTagTestAppMBean.java @@ -1,8 +1,5 @@ package org.datadog.jmxfetch; -/** - * MBean interface for testing dynamic tag resolution. - */ public interface DynamicTagTestAppMBean { String getClusterId(); String getVersion(); diff --git a/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java b/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java new file mode 100644 index 00000000..0e1c2897 --- /dev/null +++ b/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java @@ -0,0 +1,297 @@ +package org.datadog.jmxfetch; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class TestConfigDynamicTags extends TestCommon { + + @Test + public void testConfigDynamicTagsBasic() throws Exception { + registerMBean( + new DynamicTagTestApp("kafka-prod-cluster", "3.2.0", 9092), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); + + initApplicationWithYamlLines( + "init_config:", + "instances:", + " - process_name_regex: '.*surefire.*'", + " name: jmx_config_dynamic_tags_test", + " conf:", + " - include:", + " domain: org.datadog.jmxfetch.test", + " type: DynamicTagTestApp", + " tags:", + " env: test", + " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId", + " kafka_version: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#Version", + " attribute:", + " Metric:", + " metric_type: gauge", + " alias: test.config.dynamic.tags.metric" + ); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + boolean foundMetric = false; + for (Map metric : metrics) { + String metricName = (String) metric.get("name"); + if ("test.config.dynamic.tags.metric".equals(metricName)) { + foundMetric = true; + + Object tagsObj = metric.get("tags"); + assertNotNull("Metric should have tags", tagsObj); + + List tagList = new ArrayList<>(); + if (tagsObj instanceof String[]) { + for (String tag : (String[]) tagsObj) { + tagList.add(tag); + } + } else if (tagsObj instanceof List) { + for (Object tag : (List) tagsObj) { + tagList.add((String) tag); + } + } + + assertTrue("Should have cluster_id tag", + tagList.contains("cluster_id:kafka-prod-cluster")); + assertTrue("Should have kafka_version tag", + tagList.contains("kafka_version:3.2.0")); + assertTrue("Should have env tag", + tagList.contains("env:test")); + + break; + } + } + + assertTrue("Should have found the test metric", foundMetric); + } + + @Test + public void testConfigDynamicTagsMultiple() throws Exception { + registerMBean( + new DynamicTagTestApp("cluster-1", "version-1", 9092), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1"); + + registerMBean( + new DynamicTagTestApp("cluster-2", "version-2", 9093), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2"); + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); + + initApplicationWithYamlLines( + "init_config:", + "instances:", + " - process_name_regex: '.*surefire.*'", + " name: jmx_config_dynamic_tags_multi_test", + " conf:", + " - include:", + " domain: org.datadog.jmxfetch.test", + " type: DynamicTagTestApp", + " name: Instance1", + " tags:", + " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1#ClusterId", + " attribute:", + " Metric:", + " metric_type: gauge", + " alias: test.instance1.metric", + " - include:", + " domain: org.datadog.jmxfetch.test", + " type: DynamicTagTestApp", + " name: Instance2", + " tags:", + " version: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2#Version", + " attribute:", + " Metric:", + " metric_type: gauge", + " alias: test.instance2.metric" + ); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + boolean foundInstance1 = false; + boolean foundInstance2 = false; + + for (Map metric : metrics) { + String metricName = (String) metric.get("name"); + + if ("test.instance1.metric".equals(metricName)) { + foundInstance1 = true; + + List tagList = getTagsAsList(metric); + assertTrue("Instance1 should have cluster_id tag", + tagList.contains("cluster_id:cluster-1")); + + boolean hasVersionTag = false; + for (String tag : tagList) { + if (tag.startsWith("version:")) { + hasVersionTag = true; + break; + } + } + assertTrue("Instance1 should NOT have version tag", !hasVersionTag); + } + + if ("test.instance2.metric".equals(metricName)) { + foundInstance2 = true; + + List tagList = getTagsAsList(metric); + assertTrue("Instance2 should have version tag", + tagList.contains("version:version-2")); + + boolean hasClusterTag = false; + for (String tag : tagList) { + if (tag.startsWith("cluster_id:")) { + hasClusterTag = true; + break; + } + } + assertTrue("Instance2 should NOT have cluster_id tag", !hasClusterTag); + } + } + + assertTrue("Should have found instance1 metric", foundInstance1); + assertTrue("Should have found instance2 metric", foundInstance2); + } + + @Test + public void testConfigDynamicTagsWithBeanParams() throws Exception { + registerMBean( + new DynamicTagTestApp("test-cluster", "1.0.0", 9092), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean"); + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); + + initApplicationWithYamlLines( + "init_config:", + "instances:", + " - process_name_regex: '.*surefire.*'", + " name: jmx_config_dynamic_tags_bean_params_test", + " conf:", + " - include:", + " domain: org.datadog.jmxfetch.test", + " type: DynamicTagTestApp", + " tags:", + " bean_type: $type", + " bean_name: $name", + " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean#ClusterId", + " attribute:", + " Metric:", + " metric_type: gauge", + " alias: test.bean.params.metric" + ); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + boolean foundMetric = false; + for (Map metric : metrics) { + String metricName = (String) metric.get("name"); + if ("test.bean.params.metric".equals(metricName)) { + foundMetric = true; + + List tagList = getTagsAsList(metric); + + assertTrue("Should have bean_type tag", + tagList.contains("bean_type:DynamicTagTestApp")); + assertTrue("Should have bean_name tag", + tagList.contains("bean_name:TestBean")); + assertTrue("Should have cluster_id tag", + tagList.contains("cluster_id:test-cluster")); + + break; + } + } + + assertTrue("Should have found the test metric", foundMetric); + } + + @Test + public void testConfigDynamicTagsCaching() throws Exception { + registerMBean( + new DynamicTagTestApp("shared-cluster", "1.0.0", 9092), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); + registerMBean( + new SimpleTestJavaApp(), + "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); + + initApplicationWithYamlLines( + "init_config:", + "instances:", + " - process_name_regex: '.*surefire.*'", + " name: jmx_config_dynamic_tags_caching_test", + " conf:", + " - include:", + " domain: org.datadog.jmxfetch.test", + " type: DynamicTagTestApp", + " tags:", + " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId", + " attribute:", + " Metric:", + " metric_type: gauge", + " alias: test.metric1", + " - include:", + " domain: org.datadog.jmxfetch.test", + " type: DynamicTagTestApp", + " tags:", + " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId", + " attribute:", + " Port:", + " metric_type: gauge", + " alias: test.metric2" + ); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + int metricsWithClusterId = 0; + for (Map metric : metrics) { + String metricName = (String) metric.get("name"); + if (metricName != null && metricName.startsWith("test.metric")) { + List tagList = getTagsAsList(metric); + if (tagList.contains("cluster_id:shared-cluster")) { + metricsWithClusterId++; + } + } + } + + assertTrue("Found " + metricsWithClusterId + " metrics with cluster_id, expected 2", + metricsWithClusterId == 2); + } + + private List getTagsAsList(Map metric) { + List tagList = new ArrayList<>(); + Object tagsObj = metric.get("tags"); + if (tagsObj instanceof String[]) { + for (String tag : (String[]) tagsObj) { + tagList.add(tag); + } + } else if (tagsObj instanceof List) { + for (Object tag : (List) tagsObj) { + tagList.add((String) tag); + } + } + return tagList; + } +} + diff --git a/src/test/java/org/datadog/jmxfetch/TestDynamicTags.java b/src/test/java/org/datadog/jmxfetch/TestDynamicTags.java deleted file mode 100644 index dff2041d..00000000 --- a/src/test/java/org/datadog/jmxfetch/TestDynamicTags.java +++ /dev/null @@ -1,297 +0,0 @@ -package org.datadog.jmxfetch; - -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -/** - * Tests for dynamic tag resolution from JMX bean attributes. - */ -public class TestDynamicTags extends TestCommon { - - /** - * Test basic dynamic tag resolution with list-style tags. - */ - @Test - public void testDynamicTagsBasic() throws Exception { - // Register a test MBean with attributes we'll use as tags - registerMBean( - new DynamicTagTestApp("prod-kafka-cluster", "3.0.0", 9092), - "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); - - // Also register the SimpleTestJavaApp for default JVM metrics - registerMBean( - new SimpleTestJavaApp(), - "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); - - initApplication("jmx_dynamic_tags.yaml"); - - // Run the collection - run(); - - List> metrics = getMetrics(); - assertNotNull(metrics); - assertTrue("Should have collected metrics", metrics.size() > 0); - - // Verify that our test metric has the dynamic tags - List expectedTags = Arrays.asList( - "env:test", - "cluster_id:prod-kafka-cluster", - "kafka_version:3.0.0", - "instance:jmx_dynamic_tags_test", - "jmx_domain:org.datadog.jmxfetch.test", - "dd.internal.jmx_check_name:jmx_dynamic_tags", - "type:DynamicTagTestApp" - ); - - assertMetric("test.dynamic.tags.metric", 100.0, expectedTags, 7); - } - - /** - * Test dynamic tag resolution with "attribute." prefix in the reference. - */ - @Test - public void testDynamicTagsWithAttributePrefix() throws Exception { - registerMBean( - new DynamicTagTestApp("dev-kafka-cluster", "2.8.0", 9093), - "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); - - registerMBean( - new SimpleTestJavaApp(), - "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); - - initApplication("jmx_dynamic_tags_with_attribute_prefix.yaml"); - - run(); - - List> metrics = getMetrics(); - assertTrue("Should have collected metrics", metrics.size() > 0); - - // Verify the dynamic tag was resolved correctly - List expectedTags = Arrays.asList( - "env:test", - "cluster_id:dev-kafka-cluster", - "instance:jmx_dynamic_tags_attribute_test", - "jmx_domain:org.datadog.jmxfetch.test", - "dd.internal.jmx_check_name:jmx_dynamic_tags_with_attribute_prefix", - "type:DynamicTagTestApp" - ); - - assertMetric("test.dynamic.tags.metric", 100.0, expectedTags, 6); - } - - /** - * Test dynamic tag resolution with map-style tags. - */ - @Test - public void testDynamicTagsMapStyle() throws Exception { - registerMBean( - new DynamicTagTestApp("staging-cluster", "3.1.0", 9094), - "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); - - registerMBean( - new SimpleTestJavaApp(), - "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); - - initApplication("jmx_dynamic_tags_map_style.yaml"); - - run(); - - List> metrics = getMetrics(); - assertTrue("Should have collected metrics", metrics.size() > 0); - - // Verify both static and dynamic tags are present - List expectedTags = Arrays.asList( - "env:test", - "cluster_id:staging-cluster", - "static_tag:static_value", - "instance:jmx_dynamic_tags_map_test", - "jmx_domain:org.datadog.jmxfetch.test", - "dd.internal.jmx_check_name:jmx_dynamic_tags_map_style", - "type:DynamicTagTestApp" - ); - - assertMetric("test.dynamic.tags.metric", 100.0, expectedTags, 7); - } - - /** - * Test that metrics are still collected when a dynamic tag reference fails to resolve. - * The tag should simply be absent rather than causing the entire instance to fail. - */ - @Test - public void testDynamicTagsNonExistentBean() throws Exception { - // Only register the app bean, not the bean referenced in the dynamic tag - registerMBean( - new DynamicTagTestApp("test-cluster", "1.0.0", 9095), - "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); - - registerMBean( - new SimpleTestJavaApp(), - "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); - - initApplication("jmx_dynamic_tags_nonexistent.yaml"); - - run(); - - List> metrics = getMetrics(); - assertTrue("Should have collected metrics despite failed dynamic tag resolution", - metrics.size() > 0); - - // The metric should be collected, but without the failed dynamic tag - List expectedTags = Arrays.asList( - "env:test", - "instance:jmx_dynamic_tags_nonexistent_test", - "jmx_domain:org.datadog.jmxfetch.test", - "dd.internal.jmx_check_name:jmx_dynamic_tags_nonexistent", - "type:DynamicTagTestApp" - ); - - // Note: cluster_id tag should NOT be present since the bean doesn't exist - assertMetric("test.dynamic.tags.metric", 100.0, expectedTags, 5); - } - - /** - * Test that dynamic tags work with different attribute types (integer). - */ - @Test - public void testDynamicTagsWithIntegerAttribute() throws Exception { - registerMBean( - new DynamicTagTestApp("int-test-cluster", "1.0.0", 8888), - "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); - - registerMBean( - new SimpleTestJavaApp(), - "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); - - initApplicationWithYamlLines( - "init_config:", - "", - "instances:", - " - process_name_regex: .*surefire.*", - " name: jmx_dynamic_tags_int_test", - " tags:", - " - env:test", - " - port:$org.datadog.jmxfetch.test:type=DynamicTagTestApp#Port", - " conf:", - " - include:", - " domain: org.datadog.jmxfetch.test", - " type: DynamicTagTestApp", - " attribute:", - " Metric:", - " metric_type: gauge", - " alias: test.dynamic.tags.metric" - ); - - run(); - - List> metrics = getMetrics(); - assertTrue("Should have collected metrics", metrics.size() > 0); - - // Verify the port (integer) was converted to a string tag value - // Note: dd.internal.jmx_check_name will be a generated config name since we're using initApplicationWithYamlLines - - // Find the test metric - boolean foundMetric = false; - for (Map metric : metrics) { - if ("test.dynamic.tags.metric".equals(metric.get("name"))) { - String[] tags = (String[]) metric.get("tags"); - List tagList = Arrays.asList(tags); - - // Check that required tags are present - assertTrue("Should have env:test tag", tagList.contains("env:test")); - assertTrue("Should have port:8888 tag", tagList.contains("port:8888")); - - // Check for instance tag - boolean hasInstanceTag = false; - for (String tag : tagList) { - if (tag.startsWith("instance:jmx_dynamic_tags_int_test")) { - hasInstanceTag = true; - break; - } - } - assertTrue("Should have instance tag", hasInstanceTag); - assertTrue("Should have type tag", tagList.contains("type:DynamicTagTestApp")); - foundMetric = true; - break; - } - } - assertTrue("Should have found test.dynamic.tags.metric", foundMetric); - } - - /** - * Test that multiple instances can have different dynamic tag values. - */ - @Test - public void testDynamicTagsMultipleInstances() throws Exception { - // Register two different instances with different cluster IDs - registerMBean( - new DynamicTagTestApp("cluster-1", "1.0.0", 9001), - "org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1"); - - registerMBean( - new DynamicTagTestApp("cluster-2", "2.0.0", 9002), - "org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2"); - - registerMBean( - new SimpleTestJavaApp(), - "org.datadog.jmxfetch.test:type=SimpleTestJavaApp"); - - initApplicationWithYamlLines( - "init_config:", - "", - "instances:", - " - process_name_regex: .*surefire.*", - " name: jmx_dynamic_tags_multi_test", - " tags:", - " - cluster_id_1:$org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1#ClusterId", - " - cluster_id_2:$org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2#ClusterId", - " conf:", - " - include:", - " domain: org.datadog.jmxfetch.test", - " bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1", - " attribute:", - " Metric:", - " metric_type: gauge", - " alias: test.instance1.metric" - ); - - run(); - - List> metrics = getMetrics(); - assertTrue("Should have collected metrics", metrics.size() > 0); - - // Find the test metric and verify both cluster IDs are present - boolean foundMetric = false; - for (Map metric : metrics) { - if ("test.instance1.metric".equals(metric.get("name"))) { - String[] tags = (String[]) metric.get("tags"); - List tagList = Arrays.asList(tags); - - // Check that both dynamic tags are present with correct values - assertTrue("Should have cluster_id_1 tag", tagList.contains("cluster_id_1:cluster-1")); - assertTrue("Should have cluster_id_2 tag", tagList.contains("cluster_id_2:cluster-2")); - - // Check for instance tag - boolean hasInstanceTag = false; - for (String tag : tagList) { - if (tag.startsWith("instance:jmx_dynamic_tags_multi_test")) { - hasInstanceTag = true; - break; - } - } - assertTrue("Should have instance tag", hasInstanceTag); - assertTrue("Should have type tag", tagList.contains("type:DynamicTagTestApp")); - assertTrue("Should have name tag", tagList.contains("name:Instance1")); - foundMetric = true; - break; - } - } - assertTrue("Should have found test.instance1.metric", foundMetric); - } -} - diff --git a/src/test/resources/jmx_config_dynamic_tags.yaml b/src/test/resources/jmx_config_dynamic_tags.yaml new file mode 100644 index 00000000..708d5d96 --- /dev/null +++ b/src/test/resources/jmx_config_dynamic_tags.yaml @@ -0,0 +1,18 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_config_dynamic_tags_test + conf: + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + tags: + env: test + cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId + kafka_version: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#Version + attribute: + Metric: + metric_type: gauge + alias: test.config.dynamic.tags.metric + diff --git a/src/test/resources/jmx_config_dynamic_tags_multi.yaml b/src/test/resources/jmx_config_dynamic_tags_multi.yaml new file mode 100644 index 00000000..ea317623 --- /dev/null +++ b/src/test/resources/jmx_config_dynamic_tags_multi.yaml @@ -0,0 +1,30 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_config_dynamic_tags_multi_test + conf: + # Config 1: Has cluster_id dynamic tag + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + name: Instance1 + tags: + cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1#ClusterId + attribute: + Metric: + metric_type: gauge + alias: test.instance1.metric + + # Config 2: Has version dynamic tag + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + name: Instance2 + tags: + version: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2#Version + attribute: + Metric: + metric_type: gauge + alias: test.instance2.metric + diff --git a/src/test/resources/jmx_dynamic_tags.yaml b/src/test/resources/jmx_dynamic_tags.yaml deleted file mode 100644 index 5e7bd360..00000000 --- a/src/test/resources/jmx_dynamic_tags.yaml +++ /dev/null @@ -1,19 +0,0 @@ -init_config: - -instances: - - process_name_regex: .*surefire.* - name: jmx_dynamic_tags_test - tags: - - env:test - - cluster_id:$org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId - - kafka_version:$org.datadog.jmxfetch.test:type=DynamicTagTestApp#Version - conf: - - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - attribute: - Metric: - metric_type: gauge - alias: test.dynamic.tags.metric - - diff --git a/src/test/resources/jmx_dynamic_tags_map_style.yaml b/src/test/resources/jmx_dynamic_tags_map_style.yaml deleted file mode 100644 index f41c1e12..00000000 --- a/src/test/resources/jmx_dynamic_tags_map_style.yaml +++ /dev/null @@ -1,19 +0,0 @@ -init_config: - -instances: - - process_name_regex: .*surefire.* - name: jmx_dynamic_tags_map_test - tags: - env: test - cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId - static_tag: static_value - conf: - - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - attribute: - Metric: - metric_type: gauge - alias: test.dynamic.tags.metric - - diff --git a/src/test/resources/jmx_dynamic_tags_nonexistent.yaml b/src/test/resources/jmx_dynamic_tags_nonexistent.yaml deleted file mode 100644 index fd46b409..00000000 --- a/src/test/resources/jmx_dynamic_tags_nonexistent.yaml +++ /dev/null @@ -1,18 +0,0 @@ -init_config: - -instances: - - process_name_regex: .*surefire.* - name: jmx_dynamic_tags_nonexistent_test - tags: - - env:test - - cluster_id:$org.datadog.jmxfetch.test:type=NonExistentBean#Value - conf: - - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - attribute: - Metric: - metric_type: gauge - alias: test.dynamic.tags.metric - - diff --git a/src/test/resources/jmx_dynamic_tags_with_attribute_prefix.yaml b/src/test/resources/jmx_dynamic_tags_with_attribute_prefix.yaml deleted file mode 100644 index fa1f2bbe..00000000 --- a/src/test/resources/jmx_dynamic_tags_with_attribute_prefix.yaml +++ /dev/null @@ -1,18 +0,0 @@ -init_config: - -instances: - - process_name_regex: .*surefire.* - name: jmx_dynamic_tags_attribute_test - tags: - - env:test - - cluster_id:$org.datadog.jmxfetch.test:type=DynamicTagTestApp#attribute.ClusterId - conf: - - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - attribute: - Metric: - metric_type: gauge - alias: test.dynamic.tags.metric - - From 2c6b68858cb2e3e3c334142c074c987a6aa62e10 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Wed, 15 Oct 2025 20:50:10 -0600 Subject: [PATCH 03/11] Use resources files for tests --- .../jmxfetch/TestConfigDynamicTags.java | 92 +------------------ .../jmx_config_dynamic_tags_bean_params.yaml | 19 ++++ .../jmx_config_dynamic_tags_caching.yaml | 29 ++++++ 3 files changed, 52 insertions(+), 88 deletions(-) create mode 100644 src/test/resources/jmx_config_dynamic_tags_bean_params.yaml create mode 100644 src/test/resources/jmx_config_dynamic_tags_caching.yaml diff --git a/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java b/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java index 0e1c2897..26aa37d8 100644 --- a/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java +++ b/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java @@ -20,24 +20,7 @@ public void testConfigDynamicTagsBasic() throws Exception { new SimpleTestJavaApp(), "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); - initApplicationWithYamlLines( - "init_config:", - "instances:", - " - process_name_regex: '.*surefire.*'", - " name: jmx_config_dynamic_tags_test", - " conf:", - " - include:", - " domain: org.datadog.jmxfetch.test", - " type: DynamicTagTestApp", - " tags:", - " env: test", - " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId", - " kafka_version: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#Version", - " attribute:", - " Metric:", - " metric_type: gauge", - " alias: test.config.dynamic.tags.metric" - ); + initApplication("jmx_config_dynamic_tags.yaml"); run(); @@ -91,33 +74,7 @@ public void testConfigDynamicTagsMultiple() throws Exception { new SimpleTestJavaApp(), "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); - initApplicationWithYamlLines( - "init_config:", - "instances:", - " - process_name_regex: '.*surefire.*'", - " name: jmx_config_dynamic_tags_multi_test", - " conf:", - " - include:", - " domain: org.datadog.jmxfetch.test", - " type: DynamicTagTestApp", - " name: Instance1", - " tags:", - " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1#ClusterId", - " attribute:", - " Metric:", - " metric_type: gauge", - " alias: test.instance1.metric", - " - include:", - " domain: org.datadog.jmxfetch.test", - " type: DynamicTagTestApp", - " name: Instance2", - " tags:", - " version: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2#Version", - " attribute:", - " Metric:", - " metric_type: gauge", - " alias: test.instance2.metric" - ); + initApplication("jmx_config_dynamic_tags_multi.yaml"); run(); @@ -178,24 +135,7 @@ public void testConfigDynamicTagsWithBeanParams() throws Exception { new SimpleTestJavaApp(), "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); - initApplicationWithYamlLines( - "init_config:", - "instances:", - " - process_name_regex: '.*surefire.*'", - " name: jmx_config_dynamic_tags_bean_params_test", - " conf:", - " - include:", - " domain: org.datadog.jmxfetch.test", - " type: DynamicTagTestApp", - " tags:", - " bean_type: $type", - " bean_name: $name", - " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean#ClusterId", - " attribute:", - " Metric:", - " metric_type: gauge", - " alias: test.bean.params.metric" - ); + initApplication("jmx_config_dynamic_tags_bean_params.yaml"); run(); @@ -233,31 +173,7 @@ public void testConfigDynamicTagsCaching() throws Exception { new SimpleTestJavaApp(), "org.datadog.jmxfetch.test:foo=Bar,qux=Baz"); - initApplicationWithYamlLines( - "init_config:", - "instances:", - " - process_name_regex: '.*surefire.*'", - " name: jmx_config_dynamic_tags_caching_test", - " conf:", - " - include:", - " domain: org.datadog.jmxfetch.test", - " type: DynamicTagTestApp", - " tags:", - " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId", - " attribute:", - " Metric:", - " metric_type: gauge", - " alias: test.metric1", - " - include:", - " domain: org.datadog.jmxfetch.test", - " type: DynamicTagTestApp", - " tags:", - " cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId", - " attribute:", - " Port:", - " metric_type: gauge", - " alias: test.metric2" - ); + initApplication("jmx_config_dynamic_tags_caching.yaml"); run(); diff --git a/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml b/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml new file mode 100644 index 00000000..3d7cf722 --- /dev/null +++ b/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml @@ -0,0 +1,19 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_config_dynamic_tags_bean_params_test + conf: + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + tags: + bean_type: $type + bean_name: $name + cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean#ClusterId + attribute: + Metric: + metric_type: gauge + alias: test.bean.params.metric + + diff --git a/src/test/resources/jmx_config_dynamic_tags_caching.yaml b/src/test/resources/jmx_config_dynamic_tags_caching.yaml new file mode 100644 index 00000000..70213eac --- /dev/null +++ b/src/test/resources/jmx_config_dynamic_tags_caching.yaml @@ -0,0 +1,29 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_config_dynamic_tags_caching_test + conf: + # Config 1: Uses cluster_id + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + tags: + cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId + attribute: + Metric: + metric_type: gauge + alias: test.metric1 + + # Config 2: Uses same cluster_id (should be cached) + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + tags: + cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId + attribute: + Port: + metric_type: gauge + alias: test.metric2 + + From 2ed60891c16d2958ff9a64372db85bdf2b6051ae Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Thu, 16 Oct 2025 10:57:24 -0600 Subject: [PATCH 04/11] address PR comments --- .../org/datadog/jmxfetch/Configuration.java | 12 ++-- .../java/org/datadog/jmxfetch/DynamicTag.java | 66 ++++++------------- .../java/org/datadog/jmxfetch/Filter.java | 31 ++++----- .../java/org/datadog/jmxfetch/Instance.java | 5 +- .../resources/jmx_config_dynamic_tags.yaml | 9 ++- .../jmx_config_dynamic_tags_bean_params.yaml | 5 +- .../jmx_config_dynamic_tags_caching.yaml | 12 ++-- .../jmx_config_dynamic_tags_multi.yaml | 12 ++-- 8 files changed, 74 insertions(+), 78 deletions(-) diff --git a/src/main/java/org/datadog/jmxfetch/Configuration.java b/src/main/java/org/datadog/jmxfetch/Configuration.java index 0f5341f4..5ae1b832 100644 --- a/src/main/java/org/datadog/jmxfetch/Configuration.java +++ b/src/main/java/org/datadog/jmxfetch/Configuration.java @@ -50,10 +50,14 @@ public String toString() { private Boolean hasInclude() { return getInclude() != null && !getInclude().isEmptyFilter(); } - - /** Resolve dynamic tags for this configuration using cached values. */ - public void resolveDynamicTags( - MBeanServerConnection connection, Map> cache) { + + /** + * Resolves dynamic tags (bean name, attribute name) to the JMX bean value, + * and stores the result in resolvedDynamicTags. + * + * @param cache shared cache mapping "(beanName, attributeName)" to "(tagName, tagValue)" + */ + public void resolveDynamicTags(Map> cache) { if (resolvedDynamicTags != null) { return; } diff --git a/src/main/java/org/datadog/jmxfetch/DynamicTag.java b/src/main/java/org/datadog/jmxfetch/DynamicTag.java index b83f354c..1777fd25 100644 --- a/src/main/java/org/datadog/jmxfetch/DynamicTag.java +++ b/src/main/java/org/datadog/jmxfetch/DynamicTag.java @@ -2,45 +2,41 @@ import lombok.extern.slf4j.Slf4j; -import java.io.IOException; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import javax.management.AttributeNotFoundException; -import javax.management.InstanceNotFoundException; -import javax.management.MBeanException; -import javax.management.MBeanServerConnection; import javax.management.MalformedObjectNameException; import javax.management.ObjectName; -import javax.management.ReflectionException; @Slf4j public class DynamicTag { - private static final Pattern DYNAMIC_TAG_PATTERN = - Pattern.compile("^\\$([^#]+)#(?:attribute\\.)?(.+)$"); - private final String tagName; private final String beanName; private final String attributeName; - /** Parse a dynamic tag from a tag key and value. */ - public static DynamicTag parse(String tagKey, String tagValue) { - if (tagValue == null || !tagValue.startsWith("$")) { + /** Parse dynamic tag from configuration map. */ + public static DynamicTag parse(String tagKey, Object tagConfig) { + if (tagConfig == null) { + return null; + } + + if (!(tagConfig instanceof Map)) { + log.warn("Invalid dynamic tag config for '{}': expected map with 'bean' and " + + "'attribute' keys", tagKey); return null; } - Matcher matcher = DYNAMIC_TAG_PATTERN.matcher(tagValue); - if (!matcher.matches()) { - log.warn("Invalid dynamic tag format: {}. " - + "Expected format: $domain:bean_params#AttributeName", tagValue); + Map config = (Map) tagConfig; + Object beanObj = config.get("bean"); + Object attrObj = config.get("attribute"); + + if (beanObj == null || attrObj == null) { + log.warn("Invalid dynamic tag config for '{}': missing 'bean' or 'attribute' key", + tagKey); return null; } - String beanName = matcher.group(1); - String attributeName = matcher.group(2); + String beanName = beanObj.toString(); + String attributeName = attrObj.toString(); return new DynamicTag(tagKey, beanName, attributeName); } @@ -64,7 +60,7 @@ public String getAttributeName() { } /** Resolve the dynamic tag by fetching the attribute value from JMX. */ - public Map.Entry resolve(MBeanServerConnection connection) { + public Map.Entry resolve(Connection connection) { try { ObjectName objectName = new ObjectName(beanName); Object value = connection.getAttribute(objectName, attributeName); @@ -85,34 +81,14 @@ public Map.Entry resolve(MBeanServerConnection connection) { log.error("Invalid bean name '{}' for dynamic tag '{}': {}", beanName, tagName, e.getMessage()); return null; - } catch (AttributeNotFoundException | InstanceNotFoundException - | MBeanException | ReflectionException | IOException e) { + } catch (Exception e) { log.warn("Failed to resolve dynamic tag '{}' from bean '{}' attribute '{}': {}", tagName, beanName, attributeName, e.getMessage()); log.debug("Dynamic tag resolution error details", e); return null; } } - - /** Resolve multiple dynamic tags at once. */ - public static Map resolveAll( - List dynamicTags, MBeanServerConnection connection) { - Map resolvedTags = new HashMap<>(); - - if (dynamicTags == null || dynamicTags.isEmpty()) { - return resolvedTags; - } - - for (DynamicTag dynamicTag : dynamicTags) { - Map.Entry resolved = dynamicTag.resolve(connection); - if (resolved != null) { - resolvedTags.put(resolved.getKey(), resolved.getValue()); - } - } - - return resolvedTags; - } - + @Override public String toString() { return String.format("DynamicTag{name='%s', bean='%s', attribute='%s'}", diff --git a/src/main/java/org/datadog/jmxfetch/Filter.java b/src/main/java/org/datadog/jmxfetch/Filter.java index 2039b80d..b806fd90 100644 --- a/src/main/java/org/datadog/jmxfetch/Filter.java +++ b/src/main/java/org/datadog/jmxfetch/Filter.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -41,7 +42,9 @@ public String toString() { } public Set keySet() { - return filter.keySet(); + Set keys = new HashSet<>(filter.keySet()); + keys.remove("dynamic_tags"); + return keys; } @SuppressWarnings({"unchecked"}) @@ -131,25 +134,23 @@ private void parseTags() { this.additionalTags = new HashMap(); this.dynamicTags = new ArrayList(); - if (filter.get("tags") == null) { - return; + if (filter.get("tags") != null) { + Map allTags = (Map) filter.get("tags"); + this.additionalTags.putAll(allTags); } - Map allTags = (Map) filter.get("tags"); - - for (Map.Entry entry : allTags.entrySet()) { - String tagName = entry.getKey(); - String tagValue = entry.getValue(); + if (filter.get("dynamic_tags") != null) { + Map dynamicTagsConfig = + (Map) filter.get("dynamic_tags"); - if (tagValue != null && tagValue.contains("#") && tagValue.startsWith("$")) { - try { - DynamicTag dynamicTag = DynamicTag.parse(tagName, tagValue); + for (Map.Entry entry : dynamicTagsConfig.entrySet()) { + String tagName = entry.getKey(); + Object tagConfig = entry.getValue(); + + DynamicTag dynamicTag = DynamicTag.parse(tagName, tagConfig); + if (dynamicTag != null) { this.dynamicTags.add(dynamicTag); - } catch (Exception e) { - this.additionalTags.put(tagName, tagValue); } - } else { - this.additionalTags.put(tagName, tagValue); } } } diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index 3c25cf27..05e0ab04 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -485,8 +485,7 @@ private void resolveConfigurationDynamicTags() { for (DynamicTag dynamicTag : allDynamicTags) { String cacheKey = dynamicTag.getBeanName() + "#" + dynamicTag.getAttributeName(); if (!cache.containsKey(cacheKey)) { - Map.Entry resolved = - dynamicTag.resolve(connection.getMBeanServerConnection()); + Map.Entry resolved = dynamicTag.resolve(connection); if (resolved != null) { cache.put(cacheKey, resolved); } @@ -497,7 +496,7 @@ private void resolveConfigurationDynamicTags() { cache.size(), allDynamicTags.size(), instanceName); for (Configuration config : configurationList) { - config.resolveDynamicTags(connection.getMBeanServerConnection(), cache); + config.resolveDynamicTags(cache); } } diff --git a/src/test/resources/jmx_config_dynamic_tags.yaml b/src/test/resources/jmx_config_dynamic_tags.yaml index 708d5d96..c1619bf2 100644 --- a/src/test/resources/jmx_config_dynamic_tags.yaml +++ b/src/test/resources/jmx_config_dynamic_tags.yaml @@ -9,8 +9,13 @@ instances: type: DynamicTagTestApp tags: env: test - cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId - kafka_version: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#Version + dynamic_tags: + cluster_id: + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: ClusterId + kafka_version: + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: Version attribute: Metric: metric_type: gauge diff --git a/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml b/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml index 3d7cf722..892b685f 100644 --- a/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml +++ b/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml @@ -10,7 +10,10 @@ instances: tags: bean_type: $type bean_name: $name - cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean#ClusterId + dynamic_tags: + cluster_id: + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean + attribute: ClusterId attribute: Metric: metric_type: gauge diff --git a/src/test/resources/jmx_config_dynamic_tags_caching.yaml b/src/test/resources/jmx_config_dynamic_tags_caching.yaml index 70213eac..4b81b309 100644 --- a/src/test/resources/jmx_config_dynamic_tags_caching.yaml +++ b/src/test/resources/jmx_config_dynamic_tags_caching.yaml @@ -8,8 +8,10 @@ instances: - include: domain: org.datadog.jmxfetch.test type: DynamicTagTestApp - tags: - cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId + dynamic_tags: + cluster_id: + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: ClusterId attribute: Metric: metric_type: gauge @@ -19,8 +21,10 @@ instances: - include: domain: org.datadog.jmxfetch.test type: DynamicTagTestApp - tags: - cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp#ClusterId + dynamic_tags: + cluster_id: + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: ClusterId attribute: Port: metric_type: gauge diff --git a/src/test/resources/jmx_config_dynamic_tags_multi.yaml b/src/test/resources/jmx_config_dynamic_tags_multi.yaml index ea317623..8f57094f 100644 --- a/src/test/resources/jmx_config_dynamic_tags_multi.yaml +++ b/src/test/resources/jmx_config_dynamic_tags_multi.yaml @@ -9,8 +9,10 @@ instances: domain: org.datadog.jmxfetch.test type: DynamicTagTestApp name: Instance1 - tags: - cluster_id: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1#ClusterId + dynamic_tags: + cluster_id: + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1 + attribute: ClusterId attribute: Metric: metric_type: gauge @@ -21,8 +23,10 @@ instances: domain: org.datadog.jmxfetch.test type: DynamicTagTestApp name: Instance2 - tags: - version: $org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2#Version + dynamic_tags: + version: + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2 + attribute: Version attribute: Metric: metric_type: gauge From 19f1c5660018f8eaac07b48fe1f04e05ab8c53fa Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Thu, 16 Oct 2025 21:31:35 -0600 Subject: [PATCH 05/11] Move dynamic tags to same level as include --- .../org/datadog/jmxfetch/Configuration.java | 46 ++++++++++++++++--- .../java/org/datadog/jmxfetch/DynamicTag.java | 17 +++---- .../java/org/datadog/jmxfetch/Filter.java | 30 +----------- .../java/org/datadog/jmxfetch/Instance.java | 9 ++-- .../resources/jmx_config_dynamic_tags.yaml | 30 ++++++------ .../jmx_config_dynamic_tags_bean_params.yaml | 26 +++++------ .../jmx_config_dynamic_tags_caching.yaml | 40 ++++++++-------- .../jmx_config_dynamic_tags_multi.yaml | 44 +++++++++--------- 8 files changed, 123 insertions(+), 119 deletions(-) diff --git a/src/main/java/org/datadog/jmxfetch/Configuration.java b/src/main/java/org/datadog/jmxfetch/Configuration.java index 5ae1b832..805088df 100644 --- a/src/main/java/org/datadog/jmxfetch/Configuration.java +++ b/src/main/java/org/datadog/jmxfetch/Configuration.java @@ -10,7 +10,6 @@ import java.util.Map; import java.util.Map.Entry; import java.util.Set; -import javax.management.MBeanServerConnection; @Slf4j public class Configuration { @@ -18,6 +17,7 @@ public class Configuration { private Map conf; private Filter include; private Filter exclude; + private List dynamicTags = null; private Map resolvedDynamicTags = null; /** @@ -29,6 +29,37 @@ public Configuration(Map conf) { this.conf = conf; this.include = new Filter(conf.get("include")); this.exclude = new Filter(conf.get("exclude")); + this.parseDynamicTags(conf.get("dynamic_tags")); + } + + /** + * Parse dynamic tags from configuration. + * Expected format: + * dynamic_tags: + * - tag_name: cluster_id + * bean: kafka.server:type=KafkaServer,name=ClusterId + * attribute: Value + */ + private void parseDynamicTags(Object dynamicTagsConfig) { + this.dynamicTags = new ArrayList(); + + if (dynamicTagsConfig == null) { + return; + } + + if (!(dynamicTagsConfig instanceof List)) { + log.warn("Invalid dynamic_tags configuration: expected list of tag definitions"); + return; + } + + List dynamicTagsList = (List) dynamicTagsConfig; + + for (Object tagConfig : dynamicTagsList) { + DynamicTag dynamicTag = DynamicTag.parse(tagConfig); + if (dynamicTag != null) { + this.dynamicTags.add(dynamicTag); + } + } } public Map getConf() { @@ -64,11 +95,6 @@ public void resolveDynamicTags(Map> cache) { resolvedDynamicTags = new HashMap(); - if (include == null) { - return; - } - - List dynamicTags = include.getDynamicTags(); if (dynamicTags == null || dynamicTags.isEmpty()) { return; } @@ -85,6 +111,14 @@ public void resolveDynamicTags(Map> cache) { resolvedDynamicTags.size()); } + /** Get list of dynamic tags defined for this configuration. */ + public List getDynamicTags() { + if (dynamicTags == null) { + return new ArrayList(); + } + return dynamicTags; + } + /** Get all resolved dynamic tags for this configuration. */ public Map getResolvedDynamicTags() { if (resolvedDynamicTags == null) { diff --git a/src/main/java/org/datadog/jmxfetch/DynamicTag.java b/src/main/java/org/datadog/jmxfetch/DynamicTag.java index 1777fd25..c8b09ce0 100644 --- a/src/main/java/org/datadog/jmxfetch/DynamicTag.java +++ b/src/main/java/org/datadog/jmxfetch/DynamicTag.java @@ -13,32 +13,33 @@ public class DynamicTag { private final String beanName; private final String attributeName; - /** Parse dynamic tag from configuration map. */ - public static DynamicTag parse(String tagKey, Object tagConfig) { + /** Parse dynamic tag from configuration map (list entry format). */ + public static DynamicTag parse(Object tagConfig) { if (tagConfig == null) { return null; } if (!(tagConfig instanceof Map)) { - log.warn("Invalid dynamic tag config for '{}': expected map with 'bean' and " - + "'attribute' keys", tagKey); + log.warn("Invalid dynamic tag config: expected map with 'tag_name', 'bean' and " + + "'attribute' keys"); return null; } Map config = (Map) tagConfig; + Object tagNameObj = config.get("tag_name"); Object beanObj = config.get("bean"); Object attrObj = config.get("attribute"); - if (beanObj == null || attrObj == null) { - log.warn("Invalid dynamic tag config for '{}': missing 'bean' or 'attribute' key", - tagKey); + if (tagNameObj == null || beanObj == null || attrObj == null) { + log.warn("Invalid dynamic tag config: missing 'tag_name', 'bean' or 'attribute' key"); return null; } + String tagName = tagNameObj.toString(); String beanName = beanObj.toString(); String attributeName = attrObj.toString(); - return new DynamicTag(tagKey, beanName, attributeName); + return new DynamicTag(tagName, beanName, attributeName); } private DynamicTag(String tagName, String beanName, String attributeName) { diff --git a/src/main/java/org/datadog/jmxfetch/Filter.java b/src/main/java/org/datadog/jmxfetch/Filter.java index b806fd90..ce219ffd 100644 --- a/src/main/java/org/datadog/jmxfetch/Filter.java +++ b/src/main/java/org/datadog/jmxfetch/Filter.java @@ -4,7 +4,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -17,7 +16,6 @@ class Filter { List beanRegexes = null; List excludeTags = null; Map additionalTags = null; - List dynamicTags = null; boolean tagsParsed = false; /** @@ -42,9 +40,7 @@ public String toString() { } public Set keySet() { - Set keys = new HashSet<>(filter.keySet()); - keys.remove("dynamic_tags"); - return keys; + return filter.keySet(); } @SuppressWarnings({"unchecked"}) @@ -132,27 +128,11 @@ private void parseTags() { tagsParsed = true; this.additionalTags = new HashMap(); - this.dynamicTags = new ArrayList(); if (filter.get("tags") != null) { Map allTags = (Map) filter.get("tags"); this.additionalTags.putAll(allTags); } - - if (filter.get("dynamic_tags") != null) { - Map dynamicTagsConfig = - (Map) filter.get("dynamic_tags"); - - for (Map.Entry entry : dynamicTagsConfig.entrySet()) { - String tagName = entry.getKey(); - Object tagConfig = entry.getValue(); - - DynamicTag dynamicTag = DynamicTag.parse(tagName, tagConfig); - if (dynamicTag != null) { - this.dynamicTags.add(dynamicTag); - } - } - } } public Map getAdditionalTags() { @@ -162,14 +142,6 @@ public Map getAdditionalTags() { return this.additionalTags; } - - public List getDynamicTags() { - if (this.dynamicTags == null) { - parseTags(); - } - - return this.dynamicTags; - } public String getDomain() { return (String) filter.get("domain"); diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index 05e0ab04..afb50e75 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -469,12 +469,9 @@ private void resolveConfigurationDynamicTags() { List allDynamicTags = new ArrayList<>(); for (Configuration config : configurationList) { - Filter include = config.getInclude(); - if (include != null) { - List dynamicTags = include.getDynamicTags(); - if (dynamicTags != null && !dynamicTags.isEmpty()) { - allDynamicTags.addAll(dynamicTags); - } + List dynamicTags = config.getDynamicTags(); + if (dynamicTags != null && !dynamicTags.isEmpty()) { + allDynamicTags.addAll(dynamicTags); } } diff --git a/src/test/resources/jmx_config_dynamic_tags.yaml b/src/test/resources/jmx_config_dynamic_tags.yaml index c1619bf2..2c856093 100644 --- a/src/test/resources/jmx_config_dynamic_tags.yaml +++ b/src/test/resources/jmx_config_dynamic_tags.yaml @@ -5,19 +5,19 @@ instances: name: jmx_config_dynamic_tags_test conf: - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - tags: - env: test - dynamic_tags: - cluster_id: - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp - attribute: ClusterId - kafka_version: - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp - attribute: Version - attribute: - Metric: - metric_type: gauge - alias: test.config.dynamic.tags.metric + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + tags: + env: test + attribute: + Metric: + metric_type: gauge + alias: test.config.dynamic.tags.metric + dynamic_tags: + - tag_name: cluster_id + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: ClusterId + - tag_name: kafka_version + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: Version diff --git a/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml b/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml index 892b685f..808095e4 100644 --- a/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml +++ b/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml @@ -5,18 +5,18 @@ instances: name: jmx_config_dynamic_tags_bean_params_test conf: - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - tags: - bean_type: $type - bean_name: $name - dynamic_tags: - cluster_id: - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean - attribute: ClusterId - attribute: - Metric: - metric_type: gauge - alias: test.bean.params.metric + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + tags: + bean_type: $type + bean_name: $name + attribute: + Metric: + metric_type: gauge + alias: test.bean.params.metric + dynamic_tags: + - tag_name: cluster_id + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean + attribute: ClusterId diff --git a/src/test/resources/jmx_config_dynamic_tags_caching.yaml b/src/test/resources/jmx_config_dynamic_tags_caching.yaml index 4b81b309..87a90a83 100644 --- a/src/test/resources/jmx_config_dynamic_tags_caching.yaml +++ b/src/test/resources/jmx_config_dynamic_tags_caching.yaml @@ -6,28 +6,28 @@ instances: conf: # Config 1: Uses cluster_id - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - dynamic_tags: - cluster_id: - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp - attribute: ClusterId - attribute: - Metric: - metric_type: gauge - alias: test.metric1 + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + attribute: + Metric: + metric_type: gauge + alias: test.metric1 + dynamic_tags: + - tag_name: cluster_id + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: ClusterId # Config 2: Uses same cluster_id (should be cached) - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - dynamic_tags: - cluster_id: - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp - attribute: ClusterId - attribute: - Port: - metric_type: gauge - alias: test.metric2 + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + attribute: + Port: + metric_type: gauge + alias: test.metric2 + dynamic_tags: + - tag_name: cluster_id + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: ClusterId diff --git a/src/test/resources/jmx_config_dynamic_tags_multi.yaml b/src/test/resources/jmx_config_dynamic_tags_multi.yaml index 8f57094f..9ee0f68d 100644 --- a/src/test/resources/jmx_config_dynamic_tags_multi.yaml +++ b/src/test/resources/jmx_config_dynamic_tags_multi.yaml @@ -6,29 +6,29 @@ instances: conf: # Config 1: Has cluster_id dynamic tag - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - name: Instance1 - dynamic_tags: - cluster_id: - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1 - attribute: ClusterId - attribute: - Metric: - metric_type: gauge - alias: test.instance1.metric + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + name: Instance1 + attribute: + Metric: + metric_type: gauge + alias: test.instance1.metric + dynamic_tags: + - tag_name: cluster_id + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1 + attribute: ClusterId # Config 2: Has version dynamic tag - include: - domain: org.datadog.jmxfetch.test - type: DynamicTagTestApp - name: Instance2 - dynamic_tags: - version: - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2 - attribute: Version - attribute: - Metric: - metric_type: gauge - alias: test.instance2.metric + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + name: Instance2 + attribute: + Metric: + metric_type: gauge + alias: test.instance2.metric + dynamic_tags: + - tag_name: version + bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2 + attribute: Version From 67cc21f0a91e4a8e001d1a9b6be0e7a5a40e69c8 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Thu, 16 Oct 2025 21:41:24 -0600 Subject: [PATCH 06/11] remove unused functions --- .../java/org/datadog/jmxfetch/Connection.java | 5 ---- .../java/org/datadog/jmxfetch/Filter.java | 24 ++++++------------- 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/datadog/jmxfetch/Connection.java b/src/main/java/org/datadog/jmxfetch/Connection.java index 84c11e53..803cd817 100644 --- a/src/main/java/org/datadog/jmxfetch/Connection.java +++ b/src/main/java/org/datadog/jmxfetch/Connection.java @@ -68,11 +68,6 @@ public Object getAttribute(ObjectName objectName, String attributeName) } return attr; } - - /** Gets the underlying MBeanServerConnection. */ - public MBeanServerConnection getMBeanServerConnection() { - return mbs; - } /** Closes the connector. */ public void closeConnector() { diff --git a/src/main/java/org/datadog/jmxfetch/Filter.java b/src/main/java/org/datadog/jmxfetch/Filter.java index ce219ffd..eb886ed9 100644 --- a/src/main/java/org/datadog/jmxfetch/Filter.java +++ b/src/main/java/org/datadog/jmxfetch/Filter.java @@ -16,7 +16,6 @@ class Filter { List beanRegexes = null; List excludeTags = null; Map additionalTags = null; - boolean tagsParsed = false; /** * A simple class to manipulate include/exclude filter elements more easily A filter may @@ -121,25 +120,16 @@ public List getExcludeTags() { return this.excludeTags; } - private void parseTags() { - if (tagsParsed) { - return; - } - - tagsParsed = true; - this.additionalTags = new HashMap(); - - if (filter.get("tags") != null) { - Map allTags = (Map) filter.get("tags"); - this.additionalTags.putAll(allTags); - } - } - public Map getAdditionalTags() { + // Return additional tags if (this.additionalTags == null) { - parseTags(); + if (filter.get("tags") == null) { + this.additionalTags = new HashMap(); + } else { + this.additionalTags = (Map) filter.get("tags"); + } } - + return this.additionalTags; } From 4ee9eb48c360a5e78c65045fd5ebf35754d5ba50 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Thu, 16 Oct 2025 21:50:50 -0600 Subject: [PATCH 07/11] update when tags are resolved --- src/main/java/org/datadog/jmxfetch/Instance.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index afb50e75..1e2e69d0 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -448,14 +448,16 @@ public void init(boolean forceNewConnection) log.info("Trying to connect to JMX Server at " + this.toString()); connection = getConnection(instanceMap, forceNewConnection); - // Resolve configuration-level dynamic tags for all configurations - resolveConfigurationDynamicTags(); - log.info( "Trying to collect bean list for the first time for JMX Server at {}", this); this.refreshBeansList(); this.initialRefreshTime = this.lastRefreshTime; log.info("Connected to JMX Server at {} with {} beans", this, this.beans.size()); + + // Resolve configuration-level dynamic tags for all configurations + // Must be done after refreshBeansList() so the beans exist + resolveConfigurationDynamicTags(); + this.getMatchingAttributes(); log.info("Done initializing JMX Server at {}", this); } From 55eb6a7fa0d35d09abebd4a60e3001085d4e89c0 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Thu, 16 Oct 2025 22:08:14 -0600 Subject: [PATCH 08/11] separate adding dynamic tags & additional tags --- src/main/java/org/datadog/jmxfetch/JmxAttribute.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/datadog/jmxfetch/JmxAttribute.java b/src/main/java/org/datadog/jmxfetch/JmxAttribute.java index 3a762fac..43cdf986 100644 --- a/src/main/java/org/datadog/jmxfetch/JmxAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JmxAttribute.java @@ -138,7 +138,10 @@ private void addAdditionalTags() { } } } - + } + + /** Add dynamic tags that were resolved at connection time. */ + private void addDynamicTags() { Map resolvedDynamicTags = this.matchingConf.getResolvedDynamicTags(); if (resolvedDynamicTags != null && !resolvedDynamicTags.isEmpty()) { for (Map.Entry tag : resolvedDynamicTags.entrySet()) { @@ -509,6 +512,8 @@ public void setMatchingConf(Configuration matchingConf) { // Now that we have the matchingConf we can: // - add additional tags this.addAdditionalTags(); + // - add dynamic tags that were resolved at connection time + this.addDynamicTags(); // - filter out excluded tags this.applyTagsBlackList(); // Add the service tag(s) - comes last because if the service tag is blacklisted as From d6e7660b420b248080e81a7a9b7488c3d2e0bb0e Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Fri, 17 Oct 2025 08:54:56 -0600 Subject: [PATCH 09/11] address PR comments --- .../org/datadog/jmxfetch/Configuration.java | 40 +---------------- .../java/org/datadog/jmxfetch/DynamicTag.java | 10 +++-- .../java/org/datadog/jmxfetch/Instance.java | 43 ++++++++++++++++--- .../org/datadog/jmxfetch/JmxAttribute.java | 10 +++-- .../resources/jmx_config_dynamic_tags.yaml | 4 +- .../jmx_config_dynamic_tags_bean_params.yaml | 2 +- .../jmx_config_dynamic_tags_caching.yaml | 4 +- .../jmx_config_dynamic_tags_multi.yaml | 4 +- 8 files changed, 57 insertions(+), 60 deletions(-) diff --git a/src/main/java/org/datadog/jmxfetch/Configuration.java b/src/main/java/org/datadog/jmxfetch/Configuration.java index 805088df..99839453 100644 --- a/src/main/java/org/datadog/jmxfetch/Configuration.java +++ b/src/main/java/org/datadog/jmxfetch/Configuration.java @@ -18,7 +18,6 @@ public class Configuration { private Filter include; private Filter exclude; private List dynamicTags = null; - private Map resolvedDynamicTags = null; /** * Access configuration elements more easily @@ -37,7 +36,7 @@ public Configuration(Map conf) { * Expected format: * dynamic_tags: * - tag_name: cluster_id - * bean: kafka.server:type=KafkaServer,name=ClusterId + * bean_name: kafka.server:type=KafkaServer,name=ClusterId * attribute: Value */ private void parseDynamicTags(Object dynamicTagsConfig) { @@ -81,35 +80,6 @@ public String toString() { private Boolean hasInclude() { return getInclude() != null && !getInclude().isEmptyFilter(); } - - /** - * Resolves dynamic tags (bean name, attribute name) to the JMX bean value, - * and stores the result in resolvedDynamicTags. - * - * @param cache shared cache mapping "(beanName, attributeName)" to "(tagName, tagValue)" - */ - public void resolveDynamicTags(Map> cache) { - if (resolvedDynamicTags != null) { - return; - } - - resolvedDynamicTags = new HashMap(); - - if (dynamicTags == null || dynamicTags.isEmpty()) { - return; - } - - for (DynamicTag dynamicTag : dynamicTags) { - String cacheKey = dynamicTag.getBeanName() + "#" + dynamicTag.getAttributeName(); - Map.Entry cached = cache.get(cacheKey); - if (cached != null) { - resolvedDynamicTags.put(cached.getKey(), cached.getValue()); - } - } - - log.debug("Applied {} dynamic tag(s) to configuration from cache", - resolvedDynamicTags.size()); - } /** Get list of dynamic tags defined for this configuration. */ public List getDynamicTags() { @@ -118,14 +88,6 @@ public List getDynamicTags() { } return dynamicTags; } - - /** Get all resolved dynamic tags for this configuration. */ - public Map getResolvedDynamicTags() { - if (resolvedDynamicTags == null) { - return new HashMap(); - } - return resolvedDynamicTags; - } /** * Filter a configuration list to keep the ones with `include` filters. diff --git a/src/main/java/org/datadog/jmxfetch/DynamicTag.java b/src/main/java/org/datadog/jmxfetch/DynamicTag.java index c8b09ce0..457481f6 100644 --- a/src/main/java/org/datadog/jmxfetch/DynamicTag.java +++ b/src/main/java/org/datadog/jmxfetch/DynamicTag.java @@ -20,18 +20,22 @@ public static DynamicTag parse(Object tagConfig) { } if (!(tagConfig instanceof Map)) { - log.warn("Invalid dynamic tag config: expected map with 'tag_name', 'bean' and " + log.warn("Invalid dynamic tag config: expected map with 'tag_name', 'bean_name' and " + "'attribute' keys"); return null; } Map config = (Map) tagConfig; Object tagNameObj = config.get("tag_name"); - Object beanObj = config.get("bean"); + Object beanObj = config.get("bean_name"); Object attrObj = config.get("attribute"); if (tagNameObj == null || beanObj == null || attrObj == null) { - log.warn("Invalid dynamic tag config: missing 'tag_name', 'bean' or 'attribute' key"); + String missing = "Invalid dynamic tag config: missing" + + (tagNameObj == null ? " tag_name" : "") + + (beanObj == null ? " bean_name" : "") + + (attrObj == null ? " attribute" : ""); + log.warn(missing); return null; } diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index 1e2e69d0..505c4742 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -79,6 +79,7 @@ public Yaml initialValue() { private ObjectName instanceTelemetryBeanName; private MBeanServer mbs; private Boolean normalizeBeanParamTags; + private Map> dynamicTagsCache; /** Constructor, instantiates Instance based of a previous instance and appConfig. */ public Instance(Instance instance, AppConfig appConfig) { @@ -467,7 +468,7 @@ private void resolveConfigurationDynamicTags() { return; } - Map> cache = new HashMap<>(); + this.dynamicTagsCache = new HashMap<>(); List allDynamicTags = new ArrayList<>(); for (Configuration config : configurationList) { @@ -483,20 +484,46 @@ private void resolveConfigurationDynamicTags() { for (DynamicTag dynamicTag : allDynamicTags) { String cacheKey = dynamicTag.getBeanName() + "#" + dynamicTag.getAttributeName(); - if (!cache.containsKey(cacheKey)) { + if (!this.dynamicTagsCache.containsKey(cacheKey)) { Map.Entry resolved = dynamicTag.resolve(connection); if (resolved != null) { - cache.put(cacheKey, resolved); + this.dynamicTagsCache.put(cacheKey, resolved); } } } log.info("Resolved {} unique dynamic tag(s) from {} total references for instance {}", - cache.size(), allDynamicTags.size(), instanceName); + this.dynamicTagsCache.size(), allDynamicTags.size(), instanceName); + } + + /** + * Get resolved dynamic tags for a specific configuration. + * This resolves the dynamic tags defined in the configuration using the cached values. + * + * @param config the configuration to get resolved tags for + * @return map of tag name to tag value + */ + private Map getResolvedDynamicTagsForConfig(Configuration config) { + Map resolvedTags = new HashMap<>(); - for (Configuration config : configurationList) { - config.resolveDynamicTags(cache); + if (this.dynamicTagsCache == null || this.dynamicTagsCache.isEmpty()) { + return resolvedTags; + } + + List dynamicTags = config.getDynamicTags(); + if (dynamicTags == null || dynamicTags.isEmpty()) { + return resolvedTags; + } + + for (DynamicTag dynamicTag : dynamicTags) { + String cacheKey = dynamicTag.getBeanName() + "#" + dynamicTag.getAttributeName(); + Map.Entry cached = this.dynamicTagsCache.get(cacheKey); + if (cached != null) { + resolvedTags.put(cached.getKey(), cached.getValue()); + } } + + return resolvedTags; } /** Returns a string representation for the instance. */ @@ -733,7 +760,9 @@ private void getMatchingAttributes() throws IOException { for (Configuration conf : configurationList) { try { if (jmxAttribute.match(conf)) { - jmxAttribute.setMatchingConf(conf); + Map resolvedDynamicTags = + getResolvedDynamicTagsForConfig(conf); + jmxAttribute.setMatchingConf(conf, resolvedDynamicTags); metricsCount += jmxAttribute.getMetricsCount(); this.matchingAttributes.add(jmxAttribute); diff --git a/src/main/java/org/datadog/jmxfetch/JmxAttribute.java b/src/main/java/org/datadog/jmxfetch/JmxAttribute.java index 43cdf986..5a61d1f3 100644 --- a/src/main/java/org/datadog/jmxfetch/JmxAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JmxAttribute.java @@ -59,6 +59,7 @@ public abstract class JmxAttribute { new HashMap>(); protected String[] tags; private Configuration matchingConf; + private Map resolvedDynamicTags; private List defaultTagsList; private boolean cassandraAliasing; protected String checkName; @@ -142,9 +143,8 @@ private void addAdditionalTags() { /** Add dynamic tags that were resolved at connection time. */ private void addDynamicTags() { - Map resolvedDynamicTags = this.matchingConf.getResolvedDynamicTags(); - if (resolvedDynamicTags != null && !resolvedDynamicTags.isEmpty()) { - for (Map.Entry tag : resolvedDynamicTags.entrySet()) { + if (this.resolvedDynamicTags != null && !this.resolvedDynamicTags.isEmpty()) { + for (Map.Entry tag : this.resolvedDynamicTags.entrySet()) { this.defaultTagsList.add(tag.getKey() + ":" + tag.getValue()); } } @@ -506,8 +506,10 @@ public Configuration getMatchingConf() { } /** Sets a matching configuration for the attribute. */ - public void setMatchingConf(Configuration matchingConf) { + public void setMatchingConf(Configuration matchingConf, + Map resolvedDynamicTags) { this.matchingConf = matchingConf; + this.resolvedDynamicTags = resolvedDynamicTags; // Now that we have the matchingConf we can: // - add additional tags diff --git a/src/test/resources/jmx_config_dynamic_tags.yaml b/src/test/resources/jmx_config_dynamic_tags.yaml index 2c856093..b39c3d22 100644 --- a/src/test/resources/jmx_config_dynamic_tags.yaml +++ b/src/test/resources/jmx_config_dynamic_tags.yaml @@ -15,9 +15,9 @@ instances: alias: test.config.dynamic.tags.metric dynamic_tags: - tag_name: cluster_id - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp attribute: ClusterId - tag_name: kafka_version - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp attribute: Version diff --git a/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml b/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml index 808095e4..59cc2b09 100644 --- a/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml +++ b/src/test/resources/jmx_config_dynamic_tags_bean_params.yaml @@ -16,7 +16,7 @@ instances: alias: test.bean.params.metric dynamic_tags: - tag_name: cluster_id - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=TestBean attribute: ClusterId diff --git a/src/test/resources/jmx_config_dynamic_tags_caching.yaml b/src/test/resources/jmx_config_dynamic_tags_caching.yaml index 87a90a83..e0929944 100644 --- a/src/test/resources/jmx_config_dynamic_tags_caching.yaml +++ b/src/test/resources/jmx_config_dynamic_tags_caching.yaml @@ -14,7 +14,7 @@ instances: alias: test.metric1 dynamic_tags: - tag_name: cluster_id - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp attribute: ClusterId # Config 2: Uses same cluster_id (should be cached) @@ -27,7 +27,7 @@ instances: alias: test.metric2 dynamic_tags: - tag_name: cluster_id - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp attribute: ClusterId diff --git a/src/test/resources/jmx_config_dynamic_tags_multi.yaml b/src/test/resources/jmx_config_dynamic_tags_multi.yaml index 9ee0f68d..619a9b8a 100644 --- a/src/test/resources/jmx_config_dynamic_tags_multi.yaml +++ b/src/test/resources/jmx_config_dynamic_tags_multi.yaml @@ -15,7 +15,7 @@ instances: alias: test.instance1.metric dynamic_tags: - tag_name: cluster_id - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1 + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance1 attribute: ClusterId # Config 2: Has version dynamic tag @@ -29,6 +29,6 @@ instances: alias: test.instance2.metric dynamic_tags: - tag_name: version - bean: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2 + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp,name=Instance2 attribute: Version From 0971987aeee758f6943c65389aa8414174fc9ea9 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Wed, 22 Oct 2025 08:21:40 -0600 Subject: [PATCH 10/11] add negative cache and improve code readability --- .../java/org/datadog/jmxfetch/DynamicTag.java | 5 +++++ .../java/org/datadog/jmxfetch/Instance.java | 19 ++++++++++++------- .../org/datadog/jmxfetch/JmxAttribute.java | 9 ++++++--- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/datadog/jmxfetch/DynamicTag.java b/src/main/java/org/datadog/jmxfetch/DynamicTag.java index 457481f6..3e40347c 100644 --- a/src/main/java/org/datadog/jmxfetch/DynamicTag.java +++ b/src/main/java/org/datadog/jmxfetch/DynamicTag.java @@ -64,6 +64,11 @@ public String getAttributeName() { return attributeName; } + /** Gets a unique key for the bean and attribute combination. */ + public String getBeanAttributeKey() { + return beanName + "#" + attributeName; + } + /** Resolve the dynamic tag by fetching the attribute value from JMX. */ public Map.Entry resolve(Connection connection) { try { diff --git a/src/main/java/org/datadog/jmxfetch/Instance.java b/src/main/java/org/datadog/jmxfetch/Instance.java index 505c4742..82570c88 100644 --- a/src/main/java/org/datadog/jmxfetch/Instance.java +++ b/src/main/java/org/datadog/jmxfetch/Instance.java @@ -482,18 +482,22 @@ private void resolveConfigurationDynamicTags() { return; } + int successfulResolutions = 0; for (DynamicTag dynamicTag : allDynamicTags) { - String cacheKey = dynamicTag.getBeanName() + "#" + dynamicTag.getAttributeName(); + String cacheKey = dynamicTag.getBeanAttributeKey(); if (!this.dynamicTagsCache.containsKey(cacheKey)) { Map.Entry resolved = dynamicTag.resolve(connection); - if (resolved != null) { - this.dynamicTagsCache.put(cacheKey, resolved); - } + // Cache both successful and failed resolutions (null) to avoid retrying + this.dynamicTagsCache.put(cacheKey, resolved); + } + // Count successful resolutions (cached value is not null) + if (this.dynamicTagsCache.get(cacheKey) != null) { + successfulResolutions++; } } log.info("Resolved {} unique dynamic tag(s) from {} total references for instance {}", - this.dynamicTagsCache.size(), allDynamicTags.size(), instanceName); + successfulResolutions, allDynamicTags.size(), instanceName); } /** @@ -516,7 +520,7 @@ private Map getResolvedDynamicTagsForConfig(Configuration config } for (DynamicTag dynamicTag : dynamicTags) { - String cacheKey = dynamicTag.getBeanName() + "#" + dynamicTag.getAttributeName(); + String cacheKey = dynamicTag.getBeanAttributeKey(); Map.Entry cached = this.dynamicTagsCache.get(cacheKey); if (cached != null) { resolvedTags.put(cached.getKey(), cached.getValue()); @@ -762,7 +766,8 @@ private void getMatchingAttributes() throws IOException { if (jmxAttribute.match(conf)) { Map resolvedDynamicTags = getResolvedDynamicTagsForConfig(conf); - jmxAttribute.setMatchingConf(conf, resolvedDynamicTags); + jmxAttribute.setResolvedDynamicTags(resolvedDynamicTags); + jmxAttribute.setMatchingConf(conf); metricsCount += jmxAttribute.getMetricsCount(); this.matchingAttributes.add(jmxAttribute); diff --git a/src/main/java/org/datadog/jmxfetch/JmxAttribute.java b/src/main/java/org/datadog/jmxfetch/JmxAttribute.java index 5a61d1f3..d4343e35 100644 --- a/src/main/java/org/datadog/jmxfetch/JmxAttribute.java +++ b/src/main/java/org/datadog/jmxfetch/JmxAttribute.java @@ -505,11 +505,14 @@ public Configuration getMatchingConf() { return matchingConf; } + /** Sets resolved dynamic tags for the attribute. */ + public void setResolvedDynamicTags(Map resolvedDynamicTags) { + this.resolvedDynamicTags = resolvedDynamicTags; + } + /** Sets a matching configuration for the attribute. */ - public void setMatchingConf(Configuration matchingConf, - Map resolvedDynamicTags) { + public void setMatchingConf(Configuration matchingConf) { this.matchingConf = matchingConf; - this.resolvedDynamicTags = resolvedDynamicTags; // Now that we have the matchingConf we can: // - add additional tags From 91ffa171fbc69440d804d0e062beb3f9d26009e2 Mon Sep 17 00:00:00 2001 From: Piotr Wolski Date: Fri, 24 Oct 2025 08:19:59 -0600 Subject: [PATCH 11/11] test invalid dynamic tags & multiple dynamic tags --- .../jmxfetch/TestConfigDynamicTags.java | 93 +++++++++++++++++++ .../jmx_config_dynamic_tags_invalid.yaml | 34 +++++++ ...config_dynamic_tags_multiple_per_conf.yaml | 28 ++++++ 3 files changed, 155 insertions(+) create mode 100644 src/test/resources/jmx_config_dynamic_tags_invalid.yaml create mode 100644 src/test/resources/jmx_config_dynamic_tags_multiple_per_conf.yaml diff --git a/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java b/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java index 26aa37d8..fd57a6ad 100644 --- a/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java +++ b/src/test/java/org/datadog/jmxfetch/TestConfigDynamicTags.java @@ -195,6 +195,99 @@ public void testConfigDynamicTagsCaching() throws Exception { metricsWithClusterId == 2); } + @Test + public void testConfigDynamicTagsMultiplePerConf() throws Exception { + registerMBean( + new DynamicTagTestApp("cluster-1", "version-1", 9999), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); + + initApplication("jmx_config_dynamic_tags_multiple_per_conf.yaml"); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + boolean foundMetric = false; + for (Map metric : metrics) { + String metricName = (String) metric.get("name"); + + if ("test.metric.with.multiple.tags".equals(metricName)) { + foundMetric = true; + + List tagList = getTagsAsList(metric); + + // Verify all three dynamic tags are present + assertTrue("Should have cluster_id dynamic tag", + tagList.contains("cluster_id:cluster-1")); + assertTrue("Should have version dynamic tag", + tagList.contains("version:version-1")); + assertTrue("Should have port dynamic tag", + tagList.contains("port:9999")); + + // Verify normal tags are also present + assertTrue("Should have env tag", + tagList.contains("env:test")); + } + } + + assertTrue("Should have found metric with multiple dynamic tags", foundMetric); + } + + @Test + public void testConfigDynamicTagsInvalid() throws Exception { + registerMBean( + new DynamicTagTestApp("cluster-1", "version-1", 9999), + "org.datadog.jmxfetch.test:type=DynamicTagTestApp"); + + initApplication("jmx_config_dynamic_tags_invalid.yaml"); + + run(); + + List> metrics = getMetrics(); + assertTrue("Should have collected metrics", metrics.size() > 0); + + boolean foundMetric = false; + for (Map metric : metrics) { + String metricName = (String) metric.get("name"); + + if ("test.metric.with.invalid.tags".equals(metricName)) { + foundMetric = true; + + List tagList = getTagsAsList(metric); + + // Verify normal tags are still applied + assertTrue("Should have env tag", + tagList.contains("env:test")); + assertTrue("Should have region tag", + tagList.contains("region:us-east-1")); + + // Verify the valid dynamic tag is applied + assertTrue("Should have valid_port dynamic tag", + tagList.contains("valid_port:9999")); + + // Verify invalid dynamic tags are NOT applied + boolean hasMissingBeanTag = false; + boolean hasMissingAttrTag = false; + for (String tag : tagList) { + if (tag.startsWith("missing_bean:")) { + hasMissingBeanTag = true; + } + if (tag.startsWith("missing_attr:")) { + hasMissingAttrTag = true; + } + } + assertTrue("Should NOT have missing_bean tag (invalid config)", + !hasMissingBeanTag); + assertTrue("Should NOT have missing_attr tag (invalid config)", + !hasMissingAttrTag); + } + } + + assertTrue("Should have found metric (verifies invalid dynamic tags don't crash)", + foundMetric); + } + private List getTagsAsList(Map metric) { List tagList = new ArrayList<>(); Object tagsObj = metric.get("tags"); diff --git a/src/test/resources/jmx_config_dynamic_tags_invalid.yaml b/src/test/resources/jmx_config_dynamic_tags_invalid.yaml new file mode 100644 index 00000000..45f96c97 --- /dev/null +++ b/src/test/resources/jmx_config_dynamic_tags_invalid.yaml @@ -0,0 +1,34 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_config_dynamic_tags_invalid_test + conf: + # Invalid dynamic tag configurations (should be handled gracefully) + # Normal tags should still be applied, but invalid dynamic tags should be ignored + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + attribute: + Metric: + metric_type: gauge + alias: test.metric.with.invalid.tags + tags: + env: test + region: us-east-1 + dynamic_tags: + # Missing tag_name - should be ignored with warning + - bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: ClusterId + # Missing bean_name - should be ignored with warning + - tag_name: missing_bean + attribute: Version + # Missing attribute - should be ignored with warning + - tag_name: missing_attr + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp + # Valid one - should work + - tag_name: valid_port + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: Port + + diff --git a/src/test/resources/jmx_config_dynamic_tags_multiple_per_conf.yaml b/src/test/resources/jmx_config_dynamic_tags_multiple_per_conf.yaml new file mode 100644 index 00000000..54954957 --- /dev/null +++ b/src/test/resources/jmx_config_dynamic_tags_multiple_per_conf.yaml @@ -0,0 +1,28 @@ +init_config: + +instances: + - process_name_regex: .*surefire.* + name: jmx_config_dynamic_tags_multiple_per_conf_test + conf: + # Config with multiple dynamic tags (cluster_id AND version) on the same metric + - include: + domain: org.datadog.jmxfetch.test + type: DynamicTagTestApp + attribute: + Metric: + metric_type: gauge + alias: test.metric.with.multiple.tags + tags: + env: test + dynamic_tags: + - tag_name: cluster_id + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: ClusterId + - tag_name: version + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: Version + - tag_name: port + bean_name: org.datadog.jmxfetch.test:type=DynamicTagTestApp + attribute: Port + +