Skip to content

adding global_cache_ttl feature #622

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
victoramsantos wants to merge 3 commits into
base: master
Choose a base branch
from
Open

adding global_cache_ttl feature #622

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d97d910
adding global_cache_ttl feature
victoramsantos Dec 14, 2023
680d018
formating
victoramsantos Dec 14, 2023
c7e2d4e
formating
victoramsantos Dec 14, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ set_timestamp | Optional. Boolean for whether to set the Prometheus metric times
use_get_metric_data | Optional. Boolean (experimental) Use GetMetricData API to get metrics instead of GetMetricStatistics. Can be set globally and per metric.
list_metrics_cache_ttl | Optional. Number of seconds to cache the result of calling the ListMetrics API. Defaults to 0 (no cache). Can be set globally and per metric.
warn_on_empty_list_dimensions | Optional. Boolean Emit warning if the exporter cannot determine what metrics to request
global_cache_ttl | Optional. Number of seconds to cache the result from /metrics. Any value greater than 0 means that the last result will be returned. Defaults to 0 (no cache). Can be set globally.


The above config will export time series such as
Expand Down
122 changes: 97 additions & 25 deletions src/main/java/io/prometheus/cloudwatch/CloudWatchCollector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,9 @@
import java.io.IOException;
import java.io.Reader;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.time.Instant;
import java.util.*;
import java.util.Map.Entry;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.yaml.snakeyaml.LoaderOptions;
Expand All @@ -32,11 +26,7 @@
import software.amazon.awssdk.services.cloudwatch.model.Statistic;
import software.amazon.awssdk.services.resourcegroupstaggingapi.ResourceGroupsTaggingApiClient;
import software.amazon.awssdk.services.resourcegroupstaggingapi.ResourceGroupsTaggingApiClientBuilder;
import software.amazon.awssdk.services.resourcegroupstaggingapi.model.GetResourcesRequest;
import software.amazon.awssdk.services.resourcegroupstaggingapi.model.GetResourcesResponse;
import software.amazon.awssdk.services.resourcegroupstaggingapi.model.ResourceTagMapping;
import software.amazon.awssdk.services.resourcegroupstaggingapi.model.Tag;
import software.amazon.awssdk.services.resourcegroupstaggingapi.model.TagFilter;
import software.amazon.awssdk.services.resourcegroupstaggingapi.model.*;
import software.amazon.awssdk.services.sts.StsClient;
import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
Expand All @@ -50,11 +40,14 @@ static class ActiveConfig {
ResourceGroupsTaggingApiClient taggingClient;
DimensionSource dimensionSource;

Map<String, Object> globalConfig;

public ActiveConfig(ActiveConfig cfg) {
this.rules = new ArrayList<>(cfg.rules);
this.cloudWatchClient = cfg.cloudWatchClient;
this.taggingClient = cfg.taggingClient;
this.dimensionSource = cfg.dimensionSource;
this.globalConfig = cfg.globalConfig;
}

public ActiveConfig() {}
Expand Down Expand Up @@ -96,29 +89,32 @@ static class AWSTagSelect {
"ReadThrottleEvents", "WriteThrottleEvents");

public CloudWatchCollector(Reader in) {
loadConfig(in, null, null);
loadConfig(in, null, null, null);
}

public CloudWatchCollector(String yamlConfig) {
this(yamlConfig, null, null);
this(yamlConfig, null, null, null);
}

/* For unittests. */
protected CloudWatchCollector(
String jsonConfig,
CloudWatchClient cloudWatchClient,
ResourceGroupsTaggingApiClient taggingClient) {
ResourceGroupsTaggingApiClient taggingClient,
Map<String, Object> globalConfig) {
this(
(Map<String, Object>) new Yaml(new SafeConstructor(new LoaderOptions())).load(jsonConfig),
cloudWatchClient,
taggingClient);
taggingClient,
globalConfig);
}

private CloudWatchCollector(
Map<String, Object> config,
CloudWatchClient cloudWatchClient,
ResourceGroupsTaggingApiClient taggingClient) {
loadConfig(config, cloudWatchClient, taggingClient);
ResourceGroupsTaggingApiClient taggingClient,
Map<String, Object> globalConfig) {
loadConfig(config, cloudWatchClient, taggingClient, globalConfig);
}

@Override
Expand All @@ -129,26 +125,39 @@ public List<MetricFamilySamples> describe() {
protected void reloadConfig() throws IOException {
LOGGER.log(Level.INFO, "Reloading configuration");
try (FileReader reader = new FileReader(WebServer.configFilePath); ) {
loadConfig(reader, activeConfig.cloudWatchClient, activeConfig.taggingClient);
loadConfig(
reader,
activeConfig.cloudWatchClient,
activeConfig.taggingClient,
activeConfig.globalConfig);
}
}

protected void loadConfig(
Reader in, CloudWatchClient cloudWatchClient, ResourceGroupsTaggingApiClient taggingClient) {
Reader in,
CloudWatchClient cloudWatchClient,
ResourceGroupsTaggingApiClient taggingClient,
Map<String, Object> globalConfig) {
loadConfig(
(Map<String, Object>) new Yaml(new SafeConstructor(new LoaderOptions())).load(in),
cloudWatchClient,
taggingClient);
taggingClient,
globalConfig);
}

private void loadConfig(
Map<String, Object> config,
CloudWatchClient cloudWatchClient,
ResourceGroupsTaggingApiClient taggingClient) {
ResourceGroupsTaggingApiClient taggingClient,
Map<String, Object> globalConfig) {
if (config == null) { // Yaml config empty, set config to empty map.
config = new HashMap<>();
}

if (globalConfig == null) { // Yaml config empty, set config to empty map.
globalConfig = new HashMap<>();
}

int defaultPeriod = 60;
if (config.containsKey("period_seconds")) {
defaultPeriod = ((Number) config.get("period_seconds")).intValue();
Expand Down Expand Up @@ -178,6 +187,12 @@ private void loadConfig(
Duration.ofSeconds(((Number) config.get("list_metrics_cache_ttl")).intValue());
}

int defaultGlobalCacheSeconds = 0;
if (config.containsKey("global_cache_ttl")) {
defaultGlobalCacheSeconds = ((Number) config.get("global_cache_ttl")).intValue();
}
globalConfig.put("globalCacheSeconds", defaultGlobalCacheSeconds);

boolean defaultWarnOnMissingDimensions = false;
if (config.containsKey("warn_on_empty_list_dimensions")) {
defaultWarnOnMissingDimensions = (Boolean) config.get("warn_on_empty_list_dimensions");
Expand Down Expand Up @@ -331,19 +346,21 @@ private void loadConfig(
dimensionSource = new CachingDimensionSource(dimensionSource, metricCacheConfig);
}

loadConfig(rules, cloudWatchClient, taggingClient, dimensionSource);
loadConfig(rules, cloudWatchClient, taggingClient, dimensionSource, globalConfig);
}

private void loadConfig(
ArrayList<MetricRule> rules,
CloudWatchClient cloudWatchClient,
ResourceGroupsTaggingApiClient taggingClient,
DimensionSource dimensionSource) {
DimensionSource dimensionSource,
Map<String, Object> globalConfig) {
synchronized (activeConfig) {
activeConfig.cloudWatchClient = cloudWatchClient;
activeConfig.taggingClient = taggingClient;
activeConfig.rules = rules;
activeConfig.dimensionSource = dimensionSource;
activeConfig.globalConfig = globalConfig;
}
}

Expand Down Expand Up @@ -634,10 +651,46 @@ private void scrape(List<MetricFamilySamples> mfs) {
infoSamples));
}

private void updateCacheMetric(List<MetricFamilySamples> mfs, double value) {
List<MetricFamilySamples.Sample> samples = new ArrayList<>();
MetricFamilySamples cacheMetric = null;
for (MetricFamilySamples metric : mfs) {
if (metric.name.equals("cloudwatch_exporter_cached_answer")) {
cacheMetric = metric;
break;
}
}

if (cacheMetric == null) {
cacheMetric =
new MetricFamilySamples(
"cloudwatch_exporter_cached_answer",
Type.GAUGE,
"Non-zero means this scrape was from cache",
samples);
mfs.add(cacheMetric);
} else {
cacheMetric.samples.clear();
}

cacheMetric.samples.add(
new MetricFamilySamples.Sample(
"cloudwatch_exporter_cached_answer", new ArrayList<>(), new ArrayList<>(), value));
}

List<MetricFamilySamples> cachedMfs = new ArrayList<>();

public List<MetricFamilySamples> collect() {
long start = System.nanoTime();
double error = 0;
List<MetricFamilySamples> mfs = new ArrayList<>();

if (shouldCache() && shouldReturnFromCache()) {
LOGGER.log(Level.INFO, "Returning from cache");
Copy link
Contributor

@matthiasr matthiasr Jan 15, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should be debug level, otherwise it will be very noisy

this.updateCacheMetric(this.cachedMfs, 1.0);
return this.cachedMfs;
}
this.updateCacheMetric(mfs, 0.0);
try {
scrape(mfs);
} catch (Exception e) {
Expand Down Expand Up @@ -668,9 +721,28 @@ public List<MetricFamilySamples> collect() {
Type.GAUGE,
"Non-zero if this scrape failed.",
samples));
if (shouldCache()) {
this.cachedMfs = mfs;
}
this.lastCall = Instant.now();
return mfs;
}

public Instant lastCall;

private boolean shouldCache() {
return (int) this.activeConfig.globalConfig.get("globalCacheSeconds") > 0;
}

private boolean shouldReturnFromCache() {
if (this.lastCall == null) {
return false;
}
Duration elapsedTime = Duration.between(lastCall, Instant.now());
return elapsedTime.toSeconds()
<= (int) this.activeConfig.globalConfig.get("globalCacheSeconds");
}

private String extractResourceIdFromArn(String arn) {
// ARN parsing is based on
// https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
Expand Down
Loading