Skip to content

Commit fc0c624

Browse files
yaozhaoyyaozhaoy
authored
Handle log context with more than 100 metrics (#42)
* Handle log context with more than 100 metrics
1 parent 87ea5f0 commit fc0c624

File tree

13 files changed

+273
-73
lines changed

13 files changed

+273
-73
lines changed

‎src/main/java/software/amazon/cloudwatchlogs/emf/Constants.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,6 @@ public class Constants {
2020
public static final int DEFAULT_AGENT_PORT = 25888;
2121

2222
public static final String UNKNOWN = "Unknown";
23+
24+
public static final int MAX_METRICS_PER_EVENT = 100;
2325
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/Metadata.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,15 @@
2626
import java.util.HashMap;
2727
import java.util.List;
2828
import java.util.Map;
29+
import lombok.AllArgsConstructor;
2930
import lombok.Getter;
3031
import lombok.Setter;
32+
import lombok.With;
3133
import software.amazon.cloudwatchlogs.emf.serializers.InstantDeserializer;
3234
import software.amazon.cloudwatchlogs.emf.serializers.InstantSerializer;
3335

3436
/** Represents the MetaData part of the EMF schema. */
37+
@AllArgsConstructor
3538
class Metadata {
3639

3740
@Getter
@@ -44,6 +47,7 @@ class Metadata {
4447

4548
@Getter
4649
@Setter
50+
@With
4751
@JsonProperty("CloudWatchMetrics")
4852
private List<MetricDirective> cloudWatchMetrics;
4953

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinition.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,33 +16,48 @@
1616

1717
package software.amazon.cloudwatchlogs.emf.model;
1818

19+
import com.fasterxml.jackson.annotation.JsonIgnore;
1920
import com.fasterxml.jackson.annotation.JsonProperty;
2021
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
2122
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
23+
import java.util.ArrayList;
24+
import java.util.Arrays;
25+
import java.util.List;
2226
import lombok.AllArgsConstructor;
2327
import lombok.Getter;
2428
import lombok.NonNull;
25-
import lombok.Setter;
2629
import software.amazon.cloudwatchlogs.emf.serializers.UnitDeserializer;
2730
import software.amazon.cloudwatchlogs.emf.serializers.UnitSerializer;
2831

2932
/** Represents the MetricDefinition of the EMF schema. */
3033
@AllArgsConstructor
3134
class MetricDefinition {
3235
@NonNull
33-
@Setter
3436
@Getter
3537
@JsonProperty("Name")
3638
private String name;
3739

38-
@Setter
3940
@Getter
4041
@JsonProperty("Unit")
4142
@JsonSerialize(using = UnitSerializer.class)
4243
@JsonDeserialize(using = UnitDeserializer.class)
4344
private Unit unit;
4445

46+
@JsonIgnore @NonNull @Getter private List<Double> values;
47+
4548
MetricDefinition(String name) {
46-
this(name, Unit.NONE);
49+
this(name, Unit.NONE, new ArrayList<>());
50+
}
51+
52+
MetricDefinition(String name, double value) {
53+
this(name, Unit.NONE, value);
54+
}
55+
56+
MetricDefinition(String name, Unit unit, double value) {
57+
this(name, unit, new ArrayList<>(Arrays.asList(value)));
58+
}
59+
60+
void addValue(double value) {
61+
values.add(value);
4762
}
4863
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricDirective.java‎

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -16,43 +16,58 @@
1616

1717
package software.amazon.cloudwatchlogs.emf.model;
1818

19+
import com.fasterxml.jackson.annotation.JsonIgnore;
1920
import com.fasterxml.jackson.annotation.JsonProperty;
20-
import java.util.ArrayList;
21-
import java.util.Arrays;
22-
import java.util.List;
23-
import java.util.Set;
21+
import java.util.*;
2422
import java.util.stream.Collectors;
25-
import lombok.AccessLevel;
26-
import lombok.Getter;
27-
import lombok.Setter;
23+
import lombok.*;
2824

2925
/** Represents the MetricDirective part of the EMF schema. */
26+
@AllArgsConstructor
3027
class MetricDirective {
3128
@Setter
3229
@Getter
3330
@JsonProperty("Namespace")
34-
private String namespace = "aws-embedded-metrics";
31+
private String namespace;
3532

36-
@Setter
37-
@Getter
38-
@JsonProperty("Metrics")
39-
private List<MetricDefinition> metrics = new ArrayList<>();
33+
@JsonIgnore @Setter @Getter @With private Map<String, MetricDefinition> metrics;
4034

4135
@Getter(AccessLevel.PROTECTED)
42-
private List<DimensionSet> dimensions = new ArrayList<>();
36+
private List<DimensionSet> dimensions;
4337

4438
@Setter
4539
@Getter(AccessLevel.PROTECTED)
46-
private DimensionSet defaultDimensions = new DimensionSet();
40+
private DimensionSet defaultDimensions;
4741

48-
private boolean shouldUseDefaultDimension = true;
42+
private boolean shouldUseDefaultDimension;
43+
44+
MetricDirective() {
45+
namespace = "aws-embedded-metrics";
46+
metrics = new HashMap<>();
47+
dimensions = new ArrayList<>();
48+
defaultDimensions = new DimensionSet();
49+
shouldUseDefaultDimension = true;
50+
}
4951

5052
void putDimensionSet(DimensionSet dimensionSet) {
5153
dimensions.add(dimensionSet);
5254
}
5355

54-
void putMetric(MetricDefinition metric) {
55-
metrics.add(metric);
56+
void putMetric(String key, double value) {
57+
putMetric(key, value, Unit.NONE);
58+
}
59+
60+
void putMetric(String key, double value, Unit unit) {
61+
if (metrics.containsKey(key)) {
62+
metrics.get(key).addValue(value);
63+
} else {
64+
metrics.put(key, new MetricDefinition(key, unit, value));
65+
}
66+
}
67+
68+
@JsonProperty("Metrics")
69+
Collection<MetricDefinition> getAllMetrics() {
70+
return metrics.values();
5671
}
5772

5873
@JsonProperty("Dimensions")

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/MetricsContext.java‎

Lines changed: 35 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,9 @@
1717
package software.amazon.cloudwatchlogs.emf.model;
1818

1919
import com.fasterxml.jackson.core.JsonProcessingException;
20-
import java.util.Arrays;
21-
import java.util.List;
22-
import java.util.Map;
20+
import java.util.*;
2321
import lombok.Getter;
22+
import software.amazon.cloudwatchlogs.emf.Constants;
2423

2524
/** Stores metrics and their associated properties and dimensions. */
2625
public class MetricsContext {
@@ -101,8 +100,7 @@ public boolean hasDefaultDimensions() {
101100
* @param unit The unit of the metric
102101
*/
103102
public void putMetric(String key, double value, Unit unit) {
104-
metricDirective.putMetric(new MetricDefinition(key, unit));
105-
rootNode.putMetric(key, value);
103+
metricDirective.putMetric(key, value, unit);
106104
}
107105

108106
/**
@@ -201,12 +199,40 @@ public MetricsContext createCopyWithContext() {
201199
}
202200

203201
/**
204-
* Serialize the metrics in this context to a string.
202+
* Serialize the metrics in this context to strings. The EMF backend requires no more than 100
203+
* metrics in one log event. If there're more than 100 metrics, we split the metrics into
204+
* multiple log events.
205205
*
206-
* @return the serialized string
206+
* @return the serialized strings.
207207
* @throws JsonProcessingException if there's any object that cannot be serialized
208208
*/
209-
public String serialize() throws JsonProcessingException {
210-
return this.rootNode.serialize();
209+
public List<String> serialize() throws JsonProcessingException {
210+
if (rootNode.metrics().size() <= Constants.MAX_METRICS_PER_EVENT) {
211+
return Arrays.asList(this.rootNode.serialize());
212+
} else {
213+
List<RootNode> nodes = new ArrayList<>();
214+
Map<String, MetricDefinition> metrics = new HashMap<>();
215+
int count = 0;
216+
for (MetricDefinition metric : rootNode.metrics().values()) {
217+
metrics.put(metric.getName(), metric);
218+
count++;
219+
if (metrics.size() == Constants.MAX_METRICS_PER_EVENT
220+
|| count == rootNode.metrics().size()) {
221+
Metadata metadata = rootNode.getAws();
222+
MetricDirective metricDirective = metadata.getCloudWatchMetrics().get(0);
223+
Metadata clonedMetadata =
224+
metadata.withCloudWatchMetrics(
225+
Arrays.asList(metricDirective.withMetrics(metrics)));
226+
nodes.add(rootNode.withAws(clonedMetadata));
227+
metrics = new HashMap<>();
228+
}
229+
}
230+
231+
List<String> strings = new ArrayList<>();
232+
for (RootNode node : nodes) {
233+
strings.add(node.serialize());
234+
}
235+
return strings;
236+
}
211237
}
212238
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/model/RootNode.java‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,44 +22,38 @@
2222
import com.fasterxml.jackson.core.JsonProcessingException;
2323
import com.fasterxml.jackson.databind.ObjectMapper;
2424
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
25-
import java.util.ArrayList;
2625
import java.util.HashMap;
2726
import java.util.List;
2827
import java.util.Map;
28+
import lombok.AllArgsConstructor;
2929
import lombok.Getter;
30+
import lombok.With;
3031

3132
/** Represents the root of the EMF schema. */
33+
@AllArgsConstructor
3234
@JsonFilter("emptyMetricFilter")
3335
class RootNode {
3436
@Getter
37+
@With
3538
@JsonProperty("_aws")
36-
private Metadata aws = new Metadata();
39+
private Metadata aws;
3740

38-
private Map<String, Object> properties = new HashMap<>();
39-
private Map<String, List<Double>> metrics = new HashMap<>();
40-
private ObjectMapper objectMapper = new ObjectMapper();
41+
private Map<String, Object> properties;
42+
private ObjectMapper objectMapper;
4143

4244
RootNode() {
4345
final SimpleFilterProvider filterProvider =
4446
new SimpleFilterProvider().addFilter("emptyMetricFilter", new EmptyMetricsFilter());
47+
aws = new Metadata();
48+
properties = new HashMap<>();
49+
objectMapper = new ObjectMapper();
4550
objectMapper.setFilterProvider(filterProvider);
4651
}
4752

4853
public void putProperty(String key, Object value) {
4954
properties.put(key, value);
5055
}
5156

52-
/**
53-
* Add a metric measurement. Multiple calls using the same key will be stored as an array of
54-
* scalar values
55-
*/
56-
void putMetric(String key, double value) {
57-
if (!metrics.containsKey(key)) {
58-
metrics.put(key, new ArrayList<>());
59-
}
60-
metrics.get(key).add(value);
61-
}
62-
6357
Map<String, Object> getProperties() {
6458
return properties;
6559
}
@@ -70,9 +64,11 @@ Map<String, Object> getTargetMembers() {
7064
Map<String, Object> targetMembers = new HashMap<>();
7165
targetMembers.putAll(properties);
7266
targetMembers.putAll(getDimensions());
73-
for (Map.Entry<String, List<Double>> entry : metrics.entrySet()) {
74-
List<Double> values = entry.getValue();
75-
targetMembers.put(entry.getKey(), values.size() == 1 ? values.get(0) : values);
67+
for (MetricDirective metricDirective : aws.getCloudWatchMetrics()) {
68+
for (MetricDefinition metric : metricDirective.getMetrics().values()) {
69+
List<Double> values = metric.getValues();
70+
targetMembers.put(metric.getName(), values.size() == 1 ? values.get(0) : values);
71+
}
7672
}
7773
return targetMembers;
7874
}
@@ -88,6 +84,10 @@ Map<String, String> getDimensions() {
8884
return dimensions;
8985
}
9086

87+
Map<String, MetricDefinition> metrics() {
88+
return aws.getCloudWatchMetrics().get(0).getMetrics();
89+
}
90+
9191
String serialize() throws JsonProcessingException {
9292
return objectMapper.writeValueAsString(this);
9393
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/serializers/InstantSerializer.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ public class InstantSerializer extends StdSerializer<Instant> {
3737
public void serialize(Instant value, JsonGenerator jgen, SerializerProvider provider)
3838
throws IOException, JsonProcessingException {
3939

40-
// Just serialize dimensions as an array.
4140
jgen.writeNumber(value.toEpochMilli());
4241
}
4342
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/sinks/AgentSink.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,9 @@ public void accept(MetricsContext context) {
4848
}
4949

5050
try {
51-
client.sendMessage(context.serialize() + "\n");
51+
for (String event : context.serialize()) {
52+
client.sendMessage(event + "\n");
53+
}
5254
} catch (JsonProcessingException e) {
5355
log.error("Failed to serialize the metrics with the exception: ", e);
5456
}

‎src/main/java/software/amazon/cloudwatchlogs/emf/sinks/ConsoleSink.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,9 @@ public void accept(MetricsContext context) {
3333

3434
try {
3535
// CHECKSTYLE OFF
36-
System.out.println(context.serialize());
36+
for (String event : context.serialize()) {
37+
System.out.println(event);
38+
}
3739
// CHECKSTYLE ON
3840
} catch (JsonProcessingException e) {
3941
log.error("Failed to serialize a MetricsContext: ", e);

‎src/test/java/software/amazon/cloudwatchlogs/emf/model/MetricDefinitionTest.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020

2121
import com.fasterxml.jackson.core.JsonProcessingException;
2222
import com.fasterxml.jackson.databind.ObjectMapper;
23+
import java.util.Arrays;
2324
import org.junit.Test;
2425

2526
public class MetricDefinitionTest {
@@ -41,9 +42,18 @@ public void testSerializeMetricDefinitionWithoutUnit() throws JsonProcessingExce
4142
@Test
4243
public void testSerializeMetricDefinition() throws JsonProcessingException {
4344
ObjectMapper objectMapper = new ObjectMapper();
44-
MetricDefinition metricDefinition = new MetricDefinition("Time", Unit.MILLISECONDS);
45+
MetricDefinition metricDefinition = new MetricDefinition("Time", Unit.MILLISECONDS, 10);
4546
String metricString = objectMapper.writeValueAsString(metricDefinition);
4647

4748
assertEquals(metricString, "{\"Name\":\"Time\",\"Unit\":\"Milliseconds\"}");
4849
}
50+
51+
@Test
52+
public void testAddValue() {
53+
MetricDefinition md = new MetricDefinition("Time", Unit.MICROSECONDS, 10);
54+
assertEquals(Arrays.asList(10d), md.getValues());
55+
56+
md.addValue(20);
57+
assertEquals(Arrays.asList(10d, 20d), md.getValues());
58+
}
4959
}

0 commit comments

Comments
 (0)