From 3bf45397fa860a53c051b05559dcdcf2df45b4a0 Mon Sep 17 00:00:00 2001 From: Shelby Hagman Date: Wed, 12 Nov 2025 22:44:31 +0000 Subject: [PATCH] out_cloudwatch_logs: increase MAX_EVENT_LEN to 1MB with tests Increase MAX_EVENT_LEN from 262,118 bytes (256 KiB) to 1,000,000 bytes (1 MB) to better align with AWS CloudWatch's documented maximum event size of 1,048,576 bytes (1 MiB). The 1 MB limit provides a ~4.6% safety margin to account for JSON encoding overhead. Testing confirmed messages up to 1,048,546 bytes (encoding to 1,048,586 bytes) succeed, though we use a conservative limit for production safety. Add runtime tests to validate the new limit: - event_size_at_limit: Validates events at exactly MAX_EVENT_LEN (1MB) are accepted - event_size_over_limit: Validates events exceeding MAX_EVENT_LEN are truncated - event_truncation_with_backslash: Validates backslash handling at truncation boundary Signed-off-by: Shelby Hagman --- plugins/out_cloudwatch_logs/cloudwatch_api.h | 13 +- tests/runtime/out_cloudwatch.c | 138 +++++++++++++++++++ 2 files changed, 149 insertions(+), 2 deletions(-) diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.h b/plugins/out_cloudwatch_logs/cloudwatch_api.h index 837e4b85ded..38713ca0cf4 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_api.h +++ b/plugins/out_cloudwatch_logs/cloudwatch_api.h @@ -43,8 +43,17 @@ /* Maximum number of character limits including both the Attributes key and its value */ #define ATTRIBUTES_MAX_LEN 300 -/* 256KiB minus 26 bytes for the event */ -#define MAX_EVENT_LEN 262118 +/* + * https://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/API_PutLogEvents.html + * AWS CloudWatch's documented maximum event size is 1,048,576 bytes (1 MiB), + * including JSON encoding overhead (structure, escaping, etc.). + * + * Setting MAX_EVENT_LEN to 1,000,000 bytes (1 MB) provides a ~4.6% safety margin + * to account for JSON encoding overhead and ensure reliable operation. + * Testing confirmed messages up to 1,048,546 bytes (encoding to 1,048,586 bytes) + * succeed, though we use a conservative limit for production safety. + */ +#define MAX_EVENT_LEN 1000000 /* Prefix used for entity fields only */ #define AWS_ENTITY_PREFIX "aws_entity" diff --git a/tests/runtime/out_cloudwatch.c b/tests/runtime/out_cloudwatch.c index 35d785f454c..bbfbf402054 100644 --- a/tests/runtime/out_cloudwatch.c +++ b/tests/runtime/out_cloudwatch.c @@ -5,10 +5,17 @@ /* Test data */ #include "data/td/json_td.h" /* JSON_TD */ +/* CloudWatch API constants */ +#include "../../plugins/out_cloudwatch_logs/cloudwatch_api.h" + #define ERROR_ALREADY_EXISTS "{\"__type\":\"ResourceAlreadyExistsException\"}" /* not a real error code, but tests that the code can respond to any error */ #define ERROR_UNKNOWN "{\"__type\":\"UNKNOWN\"}" +/* JSON structure constants for test message generation */ +static const char *TEST_JSON_PREFIX = "{\"message\":\""; +static const char *TEST_JSON_SUFFIX = "\"}"; + /* It writes a big JSON message (copied from TD test) */ void flb_test_cloudwatch_success(void) { @@ -393,6 +400,134 @@ void flb_test_cloudwatch_error_put_retention_policy(void) flb_destroy(ctx); } +/* Helper function to create a large JSON message of specified size */ +static char* create_large_json_message(size_t target_size) +{ + size_t prefix_len = strlen(TEST_JSON_PREFIX); + size_t suffix_len = strlen(TEST_JSON_SUFFIX); + size_t overhead = prefix_len + suffix_len; + size_t data_size; + char *json; + size_t i; + + /* Ensure target size can accommodate JSON structure */ + if (target_size < overhead + 1) { + target_size = overhead + 1; + } + + json = flb_malloc(target_size + 1); + if (!json) { + return NULL; + } + + /* Build JSON: prefix + data + suffix */ + strcpy(json, TEST_JSON_PREFIX); + data_size = target_size - overhead; + + /* Fill with 'A' characters */ + for (i = 0; i < data_size; i++) { + json[prefix_len + i] = 'A'; + } + + /* Close JSON object */ + strcpy(json + prefix_len + data_size, TEST_JSON_SUFFIX); + json[target_size] = '\0'; + + return json; +} + +/* Helper to setup and run a CloudWatch test with custom JSON data */ +static void run_cloudwatch_test_with_data(char *data, size_t data_len) +{ + int ret; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + + setenv("FLB_CLOUDWATCH_PLUGIN_UNDER_TEST", "true", 1); + + ctx = flb_create(); + TEST_CHECK(ctx != NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "cloudwatch_logs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "test", NULL); + flb_output_set(ctx, out_ffd, "region", "us-west-2", NULL); + flb_output_set(ctx, out_ffd, "log_group_name", "fluent", NULL); + flb_output_set(ctx, out_ffd, "log_stream_prefix", "from-fluent-", NULL); + flb_output_set(ctx, out_ffd, "auto_create_group", "On", NULL); + flb_output_set(ctx, out_ffd, "net.keepalive", "Off", NULL); + flb_output_set(ctx, out_ffd, "Retry_Limit", "1", NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + if (data) { + flb_lib_push(ctx, in_ffd, data, data_len); + } + + sleep(2); + flb_stop(ctx); + flb_destroy(ctx); +} + +/* Test event size at maximum allowed limit (should succeed without truncation) */ +void flb_test_cloudwatch_event_size_at_limit(void) +{ + char *large_json; + + /* Create message at MAX_EVENT_LEN */ + large_json = create_large_json_message(MAX_EVENT_LEN); + TEST_CHECK(large_json != NULL); + + if (large_json) { + run_cloudwatch_test_with_data(large_json, strlen(large_json)); + flb_free(large_json); + } +} + +/* Test event size exceeding limit (should be truncated to MAX_EVENT_LEN) */ +void flb_test_cloudwatch_event_size_over_limit(void) +{ + char *large_json; + + /* Create message exceeding MAX_EVENT_LEN by 1 byte to test truncation */ + large_json = create_large_json_message(MAX_EVENT_LEN + 1); + TEST_CHECK(large_json != NULL); + + if (large_json) { + run_cloudwatch_test_with_data(large_json, strlen(large_json)); + flb_free(large_json); + } +} + +/* Test event with trailing backslash at truncation boundary */ +void flb_test_cloudwatch_event_truncation_with_backslash(void) +{ + char *large_json; + size_t prefix_len = strlen(TEST_JSON_PREFIX); + size_t i; + + /* Create base message near MAX_EVENT_LEN */ + large_json = create_large_json_message(MAX_EVENT_LEN + 100); + TEST_CHECK(large_json != NULL); + + if (large_json) { + /* Post-process: replace every 100th character with backslash */ + for (i = 99; large_json[prefix_len + i] != '\0' && + large_json[prefix_len + i] != '"'; i += 100) { + large_json[prefix_len + i] = '\\'; + } + + run_cloudwatch_test_with_data(large_json, strlen(large_json)); + flb_free(large_json); + } +} + /* Test list */ TEST_LIST = { {"success", flb_test_cloudwatch_success }, @@ -405,5 +540,8 @@ TEST_LIST = { {"put_retention_policy_success", flb_test_cloudwatch_put_retention_policy_success }, {"already_exists_create_group_put_retention_policy", flb_test_cloudwatch_already_exists_create_group_put_retention_policy }, {"error_put_retention_policy", flb_test_cloudwatch_error_put_retention_policy }, + {"event_size_at_limit", flb_test_cloudwatch_event_size_at_limit }, + {"event_size_over_limit", flb_test_cloudwatch_event_size_over_limit }, + {"event_truncation_with_backslash", flb_test_cloudwatch_event_truncation_with_backslash }, {NULL, NULL} };