From f9e15e8a952e88ee44afdd02d0678384cdd6b724 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Thu, 6 Nov 2025 13:19:17 +1300 Subject: [PATCH 01/31] network_verifier: add new Network Verifier plugin type Signed-off-by: Craig Robb --- include/fluent-bit/flb_network_verifier.h | 95 ++++++ .../fluent-bit/flb_network_verifier_plugin.h | 47 +++ src/flb_network.c | 23 +- src/flb_network_verifier.c | 308 ++++++++++++++++++ 4 files changed, 471 insertions(+), 2 deletions(-) create mode 100644 include/fluent-bit/flb_network_verifier.h create mode 100644 include/fluent-bit/flb_network_verifier_plugin.h create mode 100644 src/flb_network_verifier.c diff --git a/include/fluent-bit/flb_network_verifier.h b/include/fluent-bit/flb_network_verifier.h new file mode 100644 index 00000000000..f28993d72df --- /dev/null +++ b/include/fluent-bit/flb_network_verifier.h @@ -0,0 +1,95 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_NETWORK_VERIFIER_H +#define FLB_NETWORK_VERIFIER_H + +#include +#include +#include + +#include + +#define FLB_X509_STORE_EX_INDEX 0 + +struct flb_network_verifier_instance; + +struct flb_network_verifier_plugin { + char *name; /* Name */ + char *description; /* Description */ + + /* Config map */ + struct flb_config_map *config_map; + + /* Callbacks */ + int (*cb_init) (struct flb_network_verifier_instance *, struct flb_config *); + int (*cb_verify_tls) (int, X509_STORE_CTX *); + int (*cb_connection_failure) (struct flb_network_verifier_instance*, const char*, int, int, const char*); + int (*cb_exit) (void *, struct flb_config *); + + struct mk_list _head; /* Link to parent list (config->network_verifier_plugins) */ +}; + +/* + * Each initialized plugin must have an instance, the same plugin may be + * loaded more than one time. + * + * An instance will contain basic fixed plugin data while also + * allowing for plugin context data, generated when the plugin is invoked. + */ +struct flb_network_verifier_instance { + int id; /* instance id */ + int log_level; /* instance log level */ + char name[32]; /* numbered name */ + char *alias; /* alias name */ + void *context; /* Instance local context */ + struct flb_network_verifier_plugin *plugin; /* original plugin */ + + struct mk_list properties; /* config properties */ + struct mk_list *config_map; /* configuration map */ + + /* Keep a reference to the original context this instance belongs to */ + const struct flb_config *config; + + struct mk_list _head; /* config->network_verifiers */ +}; + +struct flb_network_verifier_instance *flb_network_verifier_new( + struct flb_config *config, const char *name); + +const char *flb_network_verifier_get_alias( + struct flb_network_verifier_instance *ins); + +int flb_network_verifier_set_property( + struct flb_network_verifier_instance *ins, const char *k, const char *v); +int flb_network_verifier_plugin_property_check( + struct flb_network_verifier_instance *ins, + struct flb_config *config); +int flb_network_verifier_init_all(struct flb_config *config); +void flb_network_verifier_exit(struct flb_config *config); + +void flb_network_verifier_instance_destroy( + struct flb_network_verifier_instance *ins); + +const struct flb_network_verifier_instance *find_network_verifier_instance( + struct flb_config *config, + const char* alias); + + +#endif \ No newline at end of file diff --git a/include/fluent-bit/flb_network_verifier_plugin.h b/include/fluent-bit/flb_network_verifier_plugin.h new file mode 100644 index 00000000000..c47a117d724 --- /dev/null +++ b/include/fluent-bit/flb_network_verifier_plugin.h @@ -0,0 +1,47 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_NETWORK_VERIFIER_PLUGIN_H +#define FLB_NETWORK_VERIFIER_PLUGIN_H + +#include +#include +#include + +#define flb_plg_log(ctx, level, fmt, ...) \ + if (flb_log_check_level(ctx->log_level, level)) \ + flb_log_print(level, NULL, 0, "[network_verifier:%s:%s] " fmt, \ + ctx->plugin->name, \ + flb_network_verifier_get_alias(ctx), ##__VA_ARGS__) + +#define flb_plg_error(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_ERROR, fmt, ##__VA_ARGS__) + +#define flb_plg_warn(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_WARN, fmt, ##__VA_ARGS__) + +#define flb_plg_info(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_INFO, fmt, ##__VA_ARGS__) + +#define flb_plg_debug(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_DEBUG, fmt, ##__VA_ARGS__) + +#define flb_plg_trace(ctx, fmt, ...) \ + flb_plg_log(ctx, FLB_LOG_TRACE, fmt, ##__VA_ARGS__) +#endif diff --git a/src/flb_network.c b/src/flb_network.c index 5d6937ca729..6216e05d7f1 100644 --- a/src/flb_network.c +++ b/src/flb_network.c @@ -1330,6 +1330,7 @@ flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, char address[41]; struct addrinfo hints; struct addrinfo *sorted_res, *res, *rp; + const char* error = NULL; if (is_async == FLB_TRUE && !u_conn) { flb_error("[net] invalid async mode with not set upstream connection"); @@ -1368,12 +1369,16 @@ flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, if (ret) { if (use_async_dns) { - flb_warn("[net] getaddrinfo(host='%s', err=%d): %s", host, ret, ares_strerror(ret)); + error = ares_strerror(ret); + flb_warn("[net] getaddrinfo(host='%s', err=%d): %s", host, ret, error); } else { - flb_warn("[net] getaddrinfo(host='%s', err=%d): %s", host, ret, gai_strerror(ret)); + error = gai_strerror(ret); + flb_warn("[net] getaddrinfo(host='%s', err=%d): %s", host, ret, error); } + flb_connection_notify_error(u_conn, host, port, ret, error); + return -1; } @@ -1382,6 +1387,9 @@ flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, flb_warn("[net] timeout detected between DNS lookup and connection attempt"); } + flb_connection_notify_error(u_conn, host, port, u_conn->net_error, + "Connection error"); + if (use_async_dns) { flb_net_free_translated_addrinfo(res); } @@ -1399,6 +1407,8 @@ flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, if (sorted_res == NULL) { flb_debug("[net] error sorting ipv4 getaddrinfo results"); + flb_connection_notify_error(u_conn, host, port, -1, + "Error sorting IPV4 results"); if (use_async_dns) { flb_net_free_translated_addrinfo(res); @@ -1415,6 +1425,8 @@ flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, if (sorted_res == NULL) { flb_debug("[net] error sorting ipv6 getaddrinfo results"); + flb_connection_notify_error(u_conn, host, port, -1, + "Error sorting IPV6 results"); if (use_async_dns) { flb_net_free_translated_addrinfo(res); @@ -1508,6 +1520,9 @@ flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, flb_debug("[net] socket #%i could not connect to %s:%s", fd, address, _port); + flb_connection_notify_error(u_conn, address, port, ret, + "Couldn't connect to end point"); + if (u_conn) { u_conn->fd = -1; u_conn->event.fd = -1; @@ -1525,6 +1540,8 @@ flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, if (fd == -1) { flb_debug("[net] could not connect to %s:%s", host, _port); + flb_connection_notify_error(u_conn, host, port, -1, + "Couldn't connect to end point file descriptor"); } if (use_async_dns) { @@ -1535,6 +1552,8 @@ flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, } if (rp == NULL) { + flb_connection_notify_error(u_conn, host, port, -1, + "Couldn't connect to remote endpoint"); return -1; } diff --git a/src/flb_network_verifier.c b/src/flb_network_verifier.c new file mode 100644 index 00000000000..d65aa881ad3 --- /dev/null +++ b/src/flb_network_verifier.c @@ -0,0 +1,308 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2025 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +static int instance_id(struct flb_config *config) +{ + struct flb_network_verifier_instance *entry; + + if (mk_list_size(&config->network_verifiers) == 0) { + return 0; + } + + entry = mk_list_entry_last(&config->network_verifiers, + struct flb_network_verifier_instance, + _head); + return (entry->id + 1); +} + +const char *flb_network_verifier_get_alias( + struct flb_network_verifier_instance *ins) +{ + if (ins->alias) { + return ins->alias; + } + + return ins->name; +} + +static int prop_key_check(const char *key, const char *kv, int k_len) +{ + int len = strlen(key); + if (strncasecmp(key, kv, k_len) == 0 && len == k_len) { + return 0; + } + + return -1; +} + +/* Initialize all network verify plugins */ +int flb_network_verifier_init_all(struct flb_config *config) +{ + int ret; + struct mk_list *tmp; + struct mk_list *head; + struct flb_network_verifier_plugin *plugin; + struct flb_network_verifier_instance *ins; + + /* Iterate all active network verify instance plugins */ + mk_list_foreach_safe(head, tmp, &config->network_verifiers) { + ins = mk_list_entry(head, struct flb_network_verifier_instance, + _head); + + if (ins->log_level == -1) { + ins->log_level = config->log->level; + } + + plugin = ins->plugin; + + /* + * Before to call the initialization callback, make sure that the received + * configuration parameters are valid if the plugin is registering a config map. + */ + if (flb_network_verifier_plugin_property_check(ins, config) == -1) { + flb_network_verifier_instance_destroy(ins); + return -1; + } + + /* Initialize the input */ + if (plugin->cb_init) { + ret = plugin->cb_init(ins, config); + if (ret != 0) { + flb_error("Failed initialize network_verifier %s", ins->name); + flb_network_verifier_instance_destroy(ins); + return -1; + } + } + } + + return 0; +} + +struct flb_network_verifier_instance *flb_network_verifier_new( + struct flb_config *config, const char *name) +{ + int id; + struct mk_list *head; + struct flb_network_verifier_plugin *plugin; + struct flb_network_verifier_instance *instance = NULL; + + if (!name) { + return NULL; + } + + mk_list_foreach(head, &config->network_verifier_plugins) { + plugin = mk_list_entry(head, struct flb_network_verifier_plugin, _head); + if (strcmp(plugin->name, name) == 0) { + break; + } + plugin = NULL; + } + + if (!plugin) { + return NULL; + } + + instance = flb_calloc(1, sizeof(struct flb_network_verifier_instance)); + if (!instance) { + flb_errno(); + return NULL; + } + instance->config = config; + + /* Get an ID */ + id = instance_id(config); + + /* format name (with instance id) */ + snprintf(instance->name, sizeof(instance->name) - 1, + "%s.%i", plugin->name, id); + + instance->id = id; + instance->alias = NULL; + instance->plugin = plugin; + instance->log_level = -1; + + mk_list_init(&instance->properties); + mk_list_add(&instance->_head, &config->network_verifiers); + + return instance; +} + +/* Override a configuration property for the given input_instance plugin */ +int flb_network_verifier_set_property(struct flb_network_verifier_instance *ins, + const char *k, const char *v) +{ + int len; + int ret; + flb_sds_t tmp; + struct flb_kv *kv; + const struct flb_config *config = ins->config; + + len = strlen(k); + tmp = flb_env_var_translate(config->env, v); + if (tmp) { + if (strlen(tmp) == 0) { + flb_sds_destroy(tmp); + tmp = NULL; + } + } + + if (prop_key_check("alias", k, len) == 0 && tmp) { + flb_utils_set_plugin_string_property("alias", &ins->alias, tmp); + } + else if (prop_key_check("log_level", k, len) == 0 && tmp) { + ret = flb_log_get_level_str(tmp); + flb_sds_destroy(tmp); + if (ret == -1) { + return -1; + } + ins->log_level = ret; + } + else { + /* + * Create the property, we don't pass the value since we will + * map it directly to avoid an extra memory allocation. + */ + kv = flb_kv_item_create(&ins->properties, (char *) k, NULL); + if (!kv) { + if (tmp) { + flb_sds_destroy(tmp); + } + return -1; + } + kv->val = tmp; + } + + return 0; +} + +int flb_network_verifier_plugin_property_check( + struct flb_network_verifier_instance *ins, struct flb_config *config) +{ + int ret = 0; + struct mk_list *config_map; + struct flb_network_verifier_plugin *plugin = ins->plugin; + + if (plugin->config_map) { + /* + * Create a dynamic version of the configmap that will be used by the specific + * instance in question. + */ + config_map = flb_config_map_create(config, plugin->config_map); + if (!config_map) { + flb_error("[network_verifier] error loading config map for '%s' plugin", + plugin->name); + return -1; + } + ins->config_map = config_map; + + if (!ins->alias || flb_sds_len(ins->alias) == 0) { + flb_error("[network_verifier] NO alias property for %s network_verifier instance.", + ins->name); + return -1; + } + + /* Validate incoming properties against config map */ + ret = flb_config_map_properties_check(ins->plugin->name, + &ins->properties, ins->config_map); + if (ret == -1) { + return -1; + } + } + + return 0; +} + +void flb_network_verifier_instance_exit(struct flb_network_verifier_instance *ins, + struct flb_config *config) +{ + struct flb_network_verifier_plugin *plugin = ins->plugin; + if (plugin->cb_exit && ins->context) { + plugin->cb_exit(ins->context, config); + } +} + +/* Invoke exit call for the network_verifier plugin */ +void flb_network_verifier_exit(struct flb_config *config) +{ + struct mk_list *tmp; + struct mk_list *head; + struct flb_network_verifier_instance *ins; + struct flb_network_verifier_plugin *plugin; + + mk_list_foreach_safe(head, tmp, &config->network_verifiers) { + ins = mk_list_entry(head, struct flb_network_verifier_instance, _head); + plugin = ins->plugin; + if (!plugin) { + continue; + } + flb_network_verifier_instance_exit(ins, config); + flb_network_verifier_instance_destroy(ins); + } +} + + +void flb_network_verifier_instance_destroy( + struct flb_network_verifier_instance *ins) +{ + if (!ins) { + return; + } + + /* destroy config map */ + if (ins->config_map) { + flb_config_map_destroy(ins->config_map); + } + + /* release properties */ + flb_kv_release(&ins->properties); + + if (ins->alias) { + flb_sds_destroy(ins->alias); + } + + mk_list_del(&ins->_head); + flb_free(ins); +} + +const struct flb_network_verifier_instance *find_network_verifier_instance( + struct flb_config *config, + const char* alias) +{ + struct mk_list *head; + struct flb_network_verifier_instance *verifier; + + if (!alias || strlen(alias) == 0) { + return NULL; + } + + mk_list_foreach(head, &config->network_verifiers) { + verifier = mk_list_entry(head, struct flb_network_verifier_instance, _head); + if (strcmp(verifier->alias, alias) == 0) { + return verifier; + } + } + + return NULL; +} From dbd740fc63e5441309301b12d3035beae74c1ca5 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Thu, 6 Nov 2025 13:20:15 +1300 Subject: [PATCH 02/31] cmakelists: added network verifier plugin register macro to cmakelists add flb_network_verifier.c and include openssl header directories Signed-off-by: Craig Robb --- CMakeLists.txt | 1 + plugins/CMakeLists.txt | 37 +++++++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 1 + 3 files changed, 39 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1059ddb8779..e53d5197e71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -743,6 +743,7 @@ if(FLB_TLS) find_package(OpenSSL) if(OPENSSL_FOUND) FLB_DEFINITION(FLB_HAVE_OPENSSL) + include_directories(${OPENSSL_INCLUDE_DIR}) endif() if (FLB_SYSTEM_WINDOWS AND NOT(OPENSSL_FOUND)) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 65d417d7cbb..5dd9aa1c73c 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -36,6 +36,43 @@ macro(REGISTER_CUSTOM_PLUGIN name) endif() endmacro() + +# REGISTER_NETWORK_VERIFIER_PLUGIN +macro(REGISTER_NETWORK_VERIFIER_PLUGIN name) + string(FIND ${name} "=" pos) + if(pos GREATER -1) + string(REPLACE "=" ";" list ${name}) + list(GET list 0 p_name) + list(GET list 1 p_path) + message(STATUS "EXTERNAL NETWORK_VERIFIER PLUGIN name='${p_name}' path='${p_path}'") + else() + set(p_name ${name}) + endif() + + string(TOUPPER ${p_name} NAME) + if(FLB_${NAME} OR p_path) + set(FLB_NETWORK_VERIFIER_PLUGINS_DECL "${FLB_NETWORK_VERIFIER_PLUGINS_DECL}extern struct flb_network_verifier_plugin ${p_name}_plugin;\n") + + # C code + set(C_CODE " network_verifier = flb_malloc(sizeof(struct flb_network_verifier_plugin));\n") + set(C_CODE "${C_CODE} if (!network_verifier) {\n") + set(C_CODE "${C_CODE} flb_errno();\n") + set(C_CODE "${C_CODE} return -1;\n") + set(C_CODE "${C_CODE} }\n") + set(C_CODE "${C_CODE} memcpy(network_verifier, &${p_name}_plugin, sizeof(struct flb_network_verifier_plugin));\n") + set(C_CODE "${C_CODE} mk_list_add(&network_verifier->_head, &config->network_verifier_plugins);\n\n") + + set(FLB_NETWORK_VERIFIER_PLUGINS_ADD "${FLB_NETWORK_VERIFIER_PLUGINS_ADD}${C_CODE}") + + if (p_path) + add_subdirectory(${p_path} ${p_path}) + else() + add_subdirectory(${p_name}) + endif() + set(flb_plugins "${flb_plugins}flb-plugin-${p_name};") + endif() +endmacro() + # REGISTER_IN_PLUGIN macro(REGISTER_IN_PLUGIN name) string(FIND ${name} "=" pos) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac5fe5ef863..e38a5089c2b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -159,6 +159,7 @@ if(FLB_TLS) # Register the TLS interface and functions set(src ${src} + flb_network_verifier.c "tls/flb_tls.c" "flb_oauth2.c" ) From b0ff1c72eb78a2fa6b60ac6a31810a30304eba0e Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Thu, 4 Sep 2025 14:01:40 +1200 Subject: [PATCH 03/31] config: added network_verifier context initialization from config Signed-off-by: Craig Robb --- include/fluent-bit/flb_config.h | 4 ++++ src/flb_config.c | 29 ++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index a27ec0c9eb9..d65531b84e3 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -136,6 +136,7 @@ struct flb_config { struct mk_list parser_plugins; /* not yet implemented */ struct mk_list filter_plugins; struct mk_list out_plugins; + struct mk_list network_verifier_plugins; /* Custom instances */ struct mk_list customs; @@ -156,6 +157,9 @@ struct flb_config { /* Filter instances */ struct mk_list filters; + /* Network Verifier instances */ + struct mk_list network_verifiers; + struct mk_event_loop *evl; /* the event loop (mk_core) */ struct flb_bucket_queue *evl_bktq; /* bucket queue for evl track event priority */ diff --git a/src/flb_config.c b/src/flb_config.c index 876e56bf072..5571beae09f 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -404,11 +404,13 @@ struct flb_config *flb_config_init() /* Initialize linked lists */ mk_list_init(&config->processor_plugins); mk_list_init(&config->custom_plugins); + mk_list_init(&config->network_verifier_plugins); mk_list_init(&config->in_plugins); mk_list_init(&config->parser_plugins); mk_list_init(&config->filter_plugins); mk_list_init(&config->out_plugins); mk_list_init(&config->customs); + mk_list_init(&config->network_verifiers); mk_list_init(&config->inputs); mk_list_init(&config->parsers); mk_list_init(&config->filters); @@ -851,6 +853,10 @@ static int configure_plugins_type(struct flb_config *config, struct flb_cf *cf, s_type = "custom"; list = &cf->customs; } + else if (type == FLB_CF_NETWORK_VERIFIER) { + s_type = "network_verifier"; + list = &cf->network_verifiers; + } else if (type == FLB_CF_INPUT) { s_type = "input"; list = &cf->inputs; @@ -884,6 +890,9 @@ static int configure_plugins_type(struct flb_config *config, struct flb_cf *cf, if (type == FLB_CF_CUSTOM) { ins = flb_custom_new(config, tmp, NULL); } + else if (type == FLB_CF_NETWORK_VERIFIER) { + ins = flb_network_verifier_new(config, tmp); + } else if (type == FLB_CF_INPUT) { ins = flb_input_new(config, tmp, NULL, FLB_TRUE); } @@ -930,6 +939,20 @@ static int configure_plugins_type(struct flb_config *config, struct flb_cf *cf, } } } + else if (type == FLB_CF_NETWORK_VERIFIER) { + if (kv->val->type == CFL_VARIANT_STRING) { + ret = flb_network_verifier_set_property(ins, + kv->key, + kv->val->data.as_string); + } else if (kv->val->type == CFL_VARIANT_ARRAY) { + for (i = 0; i < kv->val->data.as_array->entry_count; i++) { + val = kv->val->data.as_array->entries[i]; + ret = flb_network_verifier_set_property(ins, + kv->key, + val->data.as_string); + } + } + } else if (type == FLB_CF_INPUT) { if (kv->val->type == CFL_VARIANT_STRING) { ret = flb_input_set_property(ins, kv->key, kv->val->data.as_string); @@ -1048,6 +1071,7 @@ int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf) if (strcasecmp(s->name, "env") == 0 || strcasecmp(s->name, "service") == 0 || strcasecmp(s->name, "custom") == 0 || + strcasecmp(s->name, "network_verifier") == 0 || strcasecmp(s->name, "input") == 0 || strcasecmp(s->name, "filter") == 0 || strcasecmp(s->name, "output") == 0) { @@ -1106,7 +1130,10 @@ int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf) if (ret == -1) { return -1; } - + ret = configure_plugins_type(config, cf, FLB_CF_NETWORK_VERIFIER); + if (ret == -1) { + return -1; + } ret = configure_plugins_type(config, cf, FLB_CF_INPUT); if (ret == -1) { return -1; From 97821f58f8eff68a8d919e0ce7449df669a4a886 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Thu, 4 Sep 2025 14:15:40 +1200 Subject: [PATCH 04/31] tls: allow assignment of a network_verifier instance to a tls context pass through the network_verifier instance to TLS context creation and onto openssl, if non NULL will retrieve the verify callback function from the plugin and assign to the SSL_CTX_verify_callback. Also adds the network_verifier instance as a X509_STORE data variable so it is accessible for the callback function to use. Signed-off-by: Craig Robb --- include/fluent-bit/tls/flb_tls.h | 8 +++- src/tls/flb_tls.c | 12 +++++- src/tls/openssl.c | 74 +++++++++++++++++++++++++++++--- 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/include/fluent-bit/tls/flb_tls.h b/include/fluent-bit/tls/flb_tls.h index 47c261ebdf7..63db0338089 100644 --- a/include/fluent-bit/tls/flb_tls.h +++ b/include/fluent-bit/tls/flb_tls.h @@ -25,6 +25,8 @@ #include #include #include +#include + #include #define FLB_TLS_ALPN_MAX_LENGTH 16 @@ -69,7 +71,8 @@ struct flb_tls_backend { void *(*context_create) (int, int, int, const char *, const char *, const char *, const char *, - const char *, const char *); + const char *, const char *, + const struct flb_network_verifier_instance *); /* destroy backend context */ void (*context_destroy) (void *); @@ -126,7 +129,8 @@ struct flb_tls *flb_tls_create(int mode, const char *vhost, const char *ca_path, const char *ca_file, const char *crt_file, - const char *key_file, const char *key_passwd); + const char *key_file, const char *key_passwd, + const struct flb_network_verifier_instance *conn_ins); int flb_tls_destroy(struct flb_tls *tls); diff --git a/src/tls/flb_tls.c b/src/tls/flb_tls.c index 53d6cc53af5..953126580d2 100644 --- a/src/tls/flb_tls.c +++ b/src/tls/flb_tls.c @@ -98,6 +98,12 @@ struct flb_config_map tls_configmap[] = { "Specify TLS ciphers up to TLSv1.2" }, + { + FLB_CONFIG_MAP_STR, "network_verifier", NULL, + 0, FLB_FALSE, 0, + "Plugin alias to use for custom TLS / Network verification." + }, + /* EOF */ {0} }; @@ -188,7 +194,8 @@ struct flb_tls *flb_tls_create(int mode, const char *ca_file, const char *crt_file, const char *key_file, - const char *key_passwd) + const char *key_passwd, + const struct flb_network_verifier_instance *conn_ins) { void *backend; struct flb_tls *tls; @@ -199,7 +206,8 @@ struct flb_tls *flb_tls_create(int mode, backend = tls_context_create(verify, debug, mode, vhost, ca_path, ca_file, - crt_file, key_file, key_passwd); + crt_file, key_file, key_passwd, + conn_ins); if (!backend) { flb_error("[tls] could not create TLS backend"); return NULL; diff --git a/src/tls/openssl.c b/src/tls/openssl.c index 16241b7ed39..bff29cbcd86 100644 --- a/src/tls/openssl.c +++ b/src/tls/openssl.c @@ -55,12 +55,16 @@ */ #define OPENSSL_1_1_0 0x010100000L +#define FLB_ERR_CONN_HEADER "Failed to establish tls connection: " +#define FLB_ERR_CONN_HEADER_SIZE (sizeof(FLB_ERR_CONN_HEADER) - 1) + /* OpenSSL library context */ struct tls_context { int debug_level; SSL_CTX *ctx; int mode; char *alpn; + const struct flb_network_verifier_instance* verifier_ins; #if defined(FLB_SYSTEM_WINDOWS) char *certstore_name; int use_enterprise_store; @@ -78,6 +82,22 @@ struct tls_session { struct tls_context *parent; /* parent struct tls_context ref */ }; +void flb_tls_notify_error(const struct tls_context* tls_context, + int error_code, const char* error_msg) +{ + struct flb_network_verifier_instance* conn_verifier = NULL; + + if (tls_context != NULL) { + conn_verifier = tls_context->verifier_ins; + } + + if (conn_verifier && conn_verifier->plugin && + conn_verifier->plugin->cb_connection_failure) { + conn_verifier->plugin->cb_connection_failure(conn_verifier, NULL, 0, + error_code, error_msg); + } +} + static int tls_init(void) { /* @@ -608,13 +628,19 @@ static void *tls_context_create(int verify, const char *ca_file, const char *crt_file, const char *key_file, - const char *key_passwd) + const char *key_passwd, + const struct flb_network_verifier_instance *conn_ins) { int ret; SSL_CTX *ssl_ctx; struct tls_context *ctx; char err_buf[256]; char *key_log_filename; + X509_STORE* store = NULL; + SSL_verify_cb verify_cb = NULL; + if (conn_ins && conn_ins->plugin ) { + verify_cb = conn_ins->plugin->cb_verify_tls; + } /* * Init library ? based in the documentation on OpenSSL >= 1.1.0 is not longer @@ -671,6 +697,7 @@ static void *tls_context_create(int verify, ctx->mode = mode; ctx->alpn = NULL; ctx->debug_level = debug; + ctx->verifier_ins = conn_ins; #if defined(FLB_SYSTEM_WINDOWS) ctx->certstore_name = NULL; ctx->use_enterprise_store = 0; @@ -679,12 +706,28 @@ static void *tls_context_create(int verify, #endif pthread_mutex_init(&ctx->mutex, NULL); + if (verify_cb) { + store = SSL_CTX_get_cert_store(ssl_ctx); + if (!store) { + flb_error("[tls] failed to retrieve openssl certificate store."); + goto error; + } + ret = X509_STORE_set_ex_data(store, + FLB_X509_STORE_EX_INDEX, + (struct flb_network_verifier_instance *)conn_ins); + if (ret != 1) { + flb_error("[tls] Failed to set " + "network_verifier_instance in X509_STORE ex data"); + goto error; + } + } + /* Verify peer: by default OpenSSL always verify peer */ if (verify == FLB_FALSE) { SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_NONE, NULL); } else { - SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, NULL); + SSL_CTX_set_verify(ssl_ctx, SSL_VERIFY_PEER, verify_cb); } /* ca_path | ca_file */ @@ -1209,6 +1252,8 @@ static int tls_net_read(struct flb_tls_session *session, * to the net_error field. */ + flb_tls_notify_error(ctx, ret, err_buf); + session->connection->net_error = errno; ret = -1; @@ -1216,6 +1261,7 @@ static int tls_net_read(struct flb_tls_session *session, else if (ret < 0) { ERR_error_string_n(ret, err_buf, sizeof(err_buf)-1); flb_error("[tls] error: %s", err_buf); + flb_tls_notify_error(ctx, ret, err_buf); } else { ret = -1; @@ -1267,15 +1313,18 @@ static int tls_net_write(struct flb_tls_session *session, if (ERR_get_error() == 0) { if (ret == 0) { flb_debug("[tls] connection closed"); + flb_tls_notify_error(ctx, ret, "Connection Closed"); } else { flb_error("[tls] syscall error: %s", strerror(errno)); + flb_tls_notify_error(ctx, errno, strerror(errno)); } } else { err_code = ERR_get_error(); ERR_error_string_n(err_code, err_buf, sizeof(err_buf) - 1); flb_error("[tls] syscall error: %s", err_buf); + flb_tls_notify_error(ctx, err_code, err_buf); } /* According to the documentation these are non-recoverable @@ -1291,10 +1340,12 @@ static int tls_net_write(struct flb_tls_session *session, err_code = ERR_get_error(); if (err_code == 0) { flb_error("[tls] unknown error"); + flb_tls_notify_error(ctx, err_code, "Unknown error"); } else { ERR_error_string_n(err_code, err_buf, sizeof(err_buf) - 1); flb_error("[tls] error: %s", err_buf); + flb_tls_notify_error(ctx, err_code, err_buf); } ret = -1; @@ -1334,11 +1385,13 @@ static int tls_net_handshake(struct flb_tls *tls, { int ret = 0; long ssl_code = 0; - char err_buf[256]; + char err_buf[FLB_ERR_CONN_HEADER_SIZE + 256] = {0}; struct tls_session *session = ptr_session; struct tls_context *ctx; const char *x509_err; + strcpy(err_buf, FLB_ERR_CONN_HEADER); + ctx = session->parent; pthread_mutex_lock(&ctx->mutex); @@ -1407,7 +1460,7 @@ static int tls_net_handshake(struct flb_tls *tls, /* The SSL_ERROR_SYSCALL with errno value of 0 indicates unexpected * EOF from the peer. This is fixed in OpenSSL 3.0. */ - + if (ret == 0) { ssl_code = SSL_get_verify_result(session->ssl); if (ssl_code != X509_V_OK) { @@ -1416,11 +1469,18 @@ static int tls_net_handshake(struct flb_tls *tls, flb_error("[tls] certificate verification failed, reason: %s (X509 code: %ld)", x509_err, ssl_code); } else { - flb_error("[tls] error: unexpected EOF"); + strncpy(&err_buf[FLB_ERR_CONN_HEADER_SIZE], + "unexpected EOF", + sizeof(err_buf)-FLB_ERR_CONN_HEADER_SIZE); + flb_error("[tls] error: %s", &err_buf[FLB_ERR_CONN_HEADER_SIZE]); + flb_tls_notify_error(ctx, -1, err_buf); } } else { - ERR_error_string_n(ret, err_buf, sizeof(err_buf)-1); - flb_error("[tls] error: %s", err_buf); + ERR_error_string_n(ret, + &err_buf[FLB_ERR_CONN_HEADER_SIZE], + sizeof(err_buf)-FLB_ERR_CONN_HEADER_SIZE); + flb_error("[tls] error: %s", &err_buf[FLB_ERR_CONN_HEADER_SIZE]); + flb_tls_notify_error(ctx, -1, err_buf); } pthread_mutex_unlock(&ctx->mutex); From d28e22aac780630c4f5c1ea2403c357e509c0efa Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:14:11 +1200 Subject: [PATCH 05/31] input: add network_verifier property and pass to TLS context Signed-off-by: Craig Robb --- include/fluent-bit/flb_input.h | 4 ++++ src/flb_input.c | 22 +++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/include/fluent-bit/flb_input.h b/include/fluent-bit/flb_input.h index fc01d0e9dd4..4b3ebed5004 100644 --- a/include/fluent-bit/flb_input.h +++ b/include/fluent-bit/flb_input.h @@ -458,6 +458,10 @@ struct flb_input_instance { char *tls_max_version; /* Maximum protocol version of TLS */ char *tls_ciphers; /* TLS ciphers */ + + char *network_verifier; /* Network Verifier alias */ + struct flb_network_verifier_instance* verifier_ins; + struct mk_list *tls_config_map; #ifdef FLB_HAVE_TLS diff --git a/src/flb_input.c b/src/flb_input.c index 97ebb1e4ad9..5c8bafc488c 100644 --- a/src/flb_input.c +++ b/src/flb_input.c @@ -406,6 +406,7 @@ struct flb_input_instance *flb_input_new(struct flb_config *config, instance->tls_key_file = NULL; instance->tls_key_passwd = NULL; #endif + instance->network_verifier = NULL; /* Plugin requires a co-routine context ? */ if (plugin->flags & FLB_INPUT_CORO) { @@ -687,6 +688,9 @@ int flb_input_set_property(struct flb_input_instance *ins, flb_utils_set_plugin_string_property("tls.ciphers", &ins->tls_ciphers, tmp); } #endif + else if (prop_key_check("network_verifier", k, len) == 0) { + flb_utils_set_plugin_string_property("network_verifier", &ins->network_verifier, tmp); + } else if (prop_key_check("storage.type", k, len) == 0 && tmp) { /* Set the storage type */ if (strcasecmp(tmp, "filesystem") == 0) { @@ -869,6 +873,10 @@ void flb_input_instance_destroy(struct flb_input_instance *ins) flb_sds_destroy(ins->tls_ciphers); } + if (ins->network_verifier) { + flb_sds_destroy(ins->network_verifier); + } + /* release the tag if any */ flb_sds_destroy(ins->tag); @@ -1281,6 +1289,13 @@ int flb_input_instance_init(struct flb_input_instance *ins, return -1; } + ins->verifier_ins = find_network_verifier_instance(config, ins->network_verifier); + if (!ins->verifier_ins && ins->network_verifier) { + flb_error("[input %s] network_verifier '%s' not found", ins->name, + ins->network_verifier); + return -1; + } + #ifdef FLB_HAVE_TLS if (ins->use_tls == FLB_TRUE) { if ((p->flags & FLB_INPUT_NET_SERVER) != 0) { @@ -1313,7 +1328,8 @@ int flb_input_instance_init(struct flb_input_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (ins->tls == NULL) { flb_error("[input %s] error initializing TLS context", @@ -2222,6 +2238,8 @@ int flb_input_upstream_set(struct flb_upstream *u, struct flb_input_instance *in /* Set networking options 'net.*' received through instance properties */ memcpy(&u->base.net, &ins->net_setup, sizeof(struct flb_net_setup)); + u->base.verifier_ins = ins->verifier_ins; + return 0; } @@ -2242,6 +2260,8 @@ int flb_input_downstream_set(struct flb_downstream *stream, mk_list_add(&stream->base._head, &ins->downstreams); } + stream->base.verifier_ins = ins->verifier_ins; + return 0; } From bfcb6ce4118e3fcf71bb8af4221ce2f933cbf014 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:14:25 +1200 Subject: [PATCH 06/31] output: add network_verifier property and pass to TLS context Signed-off-by: Craig Robb --- include/fluent-bit/flb_output.h | 2 ++ src/flb_output.c | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index 6ad593a3652..b19b8008419 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -377,6 +377,8 @@ struct flb_output_instance { char *tls_win_thumbprints; /* CertStore Thumbprints (Windows) */ # endif #endif + char* network_verifier; /* Connection Verifier alias */ + struct flb_network_verifier_instance* verifier_ins; /* * network info: diff --git a/src/flb_output.c b/src/flb_output.c index 307905b3379..e560ef90c2b 100644 --- a/src/flb_output.c +++ b/src/flb_output.c @@ -785,6 +785,7 @@ struct flb_output_instance *flb_output_new(struct flb_config *config, instance->tls_win_thumbprints = NULL; # endif #endif + instance->network_verifier = NULL; if (plugin->flags & FLB_OUTPUT_NET) { ret = flb_net_host_set(plugin->name, &instance->host, output); @@ -1021,6 +1022,9 @@ int flb_output_set_property(struct flb_output_instance *ins, } # endif #endif + else if (prop_key_check("network_verifier", k, len) == 0 && tmp) { + flb_utils_set_plugin_string_property("network_verifier", &ins->network_verifier, tmp); + } else if (prop_key_check("storage.total_limit_size", k, len) == 0 && tmp) { if (strcasecmp(tmp, "off") == 0 || flb_utils_bool(tmp) == FLB_FALSE) { @@ -1373,6 +1377,13 @@ int flb_output_init_all(struct flb_config *config) "retried_records", ins->metrics); } #endif + ins->verifier_ins = find_network_verifier_instance(config, + ins->network_verifier); + if (!ins->verifier_ins && ins->network_verifier) { + flb_error("[output %s] network_verifier '%s' not found", ins->name, + ins->network_verifier); + return -1; + } #ifdef FLB_HAVE_TLS if (ins->use_tls == FLB_TRUE) { @@ -1384,7 +1395,8 @@ int flb_output_init_all(struct flb_config *config) ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ins->tls) { flb_error("[output %s] error initializing TLS context", ins->name); @@ -1636,6 +1648,8 @@ int flb_output_upstream_set(struct flb_upstream *u, struct flb_output_instance * /* Set networking options 'net.*' received through instance properties */ memcpy(&u->base.net, &ins->net_setup, sizeof(struct flb_net_setup)); + u->base.verifier_ins = ins->verifier_ins; + /* * If the Upstream was created using a proxy from the environment but the * final configuration asks to ignore environment proxy variables, restore From 8220be97bf38f4f72544c676217659f8431a3dd0 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:15:42 +1200 Subject: [PATCH 07/31] upstream: add network_verifier property and pass to TLS context Signed-off-by: Craig Robb --- include/fluent-bit/flb_upstream_node.h | 2 ++ src/flb_upstream_ha.c | 11 +++++++++-- src/flb_upstream_node.c | 13 ++++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/include/fluent-bit/flb_upstream_node.h b/include/fluent-bit/flb_upstream_node.h index 8f203392f28..387eb9321b0 100644 --- a/include/fluent-bit/flb_upstream_node.h +++ b/include/fluent-bit/flb_upstream_node.h @@ -43,6 +43,7 @@ struct flb_upstream_node { char *tls_crt_file; /* Certificate */ char *tls_key_file; /* Cert Key */ char *tls_key_passwd; /* Cert Key Password */ + char *network_verifier; /* Network Verifier alias */ /* context with mbedTLS contexts and data */ struct flb_tls *tls; @@ -74,6 +75,7 @@ struct flb_upstream_node *flb_upstream_node_create(flb_sds_t name, flb_sds_t hos const char *tls_crt_file, const char *tls_key_file, const char *tls_key_passwd, + const char *network_verifier, struct flb_hash_table *ht, struct flb_config *config); const char *flb_upstream_node_get_property(const char *prop, diff --git a/src/flb_upstream_ha.c b/src/flb_upstream_ha.c index d01e55ad5f3..6d83e9615b2 100644 --- a/src/flb_upstream_ha.c +++ b/src/flb_upstream_ha.c @@ -153,6 +153,7 @@ static struct flb_upstream_node *create_node(int id, char *tls_crt_file = NULL; char *tls_key_file = NULL; char *tls_key_passwd = NULL; + char *network_verifier = NULL; flb_sds_t translated_value; struct cfl_list *head; struct cfl_kvpair *entry; @@ -161,7 +162,7 @@ static struct flb_upstream_node *create_node(int id, "tls", "tls.vhost", "tls.verify", "tls.debug", "tls.ca_path", "tls.ca_file", "tls.crt_file", "tls.key_file", "tls.key_passwd", - "tls.verify_hostname", NULL}; + "tls.verify_hostname", "network_verifier", NULL}; struct flb_upstream_node *node; @@ -235,6 +236,8 @@ static struct flb_upstream_node *create_node(int id, /* tls.key_file */ tls_key_passwd = flb_cf_section_property_get_string(cf, s, "tls.key_passwd"); + network_verifier = flb_cf_section_property_get_string(cf, s, "network_verifier"); + translate_environment_variables((flb_sds_t *) &name, config, FLB_TRUE); translate_environment_variables((flb_sds_t *) &host, config, FLB_TRUE); translate_environment_variables((flb_sds_t *) &port, config, FLB_TRUE); @@ -244,6 +247,7 @@ static struct flb_upstream_node *create_node(int id, translate_environment_variables((flb_sds_t *) &tls_crt_file, config, FLB_TRUE); translate_environment_variables((flb_sds_t *) &tls_key_file, config, FLB_TRUE); translate_environment_variables((flb_sds_t *) &tls_key_passwd, config, FLB_TRUE); + translate_environment_variables((flb_sds_t *) &network_verifier, config, FLB_TRUE); /* * Create hash table to store unknown key/values that might be used @@ -322,7 +326,7 @@ static struct flb_upstream_node *create_node(int id, tls_verify_hostname, tls_debug, tls_vhost, tls_ca_path, tls_ca_file, tls_crt_file, tls_key_file, - tls_key_passwd, ht, config); + tls_key_passwd, network_verifier, ht, config); /* Teardown for created flb_sds_t stuffs by flb_cf_section_property_get_string(). */ if (tls_vhost != NULL) { @@ -349,6 +353,9 @@ static struct flb_upstream_node *create_node(int id, flb_sds_destroy(tls_key_passwd); } + if (network_verifier != NULL) { + flb_sds_destroy(network_verifier); + } return node; } diff --git a/src/flb_upstream_node.c b/src/flb_upstream_node.c index 5131efd0d6b..3a727ede485 100644 --- a/src/flb_upstream_node.c +++ b/src/flb_upstream_node.c @@ -38,6 +38,7 @@ struct flb_upstream_node *flb_upstream_node_create(flb_sds_t name, flb_sds_t hos const char *tls_crt_file, const char *tls_key_file, const char *tls_key_passwd, + const char *network_verifier, struct flb_hash_table *ht, struct flb_config *config) { @@ -46,6 +47,7 @@ struct flb_upstream_node *flb_upstream_node_create(flb_sds_t name, flb_sds_t hos int io_flags; char tmp[255]; struct flb_upstream_node *node; + const struct flb_network_verifier_instance *conn_ins; if (!host || !port) { return NULL; @@ -121,6 +123,12 @@ struct flb_upstream_node *flb_upstream_node_create(flb_sds_t name, flb_sds_t hos flb_upstream_node_destroy(node); return NULL; } + + node->network_verifier = flb_sds_create(network_verifier); + if (!node->network_verifier) { + flb_upstream_node_destroy(node); + return NULL; + } #endif /* hash table */ @@ -129,6 +137,7 @@ struct flb_upstream_node *flb_upstream_node_create(flb_sds_t name, flb_sds_t hos #ifdef FLB_HAVE_TLS /* TLS setup */ if (tls == FLB_TRUE) { + conn_ins = find_network_verifier_instance(config, network_verifier); node->tls = flb_tls_create(FLB_TLS_CLIENT_MODE, tls_verify, tls_debug, @@ -137,7 +146,8 @@ struct flb_upstream_node *flb_upstream_node_create(flb_sds_t name, flb_sds_t hos tls_ca_file, tls_crt_file, tls_key_file, - tls_key_passwd); + tls_key_passwd, + conn_ins); if (!node->tls) { flb_error("[upstream_node] error initializing TLS context " "on node '%s'", name); @@ -215,6 +225,7 @@ void flb_upstream_node_destroy(struct flb_upstream_node *node) flb_sds_destroy(node->tls_crt_file); flb_sds_destroy(node->tls_key_file); flb_sds_destroy(node->tls_key_passwd); + flb_sds_destroy(node->network_verifier); if (node->tls) { flb_tls_destroy(node->tls); } From 17053096e09cea05889ac37fbf1a0eea5eababe3 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:23:58 +1200 Subject: [PATCH 08/31] s3: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- plugins/out_s3/s3.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/out_s3/s3.c b/plugins/out_s3/s3.c index 7dc07170341..a1614631adf 100644 --- a/plugins/out_s3/s3.c +++ b/plugins/out_s3/s3.c @@ -602,7 +602,7 @@ static int cb_s3_init(struct flb_output_instance *ins, struct flb_split_entry *tok; struct mk_list *split; int list_size; - + FLB_TLS_INIT(s3_worker_info); ctx = flb_calloc(1, sizeof(struct flb_s3)); @@ -865,7 +865,8 @@ static int cb_s3_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->client_tls) { flb_plg_error(ctx->ins, "Failed to create tls context"); return -1; @@ -881,7 +882,8 @@ static int cb_s3_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->provider_tls) { flb_errno(); return -1; @@ -915,7 +917,8 @@ static int cb_s3_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->sts_provider_tls) { flb_errno(); @@ -1986,6 +1989,7 @@ static int blob_initialize_authorization_endpoint_upstream(struct flb_s3 *contex NULL, NULL, NULL, + NULL, NULL); flb_free(host); From 976a7eb9a95e993b69293abb28d0a701d321dc9c Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:16:53 +1200 Subject: [PATCH 09/31] config_format: added yaml parsing for the network_verifier plugins Signed-off-by: Craig Robb --- include/fluent-bit/config_format/flb_cf.h | 4 ++++ src/config_format/flb_cf_yaml.c | 24 +++++++++++++++++++++-- src/config_format/flb_config_format.c | 12 ++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h index 6c6233c28bc..cf296534270 100644 --- a/include/fluent-bit/config_format/flb_cf.h +++ b/include/fluent-bit/config_format/flb_cf.h @@ -60,6 +60,7 @@ enum section_type { FLB_CF_PLUGINS, /* plugins */ FLB_CF_UPSTREAM_SERVERS, /* upstream_servers */ FLB_CF_CUSTOM, /* [CUSTOM] */ + FLB_CF_NETWORK_VERIFIER, /* [network_verifier] */ FLB_CF_INPUT, /* [INPUT] */ FLB_CF_FILTER, /* [FILTER] */ FLB_CF_OUTPUT, /* [OUTPUT] */ @@ -112,6 +113,9 @@ struct flb_cf { /* 'custom' type plugins */ struct mk_list customs; + /* 'network_verifier' type plugins */ + struct mk_list network_verifiers; + /* pipeline */ struct mk_list inputs; struct mk_list filters; diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c index e5e220cc60c..2e1c66827cb 100644 --- a/src/config_format/flb_cf_yaml.c +++ b/src/config_format/flb_cf_yaml.c @@ -53,6 +53,7 @@ enum section { SECTION_SERVICE, SECTION_PIPELINE, SECTION_CUSTOM, + SECTION_NETWORK_VERIFIERS, SECTION_INPUT, SECTION_FILTER, SECTION_OUTPUT, @@ -72,6 +73,7 @@ static char *section_names[] = { "service", "pipeline", "custom", + "network_verifiers", "input", "filter", "output", @@ -124,6 +126,8 @@ enum state { STATE_OTHER, /* any other unknown section */ STATE_CUSTOM, /* custom plugins */ + STATE_NETWORK_VERIFIERS,/* Network verifiers plugins */ + STATE_PIPELINE, /* pipeline groups customs inputs, filters and outputs */ STATE_PLUGIN_INPUT, /* input plugins section */ @@ -269,6 +273,8 @@ static char *state_str(enum state val) return "other"; case STATE_CUSTOM: return "custom"; + case STATE_NETWORK_VERIFIERS: + return "network_verifiers"; case STATE_PIPELINE: return "pipeline"; case STATE_PLUGIN_INPUT: @@ -332,6 +338,9 @@ static int add_section_type(struct flb_cf *conf, struct parser_state *state) else if (state->section == SECTION_CUSTOM) { state->cf_section = flb_cf_section_create(conf, "customs", 0); } + else if (state->section == SECTION_NETWORK_VERIFIERS) { + state->cf_section = flb_cf_section_create(conf, "network_verifiers", 0); + } else if (state->section == SECTION_PARSER) { state->cf_section = flb_cf_section_create(conf, "parser", 0); } @@ -1493,10 +1502,11 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, break; /* - * 'customs' + * 'customs' & 'network_verifiers' * -------- */ case STATE_CUSTOM: + case STATE_NETWORK_VERIFIERS: switch (event->type) { case YAML_SEQUENCE_START_EVENT: break; @@ -1524,7 +1534,7 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, return YAML_FAILURE; } break; - /* end of 'customs' */ + /* end of 'customs' & 'network_verifiers' */ case STATE_PIPELINE: switch (event->type) { @@ -1648,6 +1658,16 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, return YAML_FAILURE; } } + else if (strcasecmp(value, "network_verifiers") == 0) { + state = state_push_section(ctx, + STATE_NETWORK_VERIFIERS, + SECTION_NETWORK_VERIFIERS); + + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + } else if (strcasecmp(value, "includes") == 0) { state = state_push_section(ctx, STATE_INCLUDE, SECTION_INCLUDE); diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index e6d55bf29c1..6111002d0f2 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -134,6 +134,9 @@ struct flb_cf *flb_cf_create() /* 'custom' type plugins */ mk_list_init(&ctx->customs); + /* 'network verifiers' type plugins */ + mk_list_init(&ctx->network_verifiers); + /* pipeline */ mk_list_init(&ctx->inputs); mk_list_init(&ctx->filters); @@ -194,6 +197,10 @@ static enum section_type get_section_type(char *name, int len) strncasecmp(name, "customs", len) == 0) { return FLB_CF_CUSTOM; } + else if (strncasecmp(name, "network_verifier", len) == 0 || + strncasecmp(name, "network_verifiers", len) == 0) { + return FLB_CF_NETWORK_VERIFIER; + } else if (strncasecmp(name, "input", len) == 0 || strncasecmp(name, "inputs", len) == 0) { return FLB_CF_INPUT; @@ -705,6 +712,9 @@ struct flb_cf_section *flb_cf_section_create(struct flb_cf *cf, char *name, int else if (type == FLB_CF_CUSTOM) { mk_list_add(&s->_head_section, &cf->customs); } + else if (type == FLB_CF_NETWORK_VERIFIER) { + mk_list_add(&s->_head_section, &cf->network_verifiers); + } else if (type == FLB_CF_INPUT) { mk_list_add(&s->_head_section, &cf->inputs); } @@ -804,6 +814,8 @@ static char *section_type_str(int type) return "UPSTREAM_SERVERS"; case FLB_CF_CUSTOM: return "CUSTOM"; + case FLB_CF_NETWORK_VERIFIER: + return "NETWORK_VERIFIER"; case FLB_CF_INPUT: return "INPUT"; case FLB_CF_FILTER: From 634388d2aa87f8d2f047c2e0471891651a1d05d1 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:18:11 +1200 Subject: [PATCH 10/31] tests: updated tests for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- tests/runtime/in_tcp.c | 1 + tests/runtime/out_tcp.c | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/runtime/in_tcp.c b/tests/runtime/in_tcp.c index 732af004198..e80ef98cc84 100644 --- a/tests/runtime/in_tcp.c +++ b/tests/runtime/in_tcp.c @@ -373,6 +373,7 @@ void flb_test_tcp_with_tls() NULL, NULL, NULL, + NULL, NULL); TEST_CHECK(tls != NULL); diff --git a/tests/runtime/out_tcp.c b/tests/runtime/out_tcp.c index 98f323ba3ed..99b9e21cffa 100644 --- a/tests/runtime/out_tcp.c +++ b/tests/runtime/out_tcp.c @@ -330,6 +330,7 @@ void flb_test_tcp_with_tls() NULL, TLS_CERTIFICATE_FILENAME, TLS_PRIVATE_KEY_FILENAME, + NULL, NULL); TEST_CHECK(tls != NULL); From 226e9cd07dbab6e7c4dedc945a2ab422d6ebb6de Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:19:56 +1200 Subject: [PATCH 11/31] kubernetes: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- plugins/filter_kubernetes/kube_meta.c | 6 +++--- plugins/in_kubernetes_events/kubernetes_events_conf.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/filter_kubernetes/kube_meta.c b/plugins/filter_kubernetes/kube_meta.c index c27b6371b13..5fad59642b0 100644 --- a/plugins/filter_kubernetes/kube_meta.c +++ b/plugins/filter_kubernetes/kube_meta.c @@ -2026,7 +2026,7 @@ int flb_kube_pod_association_init(struct flb_kube *ctx, struct flb_config *confi ctx->aws_pod_association_host_server_ca_file, ctx->aws_pod_association_host_client_cert_file, ctx->aws_pod_association_host_client_key_file, - NULL); + NULL, NULL); if (!ctx->aws_pod_association_tls) { flb_plg_error(ctx->ins, "[kube_meta] could not create TLS config for pod association host"); return -1; @@ -2071,7 +2071,7 @@ static int flb_kubelet_network_init(struct flb_kube *ctx, struct flb_config *con ctx->tls_vhost, ctx->tls_ca_path, ctx->tls_ca_file, - NULL, NULL, NULL); + NULL, NULL, NULL, NULL); if (!ctx->kubelet_tls) { return -1; } @@ -2126,7 +2126,7 @@ static int flb_kube_network_init(struct flb_kube *ctx, struct flb_config *config ctx->tls_vhost, ctx->tls_ca_path, ctx->tls_ca_file, - NULL, NULL, NULL); + NULL, NULL, NULL, NULL); if (!ctx->tls) { return -1; } diff --git a/plugins/in_kubernetes_events/kubernetes_events_conf.c b/plugins/in_kubernetes_events/kubernetes_events_conf.c index e40d67b415e..d367aa63eec 100644 --- a/plugins/in_kubernetes_events/kubernetes_events_conf.c +++ b/plugins/in_kubernetes_events/kubernetes_events_conf.c @@ -109,7 +109,7 @@ static int network_init(struct k8s_events *ctx, struct flb_config *config) ctx->tls_vhost, ctx->tls_ca_path, ctx->tls_ca_file, - NULL, NULL, NULL); + NULL, NULL, NULL, NULL); if (!ctx->tls) { return -1; } From 1d3efddea4c79f48c34e16b88b02f7d9f651f938 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:20:09 +1200 Subject: [PATCH 12/31] nightfall: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- plugins/filter_nightfall/nightfall.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/filter_nightfall/nightfall.c b/plugins/filter_nightfall/nightfall.c index 3e37d2a0bc4..9cb9d1c337b 100644 --- a/plugins/filter_nightfall/nightfall.c +++ b/plugins/filter_nightfall/nightfall.c @@ -96,8 +96,8 @@ static int cb_nightfall_init(struct flb_filter_instance *f_ins, ctx->tls_debug, ctx->tls_vhost, ctx->tls_ca_path, - NULL, - NULL, NULL, NULL); + NULL, NULL, NULL, + NULL, NULL); if (!ctx->tls) { flb_plg_error(f_ins, "tls initialization error"); flb_free(ctx); From 644f965c64de78e86b62cb33f262ef9a219ad5b2 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:20:27 +1200 Subject: [PATCH 13/31] azure: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- plugins/out_azure_blob/azure_blob_conf.c | 1 + plugins/out_azure_kusto/azure_kusto_conf.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_azure_blob/azure_blob_conf.c b/plugins/out_azure_blob/azure_blob_conf.c index ea883a01852..cae7654ad07 100644 --- a/plugins/out_azure_blob/azure_blob_conf.c +++ b/plugins/out_azure_blob/azure_blob_conf.c @@ -375,6 +375,7 @@ static int flb_azure_blob_apply_remote_configuration(struct flb_azure_blob *cont NULL, NULL, NULL, + NULL, NULL); if (tls_context == NULL) { diff --git a/plugins/out_azure_kusto/azure_kusto_conf.c b/plugins/out_azure_kusto/azure_kusto_conf.c index fa899eab686..48e51fe0994 100644 --- a/plugins/out_azure_kusto/azure_kusto_conf.c +++ b/plugins/out_azure_kusto/azure_kusto_conf.c @@ -89,7 +89,7 @@ static struct flb_upstream_node *flb_upstream_node_create_url(struct flb_azure_k NULL, sds_host, sds_port, FLB_TRUE, ctx->ins->tls->verify, ctx->ins->tls->verify_hostname, ctx->ins->tls->debug, ctx->ins->tls->vhost, NULL, NULL, NULL, - NULL, NULL, kv, config); + NULL, NULL, NULL, kv, config); if (!node) { flb_plg_error(ctx->ins, "error creating resource upstream node"); From 60da7727657aca24b4d7539a916f709557eed06b Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:21:29 +1200 Subject: [PATCH 14/31] kinesis: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- plugins/out_kinesis_firehose/firehose.c | 9 ++++++--- plugins/out_kinesis_streams/kinesis.c | 10 +++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/plugins/out_kinesis_firehose/firehose.c b/plugins/out_kinesis_firehose/firehose.c index 0b998599f05..a00ba0d43c0 100644 --- a/plugins/out_kinesis_firehose/firehose.c +++ b/plugins/out_kinesis_firehose/firehose.c @@ -181,7 +181,8 @@ static int cb_firehose_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->cred_tls) { flb_plg_error(ctx->ins, "Failed to create tls context"); @@ -196,7 +197,8 @@ static int cb_firehose_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->client_tls) { flb_plg_error(ctx->ins, "Failed to create tls context"); goto error; @@ -232,7 +234,8 @@ static int cb_firehose_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->sts_tls) { flb_errno(); goto error; diff --git a/plugins/out_kinesis_streams/kinesis.c b/plugins/out_kinesis_streams/kinesis.c index 03556752340..9760c528ec9 100644 --- a/plugins/out_kinesis_streams/kinesis.c +++ b/plugins/out_kinesis_streams/kinesis.c @@ -168,7 +168,8 @@ static int cb_kinesis_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->cred_tls) { flb_plg_error(ctx->ins, "Failed to create tls context"); @@ -183,7 +184,8 @@ static int cb_kinesis_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->client_tls) { flb_plg_error(ctx->ins, "Failed to create tls context"); goto error; @@ -226,7 +228,9 @@ static int cb_kinesis_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); + if (!ctx->sts_tls) { flb_errno(); goto error; From 4f63896f148db63e143cf4bc434c0246bbda7e2c Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:22:29 +1200 Subject: [PATCH 15/31] bigquery: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- plugins/out_bigquery/bigquery.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/plugins/out_bigquery/bigquery.c b/plugins/out_bigquery/bigquery.c index c5cfeeb2b25..1d75f710bb4 100644 --- a/plugins/out_bigquery/bigquery.c +++ b/plugins/out_bigquery/bigquery.c @@ -695,7 +695,8 @@ static int cb_bigquery_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->aws_tls) { flb_plg_error(ctx->ins, "Failed to create TLS context"); @@ -734,7 +735,8 @@ static int cb_bigquery_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->aws_sts_tls) { flb_plg_error(ctx->ins, "Failed to create TLS context"); @@ -765,7 +767,8 @@ static int cb_bigquery_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->google_sts_tls) { flb_plg_error(ctx->ins, "Failed to create TLS context"); @@ -793,7 +796,8 @@ static int cb_bigquery_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->google_iam_tls) { flb_plg_error(ctx->ins, "Failed to create TLS context"); From 0945db420571a3e1afecb038ad761a95c42a3c7a Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:22:47 +1200 Subject: [PATCH 16/31] cloudwatch: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- plugins/out_cloudwatch_logs/cloudwatch_logs.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/out_cloudwatch_logs/cloudwatch_logs.c b/plugins/out_cloudwatch_logs/cloudwatch_logs.c index cc03ed8a879..3c312b3dc46 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_logs.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_logs.c @@ -251,7 +251,8 @@ static int cb_cloudwatch_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->cred_tls) { flb_plg_error(ctx->ins, "Failed to create tls context"); @@ -266,7 +267,8 @@ static int cb_cloudwatch_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->client_tls) { flb_plg_error(ctx->ins, "Failed to create tls context"); goto error; @@ -302,7 +304,8 @@ static int cb_cloudwatch_init(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->sts_tls) { flb_errno(); goto error; From afcc339fb31c5e063b9dba981d26f3ffd0ba11ea Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:23:12 +1200 Subject: [PATCH 17/31] es_conf: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- plugins/out_es/es_conf.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/out_es/es_conf.c b/plugins/out_es/es_conf.c index 4bc2977c5eb..0407801c886 100644 --- a/plugins/out_es/es_conf.c +++ b/plugins/out_es/es_conf.c @@ -382,7 +382,8 @@ struct flb_elasticsearch *flb_es_conf_create(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->aws_tls) { flb_errno(); flb_es_conf_destroy(ctx); @@ -443,7 +444,8 @@ struct flb_elasticsearch *flb_es_conf_create(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->aws_sts_tls) { flb_errno(); flb_es_conf_destroy(ctx); From ffe68f51491b3c4392b3bc90a347d28fc444f07a Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:23:45 +1200 Subject: [PATCH 18/31] os_conf: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- plugins/out_opensearch/os_conf.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/out_opensearch/os_conf.c b/plugins/out_opensearch/os_conf.c index a42cedbd31d..cf247391f24 100644 --- a/plugins/out_opensearch/os_conf.c +++ b/plugins/out_opensearch/os_conf.c @@ -257,7 +257,8 @@ struct flb_opensearch *flb_os_conf_create(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->aws_tls) { flb_errno(); flb_os_conf_destroy(ctx); @@ -318,7 +319,8 @@ struct flb_opensearch *flb_os_conf_create(struct flb_output_instance *ins, ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!ctx->aws_sts_tls) { flb_errno(); flb_os_conf_destroy(ctx); From 6fc7c556983ff5bc91f2429505a38342a443f637 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:24:15 +1200 Subject: [PATCH 19/31] aws_credential: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- src/aws/flb_aws_credentials.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/aws/flb_aws_credentials.c b/src/aws/flb_aws_credentials.c index 75f13b111f2..38c316d4a3b 100644 --- a/src/aws/flb_aws_credentials.c +++ b/src/aws/flb_aws_credentials.c @@ -383,7 +383,8 @@ struct flb_aws_provider *flb_managed_chain_provider_create(struct flb_output_ins ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!cred_tls) { flb_plg_error(ins, "Failed to create TLS instance for AWS Provider"); flb_errno(); @@ -434,7 +435,8 @@ struct flb_aws_provider *flb_managed_chain_provider_create(struct flb_output_ins ins->tls_ca_file, ins->tls_crt_file, ins->tls_key_file, - ins->tls_key_passwd); + ins->tls_key_passwd, + ins->verifier_ins); if (!sts_tls) { flb_plg_error(ins, "Failed to create TLS instance for AWS STS Credential " "Provider"); From 763a7caad987b94abef8c0a4041523d9e4080b5e Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:24:35 +1200 Subject: [PATCH 20/31] oauth2: updated for network_verifier argument in tls_context_create function Signed-off-by: Craig Robb --- src/flb_oauth2.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/flb_oauth2.c b/src/flb_oauth2.c index 866fbe8e132..5921b038559 100644 --- a/src/flb_oauth2.c +++ b/src/flb_oauth2.c @@ -220,7 +220,8 @@ struct flb_oauth2 *flb_oauth2_create(struct flb_config *config, NULL, /* ca_file */ NULL, /* crt_file */ NULL, /* key_file */ - NULL); /* key_passwd */ + NULL, /* key_passwd */ + NULL); /* verifier */ if (!ctx->tls) { flb_error("[oauth2] error initializing TLS context"); goto error; From 21e8e522cafe6f2226523f0a4845acf199bb7f3d Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:26:24 +1200 Subject: [PATCH 21/31] engine: call init / exit for network_verifier plugin instances on engine start / shutdown Signed-off-by: Craig Robb --- src/flb_engine.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/flb_engine.c b/src/flb_engine.c index c4d7f54ffe5..4356b480326 100644 --- a/src/flb_engine.c +++ b/src/flb_engine.c @@ -924,6 +924,12 @@ int flb_engine_start(struct flb_config *config) return -1; } + /* Initialize network verifier plugins */ + ret = flb_network_verifier_init_all(config); + if (ret == -1) { + return -1; + } + /* Start the Storage engine */ ret = flb_storage_create(config); if (ret == -1) { @@ -1317,6 +1323,7 @@ int flb_engine_shutdown(struct flb_config *config) flb_filter_exit(config); flb_output_exit(config); flb_custom_exit(config); + flb_network_verifier_exit(config); flb_input_exit_all(config); /* scheduler */ From 353e55a4c24278485d25a561486fcf6278fee6f1 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Fri, 5 Sep 2025 07:26:59 +1200 Subject: [PATCH 22/31] reload: check network_verifier properties on hot reload Signed-off-by: Craig Robb --- src/flb_reload.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/flb_reload.c b/src/flb_reload.c index 0e6675e787c..d49b2f40970 100644 --- a/src/flb_reload.c +++ b/src/flb_reload.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -205,6 +206,32 @@ static int flb_custom_propery_check_all(struct flb_config *config) return 0; } +static int flb_network_verifier_property_check_all(struct flb_config *config) +{ + int ret; + struct mk_list *tmp; + struct mk_list *head; + struct flb_network_verifier_instance *ins; + + /* Iterate all active network verifier instance plugins */ + mk_list_foreach_safe(head, tmp, &config->network_verifiers) { + ins = mk_list_entry(head, struct flb_network_verifier_instance, _head); + + /* Check plugin property */ + ret = flb_network_verifier_plugin_property_check(ins, config); + if (ret == -1) { + return -1; + } + + /* destroy config map (will be recreated at flb_start) */ + if (ins->config_map) { + flb_config_map_destroy(ins->config_map); + ins->config_map = NULL; + } + } + return 0; +} + int flb_reload_property_check_all(struct flb_config *config) { int ret = 0; @@ -241,6 +268,14 @@ int flb_reload_property_check_all(struct flb_config *config) return -1; } + /* Check properties of tls verifier plugins */ + ret = flb_network_verifier_property_check_all(config); + if (ret == -1) { + flb_error("[reload] check properties for network verifier plugins has failed"); + + return -1; + } + return 0; } From 33ca71ddf12689ac308ecb6255d2d925a9078fea Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Tue, 9 Sep 2025 15:16:51 +1200 Subject: [PATCH 23/31] plugin: added network_verifier type for loading of plugin Signed-off-by: Craig Robb --- include/fluent-bit/flb_plugin.h | 2 ++ include/fluent-bit/flb_plugins.h.in | 11 ++++++++++ src/flb_plugin.c | 33 ++++++++++++++++++++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/include/fluent-bit/flb_plugin.h b/include/fluent-bit/flb_plugin.h index af78b1cde91..76f93d156ab 100644 --- a/include/fluent-bit/flb_plugin.h +++ b/include/fluent-bit/flb_plugin.h @@ -28,6 +28,7 @@ #define FLB_PLUGIN_FILTER 2 #define FLB_PLUGIN_OUTPUT 3 #define FLB_PLUGIN_PROCESSOR 4 +#define FLB_PLUGIN_NETWORK_VERIFIER 5 /* Informational contexts for discovered dynamic plugins */ struct flb_plugin { @@ -42,6 +43,7 @@ struct flb_plugins { struct mk_list processor; struct mk_list filter; struct mk_list output; + struct mk_list network_verifier; }; struct flb_plugins *flb_plugin_create(); diff --git a/include/fluent-bit/flb_plugins.h.in b/include/fluent-bit/flb_plugins.h.in index 7f29b401b55..363ca648bde 100644 --- a/include/fluent-bit/flb_plugins.h.in +++ b/include/fluent-bit/flb_plugins.h.in @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -34,6 +35,7 @@ extern struct flb_output_plugin *flb_zig_native_output_plugin_init(void *); @FLB_OUT_PLUGINS_DECL@ @FLB_FILTER_PLUGINS_DECL@ @FLB_PROCESSOR_PLUGINS_DECL@ +@FLB_NETWORK_VERIFIER_PLUGINS_DECL@ int flb_plugins_register(struct flb_config *config) { @@ -42,12 +44,14 @@ int flb_plugins_register(struct flb_config *config) struct flb_output_plugin *out; struct flb_filter_plugin *filter; struct flb_processor_plugin *processor; + struct flb_network_verifier_plugin *network_verifier; @FLB_CUSTOM_PLUGINS_ADD@ @FLB_IN_PLUGINS_ADD@ @FLB_OUT_PLUGINS_ADD@ @FLB_FILTER_PLUGINS_ADD@ @FLB_PROCESSOR_PLUGINS_ADD@ +@FLB_NETWORK_VERIFIER_PLUGINS_ADD@ return 0; } @@ -61,6 +65,7 @@ void flb_plugins_unregister(struct flb_config *config) struct flb_output_plugin *out; struct flb_filter_plugin *filter; struct flb_processor_plugin *processor; + struct flb_network_verifier_plugin *network_verifier; mk_list_foreach_safe(head, tmp, &config->custom_plugins) { custom = mk_list_entry(head, struct flb_custom_plugin, _head); @@ -100,6 +105,12 @@ void flb_plugins_unregister(struct flb_config *config) mk_list_del(&processor->_head); flb_free(processor); } + + mk_list_foreach_safe(head, tmp, &config->network_verifier_plugins) { + network_verifier = mk_list_entry(head, struct flb_network_verifier_plugin, _head); + mk_list_del(&network_verifier->_head); + flb_free(network_verifier); + } } #endif diff --git a/src/flb_plugin.c b/src/flb_plugin.c index 66c86083634..92b166fd32f 100644 --- a/src/flb_plugin.c +++ b/src/flb_plugin.c @@ -77,6 +77,15 @@ static int is_output(char *name) return FLB_FALSE; } +static int is_network_verifier(char *name) +{ + if (strncmp(name, "network_verifier_", 17) == 0) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + static void *get_handle(const char *path) { void *handle; @@ -155,7 +164,8 @@ static char *path_to_plugin_name(char *path) if (is_input(name) == FLB_FALSE && is_processor(name) == FLB_FALSE && is_filter(name) == FLB_FALSE && - is_output(name) == FLB_FALSE) { + is_output(name) == FLB_FALSE && + is_network_verifier(name) == FLB_FALSE) { flb_error("[plugin] invalid plugin type: %s", name); flb_free(name); return NULL; @@ -193,6 +203,7 @@ struct flb_plugins *flb_plugin_create() mk_list_init(&ctx->processor); mk_list_init(&ctx->filter); mk_list_init(&ctx->output); + mk_list_init(&ctx->network_verifier); return ctx; } @@ -209,6 +220,7 @@ int flb_plugin_load(char *path, struct flb_plugins *ctx, struct flb_processor_plugin *processor; struct flb_filter_plugin *filter; struct flb_output_plugin *output; + struct flb_network_verifier_plugin *network_verifier = NULL; /* Open the shared object file: dlopen(3) */ dso_handle = get_handle(path); @@ -286,6 +298,18 @@ int flb_plugin_load(char *path, struct flb_plugins *ctx, memcpy(output, symbol, sizeof(struct flb_output_plugin)); mk_list_add(&output->_head, &config->out_plugins); } + else if (is_network_verifier(plugin_stname) == FLB_TRUE) { + type = FLB_PLUGIN_NETWORK_VERIFIER; + network_verifier = flb_malloc(sizeof(struct flb_network_verifier_plugin)); + if (!network_verifier) { + flb_errno(); + flb_free(plugin_stname); + dlclose(dso_handle); + return -1; + } + memcpy(network_verifier, symbol, sizeof(struct flb_network_verifier_plugin)); + mk_list_add(&network_verifier->_head, &config->network_verifier_plugins); + } flb_free(plugin_stname); if (type == -1) { @@ -319,6 +343,9 @@ int flb_plugin_load(char *path, struct flb_plugins *ctx, else if (type == FLB_PLUGIN_OUTPUT) { mk_list_add(&plugin->_head, &ctx->output); } + else if (type == FLB_PLUGIN_NETWORK_VERIFIER) { + mk_list_add(&plugin->_head, &ctx->network_verifier); + } return 0; } @@ -489,5 +516,9 @@ void flb_plugin_destroy(struct flb_plugins *ctx) destroy_plugin(plugin); } + mk_list_foreach_safe(head, tmp, &ctx->network_verifier) { + plugin = mk_list_entry(head, struct flb_plugin, _head); + destroy_plugin(plugin); + } flb_free(ctx); } From 53e52e8194487d15d3aec407e382dcb99fcc3ae9 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Tue, 4 Nov 2025 10:53:15 +1300 Subject: [PATCH 24/31] connection: added Network Verifier to the connection interface. Signed-off-by: Craig Robb --- include/fluent-bit/flb_connection.h | 3 +++ src/flb_connection.c | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/include/fluent-bit/flb_connection.h b/include/fluent-bit/flb_connection.h index 67e731bd86e..31f5380ec76 100644 --- a/include/fluent-bit/flb_connection.h +++ b/include/fluent-bit/flb_connection.h @@ -181,4 +181,7 @@ void flb_connection_unset_connection_timeout(struct flb_connection *connection); void flb_connection_reset_io_timeout(struct flb_connection *connection); void flb_connection_unset_io_timeout(struct flb_connection *connection); +void flb_connection_notify_error(const struct flb_connection* conn, + const char* dest, int port, int error_code, const char* error_msg); + #endif diff --git a/src/flb_connection.c b/src/flb_connection.c index a3ed402651b..7fc035e2c26 100644 --- a/src/flb_connection.c +++ b/src/flb_connection.c @@ -3,6 +3,7 @@ #include #include #include +#include int flb_connection_setup(struct flb_connection *connection, flb_sockfd_t socket, @@ -254,4 +255,20 @@ void flb_connection_unset_io_timeout(struct flb_connection *connection) assert(connection != NULL); connection->ts_io_timeout = -1; +} + +void flb_connection_notify_error(const struct flb_connection* conn, + const char* dest, int port, int error_code, const char* error_msg) +{ + struct flb_network_verifier_instance* conn_verifier = NULL; + + if (conn && conn->stream) { + conn_verifier = conn->stream->verifier_ins; + } + + if (conn_verifier && conn_verifier->plugin && + conn_verifier->plugin->cb_connection_failure) { + conn_verifier->plugin->cb_connection_failure(conn_verifier, dest, port, + error_code, error_msg); + } } \ No newline at end of file From 3f16e79f57e3da2d57366ced264c339d2924ebc2 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Tue, 4 Nov 2025 10:53:17 +1300 Subject: [PATCH 25/31] stream: Added a pointer to Network Verifier instance to be able to notify of network errors Signed-off-by: Craig Robb --- include/fluent-bit/flb_stream.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/fluent-bit/flb_stream.h b/include/fluent-bit/flb_stream.h index cce54e3a33c..6e8585df983 100644 --- a/include/fluent-bit/flb_stream.h +++ b/include/fluent-bit/flb_stream.h @@ -48,6 +48,8 @@ struct flb_stream { struct flb_net_setup net; struct mk_list _head; + + struct flb_network_verifier_instance* verifier_ins; }; static inline int flb_stream_is_shutting_down(struct flb_stream *stream) From 9ae74919a6aa82bd9679e365236e66b89364acd3 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Tue, 4 Nov 2025 10:53:18 +1300 Subject: [PATCH 26/31] io: notify Network Verifier on network issues Signed-off-by: Craig Robb --- src/flb_io.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/flb_io.c b/src/flb_io.c index ab7e7aa3332..bbcedad94ed 100644 --- a/src/flb_io.c +++ b/src/flb_io.c @@ -144,6 +144,10 @@ int flb_io_net_connect(struct flb_connection *connection, connection->upstream->tcp_host, connection->upstream->tcp_port); + flb_connection_notify_error(connection, connection->upstream->tcp_host, + connection->upstream->tcp_port, -1, + "Couldn't connect to client proxy"); + flb_socket_close(fd); connection->fd = -1; connection->event.fd = -1; @@ -162,6 +166,10 @@ int flb_io_net_connect(struct flb_connection *connection, if (ret == -1) { flb_socket_close(fd); + + flb_connection_notify_error(connection, connection->upstream->tcp_host, + connection->upstream->tcp_port, -1, + "Error in keep-alive"); connection->fd = -1; connection->event.fd = -1; return -1; From 61fac8591da67b1bc2bf5953a89b135a8a1421d3 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Thu, 13 Nov 2025 10:01:41 +1300 Subject: [PATCH 27/31] output: added flb_network_verifier header file to have network_verifier_instance reference Signed-off-by: Craig Robb --- include/fluent-bit/flb_output.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index b19b8008419..7c214b896ff 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -48,6 +48,7 @@ #include #include #include +#include #include #include From b30d592e291d2f2082f92ab4249a3e3ee4c18a71 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Thu, 13 Nov 2025 10:02:09 +1300 Subject: [PATCH 28/31] engine: cleanup network_verifiers after all input plugins are exited Signed-off-by: Craig Robb --- src/flb_engine.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/flb_engine.c b/src/flb_engine.c index 4356b480326..175955cccbd 100644 --- a/src/flb_engine.c +++ b/src/flb_engine.c @@ -1323,9 +1323,11 @@ int flb_engine_shutdown(struct flb_config *config) flb_filter_exit(config); flb_output_exit(config); flb_custom_exit(config); - flb_network_verifier_exit(config); flb_input_exit_all(config); + /* cleanup network verifier after other plugins potentially using it */ + flb_network_verifier_exit(config); + /* scheduler */ sched_params = (struct flb_sched_timer_coro_cb_params *) FLB_TLS_GET(sched_timer_coro_cb_params); if (sched_params && sched_params->magic == FLB_SCHED_TLS_MAGIC) { From d9a4b4447d0a8282c296c193eb89127415030fae Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Thu, 13 Nov 2025 10:03:50 +1300 Subject: [PATCH 29/31] network_verifier: add null alias check when searching for instance Signed-off-by: Craig Robb --- src/flb_network_verifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_network_verifier.c b/src/flb_network_verifier.c index d65aa881ad3..4f33917d184 100644 --- a/src/flb_network_verifier.c +++ b/src/flb_network_verifier.c @@ -299,7 +299,7 @@ const struct flb_network_verifier_instance *find_network_verifier_instance( mk_list_foreach(head, &config->network_verifiers) { verifier = mk_list_entry(head, struct flb_network_verifier_instance, _head); - if (strcmp(verifier->alias, alias) == 0) { + if (verifier->alias != NULL && strcmp(verifier->alias, alias) == 0) { return verifier; } } From 0a4e6301dc032ca716cc40682e3bbc94b83ae966 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Thu, 13 Nov 2025 10:07:18 +1300 Subject: [PATCH 30/31] network: report a -1 error code when failing to connect to endpoint Signed-off-by: Craig Robb --- src/flb_network.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_network.c b/src/flb_network.c index 6216e05d7f1..5d98beb8feb 100644 --- a/src/flb_network.c +++ b/src/flb_network.c @@ -1520,7 +1520,7 @@ flb_sockfd_t flb_net_tcp_connect(const char *host, unsigned long port, flb_debug("[net] socket #%i could not connect to %s:%s", fd, address, _port); - flb_connection_notify_error(u_conn, address, port, ret, + flb_connection_notify_error(u_conn, address, port, -1, "Couldn't connect to end point"); if (u_conn) { From 2c60d5a2ac46673ed89cff0fb416a5079a9500b7 Mon Sep 17 00:00:00 2001 From: Craig Robb Date: Thu, 13 Nov 2025 10:08:12 +1300 Subject: [PATCH 31/31] reload: updated comment to use consistent terminology Signed-off-by: Craig Robb --- src/flb_reload.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_reload.c b/src/flb_reload.c index d49b2f40970..0674ba6b38d 100644 --- a/src/flb_reload.c +++ b/src/flb_reload.c @@ -268,7 +268,7 @@ int flb_reload_property_check_all(struct flb_config *config) return -1; } - /* Check properties of tls verifier plugins */ + /* Check properties of network verifier plugins */ ret = flb_network_verifier_property_check_all(config); if (ret == -1) { flb_error("[reload] check properties for network verifier plugins has failed");