diff --git a/include/fluent-bit/config_format/flb_cf.h b/include/fluent-bit/config_format/flb_cf.h index b5ae20fa327..4fd645dd176 100644 --- a/include/fluent-bit/config_format/flb_cf.h +++ b/include/fluent-bit/config_format/flb_cf.h @@ -90,8 +90,8 @@ struct flb_cf { /* global service */ struct flb_cf_section *service; - /* config environment variables (env section, availble on YAML) */ - struct mk_list env; + /* config environment variables (env section, available on YAML) */ + struct mk_list env; /* list of struct flb_cf_env_var */ /* meta commands (used by fluentbit classic mode) */ struct mk_list metas; @@ -141,9 +141,20 @@ void flb_cf_destroy(struct flb_cf *cf); int flb_cf_set_origin_format(struct flb_cf *cf, int format); void flb_cf_dump(struct flb_cf *cf); -struct flb_kv *flb_cf_env_property_add(struct flb_cf *cf, - char *k_buf, size_t k_len, - char *v_buf, size_t v_len); +struct flb_cf_env_var { + flb_sds_t name; + flb_sds_t value; + flb_sds_t uri; + int refresh_interval; + struct mk_list _head; +}; + +struct flb_cf_env_var *flb_cf_env_var_add(struct flb_cf *cf, + char *name, size_t name_len, + char *value, size_t value_len, + char *uri, size_t uri_len, + int refresh_interval); + /* metas */ struct flb_kv *flb_cf_meta_property_add(struct flb_cf *cf, char *meta, int len); diff --git a/include/fluent-bit/flb_config_map.h b/include/fluent-bit/flb_config_map.h index a41db3e4424..3afe979b53f 100644 --- a/include/fluent-bit/flb_config_map.h +++ b/include/fluent-bit/flb_config_map.h @@ -51,6 +51,7 @@ #define FLB_CONFIG_MAP_VARIANT 50 /* variant that wraps a kvlist or array */ #define FLB_CONFIG_MAP_MULT 1 +#define FLB_CONFIG_MAP_DYNAMIC_ENV 2 /* flag: resolve environment variables at runtime */ struct flb_config_map_val { union { @@ -108,6 +109,12 @@ struct mk_list *flb_config_map_create(struct flb_config *config, struct flb_config_map *map); void flb_config_map_destroy(struct mk_list *list); int flb_config_map_expected_values(int type); -int flb_config_map_set(struct mk_list *properties, struct mk_list *map, void *context); +int flb_config_map_set(struct flb_config *config, struct mk_list *properties, struct mk_list *map, void *context); + +/* Helper function to check if a config map entry has dynamic environment resolution */ +static inline int flb_config_map_has_dynamic_env(struct flb_config_map *map) +{ + return (map->flags & FLB_CONFIG_MAP_DYNAMIC_ENV) != 0; +} #endif diff --git a/include/fluent-bit/flb_custom.h b/include/fluent-bit/flb_custom.h index 537e72c71b9..e8aedcfedb0 100644 --- a/include/fluent-bit/flb_custom.h +++ b/include/fluent-bit/flb_custom.h @@ -93,7 +93,7 @@ struct flb_custom_instance { static inline int flb_custom_config_map_set(struct flb_custom_instance *ins, void *context) { - return flb_config_map_set(&ins->properties, ins->config_map, context); + return flb_config_map_set(ins->config, &ins->properties, ins->config_map, context); } int flb_custom_set_property(struct flb_custom_instance *ins, diff --git a/include/fluent-bit/flb_env.h b/include/fluent-bit/flb_env.h index ce0c59263ba..968259896d4 100644 --- a/include/fluent-bit/flb_env.h +++ b/include/fluent-bit/flb_env.h @@ -22,12 +22,24 @@ #include #include +#include +#include #define FLB_ENV_SIZE 64 +struct flb_env_var { + flb_sds_t name; /* variable name */ + flb_sds_t value; /* value (resolved) */ + flb_sds_t uri; /* optional file URI */ + int refresh_interval; /* refresh interval in seconds */ + time_t last_refresh; /* last refresh timestamp */ + struct mk_list _head; /* link to flb_env vars list */ +}; + struct flb_env { int warn_unused; /* warn about unused environment variable */ struct flb_hash_table *ht; + struct mk_list vars; /* list of environment variables */ }; static inline void flb_env_warn_unused(struct flb_env *env, int warn) @@ -38,6 +50,8 @@ static inline void flb_env_warn_unused(struct flb_env *env, int warn) struct flb_env *flb_env_create(); void flb_env_destroy(struct flb_env *env); int flb_env_set(struct flb_env *env, const char *key, const char *val); +int flb_env_set_extended(struct flb_env *env, const char *key, const char *val, + const char *uri, int refresh_interval); const char *flb_env_get(struct flb_env *env, const char *key); flb_sds_t flb_env_var_translate(struct flb_env *env, const char *value); diff --git a/include/fluent-bit/flb_filter.h b/include/fluent-bit/flb_filter.h index 4180e6bc65c..9a9343b1863 100644 --- a/include/fluent-bit/flb_filter.h +++ b/include/fluent-bit/flb_filter.h @@ -125,7 +125,7 @@ struct mk_list *flb_filter_get_global_config_map(struct flb_config *config); static inline int flb_filter_config_map_set(struct flb_filter_instance *ins, void *context) { - return flb_config_map_set(&ins->properties, ins->config_map, context); + return flb_config_map_set(ins->config, &ins->properties, ins->config_map, context); } int flb_filter_set_property(struct flb_filter_instance *ins, diff --git a/include/fluent-bit/flb_input.h b/include/fluent-bit/flb_input.h index accde0b910b..64391e2e606 100644 --- a/include/fluent-bit/flb_input.h +++ b/include/fluent-bit/flb_input.h @@ -721,7 +721,7 @@ static inline int flb_input_config_map_set(struct flb_input_instance *ins, /* Process normal properties */ if (ins->config_map) { - ret = flb_config_map_set(&ins->properties, ins->config_map, context); + ret = flb_config_map_set(ins->config, &ins->properties, ins->config_map, context); if (ret == -1) { return -1; @@ -730,7 +730,7 @@ static inline int flb_input_config_map_set(struct flb_input_instance *ins, /* Net properties */ if (ins->net_config_map) { - ret = flb_config_map_set(&ins->net_properties, ins->net_config_map, + ret = flb_config_map_set(ins->config, &ins->net_properties, ins->net_config_map, &ins->net_setup); if (ret == -1) { return -1; diff --git a/include/fluent-bit/flb_output.h b/include/fluent-bit/flb_output.h index d9675e48bc4..1e8f5bc32d7 100644 --- a/include/fluent-bit/flb_output.h +++ b/include/fluent-bit/flb_output.h @@ -1302,7 +1302,7 @@ static inline int flb_output_config_map_set(struct flb_output_instance *ins, /* Process normal properties */ if (ins->config_map) { - ret = flb_config_map_set(&ins->properties, ins->config_map, context); + ret = flb_config_map_set(ins->config, &ins->properties, ins->config_map, context); if (ret == -1) { return -1; } @@ -1310,7 +1310,7 @@ static inline int flb_output_config_map_set(struct flb_output_instance *ins, /* Net properties */ if (ins->net_config_map) { - ret = flb_config_map_set(&ins->net_properties, ins->net_config_map, + ret = flb_config_map_set(ins->config, &ins->net_properties, ins->net_config_map, &ins->net_setup); if (ret == -1) { return -1; diff --git a/include/fluent-bit/flb_processor.h b/include/fluent-bit/flb_processor.h index c6828f78bbd..551017c8ffe 100644 --- a/include/fluent-bit/flb_processor.h +++ b/include/fluent-bit/flb_processor.h @@ -278,7 +278,7 @@ static inline int flb_processor_instance_config_map_set( struct flb_processor_instance *ins, void *context) { - return flb_config_map_set(&ins->properties, ins->config_map, context); + return flb_config_map_set(ins->config, &ins->properties, ins->config_map, context); } static inline diff --git a/plugins/in_dummy/in_dummy.c b/plugins/in_dummy/in_dummy.c index 6d008b4cb2c..99257c567cc 100644 --- a/plugins/in_dummy/in_dummy.c +++ b/plugins/in_dummy/in_dummy.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "in_dummy.h" @@ -76,6 +77,15 @@ static int generate_event(struct flb_dummy *ctx) struct flb_time timestamp; msgpack_unpacked object; int result; + flb_sds_t resolved_body; + flb_sds_t resolved_metadata; + char *body_msgpack = NULL; + const char *body_template; + char *metadata_msgpack = NULL; + const char *metadata_template; + size_t body_msgpack_size = 0; + size_t metadata_msgpack_size = 0; + int root_type; result = FLB_EVENT_ENCODER_SUCCESS; body_start = 0; @@ -83,14 +93,73 @@ static int generate_event(struct flb_dummy *ctx) generate_timestamp(ctx, ×tamp); + /* Get the raw template from the property (should be raw when FLB_CONFIG_MAP_DYNAMIC_ENV is set) */ + body_template = flb_input_get_property("dummy", ctx->ins); + metadata_template = flb_input_get_property("metadata", ctx->ins); + + if (!body_template) { + body_template = DEFAULT_DUMMY_MESSAGE; + } + if (!metadata_template) { + metadata_template = DEFAULT_DUMMY_METADATA; + } + + /* Always try to resolve environment variables for dynamic content */ + resolved_body = flb_env_var_translate(ctx->ins->config->env, body_template); + if (!resolved_body) { + resolved_body = flb_sds_create(body_template); + } + + + /* Always try to resolve environment variables for dynamic content */ + resolved_metadata = flb_env_var_translate(ctx->ins->config->env, metadata_template); + if (!resolved_metadata) { + resolved_metadata = flb_sds_create(metadata_template); + } + + /* Parse the resolved JSON strings */ + if (resolved_body) { + result = flb_pack_json(resolved_body, + flb_sds_len(resolved_body), + &body_msgpack, + &body_msgpack_size, + &root_type, + NULL); + } + else { + result = 0; /* Using cached msgpack */ + } + + if (result == 0 && resolved_metadata) { + result = flb_pack_json(resolved_metadata, + flb_sds_len(resolved_metadata), + &metadata_msgpack, + &metadata_msgpack_size, + &root_type, + NULL); + } + + if (result != 0) { + flb_plg_error(ctx->ins, "failed to parse JSON template"); + flb_sds_destroy(resolved_body); + flb_sds_destroy(resolved_metadata); + if (body_msgpack) { + flb_free(body_msgpack); + } + if (metadata_msgpack) { + flb_free(metadata_msgpack); + } + return -1; + } + msgpack_unpacked_init(&object); while (result == FLB_EVENT_ENCODER_SUCCESS && msgpack_unpack_next(&object, - ctx->ref_body_msgpack, - ctx->ref_body_msgpack_size, + body_msgpack, + body_msgpack_size, &chunk_offset) == MSGPACK_UNPACK_SUCCESS) { - body_buffer = &ctx->ref_body_msgpack[body_start]; + body_buffer = &body_msgpack[body_start]; body_length = chunk_offset - body_start; if (object.data.type == MSGPACK_OBJECT_MAP) { @@ -100,8 +169,8 @@ static int generate_event(struct flb_dummy *ctx) result = flb_log_event_encoder_set_metadata_from_raw_msgpack( ctx->encoder, - ctx->ref_metadata_msgpack, - ctx->ref_metadata_msgpack_size); + metadata_msgpack, + metadata_msgpack_size); if (result == FLB_EVENT_ENCODER_SUCCESS) { result = flb_log_event_encoder_set_body_from_raw_msgpack( @@ -120,6 +189,21 @@ static int generate_event(struct flb_dummy *ctx) msgpack_unpacked_destroy(&object); + /* Clean up */ + if (resolved_body) { + flb_sds_destroy(resolved_body); + } + if (resolved_metadata) { + flb_sds_destroy(resolved_metadata); + } + /* Only free msgpack if we allocated it (not using cached) */ + if (body_msgpack && body_msgpack != ctx->ref_body_msgpack) { + flb_free(body_msgpack); + } + if (metadata_msgpack && metadata_msgpack != ctx->ref_metadata_msgpack) { + flb_free(metadata_msgpack); + } + if (result == FLB_EVENT_ENCODER_SUCCESS) { result = 0; } @@ -186,6 +270,15 @@ static int config_destroy(struct flb_dummy *ctx) flb_free(ctx->ref_metadata_msgpack); } + if (ctx->body_template != NULL) { + flb_free(ctx->body_template); + } + + if (ctx->metadata_template != NULL) { + flb_free(ctx->metadata_template); + } + + if (ctx->encoder != NULL) { flb_log_event_encoder_destroy(ctx->encoder); } @@ -200,12 +293,15 @@ static int configure(struct flb_dummy *ctx, struct flb_input_instance *in, struct timespec *tm) { - const char *msg; - int root_type; int ret = -1; + int root_type; + const char *msg; + flb_sds_t resolved_msg = NULL; ctx->ref_metadata_msgpack = NULL; ctx->ref_body_msgpack = NULL; + ctx->body_template = NULL; + ctx->metadata_template = NULL; ctx->dummy_timestamp_set = FLB_FALSE; ret = flb_input_config_map_set(in, (void *) ctx); @@ -228,7 +324,8 @@ static int configure(struct flb_dummy *ctx, /* Set using interval settings. */ tm->tv_sec = ctx->interval_sec; tm->tv_nsec = ctx->interval_nsec; - } else { + } + else { if (ctx->rate > 1) { /* Set using rate settings. */ tm->tv_sec = 0; @@ -252,14 +349,34 @@ static int configure(struct flb_dummy *ctx, flb_time_get(&ctx->base_timestamp); - /* handle it explicitly since we need to validate it is valid JSON */ + /* Store the original body template for dynamic re-parsing */ msg = flb_input_get_property("dummy", in); if (msg == NULL) { msg = DEFAULT_DUMMY_MESSAGE; } + ctx->body_template = flb_strdup(msg); + if (!ctx->body_template) { + flb_errno(); + flb_plg_error(ctx->ins, "failed to duplicate body template"); + return -1; + } + + /* Validate the template by parsing it once (with environment variables resolved) */ + resolved_msg = flb_env_var_translate(in->config->env, msg); + if (!resolved_msg || flb_sds_len(resolved_msg) == 0) { + if (resolved_msg) { + flb_sds_destroy(resolved_msg); + } + flb_plg_warn(ctx->ins, "environment variable resolution failed for dummy message, using default"); + resolved_msg = flb_sds_create(DEFAULT_DUMMY_MESSAGE); + if (!resolved_msg) { + flb_plg_error(ctx->ins, "failed to create default body template"); + return -1; + } + } - ret = flb_pack_json(msg, - strlen(msg), + ret = flb_pack_json(resolved_msg, + flb_sds_len(resolved_msg), &ctx->ref_body_msgpack, &ctx->ref_body_msgpack_size, &root_type, @@ -276,19 +393,37 @@ static int configure(struct flb_dummy *ctx, NULL); if (ret != 0) { flb_plg_error(ctx->ins, "unexpected error"); + flb_sds_destroy(resolved_msg); return -1; } } - /* handle it explicitly since we need to validate it is valid JSON */ - msg = flb_input_get_property("metadata", in); + flb_sds_destroy(resolved_msg); + /* Store the original metadata template for dynamic re-parsing */ + msg = flb_input_get_property("metadata", in); if (msg == NULL) { msg = DEFAULT_DUMMY_METADATA; } + ctx->metadata_template = flb_strdup(msg); + if (!ctx->metadata_template) { + flb_errno(); + flb_plg_error(ctx->ins, "failed to duplicate metadata template"); + return -1; + } + + /* Validate the template by parsing it once (with environment variables resolved) */ + resolved_msg = flb_env_var_translate(in->config->env, msg); + if (!resolved_msg || flb_sds_len(resolved_msg) == 0) { + if (resolved_msg) { + flb_sds_destroy(resolved_msg); + } + flb_plg_warn(ctx->ins, "environment variable resolution failed for metadata, using default"); + resolved_msg = flb_sds_create(DEFAULT_DUMMY_METADATA); + } - ret = flb_pack_json(msg, - strlen(msg), + ret = flb_pack_json(resolved_msg, + flb_sds_len(resolved_msg), &ctx->ref_metadata_msgpack, &ctx->ref_metadata_msgpack_size, &root_type, @@ -306,10 +441,13 @@ static int configure(struct flb_dummy *ctx, if (ret != 0) { flb_plg_error(ctx->ins, "unexpected error"); + flb_sds_destroy(resolved_msg); return -1; } } + flb_sds_destroy(resolved_msg); + return 0; } @@ -327,6 +465,7 @@ static int in_dummy_init(struct flb_input_instance *in, /* Allocate space for the configuration */ ctx = flb_malloc(sizeof(struct flb_dummy)); if (ctx == NULL) { + flb_errno(); return -1; } ctx->ins = in; @@ -341,7 +480,6 @@ static int in_dummy_init(struct flb_input_instance *in, } ctx->encoder = flb_log_event_encoder_create(FLB_LOG_EVENT_FORMAT_DEFAULT); - if (ctx->encoder == NULL) { flb_plg_error(in, "could not initialize event encoder"); config_destroy(ctx); @@ -405,13 +543,15 @@ static struct flb_config_map config_map[] = { }, { FLB_CONFIG_MAP_STR, "dummy", DEFAULT_DUMMY_MESSAGE, - 0, FLB_FALSE, 0, + FLB_CONFIG_MAP_DYNAMIC_ENV, FLB_FALSE, 0, "set the sample record to be generated. It should be a JSON object." + "Environment variables are resolved dynamically." }, { FLB_CONFIG_MAP_STR, "metadata", DEFAULT_DUMMY_METADATA, - 0, FLB_FALSE, 0, + FLB_CONFIG_MAP_DYNAMIC_ENV, FLB_FALSE, 0, "set the sample metadata to be generated. It should be a JSON object." + "Environment variables are resolved dynamically." }, { FLB_CONFIG_MAP_INT, "rate", DEFAULT_RATE, diff --git a/plugins/in_dummy/in_dummy.h b/plugins/in_dummy/in_dummy.h index f225fcb4eeb..865eaddffbe 100644 --- a/plugins/in_dummy/in_dummy.h +++ b/plugins/in_dummy/in_dummy.h @@ -56,6 +56,10 @@ struct flb_dummy { char *ref_body_msgpack; size_t ref_body_msgpack_size; + /* Store original JSON templates for dynamic re-parsing */ + char *body_template; + char *metadata_template; + struct flb_log_event_encoder *encoder; struct flb_input_instance *ins; diff --git a/plugins/processor_sampling/sampling_probabilistic.c b/plugins/processor_sampling/sampling_probabilistic.c index 1e84119a1d2..d883798ad5d 100644 --- a/plugins/processor_sampling/sampling_probabilistic.c +++ b/plugins/processor_sampling/sampling_probabilistic.c @@ -50,7 +50,7 @@ static int cb_init(struct flb_config *config, struct sampling *ctx) return -1; } - ret = flb_config_map_set(&ctx->plugin_settings_properties, ctx->plugin_config_map, (void *) settings); + ret = flb_config_map_set(config, &ctx->plugin_settings_properties, ctx->plugin_config_map, (void *) settings); if (ret == -1) { flb_free(settings); return -1; diff --git a/plugins/processor_sampling/sampling_tail.c b/plugins/processor_sampling/sampling_tail.c index c2d9fe76e9d..66da6f978f0 100644 --- a/plugins/processor_sampling/sampling_tail.c +++ b/plugins/processor_sampling/sampling_tail.c @@ -502,7 +502,7 @@ static int cb_init(struct flb_config *config, struct sampling *ctx) return -1; } - ret = flb_config_map_set(&ctx->plugin_settings_properties, ctx->plugin_config_map, (void *) settings); + ret = flb_config_map_set(config, &ctx->plugin_settings_properties, ctx->plugin_config_map, (void *) settings); if (ret == -1) { flb_free(settings); return -1; diff --git a/src/config_format/flb_cf_yaml.c b/src/config_format/flb_cf_yaml.c index b2e4eefdfcc..4620c203a09 100644 --- a/src/config_format/flb_cf_yaml.c +++ b/src/config_format/flb_cf_yaml.c @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include #include @@ -172,6 +174,9 @@ enum state { /* environment variables */ STATE_ENV, + STATE_ENV_LIST, + STATE_ENV_LIST_KEY, + STATE_ENV_LIST_VAL, STATE_STOP /* end state */ @@ -215,6 +220,8 @@ struct parser_state { struct file_state *file; + struct flb_cf_env_var *env_var; + struct cfl_list _head; }; @@ -297,6 +304,12 @@ static char *state_str(enum state val) return "processor"; case STATE_ENV: return "env"; + case STATE_ENV_LIST: + return "env-list"; + case STATE_ENV_LIST_KEY: + return "env-list-key"; + case STATE_ENV_LIST_VAL: + return "env-list-val"; case STATE_PARSER: return "parser"; case STATE_MULTILINE_PARSER: @@ -817,7 +830,7 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, enum status status; int ret; char *value; - struct flb_kv *keyval; + struct flb_cf_env_var *keyval; char *last_included; last_included = state_get_last(ctx); @@ -854,6 +867,113 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, } break; + case STATE_ENV_LIST: + switch (event->type) { + case YAML_MAPPING_START_EVENT: + state = state_push(ctx, STATE_ENV_LIST_KEY); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + state->env_var = flb_calloc(1, sizeof(struct flb_cf_env_var)); + if (!state->env_var) { + flb_error("unable to allocate env var"); + return YAML_FAILURE; + } + state->env_var->refresh_interval = 0; + break; + case YAML_SEQUENCE_END_EVENT: + state = state_pop(ctx); + if (state == NULL) { + flb_error("no state left"); + return YAML_FAILURE; + } + state = state_pop(ctx); + if (state == NULL) { + flb_error("no state left"); + return YAML_FAILURE; + } + break; + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + case STATE_ENV_LIST_KEY: + switch(event->type) { + case YAML_SCALAR_EVENT: + { + char *tmp_value; + struct parser_state *parent = state; + + tmp_value = (char *) event->data.scalar.value; + state = state_push_key(ctx, STATE_ENV_LIST_VAL, tmp_value); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + state->env_var = parent->env_var; + break; + } + case YAML_MAPPING_END_EVENT: + if (state->env_var && state->env_var->name) { + mk_list_add(&state->env_var->_head, &conf->env); + } + else if (state->env_var) { + if (state->env_var->name) { + flb_sds_destroy(state->env_var->name); + } + if (state->env_var->value) { + flb_sds_destroy(state->env_var->value); + } + if (state->env_var->uri) { + flb_sds_destroy(state->env_var->uri); + } + flb_free(state->env_var); + } + state = state_pop(ctx); + if (state == NULL) { + flb_error("no state left"); + return YAML_FAILURE; + } + break; + default: + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + + case STATE_ENV_LIST_VAL: + if (event->type == YAML_SCALAR_EVENT) { + value = (char *) event->data.scalar.value; + if (strcasecmp(state->key, "name") == 0) { + state->env_var->name = flb_sds_create(value); + } + else if (strcasecmp(state->key, "value") == 0) { + state->env_var->value = flb_sds_create(value); + } + else if (strcasecmp(state->key, "uri") == 0) { + state->env_var->uri = flb_sds_create(value); + } + else if (strcasecmp(state->key, "refresh_interval") == 0) { + state->env_var->refresh_interval = flb_utils_time_to_seconds(value); + } + else { + if (!state->env_var->name) { + state->env_var->name = flb_sds_create(state->key); + state->env_var->value = flb_sds_create(value); + } + } + + state = state_pop(ctx); + } + else { + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + break; + case STATE_STREAM: switch (event->type) { case YAML_DOCUMENT_START_EVENT: @@ -1675,6 +1795,19 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, case YAML_MAPPING_START_EVENT: state = state_push(ctx, STATE_SECTION_KEY); + if (state == NULL) { + flb_error("unable to allocate state"); + return YAML_FAILURE; + } + break; + case YAML_SEQUENCE_START_EVENT: + if (state->section != SECTION_ENV) { + flb_error("env list is only allowed in env section"); + yaml_error_event(ctx, state, event); + return YAML_FAILURE; + } + + state = state_push(ctx, STATE_ENV_LIST); if (state == NULL) { flb_error("unable to allocate state"); return YAML_FAILURE; @@ -1742,9 +1875,10 @@ static int consume_event(struct flb_cf *conf, struct local_ctx *ctx, /* Check if the incoming k/v pair set a config environment variable */ if (state->section == SECTION_ENV) { - keyval = flb_cf_env_property_add(conf, - state->key, flb_sds_len(state->key), - value, strlen(value)); + keyval = flb_cf_env_var_add(conf, + state->key, flb_sds_len(state->key), + value, strlen(value), + NULL, 0, 0); if (keyval == NULL) { flb_error("unable to add key value"); diff --git a/src/config_format/flb_config_format.c b/src/config_format/flb_config_format.c index 28486d6a257..030d0ab2060 100644 --- a/src/config_format/flb_config_format.c +++ b/src/config_format/flb_config_format.c @@ -150,7 +150,24 @@ struct flb_cf *flb_cf_create() void flb_cf_destroy(struct flb_cf *cf) { - flb_kv_release(&cf->env); + struct flb_cf_env_var *ev; + struct mk_list *tmp; + struct mk_list *head; + + mk_list_foreach_safe(head, tmp, &cf->env) { + ev = mk_list_entry(head, struct flb_cf_env_var, _head); + if (ev->name) { + flb_sds_destroy(ev->name); + } + if (ev->value) { + flb_sds_destroy(ev->value); + } + if (ev->uri) { + flb_sds_destroy(ev->uri); + } + flb_free(ev); + } + mk_list_init(&cf->env); flb_kv_release(&cf->metas); flb_cf_section_destroy_all(cf); flb_free(cf); @@ -421,41 +438,69 @@ struct cfl_variant * flb_cf_section_property_get(struct flb_cf *cf, struct flb_c return cfl_kvlist_fetch(s->properties, key); } -struct flb_kv *flb_cf_env_property_add(struct flb_cf *cf, - char *k_buf, size_t k_len, - char *v_buf, size_t v_len) +struct flb_cf_env_var *flb_cf_env_var_add(struct flb_cf *cf, + char *name, size_t name_len, + char *value, size_t value_len, + char *uri, size_t uri_len, + int refresh_interval) { - int ret; - struct flb_kv *kv; + struct flb_cf_env_var *ev; - if (k_len == 0) { - k_len = strlen(k_buf); + if (name_len == 0 && name) { + name_len = strlen(name); } - if (v_len == 0) { - v_len = strlen(v_buf); + if (value_len == 0 && value) { + value_len = strlen(value); + } + if (uri_len == 0 && uri) { + uri_len = strlen(uri); } - kv = flb_kv_item_create_len(&cf->env, k_buf, k_len, v_buf, v_len); - if (!kv) { + ev = flb_calloc(1, sizeof(struct flb_cf_env_var)); + if (!ev) { return NULL; } - /* sanitize key and value by removing empty spaces */ - ret = flb_sds_trim(kv->key); - if (ret == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_KEY); - flb_kv_item_destroy(kv); - return NULL; + if (name) { + ev->name = flb_sds_create_len(name, name_len); + if (!ev->name) { + flb_free(ev); + return NULL; + } } - ret = flb_sds_trim(kv->val); - if (ret == -1) { - flb_cf_error_set(cf, FLB_CF_ERROR_KV_INVALID_VAL); - flb_kv_item_destroy(kv); - return NULL; + if (value) { + ev->value = flb_sds_create_len(value, value_len); + if (!ev->value) { + if (ev->name) { + flb_sds_destroy(ev->name); + } + flb_free(ev); + return NULL; + } } - return kv; + if (uri) { + ev->uri = flb_sds_create_len(uri, uri_len); + if (!ev->uri) { + if (ev->name) { + flb_sds_destroy(ev->name); + } + if (ev->value) { + flb_sds_destroy(ev->value); + } + if (ev->uri) { + flb_sds_destroy(ev->uri); + } + flb_free(ev); + return NULL; + } + } + + ev->refresh_interval = refresh_interval; + mk_list_add(&ev->_head, &cf->env); + + return ev; } static struct flb_kv *meta_property_add(struct flb_cf *cf, @@ -827,7 +872,7 @@ static void dump_section(struct flb_cf_section *s) static void dump_env(struct mk_list *list) { struct mk_list *head; - struct flb_kv *kv; + struct flb_cf_env_var *ev; if (mk_list_size(list) == 0) { return; @@ -836,8 +881,16 @@ static void dump_env(struct mk_list *list) printf("> env:\n"); mk_list_foreach(head, list) { - kv = mk_list_entry(head, struct flb_kv, _head); - printf(" - %-15s: %s\n", kv->key, kv->val); + ev = mk_list_entry(head, struct flb_cf_env_var, _head); + if (ev->uri) { + printf(" - %-15s: %s\n", ev->name, ev->uri); + } + else if (ev->value) { + printf(" - %-15s: %s\n", ev->name, ev->value); + } + else { + printf(" - %-15s: (null)\n", ev->name); + } } } diff --git a/src/flb_config.c b/src/flb_config.c index 407464b4942..29a5fe3057f 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -145,8 +145,8 @@ struct flb_service_config service_configs[] = { {FLB_CONF_STORAGE_BL_MEM_LIMIT, FLB_CONF_TYPE_STR, offsetof(struct flb_config, storage_bl_mem_limit)}, - {FLB_CONF_STORAGE_BL_FLUSH_ON_SHUTDOWN, - FLB_CONF_TYPE_BOOL, + {FLB_CONF_STORAGE_BL_FLUSH_ON_SHUTDOWN, + FLB_CONF_TYPE_BOOL, offsetof(struct flb_config, storage_bl_flush_on_shutdown)}, {FLB_CONF_STORAGE_MAX_CHUNKS_UP, FLB_CONF_TYPE_INT, @@ -932,6 +932,7 @@ static int configure_plugins_type(struct flb_config *config, struct flb_cf *cf, int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf) { int ret; + struct flb_cf_env_var *env_var; struct flb_kv *kv; struct mk_list *head; struct cfl_kvpair *ckv; @@ -940,10 +941,13 @@ int flb_config_load_config_format(struct flb_config *config, struct flb_cf *cf) /* Process config environment vars */ mk_list_foreach(head, &cf->env) { - kv = mk_list_entry(head, struct flb_kv, _head); - ret = flb_env_set(config->env, kv->key, kv->val); + env_var = mk_list_entry(head, struct flb_cf_env_var, _head); + ret = flb_env_set_extended(config->env, + env_var->name, env_var->value, + env_var->uri, + env_var->refresh_interval); if (ret == -1) { - flb_error("could not set config environment variable '%s'", kv->key); + flb_error("could not set config environment variable '%s'", env_var->name); return -1; } } diff --git a/src/flb_config_map.c b/src/flb_config_map.c index 0b40b7becb7..5dbef891f6f 100644 --- a/src/flb_config_map.c +++ b/src/flb_config_map.c @@ -301,16 +301,23 @@ struct mk_list *flb_config_map_create(struct flb_config *config, */ flb_env_warn_unused(config->env, FLB_FALSE); - /* Translate the value */ - env = flb_env_var_translate(config->env, m->def_value); - if (env == NULL) { - flb_errno(); - flb_sds_destroy(new->name); - flb_free(new); - flb_config_map_destroy(list); - return NULL; + /* Translate the value only if not marked as dynamic */ + if (m->flags & FLB_CONFIG_MAP_DYNAMIC_ENV) { + /* For dynamic env vars, store the raw template */ + new->def_value = flb_sds_create(m->def_value); + } + else { + /* For static env vars, resolve them now */ + env = flb_env_var_translate(config->env, m->def_value); + if (env == NULL) { + flb_errno(); + flb_sds_destroy(new->name); + flb_free(new); + flb_config_map_destroy(list); + return NULL; + } + new->def_value = env; } - new->def_value = env; flb_env_warn_unused(config->env, FLB_TRUE); } @@ -579,7 +586,7 @@ int flb_config_map_expected_values(int type) * Function used by plugins that needs to populate their context structure with the * configuration properties already mapped. */ -int flb_config_map_set(struct mk_list *properties, struct mk_list *map, void *context) +int flb_config_map_set(struct flb_config *config, struct mk_list *properties, struct mk_list *map, void *context) { int ret; int len; @@ -597,6 +604,7 @@ int flb_config_map_set(struct mk_list *properties, struct mk_list *map, void *co struct mk_list *list; struct flb_config_map *m = NULL; struct flb_config_map_val *entry = NULL; + flb_sds_t resolved; base = context; @@ -712,7 +720,20 @@ int flb_config_map_set(struct mk_list *properties, struct mk_list *map, void *co /* Populate value */ if (m->type == FLB_CONFIG_MAP_STR) { - entry->val.str = flb_sds_create(kv->val); + if (m->flags & FLB_CONFIG_MAP_DYNAMIC_ENV) { + /* For dynamic env vars, store the raw template */ + entry->val.str = flb_sds_create(kv->val); + } + else { + /* For static env vars, resolve them now */ + resolved = flb_env_var_translate(config->env, kv->val); + if (resolved) { + entry->val.str = resolved; + } + else { + entry->val.str = flb_sds_create(kv->val); + } + } } else if (m->type == FLB_CONFIG_MAP_INT) { entry->val.i_num = atoi(kv->val); @@ -775,7 +796,20 @@ int flb_config_map_set(struct mk_list *properties, struct mk_list *map, void *co /* Direct write to user context */ if (m->type == FLB_CONFIG_MAP_STR) { m_str = (char **) (base + m->offset); - *m_str = kv->val; + if (m->flags & FLB_CONFIG_MAP_DYNAMIC_ENV) { + /* For dynamic env vars, store the raw template */ + *m_str = flb_sds_create(kv->val); + } + else { + /* For static env vars, resolve them now */ + resolved = flb_env_var_translate(config->env, kv->val); + if (resolved) { + *m_str = resolved; + } + else { + *m_str = flb_sds_create(kv->val); + } + } } else if (m->type == FLB_CONFIG_MAP_INT) { m_i_num = (int *) (base + m->offset); diff --git a/src/flb_env.c b/src/flb_env.c index d951145de13..b5b6ac113c4 100644 --- a/src/flb_env.c +++ b/src/flb_env.c @@ -26,6 +26,7 @@ #include #include +#include static inline flb_sds_t buf_append(flb_sds_t buf, const char *str, int len) { @@ -87,6 +88,7 @@ struct flb_env *flb_env_create() env->warn_unused = FLB_TRUE; env->ht = ht; + mk_list_init(&env->vars); env_preset(env); return env; @@ -94,11 +96,30 @@ struct flb_env *flb_env_create() void flb_env_destroy(struct flb_env *env) { + struct mk_list *tmp; + struct mk_list *head; + struct flb_env_var *var; + + mk_list_foreach_safe(head, tmp, &env->vars) { + var = mk_list_entry(head, struct flb_env_var, _head); + if (var->name) { + flb_sds_destroy(var->name); + } + if (var->value) { + flb_sds_destroy(var->value); + } + if (var->uri) { + flb_sds_destroy(var->uri); + } + flb_free(var); + } + flb_hash_table_destroy(env->ht); flb_free(env); } -int flb_env_set(struct flb_env *env, const char *key, const char *val) +int flb_env_set_extended(struct flb_env *env, const char *key, const char *val, + const char *uri, int refresh_interval) { int id; int klen; @@ -106,58 +127,87 @@ int flb_env_set(struct flb_env *env, const char *key, const char *val) void *out_buf; size_t out_size; flb_sds_t fs_buf = NULL; + const char *orig_uri = NULL; + const char *value = val; + struct flb_env_var *var; + + if (uri) { + orig_uri = uri; + value = uri; + } + + if (value == NULL) { + value = ""; + } - /* Get lengths */ klen = strlen(key); - vlen = strlen(val); + vlen = strlen(value); /* Check if the variable is a reference to a file */ - if (vlen > 7 && strncmp(val, "file://", 7) == 0) { - /* skip the file:// prefix */ + if (vlen > 7 && strncmp(value, "file://", 7) == 0) { + orig_uri = value; vlen -= 7; - val += 7; + value += 7; - /* Check if the file exists */ - if (access(val, R_OK) == -1) { - flb_error("[env] file %s not found", val); + if (access(value, R_OK) == -1) { + flb_error("[env] file %s not found", value); return -1; } - /* Read the file content */ - fs_buf = flb_file_read(val); + + fs_buf = flb_file_read(value); if (!fs_buf) { - flb_error("[env] file %s could not be read", val); + flb_error("[env] file %s could not be read", value); return -1; } - /* Set the new value */ - val = fs_buf; + value = fs_buf; vlen = flb_sds_len(fs_buf); - /* check if we have an ending \r or \n */ - if (vlen > 0 && (val[vlen - 1] == '\n' || val[vlen - 1] == '\r')) { + if (vlen > 0 && (value[vlen - 1] == '\n' || value[vlen - 1] == '\r')) { vlen--; flb_sds_len_set(fs_buf, vlen); } - /* Check if the file content is empty */ if (vlen == 0) { - flb_error("[env] file %s content is empty", val); + flb_error("[env] file %s content is empty", value); flb_sds_destroy(fs_buf); return -1; } - flb_debug("[env] file %s content read propery, length= %d", val, vlen); + flb_debug("[env] file %s content read propery, length= %d", value, vlen); } /* Check if the key is already set */ id = flb_hash_table_get(env->ht, key, klen, &out_buf, &out_size); if (id >= 0) { - /* Remove the old entry */ flb_hash_table_del(env->ht, key); } - /* Register the new key */ - id = flb_hash_table_add(env->ht, key, klen, (void *) val, vlen); + id = flb_hash_table_add(env->ht, key, klen, (void *) value, vlen); + + var = flb_calloc(1, sizeof(struct flb_env_var)); + if (!var) { + flb_errno(); + if (fs_buf) { + flb_sds_destroy(fs_buf); + } + return -1; + } + mk_list_add(&var->_head, &env->vars); + var->name = flb_sds_create(key); + if (vlen > 0) { + var->value = flb_sds_create_len(value, vlen); + } + if (orig_uri) { + var->uri = flb_sds_create(orig_uri); + } + var->refresh_interval = refresh_interval; + if (orig_uri) { + var->last_refresh = time(NULL); + } + else { + var->last_refresh = 0; + } if (fs_buf) { flb_sds_destroy(fs_buf); @@ -166,12 +216,23 @@ int flb_env_set(struct flb_env *env, const char *key, const char *val) return id; } +int flb_env_set(struct flb_env *env, const char *key, const char *val) +{ + return flb_env_set_extended(env, key, val, NULL, 0); +} + const char *flb_env_get(struct flb_env *env, const char *key) { int len; int ret; void *out_buf; size_t out_size; + struct mk_list *head; + struct flb_env_var *var; + time_t now; + flb_sds_t fs_buf; + int vlen; + const char *file; if (!key) { return NULL; @@ -179,13 +240,62 @@ const char *flb_env_get(struct flb_env *env, const char *key) len = strlen(key); - /* Try to get the value from the hash table */ + mk_list_foreach(head, &env->vars) { + var = mk_list_entry(head, struct flb_env_var, _head); + if (var->name && strcmp(var->name, key) == 0) { + if (var->uri && var->refresh_interval > 0) { + now = time(NULL); + if (var->last_refresh == 0 || + now - var->last_refresh >= var->refresh_interval) { + file = var->uri; + if (strncmp(file, "file://", 7) == 0) { + file += 7; + } + + if (access(file, R_OK) == -1) { + flb_error("[env] file %s not found", file); + break; + } + + fs_buf = flb_file_read(file); + if (!fs_buf) { + flb_error("[env] file %s could not be read", file); + break; + } + + vlen = flb_sds_len(fs_buf); + if (vlen > 0 && (fs_buf[vlen - 1] == '\n' || fs_buf[vlen - 1] == '\r')) { + vlen--; + flb_sds_len_set(fs_buf, vlen); + } + + if (vlen == 0) { + flb_error("[env] file %s content is empty", file); + flb_sds_destroy(fs_buf); + break; + } + + flb_hash_table_del(env->ht, key); + if (var->value) { + flb_sds_destroy(var->value); + } + var->value = fs_buf; + ret = flb_hash_table_add(env->ht, key, len, fs_buf, vlen); + if (ret < 0) { + break; + } + var->last_refresh = now; + } + } + break; + } + } + ret = flb_hash_table_get(env->ht, key, len, &out_buf, &out_size); if (ret >= 0) { return (char *) out_buf; } - /* If it was not found, try to get it from the real environment */ out_buf = getenv(key); if (!out_buf) { return NULL; diff --git a/src/flb_input.c b/src/flb_input.c index b42c9ac472c..b264af0d62b 100644 --- a/src/flb_input.c +++ b/src/flb_input.c @@ -565,6 +565,7 @@ int flb_input_set_property(struct flb_input_instance *ins, struct flb_kv *kv; len = strlen(k); + /* Resolve environment variables in the provided value */ tmp = flb_env_var_translate(ins->config->env, v); if (tmp) { if (flb_sds_len(tmp) == 0) { diff --git a/src/flb_reload.c b/src/flb_reload.c index 7c92b817a32..9672a5db0d2 100644 --- a/src/flb_reload.c +++ b/src/flb_reload.c @@ -305,6 +305,7 @@ int flb_reload_reconstruct_cf(struct flb_cf *src_cf, struct flb_cf *dest_cf) { struct mk_list *head; struct flb_cf_section *s; + struct flb_cf_env_var *ev; struct flb_kv *kv; mk_list_foreach(head, &src_cf->sections) { @@ -316,10 +317,12 @@ int flb_reload_reconstruct_cf(struct flb_cf *src_cf, struct flb_cf *dest_cf) /* Copy and store env. (For yaml cf.) */ mk_list_foreach(head, &src_cf->env) { - kv = mk_list_entry(head, struct flb_kv, _head); - if (!flb_cf_env_property_add(dest_cf, - kv->key, cfl_sds_len(kv->key), - kv->val, cfl_sds_len(kv->val))) { + ev = mk_list_entry(head, struct flb_cf_env_var, _head); + if (!flb_cf_env_var_add(dest_cf, + ev->name, ev->name ? flb_sds_len(ev->name) : 0, + ev->value, ev->value ? flb_sds_len(ev->value) : 0, + ev->uri, ev->uri ? flb_sds_len(ev->uri) : 0, + ev->refresh_interval)) { return -1; } diff --git a/tests/internal/config_map.c b/tests/internal/config_map.c index aab444255a0..5a7bfb0f823 100644 --- a/tests/internal/config_map.c +++ b/tests/internal/config_map.c @@ -204,7 +204,7 @@ void test_create() TEST_CHECK(map != NULL); /* Populate default values only */ - ret = flb_config_map_set(&properties, map, &ctx); + ret = flb_config_map_set(config, &properties, map, &ctx); TEST_CHECK(ret == 0); TEST_CHECK(ctx.boolean == 1); @@ -255,7 +255,7 @@ void test_override_defaults() flb_kv_item_create(&properties, "test_slist", "abc def ghi jkl m n o"); /* Populate default values only */ - ret = flb_config_map_set(&properties, map, &ctx); + ret = flb_config_map_set(config, &properties, map, &ctx); TEST_CHECK(ret == 0); TEST_CHECK(ctx.boolean == 0); @@ -346,7 +346,7 @@ void test_multiple() ret = flb_config_map_properties_check("test", &prop, map); TEST_CHECK(ret == 0); - ret = flb_config_map_set(&prop, map, &ctx); + ret = flb_config_map_set(config, &prop, map, &ctx); TEST_CHECK(ret == 0); i = 0; @@ -396,7 +396,7 @@ void test_special_properties() flb_kv_init(&prop); flb_kv_item_create(&prop, "condition", "{\"op\": \"and\", \"rules\": [{\"field\": \"$level\", \"op\": \"eq\", \"value\": \"error\"}]}"); flb_kv_item_create(&prop, "active", "true"); - + /* Add a regular property too */ flb_kv_item_create(&prop, "boolean", "true"); @@ -408,7 +408,7 @@ void test_special_properties() TEST_CHECK(ret == 0); /* Test that normal properties are still set correctly */ - ret = flb_config_map_set(&prop, map, &ctx); + ret = flb_config_map_set(config, &prop, map, &ctx); TEST_CHECK(ret == 0); TEST_CHECK(ctx.boolean == 1); diff --git a/tests/internal/env.c b/tests/internal/env.c index c0a72f0a896..750cca7b27f 100644 --- a/tests/internal/env.c +++ b/tests/internal/env.c @@ -23,6 +23,9 @@ #include #include #include +#include +#include +#include #include "flb_tests_internal.h" /* https://github.com/fluent/fluent-bit/issues/6313 */ @@ -84,8 +87,340 @@ void test_translate_long_env() flb_env_destroy(env); } +/* Test file-based environment variable with refresh interval */ +void test_file_env_var_basic() +{ + struct flb_env *env; + flb_sds_t buf = NULL; + char *test_file = "/tmp/flb_test_secret.txt"; + char *test_content = "secret_value_123"; + char *template = "${SECRET}"; + int fd; + int ret; + + /* Create test file */ + fd = open(test_file, O_CREAT | O_WRONLY | O_TRUNC, 0644); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("Failed to create test file"); + return; + } + ret = write(fd, test_content, strlen(test_content)); + close(fd); + if (!TEST_CHECK(ret == strlen(test_content))) { + TEST_MSG("Failed to write test content"); + unlink(test_file); + return; + } + + /* Test environment variable loading */ + env = flb_env_create(); + if (!TEST_CHECK(env != NULL)) { + TEST_MSG("flb_env_create failed"); + unlink(test_file); + return; + } + + /* Set file-based environment variable */ + ret = flb_env_set_extended(env, "SECRET", NULL, "file:///tmp/flb_test_secret.txt", 0); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("flb_env_set_extended failed"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + /* Test variable translation */ + buf = flb_env_var_translate(env, template); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("flb_env_var_translate failed"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + if (!TEST_CHECK(strcmp(buf, test_content) == 0)) { + TEST_MSG("Content mismatch. Got=%s expect=%s", buf, test_content); + } + + flb_sds_destroy(buf); + flb_env_destroy(env); + unlink(test_file); +} + +/* Test file-based environment variable with refresh interval */ +void test_file_env_var_refresh() +{ + struct flb_env *env; + flb_sds_t buf = NULL; + char *test_file = "/tmp/flb_test_refresh.txt"; + char *initial_content = "initial_value"; + char *updated_content = "updated_value"; + char *template = "${REFRESH_VAR}"; + int fd; + int ret; + + /* Create test file with initial content */ + fd = open(test_file, O_CREAT | O_WRONLY | O_TRUNC, 0644); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("Failed to create test file"); + return; + } + ret = write(fd, initial_content, strlen(initial_content)); + close(fd); + if (!TEST_CHECK(ret == strlen(initial_content))) { + TEST_MSG("Failed to write initial content"); + unlink(test_file); + return; + } + + /* Test environment variable loading with refresh interval */ + env = flb_env_create(); + if (!TEST_CHECK(env != NULL)) { + TEST_MSG("flb_env_create failed"); + unlink(test_file); + return; + } + + /* Set file-based environment variable with 1 second refresh interval */ + ret = flb_env_set_extended(env, "REFRESH_VAR", NULL, "file:///tmp/flb_test_refresh.txt", 1); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("flb_env_set_extended failed"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + /* Test initial value */ + buf = flb_env_var_translate(env, template); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("flb_env_var_translate failed"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + if (!TEST_CHECK(strcmp(buf, initial_content) == 0)) { + TEST_MSG("Initial content mismatch. Got=%s expect=%s", buf, initial_content); + } + flb_sds_destroy(buf); + + /* Update file content */ + fd = open(test_file, O_WRONLY | O_TRUNC, 0644); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("Failed to open test file for update"); + flb_env_destroy(env); + unlink(test_file); + return; + } + ret = write(fd, updated_content, strlen(updated_content)); + close(fd); + if (!TEST_CHECK(ret == strlen(updated_content))) { + TEST_MSG("Failed to write updated content"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + /* Wait for refresh interval to pass */ + sleep(2); + + /* Test updated value */ + buf = flb_env_var_translate(env, template); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("flb_env_var_translate failed after refresh"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + if (!TEST_CHECK(strcmp(buf, updated_content) == 0)) { + TEST_MSG("Updated content mismatch. Got=%s expect=%s", buf, updated_content); + } + + flb_sds_destroy(buf); + flb_env_destroy(env); + unlink(test_file); +} + +/* Test file-based environment variable with file:// URI */ +void test_file_env_var_uri() +{ + struct flb_env *env; + flb_sds_t buf = NULL; + char *test_file = "/tmp/flb_test_uri.txt"; + char *test_content = "uri_value_456"; + char *file_uri = "file:///tmp/flb_test_uri.txt"; + char *template = "${URI_VAR}"; + int fd; + int ret; + + /* Create test file */ + fd = open(test_file, O_CREAT | O_WRONLY | O_TRUNC, 0644); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("Failed to create test file"); + return; + } + ret = write(fd, test_content, strlen(test_content)); + close(fd); + if (!TEST_CHECK(ret == strlen(test_content))) { + TEST_MSG("Failed to write test content"); + unlink(test_file); + return; + } + + /* Test environment variable loading with file:// URI */ + env = flb_env_create(); + if (!TEST_CHECK(env != NULL)) { + TEST_MSG("flb_env_create failed"); + unlink(test_file); + return; + } + + /* Set file-based environment variable with file:// URI */ + ret = flb_env_set_extended(env, "URI_VAR", NULL, file_uri, 0); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("flb_env_set_extended failed"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + /* Test variable translation */ + buf = flb_env_var_translate(env, template); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("flb_env_var_translate failed"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + if (!TEST_CHECK(strcmp(buf, test_content) == 0)) { + TEST_MSG("Content mismatch. Got=%s expect=%s", buf, test_content); + } + + flb_sds_destroy(buf); + flb_env_destroy(env); + unlink(test_file); +} + +/* Test mixed static and dynamic environment variables */ +void test_mixed_env_vars() +{ + struct flb_env *env; + flb_sds_t buf = NULL; + char *test_file = "/tmp/flb_test_mixed.txt"; + char *file_content = "dynamic_value"; + char *template = "Static: ${STATIC_VAR}, Dynamic: ${DYNAMIC_VAR}"; + char *expected = "Static: static_value, Dynamic: dynamic_value"; + int fd; + int ret; + + /* Create test file */ + fd = open(test_file, O_CREAT | O_WRONLY | O_TRUNC, 0644); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("Failed to create test file"); + return; + } + ret = write(fd, file_content, strlen(file_content)); + close(fd); + if (!TEST_CHECK(ret == strlen(file_content))) { + TEST_MSG("Failed to write test content"); + unlink(test_file); + return; + } + + /* Test mixed environment variables */ + env = flb_env_create(); + if (!TEST_CHECK(env != NULL)) { + TEST_MSG("flb_env_create failed"); + unlink(test_file); + return; + } + + /* Set static environment variable */ + ret = flb_env_set(env, "STATIC_VAR", "static_value"); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("flb_env_set failed for static variable"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + /* Set dynamic environment variable */ + ret = flb_env_set_extended(env, "DYNAMIC_VAR", NULL, "file:///tmp/flb_test_mixed.txt", 0); + if (!TEST_CHECK(ret >= 0)) { + TEST_MSG("flb_env_set_extended failed for dynamic variable"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + /* Test mixed variable translation */ + buf = flb_env_var_translate(env, template); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("flb_env_var_translate failed"); + flb_env_destroy(env); + unlink(test_file); + return; + } + + if (!TEST_CHECK(strcmp(buf, expected) == 0)) { + TEST_MSG("Mixed content mismatch. Got=%s expect=%s", buf, expected); + } + + flb_sds_destroy(buf); + flb_env_destroy(env); + unlink(test_file); +} + +/* Test error handling for missing file */ +void test_file_env_var_missing() +{ + struct flb_env *env; + flb_sds_t buf = NULL; + char *missing_file = "/tmp/flb_nonexistent_file.txt"; + char *template = "${MISSING_VAR}"; + int ret; + + /* Test environment variable loading with missing file */ + env = flb_env_create(); + if (!TEST_CHECK(env != NULL)) { + TEST_MSG("flb_env_create failed"); + return; + } + + /* Set file-based environment variable with missing file */ + ret = flb_env_set_extended(env, "MISSING_VAR", NULL, "file:///tmp/flb_nonexistent_file.txt", 0); + if (!TEST_CHECK(ret == -1)) { + TEST_MSG("flb_env_set_extended failed"); + flb_env_destroy(env); + return; + } + + /* Test variable translation should return empty string for missing file */ + buf = flb_env_var_translate(env, template); + if (!TEST_CHECK(buf != NULL)) { + TEST_MSG("flb_env_var_translate failed"); + flb_env_destroy(env); + return; + } + + /* Should return empty string for missing file */ + if (!TEST_CHECK(strlen(buf) == 0)) { + TEST_MSG("Expected empty string for missing file. Got=%s", buf); + } + + flb_sds_destroy(buf); + flb_env_destroy(env); +} + TEST_LIST = { { "translate_long_env" , test_translate_long_env}, + { "file_env_var_basic" , test_file_env_var_basic}, + { "file_env_var_refresh" , test_file_env_var_refresh}, + { "file_env_var_uri" , test_file_env_var_uri}, + { "mixed_env_vars" , test_mixed_env_vars}, + { "file_env_var_missing" , test_file_env_var_missing}, { NULL, NULL } }; diff --git a/tests/internal/fuzzers/config_map_fuzzer.c b/tests/internal/fuzzers/config_map_fuzzer.c index d62afd8011b..444efeb1762 100644 --- a/tests/internal/fuzzers/config_map_fuzzer.c +++ b/tests/internal/fuzzers/config_map_fuzzer.c @@ -194,7 +194,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) /* Assign one of the config maps */ map = flb_config_map_create(config, configs[i]); if (map) { - if (flb_config_map_set(&prop, map, &ctx) != -1) { + if (flb_config_map_set(config, &prop, map, &ctx) != -1) { flb_config_map_properties_check(fuzz_str3, &prop, map); } flb_config_map_destroy(map);