From dc16c24be68e582be16255bc4c3930624520f80e Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 4 Jun 2026 13:53:33 +0900 Subject: [PATCH 01/14] parser: Handle IANA compatible time zone configurations Signed-off-by: Hiroshi Hatake --- include/fluent-bit/flb_parser.h | 19 +++ src/flb_parser.c | 290 +++++++++++++++++++++++++++++++- 2 files changed, 301 insertions(+), 8 deletions(-) diff --git a/include/fluent-bit/flb_parser.h b/include/fluent-bit/flb_parser.h index 62770a97109..e870d8b60e9 100644 --- a/include/fluent-bit/flb_parser.h +++ b/include/fluent-bit/flb_parser.h @@ -49,6 +49,7 @@ struct flb_parser { char *time_key; /* field name that contains the time */ int time_offset; /* fixed UTC offset */ int time_system_timezone; /* use the system timezone as a fallback */ + char *time_zone; /* IANA timezone for naive timestamps */ int time_keep; /* keep time field */ int time_strict; /* parse time field strictly */ int logfmt_no_bare_keys; /* in logfmt parsers, require all keys to have values */ @@ -91,6 +92,8 @@ static inline time_t flb_parser_tm2time(const struct flb_tm *src, return res; } +time_t flb_parser_tm2time_parser(const struct flb_tm *src, + struct flb_parser *parser); struct flb_parser *flb_parser_create(const char *name, const char *format, const char *p_regex, @@ -105,6 +108,22 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, int types_len, struct mk_list *decoders, struct flb_config *config); +struct flb_parser *flb_parser_create_with_time_zone(const char *name, + const char *format, + const char *p_regex, + int skip_empty, + const char *time_fmt, + const char *time_key, + const char *time_offset, + int time_keep, + int time_strict, + int time_system_timezone, + const char *time_zone, + int logfmt_no_bare_keys, + struct flb_parser_types *types, + int types_len, + struct mk_list *decoders, + struct flb_config *config); int flb_parser_conf_file_stat(const char *file, struct flb_config *config); int flb_parser_conf_file(const char *file, struct flb_config *config); int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, diff --git a/src/flb_parser.c b/src/flb_parser.c index c4b6b2d6036..562e76f42e0 100644 --- a/src/flb_parser.c +++ b/src/flb_parser.c @@ -42,8 +42,207 @@ #include #include #include +#include #include +#include + +static pthread_mutex_t flb_time_zone_mutex = PTHREAD_MUTEX_INITIALIZER; + +#ifdef FLB_SYSTEM_WINDOWS +static int utf8_to_wide(const char *str, wchar_t *buf, int buf_size) +{ + int ret; + + ret = MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, buf_size); + if (ret == 0) { + return -1; + } + + return 0; +} + +static int windows_time_zone_lookup(const char *windows_zone, + DYNAMIC_TIME_ZONE_INFORMATION *dtzi) +{ + DWORD index; + DWORD ret; + wchar_t wide_zone[128]; + + if (windows_zone == NULL || dtzi == NULL) { + return -1; + } + + if (utf8_to_wide(windows_zone, wide_zone, + sizeof(wide_zone) / sizeof(wide_zone[0])) != 0) { + return -1; + } + + for (index = 0; ; index++) { + memset(dtzi, 0, sizeof(DYNAMIC_TIME_ZONE_INFORMATION)); + ret = EnumDynamicTimeZoneInformation(index, dtzi); + if (ret == ERROR_NO_MORE_ITEMS) { + break; + } + if (ret != ERROR_SUCCESS) { + continue; + } + if (wcscmp(dtzi->TimeZoneKeyName, wide_zone) == 0) { + return 0; + } + } + + return -1; +} + +static int windows_systemtime_from_tm(const struct tm *tm, SYSTEMTIME *st) +{ + int year; + + year = tm->tm_year + 1900; + if (year < 1601 || year > 30827) { + return -1; + } + + memset(st, 0, sizeof(SYSTEMTIME)); + st->wYear = (WORD) year; + st->wMonth = (WORD) (tm->tm_mon + 1); + st->wDay = (WORD) tm->tm_mday; + st->wHour = (WORD) tm->tm_hour; + st->wMinute = (WORD) tm->tm_min; + st->wSecond = (WORD) tm->tm_sec; + + return 0; +} + +static time_t windows_tm2time_zone(const struct flb_tm *src, const char *iana_zone) +{ + int ret; + const char *windows_zone; + struct tm utc_tm; + SYSTEMTIME local_st; + SYSTEMTIME utc_st; + TIME_ZONE_INFORMATION tzi; + DYNAMIC_TIME_ZONE_INFORMATION dtzi; + + windows_zone = flb_time_iana_zone_to_windows(iana_zone); + if (windows_zone == NULL) { + return (time_t) -1; + } + + ret = windows_time_zone_lookup(windows_zone, &dtzi); + if (ret != 0) { + return (time_t) -1; + } + + ret = GetTimeZoneInformationForYear(src->tm.tm_year + 1900, &dtzi, &tzi); + if (ret == 0) { + return (time_t) -1; + } + + ret = windows_systemtime_from_tm(&src->tm, &local_st); + if (ret != 0) { + return (time_t) -1; + } + + ret = TzSpecificLocalTimeToSystemTime(&tzi, &local_st, &utc_st); + if (ret == 0) { + return (time_t) -1; + } + + memset(&utc_tm, 0, sizeof(struct tm)); + utc_tm.tm_year = utc_st.wYear - 1900; + utc_tm.tm_mon = utc_st.wMonth - 1; + utc_tm.tm_mday = utc_st.wDay; + utc_tm.tm_hour = utc_st.wHour; + utc_tm.tm_min = utc_st.wMinute; + utc_tm.tm_sec = utc_st.wSecond; + utc_tm.tm_isdst = 0; + + return timegm(&utc_tm); +} +#endif + +static int validate_time_zone(const char *iana_zone) +{ +#ifdef FLB_SYSTEM_WINDOWS + DYNAMIC_TIME_ZONE_INFORMATION dtzi; +#endif + const char *windows_zone; + + if (iana_zone == NULL || iana_zone[0] == '\0') { + return 0; + } + + /* + * Validate against Fluent Bit's built-in IANA timezone index first. On + * Windows the same entry also gives us the native timezone key. + */ + windows_zone = flb_time_iana_zone_to_windows(iana_zone); + if (windows_zone == NULL) { + return -1; + } + +#ifdef FLB_SYSTEM_WINDOWS + /* Ensure the mapped native timezone is available on this Windows host. */ + if (windows_time_zone_lookup(windows_zone, &dtzi) != 0) { + return -1; + } +#endif + + return 0; +} + +time_t flb_parser_tm2time_parser(const struct flb_tm *src, struct flb_parser *parser) +{ + if (parser->time_zone && parser->time_with_tz == FLB_FALSE) { +#ifdef FLB_SYSTEM_WINDOWS + return windows_tm2time_zone(src, parser->time_zone); +#else + char *old_tz = NULL; + const char *prev; + struct tm tmp; + time_t res; + + pthread_mutex_lock(&flb_time_zone_mutex); + + prev = getenv("TZ"); + if (prev) { + old_tz = flb_strdup(prev); + if (!old_tz) { + pthread_mutex_unlock(&flb_time_zone_mutex); + return (time_t) -1; + } + } + + if (setenv("TZ", parser->time_zone, 1) != 0) { + flb_free(old_tz); + pthread_mutex_unlock(&flb_time_zone_mutex); + return (time_t) -1; + } + + tzset(); + tmp = src->tm; + tmp.tm_isdst = -1; + res = mktime(&tmp); + + if (old_tz) { + setenv("TZ", old_tz, 1); + flb_free(old_tz); + } + else { + unsetenv("TZ"); + } + tzset(); + + pthread_mutex_unlock(&flb_time_zone_mutex); + return res; +#endif + } + + return flb_parser_tm2time(src, parser->time_system_timezone); +} + static inline uint32_t digits10(uint64_t v) { if (v < 10) return 1; if (v < 100) return 2; @@ -140,19 +339,25 @@ static void flb_interim_parser_destroy(struct flb_parser *parser) if (parser->time_key) { flb_free(parser->time_key); } + if (parser->time_zone) { + flb_free(parser->time_zone); + } mk_list_del(&parser->_head); flb_free(parser); } -struct flb_parser *flb_parser_create(const char *name, const char *format, +struct flb_parser *flb_parser_create_with_time_zone(const char *name, + const char *format, const char *p_regex, int skip_empty, - const char *time_fmt, const char *time_key, + const char *time_fmt, + const char *time_key, const char *time_offset, int time_keep, int time_strict, int time_system_timezone, + const char *time_zone, int logfmt_no_bare_keys, struct flb_parser_types *types, int types_len, @@ -231,6 +436,12 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, p->name = flb_strdup(name); + if (time_zone && time_zone[0] && !time_fmt) { + flb_error("[parser:%s] time_zone requires time_format", name); + flb_interim_parser_destroy(p); + return NULL; + } + if (time_fmt) { p->time_fmt_full = flb_strdup(time_fmt); if (!p->time_fmt_full) { @@ -319,11 +530,38 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, */ p->time_system_timezone = time_system_timezone; + if (time_zone && time_zone[0]) { + if (time_system_timezone) { + flb_error("[parser:%s] time_zone cannot be combined with " + "time_system_timezone", + name); + flb_interim_parser_destroy(p); + return NULL; + } + if (time_offset && time_offset[0]) { + flb_error("[parser:%s] time_zone cannot be combined with " + "time_offset", + name); + flb_interim_parser_destroy(p); + return NULL; + } + if (validate_time_zone(time_zone) != 0) { + flb_error("[parser:%s] invalid time_zone '%s'", name, time_zone); + flb_interim_parser_destroy(p); + return NULL; + } + p->time_zone = flb_strdup(time_zone); + if (!p->time_zone) { + flb_interim_parser_destroy(p); + return NULL; + } + } + /* * Optional fixed timezone offset, only applied if - * not falling back to system timezone. + * not falling back to system timezone or an IANA time_zone. */ - if (!p->time_system_timezone && time_offset) { + if (!p->time_system_timezone && !p->time_zone && time_offset) { diff = 0; len = strlen(time_offset); ret = flb_parser_tzone_offset(time_offset, len, &diff); @@ -347,6 +585,28 @@ struct flb_parser *flb_parser_create(const char *name, const char *format, return p; } +struct flb_parser *flb_parser_create(const char *name, const char *format, + const char *p_regex, + int skip_empty, + const char *time_fmt, const char *time_key, + const char *time_offset, + int time_keep, + int time_strict, + int time_system_timezone, + int logfmt_no_bare_keys, + struct flb_parser_types *types, + int types_len, + struct mk_list *decoders, + struct flb_config *config) +{ + return flb_parser_create_with_time_zone(name, format, p_regex, skip_empty, + time_fmt, time_key, time_offset, + time_keep, time_strict, + time_system_timezone, NULL, + logfmt_no_bare_keys, types, + types_len, decoders, config); +} + void flb_parser_destroy(struct flb_parser *parser) { int i = 0; @@ -367,6 +627,9 @@ void flb_parser_destroy(struct flb_parser *parser) if (parser->time_key) { flb_free(parser->time_key); } + if (parser->time_zone) { + flb_free(parser->time_zone); + } if (parser->types_len != 0) { for (i=0; itypes_len; i++){ flb_free(parser->types[i].key); @@ -492,6 +755,7 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, flb_sds_t time_fmt; flb_sds_t time_key; flb_sds_t time_offset; + flb_sds_t time_zone; flb_sds_t types_str; flb_sds_t tmp_str; int skip_empty; @@ -513,6 +777,7 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, time_fmt = NULL; time_key = NULL; time_offset = NULL; + time_zone = NULL; types_str = NULL; tmp_str = NULL; @@ -582,6 +847,9 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, /* time_offset (UTC offset) */ time_offset = get_parser_key(config, cf, s, "time_offset"); + /* time_zone (IANA name for naive timestamps) */ + time_zone = get_parser_key(config, cf, s, "time_zone"); + /* logfmt_no_bare_keys */ logfmt_no_bare_keys = FLB_FALSE; tmp_str = get_parser_key(config, cf, s, "logfmt_no_bare_keys"); @@ -603,10 +871,10 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, decoders = flb_parser_decoder_list_create(s); /* Create the parser context */ - if (!flb_parser_create(name, format, regex, skip_empty, + if (!flb_parser_create_with_time_zone(name, format, regex, skip_empty, time_fmt, time_key, time_offset, time_keep, time_strict, - time_system_timezone, logfmt_no_bare_keys, types, types_len, - decoders, config)) { + time_system_timezone, time_zone, logfmt_no_bare_keys, + types, types_len, decoders, config)) { goto fconf_error; } @@ -627,6 +895,9 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, if (time_offset) { flb_sds_destroy(time_offset); } + if (time_zone) { + flb_sds_destroy(time_zone); + } if (types_str) { flb_sds_destroy(types_str); } @@ -663,6 +934,9 @@ int flb_parser_load_parser_definitions(const char *cfg, struct flb_cf *cf, if (time_offset) { flb_sds_destroy(time_offset); } + if (time_zone) { + flb_sds_destroy(time_zone); + } if (types_str) { flb_sds_destroy(types_str); } @@ -1275,7 +1549,7 @@ int flb_parser_time_lookup(const char *time_str, size_t tsize, } } - if (parser->time_with_tz == FLB_FALSE) { + if (parser->time_with_tz == FLB_FALSE && !parser->time_zone) { flb_tm_gmtoff(tm) = parser->time_offset; } From 8a8fbfe489d9ba94fd0a25e2680bdd5af4998048 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 4 Jun 2026 13:54:39 +0900 Subject: [PATCH 02/14] parser_json: Follow API signature change Signed-off-by: Hiroshi Hatake --- src/flb_parser_json.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_parser_json.c b/src/flb_parser_json.c index 203793a7e9e..fff252085a2 100644 --- a/src/flb_parser_json.c +++ b/src/flb_parser_json.c @@ -211,7 +211,7 @@ int flb_parser_json_do(struct flb_parser *parser, skip = map_size; } else { - time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); + time_lookup = flb_parser_tm2time_parser(&tm, parser); } /* Compose a new map without the time_key field */ From e725bc4d70f9190de45150acb3007ce3682f4157 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 4 Jun 2026 13:55:08 +0900 Subject: [PATCH 03/14] parser_logfmt: Follow API signature change Signed-off-by: Hiroshi Hatake --- src/flb_parser_logfmt.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_parser_logfmt.c b/src/flb_parser_logfmt.c index ad372a07121..5f2681f2d8c 100644 --- a/src/flb_parser_logfmt.c +++ b/src/flb_parser_logfmt.c @@ -166,7 +166,7 @@ static int logfmt_parser(struct flb_parser *parser, parser->name, parser->time_fmt_full); return -1; } - *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); + *time_lookup = flb_parser_tm2time_parser(&tm, parser); } time_found = FLB_TRUE; } From 3232b2dd3c517d1385465b26b8bbff5c26909f01 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 4 Jun 2026 13:55:37 +0900 Subject: [PATCH 04/14] parser_ltsv: Follow API signature change Signed-off-by: Hiroshi Hatake --- src/flb_parser_ltsv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_parser_ltsv.c b/src/flb_parser_ltsv.c index ea099d83bf6..7661e10b80f 100644 --- a/src/flb_parser_ltsv.c +++ b/src/flb_parser_ltsv.c @@ -139,7 +139,7 @@ static int ltsv_parser(struct flb_parser *parser, parser->name, parser->time_fmt_full); return -1; } - *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); + *time_lookup = flb_parser_tm2time_parser(&tm, parser); } time_found = FLB_TRUE; } From 7ca1b0072ae70c15ba0b4dcf67f3dd31ca368e01 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 4 Jun 2026 13:56:25 +0900 Subject: [PATCH 05/14] parser_regex: Follow API Signature change Signed-off-by: Hiroshi Hatake --- src/flb_parser_regex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_parser_regex.c b/src/flb_parser_regex.c index b1baee2d693..a80e1c25906 100644 --- a/src/flb_parser_regex.c +++ b/src/flb_parser_regex.c @@ -87,7 +87,7 @@ static void cb_results(const char *name, const char *value, } pcb->time_frac = frac; - pcb->time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone); + pcb->time_lookup = flb_parser_tm2time_parser(&tm, parser); if (parser->time_keep == FLB_FALSE) { pcb->num_skipped++; From 2cad50f5893ba0587f2113682e00ed20e4b6a089 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 4 Jun 2026 13:56:49 +0900 Subject: [PATCH 06/14] tests: internal: parser: Add test cases for IANA compatible looking up cases Signed-off-by: Hiroshi Hatake --- tests/internal/parser.c | 171 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) diff --git a/tests/internal/parser.c b/tests/internal/parser.c index 8337e4f89fd..3b3bb17b72b 100644 --- a/tests/internal/parser.c +++ b/tests/internal/parser.c @@ -588,6 +588,174 @@ void test_parser_time_system_timezone_midnight() flb_config_exit(config); } +void test_parser_time_zone_iana(void) +{ + int ret; + struct flb_config *config; + struct flb_parser *parser; + struct flb_tm tm; + double ns; + time_t epoch; + + config = flb_config_init(); + TEST_CHECK(config != NULL); + if (!config) { + return; + } + + parser = flb_parser_create_with_time_zone("iana_ny", "regex", + "^(?