From 701ebf8b99c5f5cc42ba6ca1172de7c188b922d4 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 13:04:05 +0900 Subject: [PATCH 01/64] utils: Add option for esacape_unicode for choices raw utf-8 or not Signed-off-by: Hiroshi Hatake --- include/fluent-bit/flb_utils.h | 4 +- src/flb_utils.c | 92 ++++++++++++++++++++++++++++++++-- tests/internal/utils.c | 12 ++--- 3 files changed, 97 insertions(+), 11 deletions(-) diff --git a/include/fluent-bit/flb_utils.h b/include/fluent-bit/flb_utils.h index 56f286f6738..e1dc5ecbc90 100644 --- a/include/fluent-bit/flb_utils.h +++ b/include/fluent-bit/flb_utils.h @@ -57,9 +57,9 @@ void flb_utils_bytes_to_human_readable_size(size_t bytes, char *out_buf, size_t size); int flb_utils_time_split(const char *time, int *sec, long *nsec); int flb_utils_write_str(char *buf, int *off, size_t size, - const char *str, size_t str_len); + const char *str, size_t str_len, int escape_unicode); int flb_utils_write_str_buf(const char *str, size_t str_len, - char **out, size_t *out_size); + char **out, size_t *out_size, int escape_unicode); int flb_utils_url_split(const char *in_url, char **out_protocol, char **out_host, char **out_port, char **out_uri); diff --git a/src/flb_utils.c b/src/flb_utils.c index 9edc9be2e9b..34d0547f46d 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -788,7 +788,7 @@ static const struct escape_seq json_escape_table[128] = { * to escape special characters and convert utf-8 byte characters to string * representation. */ -int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_t str_len) +static int flb_utils_write_str_escaped(char *buf, int *off, size_t size, const char *str, size_t str_len) { int i, b, ret, len, hex_bytes, utf_sequence_length, utf_sequence_number; int processed_bytes = 0; @@ -1096,7 +1096,93 @@ int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_ return FLB_TRUE; } -int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t *out_size) +/* Safely copies raw UTF-8 strings, only escaping essential characters. */ +static int flb_utils_write_str_raw(char *buf, int *off, size_t size, + const char *str, size_t str_len) +{ + int i = 0; + int copypos = 0; + size_t available; + char *p; + off_t offset = 0; + + available = size - *off; + p = buf + *off; + + for (i = 0; i < str_len; i++) { + uint32_t c = (uint32_t) str[i]; + int len = 0; + char *seq = NULL; + + /* Handle essential escapes for JSON validity */ + if (c < 128 && json_escape_table[c].seq) { + seq = json_escape_table[c].seq; + len = json_escape_table[c].seq[1] == 'u' ? 6 : 2; + } + + if (seq) { + if (available < len) { + return FLB_FALSE; + } + memcpy(p, seq, len); + p += len; + offset += len; + available -= len; + } + else if (c < 0x80) { /* Regular ASCII */ + if (available < 1) { + return FLB_FALSE; + } + *p++ = c; + offset++; + available--; + } + else { /* Multibyte UTF-8 sequence */ + int utf_len = flb_utf8_len(&str[i]); + + if (utf_len == 0 || i + utf_len > str_len) { /* Invalid/truncated sequence */ + if (available < 3) { + return FLB_FALSE; + } + memcpy(p, "\xEF\xBF\xBD", 3); /* Standard replacement character */ + p += 3; + offset += 3; + available -= 3; + } + else { /* Valid sequence, copy raw */ + if (available < utf_len) { + return FLB_FALSE; + } + memcpy(p, &str[i], utf_len); + p += utf_len; + offset += utf_len; + available -= utf_len; + i += utf_len - 1; /* Advance loop counter */ + } + } + } + + *off += offset; + return FLB_TRUE; +} + +/* + * This is the wrapper public function for acting as a wrapper and calls the + * appropriate specialized function based on the escape_unicode flag. + */ +int flb_utils_write_str(char *buf, int *off, size_t size, const char *str, size_t str_len, + int escape_unicode) +{ + if (escape_unicode == FLB_TRUE) { + return flb_utils_write_str_escaped(buf, off, size, str, str_len); + } + else { + return flb_utils_write_str_raw(buf, off, size, str, str_len); + } +} + +int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t *out_size, + int escape_unicode) { int ret; int off; @@ -1113,7 +1199,7 @@ int flb_utils_write_str_buf(const char *str, size_t str_len, char **out, size_t while (1) { off = 0; - ret = flb_utils_write_str(buf, &off, s, str, str_len); + ret = flb_utils_write_str(buf, &off, s, str, str_len, escape_unicode); if (ret == FLB_FALSE) { s += 256; tmp = flb_realloc(buf, s); diff --git a/tests/internal/utils.c b/tests/internal/utils.c index a58a1c04532..3bba3b81e47 100644 --- a/tests/internal/utils.c +++ b/tests/internal/utils.c @@ -220,7 +220,7 @@ static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int bu while (!(tcase->input == 0 && tcase->output == 0)) { memset(buf, 0, size); off = 0; - ret = flb_utils_write_str(buf, &off, buf_size, tcase->input, tcase->input_len); + ret = flb_utils_write_str(buf, &off, buf_size, tcase->input, tcase->input_len, FLB_TRUE); if(!TEST_CHECK(ret == tcase->ret)) { TEST_MSG("Input string: %s", tcase->input); @@ -262,30 +262,30 @@ void test_write_str() char jp_expected_output[] = "\\u3042"; off = 0; - ret = flb_utils_write_str(buf, &off, size, "a", 1); + ret = flb_utils_write_str(buf, &off, size, "a", 1, FLB_TRUE); TEST_CHECK(ret == FLB_TRUE); TEST_CHECK(memcmp(buf, "a", off) == 0); off = 0; - ret = flb_utils_write_str(buf, &off, size, "\n", 1); + ret = flb_utils_write_str(buf, &off, size, "\n", 1, FLB_TRUE); TEST_CHECK(ret == FLB_TRUE); TEST_CHECK(memcmp(buf, "\\n", off) == 0); off = 0; - ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82", 3); + ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82", 3, FLB_TRUE); TEST_CHECK(ret == FLB_TRUE); TEST_CHECK(memcmp(buf, jp_expected_output, off) == 0); /* Truncated bytes: 'buf' should not be touched and off == 0 */ off = 0; - ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82\xe3", 1); + ret = flb_utils_write_str(buf, &off, size, "\xe3\x81\x82\xe3", 1, FLB_TRUE); TEST_CHECK(ret == FLB_TRUE); TEST_CHECK(off == 0); TEST_CHECK(memcmp(buf, jp_expected_output, off) == 0); /* Error: buffer too small */ off = 0; - ret = flb_utils_write_str(buf, &off, size, "aaaaaaaaaaa", 11); + ret = flb_utils_write_str(buf, &off, size, "aaaaaaaaaaa", 11, FLB_TRUE); TEST_CHECK(ret == FLB_FALSE); } From 81e1ddfe6c8684dafd5ec4ef36ff104edb1ad590 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 14:16:31 +0900 Subject: [PATCH 02/64] config: Introduce new config for determine whether escaping unicode or not Signed-off-by: Hiroshi Hatake --- include/fluent-bit/flb_config.h | 5 +++++ src/flb_config.c | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/include/fluent-bit/flb_config.h b/include/fluent-bit/flb_config.h index 89f837a66cb..5a3feab2aea 100644 --- a/include/fluent-bit/flb_config.h +++ b/include/fluent-bit/flb_config.h @@ -321,6 +321,8 @@ struct flb_config { struct flb_task_map *task_map; size_t task_map_size; + int json_escape_unicode; + int dry_run; }; @@ -416,4 +418,7 @@ enum conf_type { #define FLB_CONF_STR_SCHED_CAP "scheduler.cap" #define FLB_CONF_STR_SCHED_BASE "scheduler.base" +/* json escape */ +#define FLB_CONF_UNICODE_STR_JSON_ESCAPE "json.escape_unicode" + #endif diff --git a/src/flb_config.c b/src/flb_config.c index c4622db6a16..a6f74de2a77 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -145,8 +145,8 @@ struct flb_service_config service_configs[] = { {FLB_CONF_STORAGE_BL_MEM_LIMIT, FLB_CONF_TYPE_STR, offsetof(struct flb_config, storage_bl_mem_limit)}, - {FLB_CONF_STORAGE_BL_FLUSH_ON_SHUTDOWN, - FLB_CONF_TYPE_BOOL, + {FLB_CONF_STORAGE_BL_FLUSH_ON_SHUTDOWN, + FLB_CONF_TYPE_BOOL, offsetof(struct flb_config, storage_bl_flush_on_shutdown)}, {FLB_CONF_STORAGE_MAX_CHUNKS_UP, FLB_CONF_TYPE_INT, @@ -177,6 +177,11 @@ struct flb_service_config service_configs[] = { FLB_CONF_TYPE_INT, offsetof(struct flb_config, sched_base)}, + /* Escape UNicode inside of JSON */ + {FLB_CONF_UNICODE_STR_JSON_ESCAPE, + FLB_CONF_TYPE_INT, + offsetof(struct flb_config, json_escape_unicode)}, + #ifdef FLB_HAVE_STREAM_PROCESSOR {FLB_CONF_STR_STREAMS_FILE, FLB_CONF_TYPE_STR, @@ -305,6 +310,7 @@ struct flb_config *flb_config_init() config->storage_bl_flush_on_shutdown = FLB_FALSE; config->sched_cap = FLB_SCHED_CAP; config->sched_base = FLB_SCHED_BASE; + config->json_escape_unicode = FLB_TRUE; /* reload */ config->ensure_thread_safety_on_hot_reloading = FLB_TRUE; From 89c87518a56b884ba3a6e16c5395e16154ebda81 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:05:54 +0900 Subject: [PATCH 03/64] pack: Handle escape_unicode arguments Signed-off-by: Hiroshi Hatake --- include/fluent-bit/flb_pack.h | 10 ++++++---- src/flb_pack.c | 32 ++++++++++++++++---------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/include/fluent-bit/flb_pack.h b/include/fluent-bit/flb_pack.h index 875c20f4251..5c76842cde9 100644 --- a/include/fluent-bit/flb_pack.h +++ b/include/fluent-bit/flb_pack.h @@ -91,15 +91,17 @@ int flb_pack_json_valid(const char *json, size_t len); flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, int json_format, int date_format, - flb_sds_t date_key); + flb_sds_t date_key, int escape_unicode); int flb_pack_to_json_format_type(const char *str); int flb_pack_to_json_date_type(const char *str); void flb_pack_print(const char *data, size_t bytes); int flb_msgpack_to_json(char *json_str, size_t str_len, - const msgpack_object *obj); -char* flb_msgpack_to_json_str(size_t size, const msgpack_object *obj); -flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size); + const msgpack_object *obj, + int escape_unicode); +char* flb_msgpack_to_json_str(size_t size, const msgpack_object *obj, + int escape_unicode); +flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size, int escape_unicode); int flb_pack_time_now(msgpack_packer *pck); int flb_msgpack_expand_map(char *map_data, size_t map_size, diff --git a/src/flb_pack.c b/src/flb_pack.c index 28be5551fcb..e80cf0592dc 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -628,7 +628,7 @@ static inline int key_exists_in_map(msgpack_object key, msgpack_object map, int } static int msgpack2json(char *buf, int *off, size_t left, - const msgpack_object *o) + const msgpack_object *o, int escape_unicode) { int i; int dup; @@ -682,7 +682,7 @@ static int msgpack2json(char *buf, int *off, size_t left, case MSGPACK_OBJECT_STR: if (try_to_write(buf, off, left, "\"", 1) && (o->via.str.size > 0 ? - try_to_write_str(buf, off, left, o->via.str.ptr, o->via.str.size) + try_to_write_str(buf, off, left, o->via.str.ptr, o->via.str.size, escape_unicode) : 1/* nothing to do */) && try_to_write(buf, off, left, "\"", 1)) { ret = FLB_TRUE; @@ -692,7 +692,7 @@ static int msgpack2json(char *buf, int *off, size_t left, case MSGPACK_OBJECT_BIN: if (try_to_write(buf, off, left, "\"", 1) && (o->via.bin.size > 0 ? - try_to_write_str(buf, off, left, o->via.bin.ptr, o->via.bin.size) + try_to_write_str(buf, off, left, o->via.bin.ptr, o->via.bin.size, escape_unicode) : 1 /* nothing to do */) && try_to_write(buf, off, left, "\"", 1)) { ret = FLB_TRUE; @@ -729,12 +729,12 @@ static int msgpack2json(char *buf, int *off, size_t left, } if (loop != 0) { msgpack_object* p = o->via.array.ptr; - if (!msgpack2json(buf, off, left, p)) { + if (!msgpack2json(buf, off, left, p, escape_unicode)) { goto msg2json_end; } for (i=1; ikey) || + !msgpack2json(buf, off, left, &(p+i)->key, escape_unicode) || !try_to_write(buf, off, left, ":", 1) || - !msgpack2json(buf, off, left, &(p+i)->val) ) { + !msgpack2json(buf, off, left, &(p+i)->val, escape_unicode) ) { goto msg2json_end; } packed++; @@ -800,7 +800,7 @@ static int msgpack2json(char *buf, int *off, size_t left, * @return success ? a number characters filled : negative value */ int flb_msgpack_to_json(char *json_str, size_t json_size, - const msgpack_object *obj) + const msgpack_object *obj, int escape_unicode) { int ret = -1; int off = 0; @@ -809,12 +809,12 @@ int flb_msgpack_to_json(char *json_str, size_t json_size, return -1; } - ret = msgpack2json(json_str, &off, json_size - 1, obj); + ret = msgpack2json(json_str, &off, json_size - 1, obj, escape_unicode); json_str[off] = '\0'; return ret ? off: ret; } -flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size) +flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size, int escape_unicode) { int ret; size_t off = 0; @@ -849,7 +849,7 @@ flb_sds_t flb_msgpack_raw_to_json_sds(const void *in_buf, size_t in_size) root = &result.data; while (1) { - ret = flb_msgpack_to_json(out_buf, out_size, root); + ret = flb_msgpack_to_json(out_buf, out_size, root, escape_unicode); if (ret <= 0) { realloc_size *= 2; tmp_buf = flb_sds_increase(out_buf, realloc_size); @@ -959,7 +959,7 @@ static int msgpack_pack_formatted_datetime(flb_sds_t out_buf, char time_formatte flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, int json_format, int date_format, - flb_sds_t date_key) + flb_sds_t date_key, int escape_unicode) { int i; int ret; @@ -1166,7 +1166,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, json_format == FLB_PACK_JSON_FORMAT_STREAM) { /* Encode current record into JSON in a temporary variable */ - out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, FLB_TRUE); if (!out_js) { flb_sds_destroy(out_buf); msgpack_sbuffer_destroy(&tmp_sbuf); @@ -1222,7 +1222,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, /* Format to JSON */ if (json_format == FLB_PACK_JSON_FORMAT_JSON) { - out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, escape_unicode); msgpack_sbuffer_destroy(&tmp_sbuf); if (!out_buf) { return NULL; @@ -1247,7 +1247,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, * @param data The msgpack_unpacked data. * @return success ? allocated json str ptr : NULL */ -char *flb_msgpack_to_json_str(size_t size, const msgpack_object *obj) +char *flb_msgpack_to_json_str(size_t size, const msgpack_object *obj, int escape_unicode) { int ret; char *buf = NULL; @@ -1268,7 +1268,7 @@ char *flb_msgpack_to_json_str(size_t size, const msgpack_object *obj) } while (1) { - ret = flb_msgpack_to_json(buf, size, obj); + ret = flb_msgpack_to_json(buf, size, obj, escape_unicode); if (ret <= 0) { /* buffer is small. retry.*/ size *= 2; From acb0bb9e337e3fb74da8f7ed9dd6f2f5f8854f52 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:07:12 +0900 Subject: [PATCH 04/64] sds: Follow the argument increase Signed-off-by: Hiroshi Hatake --- src/flb_sds.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_sds.c b/src/flb_sds.c index 4ac36ad22e8..7d11ba1682d 100644 --- a/src/flb_sds.c +++ b/src/flb_sds.c @@ -312,7 +312,7 @@ flb_sds_t flb_sds_cat_utf8(flb_sds_t *sds, const char *str, int str_len) while (1) { offset = head->len; - ret = flb_utils_write_str(s, &offset, flb_sds_alloc(s), str, str_len); + ret = flb_utils_write_str(s, &offset, flb_sds_alloc(s), str, str_len, FLB_TRUE); if (ret == FLB_FALSE) { /* realloc */ size = flb_sds_alloc(s) * 2; From 4a2c6445596c6054bb6cd0742d0d6dd9efebd8bf Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:07:39 +0900 Subject: [PATCH 05/64] help: Follow the argument increase Signed-off-by: Hiroshi Hatake --- src/flb_help.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_help.c b/src/flb_help.c index ea292591d2a..234b121885a 100644 --- a/src/flb_help.c +++ b/src/flb_help.c @@ -825,7 +825,7 @@ flb_sds_t flb_help_build_json_schema(struct flb_config *config) } flb_mp_array_header_end(&mh); - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); return json; From aecd6dcf9d75aa053f0fd807df73e220a38eb455 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:08:06 +0900 Subject: [PATCH 06/64] record_accessor: Follow the argument change Signed-off-by: Hiroshi Hatake --- src/flb_record_accessor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_record_accessor.c b/src/flb_record_accessor.c index ed66c0f52b7..99cbfd96bbb 100644 --- a/src/flb_record_accessor.c +++ b/src/flb_record_accessor.c @@ -553,7 +553,7 @@ static flb_sds_t ra_translate_keymap(struct flb_ra_parser *rp, flb_sds_t buf, /* Check if is a map or a real bool */ if (v->o.type == MSGPACK_OBJECT_MAP) { /* Convert msgpack map to JSON string */ - js = flb_msgpack_to_json_str(1024, &v->o); + js = flb_msgpack_to_json_str(1024, &v->o, FLB_TRUE); if (js) { len = strlen(js); flb_sds_cat_safe(&buf, js, len); From e18405f7f89ef003103f2b240a55489426b696ce Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:08:50 +0900 Subject: [PATCH 07/64] bin: Follow the argument change Signed-off-by: Hiroshi Hatake --- src/fluent-bit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fluent-bit.c b/src/fluent-bit.c index 0e1b8f8840b..0a51efcb478 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -376,7 +376,8 @@ static void help_format_json(void *help_buf, size_t help_size) { flb_sds_t json; - json = flb_msgpack_raw_to_json_sds(help_buf, help_size); + /* Keep backward compatibility to format help */ + json = flb_msgpack_raw_to_json_sds(help_buf, help_size, FLB_TRUE); printf("%s\n", json); flb_sds_destroy(json); } From 0529b3bf6f7e8b598f0de4d6b957dfaff955de02 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:09:30 +0900 Subject: [PATCH 08/64] http_server: Follow to use escape_unicode argument Signed-off-by: Hiroshi Hatake --- src/http_server/api/v1/metrics.c | 2 +- src/http_server/api/v1/plugins.c | 2 +- src/http_server/api/v1/storage.c | 2 +- src/http_server/api/v1/trace.c | 4 ++-- src/http_server/api/v1/uptime.c | 2 +- src/http_server/api/v2/reload.c | 4 ++-- src/http_server/flb_hs_endpoints.c | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/http_server/api/v1/metrics.c b/src/http_server/api/v1/metrics.c index 1f5ed7136db..9f81ce3411e 100644 --- a/src/http_server/api/v1/metrics.c +++ b/src/http_server/api/v1/metrics.c @@ -153,7 +153,7 @@ static void cb_mq_metrics(mk_mq_t *queue, void *data, size_t size) } /* Convert msgpack to JSON */ - out_data = flb_msgpack_raw_to_json_sds(data, size); + out_data = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); if (!out_data) { return; } diff --git a/src/http_server/api/v1/plugins.c b/src/http_server/api/v1/plugins.c index c755173e643..4cf1c2f5675 100644 --- a/src/http_server/api/v1/plugins.c +++ b/src/http_server/api/v1/plugins.c @@ -90,7 +90,7 @@ static void cb_plugins(mk_request_t *request, void *data) } /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); mk_http_status(request, 200); diff --git a/src/http_server/api/v1/storage.c b/src/http_server/api/v1/storage.c index 56add002ed0..33d9e4de918 100644 --- a/src/http_server/api/v1/storage.c +++ b/src/http_server/api/v1/storage.c @@ -105,7 +105,7 @@ static void cb_mq_storage_metrics(mk_mq_t *queue, void *data, size_t size) } /* Convert msgpack to JSON */ - out_data = flb_msgpack_raw_to_json_sds(data, size); + out_data = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); if (!out_data) { return; } diff --git a/src/http_server/api/v1/trace.c b/src/http_server/api/v1/trace.c index 1f4021d685b..1ed6f2c32ac 100644 --- a/src/http_server/api/v1/trace.c +++ b/src/http_server/api/v1/trace.c @@ -471,7 +471,7 @@ static void cb_trace(mk_request_t *request, void *data) } /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); if (out_buf == NULL) { mk_http_status(request, 503); mk_http_done(request); @@ -638,7 +638,7 @@ static void cb_traces(mk_request_t *request, void *data) } /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); if (out_buf == NULL) { out_buf = flb_sds_create("serialization error"); } diff --git a/src/http_server/api/v1/uptime.c b/src/http_server/api/v1/uptime.c index 12a96a48cce..cb4e567ee84 100644 --- a/src/http_server/api/v1/uptime.c +++ b/src/http_server/api/v1/uptime.c @@ -88,7 +88,7 @@ static void cb_uptime(mk_request_t *request, void *data) uptime_hr(uptime, &mp_pck); /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { return; diff --git a/src/http_server/api/v2/reload.c b/src/http_server/api/v2/reload.c index 2f8c947cbd9..b57aff14930 100644 --- a/src/http_server/api/v2/reload.c +++ b/src/http_server/api/v2/reload.c @@ -114,7 +114,7 @@ static void handle_reload_request(mk_request_t *request, struct flb_config *conf #endif /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { mk_http_status(request, 400); @@ -148,7 +148,7 @@ static void handle_get_reload_status(mk_request_t *request, struct flb_config *c msgpack_pack_int64(&mp_pck, config->hot_reloaded_count); /* Export to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { mk_http_status(request, 400); diff --git a/src/http_server/flb_hs_endpoints.c b/src/http_server/flb_hs_endpoints.c index 2ea12a981bb..ffda448bfb1 100644 --- a/src/http_server/flb_hs_endpoints.c +++ b/src/http_server/flb_hs_endpoints.c @@ -93,7 +93,7 @@ static int endpoint_root(struct flb_hs *hs) flb_utils_split_free(list); /* export as JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&mp_sbuf); if (out_buf) { From 2d6fb640e43c8cc8678dfee45e0b758a5f24b913 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:10:23 +0900 Subject: [PATCH 09/64] filter_expect: Follow to use escape_unicode argument Signed-off-by: Hiroshi Hatake --- plugins/filter_expect/expect.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/filter_expect/expect.c b/plugins/filter_expect/expect.c index 425c9017136..7722f0f6f79 100644 --- a/plugins/filter_expect/expect.c +++ b/plugins/filter_expect/expect.c @@ -272,7 +272,7 @@ static char *ra_value_type_to_str(struct flb_ra_value *val) return "UNKNOWN"; } -static int rule_apply(struct flb_expect *ctx, msgpack_object map) +static int rule_apply(struct flb_expect *ctx, msgpack_object map, struct flb_config *config) { int n = 0; char *json; @@ -292,7 +292,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) continue; } - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_exists', key '%s' " "not found. Record content:\n%s", @@ -305,7 +305,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) n++; continue; } - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_not_exists', key '%s' " "exists. Record content:\n%s", @@ -316,7 +316,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) } else if (rule->type == FLB_EXP_KEY_VAL_NULL) { if (!val) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_null', " "key '%s' not found. Record content:\n%s", @@ -325,7 +325,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) return FLB_FALSE; } if (val->type != FLB_RA_NULL) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_null', " "key '%s' contains a value type '%s'. " @@ -340,7 +340,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) } else if (rule->type == FLB_EXP_KEY_VAL_NOT_NULL) { if (!val) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_not_null', " "key '%s' not found. Record content:\n%s", @@ -349,7 +349,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) return FLB_FALSE; } if (val->type == FLB_RA_NULL) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_not_null', " "key '%s' contains a value type '%s'. " @@ -364,7 +364,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) } else if (rule->type == FLB_EXP_KEY_VAL_EQ) { if (!val) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_is_null', " "key '%s' not found. Record content:\n%s", @@ -376,7 +376,7 @@ static int rule_apply(struct flb_expect *ctx, msgpack_object map) if (val->type == FLB_RA_STRING) { if (flb_sds_cmp(val->val.string, rule->expect, flb_sds_len(rule->expect)) != 0) { - json = flb_msgpack_to_json_str(size, &map); + json = flb_msgpack_to_json_str(size, &map, config->json_escape_unicode); flb_plg_error(ctx->ins, "exception on rule #%i 'key_val_eq', " "key value '%s' is different than " @@ -430,7 +430,7 @@ static int cb_expect_filter(const void *data, size_t bytes, while ((ret = flb_log_event_decoder_next( &log_decoder, &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - ret = rule_apply(ctx, *log_event.body); + ret = rule_apply(ctx, *log_event.body, config); if (ret == FLB_TRUE) { /* rule matches, we are good */ continue; From 54ffb04360f0edcd4dd37ee96a48a52f97e9c1c4 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:11:54 +0900 Subject: [PATCH 10/64] filter_nightfall: Follow to use escape_unicode argument Signed-off-by: Hiroshi Hatake --- plugins/filter_nightfall/nightfall_api.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/filter_nightfall/nightfall_api.c b/plugins/filter_nightfall/nightfall_api.c index 91ecf7948e2..accd4471128 100644 --- a/plugins/filter_nightfall/nightfall_api.c +++ b/plugins/filter_nightfall/nightfall_api.c @@ -195,8 +195,8 @@ static flb_sds_t build_request_body(struct flb_filter_nightfall *ctx, msgpack_pack_str_with_body(&req_pk, "policyUUIDs", 11); msgpack_pack_array(&req_pk, 1); msgpack_pack_str_with_body(&req_pk, ctx->policy_id, 36); - - request_body = flb_msgpack_raw_to_json_sds(req_sbuf.data, req_sbuf.size); + + request_body = flb_msgpack_raw_to_json_sds(req_sbuf.data, req_sbuf.size, FLB_TRUE); msgpack_sbuffer_destroy(&req_sbuf); flb_sds_destroy(num_str); From 0904ab5413d20fbb25c589f416c5299a29b4bb5e Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:12:30 +0900 Subject: [PATCH 11/64] filter_wasm: Refer the config whether escaping unicode characters or not Signed-off-by: Hiroshi Hatake --- plugins/filter_wasm/filter_wasm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/filter_wasm/filter_wasm.c b/plugins/filter_wasm/filter_wasm.c index a6246734646..dcc2d49be17 100644 --- a/plugins/filter_wasm/filter_wasm.c +++ b/plugins/filter_wasm/filter_wasm.c @@ -104,7 +104,7 @@ static int cb_wasm_filter(const void *data, size_t bytes, switch(ctx->event_format) { case FLB_FILTER_WASM_FMT_JSON: /* Encode as JSON from msgpack */ - buf = flb_msgpack_to_json_str(alloc_size, log_event.body); + buf = flb_msgpack_to_json_str(alloc_size, log_event.body, config->json_escape_unicode); if (buf) { /* Execute WASM program */ From a01f35bbf0ad8af4f4334dc5d712bb33c2dd4015 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:13:33 +0900 Subject: [PATCH 12/64] out_azure: Refer the config whether esacaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_azure/azure.c | 9 ++++++--- plugins/out_azure_blob/azure_blob.c | 3 ++- plugins/out_azure_kusto/azure_kusto.c | 12 ++++++++---- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/plugins/out_azure/azure.c b/plugins/out_azure/azure.c index e5eee8c3b17..a43b13594d9 100644 --- a/plugins/out_azure/azure.c +++ b/plugins/out_azure/azure.c @@ -51,7 +51,8 @@ static int cb_azure_init(struct flb_output_instance *ins, static int azure_format(const void *in_buf, size_t in_bytes, flb_sds_t tag, flb_sds_t *tag_val_out, char **out_buf, size_t *out_size, - struct flb_azure *ctx) + struct flb_azure *ctx, + struct flb_config *config) { int i; int array_size = 0; @@ -160,7 +161,8 @@ static int azure_format(const void *in_buf, size_t in_bytes, msgpack_sbuffer_destroy(&tmp_sbuf); } - record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!record) { flb_errno(); @@ -317,7 +319,8 @@ static void cb_azure_flush(struct flb_event_chunk *event_chunk, /* Convert binary logs into a JSON payload */ ret = azure_format(event_chunk->data, event_chunk->size, - event_chunk->tag, &final_log_type, &buf_data, &buf_size, ctx); + event_chunk->tag, &final_log_type, &buf_data, &buf_size, ctx, + config); /* If cannot get matching record using log_type_prefix, use log_type directly */ if (!final_log_type) { final_log_type = ctx->log_type; diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index d0ae18d503c..a650809c4d4 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -68,7 +68,8 @@ static int azure_blob_format(struct flb_config *config, out_buf = flb_pack_msgpack_to_json_format(data, bytes, FLB_PACK_JSON_FORMAT_LINES, FLB_PACK_JSON_DATE_ISO8601, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); if (!out_buf) { return -1; } diff --git a/plugins/out_azure_kusto/azure_kusto.c b/plugins/out_azure_kusto/azure_kusto.c index ccc611112d6..041460992c5 100644 --- a/plugins/out_azure_kusto/azure_kusto.c +++ b/plugins/out_azure_kusto/azure_kusto.c @@ -979,7 +979,8 @@ static int cb_azure_kusto_init(struct flb_output_instance *ins, struct flb_confi */ static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int tag_len, const void *data, size_t bytes, void **out_data, - size_t *out_size) + size_t *out_size, + struct flb_config *config) { int index; int records = 0; @@ -1086,7 +1087,8 @@ static int azure_kusto_format(struct flb_azure_kusto *ctx, const char *tag, int msgpack_pack_str_body(&mp_pck, "log_attribute_missing", 20); } - flb_sds_t json_record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + flb_sds_t json_record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!json_record) { flb_plg_error(ctx->ins, "error converting msgpack to JSON"); flb_sds_destroy(out_buf); @@ -1256,7 +1258,8 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, /* Reformat msgpack to JSON payload */ ret = azure_kusto_format(ctx, tag_name, tag_name_len, event_chunk->data, - event_chunk->size, (void **)&json, &json_size); + event_chunk->size, (void **)&json, &json_size, + config); if (ret != 0) { flb_plg_error(ctx->ins, "cannot reformat data into json"); ret = FLB_RETRY; @@ -1367,7 +1370,8 @@ static void cb_azure_kusto_flush(struct flb_event_chunk *event_chunk, /* Reformat msgpack data to JSON payload */ ret = azure_kusto_format(ctx, event_chunk->tag, tag_len, event_chunk->data, - event_chunk->size, (void **)&json, &json_size); + event_chunk->size, (void **)&json, &json_size, + config); if (ret != 0) { flb_plg_error(ctx->ins, "cannot reformat data into json"); ret = FLB_RETRY; From 98c30305c7f4aabcaac8efbd390404dbd0aab57c Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:16:03 +0900 Subject: [PATCH 13/64] out_azure_logs_ingestion: Refer to use a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_azure_logs_ingestion/azure_logs_ingestion.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c b/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c index e816488cd8a..f9f11188b5f 100644 --- a/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c +++ b/plugins/out_azure_logs_ingestion/azure_logs_ingestion.c @@ -54,7 +54,8 @@ static int cb_azure_logs_ingestion_init(struct flb_output_instance *ins, allocates sds string */ static int az_li_format(const void *in_buf, size_t in_bytes, char **out_buf, size_t *out_size, - struct flb_az_li *ctx) + struct flb_az_li *ctx, + struct flb_config *config) { int i; int array_size = 0; @@ -141,7 +142,8 @@ static int az_li_format(const void *in_buf, size_t in_bytes, msgpack_sbuffer_destroy(&tmp_sbuf); } - record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!record) { flb_errno(); msgpack_sbuffer_destroy(&mp_sbuf); @@ -266,7 +268,8 @@ static void cb_azure_logs_ingestion_flush(struct flb_event_chunk *event_chunk, /* Convert binary logs into a JSON payload */ ret = az_li_format(event_chunk->data, event_chunk->size, - &json_payload, &json_payload_size, ctx); + &json_payload, &json_payload_size, ctx, + config); if (ret == -1) { flb_upstream_conn_release(u_conn); FLB_OUTPUT_RETURN(FLB_ERROR); From 70e25f8431dc22a688c47b646e925b41ab552c68 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:16:57 +0900 Subject: [PATCH 14/64] out_bigquery: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_bigquery/bigquery.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/out_bigquery/bigquery.c b/plugins/out_bigquery/bigquery.c index b21cff5241c..c5cfeeb2b25 100644 --- a/plugins/out_bigquery/bigquery.c +++ b/plugins/out_bigquery/bigquery.c @@ -847,7 +847,8 @@ static int cb_bigquery_init(struct flb_output_instance *ins, static int bigquery_format(const void *data, size_t bytes, const char *tag, size_t tag_len, char **out_data, size_t *out_size, - struct flb_bigquery *ctx) + struct flb_bigquery *ctx, + struct flb_config *config) { int array_size = 0; flb_sds_t out_buf; @@ -937,7 +938,8 @@ static int bigquery_format(const void *data, size_t bytes, } /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); flb_log_event_decoder_destroy(&log_decoder); msgpack_sbuffer_destroy(&mp_sbuf); @@ -996,7 +998,7 @@ static void cb_bigquery_flush(struct flb_event_chunk *event_chunk, /* Reformat msgpack to bigquery JSON payload */ ret = bigquery_format(event_chunk->data, event_chunk->size, event_chunk->tag, flb_sds_len(event_chunk->tag), - &payload_buf, &payload_size, ctx); + &payload_buf, &payload_size, ctx, config); if (ret != 0) { flb_upstream_conn_release(u_conn); flb_sds_destroy(token); From 8187dd686433b48f50fd26d1b671f937c98f18b9 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:17:26 +0900 Subject: [PATCH 15/64] out_calyptia: Use a flag always to escape unicode Signed-off-by: Hiroshi Hatake --- plugins/out_calyptia/calyptia.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/out_calyptia/calyptia.c b/plugins/out_calyptia/calyptia.c index d26270583af..9966828ce37 100644 --- a/plugins/out_calyptia/calyptia.c +++ b/plugins/out_calyptia/calyptia.c @@ -311,7 +311,7 @@ static flb_sds_t get_agent_metadata(struct flb_calyptia *ctx) flb_mp_map_header_end(&mh); /* convert to json */ - meta = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + meta = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); /* could be ASCII */ msgpack_sbuffer_destroy(&mp_sbuf); return meta; @@ -532,7 +532,7 @@ static int store_session_get(struct flb_calyptia *ctx, } /* decode */ - json = flb_msgpack_raw_to_json_sds(buf, size); + json = flb_msgpack_raw_to_json_sds(buf, size, FLB_TRUE); /* TODO: could be ASCII? */ flb_free(buf); if (!json) { return -1; @@ -1029,7 +1029,8 @@ static void cb_calyptia_flush(struct flb_event_chunk *event_chunk, event_chunk->size, FLB_PACK_JSON_FORMAT_STREAM, FLB_PACK_JSON_DATE_DOUBLE, - NULL); + NULL, + FLB_TRUE); /* Trace is ASCII */ if (json == NULL) { flb_upstream_conn_release(u_conn); FLB_OUTPUT_RETURN(FLB_RETRY); From 37b0292b67c43c68740395d0b895ab72f71d9d2d Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:18:25 +0900 Subject: [PATCH 16/64] out_chronicle: Refer to use a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_chronicle/chronicle.c | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/plugins/out_chronicle/chronicle.c b/plugins/out_chronicle/chronicle.c index ac43c8b997c..855e1b7fc46 100644 --- a/plugins/out_chronicle/chronicle.c +++ b/plugins/out_chronicle/chronicle.c @@ -515,7 +515,9 @@ static int cb_chronicle_init(struct flb_output_instance *ins, return 0; } -static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, uint64_t bytes, struct flb_log_event log_event) +static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, uint64_t bytes, + struct flb_log_event log_event, + struct flb_config *config) { int i; int map_size; @@ -591,7 +593,8 @@ static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, uint64_t by } else { ret = flb_msgpack_to_json(val_buf + val_offset, - msgpack_size - val_offset, &val); + msgpack_size - val_offset, &val, + config->json_escape_unicode); if (ret < 0) { break; } @@ -677,7 +680,8 @@ static int chronicle_format(const void *data, size_t bytes, size_t last_offset, size_t threshold, size_t *out_offset, struct flb_log_event_decoder *log_decoder, - struct flb_chronicle *ctx) + struct flb_chronicle *ctx, + struct flb_config *config) { int len; int ret; @@ -722,7 +726,7 @@ static int chronicle_format(const void *data, size_t bytes, last_off = off; if (ctx->log_key != NULL) { - log_text = flb_pack_msgpack_extract_log_key(ctx, bytes, log_event); + log_text = flb_pack_msgpack_extract_log_key(ctx, bytes, log_event, config); if (log_text == NULL) { flb_plg_error(ctx->ins, "log_key extraction failed, skipping record"); continue; @@ -730,7 +734,8 @@ static int chronicle_format(const void *data, size_t bytes, log_text_size = flb_sds_len(log_text); } else { - json_str = flb_msgpack_to_json_str(alloc_size, log_event.body); + json_str = flb_msgpack_to_json_str(alloc_size, log_event.body, + config->json_escape_unicode); if (json_str == NULL) { flb_plg_error(ctx->ins, "Could not convert record to json string"); msgpack_sbuffer_destroy(&mp_sbuf); @@ -865,7 +870,8 @@ static int chronicle_format(const void *data, size_t bytes, } /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { @@ -903,7 +909,7 @@ static int cb_chronicle_format_test(struct flb_config *config, ret = chronicle_format(data, bytes, tag, tag_len, (char **)out_data, out_size, 0, bytes, &out_offset, - &log_decoder, ctx); + &log_decoder, ctx, config); flb_log_event_decoder_destroy(&log_decoder); return ret; @@ -994,7 +1000,7 @@ static void cb_chronicle_flush(struct flb_event_chunk *event_chunk, &payload_buf, &payload_size, offset, threshold, &out_offset, &log_decoder, - ctx); + ctx, config); if (ret != 0) { flb_upstream_conn_release(u_conn); flb_sds_destroy(token); From 4cf8872ff31aa8335a1375217bf97bd3e9e94de8 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:23:32 +0900 Subject: [PATCH 17/64] out_cloudwatch: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_cloudwatch_logs/cloudwatch_api.c | 37 +++++++++++-------- plugins/out_cloudwatch_logs/cloudwatch_api.h | 3 +- plugins/out_cloudwatch_logs/cloudwatch_logs.c | 2 +- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.c b/plugins/out_cloudwatch_logs/cloudwatch_api.c index d8a441f6494..f5d04bb02ec 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_api.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_api.c @@ -660,7 +660,8 @@ void remove_unneeded_field(msgpack_object *root_map, const char *nested_map_key, * which means a send must occur */ int process_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { size_t written; int ret; @@ -671,8 +672,8 @@ int process_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; ret = flb_msgpack_to_json(tmp_buf_ptr, - buf->tmp_buf_size - buf->tmp_buf_offset, - obj); + buf->tmp_buf_size - buf->tmp_buf_offset, + obj, config->json_escape_unicode); if (ret <= 0) { /* * failure to write to buffer, @@ -706,7 +707,8 @@ int process_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, } offset = 0; if (!flb_utils_write_str(buf->event_buf, &offset, size, - tmp_buf_ptr, written)) { + tmp_buf_ptr, written, + config->json_escape_unicode)) { return -1; } written = offset; @@ -833,7 +835,8 @@ int send_log_events(struct flb_cloudwatch *ctx, struct cw_flush *buf) { */ int add_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, struct log_stream *stream, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { int ret; struct cw_event *event; @@ -854,7 +857,7 @@ int add_event(struct flb_cloudwatch *ctx, struct cw_flush *buf, reset_flush_buf(ctx, buf); } - ret = process_event(ctx, buf, obj, tms); + ret = process_event(ctx, buf, obj, tms, config); if (ret < 0) { return -1; } @@ -1249,7 +1252,7 @@ void update_or_create_entity(struct flb_cloudwatch *ctx, struct log_stream *stre static int process_log_events(struct flb_cloudwatch *ctx, const char *input_plugin, struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes) + const char *data, size_t bytes, struct flb_config *config) { int i = 0; size_t map_size; @@ -1368,7 +1371,8 @@ static int process_log_events(struct flb_cloudwatch *ctx, const char *input_plug found = FLB_TRUE; val = (kv+j)->val; ret = add_event(ctx, buf, stream, &val, - &log_event.timestamp); + &log_event.timestamp, + config); if (ret < 0 ) { goto error; } @@ -1445,14 +1449,14 @@ static int process_log_events(struct flb_cloudwatch *ctx, const char *input_plug goto error; } ret = add_event(ctx, buf, stream, &emf_payload, - &log_event.timestamp); + &log_event.timestamp, config); msgpack_unpacked_destroy(&mp_emf_result); msgpack_sbuffer_destroy(&mp_sbuf); } else { ret = add_event(ctx, buf, stream, &map, - &log_event.timestamp); + &log_event.timestamp, config); } if (ret < 0 ) { @@ -1483,7 +1487,7 @@ static int process_log_events(struct flb_cloudwatch *ctx, const char *input_plug static int process_metric_events(struct flb_cloudwatch *ctx, const char *input_plugin, struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes) + const char *data, size_t bytes, struct flb_config *config) { int i = 0; int ret; @@ -1522,7 +1526,7 @@ static int process_metric_events(struct flb_cloudwatch *ctx, const char *input_p flb_time_get(&tm); ret = add_event(ctx, buf, stream, &map, - &tm); + &tm, config); if (ret < 0 ) { goto cmt_error; @@ -1551,7 +1555,8 @@ static int process_metric_events(struct flb_cloudwatch *ctx, const char *input_p */ int process_and_send(struct flb_cloudwatch *ctx, const char *input_plugin, struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes, int event_type) + const char *data, size_t bytes, int event_type, + struct flb_config *config) { int ret; int i = 0; @@ -1559,12 +1564,14 @@ int process_and_send(struct flb_cloudwatch *ctx, const char *input_plugin, if (event_type == FLB_EVENT_TYPE_LOGS) { i = process_log_events(ctx, input_plugin, buf, tag, - data, bytes); + data, bytes, + config); } else if (event_type == FLB_EVENT_TYPE_METRICS) { i = process_metric_events(ctx, input_plugin, buf, tag, - data, bytes); + data, bytes, + config); } /* send any remaining events */ ret = send_log_events(ctx, buf); diff --git a/plugins/out_cloudwatch_logs/cloudwatch_api.h b/plugins/out_cloudwatch_logs/cloudwatch_api.h index 697b15155cd..837e4b85ded 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_api.h +++ b/plugins/out_cloudwatch_logs/cloudwatch_api.h @@ -56,7 +56,8 @@ void cw_flush_destroy(struct cw_flush *buf); int process_and_send(struct flb_cloudwatch *ctx, const char *input_plugin, struct cw_flush *buf, flb_sds_t tag, - const char *data, size_t bytes, int event_type); + const char *data, size_t bytes, int event_type, + struct flb_config *config); int create_log_stream(struct flb_cloudwatch *ctx, struct log_stream *stream, int can_retry); struct log_stream *get_log_stream(struct flb_cloudwatch *ctx, flb_sds_t tag, const msgpack_object map); diff --git a/plugins/out_cloudwatch_logs/cloudwatch_logs.c b/plugins/out_cloudwatch_logs/cloudwatch_logs.c index 4ab884ba9a0..cc03ed8a879 100644 --- a/plugins/out_cloudwatch_logs/cloudwatch_logs.c +++ b/plugins/out_cloudwatch_logs/cloudwatch_logs.c @@ -455,7 +455,7 @@ static void cb_cloudwatch_flush(struct flb_event_chunk *event_chunk, } event_count = process_and_send(ctx, i_ins->p->name, buf, event_chunk->tag, event_chunk->data, event_chunk->size, - event_chunk->type); + event_chunk->type, config); if (event_count < 0) { flb_plg_error(ctx->ins, "Failed to send events"); cw_flush_destroy(buf); From 62e042e95fcbb48c5b2fa7e678cc15d0bdbf877b Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:24:07 +0900 Subject: [PATCH 18/64] out_datadog: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_datadog/datadog.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_datadog/datadog.c b/plugins/out_datadog/datadog.c index e5c6621f880..a3a3d7e8e21 100644 --- a/plugins/out_datadog/datadog.c +++ b/plugins/out_datadog/datadog.c @@ -325,7 +325,8 @@ static int datadog_format(struct flb_config *config, } /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { From 79d0ce1296fce2ecfe44edd6e42d60488561d001 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:24:34 +0900 Subject: [PATCH 19/64] out_es: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_es/es.c | 3 ++- plugins/out_file/file.c | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/out_es/es.c b/plugins/out_es/es.c index 5d8a8da3eab..f7e7a5de10e 100644 --- a/plugins/out_es/es.c +++ b/plugins/out_es/es.c @@ -559,7 +559,8 @@ static int elasticsearch_format(struct flb_config *config, } /* Convert msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&tmp_sbuf); if (!out_buf) { flb_log_event_decoder_destroy(&log_decoder); diff --git a/plugins/out_file/file.c b/plugins/out_file/file.c index c7db85fc2d6..4c6eb301dfc 100644 --- a/plugins/out_file/file.c +++ b/plugins/out_file/file.c @@ -328,11 +328,11 @@ static int template_output(FILE *fp, struct flb_time *tm, msgpack_object *obj, } -static int plain_output(FILE *fp, msgpack_object *obj, size_t alloc_size) +static int plain_output(FILE *fp, msgpack_object *obj, size_t alloc_size, int escape_unicode) { char *buf; - buf = flb_msgpack_to_json_str(alloc_size, obj); + buf = flb_msgpack_to_json_str(alloc_size, obj, escape_unicode); if (buf) { fprintf(fp, "%s" NEWLINE, buf); @@ -616,7 +616,8 @@ static void cb_file_flush(struct flb_event_chunk *event_chunk, switch (ctx->format){ case FLB_OUT_FILE_FMT_JSON: - buf = flb_msgpack_to_json_str(alloc_size, log_event.body); + buf = flb_msgpack_to_json_str(alloc_size, log_event.body, + config->json_escape_unicode); if (buf) { fprintf(fp, "%s: [%"PRIu64".%09lu, %s]" NEWLINE, event_chunk->tag, @@ -648,7 +649,7 @@ static void cb_file_flush(struct flb_event_chunk *event_chunk, log_event.body, ctx); break; case FLB_OUT_FILE_FMT_PLAIN: - plain_output(fp, log_event.body, alloc_size); + plain_output(fp, log_event.body, alloc_size, config->json_escape_unicode); break; case FLB_OUT_FILE_FMT_TEMPLATE: From 1c0e411580d890cbb0d525a3f4ba78958a232ced Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:24:59 +0900 Subject: [PATCH 20/64] out_http: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_http/http.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/out_http/http.c b/plugins/out_http/http.c index 95bf4f55212..88b62673742 100644 --- a/plugins/out_http/http.c +++ b/plugins/out_http/http.c @@ -436,7 +436,8 @@ static int compose_payload_gelf(struct flb_out_http *ctx, static int compose_payload(struct flb_out_http *ctx, const void *in_body, size_t in_size, - void **out_body, size_t *out_size) + void **out_body, size_t *out_size, + struct flb_config *config) { flb_sds_t encoded; @@ -451,7 +452,8 @@ static int compose_payload(struct flb_out_http *ctx, in_size, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); if (encoded == NULL) { flb_plg_error(ctx->ins, "failed to convert json"); return FLB_ERROR; @@ -638,7 +640,7 @@ static void cb_http_flush(struct flb_event_chunk *event_chunk, } else { ret = compose_payload(ctx, event_chunk->data, event_chunk->size, - &out_body, &out_size); + &out_body, &out_size, config); if (ret != FLB_OK) { FLB_OUTPUT_RETURN(ret); } @@ -811,7 +813,7 @@ static int cb_http_format_test(struct flb_config *config, struct flb_out_http *ctx = plugin_context; int ret; - ret = compose_payload(ctx, data, bytes, out_data, out_size); + ret = compose_payload(ctx, data, bytes, out_data, out_size, config); if (ret != FLB_OK) { flb_error("ret=%d", ret); return -1; From f264680f35bb0cf6349fadfea468bfaa790bcde5 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:34:43 +0900 Subject: [PATCH 21/64] out_influxdb: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_influxdb/influxdb.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_influxdb/influxdb.c b/plugins/out_influxdb/influxdb.c index 671dd5c16a6..2c191354b31 100644 --- a/plugins/out_influxdb/influxdb.c +++ b/plugins/out_influxdb/influxdb.c @@ -216,7 +216,8 @@ static int influxdb_format(struct flb_config *config, /* is this a string ? */ if (quote == FLB_TRUE) { ret = flb_utils_write_str_buf(val, val_len, - &str, &str_size); + &str, &str_size, + config->json_escape_unicode); if (ret == -1) { flb_errno(); goto error; From 5b617c6c28481012090991322ae62125daae22f9 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:35:58 +0900 Subject: [PATCH 22/64] out_kafka: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_kafka/kafka.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_kafka/kafka.c b/plugins/out_kafka/kafka.c index 09006bc80d8..dadd4725f74 100644 --- a/plugins/out_kafka/kafka.c +++ b/plugins/out_kafka/kafka.c @@ -284,7 +284,8 @@ int produce_message(struct flb_time *tm, msgpack_object *map, } if (ctx->format == FLB_KAFKA_FMT_JSON) { - s = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + s = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!s) { flb_plg_error(ctx->ins, "error encoding to JSON"); msgpack_sbuffer_destroy(&mp_sbuf); From 0b740221ab3c3473f545f08fe5e97b0e41688ff3 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:36:31 +0900 Subject: [PATCH 23/64] out_kafka_rest: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_kafka_rest/kafka.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/plugins/out_kafka_rest/kafka.c b/plugins/out_kafka_rest/kafka.c index a9656a69a17..ef13d079fbc 100644 --- a/plugins/out_kafka_rest/kafka.c +++ b/plugins/out_kafka_rest/kafka.c @@ -94,7 +94,8 @@ static struct flb_config_map config_map[] = { static flb_sds_t kafka_rest_format(const void *data, size_t bytes, const char *tag, int tag_len, size_t *out_size, - struct flb_kafka_rest *ctx) + struct flb_kafka_rest *ctx, + struct flb_config *config) { int i; int len; @@ -211,7 +212,8 @@ static flb_sds_t kafka_rest_format(const void *data, size_t bytes, flb_log_event_decoder_destroy(&log_decoder); /* Convert to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { return NULL; @@ -268,7 +270,7 @@ static void cb_kafka_flush(struct flb_event_chunk *event_chunk, /* Convert format */ js = kafka_rest_format(event_chunk->data, event_chunk->size, event_chunk->tag, flb_sds_len(event_chunk->tag), - &js_size, ctx); + &js_size, ctx, config); if (!js) { flb_upstream_conn_release(u_conn); FLB_OUTPUT_RETURN(FLB_ERROR); From 94825c55d79cd016ad23c985ba63f0354a9451e0 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:37:40 +0900 Subject: [PATCH 24/64] out_kinesis_firehose: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_kinesis_firehose/firehose.c | 2 +- plugins/out_kinesis_firehose/firehose_api.c | 19 +++++++++++-------- plugins/out_kinesis_firehose/firehose_api.h | 3 ++- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/plugins/out_kinesis_firehose/firehose.c b/plugins/out_kinesis_firehose/firehose.c index a10d60e24dd..0b998599f05 100644 --- a/plugins/out_kinesis_firehose/firehose.c +++ b/plugins/out_kinesis_firehose/firehose.c @@ -363,7 +363,7 @@ static void cb_firehose_flush(struct flb_event_chunk *event_chunk, } ret = process_and_send_records(ctx, buf, - event_chunk->data, event_chunk->size); + event_chunk->data, event_chunk->size, config); if (ret < 0) { flb_plg_error(ctx->ins, "Failed to send records"); flush_destroy(buf); diff --git a/plugins/out_kinesis_firehose/firehose_api.c b/plugins/out_kinesis_firehose/firehose_api.c index d12e4299f5d..a87f2008d9f 100644 --- a/plugins/out_kinesis_firehose/firehose_api.c +++ b/plugins/out_kinesis_firehose/firehose_api.c @@ -152,7 +152,8 @@ static int end_put_payload(struct flb_firehose *ctx, struct flush *buf, * which means a send must occur */ static int process_event(struct flb_firehose *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { size_t written = 0; int ret; @@ -170,8 +171,8 @@ static int process_event(struct flb_firehose *ctx, struct flush *buf, tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; ret = flb_msgpack_to_json(tmp_buf_ptr, - buf->tmp_buf_size - buf->tmp_buf_offset, - obj); + buf->tmp_buf_size - buf->tmp_buf_offset, + obj, config->json_escape_unicode); if (ret <= 0) { /* * negative value means failure to write to buffer, @@ -431,7 +432,8 @@ static int send_log_events(struct flb_firehose *ctx, struct flush *buf) { * Processes the msgpack object, sends the current batch if needed */ static int add_event(struct flb_firehose *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { int ret; struct firehose_event *event; @@ -445,7 +447,7 @@ static int add_event(struct flb_firehose *ctx, struct flush *buf, retry_add_event: retry_add = FLB_FALSE; - ret = process_event(ctx, buf, obj, tms); + ret = process_event(ctx, buf, obj, tms, config); if (ret < 0) { return -1; } @@ -510,7 +512,8 @@ static int add_event(struct flb_firehose *ctx, struct flush *buf, * return value is the number of events processed (number sent is stored in buf) */ int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, - const char *data, size_t bytes) + const char *data, size_t bytes, + struct flb_config *config) { // size_t off = 0; int i = 0; @@ -573,7 +576,7 @@ int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, if (strncmp(ctx->log_key, key_str, key_str_size) == 0) { found = FLB_TRUE; val = (kv+j)->val; - ret = add_event(ctx, buf, &val, &log_event.timestamp); + ret = add_event(ctx, buf, &val, &log_event.timestamp, config); if (ret < 0 ) { goto error; } @@ -591,7 +594,7 @@ int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, continue; } - ret = add_event(ctx, buf, &map, &log_event.timestamp); + ret = add_event(ctx, buf, &map, &log_event.timestamp, config); if (ret < 0 ) { goto error; } diff --git a/plugins/out_kinesis_firehose/firehose_api.h b/plugins/out_kinesis_firehose/firehose_api.h index 9954d2c9eb3..fde963a563d 100644 --- a/plugins/out_kinesis_firehose/firehose_api.h +++ b/plugins/out_kinesis_firehose/firehose_api.h @@ -37,7 +37,8 @@ void flush_destroy(struct flush *buf); int process_and_send_records(struct flb_firehose *ctx, struct flush *buf, - const char *data, size_t bytes); + const char *data, size_t bytes, + struct flb_config *config); int put_record_batch(struct flb_firehose *ctx, struct flush *buf, size_t payload_size, int num_records); From 3d1029dec6a38dfb4ce63b90dc9d7878d43be365 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:46:14 +0900 Subject: [PATCH 25/64] out_kinesis_streams: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_kinesis_streams/kinesis.c | 3 ++- plugins/out_kinesis_streams/kinesis_api.c | 19 +++++++++++-------- plugins/out_kinesis_streams/kinesis_api.h | 3 ++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/plugins/out_kinesis_streams/kinesis.c b/plugins/out_kinesis_streams/kinesis.c index 9a0e5d982f3..03556752340 100644 --- a/plugins/out_kinesis_streams/kinesis.c +++ b/plugins/out_kinesis_streams/kinesis.c @@ -361,7 +361,8 @@ static void cb_kinesis_flush(struct flb_event_chunk *event_chunk, ret = process_and_send_to_kinesis(ctx, buf, event_chunk->data, - event_chunk->size); + event_chunk->size, + config); if (ret < 0) { flb_plg_error(ctx->ins, "Failed to send records to kinesis"); kinesis_flush_destroy(buf); diff --git a/plugins/out_kinesis_streams/kinesis_api.c b/plugins/out_kinesis_streams/kinesis_api.c index ffd1a82e927..3f6d0f939d8 100644 --- a/plugins/out_kinesis_streams/kinesis_api.c +++ b/plugins/out_kinesis_streams/kinesis_api.c @@ -213,7 +213,8 @@ static int end_put_payload(struct flb_kinesis *ctx, struct flush *buf, * which means a send must occur */ static int process_event(struct flb_kinesis *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { size_t written = 0; int ret; @@ -230,8 +231,8 @@ static int process_event(struct flb_kinesis *ctx, struct flush *buf, tmp_buf_ptr = buf->tmp_buf + buf->tmp_buf_offset; ret = flb_msgpack_to_json(tmp_buf_ptr, - buf->tmp_buf_size - buf->tmp_buf_offset, - obj); + buf->tmp_buf_size - buf->tmp_buf_offset, + obj, config->json_escape_unicode); if (ret <= 0) { /* * negative value means failure to write to buffer, @@ -466,7 +467,8 @@ static int send_log_events(struct flb_kinesis *ctx, struct flush *buf) { * Processes the msgpack object, sends the current batch if needed */ static int add_event(struct flb_kinesis *ctx, struct flush *buf, - const msgpack_object *obj, struct flb_time *tms) + const msgpack_object *obj, struct flb_time *tms, + struct flb_config *config) { int ret; struct kinesis_event *event; @@ -480,7 +482,7 @@ static int add_event(struct flb_kinesis *ctx, struct flush *buf, retry_add_event: retry_add = FLB_FALSE; - ret = process_event(ctx, buf, obj, tms); + ret = process_event(ctx, buf, obj, tms, config); if (ret < 0) { return -1; } @@ -545,7 +547,8 @@ static int add_event(struct flb_kinesis *ctx, struct flush *buf, * return value is the number of events processed (number sent is stored in buf) */ int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, - const char *data, size_t bytes) + const char *data, size_t bytes, + struct flb_config *config) { int i = 0; size_t map_size; @@ -602,7 +605,7 @@ int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, if (strncmp(ctx->log_key, key_str, key_str_size) == 0) { found = FLB_TRUE; val = (kv+j)->val; - ret = add_event(ctx, buf, &val, &log_event.timestamp); + ret = add_event(ctx, buf, &val, &log_event.timestamp, config); if (ret < 0 ) { goto error; } @@ -620,7 +623,7 @@ int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, continue; } - ret = add_event(ctx, buf, &map, &log_event.timestamp); + ret = add_event(ctx, buf, &map, &log_event.timestamp, config); if (ret < 0 ) { goto error; } diff --git a/plugins/out_kinesis_streams/kinesis_api.h b/plugins/out_kinesis_streams/kinesis_api.h index 2ff8e08e9a4..bbc30efa96a 100644 --- a/plugins/out_kinesis_streams/kinesis_api.h +++ b/plugins/out_kinesis_streams/kinesis_api.h @@ -36,7 +36,8 @@ void kinesis_flush_destroy(struct flush *buf); int process_and_send_to_kinesis(struct flb_kinesis *ctx, struct flush *buf, - const char *data, size_t bytes); + const char *data, size_t bytes, + struct flb_config *config); int put_records(struct flb_kinesis *ctx, struct flush *buf, size_t payload_size, int num_records); From 493453712708b30ced5f5b8015a85b640b98df6e Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:47:01 +0900 Subject: [PATCH 26/64] out_lib: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_lib/out_lib.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/out_lib/out_lib.c b/plugins/out_lib/out_lib.c index a654fba31e3..6cac599184e 100644 --- a/plugins/out_lib/out_lib.c +++ b/plugins/out_lib/out_lib.c @@ -167,7 +167,8 @@ static void out_lib_flush(struct flb_event_chunk *event_chunk, #ifdef FLB_HAVE_METRICS if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { alloc_size = (off - last_off) + 4096; - buf = flb_msgpack_to_json_str(alloc_size, &result.data); + buf = flb_msgpack_to_json_str(alloc_size, &result.data, + config->json_escape_unicode); if (buf == NULL) { msgpack_unpacked_destroy(&result); FLB_OUTPUT_RETURN(FLB_ERROR); @@ -181,7 +182,8 @@ static void out_lib_flush(struct flb_event_chunk *event_chunk, alloc_size = (off - last_off) + 128; flb_time_pop_from_msgpack(&tm, &result, &obj); - buf = flb_msgpack_to_json_str(alloc_size, obj); + buf = flb_msgpack_to_json_str(alloc_size, obj, + config->json_escape_unicode); if (!buf) { msgpack_unpacked_destroy(&result); FLB_OUTPUT_RETURN(FLB_ERROR); From 0e11002d29171a4ccc146ccee02c168034c9f3b9 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:47:29 +0900 Subject: [PATCH 27/64] out_logdna: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_logdna/logdna.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/out_logdna/logdna.c b/plugins/out_logdna/logdna.c index bc834c6017d..f2b987e55ce 100644 --- a/plugins/out_logdna/logdna.c +++ b/plugins/out_logdna/logdna.c @@ -131,7 +131,8 @@ static int record_append_primary_keys(struct flb_logdna *ctx, static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, const void *data, size_t bytes, - const char *tag, int tag_len) + const char *tag, int tag_len, + struct flb_config *config) { int ret; int len; @@ -192,7 +193,7 @@ static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, msgpack_pack_str(&mp_pck, 4); msgpack_pack_str_body(&mp_pck, "line", 4); - line_json = flb_msgpack_to_json_str(1024, log_event.body); + line_json = flb_msgpack_to_json_str(1024, log_event.body, config->json_escape_unicode); len = strlen(line_json); msgpack_pack_str(&mp_pck, len); msgpack_pack_str_body(&mp_pck, line_json, len); @@ -204,7 +205,8 @@ static flb_sds_t logdna_compose_payload(struct flb_logdna *ctx, flb_log_event_decoder_destroy(&log_decoder); - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); return json; @@ -382,7 +384,8 @@ static void cb_logdna_flush(struct flb_event_chunk *event_chunk, event_chunk->data, event_chunk->size, event_chunk->tag, - flb_sds_len(event_chunk->tag)); + flb_sds_len(event_chunk->tag), + config); if (!payload) { flb_plg_error(ctx->ins, "cannot compose request payload"); FLB_OUTPUT_RETURN(FLB_RETRY); From 224e3448696b1bf631f34289338f892d25ee8a3c Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:48:07 +0900 Subject: [PATCH 28/64] out_loki: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_loki/loki.c | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/plugins/out_loki/loki.c b/plugins/out_loki/loki.c index ee7a4313a89..bdec7fd08e8 100644 --- a/plugins/out_loki/loki.c +++ b/plugins/out_loki/loki.c @@ -477,7 +477,8 @@ static void pack_maps(struct flb_loki *ctx, char *tag, int tag_len, msgpack_object *map, struct flb_mp_map_header *mh, - struct mk_list *list) + struct mk_list *list, + struct flb_config *config) { struct mk_list *head; struct flb_loki_kv *kv; @@ -539,7 +540,8 @@ static void pack_maps(struct flb_loki *ctx, */ else { accessed_map_val_json = flb_msgpack_to_json_str(1024, - &accessed_map_kv.val); + &accessed_map_kv.val, + config->json_escape_unicode); if (accessed_map_val_json) { msgpack_pack_str_with_body(mp_pck, accessed_map_val_json, strlen(accessed_map_val_json)); @@ -556,14 +558,16 @@ static void pack_maps(struct flb_loki *ctx, static flb_sds_t pack_structured_metadata(struct flb_loki *ctx, msgpack_packer *mp_pck, char *tag, int tag_len, - msgpack_object *map) + msgpack_object *map, + struct flb_config *config) { struct flb_mp_map_header mh; /* Initialize dynamic map header */ flb_mp_map_header_init(&mh, mp_pck); if (ctx->structured_metadata_map_keys) { pack_maps(ctx, mp_pck, tag, tag_len, map, &mh, - &ctx->structured_metadata_map_keys_list); + &ctx->structured_metadata_map_keys_list, + config); } /* * explicit structured_metadata entries override @@ -1415,7 +1419,8 @@ static int get_tenant_id_from_record(struct flb_loki *ctx, msgpack_object *map, static int pack_record(struct flb_loki *ctx, msgpack_packer *mp_pck, msgpack_object *rec, flb_sds_t *dynamic_tenant_id, - struct flb_mp_accessor *remove_mpa) + struct flb_mp_accessor *remove_mpa, + struct flb_config *config) { int i; int skip = 0; @@ -1506,7 +1511,7 @@ static int pack_record(struct flb_loki *ctx, } if (ctx->out_line_format == FLB_LOKI_FMT_JSON) { - line = flb_msgpack_to_json_str(size_hint, rec); + line = flb_msgpack_to_json_str(size_hint, rec, config->json_escape_unicode); if (!line) { if (tmp_sbuf_data) { flb_free(tmp_sbuf_data); @@ -1635,7 +1640,8 @@ static flb_sds_t loki_compose_payload(struct flb_loki *ctx, char *tag, int tag_len, const void *data, size_t bytes, flb_sds_t *dynamic_tenant_id, - struct flb_mp_accessor *remove_mpa) + struct flb_mp_accessor *remove_mpa, + struct flb_config *config) { // int mp_ok = MSGPACK_UNPACK_SUCCESS; // size_t off = 0; @@ -1726,9 +1732,9 @@ static flb_sds_t loki_compose_payload(struct flb_loki *ctx, /* Append the timestamp */ pack_timestamp(&mp_pck, &log_event.timestamp); - pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id, remove_mpa); + pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id, remove_mpa, config); if (ctx->structured_metadata || ctx->structured_metadata_map_keys) { - pack_structured_metadata(ctx, &mp_pck, tag, tag_len, NULL); + pack_structured_metadata(ctx, &mp_pck, tag, tag_len, NULL, config); } } } @@ -1763,16 +1769,17 @@ static flb_sds_t loki_compose_payload(struct flb_loki *ctx, /* Append the timestamp */ pack_timestamp(&mp_pck, &log_event.timestamp); - pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id, remove_mpa); + pack_record(ctx, &mp_pck, log_event.body, dynamic_tenant_id, remove_mpa, config); if (ctx->structured_metadata || ctx->structured_metadata_map_keys) { - pack_structured_metadata(ctx, &mp_pck, tag, tag_len, log_event.body); + pack_structured_metadata(ctx, &mp_pck, tag, tag_len, log_event.body, config); } } } flb_log_event_decoder_destroy(&log_decoder); - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); @@ -1856,7 +1863,8 @@ static void cb_loki_flush(struct flb_event_chunk *event_chunk, flb_sds_len(event_chunk->tag), event_chunk->data, event_chunk->size, &dynamic_tenant_id->value, - remove_mpa_entry->mpa); + remove_mpa_entry->mpa, + config); if (!payload) { flb_plg_error(ctx->ins, "cannot compose request payload"); @@ -2128,7 +2136,7 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_loki, structured_metadata), "optional structured metadata fields for API requests." }, - + { FLB_CONFIG_MAP_CLIST, "structured_metadata_map_keys", NULL, 0, FLB_TRUE, offsetof(struct flb_loki, structured_metadata_map_keys), @@ -2241,7 +2249,8 @@ static int cb_loki_format_test(struct flb_config *config, payload = loki_compose_payload(ctx, total_records, (char *) tag, tag_len, data, bytes, &dynamic_tenant_id, - ctx->remove_mpa); + ctx->remove_mpa, + config); if (payload == NULL) { if (dynamic_tenant_id != NULL) { flb_sds_destroy(dynamic_tenant_id); From 1c5cb40c761d94c81fce2abdffe6f17f82d50596 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:48:48 +0900 Subject: [PATCH 29/64] out_nats: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_nats/nats.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/out_nats/nats.c b/plugins/out_nats/nats.c index d2ed1b3caed..e74f1ca7c89 100644 --- a/plugins/out_nats/nats.c +++ b/plugins/out_nats/nats.c @@ -81,7 +81,8 @@ static int cb_nats_init(struct flb_output_instance *ins, struct flb_config *conf static int msgpack_to_json(struct flb_out_nats_config *ctx, const void *data, size_t bytes, const char *tag, int tag_len, - char **out_json, size_t *out_size) + char **out_json, size_t *out_size, + struct flb_config *config) { int i; int map_size; @@ -138,7 +139,7 @@ static int msgpack_to_json(struct flb_out_nats_config *ctx, flb_log_event_decoder_destroy(&log_decoder); - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { @@ -186,7 +187,7 @@ static void cb_nats_flush(struct flb_event_chunk *event_chunk, ret = msgpack_to_json(ctx, event_chunk->data, event_chunk->size, event_chunk->tag, flb_sds_len(event_chunk->tag), - &json_msg, &json_len); + &json_msg, &json_len, config); if (ret == -1) { flb_upstream_conn_release(u_conn); FLB_OUTPUT_RETURN(FLB_ERROR); From 570d3c564137e5c7e813955611fb322219245042 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:49:16 +0900 Subject: [PATCH 30/64] out_nrlogs: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_nrlogs/newrelic.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/out_nrlogs/newrelic.c b/plugins/out_nrlogs/newrelic.c index f50b3e048ad..e23ca90c761 100644 --- a/plugins/out_nrlogs/newrelic.c +++ b/plugins/out_nrlogs/newrelic.c @@ -142,7 +142,8 @@ static int package_record(struct flb_time *ts, msgpack_object *map, } static flb_sds_t newrelic_compose_payload(struct flb_newrelic *ctx, - const void *data, size_t bytes) + const void *data, size_t bytes, + struct flb_config *config) { int total_records; flb_sds_t json; @@ -232,7 +233,8 @@ static flb_sds_t newrelic_compose_payload(struct flb_newrelic *ctx, flb_log_event_decoder_destroy(&log_decoder); - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); @@ -368,7 +370,8 @@ static void cb_newrelic_flush(struct flb_event_chunk *event_chunk, /* Format the data to the expected Newrelic Payload */ payload = newrelic_compose_payload(ctx, - event_chunk->data, event_chunk->size); + event_chunk->data, event_chunk->size, + config); if (!payload) { flb_plg_error(ctx->ins, "cannot compose request payload"); FLB_OUTPUT_RETURN(FLB_RETRY); From 53a75b1554d59627db317e4e4756874c1f5ea983 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 17:59:27 +0900 Subject: [PATCH 31/64] out_null: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_null/null.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_null/null.c b/plugins/out_null/null.c index bf29645de98..b637bcfd9b6 100644 --- a/plugins/out_null/null.c +++ b/plugins/out_null/null.c @@ -123,7 +123,8 @@ static void cb_null_flush(struct flb_event_chunk *event_chunk, event_chunk->size, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); flb_sds_destroy(json); } From 5e9c7f6ba6debe5e3f8b2b6228304ee2e0fca35f Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:01:03 +0900 Subject: [PATCH 32/64] out_opensearch: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_opensearch/opensearch.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/out_opensearch/opensearch.c b/plugins/out_opensearch/opensearch.c index 856b00fa96e..3f52a7abd1f 100644 --- a/plugins/out_opensearch/opensearch.c +++ b/plugins/out_opensearch/opensearch.c @@ -587,7 +587,8 @@ static int opensearch_format(struct flb_config *config, } /* Convert msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&tmp_sbuf); if (!out_buf) { flb_log_event_decoder_destroy(&log_decoder); @@ -886,7 +887,8 @@ static void cb_opensearch_flush(struct flb_event_chunk *event_chunk, /* Convert format */ if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { - pack = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size); + pack = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size, + config->json_escape_unicode); if (pack) { ret = 0; From b2e8c67d820a071fcfda106a99bbe109558cc158 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:02:02 +0900 Subject: [PATCH 33/64] out_oracle_log_ayalysis: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_oracle_log_analytics/oci_logan.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_oracle_log_analytics/oci_logan.c b/plugins/out_oracle_log_analytics/oci_logan.c index 19e4529ddf3..42ff71938aa 100644 --- a/plugins/out_oracle_log_analytics/oci_logan.c +++ b/plugins/out_oracle_log_analytics/oci_logan.c @@ -1161,7 +1161,8 @@ static int total_flush(struct flb_event_chunk *event_chunk, goto clean_up; } - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); flb_log_event_decoder_destroy(&log_decoder); From 75789d0358a9b84c20571428c450fc8d4848d6ca Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:02:40 +0900 Subject: [PATCH 34/64] out_s3: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_s3/s3.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/out_s3/s3.c b/plugins/out_s3/s3.c index 3eedb9334c0..f9705b59da7 100644 --- a/plugins/out_s3/s3.c +++ b/plugins/out_s3/s3.c @@ -3347,7 +3347,7 @@ static void cb_s3_upload(struct flb_config *config, void *data) } static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, const char *data, - uint64_t bytes) + uint64_t bytes, struct flb_config *config) { int i; int records = 0; @@ -3454,7 +3454,8 @@ static flb_sds_t flb_pack_msgpack_extract_log_key(void *out_context, const char } else { ret = flb_msgpack_to_json(val_buf + val_offset, - msgpack_size - val_offset, &val); + msgpack_size - val_offset, &val, + config->json_escape_unicode); if (ret < 0) { break; } @@ -3754,14 +3755,16 @@ static void cb_s3_flush(struct flb_event_chunk *event_chunk, if (ctx->log_key) { chunk = flb_pack_msgpack_extract_log_key(ctx, event_chunk->data, - event_chunk->size); + event_chunk->size, + config); } else { chunk = flb_pack_msgpack_to_json_format(event_chunk->data, event_chunk->size, FLB_PACK_JSON_FORMAT_LINES, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); } if (chunk == NULL) { flb_plg_error(ctx->ins, "Could not marshal msgpack to output string"); From d5b49fb8a7eb1b22fb74aeb663bd85962fc6cd76 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:03:19 +0900 Subject: [PATCH 35/64] out_skywalking: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_skywalking/skywalking.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/plugins/out_skywalking/skywalking.c b/plugins/out_skywalking/skywalking.c index dd1f8fa0539..3598bd5ff09 100644 --- a/plugins/out_skywalking/skywalking.c +++ b/plugins/out_skywalking/skywalking.c @@ -152,7 +152,8 @@ static void sw_msgpack_pack_kv_int64_t(msgpack_packer* pk, const char* key, } static void sw_msgpack_pack_log_body(msgpack_packer* pk, - msgpack_object* obj, size_t obj_size) + msgpack_object* obj, size_t obj_size, + struct flb_config *config) { int i, j = 0; int log_entry_num = 0; @@ -196,7 +197,8 @@ static void sw_msgpack_pack_log_body(msgpack_packer* pk, value.via.str.ptr, value.via.str.size); } - out_body_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size); + out_body_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size, + config->json_escape_unicode); if (!out_body_str) { msgpack_sbuffer_destroy(&sbuf); flb_free(valid_log_entry); @@ -225,7 +227,7 @@ static void sw_msgpack_pack_log_body(msgpack_packer* pk, } static int sw_format(struct flb_output_sw* ctx, const void *data, size_t bytes, - void** buf, size_t* buf_len) + void** buf, size_t* buf_len, struct flb_config *config) { int ret = 0; int chunk_size = 0; @@ -270,10 +272,10 @@ static int sw_format(struct flb_output_sw* ctx, const void *data, size_t bytes, flb_sds_len(ctx->svc_name)); sw_msgpack_pack_kv_str(&pk, "serviceInstance", 15, ctx->svc_inst_name, flb_sds_len(ctx->svc_inst_name)); - sw_msgpack_pack_log_body(&pk, &map, map_size); + sw_msgpack_pack_log_body(&pk, &map, map_size, config); } - out_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size); + out_str = flb_msgpack_raw_to_json_sds(sbuf.data, sbuf.size, config->json_escape_unicode); if (!out_str) { ret = -1; goto done; @@ -323,7 +325,7 @@ static void cb_sw_flush(struct flb_event_chunk *event_chunk, tmp_ret = sw_format(ctx, event_chunk->data, event_chunk->size, - &buf, &buf_len); + &buf, &buf_len, config); if (tmp_ret != 0) { flb_plg_error(ctx->ins, "failed to create buffer"); FLB_OUTPUT_RETURN(FLB_RETRY); From 38c39fc8b3bcbbff4aefa18add13652d9cae9306 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:03:53 +0900 Subject: [PATCH 36/64] out_slack: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_slack/slack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/out_slack/slack.c b/plugins/out_slack/slack.c index 969644c2122..286fe078d85 100644 --- a/plugins/out_slack/slack.c +++ b/plugins/out_slack/slack.c @@ -230,7 +230,7 @@ static void cb_slack_flush(struct flb_event_chunk *event_chunk, flb_sds_destroy(json); /* Re-format mspgack as JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, config->json_escape_unicode); if (!out_buf) { msgpack_sbuffer_destroy(&mp_sbuf); FLB_OUTPUT_RETURN(FLB_RETRY); From b5a69f153c047b1f8d0aeff66e0c93581b6dbf17 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:05:08 +0900 Subject: [PATCH 37/64] out_splunk: Refer a config whether escaping or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_splunk/splunk.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/plugins/out_splunk/splunk.c b/plugins/out_splunk/splunk.c index 970eaa5d8d1..c2531d7b240 100644 --- a/plugins/out_splunk/splunk.c +++ b/plugins/out_splunk/splunk.c @@ -637,7 +637,7 @@ static flb_sds_t get_metadata_auth_header(struct flb_splunk *ctx) static inline int splunk_format(const void *in_buf, size_t in_bytes, char *tag, int tag_len, char **out_buf, size_t *out_size, - struct flb_splunk *ctx) + struct flb_splunk *ctx, struct flb_config *config) { int ret; char *err; @@ -726,7 +726,7 @@ static inline int splunk_format(const void *in_buf, size_t in_bytes, /* Validate packaging */ if (ret != 0) { /* Format invalid record */ - err = flb_msgpack_to_json_str(2048, &map); + err = flb_msgpack_to_json_str(2048, &map, config->json_escape_unicode); if (err) { /* Print error and continue processing other records */ flb_plg_warn(ctx->ins, "could not process or pack record: %s", err); @@ -737,7 +737,8 @@ static inline int splunk_format(const void *in_buf, size_t in_bytes, } /* Format as JSON */ - record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + record = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); if (!record) { flb_errno(); msgpack_sbuffer_destroy(&mp_sbuf); @@ -898,7 +899,8 @@ static void cb_splunk_flush(struct flb_event_chunk *event_chunk, event_chunk->size, (char *) event_chunk->tag, flb_sds_len(event_chunk->tag), - &buf_data, &buf_size, ctx); + &buf_data, &buf_size, ctx, + config); } if (ret == -1) { @@ -1183,7 +1185,7 @@ static int cb_splunk_format_test(struct flb_config *config, struct flb_splunk *ctx = plugin_context; return splunk_format(data, bytes, (char *) tag, tag_len, - (char**) out_data, out_size,ctx); + (char**) out_data, out_size, ctx, config); } struct flb_output_plugin out_splunk_plugin = { From b45269152b6a2a7b10a32982a33c04bb643b2039 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:05:46 +0900 Subject: [PATCH 38/64] out_stackdriver: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_stackdriver/stackdriver.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/out_stackdriver/stackdriver.c b/plugins/out_stackdriver/stackdriver.c index 4e1395f2d36..a322e8eca39 100644 --- a/plugins/out_stackdriver/stackdriver.c +++ b/plugins/out_stackdriver/stackdriver.c @@ -1697,7 +1697,8 @@ static int pack_payload(int insert_id_extracted, static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, int total_records, const char *tag, int tag_len, - const void *data, size_t bytes) + const void *data, size_t bytes, + struct flb_config *config) { int len; int ret; @@ -2576,7 +2577,8 @@ static flb_sds_t stackdriver_format(struct flb_stackdriver *ctx, flb_log_event_decoder_destroy(&log_decoder); /* Convert from msgpack to JSON */ - out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); msgpack_sbuffer_destroy(&mp_sbuf); if (!out_buf) { @@ -2604,7 +2606,7 @@ static int stackdriver_format_test(struct flb_config *config, total_records = flb_mp_count(data, bytes); payload = stackdriver_format(ctx, total_records, - (char *) tag, tag_len, data, bytes); + (char *) tag, tag_len, data, bytes, config); if (payload == NULL) { return -1; } @@ -2877,7 +2879,8 @@ static void cb_stackdriver_flush(struct flb_event_chunk *event_chunk, payload_buf = stackdriver_format(ctx, event_chunk->total_events, event_chunk->tag, flb_sds_len(event_chunk->tag), - event_chunk->data, event_chunk->size); + event_chunk->data, event_chunk->size, + config); if (!payload_buf) { #ifdef FLB_HAVE_METRICS cmt_counter_inc(ctx->cmt_failed_requests, From 147c7a2a8a6c61fb8f26bb37485fe17bc35faa46 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:06:36 +0900 Subject: [PATCH 39/64] out_stdout: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_stdout/stdout.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_stdout/stdout.c b/plugins/out_stdout/stdout.c index f73475476f1..11dc2c6d5a5 100644 --- a/plugins/out_stdout/stdout.c +++ b/plugins/out_stdout/stdout.c @@ -257,7 +257,8 @@ static void cb_stdout_flush(struct flb_event_chunk *event_chunk, event_chunk->size, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); write(STDOUT_FILENO, json, flb_sds_len(json)); flb_sds_destroy(json); From 91ba6e1786f64ad3cd3665f67da1b73ad09e45e1 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:07:35 +0900 Subject: [PATCH 40/64] out_tcp: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_tcp/tcp.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/plugins/out_tcp/tcp.c b/plugins/out_tcp/tcp.c index 2ea38aa0c26..7e90d025e1e 100644 --- a/plugins/out_tcp/tcp.c +++ b/plugins/out_tcp/tcp.c @@ -55,7 +55,8 @@ static int cb_tcp_init(struct flb_output_instance *ins, static int compose_payload(struct flb_out_tcp *ctx, const char *tag, int tag_len, const void *in_data, size_t in_size, - void **out_payload, size_t *out_size) + void **out_payload, size_t *out_size, + struct flb_config *config) { int ret; flb_sds_t buf = NULL; @@ -126,7 +127,8 @@ static int compose_payload(struct flb_out_tcp *ctx, in_size, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); if (!json) { flb_plg_error(ctx->ins, "error formatting JSON payload"); return FLB_ERROR; @@ -164,7 +166,8 @@ static void cb_tcp_flush(struct flb_event_chunk *event_chunk, ret = compose_payload(ctx, event_chunk->tag, flb_sds_len(event_chunk->tag), event_chunk->data, event_chunk->size, - &out_payload, &out_size); + &out_payload, &out_size, + config); if (ret != FLB_OK) { flb_upstream_conn_release(u_conn); return FLB_OUTPUT_RETURN(ret); @@ -244,7 +247,7 @@ static int cb_tcp_format_test(struct flb_config *config, struct flb_out_tcp *ctx = plugin_context; int ret; - ret = compose_payload(ctx, tag, tag_len, data, bytes, out_data, out_size); + ret = compose_payload(ctx, tag, tag_len, data, bytes, out_data, out_size, config); if (ret != FLB_OK) { flb_error("ret=%d", ret); return -1; From ea459c810a0e6f2912f875afa4a5b6dca540559f Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:08:01 +0900 Subject: [PATCH 41/64] out_udp: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_udp/udp.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/plugins/out_udp/udp.c b/plugins/out_udp/udp.c index 4ff63122622..2e854100434 100644 --- a/plugins/out_udp/udp.c +++ b/plugins/out_udp/udp.c @@ -127,7 +127,8 @@ static int deliver_chunks_raw(struct flb_out_udp *ctx, static int deliver_chunks_json(struct flb_out_udp *ctx, const char *tag, int tag_len, - const void *in_data, size_t in_size) + const void *in_data, size_t in_size, + struct flb_config *config) { int ret; size_t off = 0; @@ -158,7 +159,8 @@ static int deliver_chunks_json(struct flb_out_udp *ctx, off - previous_offset, ctx->out_format, ctx->json_date_format, - ctx->date_key); + ctx->date_key, + config->json_escape_unicode); if (!json) { flb_plg_error(ctx->ins, "error formatting JSON payload"); @@ -291,7 +293,8 @@ static void cb_udp_flush(struct flb_event_chunk *event_chunk, event_chunk->tag, flb_sds_len(event_chunk->tag), event_chunk->data, - event_chunk->size); + event_chunk->size, + config); } return FLB_OUTPUT_RETURN(ret); From 44a0d29e6822802f6ec81f862101cb631ba365a8 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:08:44 +0900 Subject: [PATCH 42/64] out_vivo: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_vivo_exporter/vivo.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/plugins/out_vivo_exporter/vivo.c b/plugins/out_vivo_exporter/vivo.c index c46b9ef1d9c..9a6289bc42d 100644 --- a/plugins/out_vivo_exporter/vivo.c +++ b/plugins/out_vivo_exporter/vivo.c @@ -27,7 +27,7 @@ #include "vivo_http.h" #include "vivo_stream.h" -static flb_sds_t format_logs(struct flb_event_chunk *event_chunk) +static flb_sds_t format_logs(struct flb_event_chunk *event_chunk, struct flb_config *config) { struct flb_log_event_decoder log_decoder; struct flb_log_event log_event; @@ -102,7 +102,8 @@ static flb_sds_t format_logs(struct flb_event_chunk *event_chunk) } /* Concatenate by using break lines */ - out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size); + out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, + config->json_escape_unicode); if (!out_js) { flb_sds_destroy(out_buf); msgpack_sbuffer_destroy(&tmp_sbuf); @@ -130,14 +131,15 @@ static flb_sds_t format_logs(struct flb_event_chunk *event_chunk) } static int logs_event_chunk_append(struct vivo_exporter *ctx, - struct flb_event_chunk *event_chunk) + struct flb_event_chunk *event_chunk, + struct flb_config *config) { size_t len; flb_sds_t json; struct vivo_stream_entry *entry; - json = format_logs(event_chunk); + json = format_logs(event_chunk, config); if (!json) { flb_plg_error(ctx->ins, "cannot convert logs chunk to JSON"); return -1; @@ -159,14 +161,16 @@ static int logs_event_chunk_append(struct vivo_exporter *ctx, static int metrics_traces_event_chunk_append(struct vivo_exporter *ctx, struct vivo_stream *vs, - struct flb_event_chunk *event_chunk) + struct flb_event_chunk *event_chunk, + struct flb_config *config) { size_t len; flb_sds_t json; struct vivo_stream_entry *entry; /* Convert msgpack to readable JSON format */ - json = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size); + json = flb_msgpack_raw_to_json_sds(event_chunk->data, event_chunk->size, + config->json_escape_unicode); if (!json) { flb_plg_error(ctx->ins, "cannot convert metrics chunk to JSON"); return -1; @@ -264,14 +268,14 @@ static void cb_vivo_flush(struct flb_event_chunk *event_chunk, #ifdef FLB_HAVE_METRICS if (event_chunk->type == FLB_EVENT_TYPE_METRICS) { - ret = metrics_traces_event_chunk_append(ctx, ctx->stream_metrics, event_chunk); + ret = metrics_traces_event_chunk_append(ctx, ctx->stream_metrics, event_chunk, config); } #endif if (event_chunk->type == FLB_EVENT_TYPE_LOGS) { - ret = logs_event_chunk_append(ctx, event_chunk); + ret = logs_event_chunk_append(ctx, event_chunk, config); } else if (event_chunk->type == FLB_EVENT_TYPE_TRACES) { - ret = metrics_traces_event_chunk_append(ctx, ctx->stream_traces, event_chunk); + ret = metrics_traces_event_chunk_append(ctx, ctx->stream_traces, event_chunk, config); } if (ret == 0) { From d8bc307b5a73e8421e50a574199f04acf85ef478 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:09:21 +0900 Subject: [PATCH 43/64] out_websocket: Refer a config whether escaping unicode or not in JSON Signed-off-by: Hiroshi Hatake --- plugins/out_websocket/websocket.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_websocket/websocket.c b/plugins/out_websocket/websocket.c index 52865e21a22..b3ffa784915 100644 --- a/plugins/out_websocket/websocket.c +++ b/plugins/out_websocket/websocket.c @@ -254,7 +254,8 @@ static void cb_ws_flush(struct flb_event_chunk *event_chunk, event_chunk->size, ctx->out_format, ctx->json_date_format, - ctx->json_date_key); + ctx->json_date_key, + config->json_escape_unicode); if (!json) { flb_error("[out_ws] error formatting JSON payload"); From f335009ef8e4328f81a25602bb9a5689c2f02c58 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:09:56 +0900 Subject: [PATCH 44/64] processor_content_modifier: Follow the argument increase Signed-off-by: Hiroshi Hatake --- plugins/processor_content_modifier/cm_utils.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/processor_content_modifier/cm_utils.c b/plugins/processor_content_modifier/cm_utils.c index e8d3f3e47a3..da68eb98a69 100644 --- a/plugins/processor_content_modifier/cm_utils.c +++ b/plugins/processor_content_modifier/cm_utils.c @@ -150,7 +150,8 @@ cfl_sds_t cm_utils_variant_convert_to_json(struct cfl_variant *value) mpack_writer_destroy(&writer); - json_result = flb_msgpack_raw_to_json_sds(data, size); + /* Using JSON escape here to keep backward compatibility */ + json_result = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); MPACK_FREE(data); return json_result; From 5bd0af25dacd15973afcd2a94024ad0e14a6b376 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:10:50 +0900 Subject: [PATCH 45/64] tests: log_event_decoder: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/internal/log_event_decoder.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/internal/log_event_decoder.c b/tests/internal/log_event_decoder.c index 6f93edcfb77..cc1509c7f57 100644 --- a/tests/internal/log_event_decoder.c +++ b/tests/internal/log_event_decoder.c @@ -180,7 +180,7 @@ void decode_object() return; } - json = flb_msgpack_to_json_str(4096, event.body); + json = flb_msgpack_to_json_str(4096, event.body, FLB_TRUE); if (!TEST_CHECK(json != NULL)) { TEST_MSG("flb_msgpack_to_json_str error"); return; @@ -250,7 +250,7 @@ void decoder_next() return; } - json = flb_msgpack_to_json_str(4096, event.body); + json = flb_msgpack_to_json_str(4096, event.body, FLB_TRUE); if (!TEST_CHECK(json != NULL)) { TEST_MSG("flb_msgpack_to_json_str error"); return; From cd402eeee6ca5a69f28c75a0a67aa7964c708e9e Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:11:27 +0900 Subject: [PATCH 46/64] tests: mp: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/internal/mp.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/internal/mp.c b/tests/internal/mp.c index 627d063d72e..7cdf6a3add8 100644 --- a/tests/internal/mp.c +++ b/tests/internal/mp.c @@ -218,7 +218,7 @@ void test_keys_remove_subkey_key() off = 0; msgpack_unpacked_init(&result_final); msgpack_unpack_next(&result_final, out_buf, out_size, &off); - flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data); + flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data, FLB_TRUE); if (!TEST_CHECK(strstr(&final_json[0] ,"kubernetes") == NULL)) { TEST_MSG("kubernetes field should be removed"); @@ -310,7 +310,7 @@ void test_remove_sibling_subkeys() off = 0; msgpack_unpacked_init(&result_final); msgpack_unpack_next(&result_final, out_buf, out_size, &off); - flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data); + flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data, FLB_TRUE); if (!TEST_CHECK(strstr(&final_json[0] ,"pod_id") == NULL)) { TEST_MSG("pod_id field should be removed"); @@ -411,7 +411,7 @@ void remove_subkey_keys(char *list[], int list_size, int index_start) off = 0; msgpack_unpacked_init(&result_final); msgpack_unpack_next(&result_final, out_buf, out_size, &off); - flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data); + flb_msgpack_to_json(&final_json[0], sizeof(final_json), &result_final.data, FLB_TRUE); if (!TEST_CHECK(strstr(&final_json[0] ,"kubernetes") == NULL)) { TEST_MSG("kubernetes field should be removed"); @@ -519,8 +519,8 @@ void test_object_to_cfl_to_msgpack() * Convert buf (msgpack 1 buffer) to JSON, and compare the strings * generated by out_buf (msgpack 2 buffer). They must match. */ - buf1 = flb_msgpack_raw_to_json_sds(buf, size); - buf2 = flb_msgpack_raw_to_json_sds(out_buf, out_size); + buf1 = flb_msgpack_raw_to_json_sds(buf, size, FLB_TRUE); + buf2 = flb_msgpack_raw_to_json_sds(out_buf, out_size, FLB_TRUE); ret = strcmp(buf1, buf2); printf("\n>> Compare JSON buf1 v/s JSON buf2 (ret=%i):\n", ret); From 246945ad7bb0ab2fb20e5a5a4d2cec4cb91ed825 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:11:48 +0900 Subject: [PATCH 47/64] tests: msgpack_append_message: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/internal/msgpack_append_message.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/internal/msgpack_append_message.c b/tests/internal/msgpack_append_message.c index abb487e877c..fe957a98a00 100644 --- a/tests/internal/msgpack_append_message.c +++ b/tests/internal/msgpack_append_message.c @@ -53,7 +53,7 @@ static inline int process_pack(char *pack, size_t size) MSGPACK_OBJECT_STR); TEST_CHECK(ret == 0); - out_buf = flb_msgpack_raw_to_json_sds(appended_buffer, appended_size); + out_buf = flb_msgpack_raw_to_json_sds(appended_buffer, appended_size, FLB_TRUE); TEST_CHECK(out_buf != NULL); p = strstr(out_buf, "\"expanding\":\"injected\""); if (!TEST_CHECK(p != NULL)) { From f2beec06f1e2e093db1c8ab4453b489db272a2f4 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:12:22 +0900 Subject: [PATCH 48/64] tests: opnentelemetry: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/internal/opentelemetry.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index 5056d71eca9..ea31931f2da 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -187,8 +187,8 @@ static struct test_output *parse_test_output(void *chunk, size_t size) if (record_type == FLB_LOG_EVENT_GROUP_START) { /* Group header */ if (group_idx < output->group_count) { - output->groups[group_idx].metadata = flb_msgpack_to_json_str(1024, event.metadata); - output->groups[group_idx].body = flb_msgpack_to_json_str(1024, event.body); + output->groups[group_idx].metadata = flb_msgpack_to_json_str(1024, event.metadata, FLB_TRUE); + output->groups[group_idx].body = flb_msgpack_to_json_str(1024, event.body, FLB_TRUE); /* Allocate records for this group */ if (output->groups[group_idx].record_count > 0) { @@ -209,8 +209,8 @@ static struct test_output *parse_test_output(void *chunk, size_t size) /* Log record within a group */ if (group_idx < output->group_count && record_idx < output->groups[group_idx].record_count) { - output->groups[group_idx].records[record_idx].metadata = flb_msgpack_to_json_str(1024, event.metadata); - output->groups[group_idx].records[record_idx].body = flb_msgpack_to_json_str(1024, event.body); + output->groups[group_idx].records[record_idx].metadata = flb_msgpack_to_json_str(1024, event.metadata, FLB_TRUE); + output->groups[group_idx].records[record_idx].body = flb_msgpack_to_json_str(1024, event.body, FLB_TRUE); record_idx++; } } @@ -308,7 +308,7 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * /* Validate group metadata */ ret = flb_otel_utils_find_map_entry_by_key(&group_obj->via.map, "metadata", 0, FLB_TRUE); if (ret >= 0) { - expected_meta = flb_msgpack_to_json_str(256, &group_obj->via.map.ptr[ret].val); + expected_meta = flb_msgpack_to_json_str(256, &group_obj->via.map.ptr[ret].val, FLB_TRUE); if (strcmp(expected_meta, actual->groups[i].metadata) != 0) { printf("Group %zu metadata mismatch:\nExpected: %s\nGot: %s\n", i, expected_meta, actual->groups[i].metadata); @@ -321,7 +321,7 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * /* Validate group body */ ret = flb_otel_utils_find_map_entry_by_key(&group_obj->via.map, "body", 0, FLB_TRUE); if (ret >= 0) { - expected_body = flb_msgpack_to_json_str(256, &group_obj->via.map.ptr[ret].val); + expected_body = flb_msgpack_to_json_str(256, &group_obj->via.map.ptr[ret].val, FLB_TRUE); if (strcmp(expected_body, actual->groups[i].body) != 0) { printf("Group %zu body mismatch:\nExpected: %s\nGot: %s\n", i, expected_body, actual->groups[i].body); @@ -359,7 +359,7 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * /* Validate record metadata */ ret = flb_otel_utils_find_map_entry_by_key(&record_obj->via.map, "metadata", 0, FLB_TRUE); if (ret >= 0) { - expected_meta = flb_msgpack_to_json_str(256, &record_obj->via.map.ptr[ret].val); + expected_meta = flb_msgpack_to_json_str(256, &record_obj->via.map.ptr[ret].val, FLB_TRUE); if (strcmp(expected_meta, actual->groups[i].records[j].metadata) != 0) { printf("Group %zu record %zu metadata mismatch:\nExpected: %s\nGot: %s\n", i, j, expected_meta, actual->groups[i].records[j].metadata); @@ -372,7 +372,7 @@ static int validate_extended_output(struct test_output *actual, msgpack_object * /* Validate record body */ ret = flb_otel_utils_find_map_entry_by_key(&record_obj->via.map, "body", 0, FLB_TRUE); if (ret >= 0) { - expected_body = flb_msgpack_to_json_str(256, &record_obj->via.map.ptr[ret].val); + expected_body = flb_msgpack_to_json_str(256, &record_obj->via.map.ptr[ret].val, FLB_TRUE); if (strcmp(expected_body, actual->groups[i].records[j].body) != 0) { printf("Group %zu record %zu body mismatch:\nExpected: %s\nGot: %s\n", i, j, expected_body, actual->groups[i].records[j].body); @@ -568,7 +568,7 @@ void test_opentelemetry_cases() ret = flb_otel_utils_find_map_entry_by_key(&case_obj->via.map, "input", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - input_json = flb_msgpack_to_json_str(1024, &case_obj->via.map.ptr[ret].val); + input_json = flb_msgpack_to_json_str(1024, &case_obj->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(input_json != NULL); ret = flb_log_event_encoder_init(&enc, FLB_LOG_EVENT_FORMAT_FLUENT_BIT_V2); @@ -602,22 +602,22 @@ void test_opentelemetry_cases() if (empty_payload == FLB_FALSE && has_groups == FLB_FALSE) { ret = flb_otel_utils_find_map_entry_by_key(&expected->via.map, "group_metadata", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - expect_group_meta = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val); + expect_group_meta = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(expect_group_meta != NULL); ret = flb_otel_utils_find_map_entry_by_key(&expected->via.map, "group_body", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - expect_group_body = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val); + expect_group_body = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(expect_group_body != NULL); ret = flb_otel_utils_find_map_entry_by_key(&expected->via.map, "log_metadata", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - expect_log_meta = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val); + expect_log_meta = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(expect_log_meta != NULL); ret = flb_otel_utils_find_map_entry_by_key(&expected->via.map, "log_body", 0, FLB_TRUE); TEST_CHECK(ret >= 0); - expect_log_body = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val); + expect_log_body = flb_msgpack_to_json_str(256, &expected->via.map.ptr[ret].val, FLB_TRUE); TEST_CHECK(expect_log_body != NULL); } From ed29fdb8944ebd10f31b5c32a7dc5de992119c7a Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:12:47 +0900 Subject: [PATCH 49/64] tests: pack: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/internal/pack.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/internal/pack.c b/tests/internal/pack.c index e7a88367ba3..6424b3a3276 100644 --- a/tests/internal/pack.c +++ b/tests/internal/pack.c @@ -283,7 +283,7 @@ void test_json_dup_keys() out_json = flb_pack_msgpack_to_json_format(out_buf, out_size, FLB_PACK_JSON_FORMAT_LINES, FLB_PACK_JSON_DATE_EPOCH, - d); + d, FLB_TRUE); TEST_CHECK(out_json != NULL); TEST_CHECK(strncmp(out_json, data_out, flb_sds_len(out_json)) == 0); @@ -497,7 +497,7 @@ void test_utf8_to_json() json_size = strlen(file_json); - out_buf = flb_msgpack_raw_to_json_sds(file_msgp, msgp_size); + out_buf = flb_msgpack_raw_to_json_sds(file_msgp, msgp_size, FLB_TRUE); TEST_CHECK(out_buf != NULL); out_size = flb_sds_len(out_buf); @@ -718,7 +718,7 @@ void test_json_pack_bug1278() msgpack_pack_str_body(&mp_pck, p_in, len); /* Pack raw string as JSON */ - json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size); + json = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, FLB_TRUE); /* Compare expected JSON output */ ret = strcmp(p_out, json); @@ -799,7 +799,7 @@ void test_json_pack_nan() msgpack_sbuffer_destroy(&mp_sbuf); // convert msgpack to json - ret = flb_msgpack_to_json(&json_str[0], sizeof(json_str), &obj); + ret = flb_msgpack_to_json(&json_str[0], sizeof(json_str), &obj, FLB_TRUE); TEST_CHECK(ret >= 0); p = strstr(&json_str[0], "nan"); @@ -810,7 +810,7 @@ void test_json_pack_nan() // convert. nan -> null memset(&json_str[0], 0, sizeof(json_str)); flb_pack_init(&config); - ret = flb_msgpack_to_json(&json_str[0], sizeof(json_str), &obj); + ret = flb_msgpack_to_json(&json_str[0], sizeof(json_str), &obj, FLB_TRUE); TEST_CHECK(ret >= 0); p = strstr(&json_str[0], "null"); @@ -1025,7 +1025,7 @@ void test_json_date(char* expect, int date_format) ret = flb_pack_msgpack_to_json_format((const char*)&input_msgpack[0], sizeof(input_msgpack), FLB_PACK_JSON_FORMAT_JSON, date_format, - json_key); + json_key, FLB_TRUE); if (!TEST_CHECK(ret != NULL)) { TEST_MSG("flb_pack_msgpack_to_json_format failed"); flb_sds_destroy(json_key); From d700219d8bc66d60ebec7713ca467d4aa723ee02 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:13:10 +0900 Subject: [PATCH 50/64] tests: stream_processor: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/internal/stream_processor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/internal/stream_processor.c b/tests/internal/stream_processor.c index aa7bf69beda..4fcd02956f0 100644 --- a/tests/internal/stream_processor.c +++ b/tests/internal/stream_processor.c @@ -886,7 +886,7 @@ static void test_conv_from_str_to_num() goto test_conv_from_str_to_num_end; } - ret = flb_msgpack_to_json(&json[0], sizeof(json), &result.data); + ret = flb_msgpack_to_json(&json[0], sizeof(json), &result.data, FLB_TRUE); if (!TEST_CHECK(ret > 0)) { TEST_MSG("flb_msgpack_to_json failed"); msgpack_unpacked_destroy(&result); From 46c72077bd430e1200c58cdbf6640fc03110e35f Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:13:55 +0900 Subject: [PATCH 51/64] tests: filter_lua: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/runtime/filter_lua.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/runtime/filter_lua.c b/tests/runtime/filter_lua.c index 70bbfc12b43..19540f221a5 100644 --- a/tests/runtime/filter_lua.c +++ b/tests/runtime/filter_lua.c @@ -185,7 +185,7 @@ static char *get_group_metadata(void *chunk, size_t size) return NULL; } - ret = flb_msgpack_to_json(out_buf, out_size, log_event.metadata); + ret = flb_msgpack_to_json(out_buf, out_size, log_event.metadata, FLB_TRUE); if (ret < 0) { flb_sds_destroy(out_buf); flb_log_event_decoder_destroy(&log_decoder); @@ -221,7 +221,7 @@ static char *get_group_body(void *chunk, size_t size) return NULL; } - ret = flb_msgpack_to_json(out_buf, out_size, log_event.body); + ret = flb_msgpack_to_json(out_buf, out_size, log_event.body, FLB_TRUE); if (ret < 0) { flb_sds_destroy(out_buf); flb_log_event_decoder_destroy(&log_decoder); @@ -252,7 +252,7 @@ static char *get_log_body(void *chunk, size_t size) return NULL; } - ret = flb_msgpack_to_json(out_buf, out_size, log_event.body); + ret = flb_msgpack_to_json(out_buf, out_size, log_event.body, FLB_TRUE); if (ret < 0) { flb_sds_destroy(out_buf); flb_log_event_decoder_destroy(&log_decoder); @@ -286,7 +286,7 @@ static char *get_record_metadata(void *chunk, size_t size) return NULL; } - ret = flb_msgpack_to_json(out_buf, out_size, log_event.metadata); + ret = flb_msgpack_to_json(out_buf, out_size, log_event.metadata, FLB_TRUE); if (ret < 0) { flb_sds_destroy(out_buf); flb_log_event_decoder_destroy(&log_decoder); @@ -1365,8 +1365,8 @@ static int cb_check_metadata_array(void *chunk, size_t size, void *data) TEST_CHECK(ret == FLB_EVENT_DECODER_SUCCESS); while ((ret = flb_log_event_decoder_next(&dec, &log_event)) == FLB_EVENT_DECODER_SUCCESS) { - char *meta = flb_msgpack_to_json_str(256, log_event.metadata); - char *body = flb_msgpack_to_json_str(256, log_event.body); + char *meta = flb_msgpack_to_json_str(256, log_event.metadata, FLB_TRUE); + char *body = flb_msgpack_to_json_str(256, log_event.body, FLB_TRUE); TEST_CHECK(meta != NULL && body != NULL); if (meta && body) { From 35eba05a3e31f2e883fb5cd4ea449820900f7b2c Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 18:14:16 +0900 Subject: [PATCH 52/64] tests: in_opentelemetry: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/runtime/in_opentelemetry.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/runtime/in_opentelemetry.c b/tests/runtime/in_opentelemetry.c index e9efabac9ca..27b5806f11f 100644 --- a/tests/runtime/in_opentelemetry.c +++ b/tests/runtime/in_opentelemetry.c @@ -98,7 +98,7 @@ static char *get_group_metadata(void *chunk, size_t size) return NULL; } - json = flb_msgpack_to_json_str(1024, log_event.metadata); + json = flb_msgpack_to_json_str(1024, log_event.metadata, FLB_TRUE); flb_log_event_decoder_destroy(&log_decoder); return json; } @@ -121,7 +121,7 @@ static char *get_group_body(void *chunk, size_t size) return NULL; } - json = flb_msgpack_to_json_str(1024, log_event.body); + json = flb_msgpack_to_json_str(1024, log_event.body, FLB_TRUE); flb_log_event_decoder_destroy(&log_decoder); return json; } @@ -146,7 +146,7 @@ static char *get_log_body(void *chunk, size_t size) flb_log_event_decoder_next(&log_decoder, &log_event); /* convert log body to json */ - json = flb_msgpack_to_json_str(1024, log_event.body); + json = flb_msgpack_to_json_str(1024, log_event.body, FLB_TRUE); flb_log_event_decoder_destroy(&log_decoder); return json; From e3621868926a26484f135808320f75e5b7d52c46 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 19:55:38 +0900 Subject: [PATCH 53/64] utils: Optimize raw string function with SIMD Signed-off-by: Hiroshi Hatake --- src/flb_utils.c | 124 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 85 insertions(+), 39 deletions(-) diff --git a/src/flb_utils.c b/src/flb_utils.c index 34d0547f46d..4815ec02433 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -1096,72 +1096,118 @@ static int flb_utils_write_str_escaped(char *buf, int *off, size_t size, const c return FLB_TRUE; } -/* Safely copies raw UTF-8 strings, only escaping essential characters. */ +/* Safely copies raw UTF-8 strings, only escaping essential characters. + * This version correctly implements the repeating SIMD fast path for performance. + */ static int flb_utils_write_str_raw(char *buf, int *off, size_t size, const char *str, size_t str_len) { - int i = 0; - int copypos = 0; + int i, b, vlen, len, utf_len, copypos = 0; size_t available; char *p; off_t offset = 0; + const size_t inst_len = FLB_SIMD_VEC8_INST_LEN; + uint32_t c; + char *seq = NULL; available = size - *off; p = buf + *off; - for (i = 0; i < str_len; i++) { - uint32_t c = (uint32_t) str[i]; - int len = 0; - char *seq = NULL; + /* align length to the nearest multiple of the vector size for safe SIMD processing */ + vlen = str_len & ~(inst_len - 1); - /* Handle essential escapes for JSON validity */ - if (c < 128 && json_escape_table[c].seq) { - seq = json_escape_table[c].seq; - len = json_escape_table[c].seq[1] == 'u' ? 6 : 2; - } + for (i = 0;;) { + /* + * Process chunks of the input string using SIMD instructions. + * This loop continues as long as it finds "safe" ASCII characters. + */ + for (; i < vlen; i += inst_len) { + flb_vector8 chunk; + flb_vector8_load(&chunk, (const uint8_t *)&str[i]); - if (seq) { - if (available < len) { - return FLB_FALSE; + /* If a special character is found, break and switch to the slow path */ + if (flb_vector8_has_le(chunk, (unsigned char) 0x1F) || + flb_vector8_has(chunk, (unsigned char) '"') || + flb_vector8_has(chunk, (unsigned char) '\\') || + flb_vector8_is_highbit_set(chunk)) { + break; } - memcpy(p, seq, len); - p += len; - offset += len; - available -= len; } - else if (c < 0x80) { /* Regular ASCII */ - if (available < 1) { + + /* Copy the 'safe' chunk processed by the SIMD loop so far */ + if (copypos < i) { + if (available < i - copypos) { return FLB_FALSE; } - *p++ = c; - offset++; - available--; + memcpy(p, &str[copypos], i - copypos); + p += i - copypos; + offset += i - copypos; + available -= (i - copypos); + copypos = i; } - else { /* Multibyte UTF-8 sequence */ - int utf_len = flb_utf8_len(&str[i]); - if (utf_len == 0 || i + utf_len > str_len) { /* Invalid/truncated sequence */ - if (available < 3) { + /* + * Process the next 16-byte chunk character by character. + * This loop runs only for a chunk that contains special characters. + */ + for (b = 0; b < inst_len; b++) { + if (i >= str_len) { + goto done; + } + + c = (uint32_t) str[i]; + len = 0; + seq = NULL; + + /* Handle essential escapes for JSON validity */ + if (c < 128 && json_escape_table[c].seq) { + seq = json_escape_table[c].seq; + len = json_escape_table[c].seq[1] == 'u' ? 6 : 2; + if (available < len) { return FLB_FALSE; } - memcpy(p, "\xEF\xBF\xBD", 3); /* Standard replacement character */ - p += 3; - offset += 3; - available -= 3; + memcpy(p, seq, len); + p += len; + offset += len; + available -= len; } - else { /* Valid sequence, copy raw */ - if (available < utf_len) { + else if (c < 0x80) { /* Regular ASCII */ + if (available < 1) { return FLB_FALSE; } - memcpy(p, &str[i], utf_len); - p += utf_len; - offset += utf_len; - available -= utf_len; - i += utf_len - 1; /* Advance loop counter */ + *p++ = c; + offset++; + available--; } + else { /* Multibyte UTF-8 sequence */ + utf_len = flb_utf8_len(&str[i]); + + if (utf_len == 0 || i + utf_len > str_len) { /* Invalid/truncated */ + if (available < 3) { + return FLB_FALSE; + } + memcpy(p, "\xEF\xBF\xBD", 3); /* Standard replacement character */ + p += 3; + offset += 3; + available -= 3; + } + else { /* Valid sequence, copy raw */ + if (available < utf_len) { + return FLB_FALSE; + } + memcpy(p, &str[i], utf_len); + p += utf_len; + offset += utf_len; + available -= utf_len; + i += utf_len - 1; /* Advance loop counter by extra bytes */ + } + } + i++; } + copypos = i; } +done: *off += offset; return FLB_TRUE; } From fc3b9c9a913a7917f3585a3f2e9fdefd99f4da63 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 20:30:20 +0900 Subject: [PATCH 54/64] tests: utils: Add test case for encoding raw UTF-8 strings Signed-off-by: Hiroshi Hatake --- tests/internal/utils.c | 58 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/tests/internal/utils.c b/tests/internal/utils.c index 3bba3b81e47..6b5987a9a23 100644 --- a/tests/internal/utils.c +++ b/tests/internal/utils.c @@ -203,13 +203,19 @@ void test_url_split() } /* test case loop for flb_utils_write_str */ -static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size); +static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size, + int escape_unicode); static void write_str_test_cases(struct write_str_case *cases) { - write_str_test_cases_w_buf_size(cases, 100); + write_str_test_cases_w_buf_size(cases, 100, FLB_TRUE); +} + +static void write_raw_str_test_cases(struct write_str_case *cases) { + write_str_test_cases_w_buf_size(cases, 100, FLB_FALSE); } /* test case loop for flb_utils_write_str */ -static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size) +static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int buf_size, + int escape_unicode) { char *buf = flb_calloc(buf_size + 1, sizeof(char)); int size = buf_size + 1; @@ -220,7 +226,8 @@ static void write_str_test_cases_w_buf_size(struct write_str_case *cases, int bu while (!(tcase->input == 0 && tcase->output == 0)) { memset(buf, 0, size); off = 0; - ret = flb_utils_write_str(buf, &off, buf_size, tcase->input, tcase->input_len, FLB_TRUE); + ret = flb_utils_write_str(buf, &off, buf_size, tcase->input, tcase->input_len, + escape_unicode); if(!TEST_CHECK(ret == tcase->ret)) { TEST_MSG("Input string: %s", tcase->input); @@ -388,6 +395,46 @@ void test_write_str_special_bytes() write_str_test_cases(cases); } +void test_write_raw_str_special_bytes() +{ + struct write_str_case cases[] = { + /* + * Input: "你好世界" (12 bytes) + * Output: "你好世界" (raw) + */ + { + "\xE4\xBD\xA0\xE5\xA5\xBD\xE4\xB8\x96\xE7\x95\x8C", 12, + "\xE4\xBD\xA0\xE5\xA5\xBD\xE4\xB8\x96\xE7\x95\x8C", + FLB_TRUE + }, + /* + * Input: "你好我来自一个汉字文化影响的地方" (48 bytes) + * Output: "你好我来自一个汉字文化影响的地方" (raw) + */ + { + "\xE4\xBD\xA0\xE5\xA5\xBD\xE6\x88\x91\xE6\x9D\xA5\xE8\x87\xAA" \ + "\xE4\xB8\x80\xE4\xB8\xAA\xE6\xB1\x89\xE5\xAD\x97\xE6\x96\x87" \ + "\xE5\x8C\x96\xE5\xBD\xB1\xE5\x93\x8D\xE7\x9A\x84\xE5\x9C\xB0" \ + "\xE6\x96\xB9", + 48, + "\xE4\xBD\xA0\xE5\xA5\xBD\xE6\x88\x91\xE6\x9D\xA5\xE8\x87\xAA" \ + "\xE4\xB8\x80\xE4\xB8\xAA\xE6\xB1\x89\xE5\xAD\x97\xE6\x96\x87" \ + "\xE5\x8C\x96\xE5\xBD\xB1\xE5\x93\x8D\xE7\x9A\x84\xE5\x9C\xB0" \ + "\xE6\x96\xB9", + FLB_TRUE + }, + /* Test string with a quote */ + { + "\"hello\"", 7, + "\\\"hello\\\"", + FLB_TRUE + }, + { 0 } + }; + + write_raw_str_test_cases(cases); +} + void test_write_str_invalid_leading_byte_case_2() { @@ -472,7 +519,7 @@ void test_write_str_buffer_overrun() }, { 0 } }; - write_str_test_cases_w_buf_size(cases, 5); + write_str_test_cases_w_buf_size(cases, 5, FLB_TRUE); } struct proxy_url_check { @@ -810,6 +857,7 @@ TEST_LIST = { { "url_split_sds", test_url_split_sds }, { "write_str", test_write_str }, { "write_str_special_bytes", test_write_str_special_bytes }, + { "write_raw_str_special_bytes", test_write_raw_str_special_bytes }, { "test_write_str_invalid_trailing_bytes", test_write_str_invalid_trailing_bytes }, { "test_write_str_invalid_leading_byte", test_write_str_invalid_leading_byte }, { "test_write_str_edge_cases", test_write_str_edge_cases }, From 8e59ec7bfa0d51fa5088a8838347d3fe2509f83e Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 20:49:14 +0900 Subject: [PATCH 55/64] in_systemd: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/runtime/in_systemd.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runtime/in_systemd.c b/tests/runtime/in_systemd.c index 098d5ceb0c4..83c83357f7c 100644 --- a/tests/runtime/in_systemd.c +++ b/tests/runtime/in_systemd.c @@ -32,7 +32,7 @@ static void cb_check_cfl_variant_properties(void *ctx, int ffd, char *result = NULL; /* Convert from msgpack to JSON */ - output = flb_msgpack_raw_to_json_sds(res_data, res_size); + output = flb_msgpack_raw_to_json_sds(res_data, res_size, FLB_TRUE); TEST_CHECK(output != NULL); result = strstr(output, "\"MESSAGE\":\"test native message with multiple values\""); From 10308d220bf7d44a75852ae54944ec63dc9876fa Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 20:54:18 +0900 Subject: [PATCH 56/64] pack: Avoid to use hard-coded value for escaping strings Signed-off-by: Hiroshi Hatake --- src/flb_pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_pack.c b/src/flb_pack.c index e80cf0592dc..50bff582c0b 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -1166,7 +1166,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes, json_format == FLB_PACK_JSON_FORMAT_STREAM) { /* Encode current record into JSON in a temporary variable */ - out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, FLB_TRUE); + out_js = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size, escape_unicode); if (!out_js) { flb_sds_destroy(out_buf); msgpack_sbuffer_destroy(&tmp_sbuf); From 171f17408034ef964f243607867d88ba0d669bb5 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 20:59:38 +0900 Subject: [PATCH 57/64] out_pqsql: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- plugins/out_pgsql/pgsql.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/out_pgsql/pgsql.c b/plugins/out_pgsql/pgsql.c index e6b24451cab..69a5ebdd14a 100644 --- a/plugins/out_pgsql/pgsql.c +++ b/plugins/out_pgsql/pgsql.c @@ -271,7 +271,8 @@ static void cb_pgsql_flush(struct flb_event_chunk *event_chunk, event_chunk->size, FLB_PACK_JSON_FORMAT_JSON, FLB_PACK_JSON_DATE_DOUBLE, - ctx->timestamp_key); + ctx->timestamp_key, + config->json_escape_unicode); if (json == NULL) { flb_errno(); flb_plg_error(ctx->ins, From 594cf85cb40a8921d0f6ac37ca43b496dcf0ad85 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 21:08:52 +0900 Subject: [PATCH 58/64] tests: fuzzer: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/internal/fuzzers/flb_json_fuzzer.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/internal/fuzzers/flb_json_fuzzer.c b/tests/internal/fuzzers/flb_json_fuzzer.c index 916b1a51265..60e6422b58f 100644 --- a/tests/internal/fuzzers/flb_json_fuzzer.c +++ b/tests/internal/fuzzers/flb_json_fuzzer.c @@ -49,7 +49,7 @@ int LLVMFuzzerTestOneInput(unsigned char *data, size_t size) if (ret2 == MSGPACK_UNPACK_SUCCESS) { msgpack_object root = result.data; char *tmp = NULL; - tmp = flb_msgpack_to_json_str(0, &root); + tmp = flb_msgpack_to_json_str(0, &root, FLB_TRUE); if (tmp != NULL) { flb_free(tmp); } @@ -60,7 +60,7 @@ int LLVMFuzzerTestOneInput(unsigned char *data, size_t size) if (decider < 0x30) { flb_sds_t ret_s = flb_pack_msgpack_to_json_format(out_buf, out_size, FLB_PACK_JSON_FORMAT_LINES, - (int)decider, d); + (int)decider, d, FLB_TRUE); free(out_buf); if (ret_s != NULL) { flb_sds_destroy(ret_s); @@ -69,7 +69,7 @@ int LLVMFuzzerTestOneInput(unsigned char *data, size_t size) else { flb_sds_t ret_s = flb_pack_msgpack_to_json_format(out_buf, out_size, FLB_PACK_JSON_FORMAT_LINES, - FLB_PACK_JSON_DATE_EPOCH, NULL); + FLB_PACK_JSON_DATE_EPOCH, NULL, FLB_TRUE); free(out_buf); if (ret_s != NULL) { flb_sds_destroy(ret_s); From 2747779150d0e2f55d4c1b8a735735991cf49768 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 22:25:34 +0900 Subject: [PATCH 59/64] tests: fuzzer: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/internal/fuzzers/msgpack_parse_fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/internal/fuzzers/msgpack_parse_fuzzer.c b/tests/internal/fuzzers/msgpack_parse_fuzzer.c index cbf2ecf1484..02011c7a888 100644 --- a/tests/internal/fuzzers/msgpack_parse_fuzzer.c +++ b/tests/internal/fuzzers/msgpack_parse_fuzzer.c @@ -23,7 +23,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size){ /* target the conversion of raw msgpack to json */ flb_sds_t record; - record = flb_msgpack_raw_to_json_sds(data, size); + record = flb_msgpack_raw_to_json_sds(data, size, FLB_TRUE); flb_sds_destroy(record); return 0; From 2d490b977b6faf2eafba2f9d128faa3310d5e207 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 31 Jul 2025 22:34:04 +0900 Subject: [PATCH 60/64] tests: fuzzer: Follow an increase of argument Signed-off-by: Hiroshi Hatake --- tests/internal/fuzzers/utils_fuzzer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/internal/fuzzers/utils_fuzzer.c b/tests/internal/fuzzers/utils_fuzzer.c index 2352adf3e43..8d70d32a144 100644 --- a/tests/internal/fuzzers/utils_fuzzer.c +++ b/tests/internal/fuzzers/utils_fuzzer.c @@ -62,7 +62,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) char *uri = NULL; char *new_dst = NULL; - if (flb_utils_write_str_buf(null_terminated, size, &new_dst, &new_size) == 0) { + if (flb_utils_write_str_buf(null_terminated, size, &new_dst, &new_size, FLB_TRUE) == 0) { flb_free(new_dst); } From abf6cb380548ed1cb023644b82ff9a2e4f5a8deb Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 18 Sep 2025 20:13:13 +0900 Subject: [PATCH 61/64] config: Use bool type to handle On and True as FLB_TRUE Signed-off-by: Hiroshi Hatake --- src/flb_config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_config.c b/src/flb_config.c index a6f74de2a77..b522bd22a44 100644 --- a/src/flb_config.c +++ b/src/flb_config.c @@ -179,7 +179,7 @@ struct flb_service_config service_configs[] = { /* Escape UNicode inside of JSON */ {FLB_CONF_UNICODE_STR_JSON_ESCAPE, - FLB_CONF_TYPE_INT, + FLB_CONF_TYPE_BOOL, offsetof(struct flb_config, json_escape_unicode)}, #ifdef FLB_HAVE_STREAM_PROCESSOR From 3420877183dade65c73706a59aaec888bf2805e5 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 18 Sep 2025 20:20:31 +0900 Subject: [PATCH 62/64] utils: Validate invalid code points inside the unescaped code path Signed-off-by: Hiroshi Hatake --- src/flb_utils.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/flb_utils.c b/src/flb_utils.c index 4815ec02433..d51555e2771 100644 --- a/src/flb_utils.c +++ b/src/flb_utils.c @@ -1096,6 +1096,65 @@ static int flb_utils_write_str_escaped(char *buf, int *off, size_t size, const c return FLB_TRUE; } +static inline int flb_utf8_validate_char(const unsigned char *str, int max_len) +{ + unsigned char c = str[0]; + int len = 0; + int i; + + if (max_len < 1) { + return 0; + } + + /* 1-byte sequence (ASCII) */ + if (c <= 0x7F) { + return 1; + } + /* 2-byte sequence */ + else if ((c & 0xE0) == 0xC0) { + if (c < 0xC2) return 0; /* Overlong encoding */ + len = 2; + } + /* 3-byte sequence */ + else if ((c & 0xF0) == 0xE0) { + if (max_len > 1 && c == 0xE0 && (unsigned char)str[1] < 0xA0) { + return 0; /* Overlong */ + } + if (max_len > 1 && c == 0xED && (unsigned char)str[1] >= 0xA0) { + return 0; /* Surrogates */ + } + len = 3; + } + /* 4-byte sequence */ + else if ((c & 0xF8) == 0xF0) { + if (max_len > 1 && c == 0xF0 && (unsigned char)str[1] < 0x90) { + return 0; /* Overlong */ + } + if (c > 0xF4) { + return 0; /* Outside of Unicode range */ + } + if (max_len > 1 && c == 0xF4 && (unsigned char)str[1] > 0x8F) { + return 0; /* Outside of Unicode range */ + } + len = 4; + } + else { + return 0; /* Invalid starting byte */ + } + + if (max_len < len) { + return 0; /* Truncated sequence */ + } + + for (i = 1; i < len; i++) { + if ((str[i] & 0xC0) != 0x80) { + return 0; /* Invalid continuation byte */ + } + } + + return len; +} + /* Safely copies raw UTF-8 strings, only escaping essential characters. * This version correctly implements the repeating SIMD fast path for performance. */ @@ -1180,7 +1239,7 @@ static int flb_utils_write_str_raw(char *buf, int *off, size_t size, available--; } else { /* Multibyte UTF-8 sequence */ - utf_len = flb_utf8_len(&str[i]); + utf_len = flb_utf8_validate_char((const unsigned char *)&str[i], str_len - i); if (utf_len == 0 || i + utf_len > str_len) { /* Invalid/truncated */ if (available < 3) { From c1dfe9adbf715f3140f2b96b3724e5ba22057c01 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Thu, 18 Sep 2025 20:30:00 +0900 Subject: [PATCH 63/64] utils: tests: Add a test case for addressing invalid code points Signed-off-by: Hiroshi Hatake --- tests/internal/utils.c | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/internal/utils.c b/tests/internal/utils.c index 6b5987a9a23..1fa385d37b9 100644 --- a/tests/internal/utils.c +++ b/tests/internal/utils.c @@ -435,6 +435,73 @@ void test_write_raw_str_special_bytes() write_raw_str_test_cases(cases); } +void test_write_raw_str_invalid_sequences() +{ + struct write_str_case cases[] = { + /* + * Case 1: Stray continuation byte (0x80) + */ + { + "hello \x80 world", 13, + "hello \xEF\xBF\xBD world", + FLB_TRUE + }, + + /* + * Case 2: Incomplete multi-byte sequence + */ + { + "a\xE6\x97""b", 4, + "a""\xEF\xBF\xBD""\xEF\xBF\xBD""b", + FLB_TRUE + }, + + /* + * Case 3: Overlong encoding + */ + { + "a\xC0\xAF""b", 4, + "a""\xEF\xBF\xBD""\xEF\xBF\xBD""b", + FLB_TRUE + }, + + /* + * Case 4: With an invalid starting byte + */ + { + "start-\xFF-end", 11, + "start-\xEF\xBF\xBD-end", + FLB_TRUE + }, + + /* + * Case 5: Mix of valid and invalid sequences + */ + { + /* Input: "你好世界" */ + "\xE4\xBD\xA0\xE5\xA5\xBD" "\x80" "\xE4\xB8\x96\xE7\x95\x8C", 13, + /* Output: "你好世界" */ + "\xE4\xBD\xA0\xE5\xA5\xBD" "\xEF\xBF\xBD" "\xE4\xB8\x96\xE7\x95\x8C", + FLB_TRUE + }, + + /* + * Case 6: Sequence with invalid continuation byte + */ + { + /* Input: "a" + 日(E6 97 A5) + ASCII "b" */ + "a\xE6\x97""b", 4, + "a""\xEF\xBF\xBD""\xEF\xBF\xBD""b", + FLB_TRUE + }, + + /* End of cases */ + { 0 } + }; + + write_raw_str_test_cases(cases); +} + void test_write_str_invalid_leading_byte_case_2() { @@ -858,6 +925,7 @@ TEST_LIST = { { "write_str", test_write_str }, { "write_str_special_bytes", test_write_str_special_bytes }, { "write_raw_str_special_bytes", test_write_raw_str_special_bytes }, + { "write_raw_str_invalid_bytes", test_write_raw_str_invalid_sequences}, { "test_write_str_invalid_trailing_bytes", test_write_str_invalid_trailing_bytes }, { "test_write_str_invalid_leading_byte", test_write_str_invalid_leading_byte }, { "test_write_str_edge_cases", test_write_str_edge_cases }, From 4510a2af811484de21aabc8712996db52af203f6 Mon Sep 17 00:00:00 2001 From: Hiroshi Hatake Date: Fri, 19 Sep 2025 15:25:38 +0900 Subject: [PATCH 64/64] workflows: Use gcc 14 and clang 14 on ubuntu noble(24.04) Signed-off-by: Hiroshi Hatake --- .github/workflows/unit-tests.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index f6b5aa4d5bd..598d122dc33 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -156,7 +156,7 @@ jobs: FLB_OPT: ${{ matrix.flb_option }} run-aarch64-unit-tests: - runs-on: ${{(github.repository == 'fluent/fluent-bit') && 'ubuntu-22.04-arm' || 'ubuntu-latest' }} + runs-on: ${{(github.repository == 'fluent/fluent-bit') && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} permissions: contents: read needs: @@ -183,8 +183,8 @@ jobs: - name: Setup environment run: | sudo apt-get update - sudo apt-get install -y gcc-9 g++-9 clang-12 flex bison libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common curl tar gzip - sudo ln -s /usr/bin/llvm-symbolizer-12 /usr/bin/llvm-symbolizer || true + sudo apt-get install -y gcc-14 g++-14 clang-14 flex bison libsystemd-dev gcovr libyaml-dev libbpf-dev linux-tools-common curl tar gzip + sudo ln -s /usr/bin/llvm-symbolizer-14 /usr/bin/llvm-symbolizer || true sudo mkdir -p "${CMAKE_HOME}" cmake_url="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-$(uname -m).tar.gz" cmake_dist="$(mktemp --suffix ".tar.gz")" @@ -199,9 +199,9 @@ jobs: - name: Build and test with arm runners run: | - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90 - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-9 90 - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-12 90 + sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 90 + sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 90 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-14 90 export nparallel=$(( $(getconf _NPROCESSORS_ONLN) > 8 ? 8 : $(getconf _NPROCESSORS_ONLN) )) export FLB_OPTION="${{ matrix.config.flb_option }}"