From 2b164dac1c46d7f2200b51868c7557b1a37a44d5 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 5 Aug 2025 12:33:11 -0600 Subject: [PATCH 1/9] in_opentelemetry: add NULL check in otlp_pack_any_value and body access - Add NULL check in otlp_pack_any_value function to handle NULL body parameter - Add NULL check for body access in conditional check to prevent segfault - Handle OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET case by packing nil value Signed-off-by: Eduardo Silva --- plugins/in_opentelemetry/opentelemetry_logs.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/in_opentelemetry/opentelemetry_logs.c b/plugins/in_opentelemetry/opentelemetry_logs.c index ba7163ca54a..ce230b905cc 100644 --- a/plugins/in_opentelemetry/opentelemetry_logs.c +++ b/plugins/in_opentelemetry/opentelemetry_logs.c @@ -176,6 +176,11 @@ static int otlp_pack_any_value(msgpack_packer *mp_pck, result = otel_pack_bytes(mp_pck, body->bytes_value); break; + case OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE__NOT_SET: + /* treat an unset value as null */ + result = msgpack_pack_nil(mp_pck); + break; + default: break; } From fc4e9db0caae7fe8bf24790d498119b8e76dcb8d Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 5 Aug 2025 12:34:43 -0600 Subject: [PATCH 2/9] opentelemetry: fix value handling for empty maps and missing values - Add proper NULL handling for empty map values (unset values) - Fix missing value handling to pack as empty string - Complete logic flow to handle all OpenTelemetry value states Signed-off-by: Eduardo Silva --- src/opentelemetry/flb_opentelemetry_utils.c | 31 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/opentelemetry/flb_opentelemetry_utils.c b/src/opentelemetry/flb_opentelemetry_utils.c index d217a2e096b..22ae9558795 100644 --- a/src/opentelemetry/flb_opentelemetry_utils.c +++ b/src/opentelemetry/flb_opentelemetry_utils.c @@ -451,6 +451,9 @@ int flb_otel_utils_json_payload_append_converted_kvlist( int value_index; int key_index; int result; + int pack_null_value = FLB_FALSE; + int pack_string_value = FLB_FALSE; + int pack_value = FLB_FALSE; size_t index; msgpack_object_array *array; msgpack_object_map *entry; @@ -476,8 +479,22 @@ int flb_otel_utils_json_payload_append_converted_kvlist( result = FLB_EVENT_ENCODER_ERROR_INVALID_ARGUMENT; } + value_index = -1; + pack_null_value = FLB_FALSE; + pack_string_value = FLB_FALSE; + pack_value = FLB_FALSE; + if (result == FLB_EVENT_ENCODER_SUCCESS) { value_index = flb_otel_utils_find_map_entry_by_key(entry, "value", 0, FLB_TRUE); + + if (value_index >= 0 && + entry->ptr[value_index].val.type == MSGPACK_OBJECT_MAP && + entry->ptr[value_index].val.via.map.size == 0) { + /* + * if value is an empty map it represents an unset value, pack as NULL + */ + pack_null_value = FLB_TRUE; + } } if (value_index == -1) { @@ -485,6 +502,10 @@ int flb_otel_utils_json_payload_append_converted_kvlist( * if value is missing basically is 'unset' and handle as Empty() in OTel world, in * this case we just pack an empty string value */ + pack_string_value = FLB_TRUE; + } + else if (!pack_null_value) { + pack_value = FLB_TRUE; } if (result == FLB_EVENT_ENCODER_SUCCESS) { @@ -495,14 +516,18 @@ int flb_otel_utils_json_payload_append_converted_kvlist( } if (result == FLB_EVENT_ENCODER_SUCCESS) { - /* if the value is not set, register an empty string as value */ - if (value_index == -1) { + if (pack_null_value) { + /* pack NULL for unset values (empty maps) */ + result = flb_log_event_encoder_append_null(encoder, target_field); + } + else if (pack_string_value) { + /* if the value is not set, register an empty string as value */ result = flb_log_event_encoder_append_string( encoder, target_field, "", 0); } - else { + else if (pack_value) { /* expected value must come in a map */ if (entry->ptr[value_index].val.type != MSGPACK_OBJECT_MAP) { result = -1; From b6effc64ed8c73159819a766eea8ae0f791563a3 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Tue, 5 Aug 2025 12:35:38 -0600 Subject: [PATCH 3/9] tests: internal: opentelemetry: add test case for issue #10672 Signed-off-by: Eduardo Silva --- .../data/opentelemetry/test_cases.json | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/internal/data/opentelemetry/test_cases.json b/tests/internal/data/opentelemetry/test_cases.json index 23bafc9b27d..3cbfab855b6 100644 --- a/tests/internal/data/opentelemetry/test_cases.json +++ b/tests/internal/data/opentelemetry/test_cases.json @@ -1225,5 +1225,31 @@ "log_metadata": {"otlp":{}}, "log_body": {"test_key": "", "normal_key": "normal_value"} } + }, + "issue_10672_empty_map_as_null": { + "input": { + "resourceLogs": [{ + "scopeLogs": [{ + "logRecords": [{ + "timeUnixNano": "1722904465173450100", + "body": { + "kvlistValue": { + "values": [ + {"key": "empty_map_key", "value": {}}, + {"key": "normal_key", "value": {"stringValue": "normal_value"}}, + {"key": "missing_value_key"} + ] + } + } + }] + }] + }] + }, + "expected": { + "group_metadata": {"schema":"otlp","resource_id":0,"scope_id":0}, + "group_body": {"resource":{}}, + "log_metadata": {"otlp":{}}, + "log_body": {"empty_map_key": null, "normal_key": "normal_value", "missing_value_key": ""} + } } } From 443ff6304e78d81cbc02ca4bd318746407c0d36e Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 6 Aug 2025 08:14:39 -0600 Subject: [PATCH 4/9] pack: relax token validation to reject tokens missing an end position Signed-off-by: Eduardo Silva --- src/flb_pack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_pack.c b/src/flb_pack.c index df1f9c196dc..ea63e0e1507 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -217,7 +217,7 @@ static char *tokens_to_msgpack(struct flb_pack_state *state, for (i = 0; i < arr_size ; i++) { t = &tokens[i]; - if (t->start == -1 || t->end == -1 || (t->start == 0 && t->end == 0)) { + if (t->start < 0 || t->end <= 0) { msgpack_sbuffer_destroy(&sbuf); return NULL; } From c514c64faae1e352c498c72b5c8bd172b9cecdaa Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 6 Aug 2025 08:17:43 -0600 Subject: [PATCH 5/9] tests: internal: pack: add test for nested empty arrays Signed-off-by: Eduardo Silva --- tests/internal/pack.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/internal/pack.c b/tests/internal/pack.c index aa73ff7defe..5836f5c0838 100644 --- a/tests/internal/pack.c +++ b/tests/internal/pack.c @@ -970,6 +970,23 @@ void test_json_pack_bug5336() } } +/* Ensure empty arrays inside nested objects are handled */ +void test_json_pack_empty_array() +{ + int ret; + int root_type; + size_t out_size; + char *out_buf; + char json[] = + "{\"resourceLogs\":[{\"resource\":{},\"scopeLogs\":[{\"scope\":{}," + "\"logRecords\":[{\"body\":{\"test\":{\"values\":[]}}}]}]}]}"; + + ret = flb_pack_json(json, strlen(json), &out_buf, &out_size, &root_type, NULL); + TEST_CHECK(ret == 0); + + flb_free(out_buf); +} + const char input_msgpack[] = {0x92,/* array 2 */ 0xd7, 0x00, /* event time*/ 0x07, 0x5b, 0xcd, 0x15, /* second = 123456789 = 1973/11/29 21:33:09 */ @@ -1115,6 +1132,7 @@ TEST_LIST = { { "json_pack_bug1278" , test_json_pack_bug1278}, { "json_pack_nan" , test_json_pack_nan}, { "json_pack_bug5336" , test_json_pack_bug5336}, + { "json_pack_empty_array", test_json_pack_empty_array}, { "json_date_iso8601" , test_json_date_iso8601}, { "json_date_double" , test_json_date_double}, { "json_date_java_sql" , test_json_date_java_sql}, From d2d2e82f3d994963c0488636daa010597462df40 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 6 Aug 2025 16:27:48 -0600 Subject: [PATCH 6/9] pack: always reset state after realloc Signed-off-by: Eduardo Silva --- src/flb_pack.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/flb_pack.c b/src/flb_pack.c index ea63e0e1507..2496ab66223 100644 --- a/src/flb_pack.c +++ b/src/flb_pack.c @@ -79,6 +79,9 @@ int flb_json_tokenise(const char *js, size_t len, state->tokens = tmp; state->tokens_size += new_tokens; + /* Reset parser to reprocess the JSON data from the beginning */ + jsmn_init(&state->parser); + ret = jsmn_parse(&state->parser, js, len, state->tokens, state->tokens_size); } From 7f68af204af63f2acf529e2f766eade6f5bcc522 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 6 Aug 2025 17:21:05 -0600 Subject: [PATCH 7/9] cmake: msgpack: increase stack size to 64 elements Signed-off-by: Eduardo Silva --- cmake/msgpack.cmake | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmake/msgpack.cmake b/cmake/msgpack.cmake index 6211e31f302..5466e283bc0 100644 --- a/cmake/msgpack.cmake +++ b/cmake/msgpack.cmake @@ -6,5 +6,9 @@ option(MSGPACK_BUILD_EXAMPLES OFF) include_directories( ${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_MSGPACK}/include ) + +# define preprocessor MSGPACK_EMBED_STACK_SIZE to 1024 +add_compile_definitions(MSGPACK_EMBED_STACK_SIZE=64) + add_subdirectory(${FLB_PATH_LIB_MSGPACK} EXCLUDE_FROM_ALL) set(MSGPACK_LIBRARIES "msgpack-c-static") From 5e6b3a7e98972efd6883c5ab93ef72702c921c11 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 6 Aug 2025 17:24:42 -0600 Subject: [PATCH 8/9] opentelemetry: fix wrapping condition for array/maps Signed-off-by: Eduardo Silva --- src/opentelemetry/flb_opentelemetry_utils.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/opentelemetry/flb_opentelemetry_utils.c b/src/opentelemetry/flb_opentelemetry_utils.c index 22ae9558795..2bcdd3b97c7 100644 --- a/src/opentelemetry/flb_opentelemetry_utils.c +++ b/src/opentelemetry/flb_opentelemetry_utils.c @@ -135,8 +135,9 @@ int flb_otel_utils_json_payload_get_wrapped_value(msgpack_object *wrapper, internal_type = MSGPACK_OBJECT_BIN; } else if (strncasecmp(kv_key->ptr, "arrayValue", kv_key->size) == 0) { - if (kv_value->type != MSGPACK_OBJECT_ARRAY) { - /* If the value is not an array, we cannot process it */ + if (kv_value->type != MSGPACK_OBJECT_ARRAY && + kv_value->type != MSGPACK_OBJECT_MAP) { + /* If the value is not an array or map, we cannot process it */ return -2; } internal_type = MSGPACK_OBJECT_ARRAY; @@ -233,7 +234,6 @@ int flb_otel_utils_json_payload_append_converted_value( target_field, (char *) object->via.str.ptr, object->via.str.size); - break; case MSGPACK_OBJECT_NIL: /* Append a null value */ @@ -262,7 +262,6 @@ int flb_otel_utils_json_payload_append_converted_value( encoder, target_field, object); - break; default: break; @@ -375,11 +374,8 @@ int flb_otel_utils_json_payload_append_converted_map( object, &encoder_result); - if (result == 0 && encoder_result == FLB_EVENT_ENCODER_SUCCESS) { - return result; - } - else if (result != 0) { - return result; + if (result == 0) { + return encoder_result; } result = flb_log_event_encoder_begin_map(encoder, target_field); From 8ecfc2c47a5dbb5aec294f7887c3bfa887ec0dcc Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Wed, 6 Aug 2025 17:26:44 -0600 Subject: [PATCH 9/9] tests: internal: opentelemetry: add cases for long nested vals Signed-off-by: Eduardo Silva --- .../data/opentelemetry/test_cases.json | 71 +++++++++++++++++++ tests/internal/pack.c | 25 +++++-- 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/tests/internal/data/opentelemetry/test_cases.json b/tests/internal/data/opentelemetry/test_cases.json index 3cbfab855b6..1ff9aa4383f 100644 --- a/tests/internal/data/opentelemetry/test_cases.json +++ b/tests/internal/data/opentelemetry/test_cases.json @@ -1251,5 +1251,76 @@ "log_metadata": {"otlp":{}}, "log_body": {"empty_map_key": null, "normal_key": "normal_value", "missing_value_key": ""} } + }, + "arrayvalue_wrapped_map": { + "input": { + "resourceLogs": [{ + "scopeLogs": [{ + "logRecords": [{ + "timeUnixNano": "1640995200000000000", + "body": { + "kvlistValue": { + "values": [{ + "key": "a", + "value": { + "kvlistValue": { + "values": [{ + "key": "d", + "value": { + "arrayValue": { + "values": [{ + "kvlistValue": { + "values": [{ + "key": "e", + "value": {"stringValue": "g"} + }] + } + }] + } + } + }] + } + } + }] + } + } + }] + }] + }] + }, + "expected": { + "group_metadata": {"schema":"otlp","resource_id":0,"scope_id":0}, + "group_body": {"resource":{}}, + "log_metadata": {"otlp":{}}, + "log_body": {"a":{"d":[{"e":"g"}]}} + } + }, + "unwrapped_plain_map": { + "input": { + "resourceLogs": [{ + "scopeLogs": [{ + "logRecords": [{ + "timeUnixNano": "1640995200000000000", + "body": { + "kvlistValue": { + "values": [{ + "key": "plain", + "value": { + "key1": {"stringValue": "v1"}, + "key2": {"intValue": 2} + } + }] + } + } + }] + }] + }] + }, + "expected": { + "group_metadata": {"schema":"otlp","resource_id":0,"scope_id":0}, + "group_body": {"resource":{}}, + "log_metadata": {"otlp":{}}, + "log_body": {"plain": {"key1": "v1", "key2": 2}} + } } } diff --git a/tests/internal/pack.c b/tests/internal/pack.c index 5836f5c0838..ae59b4d32f9 100644 --- a/tests/internal/pack.c +++ b/tests/internal/pack.c @@ -977,13 +977,30 @@ void test_json_pack_empty_array() int root_type; size_t out_size; char *out_buf; - char json[] = - "{\"resourceLogs\":[{\"resource\":{},\"scopeLogs\":[{\"scope\":{}," - "\"logRecords\":[{\"body\":{\"test\":{\"values\":[]}}}]}]}]}"; + char *json = "{\"resourceLogs\":[{\"resource\":{},\"scopeLogs\":[{\"scope\":{},\"logRecords\":[{\"observedTimeUnixNano\":\"1754059282910920618\",\"body\":{\"kvlistValue\":{\"values\":[{\"key\":\"a\",\"value\":{\"kvlistValue\":{\"values\":[{\"key\":\"b\",\"value\":{\"kvlistValue\":{\"values\":[{\"key\":\"c\",\"value\":{\"kvlistValue\":{\"values\":[{\"key\":\"d\",\"value\":{\"arrayValue\":{\"values\":[{\"kvlistValue\":{\"values\":[{\"key\":\"e\",\"value\":{\"kvlistValue\":{\"values\":[{\"key\":\"f\",\"value\":{\"stringValue\":\"g\"}}]}}}]}}]}}}]}}}]}}}]}}}]}}}]}]}]}"; - ret = flb_pack_json(json, strlen(json), &out_buf, &out_size, &root_type, NULL); + ret = flb_pack_json( (char*) json, strlen((char *) json), &out_buf, &out_size, &root_type, NULL); TEST_CHECK(ret == 0); + if (ret != 0) { + printf("flb_pack_json failed: %d\n", ret); + exit(EXIT_FAILURE); + } + + /* unpack just to validate the msgpack buffer */ + msgpack_unpacked result; + msgpack_object obj; + size_t off = 0; + msgpack_unpacked_init(&result); + off = 0; + TEST_CHECK(msgpack_unpack_next(&result, out_buf, out_size, &off) == MSGPACK_UNPACK_SUCCESS); + + printf("\nmsgpack---:\n"); + msgpack_object_print(stdout, result.data); + printf("\n"); + + msgpack_unpacked_destroy(&result); + flb_free(out_buf); }