diff --git a/plugins/in_udp/udp.c b/plugins/in_udp/udp.c index 2395d5ec407..ad5a28497ee 100644 --- a/plugins/in_udp/udp.c +++ b/plugins/in_udp/udp.c @@ -174,6 +174,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_in_udp_config, buffer_size_str), "Set the buffer size" }, + { + FLB_CONFIG_MAP_STR, "source_address_key", (char *) NULL, + 0, FLB_TRUE, offsetof(struct flb_in_udp_config, source_address_key), + "Key where the source address will be injected" + }, /* EOF */ {0} }; diff --git a/plugins/in_udp/udp.h b/plugins/in_udp/udp.h index f4b37f6c4c5..1a7bfce3056 100644 --- a/plugins/in_udp/udp.h +++ b/plugins/in_udp/udp.h @@ -43,6 +43,7 @@ struct flb_in_udp_config { char *port; /* Port */ flb_sds_t raw_separator; /* Unescaped string delimiterr */ flb_sds_t separator; /* String delimiter */ + flb_sds_t source_address_key; /* Source IP address */ int collector_id; /* Listener collector id */ struct flb_downstream *downstream; /* Client manager */ struct udp_conn *dummy_conn; /* Datagram dummy connection */ diff --git a/plugins/in_udp/udp_conn.c b/plugins/in_udp/udp_conn.c index a18f74158b3..d8cc4d5e620 100644 --- a/plugins/in_udp/udp_conn.c +++ b/plugins/in_udp/udp_conn.c @@ -32,6 +32,68 @@ static inline void consume_bytes(char *buf, int bytes, int length) memmove(buf, buf + bytes, length - bytes); } +static int append_message_to_record_data(char **result_buffer, + size_t *result_size, + flb_sds_t message_key_name, + char *base_object_buffer, + size_t base_object_size, + char *message_buffer, + size_t message_size, + int message_type) +{ + int result = FLB_MAP_NOT_MODIFIED; + char *modified_data_buffer; + int modified_data_size; + msgpack_object_kv *new_map_entries[1]; + msgpack_object_kv message_entry; + *result_buffer = NULL; + *result_size = 0; + modified_data_buffer = NULL; + + if (message_key_name != NULL) { + new_map_entries[0] = &message_entry; + + message_entry.key.type = MSGPACK_OBJECT_STR; + message_entry.key.via.str.size = flb_sds_len(message_key_name); + message_entry.key.via.str.ptr = message_key_name; + + if (message_type == MSGPACK_OBJECT_BIN) { + message_entry.val.type = MSGPACK_OBJECT_BIN; + message_entry.val.via.bin.size = message_size; + message_entry.val.via.bin.ptr = message_buffer; + } + else if (message_type == MSGPACK_OBJECT_STR) { + message_entry.val.type = MSGPACK_OBJECT_STR; + message_entry.val.via.str.size = message_size; + message_entry.val.via.str.ptr = message_buffer; + } + else { + result = FLB_MAP_EXPANSION_INVALID_VALUE_TYPE; + } + + if (result == FLB_MAP_NOT_MODIFIED) { + result = flb_msgpack_expand_map(base_object_buffer, + base_object_size, + new_map_entries, 1, + &modified_data_buffer, + &modified_data_size); + if (result == 0) { + result = FLB_MAP_EXPAND_SUCCESS; + } + else { + result = FLB_MAP_EXPANSION_ERROR; + } + } + } + + if (result == FLB_MAP_EXPAND_SUCCESS) { + *result_buffer = modified_data_buffer; + *result_size = modified_data_size; + } + + return result; +} + static inline int process_pack(struct udp_conn *conn, char *pack, size_t size) { @@ -39,7 +101,14 @@ static inline int process_pack(struct udp_conn *conn, size_t off = 0; msgpack_unpacked result; msgpack_object entry; + msgpack_sbuffer sbuf; + msgpack_packer pck; struct flb_in_udp_config *ctx; + char *appended_address_buffer; + size_t appended_address_size; + char *source_address; + int i; + int len; ctx = conn->ctx; @@ -50,6 +119,8 @@ static inline int process_pack(struct udp_conn *conn, while (msgpack_unpack_next(&result, pack, size, &off) == MSGPACK_UNPACK_SUCCESS) { entry = result.data; + appended_address_buffer = NULL; + source_address = NULL; ret = flb_log_event_encoder_begin_record(ctx->log_encoder); @@ -57,16 +128,63 @@ static inline int process_pack(struct udp_conn *conn, ret = flb_log_event_encoder_set_current_timestamp(ctx->log_encoder); } + if (ctx->source_address_key != NULL) { + source_address = flb_connection_get_remote_address(conn->connection); + } + if (ret == FLB_EVENT_ENCODER_SUCCESS) { if (entry.type == MSGPACK_OBJECT_MAP) { - ret = flb_log_event_encoder_set_body_from_msgpack_object( - ctx->log_encoder, &entry); + if (source_address != NULL) { + msgpack_sbuffer_init(&sbuf); + msgpack_packer_init(&pck, &sbuf, msgpack_sbuffer_write); + + len = entry.via.map.size; + msgpack_pack_map(&pck, len); + + for (i=0; isource_address_key, + sbuf.data, + sbuf.size, + source_address, + strlen(source_address), + MSGPACK_OBJECT_STR); + msgpack_sbuffer_destroy(&sbuf); + } + + if (ret == FLB_MAP_EXPANSION_ERROR) { + flb_plg_debug(ctx->ins, "error expanding source_address : %d", ret); + } + + if (appended_address_buffer != NULL) { + ret = flb_log_event_encoder_set_body_from_raw_msgpack( + ctx->log_encoder, appended_address_buffer, appended_address_size); + } + else { + ret = flb_log_event_encoder_set_body_from_msgpack_object( + ctx->log_encoder, &entry); + } } else if (entry.type == MSGPACK_OBJECT_ARRAY) { - ret = flb_log_event_encoder_append_body_values( + if (source_address != NULL) { + ret = flb_log_event_encoder_append_body_values( + ctx->log_encoder, + FLB_LOG_EVENT_CSTRING_VALUE("msg"), + FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&entry), + FLB_LOG_EVENT_CSTRING_VALUE(ctx->source_address_key), + FLB_LOG_EVENT_CSTRING_VALUE(source_address)); + } + else { + ret = flb_log_event_encoder_append_body_values( ctx->log_encoder, FLB_LOG_EVENT_CSTRING_VALUE("msg"), FLB_LOG_EVENT_MSGPACK_OBJECT_VALUE(&entry)); + } } else { ret = FLB_EVENT_ENCODER_ERROR_INVALID_VALUE_TYPE; @@ -76,6 +194,10 @@ static inline int process_pack(struct udp_conn *conn, ret = flb_log_event_encoder_commit_record(ctx->log_encoder); } + if (appended_address_buffer != NULL) { + flb_free(appended_address_buffer); + } + if (ret != FLB_EVENT_ENCODER_SUCCESS) { break; } diff --git a/plugins/in_udp/udp_conn.h b/plugins/in_udp/udp_conn.h index 96dbf14eb37..25b8ef3ef87 100644 --- a/plugins/in_udp/udp_conn.h +++ b/plugins/in_udp/udp_conn.h @@ -25,6 +25,10 @@ #define FLB_IN_UDP_CHUNK "32768" +#define FLB_MAP_EXPAND_SUCCESS 0 +#define FLB_MAP_NOT_MODIFIED -1 +#define FLB_MAP_EXPANSION_ERROR -2 +#define FLB_MAP_EXPANSION_INVALID_VALUE_TYPE -3 struct udp_conn_stream { char *tag; diff --git a/tests/runtime/CMakeLists.txt b/tests/runtime/CMakeLists.txt index 125b0055b83..3fd525e62c6 100644 --- a/tests/runtime/CMakeLists.txt +++ b/tests/runtime/CMakeLists.txt @@ -51,6 +51,7 @@ if(FLB_OUT_LIB) FLB_RT_TEST(FLB_IN_SPLUNK "in_splunk.c") FLB_RT_TEST(FLB_IN_SYSLOG "in_syslog.c") FLB_RT_TEST(FLB_IN_TAIL "in_tail.c") + FLB_RT_TEST(FLB_IN_UDP "in_udp.c") FLB_RT_TEST(FLB_IN_TCP "in_tcp.c") FLB_RT_TEST(FLB_IN_FORWARD "in_forward.c") FLB_RT_TEST(FLB_IN_FLUENTBIT_METRICS "in_fluentbit_metrics.c") diff --git a/tests/runtime/in_udp.c b/tests/runtime/in_udp.c new file mode 100644 index 00000000000..df60c923fad --- /dev/null +++ b/tests/runtime/in_udp.c @@ -0,0 +1,440 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2019-2022 The Fluent Bit Authors + * Copyright (C) 2015-2018 Treasure Data Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "flb_tests_runtime.h" + +#define DEFAULT_IO_TIMEOUT 10 +#define DEFAULT_HOST "127.0.0.1" +#define DEFAULT_PORT 5180 + +#define TLS_CERTIFICATE_HOSTNAME "leo.vcap.me" +#define TLS_CERTIFICATE_FILENAME FLB_TESTS_DATA_PATH "/data/tls/certificate.pem" +#define TLS_PRIVATE_KEY_FILENAME FLB_TESTS_DATA_PATH "/data/tls/private_key.pem" + +struct test_ctx { + flb_ctx_t *flb; /* Fluent Bit library context */ + int i_ffd; /* Input fd */ + int f_ffd; /* Filter fd (unused) */ + int o_ffd; /* Output fd */ +}; + +pthread_mutex_t result_mutex = PTHREAD_MUTEX_INITIALIZER; +int num_output = 0; +static int get_output_num() +{ + int ret; + pthread_mutex_lock(&result_mutex); + ret = num_output; + pthread_mutex_unlock(&result_mutex); + + return ret; +} + +static void set_output_num(int num) +{ + pthread_mutex_lock(&result_mutex); + num_output = num; + pthread_mutex_unlock(&result_mutex); +} + +static void clear_output_num() +{ + set_output_num(0); +} + +/* Callback to check expected results */ +static int cb_check_result_json(void *record, size_t size, void *data) +{ + char *p; + char *expected; + char *result; + int num = get_output_num(); + + set_output_num(num+1); + + expected = (char *) data; + result = (char *) record; + + p = strstr(result, expected); + TEST_CHECK(p != NULL); + + if (p==NULL) { + flb_error("Expected to find: '%s' in result '%s'", + expected, result); + } + /* + * If you want to debug your test + * + * printf("Expect: '%s' in result '%s'", expected, result); + */ + flb_free(record); + return 0; +} + +static struct test_ctx *test_ctx_create(struct flb_lib_out_cb *data) +{ + int ret; + int i_ffd; + int o_ffd; + struct test_ctx *ctx = NULL; + char sport[8]; + + ctx = flb_malloc(sizeof(struct test_ctx)); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("malloc failed"); + flb_errno(); + return NULL; + } + + /* Service config */ + ctx->flb = flb_create(); + flb_service_set(ctx->flb, + "Flush", "0.200000000", + "Grace", "1", + "Log_Level", "error", + NULL); + + /* Input */ + i_ffd = flb_input(ctx->flb, (char *) "udp", NULL); + TEST_CHECK(i_ffd >= 0); + ctx->i_ffd = i_ffd; + + /* Avoid to port collisions for default port of in_tcp */ + snprintf(sport, 8, "%d", DEFAULT_PORT); + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "port", sport, + NULL); + TEST_CHECK(ret == 0); + + /* Output */ + o_ffd = flb_output(ctx->flb, (char *) "lib", (void *) data); + ctx->o_ffd = o_ffd; + + return ctx; +} + +static void test_ctx_destroy(struct test_ctx *ctx) +{ + TEST_CHECK(ctx != NULL); + + sleep(1); + flb_stop(ctx->flb); + flb_destroy(ctx->flb); + flb_free(ctx); +} + +static int init_udp(char *in_host, int in_port, struct sockaddr_in *addr) +{ + int port = in_port; + char *host = in_host; + flb_sockfd_t fd; + + if (host == NULL) { + host = DEFAULT_HOST; + } + if (port < 0) { + port = DEFAULT_PORT; + } + + memset(addr, 0, sizeof(struct sockaddr_in)); + fd = socket(AF_INET, SOCK_DGRAM, 0); + if (!TEST_CHECK(fd >= 0)) { + TEST_MSG("failed to socket. host=%s port=%d errno=%d", host, port, errno); + return -1; + } + + addr->sin_family = AF_INET; + addr->sin_addr.s_addr = inet_addr(host); + addr->sin_port = htons(port); + + return fd; +} + +void flb_test_udp() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "{\"test\":\"msg\"}"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_udp_with_source_address() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "{\"test\":\"msg\"}"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"test\":\"msg\",\"source_host\":\"udp://"; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "source_address_key", "source_host", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_format_none() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "message\n"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"log\":\"message\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "format", "none", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num > 0)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +void flb_test_format_none_separator() +{ + struct flb_lib_out_cb cb_data; + struct test_ctx *ctx; + struct sockaddr_in addr; + flb_sockfd_t fd; + int ret; + int num; + ssize_t w_size; + + char *buf = "message:message:"; + size_t size = strlen(buf); + + clear_output_num(); + + cb_data.cb = cb_check_result_json; + cb_data.data = "\"log\":\"message\""; + + ctx = test_ctx_create(&cb_data); + if (!TEST_CHECK(ctx != NULL)) { + TEST_MSG("test_ctx_create failed"); + exit(EXIT_FAILURE); + } + + ret = flb_output_set(ctx->flb, ctx->o_ffd, + "match", "*", + "format", "json", + NULL); + TEST_CHECK(ret == 0); + + ret = flb_input_set(ctx->flb, ctx->i_ffd, + "format", "none", + "separator", ":", + NULL); + TEST_CHECK(ret == 0); + + /* Start the engine */ + ret = flb_start(ctx->flb); + TEST_CHECK(ret == 0); + + /* use default host/port */ + fd = init_udp(NULL, -1, &addr); + if (!TEST_CHECK(fd >= 0)) { + exit(EXIT_FAILURE); + } + + w_size = sendto(fd, buf, size, 0, (const struct sockaddr *)&addr, sizeof(addr)); + if (!TEST_CHECK(w_size == size)) { + TEST_MSG("failed to send, errno=%d", errno); + flb_socket_close(fd); + test_ctx_destroy(ctx); + exit(EXIT_FAILURE); + } + + /* waiting to flush */ + flb_time_msleep(1500); + + num = get_output_num(); + if (!TEST_CHECK(num == 2)) { + TEST_MSG("no outputs"); + } + + flb_socket_close(fd); + test_ctx_destroy(ctx); +} + +TEST_LIST = { + {"udp", flb_test_udp}, + {"udp_with_source_address", flb_test_udp_with_source_address}, + {"format_none", flb_test_format_none}, + {"format_none_separator", flb_test_format_none_separator}, + {NULL, NULL} +};