Skip to content

in_forward: Implement decompression method handlers for gzip and zstd #10710

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
edsiper merged 7 commits into from
Sep 18, 2025
Merged

in_forward: Implement decompression method handlers for gzip and zstd #10710

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
95ff5ac
in_forward: Handle with flb_compression functions to generalized both…
cosmo0920 Aug 8, 2025
b77df19
in_forward: tests: Add gzip ingestion case scenario
cosmo0920 Aug 18, 2025
9d553a1
in_forward: tests: Add zstd compression test scenario
cosmo0920 Aug 18, 2025
31a2da2
in_forward: Check head of magic bytes on payloads
cosmo0920 Sep 11, 2025
bbe1f5d
in_forward: Remove needless variables
cosmo0920 Sep 12, 2025
07fbd01
in_forward: Cleanup on errors for decompressions
cosmo0920 Sep 12, 2025
c9635e0
in_forward: Remove needless type casts
cosmo0920 Sep 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions plugins/in_forward/fw_conn.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,9 @@ struct fw_conn *fw_conn_add(struct flb_connection *connection, struct flb_in_fw_
conn->buf_size = ctx->buffer_chunk_size;
conn->in = ctx->ins;

conn->compression_type = FLB_COMPRESSION_ALGORITHM_NONE;
conn->d_ctx = NULL;

/* Register instance into the event loop */
ret = mk_event_add(flb_engine_evl_get(),
connection->fd,
Expand Down Expand Up @@ -219,6 +222,11 @@ int fw_conn_del(struct fw_conn *conn)
/* Release resources */
mk_list_del(&conn->_head);

/* Release decompression context if it exists */
if (conn->d_ctx) {
flb_decompression_context_destroy(conn->d_ctx);
}

if (conn->helo != NULL) {
if (conn->helo->nonce != NULL) {
flb_sds_destroy(conn->helo->nonce);
Expand Down
6 changes: 6 additions & 0 deletions plugins/in_forward/fw_conn.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
#ifndef FLB_IN_FW_CONN_H
#define FLB_IN_FW_CONN_H

#include <fluent-bit/flb_compression.h>

#define FLB_IN_FW_CHUNK_SIZE "1024000" /* 1MB */
#define FLB_IN_FW_CHUNK_MAX_SIZE "6144000" /* =FLB_IN_FW_CHUNK_SIZE * 6. 6MB */
#define FLB_IN_FW_NONCE_SIZE 16
Expand Down Expand Up @@ -48,6 +50,10 @@ struct fw_conn {
int buf_size; /* Buffer size */
size_t rest; /* Unpacking offset */

/* Decompression context */
int compression_type; /* e.g., FLB_COMPRESSION_ALGORITHM_GZIP */
struct flb_decompression_context *d_ctx; /* Stateful decompressor context */

struct flb_in_fw_helo *helo; /* secure forward HELO phase */

struct flb_input_instance *in; /* Parent plugin instance */
Expand Down
234 changes: 135 additions & 99 deletions plugins/in_forward/fw_prot.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,50 +90,33 @@ static int get_chunk_event_type(struct flb_input_instance *ins, msgpack_object o
return type;
}

static int is_gzip_compressed(msgpack_object options)
static int get_compression_type(msgpack_object options)
{
int i;
msgpack_object k;
msgpack_object v;
msgpack_object k, v;

if (options.type != MSGPACK_OBJECT_MAP) {
return -1;
}


for (i = 0; i < options.via.map.size; i++) {
k = options.via.map.ptr[i].key;
v = options.via.map.ptr[i].val;

if (k.type != MSGPACK_OBJECT_STR) {
return -1;
}

if (k.via.str.size != 10) {
continue;
}

if (strncmp(k.via.str.ptr, "compressed", 10) == 0) {
if (v.type != MSGPACK_OBJECT_STR) {
return -1;
}

if (v.via.str.size != 4) {
return -1;
}

if (strncmp(v.via.str.ptr, "gzip", 4) == 0) {
return FLB_TRUE;
}
else if (strncmp(v.via.str.ptr, "text", 4) == 0) {
return FLB_FALSE;
if (k.type == MSGPACK_OBJECT_STR && k.via.str.size == 10 &&
strncmp(k.via.str.ptr, "compressed", 10) == 0) {
if (v.type == MSGPACK_OBJECT_STR) {
if (v.via.str.size == 4 && strncmp(v.via.str.ptr, "gzip", 4) == 0) {
return FLB_COMPRESSION_ALGORITHM_GZIP;
}
if (v.via.str.size == 4 && strncmp(v.via.str.ptr, "zstd", 4) == 0) {
return FLB_COMPRESSION_ALGORITHM_ZSTD;
}
}

return -1;
}
}

return FLB_FALSE;
return FLB_COMPRESSION_ALGORITHM_NONE;
}

static inline void print_msgpack_error_code(struct flb_input_instance *in,
Expand Down Expand Up @@ -1246,6 +1229,18 @@ int fw_prot_secure_forward_handshake(struct flb_input_instance *ins,
return -1;
}

static int sniff_magic(const uint8_t *p, size_t n) {
if (n >= 2 && p[0]==0x1f && p[1]==0x8b) {
return FLB_COMPRESSION_ALGORITHM_GZIP;
}
if (n >= 4 && p[0]==0x28 && p[1]==0xb5 &&
p[2]==0x2f && p[3]==0xfd) {
return FLB_COMPRESSION_ALGORITHM_ZSTD;
}

return FLB_COMPRESSION_ALGORITHM_NONE;
}

int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
{
int ret;
Expand All @@ -1259,8 +1254,6 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
flb_sds_t out_tag = NULL;
size_t bytes;
size_t recv_len;
size_t gz_size;
void *gz_data;
msgpack_object tag;
msgpack_object entry;
msgpack_object map;
Expand Down Expand Up @@ -1524,93 +1517,124 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
}

if (data) {
ret = is_gzip_compressed(root.via.array.ptr[2]);
if (ret == -1) {
flb_plg_error(ctx->ins, "invalid 'compressed' option");
msgpack_unpacked_destroy(&result);
msgpack_unpacker_free(unp);
flb_sds_destroy(out_tag);
return -1;
/* Get event type early for use in both compressed/uncompressed paths */
event_type = FLB_EVENT_TYPE_LOGS;
if (contain_options) {
ret = get_chunk_event_type(ins, root.via.array.ptr[2]);
if (ret == -1) {
flb_plg_error(ctx->ins, "invalid chunk event type");
msgpack_unpacked_destroy(&result);
flb_sds_destroy(out_tag);
msgpack_unpacker_free(unp);
return -1;
}
event_type = ret;
}

if (ret == FLB_TRUE) {
size_t remaining = len;

while (remaining > 0) {
ret = flb_gzip_uncompress_multi((void *) (data + (len - remaining)), remaining,
&gz_data, &gz_size, &remaining);
/* Initialize decompressor on first compressed chunk */
if (conn->d_ctx == NULL && contain_options) {
int opt_type = contain_options ? get_compression_type(root.via.array.ptr[2]) : FLB_COMPRESSION_ALGORITHM_NONE;
int sniff = sniff_magic((const uint8_t *)data, len);
int type = opt_type;

if (sniff != FLB_COMPRESSION_ALGORITHM_NONE &&
opt_type != FLB_COMPRESSION_ALGORITHM_NONE &&
sniff != opt_type) {
flb_plg_warn(ctx->ins, "compressed=%s but magic says %s; using magic",
opt_type == FLB_COMPRESSION_ALGORITHM_ZSTD ? "zstd" : "gzip",
sniff == FLB_COMPRESSION_ALGORITHM_ZSTD ? "zstd" : "gzip");
type = sniff;
}
else if (opt_type == FLB_COMPRESSION_ALGORITHM_NONE) {
type = sniff;
}
if (type > 0) {
conn->compression_type = type;
conn->d_ctx = flb_decompression_context_create(
conn->compression_type,
FLB_DECOMPRESSION_BUFFER_SIZE);
if (!conn->d_ctx) {
flb_plg_error(ctx->ins, "failed to create decompression context");

goto cleanup_msgpack;
}
}
}

if (ret == -1) {
flb_plg_error(ctx->ins, "gzip uncompress failure");
msgpack_unpacked_destroy(&result);
msgpack_unpacker_free(unp);
flb_sds_destroy(out_tag);
return -1;
if (conn->compression_type != FLB_COMPRESSION_ALGORITHM_NONE) {
char *decomp_buf = NULL;
uint8_t *append_ptr;
size_t available_space;
size_t decomp_len;
int decomp_ret;
size_t required_size;

available_space = flb_decompression_context_get_available_space(conn->d_ctx);
if (len > available_space) {
required_size = conn->d_ctx->input_buffer_length + len;
if (flb_decompression_context_resize_buffer(conn->d_ctx, required_size) != 0) {
flb_plg_error(ctx->ins, "cannot resize decompression buffer");

goto cleanup_decompress;
}
}
append_ptr = flb_decompression_context_get_append_buffer(conn->d_ctx);
memcpy(append_ptr, data, len);
conn->d_ctx->input_buffer_length += len;

decomp_buf = flb_malloc(ctx->buffer_chunk_size);
if (!decomp_buf) {
flb_errno();

goto cleanup_decompress;
}

do {
decomp_len = ctx->buffer_chunk_size;
decomp_ret = flb_decompress(conn->d_ctx, decomp_buf, &decomp_len);

if (decomp_ret == FLB_DECOMPRESSOR_FAILURE) {
if (decomp_len > 0) {
flb_plg_error(ctx->ins, "decompression failed, data may be corrupt");
flb_free(decomp_buf);

event_type = FLB_EVENT_TYPE_LOGS;
if (contain_options) {
ret = get_chunk_event_type(ins, root.via.array.ptr[2]);
if (ret == -1) {
msgpack_unpacked_destroy(&result);
msgpack_unpacker_free(unp);
flb_sds_destroy(out_tag);
flb_free(gz_data);
return -1;
goto cleanup_decompress;
}
event_type = ret;
break;
}

ret = append_log(ins, conn,
event_type,
out_tag, gz_data, gz_size);
if (ret == -1) {
msgpack_unpacked_destroy(&result);
msgpack_unpacker_free(unp);
flb_sds_destroy(out_tag);
flb_free(gz_data);
return -1;
if (decomp_len > 0) {
if (append_log(ins, conn, event_type, out_tag, decomp_buf, decomp_len) == -1) {
flb_free(decomp_buf);

goto cleanup_decompress;
}
}
flb_free(gz_data);
}
} while (decomp_len > 0);

flb_free(decomp_buf);

flb_decompression_context_destroy(conn->d_ctx);
conn->d_ctx = NULL;
conn->compression_type = FLB_COMPRESSION_ALGORITHM_NONE;
}
else {
event_type = FLB_EVENT_TYPE_LOGS;
if (contain_options) {
ret = get_chunk_event_type(ins, root.via.array.ptr[2]);
if (ret == -1) {
msgpack_unpacked_destroy(&result);
msgpack_unpacker_free(unp);
flb_sds_destroy(out_tag);
return -1;
}
event_type = ret;
}

ret = append_log(ins, conn,
event_type,
out_tag, data, len);
if (ret == -1) {
msgpack_unpacked_destroy(&result);
msgpack_unpacker_free(unp);
flb_sds_destroy(out_tag);
return -1;
if (append_log(ins, conn, event_type, out_tag, data, len) == -1) {
goto cleanup_msgpack;
}
}
}

/* Handle ACK response */
if (chunk_id != -1) {
chunk = root.via.array.ptr[2].via.map.ptr[chunk_id].val;
send_ack(ctx->ins, conn, chunk);
}
/* Handle ACK response (common to all paths) */
if (chunk_id != -1) {
chunk = root.via.array.ptr[2].via.map.ptr[chunk_id].val;
send_ack(ctx->ins, conn, chunk);
}
}
else {
flb_plg_warn(ctx->ins, "invalid data format, type=%i",
entry.type);
msgpack_unpacked_destroy(&result);
msgpack_unpacker_free(unp);
return -1;
goto cleanup_msgpack;
}

ret = msgpack_unpacker_next(unp, &result);
Expand All @@ -1637,4 +1661,16 @@ int fw_prot_process(struct flb_input_instance *ins, struct fw_conn *conn)
};

return 0;

cleanup_decompress:
flb_decompression_context_destroy(conn->d_ctx);
conn->d_ctx = NULL;
conn->compression_type = FLB_COMPRESSION_ALGORITHM_NONE;
/* FALLTHRU */
cleanup_msgpack:
msgpack_unpacked_destroy(&result);
msgpack_unpacker_free(unp);
flb_sds_destroy(out_tag);

return -1;
}
Loading
Loading