diff --git a/plugins/in_tail/tail.c b/plugins/in_tail/tail.c index 4af656f4eb3..5e4930fabcb 100644 --- a/plugins/in_tail/tail.c +++ b/plugins/in_tail/tail.c @@ -396,13 +396,15 @@ static int in_tail_init(struct flb_input_instance *in, } #endif - /* - * After the first scan (on start time), all new files discovered needs to be - * read from head, so we switch the 'read_from_head' flag to true so any - * other file discovered after a scan or a rotation are read from the - * beginning. - */ - ctx->read_from_head = FLB_TRUE; + if (ctx->read_newly_discovered_files_from_head) { + /* + * After the first scan (on start time), all new files discovered needs to be + * read from head, so we switch the 'read_from_head' flag to true so any + * other file discovered after a scan or a rotation are read from the + * beginning. + */ + ctx->read_from_head = FLB_TRUE; + } /* Set plugin context */ flb_input_set_context(in, ctx); @@ -594,6 +596,12 @@ static struct flb_config_map config_map[] = { "For new discovered files on start (without a database offset/position), read the " "content from the head of the file, not tail." }, + { + FLB_CONFIG_MAP_BOOL, "read_newly_discovered_files_from_head", "true", + 0, FLB_TRUE, offsetof(struct flb_tail_config, read_newly_discovered_files_from_head), + "For new discovered files after start (without a database offset/position), read the " + "content from the head of the file, not tail." + }, { FLB_CONFIG_MAP_STR, "refresh_interval", "60", 0, FLB_FALSE, 0, diff --git a/plugins/in_tail/tail_config.c b/plugins/in_tail/tail_config.c index 6f32cce2024..37877680be3 100644 --- a/plugins/in_tail/tail_config.c +++ b/plugins/in_tail/tail_config.c @@ -235,6 +235,14 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins, return NULL; } + /* hash table for files lookups */ + ctx->ignored_file_sizes = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 1000, 0); + if (ctx->ignored_file_sizes == NULL) { + flb_plg_error(ctx->ins, "could not create ignored file size hash table"); + flb_tail_config_destroy(ctx); + return NULL; + } + #ifdef FLB_HAVE_SQLDB ctx->db = NULL; #endif @@ -463,10 +471,16 @@ int flb_tail_config_destroy(struct flb_tail_config *config) if (config->static_hash) { flb_hash_table_destroy(config->static_hash); } + if (config->event_hash) { flb_hash_table_destroy(config->event_hash); } + if (config->ignored_file_sizes != NULL) { + flb_hash_table_destroy(config->ignored_file_sizes); + } + flb_free(config); + return 0; } diff --git a/plugins/in_tail/tail_config.h b/plugins/in_tail/tail_config.h index dc2f6480a08..1c596c88c8d 100644 --- a/plugins/in_tail/tail_config.h +++ b/plugins/in_tail/tail_config.h @@ -82,6 +82,7 @@ struct flb_tail_config { #endif int refresh_interval_sec; /* seconds to re-scan */ long refresh_interval_nsec;/* nanoseconds to re-scan */ + int read_newly_discovered_files_from_head; /* read new files from head after startup */ int read_from_head; /* read new files from head */ int rotate_wait; /* sec to wait on rotated files */ int watcher_interval; /* watcher interval */ @@ -162,6 +163,8 @@ struct flb_tail_config { struct flb_hash_table *static_hash; struct flb_hash_table *event_hash; + struct flb_hash_table *ignored_file_sizes; + struct flb_config *config; }; diff --git a/plugins/in_tail/tail_file.c b/plugins/in_tail/tail_file.c index c20594c7f1c..65da177d6d0 100644 --- a/plugins/in_tail/tail_file.c +++ b/plugins/in_tail/tail_file.c @@ -868,13 +868,24 @@ static int set_file_position(struct flb_tail_config *ctx, return 0; } - /* tail... */ - ret = lseek(file->fd, 0, SEEK_END); - if (ret == -1) { - flb_errno(); - return -1; + if (file->offset > 0) { + ret = lseek(file->fd, file->offset, SEEK_SET); + + if (ret == -1) { + flb_errno(); + return -1; + } + } + else { + ret = lseek(file->fd, 0, SEEK_END); + + if (ret == -1) { + flb_errno(); + return -1; + } + + file->offset = ret; } - file->offset = ret; if (file->decompression_context == NULL) { file->stream_offset = ret; @@ -923,6 +934,7 @@ static int ml_flush_callback(struct flb_ml_parser *parser, } int flb_tail_file_append(char *path, struct stat *st, int mode, + ssize_t offset, struct flb_tail_config *ctx) { int fd; @@ -1012,6 +1024,10 @@ int flb_tail_file_append(char *path, struct stat *st, int mode, file->mult_flush_timeout = 0; file->mult_skipping = FLB_FALSE; + if (offset != -1) { + file->offset = offset; + } + if (strlen(path) >= 3 && strcasecmp(&path[strlen(path) - 3], ".gz") == 0) { file->decompression_context = @@ -1894,7 +1910,7 @@ int flb_tail_file_rotated(struct flb_tail_file *file) ret = stat(tmp, &st); if (ret == 0 && st.st_ino != file->inode) { if (flb_tail_file_exists(&st, ctx) == FLB_FALSE) { - ret = flb_tail_file_append(tmp, &st, FLB_TAIL_STATIC, ctx); + ret = flb_tail_file_append(tmp, &st, FLB_TAIL_STATIC, -1, ctx); if (ret == -1) { flb_tail_scan(ctx->path_list, ctx); } diff --git a/plugins/in_tail/tail_file.h b/plugins/in_tail/tail_file.h index b308bb68066..46b250394fd 100644 --- a/plugins/in_tail/tail_file.h +++ b/plugins/in_tail/tail_file.h @@ -119,6 +119,7 @@ int flb_tail_file_name_dup(char *path, struct flb_tail_file *file); int flb_tail_file_to_event(struct flb_tail_file *file); int flb_tail_file_chunk(struct flb_tail_file *file); int flb_tail_file_append(char *path, struct stat *st, int mode, + ssize_t offset, struct flb_tail_config *ctx); void flb_tail_file_remove(struct flb_tail_file *file); int flb_tail_file_remove_all(struct flb_tail_config *ctx); diff --git a/plugins/in_tail/tail_scan.c b/plugins/in_tail/tail_scan.c index dfcc79309e5..5b2dbc58442 100644 --- a/plugins/in_tail/tail_scan.c +++ b/plugins/in_tail/tail_scan.c @@ -30,6 +30,30 @@ #include "tail_scan_glob.c" #endif +void flb_tail_scan_register_ignored_file_size(struct flb_tail_config *ctx, const char *path, size_t path_length, size_t size) +{ + flb_hash_table_add(ctx->ignored_file_sizes, path, path_length, (void *) size, 0); + +} + +void flb_tail_scan_unregister_ignored_file_size(struct flb_tail_config *ctx, const char *path, size_t path_length) +{ + flb_hash_table_del(ctx->ignored_file_sizes, path); +} + +ssize_t flb_tail_scan_fetch_ignored_file_size(struct flb_tail_config *ctx, const char *path, size_t path_length) +{ + ssize_t result; + + result = (ssize_t) flb_hash_table_get_ptr(ctx->ignored_file_sizes, path, path_length); + + if (result == 0) { + result = -1; + } + + return result; +} + int flb_tail_scan(struct mk_list *path_list, struct flb_tail_config *ctx) { int ret; diff --git a/plugins/in_tail/tail_scan.h b/plugins/in_tail/tail_scan.h index 2509b79fc21..53270dcb1bc 100644 --- a/plugins/in_tail/tail_scan.h +++ b/plugins/in_tail/tail_scan.h @@ -26,4 +26,8 @@ int flb_tail_scan(struct mk_list *path, struct flb_tail_config *ctx); int flb_tail_scan_callback(struct flb_input_instance *ins, struct flb_config *config, void *context); +void flb_tail_scan_register_ignored_file_size(struct flb_tail_config *ctx, const char *path, size_t path_length, size_t size); +void flb_tail_scan_unregister_ignored_file_size(struct flb_tail_config *ctx, const char *path, size_t path_length); +ssize_t flb_tail_scan_fetch_ignored_file_size(struct flb_tail_config *ctx, const char *path, size_t path_length); + #endif diff --git a/plugins/in_tail/tail_scan_glob.c b/plugins/in_tail/tail_scan_glob.c index 603d9a514d8..60259880ab3 100644 --- a/plugins/in_tail/tail_scan_glob.c +++ b/plugins/in_tail/tail_scan_glob.c @@ -183,6 +183,7 @@ static inline int do_glob(const char *pattern, int flags, return ret; } + /* Scan a path, register the entries and return how many */ static int tail_scan_path(const char *path, struct flb_tail_config *ctx) { @@ -193,6 +194,9 @@ static int tail_scan_path(const char *path, struct flb_tail_config *ctx) time_t now; int64_t mtime; struct stat st; + ssize_t ignored_file_size; + + ignored_file_size = -1; flb_plg_debug(ctx->ins, "scanning path %s", path); @@ -245,14 +249,36 @@ static int tail_scan_path(const char *path, struct flb_tail_config *ctx) if ((now - ctx->ignore_older) > mtime) { flb_plg_debug(ctx->ins, "excluded=%s (ignore_older)", globbuf.gl_pathv[i]); + + flb_tail_scan_register_ignored_file_size( + ctx, + globbuf.gl_pathv[i], + strlen(globbuf.gl_pathv[i]), + st.st_size); + continue; } } } + if (ctx->ignore_older > 0) { + ignored_file_size = flb_tail_scan_fetch_ignored_file_size( + ctx, + globbuf.gl_pathv[i], + strlen(globbuf.gl_pathv[i])); + + flb_tail_scan_unregister_ignored_file_size( + ctx, + globbuf.gl_pathv[i], + strlen(globbuf.gl_pathv[i])); + } + /* Append file to list */ ret = flb_tail_file_append(globbuf.gl_pathv[i], &st, - FLB_TAIL_STATIC, ctx); + FLB_TAIL_STATIC, + ignored_file_size, + ctx); + if (ret == 0) { flb_plg_debug(ctx->ins, "scan_glob add(): %s, inode %" PRIu64, globbuf.gl_pathv[i], (uint64_t) st.st_ino); diff --git a/plugins/in_tail/tail_scan_win32.c b/plugins/in_tail/tail_scan_win32.c index 2cc969bfdf9..cc4d138780f 100644 --- a/plugins/in_tail/tail_scan_win32.c +++ b/plugins/in_tail/tail_scan_win32.c @@ -65,6 +65,9 @@ static int tail_register_file(const char *target, struct flb_tail_config *ctx, int64_t mtime; struct stat st; char path[MAX_PATH]; + ssize_t ignored_file_size; + + ignored_file_size = -1; if (_fullpath(path, target, MAX_PATH) == NULL) { flb_plg_error(ctx->ins, "cannot get absolute path of %s", target); @@ -81,6 +84,13 @@ static int tail_register_file(const char *target, struct flb_tail_config *ctx, if ((ts - ctx->ignore_older) > mtime) { flb_plg_debug(ctx->ins, "excluded=%s (ignore_older)", target); + + flb_tail_scan_register_ignored_file_size( + ctx, + path, + strlen(path), + st.st_size); + return -1; } } @@ -91,7 +101,19 @@ static int tail_register_file(const char *target, struct flb_tail_config *ctx, return -1; } - return flb_tail_file_append(path, &st, FLB_TAIL_STATIC, ctx); + if (ctx->ignore_older > 0) { + ignored_file_size = flb_tail_scan_fetch_ignored_file_size( + ctx, + path, + strlen(path)); + + flb_tail_scan_unregister_ignored_file_size( + ctx, + path, + strlen(path)); + } + + return flb_tail_file_append(path, &st, FLB_TAIL_STATIC, ignored_file_size, ctx); } /*