Merge pull request #2 from fluent/master

Fluent-bit from base as of Oct 13, 2020

Author: tpetrov-lp

CMakeLists.txt

@@ -3,7 +3,7 @@ project(fluent-bit)
 
 # Fluent Bit Version
 set(FLB_VERSION_MAJOR  1)
-set(FLB_VERSION_MINOR  6)
+set(FLB_VERSION_MINOR  7)
 set(FLB_VERSION_PATCH  0)
 set(FLB_VERSION_STR "${FLB_VERSION_MAJOR}.${FLB_VERSION_MINOR}.${FLB_VERSION_PATCH}")
 
@@ -191,6 +191,7 @@ option(FLB_FILTER_THROTTLE_SIZE "Enable throttle size filter"        No)
 option(FLB_FILTER_NEST        "Enable nest filter"                  Yes)
 option(FLB_FILTER_LUA         "Enable Lua scripting filter"         Yes)
 option(FLB_FILTER_RECORD_MODIFIER "Enable record_modifier filter"   Yes)
+option(FLB_FILTER_TENSORFLOW  "Enable tensorflow filter"             No)
 
 # Debug callbacks
 option(FLB_HTTP_CLIENT_DEBUG  "Enable HTTP Client debug callbacks"   No)

Dockerfile

@@ -2,9 +2,9 @@ FROM debian:buster as builder
 
 # Fluent Bit version
 ENV FLB_MAJOR 1
-ENV FLB_MINOR 6
+ENV FLB_MINOR 7
 ENV FLB_PATCH 0
-ENV FLB_VERSION 1.6.0
+ENV FLB_VERSION 1.7.0
 
 ENV DEBIAN_FRONTEND noninteractive
 

conf/fluent-bit.conf

@@ -48,7 +48,7 @@
     # storage metrics
     # ---------------
     # publish storage pipeline metrics in '/api/v1/storage'. The metrics are
-    # exported only if the 'http_server' optoin is enabled.
+    # exported only if the 'http_server' option is enabled.
     #
     storage.metrics on
 

documentation/fluentbit_users.png

@@ -16,7 +16,7 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=2ee41112a44fe7014dce33e26468ba93"
 SECTION = "net"
 
 PR = "r0"
-PV = "1.6.0"
+PV = "1.7.0"
 
 SRCREV = "v${PV}"
 SRC_URI = "git://github.com/fluent/fluent-bit.git;nobranch=1"

fluent-bit_1.6.0.bb renamed to fluent-bit_1.7.0.bb

@@ -130,8 +130,6 @@ struct flb_aws_client_generator *flb_aws_client_generator();
  */
 char *flb_aws_endpoint(char* service, char* region);
 
-char *flb_s3_endpoint(char* bucket, char* region);
-
 /* Parses AWS XML API Error responses and returns the value of the <code> tag */
 flb_sds_t flb_aws_xml_error(char *response, size_t response_len);
 

include/fluent-bit/flb_aws_util.h

@@ -28,6 +28,9 @@
 #define FLB_FSTORE_LOCK        222
 #define FLB_FSTORE_UNLOCK      333
 
+#define FLB_FSTORE_MEM         CIO_STORE_MEM
+#define FLB_FSTORE_FS          CIO_STORE_FS
+
 struct flb_fstore_file {
     flb_sds_t name;                 /* file name */
 
@@ -40,25 +43,27 @@ struct flb_fstore_file {
 };
 
 struct flb_fstore_stream {
-    struct cio_stream *stream;      /* Chunk I/O stream context */
+    char *name;                     /* reference to stream->name */
     flb_sds_t path;                 /* stream filesystem path */
+    struct cio_stream *stream;      /* Chunk I/O stream context */
     struct mk_list files;
     struct mk_list _head;
 };
 
 struct flb_fstore {
+    int store_type;
     char *root_path;
     struct cio_ctx *cio;            /* Chunk I/O context */
     struct mk_list streams;
 };
 
-struct flb_fstore *flb_fstore_create(char *path);
+struct flb_fstore *flb_fstore_create(char *path, int store_type);
 
 int flb_fstore_destroy(struct flb_fstore *fs);
 
 struct flb_fstore_stream *flb_fstore_stream_create(struct flb_fstore *fs,
                                                    char *stream_name);
-void flb_fstore_stream_destroy(struct flb_fstore_stream *stream);
+void flb_fstore_stream_destroy(struct flb_fstore_stream *stream, int delete);
 
 int flb_fstore_file_meta_set(struct flb_fstore *fs,
                              struct flb_fstore_file *fsf,
@@ -80,5 +85,6 @@ int flb_fstore_file_inactive(struct flb_fstore *fs,
                              struct flb_fstore_file *fsf);
 int flb_fstore_file_delete(struct flb_fstore *fs,
                            struct flb_fstore_file *fsf);
+void flb_fstore_dump(struct flb_fstore *fs);
 
 #endif

include/fluent-bit/flb_fstore.h

@@ -0,0 +1,48 @@
+/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
+
+/*  Fluent Bit
+ *  ==========
+ *  Copyright (C) 2020      The Fluent Bit Authors
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/* Fluent Bit intermediate metric representation */
+
+#ifndef FLB_INTERMEDIATE_METRIC_H
+#define FLB_INTERMEDIATE_METRIC_H
+
+#include <fluent-bit/flb_time.h>
+#include <monkey/mk_core.h>
+#include <msgpack.h>
+
+/* Metric Type- Gague or Counter */
+#define GAUGE 1
+#define COUNTER 2
+
+/* Metric Unit */
+#define PERCENT "Percent"
+#define BYTES "Bytes"
+
+struct flb_intermediate_metric
+{
+    msgpack_object key;
+    msgpack_object value;
+    int metric_type;
+    const char *metric_unit;
+    struct flb_time timestamp;
+
+    struct mk_list _head;
+};
+
+#endif

include/fluent-bit/flb_intermediate_metric.h

@@ -32,6 +32,7 @@ struct cio_stream {
 
 struct cio_stream *cio_stream_create(struct cio_ctx *ctx, const char *name,
                                      int type);
+int cio_stream_delete(struct cio_stream *st);
 void cio_stream_destroy(struct cio_stream *st);
 void cio_stream_destroy_all(struct cio_ctx *ctx);
 

lib/chunkio/include/chunkio/cio_stream.h

@@ -95,6 +95,12 @@ int cio_file_read_prepare(struct cio_ctx *ctx, struct cio_chunk *ch)
     return -1;
 }
 
+int cio_file_content_copy(struct cio_chunk *ch,
+                          void **out_buf, size_t *out_size)
+{
+    return -1;
+}
+
 int cio_file_is_up(struct cio_chunk *ch, struct cio_file *cf)
 {
     return CIO_FALSE;