diff --git a/lib/cmetrics/.dockerignore b/lib/cmetrics/.dockerignore new file mode 100644 index 00000000000..ff3d1ae0c6a --- /dev/null +++ b/lib/cmetrics/.dockerignore @@ -0,0 +1,3 @@ +.git +build +lib/cfl/build-review diff --git a/lib/cmetrics/.github/workflows/build.yaml b/lib/cmetrics/.github/workflows/build.yaml index efe8baf74ad..2724c5acd26 100644 --- a/lib/cmetrics/.github/workflows/build.yaml +++ b/lib/cmetrics/.github/workflows/build.yaml @@ -38,50 +38,25 @@ jobs: build-centos: name: CentOS 7 build to confirm no issues once used downstream runs-on: ubuntu-latest - container: centos:7 - env: - # workaround required for checkout@v3, https://github.com/actions/checkout/issues/1590 - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + permissions: + contents: read steps: - - name: Set up base image dependencies - run: | - sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo - sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo - yum -y update - yum install -y ca-certificates gcc gcc-c++ git make wget - yum install -y epel-release - yum install -y libcurl-devel - - shell: bash - - - name: Install CMake 3.20.0 - run: | - CMAKE_VERSION=3.20.0 - wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh - chmod +x cmake-${CMAKE_VERSION}-linux-x86_64.sh - ./cmake-${CMAKE_VERSION}-linux-x86_64.sh --skip-license --prefix=/usr/local - ln -sf /usr/local/bin/cmake /usr/bin/cmake - cmake --version - - - name: Clone repo without submodules (1.8.3 version of Git) - run: | - git clone https://github.com/fluent/cmetrics.git - shell: bash + - name: Check out the repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + submodules: recursive - - name: Check out the branch (1.8.3 version of Git) - env: - BRANCH_NAME: ${{ github.head_ref }} - run: | - git checkout "$BRANCH_NAME" - git submodule update --init --recursive - shell: bash - working-directory: cmetrics + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 - - name: Run compilation - run: | - cmake -DCMT_TESTS=on -DCMT_DEV=on . - make - working-directory: cmetrics + - name: Build sources on CentOS 7 + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ./dockerfiles/Dockerfile.centos7 + push: false + load: false + provenance: false build-debian: name: Debian Buster build to confirm no issues once used downstream diff --git a/lib/cmetrics/CMakeLists.txt b/lib/cmetrics/CMakeLists.txt index d7da71a795b..80fa4e905cf 100644 --- a/lib/cmetrics/CMakeLists.txt +++ b/lib/cmetrics/CMakeLists.txt @@ -5,8 +5,8 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version set(CMT_VERSION_MAJOR 2) -set(CMT_VERSION_MINOR 1) -set(CMT_VERSION_PATCH 5) +set(CMT_VERSION_MINOR 2) +set(CMT_VERSION_PATCH 0) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers @@ -45,9 +45,12 @@ if(CMAKE_SYSTEM_NAME MATCHES "Windows") set(CMT_SYSTEM_WINDOWS On) add_definitions(-DCMT_SYSTEM_WINDOWS) - # Disable unistd.h for flex/bison - CMT_DEFINITION(YY_NO_UNISTD_H) - message(STATUS "Specifying YY_NO_UNISTD_H") + # Disable unistd.h for flex/bison on MSVC only: MinGW provides a real + # unistd.h, and without it the generated lexer has no isatty declaration. + if(MSVC) + CMT_DEFINITION(YY_NO_UNISTD_H) + message(STATUS "Specifying YY_NO_UNISTD_H") + endif() endif() # Define macro to identify macOS system diff --git a/lib/cmetrics/dockerfiles/Dockerfile.centos7 b/lib/cmetrics/dockerfiles/Dockerfile.centos7 new file mode 100644 index 00000000000..7793c4ec0b7 --- /dev/null +++ b/lib/cmetrics/dockerfiles/Dockerfile.centos7 @@ -0,0 +1,22 @@ +# This image is used only to verify that cmetrics builds on CentOS 7. +FROM centos:7 + +RUN sed -i -e "s/^mirrorlist=http:\/\/mirrorlist.centos.org/#mirrorlist=http:\/\/mirrorlist.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \ + sed -i -e "s/^#baseurl=http:\/\/mirror.centos.org/baseurl=http:\/\/vault.centos.org/g" /etc/yum.repos.d/CentOS-Base.repo && \ + yum -y update && \ + yum install -y ca-certificates gcc gcc-c++ make wget libcurl-devel && \ + yum clean all + +ARG CMAKE_VERSION="3.20.0" + +RUN wget -q "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.sh" \ + -O /tmp/cmake-installer.sh && \ + chmod +x /tmp/cmake-installer.sh && \ + /tmp/cmake-installer.sh --skip-license --prefix=/usr/local && \ + rm -f /tmp/cmake-installer.sh + +COPY . /src/ +WORKDIR /src/build + +RUN cmake -DCMT_TESTS=On -DCMT_DEV=On .. && \ + cmake --build . -j "$(getconf _NPROCESSORS_ONLN)" diff --git a/lib/cmetrics/include/cmetrics/cmetrics.h b/lib/cmetrics/include/cmetrics/cmetrics.h index 4b5307846c5..04638c77754 100644 --- a/lib/cmetrics/include/cmetrics/cmetrics.h +++ b/lib/cmetrics/include/cmetrics/cmetrics.h @@ -80,5 +80,6 @@ struct cmt *cmt_create(); void cmt_destroy(struct cmt *cmt); int cmt_label_add(struct cmt *cmt, char *key, char *val); char *cmt_version(); +void cmt_expire(struct cmt *cmt, uint64_t expiration); #endif diff --git a/lib/cmetrics/include/cmetrics/cmt_map.h b/lib/cmetrics/include/cmetrics/cmt_map.h index f67ee15547b..00ab7ae6800 100644 --- a/lib/cmetrics/include/cmetrics/cmt_map.h +++ b/lib/cmetrics/include/cmetrics/cmt_map.h @@ -57,6 +57,9 @@ int cmt_map_metric_get_val(struct cmt_opts *opts, struct cmt_map *map, double *out_val); void cmt_map_metric_destroy(struct cmt_metric *metric); +void cmt_map_metrics_expire(struct cmt_map *, uint64_t); + void destroy_label_list(struct cfl_list *label_list); + #endif diff --git a/lib/cmetrics/src/cmetrics.c b/lib/cmetrics/src/cmetrics.c index a2bb0ffd1b3..025cd99032f 100644 --- a/lib/cmetrics/src/cmetrics.c +++ b/lib/cmetrics/src/cmetrics.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -144,6 +145,61 @@ void cmt_destroy(struct cmt *cmt) free(cmt); } +void cmt_expire(struct cmt *cmt, uint64_t expiration) +{ + struct cfl_list *tmp; + struct cfl_list *head; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_summary *summary; + struct cmt_histogram *histogram; + struct cmt_untyped *untyped; + struct cmt_exp_histogram *exp_histogram; + + if (cmt == NULL) { + return; + } + + /* Do a first pass for all regular metrics: + * * counters + * * gauges + * * summaries + * * histograms + * * untypeds + */ + cfl_list_foreach_safe(head, tmp, &cmt->counters) { + counter = cfl_list_entry(head, struct cmt_counter, _head); + cmt_map_metrics_expire(counter->map, expiration); + } + + cfl_list_foreach_safe(head, tmp, &cmt->gauges) { + gauge = cfl_list_entry(head, struct cmt_gauge, _head); + cmt_map_metrics_expire(gauge->map, expiration); + } + + cfl_list_foreach_safe(head, tmp, &cmt->summaries) { + summary = cfl_list_entry(head, struct cmt_summary, _head); + cmt_map_metrics_expire(summary->map, expiration); + } + + cfl_list_foreach_safe(head, tmp, &cmt->histograms) { + histogram = cfl_list_entry(head, struct cmt_histogram, _head); + cmt_map_metrics_expire(histogram->map, expiration); + } + + cfl_list_foreach_safe(head, tmp, &cmt->untypeds) { + untyped = cfl_list_entry(head, struct cmt_untyped, _head); + cmt_map_metrics_expire(untyped->map, expiration); + } + + /* Here we cover exp_histograms separetely. + */ + cfl_list_foreach_safe(head, tmp, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + cmt_map_metrics_expire(exp_histogram->map, expiration); + } +} + int cmt_label_add(struct cmt *cmt, char *key, char *val) { return cmt_labels_add_kv(cmt->static_labels, key, val); diff --git a/lib/cmetrics/src/cmt_decode_msgpack.c b/lib/cmetrics/src/cmt_decode_msgpack.c index 80256c20060..abd179570a4 100644 --- a/lib/cmetrics/src/cmt_decode_msgpack.c +++ b/lib/cmetrics/src/cmt_decode_msgpack.c @@ -1636,6 +1636,7 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co int result; struct cmt_summary *summary; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_counter *counter; struct cmt_msgpack_decode_context *decode_context; struct cmt_mpack_map_entry_callback_t callbacks[] = \ @@ -1684,6 +1685,12 @@ static int unpack_basic_type_meta(mpack_reader_t *reader, size_t index, void *co else { histogram->buckets = NULL; } + + histogram->aggregation_type = decode_context->aggregation_type; + } + else if (decode_context->map->type == CMT_EXP_HISTOGRAM) { + exp_histogram = (struct cmt_exp_histogram *) decode_context->map->parent; + exp_histogram->aggregation_type = decode_context->aggregation_type; } else if (decode_context->map->type == CMT_SUMMARY) { summary = (struct cmt_summary *) decode_context->map->parent; diff --git a/lib/cmetrics/src/cmt_decode_prometheus.c b/lib/cmetrics/src/cmt_decode_prometheus.c index 32b34babe55..25fb83e39c3 100644 --- a/lib/cmetrics/src/cmt_decode_prometheus.c +++ b/lib/cmetrics/src/cmt_decode_prometheus.c @@ -124,17 +124,19 @@ int cmt_decode_prometheus_create( result = cmt_decode_prometheus_parse(scanner, &context); + if (context.errcode) { + result = context.errcode; + } + if (result == 0) { *out_cmt = cmt; } else { cmt_destroy(cmt); - if (context.errcode) { - result = context.errcode; - } - reset_context(&context, true); } + reset_context(&context, true); + cmt_decode_prometheus__delete_buffer(buf, scanner); cmt_decode_prometheus_lex_destroy(scanner); @@ -1224,6 +1226,9 @@ static int cmt_decode_prometheus_error(void *yyscanner, struct cmt_decode_prometheus_context *context, const char *msg) { - report_error(context, CMT_DECODE_PROMETHEUS_SYNTAX_ERROR, msg); + if (!context->errcode) { + report_error(context, CMT_DECODE_PROMETHEUS_SYNTAX_ERROR, msg); + } + return 0; } diff --git a/lib/cmetrics/src/cmt_decode_prometheus.l b/lib/cmetrics/src/cmt_decode_prometheus.l index 8055f52deee..8f42f695154 100644 --- a/lib/cmetrics/src/cmt_decode_prometheus.l +++ b/lib/cmetrics/src/cmt_decode_prometheus.l @@ -7,11 +7,79 @@ %{ #include +#include #define STRBUF_RET \ yylval->str = context->strbuf; \ context->strbuf = NULL +static void set_allocation_error(struct cmt_decode_prometheus_context *context) +{ + context->errcode = CMT_DECODE_PROMETHEUS_ALLOCATION_ERROR; + + if (context->opts.errbuf != NULL && context->opts.errbuf_size > 0) { + snprintf(context->opts.errbuf, + context->opts.errbuf_size, + "memory allocation failed"); + } +} + +static int reset_strbuf(struct cmt_decode_prometheus_context *context) +{ + if (context->strbuf != NULL) { + cfl_sds_destroy(context->strbuf); + } + + context->strbuf = cfl_sds_create_size(256); + if (context->strbuf == NULL) { + set_allocation_error(context); + + return -1; + } + + return 0; +} + +static int append_strbuf(struct cmt_decode_prometheus_context *context, + const char *text, int length) +{ + cfl_sds_t result; + + result = cfl_sds_cat(context->strbuf, text, length); + if (result == NULL) { + set_allocation_error(context); + + return -1; + } + + context->strbuf = result; + + return 0; +} + +#define STRBUF_CREATE() \ + do { \ + if (reset_strbuf(context) != 0) { \ + return 0; \ + } \ + } while (0) + +#define STRBUF_APPEND(text, length) \ + do { \ + if (append_strbuf(context, (text), (length)) != 0) { \ + return 0; \ + } \ + } while (0) + +#define SET_STR_TOKEN() \ + do { \ + yylval->str = cfl_sds_create(yytext); \ + if (yylval->str == NULL) { \ + set_allocation_error(context); \ + return 0; \ + } \ + } while (0) + %} /* here we define some states that allow us to create rules only @@ -74,7 +142,7 @@ [^ \t]+ { // The next token will be the metric name - yylval->str = cfl_sds_create(yytext); + SET_STR_TOKEN(); return YYSTATE == HELPTAG ? HELP : TYPE; } @@ -86,7 +154,7 @@ // separate start condition for this to handle "\\" and "\n" escapes // more easily. BEGIN(INHELPTAG); - context->strbuf = cfl_sds_create_size(256); + STRBUF_CREATE(); } else { // For TYPETAG we enter INTYPETAG start condition to check only valid @@ -107,17 +175,17 @@ \\n { // Process linefeed escape sequence - context->strbuf = cfl_sds_cat(context->strbuf, "\n", 1); + STRBUF_APPEND("\n", 1); } \\\\ { // Process backslack escape sequence - context->strbuf = cfl_sds_cat(context->strbuf, "\\", 1); + STRBUF_APPEND("\\", 1); } [^\r\n\\]+ { // Put everything that is not a backslash or a line feed into strbuf - context->strbuf = cfl_sds_cat(context->strbuf, yytext, yyleng); + STRBUF_APPEND(yytext, yyleng); } counter { @@ -146,26 +214,23 @@ ["] { BEGIN(INQUOTE); - if (context->strbuf != NULL) { - cfl_sds_destroy(context->strbuf); - } - context->strbuf = cfl_sds_create_size(256); + STRBUF_CREATE(); } [\\]["] { - context->strbuf = cfl_sds_cat(context->strbuf, "\"", 1); + STRBUF_APPEND("\"", 1); } \\n { - context->strbuf = cfl_sds_cat(context->strbuf, "\n", 1); + STRBUF_APPEND("\n", 1); } \\\\ { - context->strbuf = cfl_sds_cat(context->strbuf, "\\", 1); + STRBUF_APPEND("\\", 1); } [^\r\n\\"]+ { - context->strbuf = cfl_sds_cat(context->strbuf, yytext, yyleng); + STRBUF_APPEND(yytext, yyleng); } ["] { @@ -180,7 +245,7 @@ } [a-zA-Z_][a-zA-Z_0-9]* { - yylval->str = cfl_sds_create(yytext); + SET_STR_TOKEN(); return IDENTIFIER; } diff --git a/lib/cmetrics/src/cmt_encode_msgpack.c b/lib/cmetrics/src/cmt_encode_msgpack.c index c9328055615..4341b9c651f 100644 --- a/lib/cmetrics/src/cmt_encode_msgpack.c +++ b/lib/cmetrics/src/cmt_encode_msgpack.c @@ -58,6 +58,7 @@ static void pack_header(mpack_writer_t *writer, struct cmt *cmt, struct cmt_map size_t index; struct cmt_summary *summary = NULL; struct cmt_histogram *histogram = NULL; + struct cmt_exp_histogram *exp_histogram = NULL; struct cmt_counter *counter = NULL; size_t meta_field_count; @@ -67,6 +68,11 @@ static void pack_header(mpack_writer_t *writer, struct cmt *cmt, struct cmt_map if (map->type == CMT_HISTOGRAM) { histogram = (struct cmt_histogram *) map->parent; + meta_field_count += 2; + } + else if (map->type == CMT_EXP_HISTOGRAM) { + exp_histogram = (struct cmt_exp_histogram *) map->parent; + meta_field_count++; } else if (map->type == CMT_SUMMARY) { @@ -156,6 +162,15 @@ static void pack_header(mpack_writer_t *writer, struct cmt *cmt, struct cmt_map } mpack_finish_array(writer); + + /* aggregation_type */ + mpack_write_cstr(writer, "aggregation_type"); + mpack_write_int(writer, histogram->aggregation_type); + } + else if (map->type == CMT_EXP_HISTOGRAM) { + /* aggregation_type */ + mpack_write_cstr(writer, "aggregation_type"); + mpack_write_int(writer, exp_histogram->aggregation_type); } else if (map->type == CMT_SUMMARY) { /* 'quantiles' (summary quantiles) */ diff --git a/lib/cmetrics/src/cmt_encode_prometheus.c b/lib/cmetrics/src/cmt_encode_prometheus.c index e144a1aa5f4..b8473182338 100644 --- a/lib/cmetrics/src/cmt_encode_prometheus.c +++ b/lib/cmetrics/src/cmt_encode_prometheus.c @@ -18,6 +18,7 @@ */ #include +#include #include #include @@ -407,7 +408,12 @@ static cfl_sds_t bucket_value_to_string(double val) } cfl_sds_len_set(str, len); - if (!strchr(str, '.') && !strchr(str, 'e') && !strchr(str, 'E')) { + /* + * Append .0 only when there is no decimal point and the number + * is finite and not in scientific notation. + */ + if (isfinite(val) && + !strchr(str, '.') && !strchr(str, 'e') && !strchr(str, 'E')) { cfl_sds_cat_safe(&str, ".0", 2); } diff --git a/lib/cmetrics/src/cmt_map.c b/lib/cmetrics/src/cmt_map.c index 19549d52b45..3442abda502 100644 --- a/lib/cmetrics/src/cmt_map.c +++ b/lib/cmetrics/src/cmt_map.c @@ -338,3 +338,19 @@ void destroy_label_list(struct cfl_list *label_list) free(label); } } + +/* This function can be used to expire untouched metrics. + */ +void cmt_map_metrics_expire(struct cmt_map *map, uint64_t expiration) +{ + struct cfl_list *tmp; + struct cfl_list *head; + struct cmt_metric *metric; + + cfl_list_foreach_safe(head, tmp, &map->metrics) { + metric = cfl_list_entry(head, struct cmt_metric, _head); + if (metric->timestamp < expiration) { + cmt_map_metric_destroy(metric); + } + } +} diff --git a/lib/cmetrics/src/cmt_untyped.c b/lib/cmetrics/src/cmt_untyped.c index 2df6ea1413a..fbe4cd4d291 100644 --- a/lib/cmetrics/src/cmt_untyped.c +++ b/lib/cmetrics/src/cmt_untyped.c @@ -96,7 +96,7 @@ int cmt_untyped_destroy(struct cmt_untyped *untyped) return 0; } -/* Set untyped value, new value cannot be smaller than current value */ +/* Set untyped value */ int cmt_untyped_set(struct cmt_untyped *untyped, uint64_t timestamp, double val, int labels_count, char **label_vals) { @@ -112,9 +112,6 @@ int cmt_untyped_set(struct cmt_untyped *untyped, uint64_t timestamp, double val, return -1; } - if (cmt_metric_get_value(metric) > val) { - return -1; - } cmt_metric_set(metric, timestamp, val); return 0; } diff --git a/lib/cmetrics/tests/CMakeLists.txt b/lib/cmetrics/tests/CMakeLists.txt index 40108aad95a..850e310cfda 100644 --- a/lib/cmetrics/tests/CMakeLists.txt +++ b/lib/cmetrics/tests/CMakeLists.txt @@ -15,6 +15,8 @@ set(UNIT_TESTS_FILES filter.c exp_histogram.c msgpack_abi.c + msgpack_temporality.c + expire.c ) if (CMT_BUILD_PROMETHEUS_TEXT_DECODER) diff --git a/lib/cmetrics/tests/data/issue_274.txt b/lib/cmetrics/tests/data/issue_274.txt new file mode 100644 index 00000000000..fb34a291767 --- /dev/null +++ b/lib/cmetrics/tests/data/issue_274.txt @@ -0,0 +1,3 @@ +# HELP node_zfs_arc_memory_available_bytes kstat.zfs.misc.arcstats.memory_available_bytes +# TYPE node_zfs_arc_memory_available_bytes untyped +node_zfs_arc_memory_available_bytes -8.58914944e+08 diff --git a/lib/cmetrics/tests/expire.c b/lib/cmetrics/tests/expire.c new file mode 100644 index 00000000000..94ebefc797f --- /dev/null +++ b/lib/cmetrics/tests/expire.c @@ -0,0 +1,321 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmt_tests.h" + +void test_expire_counter() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_counter *c; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create a counter metric type */ + c = cmt_counter_create(cmt, "k8s", "network", "uptime", "Network Uptime", 1, (char *[]) {"host"}); + TEST_CHECK(c != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_counter_inc(c, ts, 1, (char *[]){"valid"}); + cmt_counter_inc(c, ts-10, 1, (char *[]){"expire"}); + + TEST_CHECK(cfl_list_size(&c->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_gauge() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_gauge *g; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create a gauge metric type */ + g = cmt_gauge_create(cmt, "k8s", "network", "load", "Network load", 1, (char *[]) {"host"}); + TEST_CHECK(g != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_gauge_set(g, ts, 50, 1, (char *[]){"valid"}); + cmt_gauge_set(g, ts-10, 50, 1, (char *[]){"expire"}); + + TEST_CHECK(cfl_list_size(&g->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&g->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_histogram() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_histogram *h; + struct cmt_histogram_buckets *buckets; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create buckets */ + buckets = cmt_histogram_buckets_create(11, + 0.005, 0.01, 0.025, 0.05, + 0.1, 0.25, 0.5, 1.0, 2.5, + 5.0, 10.0); + TEST_CHECK(buckets != NULL); + + /* Create a histogram metric type */ + h = cmt_histogram_create(cmt, + "k8s", "network", "uptime", "Network Uptime", + buckets, + 1, (char *[]) {"host"}); + TEST_CHECK(h != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_histogram_observe(h, ts, 1.0, 1, (char *[]){"valid"}); + cmt_histogram_observe(h, ts-10, 1.0, 1, (char *[]){"expire"}); + + TEST_CHECK(cfl_list_size(&h->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&h->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_summary() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_summary *s; + double quantiles[6]; + double revised[6]; + double sum; + uint64_t count; + + /* set quantiles, no labels */ + quantiles[0] = 0.1; + quantiles[1] = 0.2; + quantiles[2] = 0.3; + quantiles[3] = 0.4; + quantiles[4] = 0.5; + quantiles[5] = 1.0; + + revised[0] = 1.0; + revised[1] = 2.0; + revised[2] = 3.0; + revised[3] = 4.0; + revised[4] = 5.0; + revised[5] = 6.0; + + count = 10; + sum = 51.612894511314444; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create a summary metric type */ + s = cmt_summary_create(cmt, + "k8s", "network", "uptime", "Network Uptime", + 6, + quantiles, + 1, (char *[]) {"host"}); + TEST_CHECK(s != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_summary_set_default(s, ts, revised, sum, count, 1, (char *[]){"valid"}); + cmt_summary_set_default(s, ts-10, revised, sum, count, 1, (char *[]){"expire"}); + + TEST_CHECK(cfl_list_size(&s->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&s->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_untyped() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_untyped *u; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + u = cmt_untyped_create(cmt, "cmetrics", "test", "cat_untyped", "first untyped", + 2, (char *[]) {"label5", "label6"}); + TEST_CHECK(u != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_untyped_set(u, ts, 1.3, 2, (char *[]) {"first", "valid"}); + cmt_untyped_set(u, ts-10, 1.3, 2, (char *[]) {"second", "expire"}); + + TEST_CHECK(cfl_list_size(&u->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&u->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_epxire_exp_histogram() +{ + uint64_t ts; + struct cmt *cmt; + uint64_t positive[3] = {3, 5, 7}; + uint64_t negative[2] = {2, 1}; + int result; + struct cmt_exp_histogram *exp_histogram; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + exp_histogram = cmt_exp_histogram_create(cmt, + "cm", "native", "exp_hist", "native exp histogram", + 1, (char *[]) {"endpoint"}); + TEST_CHECK(exp_histogram != NULL); + + if (exp_histogram == NULL) { + return; + } + + /* Timestamp */ + ts = cfl_time_now(); + + result = cmt_exp_histogram_set_default(exp_histogram, + ts, + 2, + 11, + 0.0, + -2, + 3, + positive, + -1, + 2, + negative, + CMT_TRUE, + 42.25, + 29, + 1, + (char *[]) {"api"}); + TEST_CHECK(result == 0); + result = cmt_exp_histogram_set_default(exp_histogram, + ts-10, + 2, + 11, + 0.0, + -2, + 3, + positive, + -1, + 2, + negative, + CMT_TRUE, + 42.25, + 29, + 1, + (char *[]) {"http"}); + TEST_CHECK(result == 0); + + TEST_CHECK(cfl_list_size(&exp_histogram->map->metrics) == 2); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&exp_histogram->map->metrics) == 1); + + cmt_destroy(cmt); +} + +void test_expire_off_by_one() +{ + uint64_t ts; + struct cmt *cmt; + struct cmt_counter *c; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + /* Create a counter metric type */ + c = cmt_counter_create(cmt, "k8s", "network", "uptime", "Network Uptime", 1, (char *[]) {"host"}); + TEST_CHECK(c != NULL); + + /* Timestamp */ + ts = cfl_time_now(); + + cmt_counter_inc(c, ts, 1, (char *[]){"fullyvalid"}); + cmt_counter_inc(c, ts-1, 1, (char *[]){"bordervalid"}); + cmt_counter_inc(c, ts-2, 1, (char *[]){"borderexpire"}); + + TEST_CHECK(cfl_list_size(&c->map->metrics) == 3); + cmt_expire(cmt, ts-2); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 3); + cmt_expire(cmt, ts-1); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 2); + cmt_expire(cmt, ts); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 1); + cmt_expire(cmt, ts+1); + TEST_CHECK(cfl_list_size(&c->map->metrics) == 0); + + cmt_destroy(cmt); +} + +TEST_LIST = { + {"expire_counter" , test_expire_counter}, + {"expire_gauge", test_expire_gauge}, + {"expire_histogram", test_expire_histogram}, + {"expire_summary", test_expire_summary}, + {"expire_untyped", test_expire_untyped}, + {"expire_exp_histogram", test_epxire_exp_histogram}, + {"expire_off_by_one", test_expire_off_by_one}, + { 0 } +}; diff --git a/lib/cmetrics/tests/histogram.c b/lib/cmetrics/tests/histogram.c index bf3662bc589..8cbd6ff04b9 100644 --- a/lib/cmetrics/tests/histogram.c +++ b/lib/cmetrics/tests/histogram.c @@ -109,6 +109,51 @@ static void prometheus_encode_test(struct cmt *cmt) cmt_encode_prometheus_destroy(buf); } +void test_histogram_non_finite_bucket_labels() +{ + /* Cover non-finite %g outputs like inf/nan. */ + uint64_t ts; + cfl_sds_t buf; + struct cmt *cmt; + struct cmt_histogram *h; + struct cmt_histogram_buckets *buckets; + + cmt_initialize(); + + ts = 0; + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + buckets = cmt_histogram_buckets_create(3, -INFINITY, NAN, INFINITY); + TEST_CHECK(buckets != NULL); + + h = cmt_histogram_create(cmt, + "cm", "encoding", "non_finite_bucket", + "Histogram non-finite bucket label", + buckets, + 0, NULL); + TEST_CHECK(h != NULL); + + cmt_histogram_observe(h, ts, 42.0, 0, NULL); + + buf = cmt_encode_prometheus_create(cmt, CMT_TRUE); + TEST_CHECK(buf != NULL); + if (buf != NULL) { + TEST_CHECK(strstr(buf, + "cm_encoding_non_finite_bucket_bucket{le=\"-inf\"} 0 0") != NULL); + TEST_CHECK(strstr(buf, + "cm_encoding_non_finite_bucket_bucket{le=\"nan\"} 1 0") != NULL); + TEST_CHECK(strstr(buf, + "cm_encoding_non_finite_bucket_bucket{le=\"inf\"} 1 0") != NULL); + TEST_CHECK(strstr(buf, "le=\"-inf.0\"") == NULL); + TEST_CHECK(strstr(buf, "le=\"nan.0\"") == NULL); + TEST_CHECK(strstr(buf, "le=\"inf.0\"") == NULL); + cmt_encode_prometheus_destroy(buf); + } + + cmt_destroy(cmt); +} + void test_histogram() { @@ -257,6 +302,7 @@ void test_prometheus_large_integer_bucket_precision() } TEST_LIST = { + {"non_finite_bucket_labels" , test_histogram_non_finite_bucket_labels}, {"histogram" , test_histogram}, {"set_defaults" , test_set_defaults}, {"prometheus_large_integer_bucket_precision", test_prometheus_large_integer_bucket_precision}, diff --git a/lib/cmetrics/tests/msgpack_temporality.c b/lib/cmetrics/tests/msgpack_temporality.c new file mode 100644 index 00000000000..b4a36e94aa0 --- /dev/null +++ b/lib/cmetrics/tests/msgpack_temporality.c @@ -0,0 +1,213 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Regression test: histogram and exp_histogram aggregation_type (delta vs + * cumulative temporality) must survive a cmt_encode_msgpack / + * cmt_decode_msgpack round trip. + * + * Before this fix, pack_header() in cmt_encode_msgpack.c only wrote the + * "aggregation_type" meta field for CMT_COUNTER, so histograms and + * exp_histograms always decoded back with CMT_AGGREGATION_TYPE_UNSPECIFIED + * regardless of what temporality they were created with. This is the path + * Fluent Bit's in_opentelemetry -> (internal msgpack buffer) -> + * out_opentelemetry pipeline uses, so any OTLP histogram passing through + * Fluent Bit lost its temporality and Prometheus's OTLP receiver rejected it + * with "invalid temporality and type combination". + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cmt_tests.h" + +static void test_histogram_temporality_survives_msgpack_roundtrip() +{ + int result; + char *encoded_buffer; + size_t encoded_size; + size_t offset; + struct cmt *cmt; + struct cmt *decoded_cmt; + struct cmt_histogram *histogram; + struct cmt_histogram *decoded_histogram; + struct cmt_histogram_buckets *buckets; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + buckets = cmt_histogram_buckets_default_create(); + TEST_CHECK(buckets != NULL); + + histogram = cmt_histogram_create(cmt, "k6", "http", "req_duration", + "request duration", buckets, 0, NULL); + TEST_CHECK(histogram != NULL); + + result = cmt_histogram_observe(histogram, 12345, 1.5, 0, NULL); + TEST_CHECK(result == 0); + + /* Simulate what an OTLP delta-temporality histogram looks like once + * decoded from OTLP into a cmt_histogram (cmt_decode_opentelemetry.c + * already sets this correctly; the bug is losing it in the internal + * msgpack buffer that sits between Fluent Bit's OTLP input and output + * plugins). + */ + histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + + offset = 0; + result = cmt_encode_msgpack_create(cmt, &encoded_buffer, &encoded_size); + TEST_CHECK(result == 0); + + result = cmt_decode_msgpack_create(&decoded_cmt, encoded_buffer, + encoded_size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + TEST_CHECK(cfl_list_size(&decoded_cmt->histograms) == 1); + + decoded_histogram = cfl_list_entry_first(&decoded_cmt->histograms, + struct cmt_histogram, _head); + TEST_CHECK(decoded_histogram != NULL); + + if (decoded_histogram != NULL) { + TEST_CHECK(decoded_histogram->aggregation_type == + CMT_AGGREGATION_TYPE_DELTA); + } + + cmt_decode_msgpack_destroy(decoded_cmt); + } + + cmt_encode_msgpack_destroy(encoded_buffer); + cmt_destroy(cmt); +} + +static void test_exp_histogram_temporality_survives_msgpack_roundtrip() +{ + int result; + char *encoded_buffer; + size_t encoded_size; + size_t offset; + struct cmt *cmt; + struct cmt *decoded_cmt; + struct cmt_exp_histogram *exp_histogram; + struct cmt_exp_histogram *decoded_exp_histogram; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + exp_histogram = cmt_exp_histogram_create(cmt, "k6", "http", "req_duration", + "request duration", 0, NULL); + TEST_CHECK(exp_histogram != NULL); + + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + + offset = 0; + result = cmt_encode_msgpack_create(cmt, &encoded_buffer, &encoded_size); + TEST_CHECK(result == 0); + + result = cmt_decode_msgpack_create(&decoded_cmt, encoded_buffer, + encoded_size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + TEST_CHECK(cfl_list_size(&decoded_cmt->exp_histograms) == 1); + + decoded_exp_histogram = cfl_list_entry_first(&decoded_cmt->exp_histograms, + struct cmt_exp_histogram, + _head); + TEST_CHECK(decoded_exp_histogram != NULL); + + if (decoded_exp_histogram != NULL) { + TEST_CHECK(decoded_exp_histogram->aggregation_type == + CMT_AGGREGATION_TYPE_DELTA); + } + + cmt_decode_msgpack_destroy(decoded_cmt); + } + + cmt_encode_msgpack_destroy(encoded_buffer); + cmt_destroy(cmt); +} + +static void test_counter_temporality_still_survives_msgpack_roundtrip() +{ + int result; + char *encoded_buffer; + size_t encoded_size; + size_t offset; + struct cmt *cmt; + struct cmt *decoded_cmt; + struct cmt_counter *counter; + struct cmt_counter *decoded_counter; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + + counter = cmt_counter_create(cmt, "k6", "http", "reqs", "requests", 0, NULL); + TEST_CHECK(counter != NULL); + + counter->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + + offset = 0; + result = cmt_encode_msgpack_create(cmt, &encoded_buffer, &encoded_size); + TEST_CHECK(result == 0); + + result = cmt_decode_msgpack_create(&decoded_cmt, encoded_buffer, + encoded_size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + TEST_CHECK(cfl_list_size(&decoded_cmt->counters) == 1); + + decoded_counter = cfl_list_entry_first(&decoded_cmt->counters, + struct cmt_counter, _head); + TEST_CHECK(decoded_counter != NULL); + + if (decoded_counter != NULL) { + TEST_CHECK(decoded_counter->aggregation_type == + CMT_AGGREGATION_TYPE_DELTA); + } + + cmt_decode_msgpack_destroy(decoded_cmt); + } + + cmt_encode_msgpack_destroy(encoded_buffer); + cmt_destroy(cmt); +} + +TEST_LIST = { + {"histogram_temporality_survives_msgpack_roundtrip", + test_histogram_temporality_survives_msgpack_roundtrip}, + {"exp_histogram_temporality_survives_msgpack_roundtrip", + test_exp_histogram_temporality_survives_msgpack_roundtrip}, + {"counter_temporality_still_survives_msgpack_roundtrip", + test_counter_temporality_still_survives_msgpack_roundtrip}, + {NULL, NULL} +}; diff --git a/lib/cmetrics/tests/prometheus_parser.c b/lib/cmetrics/tests/prometheus_parser.c index 28643fe8418..2f9c8fafb42 100644 --- a/lib/cmetrics/tests/prometheus_parser.c +++ b/lib/cmetrics/tests/prometheus_parser.c @@ -1710,6 +1710,20 @@ void test_issue_fluent_bit_9267() cfl_sds_destroy(in_buf); } +// reproduces https://github.com/fluent/cmetrics/issues/274 +void test_issue_274() +{ + int status; + struct cmt *cmt; + cfl_sds_t in_buf = read_file(CMT_TESTS_DATA_PATH "/issue_274.txt"); + size_t in_size = cfl_sds_len(in_buf); + + status = cmt_decode_prometheus_create(&cmt, in_buf, in_size, NULL); + TEST_CHECK(status == 0); + cfl_sds_destroy(in_buf); + cmt_decode_prometheus_destroy(cmt); +} + TEST_LIST = { {"header_help", test_header_help}, {"header_type", test_header_type}, @@ -1744,5 +1758,6 @@ TEST_LIST = { {"histogram_different_label_count", test_histogram_different_label_count}, {"issue_fluent_bit_6534", test_issue_fluent_bit_6534}, {"issue_fluent_bit_9267", test_issue_fluent_bit_9267}, + {"issue_274", test_issue_274}, { 0 } };