diff --git a/.github/scripts/commit_prefix_check.py b/.github/scripts/commit_prefix_check.py index 8558639f9df..afcf0df0298 100644 --- a/.github/scripts/commit_prefix_check.py +++ b/.github/scripts/commit_prefix_check.py @@ -21,7 +21,7 @@ repo = Repo(".") # Regex patterns -PREFIX_RE = re.compile(r"^([a-z0-9_]+:)\s+\S", re.IGNORECASE) +PREFIX_RE = re.compile(r"^((?:[a-z0-9_]+:\s+)+)\S", re.IGNORECASE) SIGNED_OFF_RE = re.compile(r"Signed-off-by:", re.IGNORECASE) FENCED_BLOCK_RE = re.compile( r""" @@ -33,6 +33,15 @@ re.DOTALL | re.VERBOSE, ) + +def extract_subject_prefix(line: str): + match = PREFIX_RE.match(line) + + if not match: + return None + + return match.group(1).rstrip() + def strip_fenced_code_blocks(text: str) -> str: """ Remove fenced code blocks (``` or ~~~) from commit message body. @@ -78,6 +87,8 @@ def infer_prefix_from_paths(paths): if name: component_prefixes.add(f"{name}:") component_prefixes.add("tests:") + if p.startswith("tests/integration/"): + component_prefixes.add("tests: integration:") else: component_prefixes.add("tests:") @@ -226,12 +237,10 @@ def is_version_bump(commit): body = strip_fenced_code_blocks(body) # Subject must start with a prefix - subject_prefix_match = PREFIX_RE.match(first_line) - if not subject_prefix_match: + subject_prefix = extract_subject_prefix(first_line) + if not subject_prefix: return False, f"Missing prefix in commit subject: '{first_line}'" - subject_prefix = subject_prefix_match.group(1) - # Run squash detection (but ignore multi-signoff errors) bad_squash, reason = detect_bad_squash(body) @@ -313,7 +322,7 @@ def is_version_bump(commit): } # Prefixes that are allowed to cover multiple subcomponents - umbrella_prefixes = {"lib:", "tests:", "http_server:"} + umbrella_prefixes = {"lib:", "tests:", "tests: integration:", "http_server:"} # If more than one non-build prefix is inferred AND the subject is not an umbrella # prefix, check if the subject prefix is in the expected list. If it is, allow it @@ -341,6 +350,15 @@ def is_version_bump(commit): f"Expected one of: {expected_str}" ) + elif subj_lower == "tests: integration:": + if not all(p.startswith("tests/integration/") for p in norm_paths): + expected_list = sorted(expected) + expected_str = ", ".join(expected_list) + return False, ( + f"Subject prefix '{subject_prefix}' does not match files changed.\n" + f"Expected one of: {expected_str}" + ) + elif subj_lower == "http_server:": if not all(is_http_server_interface_path(p) for p in norm_paths): expected_list = sorted(expected) diff --git a/.gitignore b/.gitignore index ba37a7bb55f..aa5b6f9871b 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,7 @@ workflow/ examples/wasi_serde_json/target/ # WASM test data tests/runtime/wasm/go/*.wasm +tests/integration/.venv/ +tests/integration/.pytest_cache/ +tests/integration/**/__pycache__/ +tests/integration/results/ diff --git a/AGENTS.md b/AGENTS.md index aebdc13146d..8421b7e3e76 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,6 +8,10 @@ when the affected area is known, because the full enabled suite can be slow. - Run a focused integration test with `ctest --test-dir build -R flb-it-opentelemetry --output-on-failure` +- Run the in-tree Python integration suite with: + `cd tests/integration && ./setup-venv.sh && ./run_tests.py` +- List available Python integration scenarios with: + `cd tests/integration && ./run_tests.py --list` - Run locally with `./build/bin/fluent-bit -c conf/fluent-bit.conf` ## Project Structure & Module Organization @@ -18,6 +22,9 @@ Fluent Bit is a C/C++ monorepo built with CMake. - `plugins/`: input/filter/processor/output plugins (`in_*`, `filter_*`, `processor_*`, `out_*`). - `lib/`: bundled libraries (e.g., `cprofiles`, `ctraces`, `cmetrics`, `chunkio`). - `tests/`: integration/runtime tests and fixtures. +- `tests/integration/`: in-tree Python integration test suite for end-to-end + plugin and protocol validation; introduced from the original + `github.com/fluent/fluent-bit-test-suite` project. - `conf/`: sample configurations for local validation. Keep changes scoped: plugin logic in its plugin directory, shared behavior in `src/` or `lib/`. @@ -27,6 +34,14 @@ Keep changes scoped: plugin logic in its plugin directory, shared behavior in `s - `cmake --build build -j8`: compile Fluent Bit and tests. - `ctest --test-dir build --output-on-failure`: run enabled tests. - `ctest --test-dir build -R flb-it-opentelemetry --output-on-failure`: run a focused integration test. +- `cd tests/integration && ./setup-venv.sh`: create the local virtualenv for + the Python integration suite. +- `cd tests/integration && ./run_tests.py --list`: list available Python + integration scenarios. +- `cd tests/integration && ./run_tests.py`: run the full Python integration + suite against `build/bin/fluent-bit`. +- `cd tests/integration && FLUENT_BIT_BINARY=/path/to/fluent-bit ./run_tests.py`: + run the Python integration suite against a specific binary. - `./build/bin/fluent-bit -c conf/fluent-bit.conf`: run locally with a config. ## Coding Style & Naming Conventions @@ -48,9 +63,17 @@ Keep changes scoped: plugin logic in its plugin directory, shared behavior in `s - Add or update tests for behavior changes, especially protocol parsing and encoder/decoder paths. - Prefer targeted tests close to the changed module (`tests/internal`, plugin runtime tests). - Prefer focused `ctest -R ...` runs or specific test binaries when the touched area is known. +- Use `tests/integration` when validating end-to-end plugin behavior, network + protocols, downstream request generation, or local fake-server interactions + that are awkward to cover in `ctest` binaries alone. +- The Python integration suite is not part of the default CMake `ctest` targets; + run it explicitly from `tests/integration`. - Run broader test coverage when changing shared lifecycle, routing, storage, or accounting code. - Validate both success and failure paths (invalid payloads, boundary sizes, null/missing fields). - You can also run specific binaries from `build/bin` (e.g., `./bin/flb-it-opentelemetry`). +- Keep generated integration artifacts out of git. Do not commit + `.venv/`, `.pytest_cache/`, `results/`, or `__pycache__/` under + `tests/integration`. ## Commit & Pull Request Guidelines - Prefix commit subjects with the component/plugin name in lowercase, e.g.: diff --git a/tests/integration/.gitignore b/tests/integration/.gitignore new file mode 100644 index 00000000000..0471efc7161 --- /dev/null +++ b/tests/integration/.gitignore @@ -0,0 +1,7 @@ +__pycache__ +.pytest_cache +results/* +valgrind.log +env + + diff --git a/tests/integration/CONTRIBUTING.md b/tests/integration/CONTRIBUTING.md new file mode 100644 index 00000000000..3db79c190cc --- /dev/null +++ b/tests/integration/CONTRIBUTING.md @@ -0,0 +1,88 @@ +# Contributing to Fluent Bit Test Suite + +> note: this guide is still work in process + +The way to contribute to this project is through the official Github Repository [fluent/fluent-bit-test-suite](https://github.com/fluent/fluent-bit-test-suite). All contributions must adhere to this guidelines that aims to make easier it maintenance and high quality over time. + +## Why contributing to this project ? + +Fluent Bit is one of the widely deployed Telemetry Agent around the globe, millions of new deployments happens every single day. Contributing to this project aims to extend the testing of different areas under different complex configurations. + +Our goal is to avoid regressions and making sure Fluent Bit can continue to grow in a healthy way. + +## Guidelines + +All the code in this project is based on Python 3.x and we have a few requirements: + +### Code Style + +Avoid using camelCase in variables, functions and method names, use the underscore (`_`) instead. + +### GIT Commits + +In open source project maintenance, having a clear history is key for us, for hence we expect full clarity in the commits. Clarity must be in the following places: + +- commit prefix +- commit description + +#### Commit Prefix and Descriptions + +Commit _prefix_ __must not exceed 80__ characters in length. This help to readibility in the GIT log viewers. Make it short and straight to the point. + +In _commit description_, make sure each line do not exceed __120 characters__. Here you have more flexibility to write more about the changes, just try to be inside the length limit of each line. + +The project have specific components, we enforce that every commit that touches an interface or component be prefixed with that name. As of today we register the following components: + +__scenarios__ + +An scenario defines a main type of pipeline or Fluent Bit component being tested, an example is: + +- in-opentelemetry: changes applicable to OTLP input and OTLP round-trip tests +- in-splunk: changes applicable to Splunk input protocol tests +- in-elasticsearch: changes applicable to Elasticsearch input compatibility tests +- in-http: changes applicable to HTTP input end-to-end tests + +For any code change that is happening inside [scenarios/in_opentelemetry](https://github.com/fluent/fluent-bit-test-suite/tree/main/scenarios/in_opentelemetry) the commit must be prefixed like this: + +``` +scenarios: in-opentelemetry: description of the change +``` + +Optionally you can add a third component that represents another file interface (without the extension `.py`) . + +__server__ + +In the server components, we have helpers to implement 'fake servers' who mimic other projects that we use as receivers, e.g: + +| server | description | +|--|--| +| [http](https://github.com/fluent/fluent-bit-test-suite/blob/main/src/server/http_server.py) | Simple HTTP server | +| [otlp](https://github.com/fluent/fluent-bit-test-suite/blob/main/src/server/otlp_server.py) | OpenTelemetry HTTP Server | +| [splunk](https://github.com/fluent/fluent-bit-test-suite/blob/main/src/server/splunk_server.py) | Splunk HTTP Server | + +When modifying any of those servers or adding new ones, the commits must be prefixed like this: + +``` +server: http: some example descripition +``` + +### Naming + +Prefer Fluent Bit internal plugin naming in test names and descriptions: + +- `in_splunk` +- `in_elasticsearch` +- `in_http` + +Scenario directory names may be broader for now, but new tests and updated assertions should follow the internal plugin names in function names, docs, and commit prefixes whenever practical. +Scenario directory names are now expected to follow the internal plugin names for maintained scenarios whenever possible. + + +##### Others + +Commits should not modify files outside of the scope defined in the prefix, while there might be cases for exceptions we will handle those in the Pull Request review process. + + +### License + +All code contributed to this project is under the terms of the Apache v2 License. All commits must be signed (DCO). diff --git a/tests/integration/LICENSE b/tests/integration/LICENSE new file mode 100644 index 00000000000..f433b1a53f5 --- /dev/null +++ b/tests/integration/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 00000000000..ab3986d22fb --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,594 @@ +# Fluent Bit Python Integration Test Suite + +## Status In Fluent Bit + +This suite originates from the original +[`github.com/fluent/fluent-bit-test-suite`](https://github.com/fluent/fluent-bit-test-suite) +project. + +Inside the Fluent Bit repository it is now referred to as the integration test +suite and is included under `tests/fluent-bit-test-suite` as an in-tree +developer test harness. + +It is intended for local development, plugin validation, and focused regression work. + +It is not wired into the default Fluent Bit CMake test targets, `ctest`, or the default GitHub Actions workflows in this repository. + +## Quick Start + +From the repository root: + +```bash +cd tests/fluent-bit-test-suite +./setup-venv.sh +./run_tests.py --list +./run_tests.py +``` + +By default the suite looks for `build/bin/fluent-bit`. You can override that with `FLUENT_BIT_BINARY=/path/to/fluent-bit`. + +## What This Is + +This is a binary-level integration test harness for Fluent Bit. + +This project is distributed under the Apache License, Version 2.0. + +It starts a real Fluent Bit process, drives real network traffic into it, captures what Fluent Bit emits, and asserts on observable behavior: + +- listener behavior +- protocol negotiation +- payload parsing +- payload transformation +- downstream request generation +- exporter endpoint output +- negative-path handling +- memory diagnostics through optional Valgrind execution + +The suite is designed as a reusable tool for testing Fluent Bit plugins and protocol behavior with deterministic local infrastructure. + +## What This Solves + +This framework gives you a controlled environment for testing Fluent Bit end to end without depending on external services. + +It provides: + +- dynamic port allocation +- local fake HTTP servers +- local fake OTLP receivers over HTTP and gRPC +- TLS-enabled endpoints with reusable local certificates +- HTTP/1.1 and HTTP/2 matrix execution +- bounded waits instead of ad hoc sleeps +- per-run logs and result directories +- optional Valgrind wrapping and parsing + +That makes it useful both for plugin development and for runtime regression testing. + +## High-Level Architecture + +```text + +----------------------+ + | Python test case | + | (pytest) | + +----------+-----------+ + | + v + +----------------------+ + | FluentBitTestService | + | port/env orchestration + | server lifecycle + | bounded waits + +----------+-----------+ + | + v + +----------------------+ + | FluentBitManager | + | start/stop binary + | logs/results + | readiness + | valgrind integration + +----------+-----------+ + | + v + +----------------------+ + | fluent-bit binary | + +-----+----------+-----+ + | | + input side | | output side + v v + real clients/tests fake receivers/exporters +``` + +## Main Components + +### Process management + +[`src/utils/fluent_bit_manager.py`](src/utils/fluent_bit_manager.py) + +- resolves the Fluent Bit binary +- starts Fluent Bit with a scenario config +- captures logs in a per-run results directory +- exposes monitoring-based readiness checks +- supports `VALGRIND=1` + +[`src/utils/test_service.py`](src/utils/test_service.py) + +- allocates dynamic ports +- injects environment variables into scenario configs +- starts and stops helper servers +- exposes deterministic wait helpers for assertions + +### Transport matrix + +[`src/utils/http_matrix.py`](src/utils/http_matrix.py) + +- drives HTTP/1.1 cleartext +- drives HTTP/2 cleartext +- drives HTTP/1.1 over TLS +- drives HTTP/2 over TLS +- exercises upgrade and fallback behavior when relevant + +### Helper servers + +[`src/server/http_server.py`](src/server/http_server.py) + +- fake HTTP destination +- captures request headers, body, path, auth, and metadata +- can inject response codes, delays, OAuth token responses, and JWKS responses + +[`src/server/otlp_server.py`](src/server/otlp_server.py) + +- fake OTLP receiver +- accepts OTLP over HTTP and gRPC +- decodes protobuf payloads +- supports gzip and zstd request decoding +- captures method/path/headers/transport for assertions + +[`src/server/splunk_server.py`](src/server/splunk_server.py) + +- Splunk-oriented downstream capture helper + +[`src/server/forward_server.py`](src/server/forward_server.py) + +- Forward protocol receiver for end-to-end input and output validation +- captures message mode and packed-forward payloads, chunk metadata, and signal options + +[`src/server/kafka_server.py`](src/server/kafka_server.py) + +- minimal Kafka-compatible server for output plugin validation +- captures produced records, keys, topics, and payload encodings + +[`src/server/s3_server.py`](src/server/s3_server.py) + +- fake S3-compatible HTTP receiver for `out_s3` +- captures PUT requests, object paths, headers, and uploaded payloads + +## Data Flow + +### Input plugin tests + +```text +test client + | + | real protocol payload + v ++------------------+ +| Fluent Bit | +| input plugin | ++------------------+ + | + | forwarded output + v ++------------------+ +| fake receiver | +| http / otlp | ++------------------+ + | + v +pytest assertions +``` + +Examples: + +- `in_http` +- `in_splunk` +- `in_elasticsearch` +- `in_opentelemetry` +- `in_syslog` + +### Output plugin tests + +```text +source input inside Fluent Bit +dummy / metrics / otlp / etc + | + v ++------------------+ +| Fluent Bit | +| output plugin | ++------------------+ + | + | outbound request + v ++------------------+ +| fake receiver | +| http / otlp | ++------------------+ + | + v +pytest assertions +``` + +Examples: + +- `out_http` +- `out_opentelemetry` +- `out_prometheus_exporter` +- `out_vivo_exporter` + +### Internal endpoint tests + +```text +pytest client + | + v ++------------------+ +| Fluent Bit | +| internal server | ++------------------+ + | + v +response validation +``` + +Examples: + +- `internal_http_server` +- `out_prometheus_exporter` +- `out_vivo_exporter` + +## TLS And Protocol Matrix + +The suite reuses a local certificate pair from: + +- [`scenarios/in_splunk/certificate/certificate.pem`](scenarios/in_splunk/certificate/certificate.pem) +- [`scenarios/in_splunk/certificate/private_key.pem`](scenarios/in_splunk/certificate/private_key.pem) + +These assets are shared across HTTP and OTLP TLS scenarios. + +The HTTP matrix covers, depending on plugin support: + +- HTTP/1.1 cleartext +- HTTP/2 cleartext with prior knowledge +- cleartext upgrade attempts +- fallback to HTTP/1.1 +- HTTP/1.1 over TLS +- HTTP/2 over TLS via ALPN +- HTTP/2 TLS fallback to HTTP/1.1 + +This lets the suite validate not only payload handling, but also the transport behavior exposed by Fluent Bit listeners and endpoints. + +## Current Coverage + +The suite currently exercises: + +- HTTP input plugins +- Forward input behavior +- MQTT input behavior +- OTLP input and output paths +- Splunk HEC input behavior +- Elasticsearch-compatible input behavior +- syslog ingestion over TCP, TLS, UDP, and Unix sockets +- Prometheus remote write ingestion +- Kafka output behavior +- S3 output behavior +- stdout output behavior +- Azure Logs Ingestion output behavior +- Prometheus and Vivo exporters +- internal HTTP server endpoints +- connection limiting behavior +- OAuth2 and JWT flows for supported plugins +- compression with gzip and zstd +- selected end-to-end plugin-to-plugin behavior + +## Scenario Index + +### `in_http` + +Path: [`scenarios/in_http`](scenarios/in_http) + +Entry point: [`scenarios/in_http/tests/test_in_http_001.py`](scenarios/in_http/tests/test_in_http_001.py) + +Covers: + +- request acceptance and downstream forwarding +- HTTP transport matrix +- malformed JSON rejection +- invalid method rejection +- JWT/OAuth2 protected input behavior + +### `in_elasticsearch` + +Path: [`scenarios/in_elasticsearch`](scenarios/in_elasticsearch) + +Entry point: [`scenarios/in_elasticsearch/tests/test_in_elasticsearch_001.py`](scenarios/in_elasticsearch/tests/test_in_elasticsearch_001.py) + +Covers: + +- root and `/_nodes/http` endpoints +- bulk create, update, and delete operations +- HTTP transport matrix +- worker and small-buffer variants +- invalid bulk request handling + +### `in_forward` + +Path: [`scenarios/in_forward`](scenarios/in_forward) + +Entry point: [`scenarios/in_forward/tests/test_in_forward_001.py`](scenarios/in_forward/tests/test_in_forward_001.py) + +Covers: + +- forward message mode and packed-forward mode +- gzip and zstd packed-forward payloads +- chunk acknowledgements and metadata +- tag rewriting and forced-tag behavior +- Unix socket transport +- TLS and secure-forward authentication +- end-to-end forwarding into a local forward receiver +- storage-limit behavior where supported by the current binary + +### `in_opentelemetry` + +Path: [`scenarios/in_opentelemetry`](scenarios/in_opentelemetry) + +Entry point: [`scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py`](scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py) + +Covers: + +- OTLP logs, metrics, and traces ingestion +- semantic validation of re-emitted OTLP +- histogram and gauge metrics +- parent/child traces +- invalid payload handling +- receiver error visibility +- HTTP transport matrix + +### `in_splunk` + +Path: [`scenarios/in_splunk`](scenarios/in_splunk) + +Entry point: [`scenarios/in_splunk/tests/test_in_splunk_001.py`](scenarios/in_splunk/tests/test_in_splunk_001.py) + +Covers: + +- Splunk HEC URI variants +- HTTP transport matrix +- keepalive variations +- invalid request handling +- output-token precedence in `in_splunk -> out_splunk` + +### `in_prometheus_remote_write` + +Path: [`scenarios/in_prometheus_remote_write`](scenarios/in_prometheus_remote_write) + +Entry point: [`scenarios/in_prometheus_remote_write/tests/test_in_prometheus_remote_write_001.py`](scenarios/in_prometheus_remote_write/tests/test_in_prometheus_remote_write_001.py) + +Covers: + +- remote-write ingestion using a real Fluent Bit sender +- HTTP/1 and HTTP/2 receiver modes +- cleartext and TLS receiver modes + +### `in_http_max_connections` + +Path: [`scenarios/in_http_max_connections`](scenarios/in_http_max_connections) + +Entry point: [`scenarios/in_http_max_connections/tests/test_in_http_max_connections_001.py`](scenarios/in_http_max_connections/tests/test_in_http_max_connections_001.py) + +Covers: + +- `http_server.max_connections` +- deterministic block and recovery behavior + +### `in_mqtt` + +Path: [`scenarios/in_mqtt`](scenarios/in_mqtt) + +Entry point: [`scenarios/in_mqtt/tests/test_in_mqtt_001.py`](scenarios/in_mqtt/tests/test_in_mqtt_001.py) + +Covers: + +- valid MQTT publish ingestion +- truncated and malformed publish recovery +- invalid topic-length handling +- payload wrapping via `payload_key` + +### `in_syslog` + +Path: [`scenarios/in_syslog`](scenarios/in_syslog) + +Entry point: [`scenarios/in_syslog/tests/test_in_syslog_001.py`](scenarios/in_syslog/tests/test_in_syslog_001.py) + +Covers: + +- TCP plaintext +- TCP TLS +- UDP plaintext +- Unix stream sockets +- Unix datagram sockets + +### `internal_http_server` + +Path: [`scenarios/internal_http_server`](scenarios/internal_http_server) + +Entry point: [`scenarios/internal_http_server/tests/test_internal_http_server_001.py`](scenarios/internal_http_server/tests/test_internal_http_server_001.py) + +Covers: + +- representative internal endpoints +- response headers +- concurrency behavior +- selected HTTP/2 access + +### `out_http` + +Path: [`scenarios/out_http`](scenarios/out_http) + +Entry point: [`scenarios/out_http/tests/test_out_http_001.py`](scenarios/out_http/tests/test_out_http_001.py) + +Covers: + +- outbound JSON delivery +- receiver error observability +- OAuth2 client credentials +- OAuth2 private key JWT + +### `out_azure_logs_ingestion` + +Path: [`scenarios/out_azure_logs_ingestion`](scenarios/out_azure_logs_ingestion) + +Entry point: [`scenarios/out_azure_logs_ingestion/tests/test_out_azure_logs_ingestion_001.py`](scenarios/out_azure_logs_ingestion/tests/test_out_azure_logs_ingestion_001.py) + +Covers: + +- Azure Logs Ingestion delivery with OAuth2 +- token and data-plane request validation + +### `out_kafka` + +Path: [`scenarios/out_kafka`](scenarios/out_kafka) + +Entry point: [`scenarios/out_kafka/tests/test_out_kafka_001.py`](scenarios/out_kafka/tests/test_out_kafka_001.py) + +Covers: + +- JSON, raw, and msgpack output +- dynamic topic routing +- message-key mapping +- OTLP JSON and OTLP protobuf output for logs, metrics, and traces +- multi-resource preservation checks + +### `out_opentelemetry` + +Path: [`scenarios/out_opentelemetry`](scenarios/out_opentelemetry) + +Entry point: [`scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py`](scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py) + +Covers: + +- OTLP logs, metrics, and traces output +- HTTP and gRPC transport +- custom HTTP and gRPC URIs +- OAuth2 client credentials +- OAuth2 private key JWT +- TLS verification and vhost/SNI behavior +- custom headers and basic auth +- gzip and zstd compression +- `logs_body_key` +- `logs_body_key_attributes` +- metadata and message key mapping +- `add_label` +- `batch_size` +- `logs_max_resources` +- `logs_max_scopes` + +### `out_prometheus_exporter` + +Path: [`scenarios/out_prometheus_exporter`](scenarios/out_prometheus_exporter) + +Entry point: [`scenarios/out_prometheus_exporter/tests/test_out_prometheus_exporter_001.py`](scenarios/out_prometheus_exporter/tests/test_out_prometheus_exporter_001.py) + +Covers: + +- scrapeable `/metrics` +- selected HTTP/2 access + +### `out_s3` + +Path: [`scenarios/out_s3`](scenarios/out_s3) + +Entry point: [`scenarios/out_s3/tests/test_out_s3_001.py`](scenarios/out_s3/tests/test_out_s3_001.py) + +Covers: + +- `use_put_object` uploads +- JSON-lines payload delivery +- gzip-compressed uploads +- newer S3 output formats when supported by the current binary + +### `out_stdout` + +Path: [`scenarios/out_stdout`](scenarios/out_stdout) + +Entry point: [`scenarios/out_stdout/tests/test_out_stdout_001.py`](scenarios/out_stdout/tests/test_out_stdout_001.py) + +Covers: + +- default stdout formatting +- JSON-lines formatting +- metrics and traces text output +- OTLP JSON ingestion paths rendered to stdout + +### `out_vivo_exporter` + +Path: [`scenarios/out_vivo_exporter`](scenarios/out_vivo_exporter) + +Entry point: [`scenarios/out_vivo_exporter/tests/test_out_vivo_exporter_001.py`](scenarios/out_vivo_exporter/tests/test_out_vivo_exporter_001.py) + +Covers: + +- exporter endpoints +- headers +- selected HTTP/2 access + +## Running + +Run the full suite with the wrapper: + +```bash +./tests/fluent-bit-test-suite/run_tests.py +``` + +Run the full suite with raw pytest: + +```bash +./tests/fluent-bit-test-suite/.venv/bin/pytest -q tests/fluent-bit-test-suite +``` + +List tests with the local wrapper: + +```bash +./tests/fluent-bit-test-suite/run_tests.py --list +``` + +Run tests with a simple checkbox progress view: + +```bash +./tests/fluent-bit-test-suite/run_tests.py +``` + +Run a subset: + +```bash +./tests/fluent-bit-test-suite/run_tests.py scenarios/in_opentelemetry -k oauth2 +``` + +Run against a different binary: + +```bash +FLUENT_BIT_BINARY=/path/to/fluent-bit \ +./tests/fluent-bit-test-suite/.venv/bin/pytest -q tests/fluent-bit-test-suite +``` + +Run under Valgrind: + +```bash +VALGRIND=1 ./tests/fluent-bit-test-suite/.venv/bin/pytest -q tests/fluent-bit-test-suite +``` + +Require Valgrind-clean runs: + +```bash +VALGRIND=1 VALGRIND_STRICT=1 \ +./tests/fluent-bit-test-suite/.venv/bin/pytest -q tests/fluent-bit-test-suite +``` diff --git a/tests/integration/config.yaml b/tests/integration/config.yaml new file mode 100644 index 00000000000..239989bdd2e --- /dev/null +++ b/tests/integration/config.yaml @@ -0,0 +1,2 @@ +fluent_bit_binary: build/bin/fluent-bit +config_path: fluent-bit.yaml diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 00000000000..a67afaaa66a --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,61 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2024 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import yaml +import logging +import pytest + +# Configure logging +def configure_logging(): + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + + if not logger.handlers: + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger + +logger = configure_logging() + +def load_global_config(): + config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), 'config.yaml')) + with open(config_file, 'r') as file: + return yaml.safe_load(file) + +GLOBAL_CONFIG = load_global_config() + +@pytest.hookimpl(tryfirst=True) +def pytest_configure(config): + logger.info("Configuring pytest") + +@pytest.hookimpl(tryfirst=True) +def pytest_sessionstart(session): + logger.info("Starting pytest session") + #flb = FluentBitManager(GLOBAL_CONFIG['fluent_bit']['config_path']) + #flb = FluentBitManager() + +@pytest.hookimpl(trylast=True) +def pytest_sessionfinish(session, exitstatus): + pass #logger.info("Finishing pytest session") + +@pytest.hookimpl(trylast=True) +def pytest_unconfigure(config): + logger.info("Unconfiguring pytest") diff --git a/tests/integration/pytest.ini b/tests/integration/pytest.ini new file mode 100644 index 00000000000..61bed84f7cf --- /dev/null +++ b/tests/integration/pytest.ini @@ -0,0 +1,24 @@ +# Fluent Bit +# ========== +# Copyright (C) 2024 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[pytest] +testpaths = scenarios +pythonpath = . src +log_cli = 1 +log_cli_level = INFO +log_cli_format = %(asctime)s - %(levelname)s - %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S + diff --git a/tests/integration/requirements.txt b/tests/integration/requirements.txt new file mode 100644 index 00000000000..5a70170b2b0 --- /dev/null +++ b/tests/integration/requirements.txt @@ -0,0 +1,39 @@ +beautifulsoup4==4.12.3 +blinker==1.8.2 +certifi==2024.6.2 +charset-normalizer==3.3.2 +click==8.1.7 +deepdiff==7.0.1 +Deprecated==1.2.14 +Flask==3.0.3 +google==3.0.0 +googleapis-common-protos==1.63.2 +grpcio==1.64.1 +idna==3.7 +importlib_metadata==7.1.0 +iniconfig==2.0.0 +itsdangerous==2.2.0 +Jinja2==3.1.4 +MarkupSafe==2.1.5 +opentelemetry-api==1.25.0 +opentelemetry-exporter-otlp==1.25.0 +opentelemetry-exporter-otlp-proto-common==1.25.0 +opentelemetry-exporter-otlp-proto-grpc==1.25.0 +opentelemetry-exporter-otlp-proto-http==1.25.0 +opentelemetry-proto==1.25.0 +opentelemetry-sdk==1.25.0 +opentelemetry-semantic-conventions==0.46b0 +ordered-set==4.1.0 +packaging==24.1 +pluggy==1.5.0 +protobuf==4.25.3 +pytest==8.2.2 +PyYAML==6.0.1 +requests==2.32.3 +soupsieve==2.5 +typing_extensions==4.12.2 +urllib3==2.2.2 +waitress==3.0.0 +Werkzeug==3.0.3 +wrapt==1.16.0 +zipp==3.19.2 diff --git a/tests/integration/run_tests.py b/tests/integration/run_tests.py new file mode 100755 index 00000000000..40c0c1006f7 --- /dev/null +++ b/tests/integration/run_tests.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import os +import sys +from collections import OrderedDict +from pathlib import Path + + +SUITE_ROOT = Path(__file__).resolve().parent +VENV_PYTHON = SUITE_ROOT / ".venv" / "bin" / "python3" +REEXEC_ENV = "FLB_SUITE_WRAPPER_REEXEC" + + +def _maybe_reexec_in_venv() -> None: + if os.environ.get(REEXEC_ENV) == "1": + return + + if not VENV_PYTHON.is_file(): + return + + current = Path(sys.executable).resolve() + target = VENV_PYTHON.resolve() + + if current == target: + return + + env = os.environ.copy() + env[REEXEC_ENV] = "1" + os.execve(str(target), [str(target), str(Path(__file__).resolve()), *sys.argv[1:]], env) + + +_maybe_reexec_in_venv() + +try: + import pytest # noqa: E402 +except ModuleNotFoundError: + if VENV_PYTHON.is_file() and os.environ.get(REEXEC_ENV) != "1": + env = os.environ.copy() + env[REEXEC_ENV] = "1" + os.execve(str(VENV_PYTHON), [str(VENV_PYTHON), str(Path(__file__).resolve()), *sys.argv[1:]], env) + raise + + +STATUS_PENDING = "pending" +STATUS_RUNNING = "running" +STATUS_PASSED = "passed" +STATUS_FAILED = "failed" +STATUS_SKIPPED = "skipped" + +STATUS_ICON = { + STATUS_PENDING: "[ ]", + STATUS_RUNNING: "[>]", + STATUS_PASSED: "[x]", + STATUS_FAILED: "[!]", + STATUS_SKIPPED: "[-]", +} + + +def scenario_name_from_nodeid(nodeid: str) -> str: + path = nodeid.split("::", 1)[0] + parts = path.split("/") + if "scenarios" in parts: + index = parts.index("scenarios") + if index + 1 < len(parts): + return parts[index + 1] + return Path(path).stem + + +def short_name_from_nodeid(nodeid: str) -> str: + path, _, test_name = nodeid.partition("::") + return f"{Path(path).name}::{test_name}" if test_name else Path(path).name + + +class CollectPlugin: + def __init__(self) -> None: + self.nodeids: list[str] = [] + + def pytest_collection_modifyitems(self, session, config, items): + self.nodeids = [item.nodeid for item in items] + + +class CheckboxProgressPlugin: + def __init__(self) -> None: + self.nodeids: list[str] = [] + self.statuses: OrderedDict[str, str] = OrderedDict() + self.terminal_reporter = None + self.use_tty = sys.stdout.isatty() + self._listed_non_tty = False + + def pytest_configure(self, config): + self.terminal_reporter = config.pluginmanager.getplugin("terminalreporter") + + def pytest_collection_modifyitems(self, session, config, items): + self.nodeids = [item.nodeid for item in items] + self.statuses = OrderedDict((nodeid, STATUS_PENDING) for nodeid in self.nodeids) + self._render() + + def pytest_runtest_logstart(self, nodeid, location): + if nodeid in self.statuses and self.statuses[nodeid] == STATUS_PENDING: + self.statuses[nodeid] = STATUS_RUNNING + self._render(changed_nodeid=nodeid) + + def pytest_runtest_logreport(self, report): + nodeid = report.nodeid + if nodeid not in self.statuses: + return + + if report.when == "setup" and report.skipped: + self.statuses[nodeid] = STATUS_SKIPPED + self._render(changed_nodeid=nodeid) + return + + if report.when == "call": + if report.passed: + self.statuses[nodeid] = STATUS_PASSED + elif report.failed: + self.statuses[nodeid] = STATUS_FAILED + elif report.skipped: + self.statuses[nodeid] = STATUS_SKIPPED + self._render(changed_nodeid=nodeid) + return + + if report.when == "teardown" and report.failed: + self.statuses[nodeid] = STATUS_FAILED + self._render(changed_nodeid=nodeid) + + def pytest_sessionfinish(self, session, exitstatus): + self._render(final=True) + + def _summary(self) -> dict[str, int]: + counts = { + STATUS_PENDING: 0, + STATUS_RUNNING: 0, + STATUS_PASSED: 0, + STATUS_FAILED: 0, + STATUS_SKIPPED: 0, + } + for status in self.statuses.values(): + counts[status] += 1 + return counts + + def _grouped_lines(self) -> list[str]: + groups: OrderedDict[str, list[str]] = OrderedDict() + for nodeid in self.nodeids: + groups.setdefault(scenario_name_from_nodeid(nodeid), []).append(nodeid) + + lines: list[str] = [] + for scenario, nodeids in groups.items(): + lines.append(f"{scenario}") + for nodeid in nodeids: + lines.append(f" {STATUS_ICON[self.statuses[nodeid]]} {short_name_from_nodeid(nodeid)}") + return lines + + def _render(self, final: bool = False, changed_nodeid: str | None = None): + if not self.terminal_reporter or not self.nodeids: + return + + summary = self._summary() + done = ( + summary[STATUS_PASSED] + + summary[STATUS_FAILED] + + summary[STATUS_SKIPPED] + ) + total = len(self.nodeids) + lines = [ + "Suite Progress", + ( + f" done {done}/{total} " + f"passed {summary[STATUS_PASSED]} " + f"failed {summary[STATUS_FAILED]} " + f"skipped {summary[STATUS_SKIPPED]} " + f"running {summary[STATUS_RUNNING]} " + f"pending {summary[STATUS_PENDING]}" + ), + "", + *self._grouped_lines(), + ] + output = "\n".join(lines) + + if self.use_tty: + self.terminal_reporter.write("\x1b[2J\x1b[H" + output + ("\n" if final else ""), flush=True) + else: + if not self._listed_non_tty: + self.terminal_reporter.write_line(f"Collected {total} tests") + for line in self._grouped_lines(): + self.terminal_reporter.write_line(line) + self._listed_non_tty = True + return + + if changed_nodeid is not None: + self.terminal_reporter.write_line( + f"{STATUS_ICON[self.statuses[changed_nodeid]]} {changed_nodeid}" + ) + + if final: + self.terminal_reporter.write_line( + ( + f"Summary: passed {summary[STATUS_PASSED]}, " + f"failed {summary[STATUS_FAILED]}, " + f"skipped {summary[STATUS_SKIPPED]}" + ) + ) + + +def build_pytest_args(args, passthrough: list[str]) -> list[str]: + pytest_args = [ + "--rootdir", + str(SUITE_ROOT), + ] + + if not args.show_logs: + pytest_args.extend(["-o", "log_cli=false"]) + + if args.list_only: + pytest_args.extend(["--collect-only", "-q"]) + elif args.quiet: + pytest_args.append("-q") + else: + pytest_args.append("-vv") + + pytest_args.extend(passthrough) + + if not passthrough: + pytest_args.append("scenarios") + + return pytest_args + + +def print_collected_tests(nodeids: list[str]) -> None: + groups: OrderedDict[str, list[str]] = OrderedDict() + for nodeid in nodeids: + groups.setdefault(scenario_name_from_nodeid(nodeid), []).append(nodeid) + + total = len(nodeids) + print(f"Collected {total} tests") + for scenario, scenario_tests in groups.items(): + print() + print(f"{scenario} ({len(scenario_tests)})") + for nodeid in scenario_tests: + print(f" [ ] {short_name_from_nodeid(nodeid)}") + + +def parse_args(argv: list[str]) -> tuple[argparse.Namespace, list[str]]: + parser = argparse.ArgumentParser( + description="List and run the Fluent Bit Python test suite with a simple checkbox progress view." + ) + parser.add_argument("--list", dest="list_only", action="store_true", help="List collected tests and exit.") + parser.add_argument("--binary", help="Set FLUENT_BIT_BINARY for this run.") + parser.add_argument("--valgrind", action="store_true", help="Run with VALGRIND=1.") + parser.add_argument( + "--valgrind-strict", + action="store_true", + help="Run with VALGRIND=1 and VALGRIND_STRICT=1.", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="Use quieter pytest output; checkbox progress still renders.", + ) + parser.add_argument( + "--show-logs", + action="store_true", + help="Keep pytest live logs enabled instead of the cleaner wrapper view.", + ) + return parser.parse_known_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args, passthrough = parse_args(argv or sys.argv[1:]) + + os.chdir(SUITE_ROOT) + + if args.binary: + os.environ["FLUENT_BIT_BINARY"] = args.binary + if args.valgrind or args.valgrind_strict: + os.environ["VALGRIND"] = "1" + if args.valgrind_strict: + os.environ["VALGRIND_STRICT"] = "1" + + if args.list_only: + collector = CollectPlugin() + exit_code = pytest.main(build_pytest_args(args, passthrough), plugins=[collector]) + if exit_code != 0: + return exit_code + print_collected_tests(collector.nodeids) + return 0 + + progress = CheckboxProgressPlugin() + return pytest.main(build_pytest_args(args, passthrough), plugins=[progress]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-after.yaml b/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-after.yaml new file mode 100644 index 00000000000..0fbbdbd256e --- /dev/null +++ b/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-after.yaml @@ -0,0 +1,25 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + hot_reload: on + hot_reload.watch: on + hot_reload.watch_interval: 1 + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "after" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /v1/logs + logs_body_key: $message diff --git a/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-before.yaml b/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-before.yaml new file mode 100644 index 00000000000..5651f1cd9c6 --- /dev/null +++ b/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-before.yaml @@ -0,0 +1,25 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + hot_reload: on + hot_reload.watch: on + hot_reload.watch_interval: 1 + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "before" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /v1/logs + logs_body_key: $message diff --git a/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-manual-after.yaml b/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-manual-after.yaml new file mode 100644 index 00000000000..064f58bc948 --- /dev/null +++ b/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-manual-after.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + hot_reload: on + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "after" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /v1/logs + logs_body_key: $message diff --git a/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-manual-before.yaml b/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-manual-before.yaml new file mode 100644 index 00000000000..2f32af9efb9 --- /dev/null +++ b/tests/integration/scenarios/hot_reload_watch/config/fluent-bit-manual-before.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + hot_reload: on + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "before" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /v1/logs + logs_body_key: $message diff --git a/tests/integration/scenarios/hot_reload_watch/tests/test_hot_reload_watch_001.py b/tests/integration/scenarios/hot_reload_watch/tests/test_hot_reload_watch_001.py new file mode 100644 index 00000000000..d356ab6f4ed --- /dev/null +++ b/tests/integration/scenarios/hot_reload_watch/tests/test_hot_reload_watch_001.py @@ -0,0 +1,155 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2024 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import os +import shutil +import tempfile +import time + +import pytest +import requests +from google.protobuf import json_format + +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from src.server.otlp_server import data_storage, otlp_server_run +from src.utils.fluent_bit_manager import FluentBitManager +from src.utils.network import find_available_port + +logger = logging.getLogger(__name__) + + +class Service: + def __init__(self, before_name, after_name): + self.test_path = os.path.dirname(os.path.abspath(__file__)) + self.config_dir = os.path.abspath(os.path.join(self.test_path, "../config")) + self.before_config = os.path.join(self.config_dir, before_name) + self.after_config = os.path.join(self.config_dir, after_name) + self.runtime_dir = tempfile.mkdtemp(prefix="flb-hot-reload-watch-") + self.runtime_config = os.path.join(self.runtime_dir, "fluent-bit.yaml") + data_storage["logs"] = [] + + def start(self): + shutil.copyfile(self.before_config, self.runtime_config) + + self.flb = FluentBitManager(self.runtime_config) + self.test_suite_http_port = find_available_port(starting_port=50000) + os.environ["TEST_SUITE_HTTP_PORT"] = str(self.test_suite_http_port) + logger.info(f"test suite http port: {self.test_suite_http_port}") + + otlp_server_run(self.test_suite_http_port) + + url = f"http://127.0.0.1:{self.test_suite_http_port}/ping" + start_time = time.time() + while time.time() - start_time < 10: + try: + response = requests.get(url) + if response.status_code == 200: + break + except requests.exceptions.ConnectionError: + pass + time.sleep(0.5) + + self.flb.start() + + def stop(self): + if getattr(self, "flb", None) is not None and self.flb.process is not None: + self.flb.stop() + requests.post(f"http://127.0.0.1:{self.test_suite_http_port}/shutdown") + shutil.rmtree(self.runtime_dir, ignore_errors=True) + + def wait_for_log_count(self, expected_count, timeout=15): + start_time = time.time() + + while time.time() - start_time < timeout: + if len(data_storage["logs"]) >= expected_count: + return + time.sleep(0.5) + + raise TimeoutError(f"Timed out waiting for {expected_count} log payloads") + + def read_message(self, index): + request = data_storage["logs"][index] + json_str = json_format.MessageToJson(request) + payload = json.loads(json_str) + return payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0]["body"]["stringValue"] + + def replace_config(self): + pending_path = os.path.join(self.runtime_dir, "fluent-bit.yaml.tmp") + shutil.copyfile(self.after_config, pending_path) + os.replace(pending_path, self.runtime_config) + + +def assert_reload_result(service): + service.wait_for_log_count(2) + assert service.read_message(1) == "after" + + +def test_hot_reload_watch_yaml_config_change(): + service = Service("fluent-bit-before.yaml", "fluent-bit-after.yaml") + + try: + service.start() + service.wait_for_log_count(1) + assert service.read_message(0) == "before" + + service.replace_config() + service.flb.wait_for_hot_reload_count(1) + assert_reload_result(service) + finally: + service.stop() + + +def test_hot_reload_sighup_yaml_config_change(): + service = Service("fluent-bit-manual-before.yaml", "fluent-bit-manual-after.yaml") + + try: + service.start() + service.wait_for_log_count(1) + assert service.read_message(0) == "before" + + service.replace_config() + + with pytest.raises(TimeoutError): + service.flb.wait_for_hot_reload_count(1, timeout=2) + + service.flb.send_sighup() + service.flb.wait_for_hot_reload_count(1) + assert_reload_result(service) + finally: + service.stop() + + +def test_hot_reload_http_yaml_config_change(): + service = Service("fluent-bit-manual-before.yaml", "fluent-bit-manual-after.yaml") + + try: + service.start() + service.wait_for_log_count(1) + assert service.read_message(0) == "before" + + service.replace_config() + + with pytest.raises(TimeoutError): + service.flb.wait_for_hot_reload_count(1, timeout=2) + + payload = service.flb.trigger_http_reload() + assert payload["reload"] == "done" + service.flb.wait_for_hot_reload_count(1) + assert_reload_result(service) + finally: + service.stop() diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch new file mode 100644 index 00000000000..3ed67dc926d --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch @@ -0,0 +1,14 @@ +[SERVICE] + Flush 1 + Log_Level info + HTTP_Server on + HTTP_Port ${FLUENT_BIT_HTTP_MONITORING_PORT} + +[INPUT] + Name elasticsearch + Listen 0.0.0.0 + Port ${FLUENT_BIT_TEST_LISTENER_PORT} + +[OUTPUT] + Name stdout + Match * \ No newline at end of file diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_cleartext.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_cleartext.yaml new file mode 100644 index 00000000000..dcaf6527c0d --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_cleartext.yaml @@ -0,0 +1,17 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_cleartext_small_buffer.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_cleartext_small_buffer.yaml new file mode 100644 index 00000000000..e094a32541f --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_cleartext_small_buffer.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + buffer_max_size: 112 + buffer_chunk_size: 112 + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_cleartext_workers.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_cleartext_workers.yaml new file mode 100644 index 00000000000..cf937b14a4c --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_cleartext_workers.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + http_server.workers: 4 + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_tls.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_tls.yaml new file mode 100644 index 00000000000..76c80904486 --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_tls.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_tls_workers.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_tls_workers.yaml new file mode 100644 index 00000000000..cd3ce516c1d --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http1_tls_workers.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + http_server.workers: 4 + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext.yaml new file mode 100644 index 00000000000..142f09ed3d2 --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext.yaml @@ -0,0 +1,17 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: off + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext_small_buffer.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext_small_buffer.yaml new file mode 100644 index 00000000000..d3b2cebb66c --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext_small_buffer.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: off + buffer_max_size: 112 + buffer_chunk_size: 112 + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext_small_buffer_workers.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext_small_buffer_workers.yaml new file mode 100644 index 00000000000..cf7dbe366d0 --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext_small_buffer_workers.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: off + http_server.workers: 4 + buffer_max_size: 112 + buffer_chunk_size: 112 + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext_workers.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext_workers.yaml new file mode 100644 index 00000000000..fa3225722af --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_cleartext_workers.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: off + http_server.workers: 4 + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_tls.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_tls.yaml new file mode 100644 index 00000000000..98381ecdd94 --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_tls.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_tls_workers.yaml b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_tls_workers.yaml new file mode 100644 index 00000000000..09664880c2d --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/config/in_elasticsearch_http2_tls_workers.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: elasticsearch + listen: 0.0.0.0 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + http_server.workers: 4 + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_elasticsearch/tests/data_files/create_index.json b/tests/integration/scenarios/in_elasticsearch/tests/data_files/create_index.json new file mode 100644 index 00000000000..7556a120db9 --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/tests/data_files/create_index.json @@ -0,0 +1,2 @@ +{ "index": { "_index": "new_index", "_id": "1" } } +{ "title": "Test for creation of new_index", "description": "create the index" } \ No newline at end of file diff --git a/tests/integration/scenarios/in_elasticsearch/tests/data_files/create_multiple_documents.json b/tests/integration/scenarios/in_elasticsearch/tests/data_files/create_multiple_documents.json new file mode 100644 index 00000000000..226ecdd5f4f --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/tests/data_files/create_multiple_documents.json @@ -0,0 +1,8 @@ +{ "index": { "_index": "new_index", "_id": "2" } } +{ "title": "second doc", "description": "Description of the first document." } +{ "index": { "_index": "new_index", "_id": "3" } } +{ "title": "Tercero doc", "description": "Description of the second document." } +{ "index": { "_index": "new_index", "_id": "4" } } +{ "title": "four Doc", "description": "Description of the second document." } +{ "index": { "_index": "new_index", "_id": "5" } } +{ "title": "five doc", "description": "Description of the second document." } \ No newline at end of file diff --git a/tests/integration/scenarios/in_elasticsearch/tests/data_files/delete_multiple_documents.json b/tests/integration/scenarios/in_elasticsearch/tests/data_files/delete_multiple_documents.json new file mode 100644 index 00000000000..ce8a66bc1c9 --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/tests/data_files/delete_multiple_documents.json @@ -0,0 +1,2 @@ +{ "update": { "_index": "new_index", "_id": "2" } } +{ "doc": { "title": "Updated Title", "description": "Updated description." } } \ No newline at end of file diff --git a/tests/integration/scenarios/in_elasticsearch/tests/data_files/update_multiple_documents.json b/tests/integration/scenarios/in_elasticsearch/tests/data_files/update_multiple_documents.json new file mode 100644 index 00000000000..f0371a4f3f1 --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/tests/data_files/update_multiple_documents.json @@ -0,0 +1 @@ +{ "delete": { "_index": "new_index", "_id": "2" } } \ No newline at end of file diff --git a/tests/integration/scenarios/in_elasticsearch/tests/test_in_elasticsearch_001.py b/tests/integration/scenarios/in_elasticsearch/tests/test_in_elasticsearch_001.py new file mode 100644 index 00000000000..489da8928a0 --- /dev/null +++ b/tests/integration/scenarios/in_elasticsearch/tests/test_in_elasticsearch_001.py @@ -0,0 +1,401 @@ +import http.client, json, os, logging, time, subprocess + +import pytest + +from server.otlp_server import data_storage +from utils.http_matrix import PROTOCOL_CASES, run_curl_request +from utils.test_service import FluentBitTestService + +logger = logging.getLogger(__name__) + +IN_ELASTICSEARCH_PROTOCOL_CONFIGS = { + False: { + "http1_cleartext": "in_elasticsearch_http1_cleartext.yaml", + "http2_cleartext": "in_elasticsearch_http2_cleartext.yaml", + "http1_tls": "in_elasticsearch_http1_tls.yaml", + "http2_tls": "in_elasticsearch_http2_tls.yaml", + }, + True: { + "http1_cleartext": "in_elasticsearch_http1_cleartext_workers.yaml", + "http2_cleartext": "in_elasticsearch_http2_cleartext_workers.yaml", + "http1_tls": "in_elasticsearch_http1_tls_workers.yaml", + "http2_tls": "in_elasticsearch_http2_tls_workers.yaml", + }, +} + +IN_ELASTICSEARCH_SMALL_BUFFER_REGRESSION_CASES = [ + { + "id": "legacy_http1_cleartext", + "config_file": "in_elasticsearch_http1_cleartext_small_buffer.yaml", + "http_mode": "http1.1", + }, + { + "id": "http2_cleartext", + "config_file": "in_elasticsearch_http2_cleartext_small_buffer.yaml", + "http_mode": "http2-prior-knowledge", + }, +] + + +def parse_single_item_response(response_text): + payload = json.loads(response_text) + assert "items" in payload + assert len(payload["items"]) == 1 + operation, details = next(iter(payload["items"][0].items())) + return payload, operation, details + +# Definition of constant url elastic +# PORT_FAKE_ELASTIC = 9200 +def create_connection(server, port): + return http.client.HTTPConnection(server, port) + + +def create_headers(): + return { + 'Content-Type': 'application/json' + } +""" +Load json file to generate the payload +""" +def create_payload(json_filename): + try: + file_name = os.path.abspath(os.path.join(os.path.dirname(__file__), './data_files/', json_filename)) + with open(file_name, 'r') as file: + # Read the entire file as text and strip unnecessary whitespace + data = file.read().strip() + return data + except FileNotFoundError: + return json.dumps({"error": "File not found"}, indent=4) + except json.JSONDecodeError: + return json.dumps({"error": "Invalid JSON format"}, indent=4) +""" +Generic send request +""" +def send_requests(conn, num_requests, headers, json_payload): + responses = [] + for i in range(num_requests): + conn.request("POST", "/_bulk", body=json_payload, headers=headers) + response = conn.getresponse() + responses.append({ + 'status': response.status, + 'reason': response.reason, + 'data': response.read().decode() + }) + return responses + +""" +CREATE INDEX +""" +def send_request(conn, method, path, headers=None, body=None): + conn.request(method, path, body=body, headers=headers or {}) + response = conn.getresponse() + return { + 'status': response.status, + 'reason': response.reason, + 'data': response.read().decode() + } + + +def test_in_elasticsearch_root_info(): + service = Service("in_elasticsearch") + service.start() + conn = create_connection('localhost', service.flb_listener_port) + response = send_request(conn, "GET", "/") + conn.close() + service.stop() + + assert response['status'] == 200 + assert response['reason'] == 'OK' + assert '"version"' in response['data'] + + +def test_in_elasticsearch_nodes_http(): + service = Service("in_elasticsearch") + service.start() + conn = create_connection('localhost', service.flb_listener_port) + response = send_request(conn, "GET", "/_nodes/http") + conn.close() + service.stop() + + assert response['status'] == 200 + assert response['reason'] == 'OK' + assert '"_nodes"' in response['data'] + assert '"nodes"' in response['data'] + + +def test_in_elasticsearch_create_index(): + try: + service = Service("in_elasticsearch") + service.start() + output = service.runtest_create_index('localhost', service.flb_listener_port,'create_index.json') + logger.info(f"response: {output}") + service.stop() + assert len(output) == 1 + + # Verify response details + for response in output: + assert response['status'] == 200 + assert response['reason'] == 'OK' + assert response['data'] == '{"errors":false,"items":[{"index":{"status":201,"result":"created"}}]}' + except Exception as e: + logger.error(f"An error occurred: {e}") + if service.flb.process is not None: + service.stop() + raise + +""" +CREATE MULTIPLOE DOCUMENTS +""" +def test_in_elasticsearch_create_multiple_documents(): + try: + service = Service("in_elasticsearch") + service.start() + output = service.runtest_create_multiple_documents('localhost', service.flb_listener_port,'create_multiple_documents.json') + logger.info(f"response: {output}") + service.stop() + assert len(output) == 1 + + # Verify response details + for response in output: + assert response['status'] == 200 + assert response['reason'] == 'OK' + assert response['data'] == '{"errors":false,"items":[{"index":{"status":201,"result":"created"}},{"index":{"status":201,"result":"created"}},{"index":{"status":201,"result":"created"}},{"index":{"status":201,"result":"created"}}]}' + except Exception as e: + logger.error(f"An error occurred: {e}") + if service.flb.process is not None: + service.stop() + raise + +""" +UOPDATE MULTIPLE DOCUMENTS +""" +def test_in_elasticsearch_update_multiple_documents(): + try: + service = Service("in_elasticsearch") + service.start() + output = service.runtest_update_multiple_documents('localhost', service.flb_listener_port,'update_multiple_documents.json') + logger.info(f"response: {output}") + service.stop() + assert len(output) == 1 + + # Verify response details + for response in output: + assert response['status'] == 200 + assert response['reason'] == 'OK' + payload, operation, details = parse_single_item_response(response['data']) + assert payload["errors"] is True + assert operation == "delete" + assert details["status"] == 404 + assert details["result"] == "not_found" + except Exception as e: + logger.error(f"An error occurred: {e}") + if service.flb.process is not None: + service.stop() + raise + +""" +DELETE MULTIPLE DOCUMENTS +""" +def test_in_elasticsearch_delete_multiple_documents(): + try: + service = Service("in_elasticsearch") + service.start() + output = service.runtest_delete_multiple_documents('localhost', service.flb_listener_port,'delete_multiple_documents.json') + service.stop() + logger.info(f"response: {output}") + assert len(output) == 1 + + # Verify response details + for response in output: + assert response['status'] == 200 + assert response['reason'] == 'OK' + payload, operation, details = parse_single_item_response(response['data']) + assert payload["errors"] is True + assert operation == "update" + assert details["status"] == 403 + assert details["result"] == "forbidden" + except Exception as e: + logger.error(f"An error occurred: {e}") + if service.flb.process is not None: + service.stop() + raise + + +@pytest.mark.parametrize("workers_enabled", [False, True], ids=["single_listener", "workers_4"]) +@pytest.mark.parametrize("case", PROTOCOL_CASES, ids=[case["id"] for case in PROTOCOL_CASES]) +def test_in_elasticsearch_bulk_protocol_matrix(case, workers_enabled): + service = Service(IN_ELASTICSEARCH_PROTOCOL_CONFIGS[workers_enabled][case["config_key"]]) + service.start() + if workers_enabled: + service.wait_for_log_message("with 4 workers", timeout=10) + + scheme = "https" if case["use_tls"] else "http" + result = run_curl_request( + f"{scheme}://localhost:{service.flb_listener_port}/_bulk", + create_payload("create_index.json"), + headers=["Content-Type: application/json"], + http_mode=case["http_mode"], + ca_cert_path=service.tls_crt_file if case["use_tls"] else None, + ) + + service.stop() + + assert result["status_code"] == 200 + assert result["http_version"] == case["expected_http_version"] + payload, operation, details = parse_single_item_response(result["body"]) + assert payload["errors"] is False + assert operation == "index" + assert details["status"] == 201 + assert details["result"] == "created" + + +def test_in_elasticsearch_rejects_unknown_bulk_operation(): + service = Service("in_elasticsearch") + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/_bulk", + '{"nonexistent":{"_index":"fluent-bit","_id":"1"}}\n{"message":"hello"}\n', + headers=["Content-Type: application/json"], + http_mode="http1.1", + ) + + service.stop() + + assert result["status_code"] == 200 + payload, operation, details = parse_single_item_response(result["body"]) + assert payload["errors"] is True + assert operation == "unknown" + assert details["status"] == 400 + + +@pytest.mark.parametrize( + "case", + IN_ELASTICSEARCH_SMALL_BUFFER_REGRESSION_CASES, + ids=[case["id"] for case in IN_ELASTICSEARCH_SMALL_BUFFER_REGRESSION_CASES], +) +def test_in_elasticsearch_bulk_small_status_buffer_does_not_crash(case): + service = Service(case["config_file"]) + service.start() + + trigger_payload = '{"index":{}}\n{}\n{"index":{}}\n{}\n' + curl_command = [ + "curl", + "--silent", + "--show-error", + "--output", + "-", + "--write-out", + "\n__META__%{http_code} %{http_version}", + "--max-time", + "10", + "-X", + "POST", + "-H", + "Content-Type: application/x-ndjson", + "--data-binary", + "@-", + ] + if case["http_mode"] == "http2-prior-knowledge": + curl_command.append("--http2-prior-knowledge") + else: + curl_command.append("--http1.1") + + curl_command.append(f"http://localhost:{service.flb_listener_port}/_bulk") + + curl_result = subprocess.run( + curl_command, + input=trigger_payload.encode(), + capture_output=True, + check=False, + ) + + assert service.flb.process is not None + assert service.flb.process.poll() is None + + health_result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/_nodes/http", + None, + method="GET", + http_mode="http1.1", + ) + + service.stop() + + # The regression condition is process termination from a reachable double-free. + # Response formatting can still vary under constrained buffer settings, but the + # parser must not take down the process. + assert curl_result.returncode in {0, 56} + assert health_result["status_code"] == 200 + + +class Service: + def __init__(self, config_file): + # Compose the absolute path for the Fluent Bit configuration file + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), '../config/', config_file)) + test_path = os.path.dirname(os.path.abspath(__file__)) + cert_dir = os.path.abspath(os.path.join(test_path, "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["logs"], + extra_env={ + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + }, + ) + + def start(self): + self.service.start() + self.flb = self.service.flb + self.flb_listener_port = self.service.flb_listener_port + + def wait_for_log_message(self, pattern, timeout=10, interval=0.25): + deadline = time.time() + timeout + while time.time() < deadline: + if self.flb and self.flb.log_file and os.path.exists(self.flb.log_file): + with open(self.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + if pattern in log_file.read(): + return True + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for log pattern: {pattern}") + self.test_suite_http_port = self.service.test_suite_http_port + logger.info(f"Fluent Bit listener port: {self.flb_listener_port}") + logger.info(f"test suite http port: {self.test_suite_http_port}") + + def runtest_create_index(self,server, port, json_filename): + conn = create_connection(server, port) + headers = create_headers() + json_payload = create_payload(json_filename) + responses = send_requests(conn, 1, headers, json_payload) + conn.close() + return responses + + def runtest_create_multiple_documents(self,server, port, json_filename): + conn = create_connection(server, port) + headers = create_headers() + json_payload = create_payload(json_filename) + responses = send_requests(conn, 1, headers, json_payload) + conn.close() + return responses + + def runtest_update_multiple_documents(self,server, port, json_filename): + conn = create_connection(server, port) + headers = create_headers() + json_payload = create_payload(json_filename) + responses = send_requests(conn, 1, headers, json_payload) + conn.close() + return responses + + def runtest_delete_multiple_documents(self,server, port, json_filename): + conn = create_connection(server, port) + headers = create_headers() + json_payload = create_payload(json_filename) + responses = send_requests(conn, 1, headers, json_payload) + conn.close() + return responses + + def stop(self): + self.service.stop() diff --git a/tests/integration/scenarios/in_forward/config/in_forward.yaml b/tests/integration/scenarios/in_forward/config/in_forward.yaml new file mode 100644 index 00000000000..f2eba902596 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_forced_tag.yaml b/tests/integration/scenarios/in_forward/config/in_forward_forced_tag.yaml new file mode 100644 index 00000000000..5912e8800f1 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_forced_tag.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + tag: forced.tag + + outputs: + - name: http + match: forced.tag + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_secure.yaml b/tests/integration/scenarios/in_forward/config/in_forward_secure.yaml new file mode 100644 index 00000000000..0b97c0b31b4 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_secure.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + self_hostname: server-node + shared_key: shared-secret + security.users: alice s3cr3t + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_multi_output.yaml b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_multi_output.yaml new file mode 100644 index 00000000000..cce440ef5f5 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_multi_output.yaml @@ -0,0 +1,35 @@ +service: + flush: 60 + grace: 5 + log_level: info + storage.path: ${FORWARD_STORAGE_PATH} + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + storage.type: filesystem + + outputs: + - name: http + match: shared.* + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /shared + format: json + json_date_key: false + retry_limit: false + storage.total_limit_size: 10M + + - name: http + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /solo + format: json + json_date_key: false + retry_limit: false + storage.total_limit_size: 10K diff --git a/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_single_output.yaml b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_single_output.yaml new file mode 100644 index 00000000000..cd89701471b --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_storage_limit_single_output.yaml @@ -0,0 +1,25 @@ +service: + flush: 60 + grace: 5 + log_level: info + storage.path: ${FORWARD_STORAGE_PATH} + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + storage.type: filesystem + + outputs: + - name: http + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /solo + format: json + json_date_key: false + retry_limit: false + storage.total_limit_size: 10K diff --git a/tests/integration/scenarios/in_forward/config/in_forward_tag_prefix.yaml b/tests/integration/scenarios/in_forward/config/in_forward_tag_prefix.yaml new file mode 100644 index 00000000000..d2d8f79d172 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_tag_prefix.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + tag_prefix: edge. + + outputs: + - name: http + match: edge.* + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_tls.yaml b/tests/integration/scenarios/in_forward/config/in_forward_tls.yaml new file mode 100644 index 00000000000..011307e74dc --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_tls.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_to_forward_receiver.yaml b/tests/integration/scenarios/in_forward/config/in_forward_to_forward_receiver.yaml new file mode 100644 index 00000000000..b320f5606c8 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_to_forward_receiver.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: forward + match: test + host: 127.0.0.1 + port: ${FORWARD_RECEIVER_PORT} + send_options: true + require_ack_response: true + retain_metadata_in_forward_mode: true diff --git a/tests/integration/scenarios/in_forward/config/in_forward_to_forward_receiver_gzip.yaml b/tests/integration/scenarios/in_forward/config/in_forward_to_forward_receiver_gzip.yaml new file mode 100644 index 00000000000..15a52ae95fa --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_to_forward_receiver_gzip.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: forward + match: test + host: 127.0.0.1 + port: ${FORWARD_RECEIVER_PORT} + send_options: true + require_ack_response: true + retain_metadata_in_forward_mode: true + compress: gzip diff --git a/tests/integration/scenarios/in_forward/config/in_forward_unix.yaml b/tests/integration/scenarios/in_forward/config/in_forward_unix.yaml new file mode 100644 index 00000000000..059d1660a1d --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_unix.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + unix_path: ${FORWARD_UNIX_PATH} + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_forward_unix_perm.yaml b/tests/integration/scenarios/in_forward/config/in_forward_unix_perm.yaml new file mode 100644 index 00000000000..05fcc9899d3 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_forward_unix_perm.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: forward + unix_path: ${FORWARD_UNIX_PATH} + unix_perm: "0600" + + outputs: + - name: http + match: test + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_forward/config/in_opentelemetry_to_forward_receiver.yaml b/tests/integration/scenarios/in_forward/config/in_opentelemetry_to_forward_receiver.yaml new file mode 100644 index 00000000000..0b08d179d99 --- /dev/null +++ b/tests/integration/scenarios/in_forward/config/in_opentelemetry_to_forward_receiver.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + + outputs: + - name: forward + match: '*' + host: 127.0.0.1 + port: ${FORWARD_RECEIVER_PORT} + send_options: true + require_ack_response: true diff --git a/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py new file mode 100644 index 00000000000..91aa0134f09 --- /dev/null +++ b/tests/integration/scenarios/in_forward/tests/test_in_forward_001.py @@ -0,0 +1,991 @@ +import gzip +import hashlib +import json +import os +import shutil +import socket +import ssl +import subprocess +import tempfile +import uuid +from pathlib import Path + +import pytest +import requests +from google.protobuf import json_format +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from server.forward_server import ( + data_storage as forward_data_storage, + forward_server_run, + forward_server_stop, +) +from server.http_server import configure_http_response, data_storage, http_server_run +from utils.data_utils import read_json_file +from utils.test_service import FluentBitTestService + + +TEST_TAG = "test" +TEST_TS = 1234567890 +SECURE_SHARED_KEY = "shared-secret" +SECURE_USERNAME = "alice" +SECURE_PASSWORD = "s3cr3t" +SECURE_SELF_HOSTNAME = "server-node" + + +class Service: + def __init__(self, config_file, *, use_unix_socket=False): + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + self.use_unix_socket = use_unix_socket + self.socket_path = None + test_path = os.path.dirname(os.path.abspath(__file__)) + cert_dir = os.path.abspath(os.path.join(test_path, "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + extra_env = { + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + } + + if use_unix_socket: + self.socket_path = os.path.join(tempfile.gettempdir(), f"fluent_bit_forward_{uuid.uuid4().hex}.sock") + extra_env["FORWARD_UNIX_PATH"] = self.socket_path + + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["payloads"], + extra_env=extra_env, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + http_server_run(service.test_suite_http_port) + self.service.wait_for_http_endpoint( + f"http://127.0.0.1:{service.test_suite_http_port}/ping", + timeout=10, + interval=0.5, + ) + + def _stop_receiver(self, service): + try: + requests.post(f"http://127.0.0.1:{service.test_suite_http_port}/shutdown", timeout=2) + except requests.RequestException: + pass + if self.socket_path: + try: + os.unlink(self.socket_path) + except FileNotFoundError: + pass + + def start(self): + self.service.start() + self.flb_listener_port = self.service.flb_listener_port + + def stop(self): + self.service.stop() + + def flattened_records(self): + records = [] + for payload in data_storage["payloads"]: + if isinstance(payload, list): + records.extend(payload) + elif payload is not None: + records.append(payload) + return records + + def wait_for_record_count(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: self.flattened_records() if len(self.flattened_records()) >= minimum_count else None, + timeout=timeout, + interval=0.2, + description=f"{minimum_count} forwarded forward records", + ) + + +class StorageLimitService(Service): + def __init__(self, config_file): + super().__init__(config_file) + self.storage_path = tempfile.mkdtemp(prefix="fluent_bit_forward_storage_") + self.service.extra_env["FORWARD_STORAGE_PATH"] = self.storage_path + + def stop(self): + try: + super().stop() + finally: + shutil.rmtree(self.storage_path, ignore_errors=True) + + def count_chunk_files(self): + stream_dir = Path(self.storage_path) / "forward.0" + if not stream_dir.exists(): + return 0 + + return sum(1 for path in stream_dir.rglob("*.flb") if path.is_file()) + + def chunk_file_contents(self): + stream_dir = Path(self.storage_path) / "forward.0" + if not stream_dir.exists(): + return [] + + return [path.read_bytes() for path in stream_dir.rglob("*.flb") if path.is_file()] + + +class ForwardReceiverService: + def __init__(self, config_file): + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + self.service = FluentBitTestService( + self.config_file, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + self.forward_receiver_port = service.allocate_port_env("FORWARD_RECEIVER_PORT") + forward_server_run(self.forward_receiver_port) + + def _stop_receiver(self, service): + forward_server_stop() + + def start(self): + self.service.start() + self.flb_listener_port = self.service.flb_listener_port + + def stop(self): + self.service.stop() + + def wait_for_forward_messages(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: forward_data_storage["messages"] if len(forward_data_storage["messages"]) >= minimum_count else None, + timeout=timeout, + interval=0.2, + description=f"{minimum_count} captured forward messages", + ) + + def send_request(self, endpoint, payload, content_type="application/x-protobuf"): + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}{endpoint}", + data=payload.SerializeToString(), + headers={"Content-Type": content_type}, + timeout=5, + ) + response.raise_for_status() + return response + + def send_json_as_otel_protobuf(self, json_input, signal_type): + base_path = os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../../in_opentelemetry/tests/data_files", + ) + ) + json_payload_dict = read_json_file(os.path.join(base_path, json_input)) + request_map = { + "metrics": (ExportMetricsServiceRequest(), "/v1/metrics"), + "traces": (ExportTraceServiceRequest(), "/v1/traces"), + } + request_message, endpoint = request_map[signal_type] + protobuf_payload = json_format.Parse(json.dumps(json_payload_dict), request_message) + return self.send_request(endpoint, protobuf_payload) + + +def _pack_uint(value): + if value < 0x80: + return bytes([value]) + if value <= 0xFF: + return b"\xCC" + bytes([value]) + if value <= 0xFFFF: + return b"\xCD" + value.to_bytes(2, "big") + if value <= 0xFFFFFFFF: + return b"\xCE" + value.to_bytes(4, "big") + return b"\xCF" + value.to_bytes(8, "big") + + +def _pack_bool(value): + return b"\xC3" if value else b"\xC2" + + +def _pack_str(value): + data = value.encode() + length = len(data) + if length <= 31: + return bytes([0xA0 | length]) + data + if length <= 0xFF: + return b"\xD9" + bytes([length]) + data + if length <= 0xFFFF: + return b"\xDA" + length.to_bytes(2, "big") + data + return b"\xDB" + length.to_bytes(4, "big") + data + + +def _pack_bin(value): + length = len(value) + if length <= 0xFF: + return b"\xC4" + bytes([length]) + value + if length <= 0xFFFF: + return b"\xC5" + length.to_bytes(2, "big") + value + return b"\xC6" + length.to_bytes(4, "big") + value + + +def _pack_ext(type_code, payload): + length = len(payload) + if length == 1: + return b"\xD4" + type_code.to_bytes(1, "big", signed=True) + payload + if length == 2: + return b"\xD5" + type_code.to_bytes(1, "big", signed=True) + payload + if length == 4: + return b"\xD6" + type_code.to_bytes(1, "big", signed=True) + payload + if length == 8: + return b"\xD7" + type_code.to_bytes(1, "big", signed=True) + payload + if length == 16: + return b"\xD8" + type_code.to_bytes(1, "big", signed=True) + payload + raise ValueError(f"Unsupported ext payload size {length}") + + +def _pack_array(items): + length = len(items) + if length <= 15: + prefix = bytes([0x90 | length]) + elif length <= 0xFFFF: + prefix = b"\xDC" + length.to_bytes(2, "big") + else: + prefix = b"\xDD" + length.to_bytes(4, "big") + return prefix + b"".join(_pack_obj(item) for item in items) + + +def _pack_map(mapping): + items = list(mapping.items()) + length = len(items) + if length <= 15: + prefix = bytes([0x80 | length]) + elif length <= 0xFFFF: + prefix = b"\xDE" + length.to_bytes(2, "big") + else: + prefix = b"\xDF" + length.to_bytes(4, "big") + encoded = [] + for key, value in items: + encoded.append(_pack_obj(key)) + encoded.append(_pack_obj(value)) + return prefix + b"".join(encoded) + + +def _pack_obj(value): + if value is None: + return b"\xC0" + if value is False: + return b"\xC2" + if value is True: + return b"\xC3" + if isinstance(value, int): + return _pack_uint(value) + if isinstance(value, bool): + return _pack_bool(value) + if isinstance(value, str): + return _pack_str(value) + if isinstance(value, bytes): + return _pack_bin(value) + if isinstance(value, tuple) and len(value) == 3 and value[0] == "__ext__": + return _pack_ext(value[1], value[2]) + if isinstance(value, list): + return _pack_array(value) + if isinstance(value, dict): + return _pack_map(value) + raise TypeError(f"Unsupported value type {type(value)!r}") + + +def _message_mode_payload(tag, body): + return _pack_obj([tag, TEST_TS, body]) + + +def _message_mode_eventtime_payload(tag, body, *, seconds, nanoseconds): + ext_payload = seconds.to_bytes(4, "big") + nanoseconds.to_bytes(4, "big") + return _pack_obj([tag, ("__ext__", 0, ext_payload), body]) + + +def _forward_mode_payload(tag, entries): + return _pack_obj([tag, [[TEST_TS, entry] for entry in entries]]) + + +def _packed_forward_payload(tag, packed_entries, *, compressed=None): + options = {} + if compressed: + options["compressed"] = compressed + payload = [tag, packed_entries] + if options: + payload.append(options) + return _pack_obj(payload) + + +def _gzip_bytes(data): + return gzip.compress(data) + + +def _zstd_bytes(data): + if not shutil.which("zstd"): + pytest.skip("zstd binary is required for this test") + + result = subprocess.run( + ["zstd", "-q", "-c"], + input=data, + capture_output=True, + check=True, + ) + return result.stdout + + +def _send_tcp_payload(port, payload): + with socket.create_connection(("127.0.0.1", port), timeout=5) as sock: + sock.sendall(payload) + + +def _send_unix_payload(path, payload): + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.settimeout(5) + sock.connect(path) + sock.sendall(payload) + + +def _send_tls_payload(port, payload, cafile): + context = ssl.create_default_context(cafile=cafile) + with socket.create_connection(("127.0.0.1", port), timeout=5) as raw_sock: + with context.wrap_socket(raw_sock, server_hostname="localhost") as tls_sock: + tls_sock.sendall(payload) + + +def _recv_msgpack_value(sock): + sock.settimeout(5) + data = sock.recv(4096) + assert data + value, offset = _unpack_obj(data, 0) + assert offset == len(data) + return value + + +def _decode_str_like(raw): + try: + return raw.decode() + except UnicodeDecodeError: + return raw + + +def _unpack_obj(data, offset): + first = data[offset] + offset += 1 + + if first <= 0x7F: + return first, offset + if 0x80 <= first <= 0x8F: + size = first & 0x0F + result = {} + for _ in range(size): + key, offset = _unpack_obj(data, offset) + value, offset = _unpack_obj(data, offset) + result[key] = value + return result, offset + if 0x90 <= first <= 0x9F: + size = first & 0x0F + result = [] + for _ in range(size): + value, offset = _unpack_obj(data, offset) + result.append(value) + return result, offset + if 0xA0 <= first <= 0xBF: + size = first & 0x1F + raw = data[offset:offset + size] + return _decode_str_like(raw), offset + size + if first == 0xC0: + return None, offset + if first == 0xC2: + return False, offset + if first == 0xC3: + return True, offset + if first == 0xC4: + size = data[offset] + offset += 1 + return data[offset:offset + size], offset + size + if first == 0xCC: + return data[offset], offset + 1 + if first == 0xCD: + return int.from_bytes(data[offset:offset + 2], "big"), offset + 2 + if first == 0xCE: + return int.from_bytes(data[offset:offset + 4], "big"), offset + 4 + if first == 0xCF: + return int.from_bytes(data[offset:offset + 8], "big"), offset + 8 + if first == 0xD9: + size = data[offset] + offset += 1 + raw = data[offset:offset + size] + return _decode_str_like(raw), offset + size + if first == 0xDA: + size = int.from_bytes(data[offset:offset + 2], "big") + offset += 2 + raw = data[offset:offset + size] + return _decode_str_like(raw), offset + size + if first == 0xDE: + size = int.from_bytes(data[offset:offset + 2], "big") + offset += 2 + result = {} + for _ in range(size): + key, offset = _unpack_obj(data, offset) + value, offset = _unpack_obj(data, offset) + result[key] = value + return result, offset + + raise ValueError(f"Unsupported MessagePack type 0x{first:02x}") + + +def _sha512_hex(*parts): + hasher = hashlib.sha512() + for part in parts: + if isinstance(part, str): + part = part.encode() + hasher.update(part) + return hasher.hexdigest() + + +def _secure_forward_handshake(sock, *, username, password, shared_key, hostname="client-node", shared_key_salt="client-salt-1234"): + helo = _recv_msgpack_value(sock) + assert helo[0] == "HELO" + + helo_options = helo[1] + nonce = helo_options["nonce"] + auth_salt = helo_options["auth"] + + shared_key_digest = _sha512_hex(shared_key_salt, hostname, nonce, shared_key) + password_digest = _sha512_hex(auth_salt, username, password) + + ping = _pack_obj(["PING", hostname, shared_key_salt, shared_key_digest, username, password_digest]) + sock.sendall(ping) + + return _recv_msgpack_value(sock) + + +def test_in_forward_message_mode_tcp(): + service = Service("in_forward.yaml") + service.start() + + try: + payload = _message_mode_payload(TEST_TAG, {"message": "message-mode"}) + _send_tcp_payload(service.flb_listener_port, payload) + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert records[0]["message"] == "message-mode" + + +def test_in_forward_message_mode_partial_tcp_writes(): + service = Service("in_forward.yaml") + service.start() + + try: + payload = _message_mode_payload(TEST_TAG, {"message": "partial"}) + midpoint = len(payload) // 2 + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(payload[:midpoint]) + sock.sendall(payload[midpoint:]) + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert records[0]["message"] == "partial" + + +def test_in_forward_message_mode_eventtime_ext(): + service = Service("in_forward.yaml") + service.start() + + try: + payload = _message_mode_eventtime_payload( + TEST_TAG, + {"message": "eventtime"}, + seconds=TEST_TS, + nanoseconds=123456789, + ) + _send_tcp_payload(service.flb_listener_port, payload) + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert records[0]["message"] == "eventtime" + + +def test_in_forward_forward_mode_multiple_entries(): + service = Service("in_forward.yaml") + service.start() + + try: + payload = _forward_mode_payload(TEST_TAG, [{"message": "entry-1"}, {"message": "entry-2"}]) + _send_tcp_payload(service.flb_listener_port, payload) + records = service.wait_for_record_count(2) + finally: + service.stop() + + assert [record["message"] for record in records[:2]] == ["entry-1", "entry-2"] + + +def test_in_forward_packed_forward_gzip(): + service = Service("in_forward.yaml") + service.start() + + try: + packed_entries = _pack_obj([TEST_TS, {"message": "gzip-packed-forward"}]) + payload = _packed_forward_payload(TEST_TAG, _gzip_bytes(packed_entries), compressed="gzip") + _send_tcp_payload(service.flb_listener_port, payload) + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert records[0]["message"] == "gzip-packed-forward" + + +def test_in_forward_packed_forward_uncompressed_with_ack(): + service = Service("in_forward.yaml") + service.start() + + try: + chunk = "packed-chunk-001" + packed_entries = _pack_obj([TEST_TS, {"message": "packed-uncompressed"}]) + payload = _packed_forward_payload(TEST_TAG, packed_entries, compressed=None) + payload = _pack_obj([TEST_TAG, packed_entries, {"chunk": chunk}]) + + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(payload) + ack = _recv_msgpack_value(sock) + + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert ack == {"ack": chunk} + assert records[0]["message"] == "packed-uncompressed" + + +def test_in_forward_packed_forward_zstd(): + service = Service("in_forward.yaml") + service.start() + + try: + packed_entries = _pack_obj([TEST_TS, {"message": "zstd-packed-forward"}]) + payload = _packed_forward_payload(TEST_TAG, _zstd_bytes(packed_entries), compressed="zstd") + _send_tcp_payload(service.flb_listener_port, payload) + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert records[0]["message"] == "zstd-packed-forward" + + +def test_in_forward_message_mode_chunk_ack_and_metadata(): + service = Service("in_forward.yaml") + service.start() + + try: + chunk = "chunk-001" + payload = _pack_obj( + [ + TEST_TAG, + TEST_TS, + {"message": "metadata-ack"}, + {"chunk": chunk, "metadata": {"source": "suite", "path": "message-mode"}}, + ] + ) + + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(payload) + ack = _recv_msgpack_value(sock) + + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert ack == {"ack": chunk} + assert records[0]["message"] == "metadata-ack" + + +def test_in_forward_forward_mode_chunk_ack(): + service = Service("in_forward.yaml") + service.start() + + try: + chunk = "forward-chunk-001" + payload = _pack_obj( + [ + TEST_TAG, + [[TEST_TS, {"message": "forward-ack"}]], + {"chunk": chunk}, + ] + ) + + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(payload) + ack = _recv_msgpack_value(sock) + + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert ack == {"ack": chunk} + assert records[0]["message"] == "forward-ack" + + +def test_in_forward_tag_prefix_routes_records(): + service = Service("in_forward_tag_prefix.yaml") + service.start() + + try: + payload = _message_mode_payload(TEST_TAG, {"message": "prefixed"}) + _send_tcp_payload(service.flb_listener_port, payload) + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert records[0]["message"] == "prefixed" + + +def test_in_forward_forced_input_tag_overrides_incoming_tag(): + service = Service("in_forward_forced_tag.yaml") + service.start() + + try: + payload = _message_mode_payload("ignored.incoming.tag", {"message": "forced-tag"}) + _send_tcp_payload(service.flb_listener_port, payload) + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert records[0]["message"] == "forced-tag" + + +def test_in_forward_unix_socket_message_mode(): + service = Service("in_forward_unix.yaml", use_unix_socket=True) + service.start() + + try: + service.service.wait_for_condition( + lambda: os.path.exists(service.socket_path), + timeout=10, + interval=0.2, + description="forward unix socket", + ) + payload = _message_mode_payload(TEST_TAG, {"message": "unix-socket"}) + _send_unix_payload(service.socket_path, payload) + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert records[0]["message"] == "unix-socket" + + +def test_in_forward_unix_socket_permissions(): + service = Service("in_forward_unix_perm.yaml", use_unix_socket=True) + service.start() + + try: + service.service.wait_for_condition( + lambda: os.path.exists(service.socket_path), + timeout=10, + interval=0.2, + description="forward unix socket with permissions", + ) + mode = os.stat(service.socket_path).st_mode & 0o777 + finally: + service.stop() + + assert mode == 0o600 + + +def test_in_forward_tls_message_mode(): + service = Service("in_forward_tls.yaml") + service.start() + + try: + payload = _message_mode_payload(TEST_TAG, {"message": "tls-message"}) + _send_tls_payload(service.flb_listener_port, payload, service.tls_crt_file) + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert records[0]["message"] == "tls-message" + + +def test_in_forward_secure_forward_auth_success(): + service = Service("in_forward_secure.yaml") + service.start() + + try: + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + pong = _secure_forward_handshake( + sock, + username=SECURE_USERNAME, + password=SECURE_PASSWORD, + shared_key=SECURE_SHARED_KEY, + ) + sock.sendall(_message_mode_payload(TEST_TAG, {"message": "secure-success"})) + + records = service.wait_for_record_count(1) + finally: + service.stop() + + assert pong[0] == "PONG" + assert pong[1] is True + assert pong[2] == "" + assert pong[3] == SECURE_SELF_HOSTNAME + assert records[0]["message"] == "secure-success" + + +def test_in_forward_secure_forward_auth_failure(): + service = Service("in_forward_secure.yaml") + service.start() + + try: + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + pong = _secure_forward_handshake( + sock, + username=SECURE_USERNAME, + password="wrong-password", + shared_key=SECURE_SHARED_KEY, + ) + sock.sendall(_message_mode_payload(TEST_TAG, {"message": "should-not-pass"})) + + with pytest.raises(TimeoutError): + service.wait_for_record_count(1, timeout=2) + finally: + service.stop() + + assert pong[0] == "PONG" + assert pong[1] is False + assert "username/password mismatch" in pong[2] + + +def test_in_forward_e2e_forward_receiver_preserves_metadata_and_signal_options(): + service = ForwardReceiverService("in_forward_to_forward_receiver.yaml") + service.start() + + try: + payload = _pack_obj( + [ + TEST_TAG, + TEST_TS, + {"message": "end-to-end-forward"}, + {"metadata": {"source": "suite", "scope": "log"}, "chunk": "input-chunk"}, + ] + ) + _send_tcp_payload(service.flb_listener_port, payload) + messages = service.wait_for_forward_messages(1) + finally: + service.stop() + + message = messages[0] + record = message["records"][0] + raw_record = record["raw"] + + assert message["mode"] == "forward" + assert message["tag"] == TEST_TAG + assert message["options"]["fluent_signal"] == 0 + assert message["options"]["size"] == 1 + assert message["options"]["chunk"] + assert "metadata" not in message["options"] + assert len(raw_record) == 2 + assert len(raw_record[0]) == 2 + assert raw_record[0][1] == {"source": "suite", "scope": "log"} + assert record["body"]["message"] == "end-to-end-forward" + assert record["metadata"]["source"] == "suite" + assert record["metadata"]["scope"] == "log" + + +def test_in_forward_e2e_forward_receiver_gzip_preserves_metadata_and_signal_options(): + service = ForwardReceiverService("in_forward_to_forward_receiver_gzip.yaml") + service.start() + + try: + payload = _pack_obj( + [ + TEST_TAG, + TEST_TS, + {"message": "end-to-end-gzip"}, + {"metadata": {"source": "suite", "scope": "gzip"}, "chunk": "input-chunk-gzip"}, + ] + ) + _send_tcp_payload(service.flb_listener_port, payload) + messages = service.wait_for_forward_messages(1) + finally: + service.stop() + + message = messages[0] + record = message["records"][0] + raw_record = record["raw"] + + assert message["mode"] == "packed_forward" + assert message["tag"] == TEST_TAG + assert message["options"]["compressed"] == "gzip" + assert message["options"]["fluent_signal"] == 0 + assert message["options"]["size"] == 1 + assert message["options"]["chunk"] + assert "metadata" not in message["options"] + assert len(raw_record) == 2 + assert len(raw_record[0]) == 2 + assert raw_record[0][1] == {"source": "suite", "scope": "gzip"} + assert record["body"]["message"] == "end-to-end-gzip" + assert record["metadata"]["source"] == "suite" + assert record["metadata"]["scope"] == "gzip" + + +def test_out_forward_metrics_signal_e2e_with_forward_receiver(): + service = ForwardReceiverService("in_opentelemetry_to_forward_receiver.yaml") + service.start() + + try: + service.send_json_as_otel_protobuf("test_metrics_001.in.json", "metrics") + messages = service.wait_for_forward_messages(1) + finally: + service.stop() + + message = messages[0] + record_payload = message["records"][0]["raw"] + + assert message["mode"] == "packed_forward" + assert message["tag"] == "v1_metrics" + assert message["options"]["fluent_signal"] == 1 + assert message["options"]["chunk"] + assert "size" not in message["options"] + assert len(message["records"]) >= 1 + assert record_payload["meta"]["external"]["scope"]["metadata"]["name"] == "metrics-scope" + assert record_payload["metrics"][0]["meta"]["opts"]["name"] == "requests_total" + assert record_payload["metrics"][0]["values"][0]["labels"] == ["checkout"] + assert record_payload["metrics"][0]["values"][0]["value_int64"] == 42 + + +def test_out_forward_traces_signal_e2e_with_forward_receiver(): + service = ForwardReceiverService("in_opentelemetry_to_forward_receiver.yaml") + service.start() + + try: + service.send_json_as_otel_protobuf("test_traces_001.in.json", "traces") + messages = service.wait_for_forward_messages(1) + finally: + service.stop() + + message = messages[0] + record_payload = message["records"][0]["raw"] + + assert message["mode"] == "packed_forward" + assert message["tag"] == "v1_traces" + assert message["options"]["fluent_signal"] == 2 + assert message["options"]["chunk"] + assert "size" not in message["options"] + assert len(message["records"]) >= 1 + span = record_payload["resourceSpans"][0]["scope_spans"][0]["spans"][0] + assert record_payload["resourceSpans"][0]["scope_spans"][0]["scope"]["name"] == "trace-scope" + assert record_payload["resourceSpans"][0]["resource"]["attributes"]["service.name"] == "checkout" + assert span["name"] == "checkout-span" + assert span["attributes"]["http.method"] == "GET" + + +def test_in_forward_storage_limit_single_output_prefers_actual_chunk_deletion(): + service = StorageLimitService("in_forward_storage_limit_single_output.yaml") + service.start() + + try: + configure_http_response(status_code=500, body={"status": "retry"}) + + _send_tcp_payload(service.flb_listener_port, _message_mode_payload("solo.one", {"message": "single-one"})) + _send_tcp_payload(service.flb_listener_port, _message_mode_payload("solo.two", {"message": "single-two"})) + + service.service.wait_for_condition( + lambda: service.count_chunk_files() == 2, + timeout=10, + interval=0.2, + description="2 chunk files after two solo messages", + ) + + _send_tcp_payload(service.flb_listener_port, _message_mode_payload("solo.three", {"message": "single-three"})) + + def single_output_eviction_snapshot(): + chunk_contents = service.chunk_file_contents() + + if len(chunk_contents) != 2: + return None + + if any(b"solo.one" in content for content in chunk_contents): + return None + + if not any(b"solo.three" in content for content in chunk_contents): + return None + + return chunk_contents + + try: + chunk_contents = service.service.wait_for_condition( + single_output_eviction_snapshot, + timeout=10, + interval=0.2, + description="single-output storage eviction snapshot", + ) + except TimeoutError: + if service.count_chunk_files() >= 3: + pytest.skip( + "forward storage eviction preference is not supported by this Fluent Bit binary" + ) + raise + finally: + service.stop() + + assert not any(b"solo.one" in content for content in chunk_contents) + assert any(b"solo.three" in content for content in chunk_contents) + + +def test_in_forward_storage_limit_multi_output_prefers_deletable_solo_chunk(): + service = StorageLimitService("in_forward_storage_limit_multi_output.yaml") + service.start() + + try: + configure_http_response(status_code=500, body={"status": "retry"}) + + _send_tcp_payload(service.flb_listener_port, _message_mode_payload("shared.one", {"message": "shared-one"})) + _send_tcp_payload(service.flb_listener_port, _message_mode_payload("solo.one", {"message": "solo-one"})) + + service.service.wait_for_condition( + lambda: service.count_chunk_files() == 2, + timeout=10, + interval=0.2, + description="2 chunk files after shared and solo messages", + ) + + _send_tcp_payload(service.flb_listener_port, _message_mode_payload("solo.two", {"message": "solo-two"})) + + def multi_output_eviction_snapshot(): + chunk_contents = service.chunk_file_contents() + + if len(chunk_contents) != 2: + return None + + if any(b"solo.one" in content for content in chunk_contents): + return None + + if not any(b"shared.one" in content for content in chunk_contents): + return None + + if not any(b"solo.two" in content for content in chunk_contents): + return None + + return chunk_contents + + try: + chunk_contents = service.service.wait_for_condition( + multi_output_eviction_snapshot, + timeout=10, + interval=0.2, + description="multi-output storage eviction snapshot", + ) + except TimeoutError: + if service.count_chunk_files() >= 3: + pytest.skip( + "forward storage eviction preference is not supported by this Fluent Bit binary" + ) + raise + finally: + service.stop() + + assert any(b"shared.one" in content for content in chunk_contents) + assert not any(b"solo.one" in content for content in chunk_contents) diff --git a/tests/integration/scenarios/in_http/config/in_http_config b/tests/integration/scenarios/in_http/config/in_http_config new file mode 100644 index 00000000000..dde514b44d9 --- /dev/null +++ b/tests/integration/scenarios/in_http/config/in_http_config @@ -0,0 +1,17 @@ +[SERVICE] + Flush 1 + Log_Level info + HTTP_Server on + HTTP_Port ${FLUENT_BIT_HTTP_MONITORING_PORT} + +[INPUT] + Name http + Port ${FLUENT_BIT_TEST_LISTENER_PORT} + +[OUTPUT] + Name http + Match * + Host 127.0.0.1 + Port ${TEST_SUITE_HTTP_PORT} + URI /data + Format json diff --git a/tests/integration/scenarios/in_http/config/in_http_http1_cleartext.yaml b/tests/integration/scenarios/in_http/config/in_http_http1_cleartext.yaml new file mode 100644 index 00000000000..31a16c19c42 --- /dev/null +++ b/tests/integration/scenarios/in_http/config/in_http_http1_cleartext.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: http + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + + outputs: + - name: http + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_http/config/in_http_http1_tls.yaml b/tests/integration/scenarios/in_http/config/in_http_http1_tls.yaml new file mode 100644 index 00000000000..907717bf311 --- /dev/null +++ b/tests/integration/scenarios/in_http/config/in_http_http1_tls.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: http + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: http + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_http/config/in_http_http2_cleartext.yaml b/tests/integration/scenarios/in_http/config/in_http_http2_cleartext.yaml new file mode 100644 index 00000000000..31ac8be839f --- /dev/null +++ b/tests/integration/scenarios/in_http/config/in_http_http2_cleartext.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: http + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: off + + outputs: + - name: http + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_http/config/in_http_http2_tls.yaml b/tests/integration/scenarios/in_http/config/in_http_http2_tls.yaml new file mode 100644 index 00000000000..a9c54aa9cfb --- /dev/null +++ b/tests/integration/scenarios/in_http/config/in_http_http2_tls.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: http + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: http + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_http/config/in_http_oauth2.yaml b/tests/integration/scenarios/in_http/config/in_http_oauth2.yaml new file mode 100644 index 00000000000..1930d03372a --- /dev/null +++ b/tests/integration/scenarios/in_http/config/in_http_oauth2.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: http + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + oauth2.validate: true + oauth2.issuer: issuer + oauth2.jwks_url: http://127.0.0.1:${TEST_SUITE_HTTP_PORT}/jwks + oauth2.allowed_audience: audience + oauth2.allowed_clients: client1 + + outputs: + - name: http + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json + json_date_key: false diff --git a/tests/integration/scenarios/in_http/tests/data_files/sample_data.json b/tests/integration/scenarios/in_http/tests/data_files/sample_data.json new file mode 100644 index 00000000000..fce478267b9 --- /dev/null +++ b/tests/integration/scenarios/in_http/tests/data_files/sample_data.json @@ -0,0 +1,7 @@ +[ + { + "timestamp": "2024-07-29T10:00:00Z", + "message": "Este es un mensaje de prueba", + "level": "info" + } +] \ No newline at end of file diff --git a/tests/integration/scenarios/in_http/tests/test_in_http_001.py b/tests/integration/scenarios/in_http/tests/test_in_http_001.py new file mode 100644 index 00000000000..1d4d9c7d66e --- /dev/null +++ b/tests/integration/scenarios/in_http/tests/test_in_http_001.py @@ -0,0 +1,241 @@ +import http.client +import json +import os +import logging +import time + +import pytest +import requests + +from server.http_server import data_storage, http_server_run +from utils.http_matrix import PROTOCOL_CASES, run_curl_request +from utils.test_service import FluentBitTestService + +logger = logging.getLogger(__name__) +MOCK_VALID_JWT = ( + "eyJhbGciOiJSUzI1NiIsImtpZCI6InRlc3QiLCJ0eXAiOiJKV1QifQ." + "eyJleHAiOjE4OTM0NTYwMDAsImlzcyI6Imlzc3VlciIsImF1ZCI6ImF1ZGllbmNlIiwiYXpwIjoiY2xpZW50MSJ9." + "TqWs06LUpQa0FGLejnOkWAD6v562d5CUh2NwsJ7iAuae9-WNFBKU6mP1zAaoafla6o5npee7RfbSzZNFI4PKhqAj69789JjAYV7IW-GSuMwJejHdVOWmCc5lmcZPH0EVxEkHA6lFQxYQwDCrfQ8Sd4Q3vYCV6sLPENcuNpQi9ytjVjaZs_7ONH2oA-sZ7EUchqJJoIBPfjit2yYsq9NeemxCzYMtngiC-IX12eEfaQ1cVYPIjhhN_NaMvapznp-BW4gnXkNoAZ1S-p1axWWY-6UgRdMYOr0Hy5PHQ9fCuHJ6Z-blYdtuGavCUGHK5ghX-JdH1WJ51F89992dQ5yF_w" +) + +IN_HTTP_PROTOCOL_CONFIGS = { + "http1_cleartext": "in_http_http1_cleartext.yaml", + "http2_cleartext": "in_http_http2_cleartext.yaml", + "http1_tls": "in_http_http1_tls.yaml", + "http2_tls": "in_http_http2_tls.yaml", +} + +def create_connection(server, port): + return http.client.HTTPConnection(server, port) + +def create_headers(): + return { + 'Content-Type': 'application/json' + } + +def create_payload(json_filename): + try: + file_name = os.path.abspath(os.path.join(os.path.dirname(__file__), './data_files/', json_filename)) + with open(file_name, 'r') as file: + data = file.read().strip() + return data + except FileNotFoundError: + return json.dumps({"error": "File not found"}, indent=4) + except json.JSONDecodeError: + return json.dumps({"error": "Invalid JSON format"}, indent=4) + +def send_requests(conn, num_requests, headers, json_payload): + responses = [] + for i in range(num_requests): + conn.request("POST", "/", body=json_payload, headers=headers) + response = conn.getresponse() + responses.append({ + 'status': response.status, + 'reason': response.reason, + 'data': response.read().decode() + }) + return responses + + +def test_send_data(): + try: + service = Service("in_http_config") + service.start() + output = service.runtest_send_data('localhost', service.flb_listener_port, 'sample_data.json') + forwarded_payloads = service.read_forwarded_payloads() + logger.info(f"response: {output}") + service.stop() + assert len(output) > 0 + + # Verify response details if necessary + for response in output: + assert response['status'] == 201 + assert response['reason'] == 'Created' + + assert len(forwarded_payloads) == 1 + assert isinstance(forwarded_payloads[0], list) + assert len(forwarded_payloads[0]) == 1 + record = forwarded_payloads[0][0] + assert record["message"] == "Este es un mensaje de prueba" + assert record["level"] == "info" + assert record["timestamp"] == "2024-07-29T10:00:00Z" + except Exception as e: + logger.error(f"An error occurred: {e}") + if service.flb.process is not None: + service.stop() + raise + + +@pytest.mark.parametrize("case", PROTOCOL_CASES, ids=[case["id"] for case in PROTOCOL_CASES]) +def test_in_http_protocol_matrix(case): + service = Service(IN_HTTP_PROTOCOL_CONFIGS[case["config_key"]]) + service.start() + + scheme = "https" if case["use_tls"] else "http" + result = run_curl_request( + f"{scheme}://localhost:{service.flb_listener_port}/", + create_payload("sample_data.json"), + headers=["Content-Type: application/json"], + http_mode=case["http_mode"], + ca_cert_path=service.tls_crt_file if case["use_tls"] else None, + ) + forwarded_payloads = service.read_forwarded_payloads() + + service.stop() + + assert result["status_code"] == 201 + assert result["http_version"] == case["expected_http_version"] + assert len(forwarded_payloads) == 1 + assert forwarded_payloads[0][0]["message"] == "Este es un mensaje de prueba" + + +def test_in_http_rejects_bad_json(): + service = Service("in_http_config") + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/", + '{"message":"broken"', + headers=["Content-Type: application/json"], + http_mode="http1.1", + ) + + service.stop() + + assert result["status_code"] == 400 + + +def test_in_http_rejects_get_requests(): + service = Service("in_http_config") + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/", + None, + method="GET", + http_mode="http1.1", + ) + + service.stop() + + assert result["status_code"] >= 400 + + +def test_in_http_oauth2_requires_bearer_token(): + service = Service("in_http_oauth2.yaml") + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/", + create_payload("sample_data.json"), + headers=["Content-Type: application/json"], + http_mode="http1.1", + ) + + service.stop() + + assert result["status_code"] == 401 + assert data_storage["payloads"] == [] + + +def test_in_http_oauth2_accepts_valid_jwt(): + service = Service("in_http_oauth2.yaml") + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/", + create_payload("sample_data.json"), + headers=[ + "Content-Type: application/json", + f"Authorization: Bearer {MOCK_VALID_JWT}", + ], + http_mode="http1.1", + ) + forwarded_payloads = service.read_forwarded_payloads() + + service.stop() + + assert result["status_code"] == 201 + assert len(forwarded_payloads) == 1 + assert forwarded_payloads[0][0]["message"] == "Este es un mensaje de prueba" + + +class Service: + def __init__(self, config_file): + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), '../config/', config_file)) + test_path = os.path.dirname(os.path.abspath(__file__)) + cert_dir = os.path.abspath(os.path.join(test_path, "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["payloads"], + extra_env={ + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + }, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + http_server_run(service.test_suite_http_port) + self.service.wait_for_http_endpoint( + f"http://127.0.0.1:{service.test_suite_http_port}/ping", + timeout=10, + interval=0.5, + ) + + def _stop_receiver(self, service): + try: + requests.post(f"http://127.0.0.1:{service.test_suite_http_port}/shutdown", timeout=2) + except requests.RequestException: + pass + + def start(self): + self.service.start() + self.flb = self.service.flb + self.flb_listener_port = self.service.flb_listener_port + self.test_suite_http_port = self.service.test_suite_http_port + logger.info(f"Fluent Bit listener port: {self.flb_listener_port}") + logger.info(f"test suite http port: {self.test_suite_http_port}") + + def runtest_send_data(self, server, port, json_filename): + conn = create_connection(server, port) + headers = create_headers() + json_payload = create_payload(json_filename) + responses = send_requests(conn, 1, headers, json_payload) + conn.close() + return responses + + def read_forwarded_payloads(self, timeout=10): + deadline = time.time() + timeout + while time.time() < deadline: + if data_storage["payloads"]: + return data_storage["payloads"] + time.sleep(0.5) + raise TimeoutError("Timed out waiting for forwarded HTTP payloads") + + def stop(self): + self.service.stop() diff --git a/tests/integration/scenarios/in_http_max_connections/config/in_http_max_connections.yaml b/tests/integration/scenarios/in_http_max_connections/config/in_http_max_connections.yaml new file mode 100644 index 00000000000..cccb8bf133e --- /dev/null +++ b/tests/integration/scenarios/in_http_max_connections/config/in_http_max_connections.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: http + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http_server.max_connections: 1 + + outputs: + - name: http + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_http_max_connections/tests/test_in_http_max_connections_001.py b/tests/integration/scenarios/in_http_max_connections/tests/test_in_http_max_connections_001.py new file mode 100644 index 00000000000..c9e3766aa4b --- /dev/null +++ b/tests/integration/scenarios/in_http_max_connections/tests/test_in_http_max_connections_001.py @@ -0,0 +1,95 @@ +import os +import socket + +import pytest +import requests + +from server.http_server import data_storage, http_server_run +from utils.fluent_bit_manager import FluentBitStartupError +from utils.http_matrix import run_curl_request +from utils.test_service import FluentBitTestService + + +class Service: + def __init__(self): + self.config_file = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../config/in_http_max_connections.yaml") + ) + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["payloads"], + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + http_server_run(service.test_suite_http_port) + self.service.wait_for_http_endpoint(f"http://127.0.0.1:{service.test_suite_http_port}/ping") + + def _stop_receiver(self, service): + try: + requests.post(f"http://127.0.0.1:{service.test_suite_http_port}/shutdown", timeout=2) + except requests.RequestException: + pass + + def start(self): + self.service.start() + self.flb_listener_port = self.service.flb_listener_port + + def stop(self): + self.service.stop() + + +def test_in_http_max_connections_blocks_and_recovers(): + service = Service() + try: + service.start() + except FluentBitStartupError as error: + log_contents = "" + if service.service.flb and service.service.flb.log_file: + with open(service.service.flb.log_file, "r", encoding="utf-8", errors="replace") as file: + log_contents = file.read() + if "http_server.max_connections" in str(error) or "unknown configuration property 'http_server.max_connections'" in log_contents: + pytest.skip("http_server.max_connections is not supported by this Fluent Bit binary") + raise + + held_connection = None + try: + held_connection = socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=2) + held_connection.settimeout(2) + + overflow_rejected = False + try: + response = run_curl_request( + f"http://127.0.0.1:{service.flb_listener_port}/", + payload='{"message":"max-connections"}', + headers=["Content-Type: application/json"], + http_mode="http1.1", + ) + overflow_rejected = response["status_code"] != 201 + except Exception: + overflow_rejected = True + + assert overflow_rejected + finally: + if held_connection: + held_connection.close() + + accepted = run_curl_request( + f"http://127.0.0.1:{service.flb_listener_port}/", + payload='{"message":"max-connections"}', + headers=["Content-Type: application/json"], + http_mode="http1.1", + ) + forwarded_payloads = service.service.wait_for_condition( + lambda: data_storage["payloads"] if data_storage["payloads"] else None, + timeout=10, + interval=0.5, + description="forwarded max-connections payload", + ) + + service.stop() + + assert accepted["status_code"] == 201 + assert forwarded_payloads[0][0]["message"] == "max-connections" diff --git a/tests/integration/scenarios/in_mqtt/config/in_mqtt.yaml b/tests/integration/scenarios/in_mqtt/config/in_mqtt.yaml new file mode 100644 index 00000000000..1b78040e39b --- /dev/null +++ b/tests/integration/scenarios/in_mqtt/config/in_mqtt.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: mqtt + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_mqtt/config/in_mqtt_payload_key.yaml b/tests/integration/scenarios/in_mqtt/config/in_mqtt_payload_key.yaml new file mode 100644 index 00000000000..02522428b89 --- /dev/null +++ b/tests/integration/scenarios/in_mqtt/config/in_mqtt_payload_key.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: mqtt + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + payload_key: payload_k + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_mqtt/tests/test_in_mqtt_001.py b/tests/integration/scenarios/in_mqtt/tests/test_in_mqtt_001.py new file mode 100644 index 00000000000..5482bca6d35 --- /dev/null +++ b/tests/integration/scenarios/in_mqtt/tests/test_in_mqtt_001.py @@ -0,0 +1,227 @@ +import os +import socket +import time + +import requests + +from server.http_server import data_storage, http_server_run +from utils.test_service import FluentBitTestService + + +MQTT_CONNECT_PACKET = bytes( + [0x10, 0x0A, 0x00, 0x04, ord("M"), ord("Q"), ord("T"), ord("T"), 0x04, 0xCE, 0x00, 0x0A] +) +MQTT_TRUNCATED_QOS1_PUBLISH_PACKET = bytes([0x32, 0x04, 0x00, 0x01, ord("X"), 0x00]) +MQTT_EMPTY_PUBLISH_PACKET = bytes([0x30, 0x03, 0x00, 0x01, ord("a")]) +MQTT_INVALID_TOPIC_LENGTH_PACKET = bytes([0x30, 0x04, 0x00, 0x05, ord("a"), ord("b")]) +MQTT_VALID_PAYLOAD = b'{"key":"val"}' +MQTT_INVALID_JSON_PAYLOAD = b'{"key"' + + +class Service: + def __init__(self, config_file="in_mqtt.yaml"): + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["payloads"], + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + http_server_run(service.test_suite_http_port) + self.service.wait_for_http_endpoint( + f"http://127.0.0.1:{service.test_suite_http_port}/ping", + timeout=10, + interval=0.5, + ) + + def _stop_receiver(self, service): + try: + requests.post(f"http://127.0.0.1:{service.test_suite_http_port}/shutdown", timeout=2) + except requests.RequestException: + pass + + def start(self): + self.service.start() + self.flb_listener_port = self.service.flb_listener_port + + def stop(self): + self.service.stop() + + def assert_running(self): + assert self.service.flb.process is not None + assert self.service.flb.process.poll() is None + + def read_forwarded_payloads(self, timeout=10): + return self.service.wait_for_condition( + lambda: data_storage["payloads"] if data_storage["payloads"] else None, + timeout=timeout, + interval=0.2, + description="forwarded mqtt payloads", + ) + + +def _recv_connack(sock): + response = sock.recv(4) + assert response + assert response[0] == 0x20 + + +def _build_publish_packet(payload, topic=b"a/b"): + remaining_length = 2 + len(topic) + len(payload) + return bytes([0x30, remaining_length]) + len(topic).to_bytes(2, "big") + topic + payload + + +def _assert_record(payloads): + assert len(payloads) == 1 + assert isinstance(payloads[0], list) + assert len(payloads[0]) == 1 + record = payloads[0][0] + assert record["topic"] == "a/b" + assert record["key"] == "val" + + +def _assert_payload_key_record(payloads): + assert len(payloads) == 1 + assert isinstance(payloads[0], list) + assert len(payloads[0]) == 1 + record = payloads[0][0] + assert record["topic"] == "a/b" + assert record["payload_k"]["key"] == "val" + + +def test_in_mqtt_publish_forwards_json(): + service = Service() + service.start() + + try: + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(MQTT_CONNECT_PACKET) + _recv_connack(sock) + sock.sendall(_build_publish_packet(MQTT_VALID_PAYLOAD)) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_record(payloads) + + +def test_in_mqtt_truncated_publish_recovers(): + service = Service() + service.start() + + try: + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(MQTT_CONNECT_PACKET) + _recv_connack(sock) + sock.sendall(MQTT_TRUNCATED_QOS1_PUBLISH_PACKET) + + time.sleep(0.2) + assert data_storage["payloads"] == [] + service.assert_running() + + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as recovery_sock: + recovery_sock.sendall(MQTT_CONNECT_PACKET) + _recv_connack(recovery_sock) + recovery_sock.sendall(_build_publish_packet(MQTT_VALID_PAYLOAD)) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_record(payloads) + + +def test_in_mqtt_empty_publish_recovers(): + service = Service() + service.start() + + try: + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(MQTT_CONNECT_PACKET) + _recv_connack(sock) + sock.sendall(MQTT_EMPTY_PUBLISH_PACKET) + + time.sleep(0.2) + assert data_storage["payloads"] == [] + service.assert_running() + + sock.sendall(_build_publish_packet(MQTT_VALID_PAYLOAD)) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_record(payloads) + + +def test_in_mqtt_invalid_topic_length_recovers(): + service = Service() + service.start() + + try: + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(MQTT_CONNECT_PACKET) + _recv_connack(sock) + sock.sendall(MQTT_INVALID_TOPIC_LENGTH_PACKET) + + time.sleep(0.2) + assert data_storage["payloads"] == [] + service.assert_running() + + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as recovery_sock: + recovery_sock.sendall(MQTT_CONNECT_PACKET) + _recv_connack(recovery_sock) + recovery_sock.sendall(_build_publish_packet(MQTT_VALID_PAYLOAD)) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_record(payloads) + + +def test_in_mqtt_invalid_json_payload_recovers(): + service = Service() + service.start() + + try: + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(MQTT_CONNECT_PACKET) + _recv_connack(sock) + sock.sendall(_build_publish_packet(MQTT_INVALID_JSON_PAYLOAD)) + + time.sleep(0.2) + assert data_storage["payloads"] == [] + service.assert_running() + + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as recovery_sock: + recovery_sock.sendall(MQTT_CONNECT_PACKET) + _recv_connack(recovery_sock) + recovery_sock.sendall(_build_publish_packet(MQTT_VALID_PAYLOAD)) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_record(payloads) + + +def test_in_mqtt_payload_key_wraps_payload(): + service = Service("in_mqtt_payload_key.yaml") + service.start() + + try: + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(MQTT_CONNECT_PACKET) + _recv_connack(sock) + sock.sendall(_build_publish_packet(MQTT_VALID_PAYLOAD)) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_payload_key_record(payloads) diff --git a/tests/integration/scenarios/in_opentelemetry/config/001-fluent-bit.yaml b/tests/integration/scenarios/in_opentelemetry/config/001-fluent-bit.yaml new file mode 100644 index 00000000000..6e3d55cd3bf --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/001-fluent-bit.yaml @@ -0,0 +1,47 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + processors: + logs: + - name: content_modifier + context: otel_resource_attributes + action: upsert + key: "aaa" + value: "bbb" + + - name: content_modifier + context: otel_resource_attributes + action: delete + key: "service.name" + + - name: content_modifier + context: otel_scope_attributes + action: upsert + key: "mynewscope" + value: "123" + + - name: content_modifier + context: otel_scope_name + action: upsert + value: "new scope name" + + - name: content_modifier + context: otel_scope_version + action: upsert + value: "3.1.0" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/002-fluent-bit.yaml b/tests/integration/scenarios/in_opentelemetry/config/002-fluent-bit.yaml new file mode 100644 index 00000000000..988537733f0 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/002-fluent-bit.yaml @@ -0,0 +1,32 @@ +service: + flush: 1 + log_level: debug + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message:": "something", + "spanid": "251484295a9df731", + "trace_id": { + "w3c": "63560bd4d8de74fae7d1e4160f2ee099" + }, + "traceid": "63560bd4d8de74fae7d1e4160f2ee099" + } + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /v1/logs + logs_body_key: $message + logs_trace_id_message_key: $traceid + logs_span_id_message_key: $spanid diff --git a/tests/integration/scenarios/in_opentelemetry/config/003-stdout-otlp-json.yaml b/tests/integration/scenarios/in_opentelemetry/config/003-stdout-otlp-json.yaml new file mode 100644 index 00000000000..63f5a8af2c6 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/003-stdout-otlp-json.yaml @@ -0,0 +1,15 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: stdout + match: '*' + format: otlp_json diff --git a/tests/integration/scenarios/in_opentelemetry/config/004-stdout-otlp-json-pretty.yaml b/tests/integration/scenarios/in_opentelemetry/config/004-stdout-otlp-json-pretty.yaml new file mode 100644 index 00000000000..4258f74f17e --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/004-stdout-otlp-json-pretty.yaml @@ -0,0 +1,15 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: stdout + match: '*' + format: otlp_json_pretty diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext.yaml new file mode 100644 index 00000000000..453bdd9d330 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext.yaml @@ -0,0 +1,49 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + processors: + logs: + - name: content_modifier + context: otel_resource_attributes + action: upsert + key: "aaa" + value: "bbb" + + - name: content_modifier + context: otel_resource_attributes + action: delete + key: "service.name" + + - name: content_modifier + context: otel_scope_attributes + action: upsert + key: "mynewscope" + value: "123" + + - name: content_modifier + context: otel_scope_name + action: upsert + value: "new scope name" + + - name: content_modifier + context: otel_scope_version + action: upsert + value: "3.1.0" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_oauth2.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_oauth2.yaml new file mode 100644 index 00000000000..ec8c6479be5 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_oauth2.yaml @@ -0,0 +1,54 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + oauth2.validate: true + oauth2.issuer: issuer + oauth2.jwks_url: http://127.0.0.1:${TEST_SUITE_JWKS_PORT}/jwks + oauth2.allowed_audience: audience + oauth2.allowed_clients: client1 + processors: + logs: + - name: content_modifier + context: otel_resource_attributes + action: upsert + key: "aaa" + value: "bbb" + + - name: content_modifier + context: otel_resource_attributes + action: delete + key: "service.name" + + - name: content_modifier + context: otel_scope_attributes + action: upsert + key: "mynewscope" + value: "123" + + - name: content_modifier + context: otel_scope_name + action: upsert + value: "new scope name" + + - name: content_modifier + context: otel_scope_version + action: upsert + value: "3.1.0" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers.yaml new file mode 100644 index 00000000000..1bb6e6576b0 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_cleartext_workers.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + http_server.workers: 4 + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_tls.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_tls.yaml new file mode 100644 index 00000000000..5f1eb3d2f22 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_tls.yaml @@ -0,0 +1,51 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + processors: + logs: + - name: content_modifier + context: otel_resource_attributes + action: upsert + key: "aaa" + value: "bbb" + + - name: content_modifier + context: otel_resource_attributes + action: delete + key: "service.name" + + - name: content_modifier + context: otel_scope_attributes + action: upsert + key: "mynewscope" + value: "123" + + - name: content_modifier + context: otel_scope_name + action: upsert + value: "new scope name" + + - name: content_modifier + context: otel_scope_version + action: upsert + value: "3.1.0" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_tls_oauth2.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_tls_oauth2.yaml new file mode 100644 index 00000000000..46b602888d0 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_tls_oauth2.yaml @@ -0,0 +1,57 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: on + tls.verify: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + oauth2.validate: true + oauth2.issuer: issuer + oauth2.jwks_url: http://127.0.0.1:${TEST_SUITE_JWKS_PORT}/jwks + oauth2.allowed_audience: audience + oauth2.allowed_clients: client1 + processors: + logs: + - name: content_modifier + context: otel_resource_attributes + action: upsert + key: "aaa" + value: "bbb" + + - name: content_modifier + context: otel_resource_attributes + action: delete + key: "service.name" + + - name: content_modifier + context: otel_scope_attributes + action: upsert + key: "mynewscope" + value: "123" + + - name: content_modifier + context: otel_scope_name + action: upsert + value: "new scope name" + + - name: content_modifier + context: otel_scope_version + action: upsert + value: "3.1.0" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_tls_workers.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_tls_workers.yaml new file mode 100644 index 00000000000..40498f741ca --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http1_tls_workers.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + http_server.workers: 4 + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_cleartext.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_cleartext.yaml new file mode 100644 index 00000000000..017f7fd2182 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_cleartext.yaml @@ -0,0 +1,49 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: off + processors: + logs: + - name: content_modifier + context: otel_resource_attributes + action: upsert + key: "aaa" + value: "bbb" + + - name: content_modifier + context: otel_resource_attributes + action: delete + key: "service.name" + + - name: content_modifier + context: otel_scope_attributes + action: upsert + key: "mynewscope" + value: "123" + + - name: content_modifier + context: otel_scope_name + action: upsert + value: "new scope name" + + - name: content_modifier + context: otel_scope_version + action: upsert + value: "3.1.0" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_cleartext_oauth2.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_cleartext_oauth2.yaml new file mode 100644 index 00000000000..cb00af32759 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_cleartext_oauth2.yaml @@ -0,0 +1,54 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: off + oauth2.validate: true + oauth2.issuer: issuer + oauth2.jwks_url: http://127.0.0.1:${TEST_SUITE_JWKS_PORT}/jwks + oauth2.allowed_audience: audience + oauth2.allowed_clients: client1 + processors: + logs: + - name: content_modifier + context: otel_resource_attributes + action: upsert + key: "aaa" + value: "bbb" + + - name: content_modifier + context: otel_resource_attributes + action: delete + key: "service.name" + + - name: content_modifier + context: otel_scope_attributes + action: upsert + key: "mynewscope" + value: "123" + + - name: content_modifier + context: otel_scope_name + action: upsert + value: "new scope name" + + - name: content_modifier + context: otel_scope_version + action: upsert + value: "3.1.0" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_cleartext_workers.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_cleartext_workers.yaml new file mode 100644 index 00000000000..fba8fcfe915 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_cleartext_workers.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: off + http_server.workers: 4 + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_tls.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_tls.yaml new file mode 100644 index 00000000000..88b47cc10ba --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_tls.yaml @@ -0,0 +1,51 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + processors: + logs: + - name: content_modifier + context: otel_resource_attributes + action: upsert + key: "aaa" + value: "bbb" + + - name: content_modifier + context: otel_resource_attributes + action: delete + key: "service.name" + + - name: content_modifier + context: otel_scope_attributes + action: upsert + key: "mynewscope" + value: "123" + + - name: content_modifier + context: otel_scope_name + action: upsert + value: "new scope name" + + - name: content_modifier + context: otel_scope_version + action: upsert + value: "3.1.0" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_tls_oauth2.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_tls_oauth2.yaml new file mode 100644 index 00000000000..23654540b8f --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_tls_oauth2.yaml @@ -0,0 +1,57 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: on + tls.verify: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + oauth2.validate: true + oauth2.issuer: issuer + oauth2.jwks_url: http://127.0.0.1:${TEST_SUITE_JWKS_PORT}/jwks + oauth2.allowed_audience: audience + oauth2.allowed_clients: client1 + processors: + logs: + - name: content_modifier + context: otel_resource_attributes + action: upsert + key: "aaa" + value: "bbb" + + - name: content_modifier + context: otel_resource_attributes + action: delete + key: "service.name" + + - name: content_modifier + context: otel_scope_attributes + action: upsert + key: "mynewscope" + value: "123" + + - name: content_modifier + context: otel_scope_name + action: upsert + value: "new scope name" + + - name: content_modifier + context: otel_scope_version + action: upsert + value: "3.1.0" + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_tls_workers.yaml b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_tls_workers.yaml new file mode 100644 index 00000000000..12d7ab0c2d1 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/config/otlp_http2_tls_workers.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: on + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + http_server.workers: 4 + + outputs: + - name: stdout + match: '*' + + - name: opentelemetry + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} diff --git a/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_logs_001.in.json b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_logs_001.in.json new file mode 100644 index 00000000000..a27b1455b4d --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_logs_001.in.json @@ -0,0 +1,128 @@ +{ + "resource_logs": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "string_value": "example-service" + } + } + ] + }, + "scope_logs": [ + { + "scope": { + "name": "example-scope", + "version": "1.0.0" + }, + "log_records": [ + { + "time_unix_nano": "1650917400000000000", + "severity_number": 9, + "severity_text": "INFO", + "body": { + "string_value": "This is an example log message." + }, + "attributes": [ + { + "key": "example_key", + "value": { + "string_value": "example_value" + } + } + ] + } + ] + }, + { + "scope": { + "name": "example-scope", + "version": "2.0.0" + }, + "log_records": [ + { + "time_unix_nano": "1650917400000000000", + "severity_number": 9, + "severity_text": "INFO", + "body": { + "string_value": "This is another example log message." + }, + "attributes": [ + { + "key": "example_key", + "value": { + "string_value": "example_value" + } + } + ] + } + ] + } + ] + }, + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "string_value": "example-service" + } + } + ] + }, + "scope_logs": [ + { + "scope": { + "name": "example-scope", + "version": "1.0.0" + }, + "log_records": [ + { + "time_unix_nano": "1650917400000000000", + "severity_number": 9, + "severity_text": "INFO", + "body": { + "string_value": "This is an example log message." + }, + "attributes": [ + { + "key": "example_key", + "value": { + "string_value": "example_value" + } + } + ] + } + ] + }, + { + "scope": { + "name": "example-scope", + "version": "2.0.0" + }, + "log_records": [ + { + "time_unix_nano": "1650917400000000000", + "severity_number": 9, + "severity_text": "INFO", + "body": { + "string_value": "This is another example log message." + }, + "attributes": [ + { + "key": "example_key", + "value": { + "string_value": "example_value" + } + } + ] + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_logs_001.out.json b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_logs_001.out.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_metrics_001.in.json b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_metrics_001.in.json new file mode 100644 index 00000000000..aa0fc251d85 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_metrics_001.in.json @@ -0,0 +1,47 @@ +{ + "resource_metrics": [ + { + "resource": { + "attributes": [ + { + "key": "service.instance.id", + "value": { + "string_value": "instance-1" + } + } + ] + }, + "scope_metrics": [ + { + "scope": { + "name": "metrics-scope", + "version": "1.0.0" + }, + "metrics": [ + { + "name": "requests_total", + "sum": { + "aggregation_temporality": 2, + "is_monotonic": true, + "data_points": [ + { + "attributes": [ + { + "key": "service.name", + "value": { + "string_value": "checkout" + } + } + ], + "time_unix_nano": "1650917400000000000", + "as_int": "42" + } + ] + } + } + ] + } + ] + } + ] +} diff --git a/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_metrics_002.in.json b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_metrics_002.in.json new file mode 100644 index 00000000000..91ebeae5fac --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_metrics_002.in.json @@ -0,0 +1,75 @@ +{ + "resource_metrics": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "string_value": "payments" + } + } + ] + }, + "scope_metrics": [ + { + "scope": { + "name": "metrics-advanced-scope", + "version": "2.0.0" + }, + "metrics": [ + { + "name": "request.duration", + "unit": "ms", + "histogram": { + "aggregation_temporality": 1, + "data_points": [ + { + "attributes": [ + { + "key": "http.route", + "value": { + "string_value": "/checkout" + } + } + ], + "time_unix_nano": "1650917402000000000", + "count": "3", + "sum": 245.0, + "bucket_counts": [ + "1", + "2" + ], + "explicit_bounds": [ + 100.0 + ] + } + ] + } + }, + { + "name": "cpu.usage", + "unit": "1", + "gauge": { + "data_points": [ + { + "attributes": [ + { + "key": "host.name", + "value": { + "string_value": "node-a" + } + } + ], + "time_unix_nano": "1650917403000000000", + "as_double": 0.82 + } + ] + } + } + ] + } + ] + } + ] +} diff --git a/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_traces_001.in.json b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_traces_001.in.json new file mode 100644 index 00000000000..9478a5ec351 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_traces_001.in.json @@ -0,0 +1,42 @@ +{ + "resource_spans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "string_value": "checkout" + } + } + ] + }, + "scope_spans": [ + { + "scope": { + "name": "trace-scope", + "version": "1.0.0" + }, + "spans": [ + { + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "eee19b7ec3c1b174", + "name": "checkout-span", + "kind": 2, + "start_time_unix_nano": "1650917400000000000", + "end_time_unix_nano": "1650917401000000000", + "attributes": [ + { + "key": "http.method", + "value": { + "string_value": "GET" + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_traces_002.in.json b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_traces_002.in.json new file mode 100644 index 00000000000..4812543eb4b --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/tests/data_files/test_traces_002.in.json @@ -0,0 +1,65 @@ +{ + "resource_spans": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "string_value": "checkout" + } + } + ] + }, + "scope_spans": [ + { + "scope": { + "name": "trace-advanced-scope", + "version": "2.0.0" + }, + "spans": [ + { + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "1111111111111111", + "name": "parent-span", + "kind": 2, + "start_time_unix_nano": "1650917400000000000", + "end_time_unix_nano": "1650917401000000000", + "events": [ + { + "time_unix_nano": "1650917400500000000", + "name": "cache.miss" + } + ], + "status": { + "code": 1, + "message": "ok" + } + }, + { + "trace_id": "5b8efff798038103d269b633813fc60c", + "span_id": "2222222222222222", + "parent_span_id": "1111111111111111", + "name": "child-span", + "kind": 3, + "start_time_unix_nano": "1650917400100000000", + "end_time_unix_nano": "1650917400900000000", + "attributes": [ + { + "key": "db.system", + "value": { + "string_value": "postgresql" + } + } + ], + "status": { + "code": 2, + "message": "db timeout" + } + } + ] + } + ] + } + ] +} diff --git a/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py new file mode 100644 index 00000000000..db2cc4357e4 --- /dev/null +++ b/tests/integration/scenarios/in_opentelemetry/tests/test_in_opentelemetry_001.py @@ -0,0 +1,927 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2024 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json +import logging +import time +import base64 +from concurrent.futures import ThreadPoolExecutor +import grpc +import requests +import pytest + +# OTel imports to convert from JSON to OTLP Protobuf +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceResponse +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceResponse +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceResponse +from google.protobuf import json_format + +# local imports +from utils.data_utils import read_json_file +from utils.http_matrix import PROTOCOL_CASES, run_curl_request +from utils.test_service import FluentBitTestService + +from server.http_server import http_server_run +from server.otlp_server import configure_otlp_response, otlp_server_run, data_storage + +logger = logging.getLogger(__name__) +MOCK_VALID_JWT = ( + "eyJhbGciOiJSUzI1NiIsImtpZCI6InRlc3QiLCJ0eXAiOiJKV1QifQ." + "eyJleHAiOjE4OTM0NTYwMDAsImlzcyI6Imlzc3VlciIsImF1ZCI6ImF1ZGllbmNlIiwiYXpwIjoiY2xpZW50MSJ9." + "TqWs06LUpQa0FGLejnOkWAD6v562d5CUh2NwsJ7iAuae9-WNFBKU6mP1zAaoafla6o5npee7RfbSzZNFI4PKhqAj69789JjAYV7IW-GSuMwJejHdVOWmCc5lmcZPH0EVxEkHA6lFQxYQwDCrfQ8Sd4Q3vYCV6sLPENcuNpQi9ytjVjaZs_7ONH2oA-sZ7EUchqJJoIBPfjit2yYsq9NeemxCzYMtngiC-IX12eEfaQ1cVYPIjhhN_NaMvapznp-BW4gnXkNoAZ1S-p1axWWY-6UgRdMYOr0Hy5PHQ9fCuHJ6Z-blYdtuGavCUGHK5ghX-JdH1WJ51F89992dQ5yF_w" +) + +IN_OPENTELEMETRY_PROTOCOL_CONFIGS = { + "http1_cleartext": "otlp_http1_cleartext.yaml", + "http2_cleartext": "otlp_http2_cleartext.yaml", + "http1_tls": "otlp_http1_tls.yaml", + "http2_tls": "otlp_http2_tls.yaml", +} + +IN_OPENTELEMETRY_WORKER_PROTOCOL_CONFIGS = { + "http1_cleartext": "otlp_http1_cleartext_workers.yaml", + "http2_cleartext": "otlp_http2_cleartext_workers.yaml", + "http1_tls": "otlp_http1_tls_workers.yaml", + "http2_tls": "otlp_http2_tls_workers.yaml", +} + +IN_OPENTELEMETRY_OAUTH2_PROTOCOL_CONFIGS = { + "http1_cleartext": "otlp_http1_cleartext_oauth2.yaml", + "http2_cleartext": "otlp_http2_cleartext_oauth2.yaml", + "http1_tls": "otlp_http1_tls_oauth2.yaml", + "http2_tls": "otlp_http2_tls_oauth2.yaml", +} + +IN_OPENTELEMETRY_GRPC_OAUTH2_CASES = [ + { + "id": "grpc_cleartext", + "config_file": "otlp_http2_cleartext_oauth2.yaml", + "use_tls": False, + }, + { + "id": "grpc_tls", + "config_file": "otlp_http2_tls_oauth2.yaml", + "use_tls": True, + }, +] + +# [Fluent Bit Test Suite] +# - Start Fluent Bit: +# - a custom configuration file +# - set 3 environment variables: +# - FLUENT_BIT_HTTP_MONITORING_PORT: port where Fluent Bit will expose internal metrics +# - FLUENT_BIT_TEST_LISTENER_PORT: port used by the config file to define where to listen +# for incoming connections +# - TEST_SUITE_HTTP_PORT: local port on this suite which is used by Fluent Bit to send the +# data back +# +# [Test Suite] --> writes a request --> [Fluent Bit] --> forwards the request --> [Test Suite] +# +# In the process above, Fluent Bit decode the request, process it and encode it back. + + +def iter_log_records(output): + for resource_log in output["resourceLogs"]: + resource_attributes = { + item["key"]: next(iter(item["value"].values())) + for item in resource_log.get("resource", {}).get("attributes", []) + } + for scope_log in resource_log.get("scopeLogs", []): + scope = scope_log.get("scope", {}) + scope_attributes = { + item["key"]: next(iter(item["value"].values())) + for item in scope.get("attributes", []) + } + for record in scope_log.get("logRecords", []): + record_attributes = { + item["key"]: next(iter(item["value"].values())) + for item in record.get("attributes", []) + } + yield { + "resource_attributes": resource_attributes, + "scope_name": scope.get("name"), + "scope_version": scope.get("version"), + "scope_attributes": scope_attributes, + "record": record, + "record_attributes": record_attributes, + "body": record.get("body", {}).get("stringValue"), + } + + +def iter_metric_entries(output): + for resource_metric in output.get("resourceMetrics", []): + resource_attributes = { + item["key"]: next(iter(item["value"].values())) + for item in resource_metric.get("resource", {}).get("attributes", []) + } + for scope_metric in resource_metric.get("scopeMetrics", []): + scope = scope_metric.get("scope", {}) + for metric in scope_metric.get("metrics", []): + yield { + "resource_attributes": resource_attributes, + "scope_name": scope.get("name"), + "scope_version": scope.get("version"), + "metric": metric, + } + + +def iter_spans(output): + for resource_span in output.get("resourceSpans", []): + resource_attributes = { + item["key"]: next(iter(item["value"].values())) + for item in resource_span.get("resource", {}).get("attributes", []) + } + for scope_span in resource_span.get("scopeSpans", []): + scope = scope_span.get("scope", {}) + for span in scope_span.get("spans", []): + span_attributes = { + item["key"]: next(iter(item["value"].values())) + for item in span.get("attributes", []) + } + yield { + "resource_attributes": resource_attributes, + "scope_name": scope.get("name"), + "scope_version": scope.get("version"), + "span": span, + "span_attributes": span_attributes, + } + + +def read_stdout_otlp_json(service, root_key, timeout=10, interval=0.25): + deadline = time.time() + timeout + decoder = json.JSONDecoder() + + while time.time() < deadline: + if service.flb and service.flb.log_file and os.path.exists(service.flb.log_file): + with open(service.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + content = log_file.read() + + matches = [] + + for offset, character in enumerate(content): + if character != "{": + continue + + try: + payload, _ = decoder.raw_decode(content[offset:]) + except json.JSONDecodeError: + continue + + if isinstance(payload, dict) and root_key in payload: + matches.append(payload) + + if matches: + return matches[-1] + + time.sleep(interval) + + raise TimeoutError(f"Timed out waiting for stdout OTLP JSON payload with root key {root_key}") + + +def read_stdout_otlp_json_text(service, root_key, timeout=10, interval=0.25): + deadline = time.time() + timeout + decoder = json.JSONDecoder() + + while time.time() < deadline: + if service.flb and service.flb.log_file and os.path.exists(service.flb.log_file): + with open(service.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + content = log_file.read() + + matches = [] + + for offset, character in enumerate(content): + if character != "{": + continue + + try: + payload, end = decoder.raw_decode(content[offset:]) + except json.JSONDecodeError: + continue + + if isinstance(payload, dict) and root_key in payload: + matches.append(content[offset:offset + end]) + + if matches: + return matches[-1] + + time.sleep(interval) + + raise TimeoutError(f"Timed out waiting for stdout OTLP JSON payload with root key {root_key}") + +class Service: + def __init__(self, config_file, *, use_auth_server=False): + # Compose the absolute path for the Fluent Bit configuration file + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), '../config/', config_file)) + test_path = os.path.dirname(os.path.abspath(__file__)) + cert_dir = os.path.abspath(os.path.join(test_path, "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + self.use_auth_server = use_auth_server + self.auth_server_port = None + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["logs", "metrics", "traces", "requests"], + extra_env={ + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + }, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + if self.use_auth_server: + self.auth_server_port = service.allocate_port_env("TEST_SUITE_JWKS_PORT") + http_server_run(self.auth_server_port) + self.service.wait_for_http_endpoint( + f"http://127.0.0.1:{self.auth_server_port}/ping", + timeout=10, + interval=0.5, + ) + otlp_server_run(service.test_suite_http_port) + url = f'http://127.0.0.1:{service.test_suite_http_port}/ping' + self.service.wait_for_http_endpoint(url, timeout=10, interval=0.5) + + def _stop_receiver(self, service): + if self.auth_server_port is not None: + try: + requests.post(f"http://127.0.0.1:{self.auth_server_port}/shutdown", timeout=2) + except requests.RequestException: + pass + try: + requests.post(f'http://localhost:{service.test_suite_http_port}/shutdown', timeout=2) + except requests.RequestException: + pass + + def start(self): + self.service.start() + self.flb = self.service.flb + self.flb_listener_port = self.service.flb_listener_port + self.test_suite_http_port = self.service.test_suite_http_port + logger.info(f"Fluent Bit listener port: {self.flb_listener_port}") + logger.info(f"test suite http port: {self.test_suite_http_port}") + + def wait_for_log_message(self, pattern, timeout=10, interval=0.25): + deadline = time.time() + timeout + + while time.time() < deadline: + if self.flb and self.flb.log_file and os.path.exists(self.flb.log_file): + with open(self.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + if pattern in log_file.read(): + return True + + time.sleep(interval) + + raise TimeoutError(f"Timed out waiting for log pattern: {pattern}") + + def read_response(self, signal_type, timeout=10, interval=0.5): + deadline = time.time() + timeout + while len(data_storage[signal_type]) <= 0: + if time.time() >= deadline: + raise TimeoutError(f"Timed out waiting for OTLP {signal_type} response") + time.sleep(0.5) + logger.info("waiting for %s response...", signal_type) + + json_str = json_format.MessageToJson(data_storage[signal_type][0]) + logger.info(f"{json_str}") + return json.loads(json_str) + + def send_request(self, endpoint, payload, content_type='application/x-protobuf'): + # Send the protobuf payload + url = f'http://localhost:{self.flb_listener_port}{endpoint}' + headers = {'Content-Type': content_type} + response = requests.post(url, data=payload.SerializeToString(), headers=headers) + print(f'Status code: {response.status_code}') + print(f'Response text: {response.text}') + return response + + def send_raw_request(self, endpoint, payload, content_type='application/x-protobuf'): + url = f'http://localhost:{self.flb_listener_port}{endpoint}' + headers = {'Content-Type': content_type} + return requests.post(url, data=payload, headers=headers, timeout=5) + + def send_json_as_otel_protobuf(self, json_input, signal_type): + base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../tests', 'data_files')) + json_payload_dict = read_json_file(os.path.join(base_path, json_input)) + json_payload_str = json.dumps(json_payload_dict) + + request_map = { + "logs": (ExportLogsServiceRequest(), "/v1/logs"), + "metrics": (ExportMetricsServiceRequest(), "/v1/metrics"), + "traces": (ExportTraceServiceRequest(), "/v1/traces"), + } + request_message, endpoint = request_map[signal_type] + protobuf_payload = json_format.Parse(json_payload_str, request_message) + + self.send_request(endpoint, protobuf_payload) + + return self.read_response(signal_type) + + def send_grpc_request(self, signal_type, payload, *, authorization=None, use_tls=False): + method_map = { + "logs": ( + "/opentelemetry.proto.collector.logs.v1.LogsService/Export", + ExportLogsServiceRequest.SerializeToString, + ExportLogsServiceResponse.FromString, + ), + "metrics": ( + "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export", + ExportMetricsServiceRequest.SerializeToString, + ExportMetricsServiceResponse.FromString, + ), + "traces": ( + "/opentelemetry.proto.collector.trace.v1.TraceService/Export", + ExportTraceServiceRequest.SerializeToString, + ExportTraceServiceResponse.FromString, + ), + } + method_path, serializer, deserializer = method_map[signal_type] + target = f"127.0.0.1:{self.flb_listener_port}" + + if use_tls: + with open(self.tls_crt_file, "rb") as certificate_file: + channel_credentials = grpc.ssl_channel_credentials(certificate_file.read()) + channel = grpc.secure_channel(target, channel_credentials) + else: + channel = grpc.insecure_channel(target) + + metadata = [] + if authorization is not None: + metadata.append(("authorization", authorization)) + + try: + grpc.channel_ready_future(channel).result(timeout=5) + rpc = channel.unary_unary( + method_path, + request_serializer=serializer, + response_deserializer=deserializer, + ) + return rpc(payload, metadata=metadata, timeout=5) + finally: + channel.close() + + def build_otel_payload(self, json_input, signal_type): + base_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../tests', 'data_files')) + json_payload_dict = read_json_file(os.path.join(base_path, json_input)) + json_payload_str = json.dumps(json_payload_dict) + + request_map = { + "logs": ExportLogsServiceRequest(), + "metrics": ExportMetricsServiceRequest(), + "traces": ExportTraceServiceRequest(), + } + request_message = request_map[signal_type] + protobuf_payload = json_format.Parse(json_payload_str, request_message) + return protobuf_payload.SerializeToString() + + def wait_for_signal_count(self, signal_type, minimum_count, timeout=10, interval=0.25): + return self.service.wait_for_condition( + lambda: len(data_storage[signal_type]) if len(data_storage[signal_type]) >= minimum_count else None, + timeout=timeout, + interval=interval, + description=f"{minimum_count} OTLP {signal_type} payloads", + ) + + def stop(self): + self.service.stop() + + +# This is a full pipeline test, the file tests_logs_001.in.json, represents an OpenTelemetry Log payload in +# JSON format which gets converted to Protobuf. +# +# Then Fluent Bit is started having an OpenTelemetry listener (input plugin) and OpenTelemetry output plugin +# that sends the data back to the test suite. Note that we instance a dummy OTLP server in this test suite +# so we can check the data that is being sent back. +def test_opentelemetry_to_opentelemetry_basic_log(): + service = Service("001-fluent-bit.yaml") + service.start() + output = service.send_json_as_otel_protobuf("test_logs_001.in.json", "logs") + logger.info(f"response: {output}") + service.stop() + + records = list(iter_log_records(output)) + assert len(records) >= 2 + + expected_bodies = { + "This is an example log message.", + "This is another example log message.", + } + observed_bodies = {record["body"] for record in records} + + assert observed_bodies == expected_bodies + + for item in records: + assert item["resource_attributes"]["aaa"] == "bbb" + assert "service.name" not in item["resource_attributes"] + assert item["scope_name"] == "new scope name" + assert item["scope_version"] == "3.1.0" + assert item["scope_attributes"]["mynewscope"] == "123" + assert item["record"]["severityText"] == "INFO" + assert item["record_attributes"]["example_key"] == "example_value" + + +# Start a Fluent Bit Pipeline with Dummy message and then it gets handle by OpenTelemetry output, the config +# aims to populate traceId and spanId fields with the values from the Dummy message. +# +# issue : https://github.com/fluent/fluent-bit/issues/9071 +# fixed : https://github.com/fluent/fluent-bit/pull/9074 +def test_dummy_to_opentelemetry_log(): + service = Service("002-fluent-bit.yaml") + service.start() + output = service.read_response("logs") + logger.info(f"response: {output}") + service.stop() + + # direct reference to the record + record = output['resourceLogs'][0]['scopeLogs'][0]['logRecords'][0] + + # notes on traceid and spanid: the test case encodes the values as hex strings, Fluent Bit OpenTelemetry + # output plugin will decode and pack them as bytes. When the data is sent back to the test suite, the values + # are encoded as base64 strings (Python thing). So we need to decode them back to bytes and compare them. + assert base64.b64decode(record['traceId']) == bytes.fromhex('63560bd4d8de74fae7d1e4160f2ee099') + assert base64.b64decode(record['spanId']) == bytes.fromhex('251484295a9df731') + + +def test_opentelemetry_to_opentelemetry_basic_metrics(): + service = Service("001-fluent-bit.yaml") + service.start() + output = service.send_json_as_otel_protobuf("test_metrics_001.in.json", "metrics") + logger.info(f"response: {output}") + service.stop() + + assert len(output["resourceMetrics"]) == 1 + resource_metric = output["resourceMetrics"][0] + metric = resource_metric["scopeMetrics"][0]["metrics"][0] + datapoint = metric["sum"]["dataPoints"][0] + + assert metric["name"] == "requests_total" + assert metric["sum"]["aggregationTemporality"] == "AGGREGATION_TEMPORALITY_CUMULATIVE" + assert datapoint["asInt"] == "42" + assert datapoint["attributes"][0]["key"] == "service.name" + assert datapoint["attributes"][0]["value"]["stringValue"] == "checkout" + + +def test_opentelemetry_to_opentelemetry_histogram_and_gauge_metrics(): + service = Service("001-fluent-bit.yaml") + service.start() + output = service.send_json_as_otel_protobuf("test_metrics_002.in.json", "metrics") + logger.info(f"response: {output}") + service.stop() + + metrics = {item["metric"]["name"]: item for item in iter_metric_entries(output)} + + assert set(metrics) == {"request.duration", "cpu.usage"} + + histogram = metrics["request.duration"]["metric"]["histogram"] + gauge = metrics["cpu.usage"]["metric"]["gauge"] + histogram_datapoint = histogram["dataPoints"][0] + gauge_datapoint = gauge["dataPoints"][0] + + assert metrics["request.duration"]["resource_attributes"]["service.name"] == "payments" + assert metrics["request.duration"]["scope_name"] == "metrics-advanced-scope" + assert histogram_datapoint["count"] == "3" + assert histogram_datapoint["sum"] == 245.0 + assert histogram_datapoint["bucketCounts"] == ["1", "2"] + assert histogram_datapoint["explicitBounds"] == [100.0] + assert histogram_datapoint["attributes"][0]["key"] == "http.route" + assert histogram_datapoint["attributes"][0]["value"]["stringValue"] == "/checkout" + + assert metrics["cpu.usage"]["scope_version"] == "2.0.0" + assert gauge_datapoint["asDouble"] == 0.82 + assert gauge_datapoint["attributes"][0]["key"] == "host.name" + assert gauge_datapoint["attributes"][0]["value"]["stringValue"] == "node-a" + + +def test_opentelemetry_to_opentelemetry_basic_traces(): + service = Service("001-fluent-bit.yaml") + service.start() + output = service.send_json_as_otel_protobuf("test_traces_001.in.json", "traces") + logger.info(f"response: {output}") + service.stop() + + assert len(output["resourceSpans"]) == 1 + resource_span = output["resourceSpans"][0] + span = resource_span["scopeSpans"][0]["spans"][0] + + assert span["name"] == "checkout-span" + assert span["kind"] == "SPAN_KIND_SERVER" + assert span["traceId"] == "5b8efff798038103d269b633813fc60c" + assert span["spanId"] == "eee19b7ec3c1b174" + assert span["attributes"][0]["key"] == "http.method" + assert span["attributes"][0]["value"]["stringValue"] == "GET" + + +def test_opentelemetry_to_opentelemetry_parent_child_traces(): + service = Service("001-fluent-bit.yaml") + service.start() + output = service.send_json_as_otel_protobuf("test_traces_002.in.json", "traces") + logger.info(f"response: {output}") + service.stop() + + spans = {item["span"]["name"]: item for item in iter_spans(output)} + + assert set(spans) == {"parent-span", "child-span"} + + parent_span = spans["parent-span"]["span"] + child_span = spans["child-span"]["span"] + + assert spans["parent-span"]["resource_attributes"]["service.name"] == "checkout" + assert spans["parent-span"]["scope_name"] == "trace-advanced-scope" + assert spans["child-span"]["span_attributes"]["db.system"] == "postgresql" + + assert parent_span["traceId"] == child_span["traceId"] + assert child_span["parentSpanId"] == parent_span["spanId"] + assert parent_span["events"][0]["name"] == "cache.miss" + assert parent_span["status"]["message"] == "ok" + assert child_span["status"]["code"] == "STATUS_CODE_ERROR" + + +def test_in_opentelemetry_rejects_invalid_logs_payload(): + service = Service("001-fluent-bit.yaml") + service.start() + response = service.send_raw_request("/v1/logs", b"not-a-valid-otlp-payload") + service.stop() + + assert response.status_code >= 400 + assert len(data_storage["logs"]) == 0 + + +def test_in_opentelemetry_rejects_invalid_metrics_payload(): + service = Service("001-fluent-bit.yaml") + service.start() + response = service.send_raw_request("/v1/metrics", b"not-a-valid-otlp-payload") + service.stop() + + assert response.status_code >= 400 + assert len(data_storage["metrics"]) == 0 + + +def test_in_opentelemetry_rejects_invalid_traces_payload(): + service = Service("001-fluent-bit.yaml") + service.start() + response = service.send_raw_request("/v1/traces", b"not-a-valid-otlp-payload") + service.stop() + + assert response.status_code >= 400 + assert len(data_storage["traces"]) == 0 + + +def test_in_opentelemetry_stdout_otlp_json_logs(): + service = Service("003-stdout-otlp-json.yaml") + service.start() + + payload = service.build_otel_payload("test_logs_001.in.json", "logs") + response = service.send_raw_request("/v1/logs", payload) + assert 200 <= response.status_code < 300 + + output = read_stdout_otlp_json(service, "resourceLogs") + service.stop() + + records = list(iter_log_records(output)) + assert len(records) >= 2 + assert records[0]["record"]["severityText"] == "INFO" + assert records[0]["record"]["timeUnixNano"] == "1650917400000000000" + assert records[0]["resource_attributes"]["service.name"] == "example-service" + + +def test_in_opentelemetry_stdout_otlp_json_metrics(): + service = Service("003-stdout-otlp-json.yaml") + service.start() + + payload = service.build_otel_payload("test_metrics_001.in.json", "metrics") + response = service.send_raw_request("/v1/metrics", payload) + assert 200 <= response.status_code < 300 + + output = read_stdout_otlp_json(service, "resourceMetrics") + service.stop() + + metric_entry = list(iter_metric_entries(output))[0] + metric = metric_entry["metric"] + + assert metric["name"] == "requests_total" + assert metric["sum"]["dataPoints"][0]["asInt"] == "42" + assert metric_entry["resource_attributes"]["service.instance.id"] == "instance-1" + assert metric["sum"]["dataPoints"][0]["attributes"][0]["key"] == "service.name" + assert metric["sum"]["dataPoints"][0]["attributes"][0]["value"]["stringValue"] == "checkout" + + +def test_in_opentelemetry_stdout_otlp_json_traces(): + service = Service("003-stdout-otlp-json.yaml") + service.start() + + payload = service.build_otel_payload("test_traces_001.in.json", "traces") + response = service.send_raw_request("/v1/traces", payload) + assert 200 <= response.status_code < 300 + + output = read_stdout_otlp_json(service, "resourceSpans") + service.stop() + + span_entry = list(iter_spans(output))[0] + span = span_entry["span"] + + assert span["name"] == "checkout-span" + assert span["traceId"] == "e5bf1e7df7fbf7cd37f35d37776ebd6fadf7f35ddf73ad1c" + assert span["spanId"] == "79e7b5f5bede7377356f5ef8" + assert span_entry["resource_attributes"]["service.name"] == "checkout" + + +def test_in_opentelemetry_stdout_otlp_json_pretty_logs(): + service = Service("004-stdout-otlp-json-pretty.yaml") + service.start() + + payload = service.build_otel_payload("test_logs_001.in.json", "logs") + response = service.send_raw_request("/v1/logs", payload) + assert 200 <= response.status_code < 300 + + output = read_stdout_otlp_json(service, "resourceLogs") + output_text = read_stdout_otlp_json_text(service, "resourceLogs") + service.stop() + + records = list(iter_log_records(output)) + assert len(records) >= 2 + assert records[0]["record"]["severityText"] == "INFO" + assert "\n \"resourceLogs\": [" in output_text + assert "\n \"logRecords\": [" in output_text + + +def test_in_opentelemetry_stdout_otlp_json_pretty_metrics(): + service = Service("004-stdout-otlp-json-pretty.yaml") + service.start() + + payload = service.build_otel_payload("test_metrics_001.in.json", "metrics") + response = service.send_raw_request("/v1/metrics", payload) + assert 200 <= response.status_code < 300 + + output = read_stdout_otlp_json(service, "resourceMetrics") + output_text = read_stdout_otlp_json_text(service, "resourceMetrics") + service.stop() + + metric_entry = list(iter_metric_entries(output))[0] + assert metric_entry["metric"]["name"] == "requests_total" + assert "\n \"resourceMetrics\": [" in output_text + assert "\n \"metrics\": [" in output_text + + +def test_in_opentelemetry_stdout_otlp_json_pretty_traces(): + service = Service("004-stdout-otlp-json-pretty.yaml") + service.start() + + payload = service.build_otel_payload("test_traces_001.in.json", "traces") + response = service.send_raw_request("/v1/traces", payload) + assert 200 <= response.status_code < 300 + + output = read_stdout_otlp_json(service, "resourceSpans") + output_text = read_stdout_otlp_json_text(service, "resourceSpans") + service.stop() + + span_entry = list(iter_spans(output))[0] + assert span_entry["span"]["name"] == "checkout-span" + assert "\n \"resourceSpans\": [" in output_text + assert "\n \"spans\": [" in output_text + + +@pytest.mark.parametrize("case", PROTOCOL_CASES, ids=[case["id"] for case in PROTOCOL_CASES]) +def test_in_opentelemetry_oauth2_requires_bearer_token(case): + service = Service( + IN_OPENTELEMETRY_OAUTH2_PROTOCOL_CONFIGS[case["config_key"]], + use_auth_server=True, + ) + service.start() + + scheme = "https" if case["use_tls"] else "http" + payload = service.build_otel_payload("test_logs_001.in.json", "logs") + result = run_curl_request( + f"{scheme}://localhost:{service.flb_listener_port}/v1/logs", + payload, + headers=["Content-Type: application/x-protobuf"], + http_mode=case["http_mode"], + ca_cert_path=service.tls_crt_file if case["use_tls"] else None, + ) + + service.stop() + + assert result["status_code"] == 401 + assert len(data_storage["logs"]) == 0 + + +@pytest.mark.parametrize("case", PROTOCOL_CASES, ids=[case["id"] for case in PROTOCOL_CASES]) +def test_in_opentelemetry_oauth2_accepts_valid_jwt(case): + service = Service( + IN_OPENTELEMETRY_OAUTH2_PROTOCOL_CONFIGS[case["config_key"]], + use_auth_server=True, + ) + service.start() + + scheme = "https" if case["use_tls"] else "http" + payload = service.build_otel_payload("test_logs_001.in.json", "logs") + result = run_curl_request( + f"{scheme}://localhost:{service.flb_listener_port}/v1/logs", + payload, + headers=[ + "Content-Type: application/x-protobuf", + f"Authorization: Bearer {MOCK_VALID_JWT}", + ], + http_mode=case["http_mode"], + ca_cert_path=service.tls_crt_file if case["use_tls"] else None, + ) + response_payload = service.read_response("logs") + + service.stop() + + assert result["status_code"] == 201 + assert result["http_version"] == case["expected_http_version"] + assert len(response_payload["resourceLogs"]) > 0 + + +@pytest.mark.parametrize("case", IN_OPENTELEMETRY_GRPC_OAUTH2_CASES, ids=[case["id"] for case in IN_OPENTELEMETRY_GRPC_OAUTH2_CASES]) +def test_in_opentelemetry_grpc_oauth2_requires_bearer_token(case): + service = Service(case["config_file"], use_auth_server=True) + service.start() + + payload = json_format.Parse( + json.dumps( + read_json_file( + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../tests/data_files/test_logs_001.in.json") + ) + ) + ), + ExportLogsServiceRequest(), + ) + + with pytest.raises(grpc.RpcError) as exc_info: + service.send_grpc_request("logs", payload, use_tls=case["use_tls"]) + + service.stop() + + assert exc_info.value.code() == grpc.StatusCode.UNAUTHENTICATED + assert len(data_storage["logs"]) == 0 + + +@pytest.mark.parametrize("case", IN_OPENTELEMETRY_GRPC_OAUTH2_CASES, ids=[case["id"] for case in IN_OPENTELEMETRY_GRPC_OAUTH2_CASES]) +def test_in_opentelemetry_grpc_oauth2_accepts_valid_jwt(case): + service = Service(case["config_file"], use_auth_server=True) + service.start() + + payload = json_format.Parse( + json.dumps( + read_json_file( + os.path.abspath( + os.path.join(os.path.dirname(__file__), "../tests/data_files/test_logs_001.in.json") + ) + ) + ), + ExportLogsServiceRequest(), + ) + service.send_grpc_request( + "logs", + payload, + authorization=f"Bearer {MOCK_VALID_JWT}", + use_tls=case["use_tls"], + ) + response_payload = service.read_response("logs") + + service.stop() + + assert len(response_payload["resourceLogs"]) > 0 + + +def test_out_opentelemetry_receiver_error_is_observable(): + service = Service("001-fluent-bit.yaml") + service.start() + configure_otlp_response(status_code=500, body={"status": "error"}) + + service.send_json_as_otel_protobuf("test_logs_001.in.json", "logs") + requests_seen = service.service.wait_for_condition( + lambda: data_storage["requests"] if len(data_storage["requests"]) >= 1 else None, + timeout=10, + interval=0.5, + description="at least one OTLP output attempt", + ) + service.stop() + + assert len(requests_seen) >= 1 + + +@pytest.mark.parametrize("signal_type,json_input,endpoint,storage_key", [ + ("logs", "test_logs_001.in.json", "/v1/logs", "logs"), + ("metrics", "test_metrics_001.in.json", "/v1/metrics", "metrics"), + ("traces", "test_traces_001.in.json", "/v1/traces", "traces"), +]) +@pytest.mark.parametrize("case", PROTOCOL_CASES, ids=[case["id"] for case in PROTOCOL_CASES]) +def test_in_opentelemetry_protocol_matrix(case, signal_type, json_input, endpoint, storage_key): + service = Service(IN_OPENTELEMETRY_PROTOCOL_CONFIGS[case["config_key"]]) + service.start() + + scheme = "https" if case["use_tls"] else "http" + payload = service.build_otel_payload(json_input, signal_type) + result = run_curl_request( + f"{scheme}://localhost:{service.flb_listener_port}{endpoint}", + payload, + headers=["Content-Type: application/x-protobuf"], + http_mode=case["http_mode"], + ca_cert_path=service.tls_crt_file if case["use_tls"] else None, + ) + response_payload = service.read_response(storage_key) + + service.stop() + + assert result["status_code"] == 201 + assert result["http_version"] == case["expected_http_version"] + assert len(response_payload) > 0 + + +# This test is branch-specific coverage for the generic HTTP listener worker mode. +# It does three things: +# 1. enables http_server.workers on the in_opentelemetry listener, +# 2. sends concurrent mixed OTLP requests across representative matrix variants, +# 3. verifies end-to-end delivery for logs, metrics and traces under that load. +# +# The current OTLP input path does not expose the serving worker id back to the +# client or the forwarded payload, so this test validates the multi-worker +# transport path with concurrent mixed traffic and confirms the listener started +# with multiple workers. Once the branch exposes per-worker request identity, +# this should be tightened to assert distinct worker ids directly. +@pytest.mark.parametrize("case", [ + {"id": "http1_cleartext", "config_key": "http1_cleartext", "http_mode": "http1.1", "use_tls": False}, + {"id": "http2_cleartext_prior_knowledge", "config_key": "http2_cleartext", "http_mode": "http2-prior-knowledge", "use_tls": False}, + {"id": "http1_tls", "config_key": "http1_tls", "http_mode": "http1.1", "use_tls": True}, + {"id": "http2_tls_alpn", "config_key": "http2_tls", "http_mode": "http2", "use_tls": True}, +], ids=lambda case: case["id"]) +def test_in_opentelemetry_http_workers_mixed_signal_matrix(case): + request_plan = [ + ("logs", "test_logs_001.in.json", "/v1/logs"), + ("metrics", "test_metrics_001.in.json", "/v1/metrics"), + ("traces", "test_traces_001.in.json", "/v1/traces"), + ] + repeats_per_signal = 4 + total_requests = len(request_plan) * repeats_per_signal + + service = Service(IN_OPENTELEMETRY_WORKER_PROTOCOL_CONFIGS[case["config_key"]]) + service.start() + service.wait_for_log_message("with 4 workers", timeout=10) + + scheme = "https" if case["use_tls"] else "http" + request_jobs = [] + for _ in range(repeats_per_signal): + for signal_type, json_input, endpoint in request_plan: + request_jobs.append( + { + "signal_type": signal_type, + "endpoint": endpoint, + "payload": service.build_otel_payload(json_input, signal_type), + } + ) + + def send_job(job): + return run_curl_request( + f"{scheme}://localhost:{service.flb_listener_port}{job['endpoint']}", + job["payload"], + headers=["Content-Type: application/x-protobuf", "Connection: close"], + http_mode=case["http_mode"], + ca_cert_path=service.tls_crt_file if case["use_tls"] else None, + ) + + with ThreadPoolExecutor(max_workers=total_requests) as executor: + results = list(executor.map(send_job, request_jobs)) + + for result in results: + assert result["status_code"] == 201 + + service.wait_for_signal_count("logs", 1, timeout=20) + service.wait_for_signal_count("metrics", 1, timeout=20) + service.wait_for_signal_count("traces", 1, timeout=20) + + requests_seen = service.service.wait_for_condition( + lambda: list(data_storage["requests"]) if len(data_storage["requests"]) >= 3 else None, + timeout=20, + interval=0.25, + description="mixed OTLP output requests", + ) + + service.stop() + + paths_seen = {request["path"] for request in requests_seen} + + assert len(requests_seen) >= 3 + assert "/v1/logs" in paths_seen + assert "/v1/metrics" in paths_seen + assert "/v1/traces" in paths_seen diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_cleartext.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_cleartext.yaml new file mode 100644 index 00000000000..6cd85dfccc3 --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_cleartext.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: prometheus_remote_write + listen: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + http2: off + successful_response_code: 201 + + outputs: + - name: stdout + match: "*" diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_cleartext_workers.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_cleartext_workers.yaml new file mode 100644 index 00000000000..2f6df91851b --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_cleartext_workers.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: prometheus_remote_write + listen: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + http2: off + successful_response_code: 201 + http_server.workers: 4 + + outputs: + - name: stdout + match: "*" diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_tls.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_tls.yaml new file mode 100644 index 00000000000..a59fbf87aa8 --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_tls.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: prometheus_remote_write + listen: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + http2: off + successful_response_code: 201 + tls: on + tls.verify: no + tls.vhost: localhost + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: stdout + match: "*" diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_tls_workers.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_tls_workers.yaml new file mode 100644 index 00000000000..5a6c42f1709 --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http1_tls_workers.yaml @@ -0,0 +1,25 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: prometheus_remote_write + listen: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + http2: off + successful_response_code: 201 + tls: on + tls.verify: no + tls.vhost: localhost + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + http_server.workers: 4 + + outputs: + - name: stdout + match: "*" diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_cleartext.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_cleartext.yaml new file mode 100644 index 00000000000..7d2ef3a8dec --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_cleartext.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: prometheus_remote_write + listen: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + http2: on + successful_response_code: 201 + + outputs: + - name: stdout + match: "*" diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_cleartext_workers.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_cleartext_workers.yaml new file mode 100644 index 00000000000..00391a51eea --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_cleartext_workers.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: prometheus_remote_write + listen: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + http2: on + successful_response_code: 201 + http_server.workers: 4 + + outputs: + - name: stdout + match: "*" diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_tls.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_tls.yaml new file mode 100644 index 00000000000..f103eeafc77 --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_tls.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: prometheus_remote_write + listen: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + http2: on + successful_response_code: 201 + tls: on + tls.verify: no + tls.vhost: localhost + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: stdout + match: "*" diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_tls_workers.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_tls_workers.yaml new file mode 100644 index 00000000000..22784b8c281 --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/receiver_http2_tls_workers.yaml @@ -0,0 +1,25 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: prometheus_remote_write + listen: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + http2: on + successful_response_code: 201 + tls: on + tls.verify: no + tls.vhost: localhost + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + http_server.workers: 4 + + outputs: + - name: stdout + match: "*" diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/sender_cleartext.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/sender_cleartext.yaml new file mode 100644 index 00000000000..ca46d9f470f --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/sender_cleartext.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: fluentbit_metrics + scrape_interval: 1 + scrape_on_start: true + + outputs: + - name: prometheus_remote_write + match: "*" + host: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + compression: snappy diff --git a/tests/integration/scenarios/in_prometheus_remote_write/config/sender_tls.yaml b/tests/integration/scenarios/in_prometheus_remote_write/config/sender_tls.yaml new file mode 100644 index 00000000000..17a58ef1d3c --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/config/sender_tls.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: fluentbit_metrics + scrape_interval: 1 + scrape_on_start: true + + outputs: + - name: prometheus_remote_write + match: "*" + host: 127.0.0.1 + port: ${PROM_RW_RECEIVER_PORT} + uri: /write + compression: snappy + tls: on + tls.verify: no diff --git a/tests/integration/scenarios/in_prometheus_remote_write/tests/test_in_prometheus_remote_write_001.py b/tests/integration/scenarios/in_prometheus_remote_write/tests/test_in_prometheus_remote_write_001.py new file mode 100644 index 00000000000..88bf30f941c --- /dev/null +++ b/tests/integration/scenarios/in_prometheus_remote_write/tests/test_in_prometheus_remote_write_001.py @@ -0,0 +1,130 @@ +import os +import time + +import pytest + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port + + +PROM_RW_CASES = [ + { + "id": "http1_cleartext", + "receiver_config": { + False: "receiver_http1_cleartext.yaml", + True: "receiver_http1_cleartext_workers.yaml", + }, + "sender_config": "sender_cleartext.yaml", + }, + { + "id": "http2_cleartext", + "receiver_config": { + False: "receiver_http2_cleartext.yaml", + True: "receiver_http2_cleartext_workers.yaml", + }, + "sender_config": "sender_cleartext.yaml", + }, + { + "id": "http1_tls", + "receiver_config": { + False: "receiver_http1_tls.yaml", + True: "receiver_http1_tls_workers.yaml", + }, + "sender_config": "sender_tls.yaml", + }, + { + "id": "http2_tls", + "receiver_config": { + False: "receiver_http2_tls.yaml", + True: "receiver_http2_tls_workers.yaml", + }, + "sender_config": "sender_tls.yaml", + }, +] + + +def _read_file(path): + with open(path, "r", encoding="utf-8", errors="replace") as file: + return file.read() + + +class Service: + def __init__(self, receiver_config, sender_config): + base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config")) + self.receiver_config = os.path.join(base_dir, receiver_config) + self.sender_config = os.path.join(base_dir, sender_config) + cert_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + self.receiver = None + self.sender = None + self._previous_env = {} + + def _set_env(self, key, value): + self._previous_env.setdefault(key, os.environ.get(key)) + os.environ[key] = str(value) + + def _restore_env(self): + for key, value in self._previous_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self._previous_env.clear() + + def start(self): + self._set_env("PROM_RW_RECEIVER_PORT", find_available_port()) + self._set_env("CERTIFICATE_TEST", self.tls_crt_file) + self._set_env("PRIVATE_KEY_TEST", self.tls_key_file) + + self.receiver = FluentBitManager(self.receiver_config) + self.receiver.start() + self.receiver_port = int(os.environ["PROM_RW_RECEIVER_PORT"]) + self.wait_for_log(self.receiver.log_file, f"listening on 127.0.0.1:{self.receiver_port}") + + self.sender = FluentBitManager(self.sender_config) + self.sender.start() + + def stop(self): + try: + if self.sender: + self.sender.stop() + if self.receiver: + self.receiver.stop() + finally: + self._restore_env() + + def wait_for_log(self, path, pattern, *, timeout=20, interval=0.5): + deadline = time.time() + timeout + while time.time() < deadline: + contents = _read_file(path) + if pattern in contents: + return contents + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for {pattern} in {path}") + + +@pytest.mark.parametrize("workers_enabled", [False, True], ids=["single_listener", "workers_4"]) +@pytest.mark.parametrize("case", PROM_RW_CASES, ids=[case["id"] for case in PROM_RW_CASES]) +def test_in_prometheus_remote_write_matrix(case, workers_enabled): + service = Service(case["receiver_config"][workers_enabled], case["sender_config"]) + service.start() + + try: + if workers_enabled: + service.wait_for_log( + service.receiver.log_file, + "with 4 workers", + timeout=20, + interval=0.5, + ) + receiver_log = service.wait_for_log( + service.receiver.log_file, + "fluentbit_input_metrics_scrapes_total", + timeout=40, + interval=1, + ) + assert f"listening on 127.0.0.1:{service.receiver_port}" in receiver_log + assert "fluentbit_input_metrics_scrapes_total" in receiver_log + finally: + service.stop() diff --git a/tests/integration/scenarios/in_splunk/certificate/certificate.pem b/tests/integration/scenarios/in_splunk/certificate/certificate.pem new file mode 100644 index 00000000000..019540f3b5f --- /dev/null +++ b/tests/integration/scenarios/in_splunk/certificate/certificate.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJTCCAg2gAwIBAgIUcBSPu8GQTC+FMNXcPoCtPnf0GI4wDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI2MDMxMTE4MzcwNloXDTMwMTIz +MTE4MzcwNlowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAtO/AEXXIOj7W+BXkw4gNgNQS7GQ42lQnAXf24Zdy6D24 +Ts0lerjTT6jgNjcpFO1MbIr1a6DqIMpwAqTwPblf8x+UEXXgIeTBAsUsmrOeFsN6 +74rh4+uQ9OVtKRqT9sw74DoSC4KLw2RkO1/Gvpq8VV9SDCd1Y9MBPN3heNWtOMUO +RPfDIOOXDJnaxkfp+q9t87vZpVfcwkrWX7iwx0OtK7NUrPYSJCW7OSNGr9q/H3WH +D7Z96O+1P07Txou5yGCvVCOlDBapMoYmMs/el/ZocWHpEbO72X5z6YlrtvcHSVRP +H6rJCdUHuA9EOiM1elfp+WHu5Tu0Z4t4uMFcClhnvQIDAQABo28wbTAdBgNVHQ4E +FgQUsjJJ43VR8heAO76MFdidqMVGS3UwHwYDVR0jBBgwFoAUsjJJ43VR8heAO76M +FdidqMVGS3UwDwYDVR0TAQH/BAUwAwEB/zAaBgNVHREEEzARgglsb2NhbGhvc3SH +BH8AAAEwDQYJKoZIhvcNAQELBQADggEBACAcKzsDaxLBULa3vxF1U9TMkyYuqm7Q +t1iFfScO6tSL5lp7NUSH9bszxypL8JlWIKypxS16MjefVkCQnXoseUcIUF2YyjE7 +du7uzPsBCkdSekeHVEHxyMjYcb5A95G5gJVeRodkAlxNnf7D1gLc38zRSUSP4N+H +adS3uvG1tHABC+XK2fA8k5f3P/JcBml8rB6u7YRsoBDyf+ZhDrO9sqVDyAIS5WSW +/nQFBqonNsdFs6ZyVrx3M2heVE6RhHLxcHoLrSYGpfDKEgjRn6sVZqvWb4m4TyP2 +kZmyxufuEVfOkNVoqeD2n2BSFYyiIdx+/BwdH8cH2nIPUZAtoNxdPVI= +-----END CERTIFICATE----- diff --git a/tests/integration/scenarios/in_splunk/certificate/private_key.pem b/tests/integration/scenarios/in_splunk/certificate/private_key.pem new file mode 100644 index 00000000000..d76dae0e44f --- /dev/null +++ b/tests/integration/scenarios/in_splunk/certificate/private_key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC078ARdcg6Ptb4 +FeTDiA2A1BLsZDjaVCcBd/bhl3LoPbhOzSV6uNNPqOA2NykU7UxsivVroOogynAC +pPA9uV/zH5QRdeAh5MECxSyas54Ww3rviuHj65D05W0pGpP2zDvgOhILgovDZGQ7 +X8a+mrxVX1IMJ3Vj0wE83eF41a04xQ5E98Mg45cMmdrGR+n6r23zu9mlV9zCStZf +uLDHQ60rs1Ss9hIkJbs5I0av2r8fdYcPtn3o77U/TtPGi7nIYK9UI6UMFqkyhiYy +z96X9mhxYekRs7vZfnPpiWu29wdJVE8fqskJ1Qe4D0Q6IzV6V+n5Ye7lO7Rni3i4 +wVwKWGe9AgMBAAECggEAEp8uPF3L6rq28zeFoN4qzZyGChBSjZ3lmdAX96XXrLFV +e76/Yb8N+H/f+5R3xPCbUfyHP7mC/zb2ij8hfqjKiEIywg1IsRRTDQ7wBUgBOESM +LYU0tHl1JM9MCdPzcI4ah3xSiR65dbWqXzX/RXS1d2/lCN0yxsBbu0D2yLHtLw9k +VieZeAQu77smYODKSC76dK/IV2v19IqHbYVIWGBLJiQ1PH1qKk/bCP1lrHmLcwpx +T3gBckRhxgSayibFVu88ueq8xFPToUuy6s4zlQkPV0XhQdWXZzVg13IvUtqFzS8B +SNOvoI6jlL2kBoDlFIiCHpjuRqO8hdAw2asexYsRQQKBgQDvycZ7+q5l/+lofKe3 +Wb94ra/iESTh6iigLcj/uT6NnN3/Be2NqIOmufwQxvPaQUUnkbxSTVn9+dpADwli +yYxzz1j0eWsvAgaYz56ZLlFZ8VuthfcHG3CIdEWD14PI2ZEcuBRWGNP1DMMcQmfx +oTSvt/GTeG5RZWhIBNo+GhHY8QKBgQDBK2CoqvI+EjB2FPokjHhg7IrFPnzJQdrv +aK1QqhnJzaI77ly2cNp3cuCqsv+4BaCW+/j/OqgfQA0ykFG9cc7RZLP3ntSfAA5w +8NUaoSWrKWej99+6hN2tozoCLG1Cn2bzaEWgqrQFXuzwNS6fh2Z1FJNHl2LhrZY0 +KXKU0sqbjQKBgENvjJmLa4aWDQ9sl0JSRC3tMtyyE5xRXTGHDtMURspOxoQVJ7TC +Ipw+C+yv9x53Yrp5GuxOgrORk9aWI/6bThu3wX2ntAQZXr+VDDZqN0jDPxQy68Ec +724AvRgSASb4QP5Bqr535wwwlaKZ7l+fBZExewgNQ4EysrwmWTZD4KcRAoGAAWHj +FjHq++C7cgziYWKT9fWbZJ/22qXbAD2ah+o/tv7+uzkQdsnF9nbe/rm7NMDtjkcN +WB4+V8LolUUNILLwzPTQiOQdF2ozsEE49TDUCS6JrFW4xyfuQjDZ2Gwi+AgV/4Xu +gaDXGva78VggFkosxIe6Khf+QCky2vq09DWx930CgYA3GmotUdlOwKxXkLqgo1bQ +v4+fbXkXeL9o2xSvcx5BH0pznh1z4vvCdoRFKY7vVmwaokjzvshWAW56C6zfYo3V +gKDLT38dulbBOvqliyWJQJgUjP4T1GjEKdwKHB4a22xI0qP+Q4+OHT3ZLieivU6z +Tys72yk94uTEz6gi5RuDmQ== +-----END PRIVATE KEY----- diff --git a/tests/integration/scenarios/in_splunk/config/in_splunk_to_out_splunk.yaml b/tests/integration/scenarios/in_splunk/config/in_splunk_to_out_splunk.yaml new file mode 100644 index 00000000000..d262776c333 --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/in_splunk_to_out_splunk.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: off + tls: off + splunk_token: secret-token + store_token_in_metadata: true + outputs: + - name: splunk + match: '*' + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + tls: off + splunk_token: fallback-token diff --git a/tests/integration/scenarios/in_splunk/config/splunk_http1_keepalive.yaml b/tests/integration/scenarios/in_splunk/config/splunk_http1_keepalive.yaml new file mode 100644 index 00000000000..8ee2ed55096 --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_http1_keepalive.yaml @@ -0,0 +1,16 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: off + net.keepalive: on + tls: off + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_http1_keepalive_workers.yaml b/tests/integration/scenarios/in_splunk/config/splunk_http1_keepalive_workers.yaml new file mode 100644 index 00000000000..80216304394 --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_http1_keepalive_workers.yaml @@ -0,0 +1,17 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: off + net.keepalive: on + tls: off + http_server.workers: 4 + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_http1_no_keepalive.yaml b/tests/integration/scenarios/in_splunk/config/splunk_http1_no_keepalive.yaml new file mode 100644 index 00000000000..11eaa1855d4 --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_http1_no_keepalive.yaml @@ -0,0 +1,16 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: off + net.keepalive: off + tls: off + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_http1_tls_keepalive.yaml b/tests/integration/scenarios/in_splunk/config/splunk_http1_tls_keepalive.yaml new file mode 100644 index 00000000000..19b5c1bfe04 --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_http1_tls_keepalive.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: off + net.keepalive: on + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_http1_tls_keepalive_workers.yaml b/tests/integration/scenarios/in_splunk/config/splunk_http1_tls_keepalive_workers.yaml new file mode 100644 index 00000000000..5b10c37a70c --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_http1_tls_keepalive_workers.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: off + net.keepalive: on + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + http_server.workers: 4 + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_on_http2_keepalive.yaml b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_keepalive.yaml new file mode 100644 index 00000000000..f2fc9118ade --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_keepalive.yaml @@ -0,0 +1,16 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: on + net.keepalive: on + tls: off + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_on_http2_keepalive_tls_on.yaml b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_keepalive_tls_on.yaml new file mode 100644 index 00000000000..88515adff91 --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_keepalive_tls_on.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: off + net.keepalive: on + tls: on + #tls.crt_file: ../certificate/certificate.pem + #tls.key_file: ../certificate/private_key.pem + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_on_http2_keepalive_workers.yaml b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_keepalive_workers.yaml new file mode 100644 index 00000000000..b79a6cf329d --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_keepalive_workers.yaml @@ -0,0 +1,17 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: on + net.keepalive: on + tls: off + http_server.workers: 4 + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_on_http2_no_keepalive.yaml b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_no_keepalive.yaml new file mode 100644 index 00000000000..f25cd523fc4 --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_no_keepalive.yaml @@ -0,0 +1,16 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: on + net.keepalive: off + tls: off + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_on_http2_on_keepalive_on_tls_on.yaml b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_on_keepalive_on_tls_on.yaml new file mode 100644 index 00000000000..cb9d9374f59 --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_on_keepalive_on_tls_on.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: on + net.keepalive: on + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/config/splunk_on_http2_on_keepalive_on_tls_on_workers.yaml b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_on_keepalive_on_tls_on_workers.yaml new file mode 100644 index 00000000000..dd8aa77857d --- /dev/null +++ b/tests/integration/scenarios/in_splunk/config/splunk_on_http2_on_keepalive_on_tls_on_workers.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} +pipeline: + inputs: + - name: splunk + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + host: 0.0.0.0 + http2: on + net.keepalive: on + tls: on + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + http_server.workers: 4 + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/in_splunk/tests/test_in_splunk_001.py b/tests/integration/scenarios/in_splunk/tests/test_in_splunk_001.py new file mode 100644 index 00000000000..aeb100b3775 --- /dev/null +++ b/tests/integration/scenarios/in_splunk/tests/test_in_splunk_001.py @@ -0,0 +1,286 @@ +import json +import os +import time + +import pytest +import requests + +from server.http_server import configure_http_response, data_storage, http_server_run +from utils.test_service import FluentBitTestService +from utils.http_matrix import PROTOCOL_CASES, run_curl_request + + +SUCCESS_BODY = '{"text":"Success","code":0}' + + +class Service: + def __init__(self, config_file): + self.test_path = os.path.dirname(os.path.abspath(__file__)) + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + self.tls_crt_file = f"{self.test_path}/../certificate/certificate.pem" + self.tls_key_file = f"{self.test_path}/../certificate/private_key.pem" + self.service = FluentBitTestService( + self.config_file, + extra_env={ + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + }, + ) + + def start(self): + self.service.start() + self.flb = self.service.flb + self.flb_listener_port = self.service.flb_listener_port + + def wait_for_log_message(self, pattern, timeout=10, interval=0.25): + deadline = time.time() + timeout + while time.time() < deadline: + if self.flb and self.flb.log_file and os.path.exists(self.flb.log_file): + with open(self.flb.log_file, "r", encoding="utf-8", errors="replace") as log_file: + if pattern in log_file.read(): + return True + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for log pattern: {pattern}") + + def stop(self): + self.service.stop() + + +class ForwardingService(Service): + def __init__(self, config_file): + self.test_path = os.path.dirname(os.path.abspath(__file__)) + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + self.tls_crt_file = f"{self.test_path}/../certificate/certificate.pem" + self.tls_key_file = f"{self.test_path}/../certificate/private_key.pem" + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["payloads", "requests"], + extra_env={ + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + }, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + http_server_run(service.test_suite_http_port) + self.service.wait_for_http_endpoint( + f"http://127.0.0.1:{service.test_suite_http_port}/ping", + timeout=10, + interval=0.5, + ) + + def _stop_receiver(self, service): + try: + requests.post(f"http://127.0.0.1:{service.test_suite_http_port}/shutdown", timeout=2) + except requests.RequestException: + pass + + def start(self): + super().start() + self.test_suite_http_port = self.service.test_suite_http_port + + def wait_for_forwarded_requests(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: data_storage["requests"] if len(data_storage["requests"]) >= minimum_count else None, + timeout=timeout, + interval=0.5, + description=f"{minimum_count} forwarded Splunk requests", + ) + + +def create_splunk_headers(content_type="application/json"): + return [ + "Authorization: Splunk secret-token", + f"Content-Type: {content_type}", + ] + + +SPLUNK_PROTOCOL_CONFIGS = { + False: { + "http1_cleartext": "splunk_http1_keepalive.yaml", + "http2_cleartext": "splunk_on_http2_keepalive.yaml", + "http1_tls": "splunk_http1_tls_keepalive.yaml", + "http2_tls": "splunk_on_http2_on_keepalive_on_tls_on.yaml", + }, + True: { + "http1_cleartext": "splunk_http1_keepalive_workers.yaml", + "http2_cleartext": "splunk_on_http2_keepalive_workers.yaml", + "http1_tls": "splunk_http1_tls_keepalive_workers.yaml", + "http2_tls": "splunk_on_http2_on_keepalive_on_tls_on_workers.yaml", + }, +} + + +@pytest.mark.parametrize("workers_enabled", [False, True], ids=["single_listener", "workers_4"]) +@pytest.mark.parametrize("case", PROTOCOL_CASES, ids=[case["id"] for case in PROTOCOL_CASES]) +def test_splunk_protocol_matrix(case, workers_enabled): + service = Service(SPLUNK_PROTOCOL_CONFIGS[workers_enabled][case["config_key"]]) + service.start() + if workers_enabled: + service.wait_for_log_message("with 4 workers", timeout=10) + + scheme = "https" if case["use_tls"] else "http" + url = f"{scheme}://localhost:{service.flb_listener_port}/services/collector" + result = run_curl_request( + url, + json.dumps({"Event": "Some text in the event"}), + headers=create_splunk_headers(), + http_mode=case["http_mode"], + ca_cert_path=service.tls_crt_file if case["use_tls"] else None, + ) + + service.stop() + + assert result["status_code"] == 200 + assert result["body"] == SUCCESS_BODY + assert result["http_version"] == case["expected_http_version"] + + +def test_splunk_http1_no_keepalive(): + service = Service("splunk_http1_no_keepalive.yaml") + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/services/collector", + json.dumps({"Event": "Some text in the event"}), + headers=create_splunk_headers(), + http_mode="http1.1", + ) + + service.stop() + + assert result["status_code"] == 200 + assert result["body"] == SUCCESS_BODY + assert result["http_version"] == "1.1" + + +@pytest.mark.parametrize( + ("method", "endpoint"), + [ + ("POST", "/services/collector/unsupported"), + ("GET", "/services/collector"), + ], + ids=["unsupported_uri", "unsupported_method"], +) +def test_in_splunk_rejects_invalid_requests(method, endpoint): + service = Service("splunk_http1_keepalive.yaml") + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}{endpoint}", + json.dumps({"event": "Some text in the event"}) if method == "POST" else None, + method=method, + headers=create_splunk_headers() if method == "POST" else [], + http_mode="http1.1", + ) + + service.stop() + + assert result["status_code"] >= 400 + + +def test_in_splunk_accepts_missing_authorization_header_by_default(): + service = Service("splunk_http1_keepalive.yaml") + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/services/collector", + json.dumps({"event": "Some text in the event"}), + headers=["Content-Type: application/json"], + http_mode="http1.1", + ) + + service.stop() + + assert result["status_code"] == 200 + + +def test_in_splunk_to_out_splunk_prefers_configured_output_token(): + service = ForwardingService("in_splunk_to_out_splunk.yaml") + service.start() + configure_http_response(status_code=200, body={"text": "Success", "code": 0}) + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}/services/collector/event", + json.dumps({"event": "Some text in the event"}), + headers=create_splunk_headers(), + http_mode="http1.1", + ) + + forwarded_requests = service.wait_for_forwarded_requests(1) + service.stop() + + assert result["status_code"] == 200 + assert result["body"] == SUCCESS_BODY + assert forwarded_requests[0]["path"] == "/services/collector/event" + assert forwarded_requests[0]["headers"].get("Authorization") == "Splunk fallback-token" + + +SPLUNK_URI_CASES = [ + { + "id": "collector", + "method": "POST", + "endpoint": "/services/collector", + "payload": json.dumps({"Event": "Some text in the event"}), + "headers": create_splunk_headers(), + }, + { + "id": "collector_event", + "method": "POST", + "endpoint": "/services/collector/event", + "payload": json.dumps({"event": "Some text in the event"}), + "headers": create_splunk_headers(), + }, + { + "id": "collector_raw", + "method": "POST", + "endpoint": "/services/collector/raw", + "payload": "1, 2, 3... Hello, world!", + "headers": create_splunk_headers("application/json"), + }, + { + "id": "collector_event_1_0", + "method": "POST", + "endpoint": "/services/collector/event/1.0", + "payload": json.dumps({"event": "Some text in the event"}), + "headers": create_splunk_headers(), + }, + { + "id": "collector_raw_1_0", + "method": "POST", + "endpoint": "/services/collector/raw/1.0", + "payload": "1, 2, 3... Hello, world!", + "headers": create_splunk_headers("application/json"), + }, + { + "id": "collector_health", + "method": "GET", + "endpoint": "/services/collector/health", + "payload": None, + "headers": [], + "expected_body": '{"text":"Success","code":200}', + }, +] + + +@pytest.mark.parametrize("case", SPLUNK_URI_CASES, ids=[case["id"] for case in SPLUNK_URI_CASES]) +def test_in_splunk_uri_variants(case): + service = Service("splunk_http1_keepalive.yaml") + service.start() + + result = run_curl_request( + f"http://localhost:{service.flb_listener_port}{case['endpoint']}", + case["payload"], + method=case["method"], + headers=case["headers"], + http_mode="http1.1", + ) + + service.stop() + + assert result["status_code"] == 200 + assert result["body"] == case.get("expected_body", SUCCESS_BODY) + assert result["http_version"] == "1.1" diff --git a/tests/integration/scenarios/in_syslog/config/in_syslog_tcp_plaintext.yaml b/tests/integration/scenarios/in_syslog/config/in_syslog_tcp_plaintext.yaml new file mode 100644 index 00000000000..dc87d5b097b --- /dev/null +++ b/tests/integration/scenarios/in_syslog/config/in_syslog_tcp_plaintext.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: syslog + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + mode: tcp + parser: syslog-rfc5424 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_syslog/config/in_syslog_tcp_tls.yaml b/tests/integration/scenarios/in_syslog/config/in_syslog_tcp_tls.yaml new file mode 100644 index 00000000000..6d58a4e36c8 --- /dev/null +++ b/tests/integration/scenarios/in_syslog/config/in_syslog_tcp_tls.yaml @@ -0,0 +1,29 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: syslog + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + mode: tcp + parser: syslog-rfc5424 + tls: on + tls.verify: no + tls.vhost: localhost + tls.crt_file: ${CERTIFICATE_TEST} + tls.key_file: ${PRIVATE_KEY_TEST} + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_syslog/config/in_syslog_udp_plaintext.yaml b/tests/integration/scenarios/in_syslog/config/in_syslog_udp_plaintext.yaml new file mode 100644 index 00000000000..c49ff7f13da --- /dev/null +++ b/tests/integration/scenarios/in_syslog/config/in_syslog_udp_plaintext.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: syslog + tag: target_input + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + mode: udp + parser: syslog-rfc5424 + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_syslog/config/in_syslog_uds_dgram_plaintext.yaml b/tests/integration/scenarios/in_syslog/config/in_syslog_uds_dgram_plaintext.yaml new file mode 100644 index 00000000000..9c36cc78f37 --- /dev/null +++ b/tests/integration/scenarios/in_syslog/config/in_syslog_uds_dgram_plaintext.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: syslog + tag: target_input + path: ${SYSLOG_SOCKET_PATH} + mode: unix_udp + parser: syslog-rfc3164-local + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_syslog/config/in_syslog_uds_stream_plaintext.yaml b/tests/integration/scenarios/in_syslog/config/in_syslog_uds_stream_plaintext.yaml new file mode 100644 index 00000000000..20f9dab03c2 --- /dev/null +++ b/tests/integration/scenarios/in_syslog/config/in_syslog_uds_stream_plaintext.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + parsers_file: ${PARSERS_FILE_TEST} + +pipeline: + inputs: + - name: syslog + tag: target_input + path: ${SYSLOG_SOCKET_PATH} + mode: unix_tcp + parser: syslog-rfc3164-local + + outputs: + - name: http + match: target_input + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json diff --git a/tests/integration/scenarios/in_syslog/tests/test_in_syslog_001.py b/tests/integration/scenarios/in_syslog/tests/test_in_syslog_001.py new file mode 100644 index 00000000000..615b4845399 --- /dev/null +++ b/tests/integration/scenarios/in_syslog/tests/test_in_syslog_001.py @@ -0,0 +1,177 @@ +import os +import socket +import ssl +import sys +import uuid + +import pytest +import requests + +from server.http_server import data_storage, http_server_run +from utils.test_service import FluentBitTestService + + +TCP_RFC5424_MESSAGE = b"<13>1 1970-01-01T00:00:00.000000+00:00 testhost testuser - - [] Hello!\n" +UDS_RFC3164_MESSAGE = b"<13>Jan 1 00:00:00 testuser: Hello!\n" + + +class Service: + def __init__(self, config_file): + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + test_path = os.path.dirname(os.path.abspath(__file__)) + cert_dir = os.path.abspath(os.path.join(test_path, "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + self.parsers_file = os.environ.get("FLUENT_BIT_PARSERS_FILE") or os.path.abspath(os.path.join(test_path, "../../../../../conf/parsers.conf")) + self.socket_path = f"/tmp/fluent_bit_syslog_{uuid.uuid4().hex}.sock" + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["payloads"], + extra_env={ + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + "PARSERS_FILE_TEST": self.parsers_file, + "SYSLOG_SOCKET_PATH": self.socket_path, + }, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + http_server_run(service.test_suite_http_port) + self.service.wait_for_http_endpoint( + f"http://127.0.0.1:{service.test_suite_http_port}/ping", + timeout=10, + interval=0.5, + ) + + def _stop_receiver(self, service): + try: + requests.post(f"http://127.0.0.1:{service.test_suite_http_port}/shutdown", timeout=2) + except requests.RequestException: + pass + try: + os.unlink(self.socket_path) + except FileNotFoundError: + pass + + def start(self): + self.service.start() + self.flb_listener_port = self.service.flb_listener_port + + def stop(self): + self.service.stop() + + def read_forwarded_payloads(self, timeout=10): + return self.service.wait_for_condition( + lambda: data_storage["payloads"] if data_storage["payloads"] else None, + timeout=timeout, + interval=0.5, + description="forwarded syslog payloads", + ) + + +def _assert_message(payloads): + assert len(payloads) == 1 + assert isinstance(payloads[0], list) + assert len(payloads[0]) == 1 + record = payloads[0][0] + assert record["message"] == "Hello!" + + +def test_in_syslog_tcp_plaintext(): + service = Service("in_syslog_tcp_plaintext.yaml") + service.start() + + try: + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as sock: + sock.sendall(TCP_RFC5424_MESSAGE) + sock.shutdown(socket.SHUT_WR) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_message(payloads) + + +def test_in_syslog_tcp_tls(): + service = Service("in_syslog_tcp_tls.yaml") + service.start() + + try: + context = ssl.create_default_context(cafile=service.tls_crt_file) + with socket.create_connection(("127.0.0.1", service.flb_listener_port), timeout=5) as raw_sock: + with context.wrap_socket(raw_sock, server_hostname="localhost") as tls_sock: + tls_sock.sendall(TCP_RFC5424_MESSAGE) + tls_sock.shutdown(socket.SHUT_WR) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_message(payloads) + + +def test_in_syslog_udp_plaintext(): + service = Service("in_syslog_udp_plaintext.yaml") + service.start() + + try: + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: + sock.sendto(TCP_RFC5424_MESSAGE, ("127.0.0.1", service.flb_listener_port)) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_message(payloads) + + +def test_in_syslog_uds_stream_plaintext(): + service = Service("in_syslog_uds_stream_plaintext.yaml") + service.start() + + try: + service.service.wait_for_condition( + lambda: os.path.exists(service.socket_path), + timeout=10, + interval=0.2, + description="syslog unix stream socket", + ) + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.connect(service.socket_path) + sock.sendall(UDS_RFC3164_MESSAGE) + sock.shutdown(socket.SHUT_WR) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_message(payloads) + + +@pytest.mark.skipif(sys.platform == "darwin", reason="unix datagram syslog runtime-shell test is skipped on Darwin") +def test_in_syslog_uds_dgram_plaintext(): + service = Service("in_syslog_uds_dgram_plaintext.yaml") + service.start() + + try: + service.service.wait_for_condition( + lambda: os.path.exists(service.socket_path), + timeout=10, + interval=0.2, + description="syslog unix datagram socket", + ) + + with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock: + sock.connect(service.socket_path) + sock.sendall(UDS_RFC3164_MESSAGE) + + payloads = service.read_forwarded_payloads() + finally: + service.stop() + + _assert_message(payloads) diff --git a/tests/integration/scenarios/internal_http_server/config/internal_http_server.yaml b/tests/integration/scenarios/internal_http_server/config/internal_http_server.yaml new file mode 100644 index 00000000000..5fca09ad6be --- /dev/null +++ b/tests/integration/scenarios/internal_http_server/config/internal_http_server.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_listen: 127.0.0.1 + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + health_check: on + storage.metrics: on + hot_reload: on + enable_chunk_trace: on + +pipeline: + inputs: + - name: dummy + tag: test + dummy: '{"message":"hello"}' + + outputs: + - name: stdout + match: "*" diff --git a/tests/integration/scenarios/internal_http_server/tests/test_internal_http_server_001.py b/tests/integration/scenarios/internal_http_server/tests/test_internal_http_server_001.py new file mode 100644 index 00000000000..f1bf18a03f8 --- /dev/null +++ b/tests/integration/scenarios/internal_http_server/tests/test_internal_http_server_001.py @@ -0,0 +1,148 @@ +import concurrent.futures +import os +import subprocess + +import pytest +from utils.http_matrix import curl_supports_http2, run_curl_request +from utils.test_service import FluentBitTestService + + +def _headers_map(headers_raw): + headers = {} + for line in headers_raw.splitlines(): + if ":" not in line or line.startswith("HTTP/"): + continue + key, value = line.split(":", 1) + headers[key.strip().lower()] = value.strip() + return headers + + +class Service: + def __init__(self): + self.config_file = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../config/internal_http_server.yaml") + ) + self.service = FluentBitTestService(self.config_file) + + def start(self): + self.service.start() + self.flb = self.service.flb + self.base_url = f"http://127.0.0.1:{self.flb.http_monitoring_port}" + + def stop(self): + self.service.stop() + + def request(self, path, *, method="GET", http_mode="http1.1", include_headers=False): + return run_curl_request( + f"{self.base_url}{path}", + method=method, + payload=None, + http_mode=http_mode, + include_headers=include_headers, + ) + + +def test_internal_http_server_endpoints(): + service = Service() + service.start() + root_headers = _headers_map(service.request("/", include_headers=True)["headers_raw"]) + require_unified_v2_health = ( + root_headers.get("server") == "Fluent Bit" + and root_headers.get("x-http-engine") == "Monkey heritage" + ) + + checks = [ + ("/", "200", "fluent-bit"), + ("/api/v1/uptime", "200", "uptime_sec"), + ("/api/v1/plugins", "200", "inputs"), + ("/api/v1/health", "200", "ok"), + ("/api/v1/metrics", "200", "output"), + ("/api/v1/metrics/prometheus", "200", "fluentbit_uptime"), + ("/api/v1/storage", "200", "chunks"), + ("/api/v2/metrics", "200", "fluentbit_uptime"), + ("/api/v2/metrics/prometheus", "200", "fluentbit_uptime"), + ("/api/v2/reload", "200", "hot_reload_count"), + ] + + try: + for path, expected_status, pattern in checks: + result = service.service.wait_for_condition( + lambda: ( + response + if response["status_code"] == int(expected_status) and pattern in response["body"] + else None + ) if (response := service.request(path)) else None, + timeout=10, + interval=0.5, + description=f"internal endpoint {path}", + ) + assert result["status_code"] == int(expected_status) + assert pattern in result["body"] + + v2_health = service.request("/api/v2/health") + if require_unified_v2_health: + assert v2_health["status_code"] == 200 + assert "ok" in v2_health["body"] + + trace_enable = service.request("/api/v1/trace/dummy.0") + assert trace_enable["status_code"] == 200 + assert '"status":"ok"' in trace_enable["body"] + + trace_disable = service.request("/api/v1/trace/dummy.0", method="DELETE") + assert trace_disable["status_code"] == 201 + assert '"status":"ok"' in trace_disable["body"] + finally: + service.stop() + + +def test_internal_http_server_headers_and_concurrency(): + service = Service() + service.start() + + try: + result = service.request("/", include_headers=True) + headers = _headers_map(result["headers_raw"]) + + assert result["status_code"] == 200 + if headers.get("server") != "Fluent Bit" or headers.get("x-http-engine") != "Monkey heritage": + pytest.skip("Unified internal HTTP server headers are not available in this Fluent Bit binary") + assert headers["server"] == "Fluent Bit" + assert headers["x-http-engine"] == "Monkey heritage" + + header_lines = [line.lower() for line in result["headers_raw"].splitlines()] + server_index = next(index for index, line in enumerate(header_lines) if line.startswith("server:")) + engine_index = next(index for index, line in enumerate(header_lines) if line.startswith("x-http-engine:")) + content_type_index = next(index for index, line in enumerate(header_lines) if line.startswith("content-type:")) + assert server_index < content_type_index + assert engine_index < content_type_index + + def fetch_metrics(): + response = service.request("/api/v1/metrics/prometheus") + assert response["status_code"] == 200 + assert "fluentbit_uptime" in response["body"] + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: + futures = [executor.submit(fetch_metrics) for _ in range(50)] + for future in futures: + future.result() + finally: + service.stop() + + +def test_internal_http_server_http2_subset(): + if not curl_supports_http2(): + return + + service = Service() + service.start() + + try: + for path in ["/", "/api/v1/uptime", "/api/v1/metrics/prometheus", "/api/v2/metrics/prometheus", "/api/v2/reload"]: + try: + result = service.request(path, http_mode="http2-prior-knowledge") + except subprocess.CalledProcessError: + pytest.skip("Internal HTTP server does not support HTTP/2 prior knowledge in this Fluent Bit binary") + assert result["status_code"] == 200 + assert result["http_version"] == "2" + finally: + service.stop() diff --git a/tests/integration/scenarios/out_azure_logs_ingestion/config/out_azure_logs_ingestion_oauth2.yaml b/tests/integration/scenarios/out_azure_logs_ingestion/config/out_azure_logs_ingestion_oauth2.yaml new file mode 100644 index 00000000000..c7b86fdbfa8 --- /dev/null +++ b/tests/integration/scenarios/out_azure_logs_ingestion/config/out_azure_logs_ingestion_oauth2.yaml @@ -0,0 +1,28 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: azure_logs_ingestion + dummy: '{"message":"hello from azure logs ingestion","source":"dummy","level":"info"}' + samples: 1 + + outputs: + - name: azure_logs_ingestion + match: azure_logs_ingestion + tenant_id: suite-tenant + client_id: suite-client + client_secret: suite-secret + auth_url: http://127.0.0.1:${TEST_SUITE_OAUTH_PORT}/oauth/token + dce_url: https://localhost:${TEST_SUITE_HTTP_PORT} + dcr_id: dcr-suite + table_name: suite_CL + compress: on + tls.verify: on + tls.verify_hostname: on + tls.vhost: localhost + tls.ca_file: ${CERTIFICATE_TEST} diff --git a/tests/integration/scenarios/out_azure_logs_ingestion/tests/test_out_azure_logs_ingestion_001.py b/tests/integration/scenarios/out_azure_logs_ingestion/tests/test_out_azure_logs_ingestion_001.py new file mode 100644 index 00000000000..4f0dcb4d5e2 --- /dev/null +++ b/tests/integration/scenarios/out_azure_logs_ingestion/tests/test_out_azure_logs_ingestion_001.py @@ -0,0 +1,156 @@ +import logging +import os + +import requests + +from server.http_server import ( + configure_http_response, + configure_oauth_token_response, + data_storage, + http_server_run, +) +from utils.test_service import FluentBitTestService + +logger = logging.getLogger(__name__) + + +class Service: + def __init__(self, config_file): + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + cert_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + self.oauth_server_port = None + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["payloads", "requests"], + extra_env={ + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + }, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + self.oauth_server_port = service.allocate_port_env("TEST_SUITE_OAUTH_PORT") + http_server_run(self.oauth_server_port) + http_server_run( + service.test_suite_http_port, + use_tls=True, + tls_crt_file=self.tls_crt_file, + tls_key_file=self.tls_key_file, + reset_state=False, + ) + + def _http_ready(): + try: + response = requests.get( + f"http://127.0.0.1:{self.oauth_server_port}/ping", + timeout=1, + ) + return response.status_code == 200 + except requests.RequestException: + return False + + def _https_ready(): + try: + response = requests.get( + f"https://localhost:{service.test_suite_http_port}/ping", + timeout=1, + verify=self.tls_crt_file, + ) + return response.status_code == 200 + except requests.RequestException: + return False + + self.service.wait_for_condition( + _http_ready, + timeout=10, + interval=0.5, + description="azure logs ingestion oauth receiver readiness", + ) + + self.service.wait_for_condition( + _https_ready, + timeout=10, + interval=0.5, + description="azure logs ingestion receiver readiness", + ) + + def _stop_receiver(self, service): + try: + if self.oauth_server_port is not None: + requests.post( + f"http://127.0.0.1:{self.oauth_server_port}/shutdown", + timeout=2, + ) + except requests.RequestException: + pass + + try: + requests.post( + f"https://localhost:{service.test_suite_http_port}/shutdown", + timeout=2, + verify=self.tls_crt_file, + ) + except requests.RequestException: + pass + + def start(self): + self.service.start() + self.flb = self.service.flb + self.flb_listener_port = self.service.flb_listener_port + self.test_suite_http_port = self.service.test_suite_http_port + + def stop(self): + self.service.stop() + + def wait_for_requests(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: data_storage["requests"] if len(data_storage["requests"]) >= minimum_count else None, + timeout=timeout, + interval=0.5, + description=f"{minimum_count} azure logs ingestion requests", + ) + + +def test_out_azure_logs_ingestion_legacy_oauth2_and_payload_format(): + service = Service("out_azure_logs_ingestion_oauth2.yaml") + service.start() + configure_http_response(status_code=200, body={"status": "received"}) + configure_oauth_token_response( + status_code=200, + body={"access_token": "oauth-access-token", "token_type": "Bearer", "expires_in": 300}, + ) + + requests_seen = service.wait_for_requests(2, timeout=15) + service.stop() + + token_request = next(request for request in requests_seen if request["path"] == "/oauth/token") + data_request = next( + request + for request in requests_seen + if request["path"] == "/dataCollectionRules/dcr-suite/streams/Custom-suite_CL" + ) + + assert token_request["method"] == "POST" + assert "grant_type=client_credentials" in token_request["raw_data"] + assert "scope=https://monitor.azure.com/.default" in token_request["raw_data"] + assert "client_id=suite-client" in token_request["raw_data"] + assert "client_secret=suite-secret" in token_request["raw_data"] + + assert data_request["method"] == "POST" + assert data_request["query_string"] == "api-version=2021-11-01-preview" + assert data_request["headers"].get("Authorization") == "Bearer oauth-access-token" + assert data_request["headers"].get("Content-Encoding") == "gzip" + assert data_request["headers"].get("Content-Type") == "application/json" + + payload = data_request["json"] + assert isinstance(payload, list) + assert len(payload) == 1 + assert payload[0]["message"] == "hello from azure logs ingestion" + assert payload[0]["source"] == "dummy" + assert payload[0]["level"] == "info" + assert isinstance(payload[0]["@timestamp"], (int, float)) diff --git a/tests/integration/scenarios/out_http/config/out_http_basic.yaml b/tests/integration/scenarios/out_http/config/out_http_basic.yaml new file mode 100644 index 00000000000..4d48b458aa1 --- /dev/null +++ b/tests/integration/scenarios/out_http/config/out_http_basic.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_http + dummy: '{"message":"hello from out_http","source":"dummy"}' + samples: 1 + + outputs: + - name: http + match: out_http + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json + json_date_key: false diff --git a/tests/integration/scenarios/out_http/config/out_http_oauth2_basic.yaml b/tests/integration/scenarios/out_http/config/out_http_oauth2_basic.yaml new file mode 100644 index 00000000000..bb2801ef863 --- /dev/null +++ b/tests/integration/scenarios/out_http/config/out_http_oauth2_basic.yaml @@ -0,0 +1,26 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_http + dummy: '{"message":"hello from oauth2 basic","source":"dummy"}' + samples: 1 + + outputs: + - name: http + match: out_http + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json + json_date_key: false + oauth2.enable: true + oauth2.token_url: http://127.0.0.1:${TEST_SUITE_HTTP_PORT}/oauth/token + oauth2.client_id: client1 + oauth2.client_secret: secret1 + oauth2.scope: logs.write diff --git a/tests/integration/scenarios/out_http/config/out_http_oauth2_private_key_jwt.yaml b/tests/integration/scenarios/out_http/config/out_http_oauth2_private_key_jwt.yaml new file mode 100644 index 00000000000..b6e6fc62c64 --- /dev/null +++ b/tests/integration/scenarios/out_http/config/out_http_oauth2_private_key_jwt.yaml @@ -0,0 +1,28 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_http + dummy: '{"message":"hello from oauth2 jwt","source":"dummy"}' + samples: 1 + + outputs: + - name: http + match: out_http + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json + json_date_key: false + oauth2.enable: true + oauth2.token_url: http://127.0.0.1:${TEST_SUITE_HTTP_PORT}/oauth/token + oauth2.client_id: client1 + oauth2.auth_method: private_key_jwt + oauth2.jwt_key_file: ${PRIVATE_KEY_TEST} + oauth2.jwt_cert_file: ${CERTIFICATE_TEST} + oauth2.jwt_aud: http://127.0.0.1:${TEST_SUITE_HTTP_PORT}/oauth/token diff --git a/tests/integration/scenarios/out_http/config/out_http_retry.yaml b/tests/integration/scenarios/out_http/config/out_http_retry.yaml new file mode 100644 index 00000000000..83aa275234c --- /dev/null +++ b/tests/integration/scenarios/out_http/config/out_http_retry.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_http + dummy: '{"message":"retry me","source":"dummy"}' + samples: 1 + + outputs: + - name: http + match: out_http + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + uri: /data + format: json + json_date_key: false + retry_limit: 1 diff --git a/tests/integration/scenarios/out_http/tests/test_out_http_001.py b/tests/integration/scenarios/out_http/tests/test_out_http_001.py new file mode 100644 index 00000000000..52f444b9765 --- /dev/null +++ b/tests/integration/scenarios/out_http/tests/test_out_http_001.py @@ -0,0 +1,140 @@ +import json +import logging +import os + +import requests + +from server.http_server import ( + configure_http_response, + configure_oauth_token_response, + data_storage, + http_server_run, +) +from utils.test_service import FluentBitTestService + +logger = logging.getLogger(__name__) + + +class Service: + def __init__(self, config_file): + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + test_path = os.path.dirname(os.path.abspath(__file__)) + cert_dir = os.path.abspath(os.path.join(test_path, "../../in_splunk/certificate")) + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["payloads", "requests"], + extra_env={ + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + }, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + http_server_run(service.test_suite_http_port) + self.service.wait_for_http_endpoint( + f"http://127.0.0.1:{service.test_suite_http_port}/ping", + timeout=10, + interval=0.5, + ) + + def _stop_receiver(self, service): + try: + requests.post(f"http://127.0.0.1:{service.test_suite_http_port}/shutdown", timeout=2) + except requests.RequestException: + pass + + def start(self): + self.service.start() + self.flb = self.service.flb + self.flb_listener_port = self.service.flb_listener_port + self.test_suite_http_port = self.service.test_suite_http_port + + def stop(self): + self.service.stop() + + def wait_for_requests(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: data_storage["requests"] if len(data_storage["requests"]) >= minimum_count else None, + timeout=timeout, + interval=0.5, + description=f"{minimum_count} outbound HTTP requests", + ) + + +def test_out_http_sends_json_payload(): + service = Service("out_http_basic.yaml") + service.start() + configure_http_response(status_code=200, body={"status": "received"}) + + requests_seen = service.wait_for_requests(1) + service.stop() + + first_request = requests_seen[0] + assert first_request["path"] == "/data" + assert first_request["method"] == "POST" + assert "application/json" in first_request["headers"].get("Content-Type", "") + + payload = json.loads(first_request["raw_data"]) + assert isinstance(payload, list) + assert payload[0]["message"] == "hello from out_http" + assert payload[0]["source"] == "dummy" + + +def test_out_http_receiver_error_is_observable(): + service = Service("out_http_retry.yaml") + service.start() + configure_http_response(status_code=500, body={"status": "error"}) + + requests_seen = service.wait_for_requests(1, timeout=10) + service.stop() + + assert len(requests_seen) >= 1 + + +def test_out_http_oauth2_basic_adds_bearer_token(): + service = Service("out_http_oauth2_basic.yaml") + service.start() + configure_http_response(status_code=200, body={"status": "received"}) + configure_oauth_token_response( + status_code=200, + body={"access_token": "oauth-access-token", "token_type": "Bearer", "expires_in": 300}, + ) + + requests_seen = service.wait_for_requests(2) + service.stop() + + token_request = next(request for request in requests_seen if request["path"] == "/oauth/token") + data_request = next(request for request in requests_seen if request["path"] == "/data") + + assert token_request["method"] == "POST" + assert "Basic " in token_request["headers"].get("Authorization", "") + assert "grant_type=client_credentials" in token_request["raw_data"] + assert "scope=logs.write" in token_request["raw_data"] + assert data_request["headers"].get("Authorization") == "Bearer oauth-access-token" + + +def test_out_http_oauth2_private_key_jwt_adds_bearer_token(): + service = Service("out_http_oauth2_private_key_jwt.yaml") + service.start() + configure_http_response(status_code=200, body={"status": "received"}) + configure_oauth_token_response( + status_code=200, + body={"access_token": "oauth-access-token", "token_type": "Bearer", "expires_in": 300}, + ) + + requests_seen = service.wait_for_requests(2) + service.stop() + + token_request = next(request for request in requests_seen if request["path"] == "/oauth/token") + data_request = next(request for request in requests_seen if request["path"] == "/data") + + assert token_request["method"] == "POST" + assert "client_assertion_type=" in token_request["raw_data"] + assert "client_assertion=" in token_request["raw_data"] + assert "client_id=client1" in token_request["raw_data"] + assert data_request["headers"].get("Authorization") == "Bearer oauth-access-token" diff --git a/tests/integration/scenarios/out_kafka/config/out_kafka_basic.yaml b/tests/integration/scenarios/out_kafka/config/out_kafka_basic.yaml new file mode 100644 index 00000000000..82361945192 --- /dev/null +++ b/tests/integration/scenarios/out_kafka/config/out_kafka_basic.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_kafka + dummy: '{"message":"hello from out_kafka","source":"dummy"}' + samples: 1 + + outputs: + - name: kafka + match: out_kafka + brokers: 127.0.0.1:${TEST_SUITE_KAFKA_PORT} + topics: test + format: json + queue_full_retries: 1 + rdkafka.api.version.request: false + rdkafka.broker.version.fallback: 0.8.2.0 diff --git a/tests/integration/scenarios/out_kafka/config/out_kafka_dynamic_topic.yaml b/tests/integration/scenarios/out_kafka/config/out_kafka_dynamic_topic.yaml new file mode 100644 index 00000000000..091845f32dc --- /dev/null +++ b/tests/integration/scenarios/out_kafka/config/out_kafka_dynamic_topic.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_kafka + dummy: '{"message":"hello dynamic topic","topic_name":"topic-dynamic","source":"dummy"}' + samples: 1 + + outputs: + - name: kafka + match: out_kafka + brokers: 127.0.0.1:${TEST_SUITE_KAFKA_PORT} + topics: test + dynamic_topic: true + topic_key: topic_name + format: json + queue_full_retries: 1 + rdkafka.api.version.request: false + rdkafka.broker.version.fallback: 0.8.2.0 diff --git a/tests/integration/scenarios/out_kafka/config/out_kafka_message_key_field.yaml b/tests/integration/scenarios/out_kafka/config/out_kafka_message_key_field.yaml new file mode 100644 index 00000000000..b7200836dcc --- /dev/null +++ b/tests/integration/scenarios/out_kafka/config/out_kafka_message_key_field.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_kafka + dummy: '{"message":"hello with key","message_key":"key-123","source":"dummy"}' + samples: 1 + + outputs: + - name: kafka + match: out_kafka + brokers: 127.0.0.1:${TEST_SUITE_KAFKA_PORT} + topics: test + format: json + message_key_field: message_key + queue_full_retries: 1 + rdkafka.api.version.request: false + rdkafka.broker.version.fallback: 0.8.2.0 diff --git a/tests/integration/scenarios/out_kafka/config/out_kafka_msgpack.yaml b/tests/integration/scenarios/out_kafka/config/out_kafka_msgpack.yaml new file mode 100644 index 00000000000..9e258fac34b --- /dev/null +++ b/tests/integration/scenarios/out_kafka/config/out_kafka_msgpack.yaml @@ -0,0 +1,22 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_kafka + dummy: '{"message":"hello msgpack","count":7,"source":"dummy"}' + samples: 1 + + outputs: + - name: kafka + match: out_kafka + brokers: 127.0.0.1:${TEST_SUITE_KAFKA_PORT} + topics: test + format: msgpack + queue_full_retries: 1 + rdkafka.api.version.request: false + rdkafka.broker.version.fallback: 0.8.2.0 diff --git a/tests/integration/scenarios/out_kafka/config/out_kafka_otlp_json.yaml b/tests/integration/scenarios/out_kafka/config/out_kafka_otlp_json.yaml new file mode 100644 index 00000000000..63d16d5d782 --- /dev/null +++ b/tests/integration/scenarios/out_kafka/config/out_kafka_otlp_json.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: kafka + match: "*" + brokers: 127.0.0.1:${TEST_SUITE_KAFKA_PORT} + topics: otlp-topic + format: otlp_json + message_key: static-otlp-key + raw_log_key: ignored_raw_key + message_key_field: ignored_message_key_field + topic_key: ignored_topic_key + queue_full_retries: 1 + rdkafka.api.version.request: false + rdkafka.broker.version.fallback: 0.8.2.0 diff --git a/tests/integration/scenarios/out_kafka/config/out_kafka_otlp_proto.yaml b/tests/integration/scenarios/out_kafka/config/out_kafka_otlp_proto.yaml new file mode 100644 index 00000000000..32ce0808d16 --- /dev/null +++ b/tests/integration/scenarios/out_kafka/config/out_kafka_otlp_proto.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: kafka + match: "*" + brokers: 127.0.0.1:${TEST_SUITE_KAFKA_PORT} + topics: otlp-topic + format: otlp_proto + message_key: static-otlp-key + raw_log_key: ignored_raw_key + message_key_field: ignored_message_key_field + topic_key: ignored_topic_key + queue_full_retries: 1 + rdkafka.api.version.request: false + rdkafka.broker.version.fallback: 0.8.2.0 diff --git a/tests/integration/scenarios/out_kafka/config/out_kafka_raw.yaml b/tests/integration/scenarios/out_kafka/config/out_kafka_raw.yaml new file mode 100644 index 00000000000..536898ca3b4 --- /dev/null +++ b/tests/integration/scenarios/out_kafka/config/out_kafka_raw.yaml @@ -0,0 +1,23 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_kafka + dummy: '{"key_0":"raw value","source":"dummy"}' + samples: 1 + + outputs: + - name: kafka + match: out_kafka + brokers: 127.0.0.1:${TEST_SUITE_KAFKA_PORT} + topics: test + format: raw + raw_log_key: key_0 + queue_full_retries: 1 + rdkafka.api.version.request: false + rdkafka.broker.version.fallback: 0.8.2.0 diff --git a/tests/integration/scenarios/out_kafka/tests/test_out_kafka_001.py b/tests/integration/scenarios/out_kafka/tests/test_out_kafka_001.py new file mode 100644 index 00000000000..50a670b6ead --- /dev/null +++ b/tests/integration/scenarios/out_kafka/tests/test_out_kafka_001.py @@ -0,0 +1,524 @@ +import json +import os +import struct +from copy import deepcopy + +import requests +import pytest +from google.protobuf import json_format +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from server.kafka_server import data_storage, kafka_server_run, kafka_server_stop +from utils.data_utils import read_json_file +from utils.test_service import FluentBitTestService + + +class Service: + def __init__(self, config_file): + self.config_file = os.path.abspath(os.path.join(os.path.dirname(__file__), "../config", config_file)) + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["connections", "requests", "messages"], + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + self.kafka_port = service.allocate_port_env("TEST_SUITE_KAFKA_PORT") + kafka_server_run(self.kafka_port) + + def _stop_receiver(self, service): + kafka_server_stop() + + def start(self): + self.service.start() + self.flb = self.service.flb + self.flb_listener_port = self.service.flb_listener_port + + def stop(self): + self.service.stop() + + def wait_for_messages(self, minimum_count=1, timeout=10): + return self.service.wait_for_condition( + lambda: data_storage["messages"] if len(data_storage["messages"]) >= minimum_count else None, + timeout=timeout, + interval=0.5, + description=f"{minimum_count} Kafka messages", + ) + + def send_json_logs_payload(self, json_file): + payload = self._build_signal_payload(json_file, "logs") + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/logs", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def send_json_metrics_payload(self, json_file): + payload = self._build_signal_payload(json_file, "metrics") + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/metrics", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def send_json_traces_payload(self, json_file): + payload = self._build_signal_payload(json_file, "traces") + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/traces", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def send_payload_dict(self, payload_dict, signal_type): + payload = self._build_signal_payload_from_dict(payload_dict, signal_type) + endpoints = { + "logs": "/v1/logs", + "metrics": "/v1/metrics", + "traces": "/v1/traces", + } + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}{endpoints[signal_type]}", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def _resolve_json_fixture(self, json_file): + return os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../../in_opentelemetry/tests/data_files", + json_file, + ) + ) + + def _build_signal_payload(self, json_file, signal_type): + messages = { + "logs": ExportLogsServiceRequest(), + "metrics": ExportMetricsServiceRequest(), + "traces": ExportTraceServiceRequest(), + } + return json_format.Parse( + json.dumps(read_json_file(self._resolve_json_fixture(json_file))), + messages[signal_type], + ) + + def _build_signal_payload_from_dict(self, payload_dict, signal_type): + messages = { + "logs": ExportLogsServiceRequest(), + "metrics": ExportMetricsServiceRequest(), + "traces": ExportTraceServiceRequest(), + } + return json_format.Parse(json.dumps(payload_dict), messages[signal_type]) + + +def _decode_simple_msgpack(data, offset=0): + first = data[offset] + offset += 1 + + if first <= 0x7F: + return first, offset + if 0xA0 <= first <= 0xBF: + size = first & 0x1F + end = offset + size + return data[offset:end].decode("utf-8"), end + if 0x80 <= first <= 0x8F: + size = first & 0x0F + mapping = {} + for _ in range(size): + key, offset = _decode_simple_msgpack(data, offset) + value, offset = _decode_simple_msgpack(data, offset) + mapping[key] = value + return mapping, offset + if first == 0xC0: + return None, offset + if first == 0xC2: + return False, offset + if first == 0xC3: + return True, offset + if first == 0xCC: + return data[offset], offset + 1 + if first == 0xCD: + return int.from_bytes(data[offset:offset + 2], "big"), offset + 2 + if first == 0xCE: + return int.from_bytes(data[offset:offset + 4], "big"), offset + 4 + if first == 0xCA: + return struct.unpack(">f", data[offset:offset + 4])[0], offset + 4 + if first == 0xCB: + return struct.unpack(">d", data[offset:offset + 8])[0], offset + 8 + if first == 0xD9: + size = data[offset] + offset += 1 + end = offset + size + return data[offset:end].decode("utf-8"), end + + raise ValueError(f"Unsupported MessagePack type 0x{first:02x}") + + +def _decode_otlp_proto(data, signal_type): + messages = { + "logs": ExportLogsServiceRequest(), + "metrics": ExportMetricsServiceRequest(), + "traces": ExportTraceServiceRequest(), + } + message = messages[signal_type] + message.ParseFromString(data) + return json.loads(json_format.MessageToJson(message)) + + +def _resource_key(signal_type): + return { + "logs": "resource_logs", + "metrics": "resource_metrics", + "traces": "resource_spans", + }[signal_type] + + +def _resource_key_camel(signal_type): + return { + "logs": "resourceLogs", + "metrics": "resourceMetrics", + "traces": "resourceSpans", + }[signal_type] + + +def _load_signal_fixture(service, json_file): + return read_json_file(service._resolve_json_fixture(json_file)) + + +def _build_multi_resource_payload(service, signal_type, json_file): + payload = _load_signal_fixture(service, json_file) + key = _resource_key(signal_type) + resources = payload[key] + base = resources[0] + + clone = deepcopy(base) + if signal_type == "logs": + clone["resource"]["attributes"][0]["value"]["string_value"] = "example-service-bulk" + clone["scope_logs"][0]["log_records"][0]["body"]["string_value"] = "bulk log resource" + elif signal_type == "metrics": + clone["resource"]["attributes"][0]["value"]["string_value"] = "instance-bulk" + clone["scope_metrics"][0]["metrics"][0]["name"] = "requests_total_bulk" + else: + clone["resource"]["attributes"][0]["value"]["string_value"] = "checkout-bulk" + clone["scope_spans"][0]["spans"][0]["name"] = "bulk-trace-span" + + resources.append(clone) + return payload + + +def _decode_kafka_payload(message, format_name, signal_type): + if format_name == "otlp_json": + return json.loads(message["value"].decode("utf-8")) + return _decode_otlp_proto(message["value"], signal_type) + + +def _collect_resources(messages, format_name, signal_type): + resource_key = _resource_key_camel(signal_type) + resources = [] + + for message in messages: + payload = _decode_kafka_payload(message, format_name, signal_type) + resources.extend(payload[resource_key]) + + return resources + + +def test_out_kafka_sends_json_payload(): + service = Service("out_kafka_basic.yaml") + service.start() + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + assert message["topic"] == "test" + assert message["partition"] == 0 + assert message["key"] is None + + payload = json.loads(message["value"].decode("utf-8")) + assert payload["message"] == "hello from out_kafka" + assert payload["source"] == "dummy" + assert any(request["api_key"] == 3 for request in data_storage["requests"]) + assert any(request["api_key"] == 0 for request in data_storage["requests"]) + + +def test_out_kafka_raw_format_uses_selected_field(): + service = Service("out_kafka_raw.yaml") + service.start() + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + assert message["topic"] == "test" + assert message["value"] == b"raw value" + + +def test_out_kafka_message_key_field_sets_kafka_key(): + service = Service("out_kafka_message_key_field.yaml") + service.start() + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + assert message["topic"] == "test" + assert message["key"] == b"key-123" + + payload = json.loads(message["value"].decode("utf-8")) + assert payload["message"] == "hello with key" + assert payload["message_key"] == "key-123" + + +def test_out_kafka_dynamic_topic_routes_to_record_topic(): + service = Service("out_kafka_dynamic_topic.yaml") + service.start() + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + assert message["topic"] == "topic-dynamic" + + payload = json.loads(message["value"].decode("utf-8")) + assert payload["message"] == "hello dynamic topic" + assert payload["topic_name"] == "topic-dynamic" + + +def test_out_kafka_msgpack_format_sends_msgpack_payload(): + service = Service("out_kafka_msgpack.yaml") + service.start() + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + assert message["topic"] == "test" + assert message["key"] is None + + payload, offset = _decode_simple_msgpack(message["value"]) + assert offset == len(message["value"]) + assert payload["message"] == "hello msgpack" + assert payload["count"] == 7 + assert payload["source"] == "dummy" + + +def test_out_kafka_otlp_json_logs(): + service = Service("out_kafka_otlp_json.yaml") + service.start() + service.send_json_logs_payload("test_logs_001.in.json") + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + payload = json.loads(message["value"].decode("utf-8")) + record = payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0] + + assert message["topic"] == "otlp-topic" + assert message["key"] == b"static-otlp-key" + assert payload["resourceLogs"] + assert record["body"]["stringValue"] == "This is an example log message." + assert payload["resourceLogs"][0]["resource"]["attributes"][0]["key"] == "service.name" + + +def test_out_kafka_otlp_json_metrics(): + service = Service("out_kafka_otlp_json.yaml") + service.start() + service.send_json_metrics_payload("test_metrics_001.in.json") + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + payload = json.loads(message["value"].decode("utf-8")) + metric = payload["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0] + data_point = metric["sum"]["dataPoints"][0] + + assert message["topic"] == "otlp-topic" + assert message["key"] == b"static-otlp-key" + assert payload["resourceMetrics"] + assert metric["name"] == "requests_total" + assert data_point["attributes"][0]["key"] == "service.name" + assert data_point["attributes"][0]["value"]["stringValue"] == "checkout" + + +def test_out_kafka_otlp_json_traces(): + service = Service("out_kafka_otlp_json.yaml") + service.start() + service.send_json_traces_payload("test_traces_001.in.json") + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + payload = json.loads(message["value"].decode("utf-8")) + span = payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0] + + assert message["topic"] == "otlp-topic" + assert message["key"] == b"static-otlp-key" + assert payload["resourceSpans"] + assert span["name"] == "checkout-span" + + +def test_out_kafka_otlp_proto_logs(): + service = Service("out_kafka_otlp_proto.yaml") + service.start() + service.send_json_logs_payload("test_logs_001.in.json") + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + payload = _decode_otlp_proto(message["value"], "logs") + record = payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0] + + assert message["topic"] == "otlp-topic" + assert message["key"] == b"static-otlp-key" + assert payload["resourceLogs"] + assert record["body"]["stringValue"] == "This is an example log message." + assert payload["resourceLogs"][0]["resource"]["attributes"][0]["key"] == "service.name" + + +def test_out_kafka_otlp_proto_metrics(): + service = Service("out_kafka_otlp_proto.yaml") + service.start() + service.send_json_metrics_payload("test_metrics_001.in.json") + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + payload = _decode_otlp_proto(message["value"], "metrics") + metric = payload["resourceMetrics"][0]["scopeMetrics"][0]["metrics"][0] + data_point = metric["sum"]["dataPoints"][0] + + assert message["topic"] == "otlp-topic" + assert message["key"] == b"static-otlp-key" + assert payload["resourceMetrics"] + assert metric["name"] == "requests_total" + assert data_point["attributes"][0]["key"] == "service.name" + assert data_point["attributes"][0]["value"]["stringValue"] == "checkout" + + +def test_out_kafka_otlp_proto_traces(): + service = Service("out_kafka_otlp_proto.yaml") + service.start() + service.send_json_traces_payload("test_traces_001.in.json") + + messages = service.wait_for_messages(1) + service.stop() + + message = messages[0] + payload = _decode_otlp_proto(message["value"], "traces") + span = payload["resourceSpans"][0]["scopeSpans"][0]["spans"][0] + + assert message["topic"] == "otlp-topic" + assert message["key"] == b"static-otlp-key" + assert payload["resourceSpans"] + assert span["name"] == "checkout-span" + + +@pytest.mark.parametrize( + "format_name,config_file,signal_type,json_file", + [ + ("otlp_json", "out_kafka_otlp_json.yaml", "logs", "test_logs_001.in.json"), + ("otlp_json", "out_kafka_otlp_json.yaml", "metrics", "test_metrics_001.in.json"), + ("otlp_json", "out_kafka_otlp_json.yaml", "traces", "test_traces_001.in.json"), + ("otlp_proto", "out_kafka_otlp_proto.yaml", "logs", "test_logs_001.in.json"), + ("otlp_proto", "out_kafka_otlp_proto.yaml", "metrics", "test_metrics_001.in.json"), + ("otlp_proto", "out_kafka_otlp_proto.yaml", "traces", "test_traces_001.in.json"), + ], + ids=[ + "otlp_json_logs", + "otlp_json_metrics", + "otlp_json_traces", + "otlp_proto_logs", + "otlp_proto_metrics", + "otlp_proto_traces", + ], +) +def test_out_kafka_otlp_formats_preserve_multiple_resources( + format_name, + config_file, + signal_type, + json_file, +): + service = Service(config_file) + service.start() + payload_dict = _build_multi_resource_payload(service, signal_type, json_file) + service.send_payload_dict(payload_dict, signal_type) + + expected_message_count = 2 if signal_type == "metrics" else 1 + messages = service.wait_for_messages(expected_message_count) + service.stop() + + resources = _collect_resources(messages, format_name, signal_type) + + for message in messages: + assert message["topic"] == "otlp-topic" + assert message["key"] == b"static-otlp-key" + + assert len(resources) >= 2 + + if signal_type == "logs": + bodies = [ + record["body"]["stringValue"] + for resource in resources + for scope in resource["scopeLogs"] + for record in scope["logRecords"] + ] + resource_names = [ + attribute["value"]["stringValue"] + for resource in resources + for attribute in resource["resource"]["attributes"] + if attribute["key"] == "service.name" + ] + assert "This is an example log message." in bodies + assert "bulk log resource" in bodies + assert "example-service-bulk" in resource_names + elif signal_type == "metrics": + metric_names = [ + metric["name"] + for resource in resources + for scope in resource["scopeMetrics"] + for metric in scope["metrics"] + ] + assert "requests_total" in metric_names + assert "requests_total_bulk" in metric_names + if format_name == "otlp_json": + instance_ids = [ + attribute["value"]["stringValue"] + for resource in resources + for attribute in resource["resource"]["attributes"] + if attribute["key"] == "service.instance.id" + ] + assert "instance-bulk" in instance_ids + else: + span_names = [ + span["name"] + for resource in resources + for scope in resource["scopeSpans"] + for span in scope["spans"] + ] + service_names = [ + attribute["value"]["stringValue"] + for resource in resources + for attribute in resource["resource"]["attributes"] + if attribute["key"] == "service.name" + ] + assert "checkout-span" in span_names + assert "bulk-trace-span" in span_names + assert "checkout-bulk" in service_names diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_logs.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_logs.yaml new file mode 100644 index 00000000000..ed1634a2408 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_logs.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "hello via grpc" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + http2: on + grpc: on + grpc_logs_uri: /custom.logs.v1.Logs/Push + header: "x-grpc otlp-test" diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_logs_oauth2_basic.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_logs_oauth2_basic.yaml new file mode 100644 index 00000000000..964e744d6c3 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_logs_oauth2_basic.yaml @@ -0,0 +1,28 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "hello via grpc oauth2 basic" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + http2: on + grpc: on + grpc_logs_uri: /custom.logs.v1.Logs/Push + oauth2.enable: true + oauth2.token_url: http://127.0.0.1:${TEST_SUITE_OAUTH_PORT}/oauth/token + oauth2.client_id: client1 + oauth2.client_secret: secret1 + oauth2.scope: logs.write diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_logs_oauth2_private_key_jwt.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_logs_oauth2_private_key_jwt.yaml new file mode 100644 index 00000000000..cf2e50039cf --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_logs_oauth2_private_key_jwt.yaml @@ -0,0 +1,30 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "hello via grpc oauth2 private key jwt" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + http2: on + grpc: on + grpc_logs_uri: /custom.logs.v1.Logs/Push + oauth2.enable: true + oauth2.token_url: http://127.0.0.1:${TEST_SUITE_OAUTH_PORT}/oauth/token + oauth2.client_id: client1 + oauth2.auth_method: private_key_jwt + oauth2.jwt_key_file: ${PRIVATE_KEY_TEST} + oauth2.jwt_cert_file: ${CERTIFICATE_TEST} + oauth2.jwt_aud: http://127.0.0.1:${TEST_SUITE_OAUTH_PORT}/oauth/token diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_metrics.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_metrics.yaml new file mode 100644 index 00000000000..24173ca7b1e --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_grpc_metrics.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: fluentbit_metrics + scrape_interval: 1 + scrape_on_start: on + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + http2: on + grpc: on + grpc_metrics_uri: /custom.metrics.v1.Metrics/Push + header: "x-metrics grpc" diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_limited_logs.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_limited_logs.yaml new file mode 100644 index 00000000000..7a49d0fa637 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_limited_logs.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /limited/logs + logs_max_resources: 1 + logs_max_scopes: 1 diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_limited_scopes_logs.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_limited_scopes_logs.yaml new file mode 100644 index 00000000000..e5f79fa5958 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_limited_scopes_logs.yaml @@ -0,0 +1,21 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /limited/logs + logs_max_resources: 0 + logs_max_scopes: 1 diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs.yaml new file mode 100644 index 00000000000..a88ec8725db --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs.yaml @@ -0,0 +1,29 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "hello from out_opentelemetry", + "source": "dummy", + "traceid": "63560bd4d8de74fae7d1e4160f2ee099", + "spanid": "251484295a9df731" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /custom/logs + header: "x-suite otel-test" + http_user: otel + http_passwd: secret + logs_trace_id_message_key: $traceid + logs_span_id_message_key: $spanid diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_batch_size.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_batch_size.yaml new file mode 100644 index 00000000000..2a92ad64044 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_batch_size.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /batched/logs + batch_size: 1 diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_gzip.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_gzip.yaml new file mode 100644 index 00000000000..7aa917ca160 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_gzip.yaml @@ -0,0 +1,25 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "body only", + "source": "dummy", + "level": "info" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + compress: gzip + logs_body_key: $message + logs_body_key_attributes: on diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_message_keys.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_message_keys.yaml new file mode 100644 index 00000000000..4adf1a73fd1 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_message_keys.yaml @@ -0,0 +1,25 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "message-key body", + "severity_text_key": "ERROR", + "severity_number_key": 17 + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_body_key: $message + logs_severity_text_message_key: $severity_text_key + logs_severity_number_message_key: $severity_number_key diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_oauth2_basic.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_oauth2_basic.yaml new file mode 100644 index 00000000000..ee42c6824dd --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_oauth2_basic.yaml @@ -0,0 +1,27 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "hello from out_opentelemetry oauth2 basic", + "source": "dummy" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /custom/logs + oauth2.enable: true + oauth2.token_url: http://127.0.0.1:${TEST_SUITE_OAUTH_PORT}/oauth/token + oauth2.client_id: client1 + oauth2.client_secret: secret1 + oauth2.scope: logs.write diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_oauth2_private_key_jwt.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_oauth2_private_key_jwt.yaml new file mode 100644 index 00000000000..bef38299a82 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_oauth2_private_key_jwt.yaml @@ -0,0 +1,29 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "hello from out_opentelemetry oauth2 jwt", + "source": "dummy" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /custom/logs + oauth2.enable: true + oauth2.token_url: http://127.0.0.1:${TEST_SUITE_OAUTH_PORT}/oauth/token + oauth2.client_id: client1 + oauth2.auth_method: private_key_jwt + oauth2.jwt_key_file: ${PRIVATE_KEY_TEST} + oauth2.jwt_cert_file: ${CERTIFICATE_TEST} + oauth2.jwt_aud: http://127.0.0.1:${TEST_SUITE_OAUTH_PORT}/oauth/token diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_tls.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_tls.yaml new file mode 100644 index 00000000000..0fe7d7db47f --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_tls.yaml @@ -0,0 +1,25 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "hello over tls" + } + + outputs: + - name: opentelemetry + match: "*" + host: localhost + port: ${TEST_SUITE_HTTP_PORT} + tls: on + tls.verify: on + tls.verify_hostname: on + tls.vhost: localhost + tls.ca_file: ${CERTIFICATE_TEST} diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_zstd.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_zstd.yaml new file mode 100644 index 00000000000..d34cc165e6d --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_logs_zstd.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + samples: 1 + dummy: | + { + "message": "zstd body", + "source": "dummy" + } + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + compress: zstd + logs_body_key: $message + logs_body_key_attributes: on diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_metadata_keys.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_metadata_keys.yaml new file mode 100644 index 00000000000..29fadcc7d7b --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_metadata_keys.yaml @@ -0,0 +1,34 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + processors: + logs: + - name: content_modifier + context: metadata + action: rename + key: otlp + value: custom_otel + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + logs_uri: /metadata/logs + logs_observed_timestamp_metadata_key: "$custom_otel['observed_timestamp']" + logs_timestamp_metadata_key: "$custom_otel['timestamp']" + logs_severity_text_metadata_key: "$custom_otel['severity_text']" + logs_severity_number_metadata_key: "$custom_otel['severity_number']" + logs_trace_flags_metadata_key: "$custom_otel['trace_flags']" + logs_span_id_metadata_key: "$custom_otel['span_id']" + logs_trace_id_metadata_key: "$custom_otel['trace_id']" + logs_attributes_metadata_key: "$custom_otel['attributes']" diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_metrics.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_metrics.yaml new file mode 100644 index 00000000000..811ee1e91bb --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_metrics.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: fluentbit_metrics + scrape_interval: 1 + scrape_on_start: on + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + metrics_uri: /custom/metrics + add_label: "cluster ci" diff --git a/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_traces.yaml b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_traces.yaml new file mode 100644 index 00000000000..9891c70f5ca --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/config/out_otel_http_traces.yaml @@ -0,0 +1,19 @@ +service: + flush: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + http2: off + tls: off + + outputs: + - name: opentelemetry + match: "*" + host: 127.0.0.1 + port: ${TEST_SUITE_HTTP_PORT} + traces_uri: /custom/traces diff --git a/tests/integration/scenarios/out_opentelemetry/tests/data_files/test_logs_metadata_001.in.json b/tests/integration/scenarios/out_opentelemetry/tests/data_files/test_logs_metadata_001.in.json new file mode 100644 index 00000000000..e8cebdd1233 --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/tests/data_files/test_logs_metadata_001.in.json @@ -0,0 +1,52 @@ +{ + "resource_logs": [ + { + "resource": { + "attributes": [ + { + "key": "service.name", + "value": { + "string_value": "metadata-service" + } + } + ] + }, + "scope_logs": [ + { + "scope": { + "name": "metadata-scope", + "version": "9.9.9" + }, + "log_records": [ + { + "time_unix_nano": "1650917400000000000", + "observed_time_unix_nano": "1650917401000000000", + "severity_number": 13, + "severity_text": "WARN", + "trace_id": "Y1YL1NjedPrn0eQWDy7gmQ==", + "span_id": "JRSEKVqd9zE=", + "flags": 1, + "body": { + "string_value": "metadata driven log" + }, + "attributes": [ + { + "key": "example_key", + "value": { + "string_value": "example_value" + } + }, + { + "key": "custom_attr", + "value": { + "string_value": "custom_value" + } + } + ] + } + ] + } + ] + } + ] +} diff --git a/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py b/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py new file mode 100644 index 00000000000..3d86154c44b --- /dev/null +++ b/tests/integration/scenarios/out_opentelemetry/tests/test_out_opentelemetry_001.py @@ -0,0 +1,552 @@ +import base64 +import json +import logging +import os +import socket + +import requests +import pytest +from google.protobuf import json_format +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from server.http_server import ( + configure_oauth_token_response, + data_storage as http_data_storage, + http_server_run, +) +from server.otlp_server import ( + configure_otlp_grpc_methods, + data_storage, + otlp_server_run, + stop_otlp_server, +) +from utils.data_utils import read_json_file +from utils.test_service import FluentBitTestService + +logger = logging.getLogger(__name__) + + +def _repo_relative(*parts): + return os.path.abspath(os.path.join(os.path.dirname(__file__), *parts)) + + +def iter_log_records(output): + for resource_log in output.get("resourceLogs", []): + for scope_log in resource_log.get("scopeLogs", []): + for record in scope_log.get("logRecords", []): + attributes = { + item["key"]: next(iter(item["value"].values())) + for item in record.get("attributes", []) + } + yield record, attributes + + +def iter_metric_attributes(output): + for resource_metric in output.get("resourceMetrics", []): + for scope_metric in resource_metric.get("scopeMetrics", []): + for metric in scope_metric.get("metrics", []): + if "sum" in metric: + points = metric["sum"].get("dataPoints", []) + elif "gauge" in metric: + points = metric["gauge"].get("dataPoints", []) + else: + points = metric.get("histogram", {}).get("dataPoints", []) + for point in points: + yield { + item["key"]: next(iter(item["value"].values())) + for item in point.get("attributes", []) + } + + +class Service: + def __init__( + self, + config_file, + *, + receiver_mode="http", + use_tls=False, + grpc_methods=None, + use_oauth_server=False, + ): + self.config_file = _repo_relative("../config", config_file) + cert_dir = _repo_relative("../../in_splunk/certificate") + self.tls_crt_file = os.path.join(cert_dir, "certificate.pem") + self.tls_key_file = os.path.join(cert_dir, "private_key.pem") + self.receiver_mode = receiver_mode + self.use_tls = use_tls + self.grpc_methods = grpc_methods or {} + self.use_oauth_server = use_oauth_server + self.oauth_server_port = None + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["logs", "metrics", "traces", "requests"], + extra_env={ + "CERTIFICATE_TEST": self.tls_crt_file, + "PRIVATE_KEY_TEST": self.tls_key_file, + }, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _wait_for_tcp_port(self, port, timeout=10): + def _ready(): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(0.5) + return sock.connect_ex(("127.0.0.1", port)) == 0 + + self.service.wait_for_condition( + _ready, + timeout=timeout, + interval=0.25, + description=f"OTLP receiver port {port}", + ) + + def _start_receiver(self, service): + if self.use_oauth_server: + self.oauth_server_port = service.allocate_port_env("TEST_SUITE_OAUTH_PORT") + http_server_run(self.oauth_server_port) + self.service.wait_for_http_endpoint( + f"http://127.0.0.1:{self.oauth_server_port}/ping", + timeout=10, + interval=0.5, + ) + + configure_otlp_grpc_methods(**self.grpc_methods) + otlp_server_run( + service.test_suite_http_port, + use_tls=self.use_tls, + tls_crt_file=self.tls_crt_file, + tls_key_file=self.tls_key_file, + use_grpc=self.receiver_mode == "grpc", + ) + + if self.receiver_mode == "grpc": + self._wait_for_tcp_port(service.test_suite_http_port) + return + + scheme = "https" if self.use_tls else "http" + url = f"{scheme}://localhost:{service.test_suite_http_port}/ping" + + def _http_ready(): + try: + response = requests.get( + url, + timeout=1, + verify=self.tls_crt_file if self.use_tls else True, + ) + return response.status_code == 200 + except requests.RequestException: + return False + + self.service.wait_for_condition( + _http_ready, + timeout=10, + interval=0.5, + description=f"OTLP {self.receiver_mode} receiver readiness", + ) + + def _stop_receiver(self, service): + if self.oauth_server_port is not None: + try: + requests.post(f"http://127.0.0.1:{self.oauth_server_port}/shutdown", timeout=2) + except requests.RequestException: + pass + stop_otlp_server() + + def start(self): + self.service.start() + self.flb = self.service.flb + self.flb_listener_port = self.service.flb_listener_port + self.test_suite_http_port = self.service.test_suite_http_port + + def stop(self): + self.service.stop() + + def wait_for_requests(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: data_storage["requests"] if len(data_storage["requests"]) >= minimum_count else None, + timeout=timeout, + interval=0.5, + description=f"{minimum_count} OTLP requests", + ) + + def wait_for_oauth_requests(self, minimum_count, timeout=10): + return self.service.wait_for_condition( + lambda: http_data_storage["requests"] if len(http_data_storage["requests"]) >= minimum_count else None, + timeout=timeout, + interval=0.5, + description=f"{minimum_count} OAuth requests", + ) + + def wait_for_signal(self, signal_type, minimum_count=1, timeout=10): + return self.service.wait_for_condition( + lambda: data_storage[signal_type] if len(data_storage[signal_type]) >= minimum_count else None, + timeout=timeout, + interval=0.5, + description=f"{minimum_count} OTLP {signal_type} payloads", + ) + + def send_json_logs_payload(self, json_file): + payload = self._build_signal_payload(json_file, "logs") + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/logs", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def send_json_traces_payload(self, json_file): + payload = self._build_signal_payload(json_file, "traces") + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/traces", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def _resolve_json_fixture(self, json_file): + scenario_fixture = _repo_relative("../tests/data_files", json_file) + if os.path.exists(scenario_fixture): + return scenario_fixture + + return _repo_relative("../../in_opentelemetry/tests/data_files", json_file) + + def _build_signal_payload(self, json_file, signal_type): + messages = { + "logs": ExportLogsServiceRequest(), + "metrics": ExportMetricsServiceRequest(), + "traces": ExportTraceServiceRequest(), + } + return json_format.Parse( + json.dumps(read_json_file(self._resolve_json_fixture(json_file))), + messages[signal_type], + ) + + +def test_out_opentelemetry_http_logs_uri_headers_and_basic_auth(): + service = Service("out_otel_http_logs.yaml") + service.start() + requests_seen = service.wait_for_requests(1) + logs_seen = service.wait_for_signal("logs") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(logs_seen[0])) + record = output["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0] + + assert request_seen["transport"] == "http" + assert request_seen["path"] == "/custom/logs" + assert request_seen["headers"]["Authorization"] == "Basic " + base64.b64encode(b"otel:secret").decode() + assert request_seen["headers"]["X-Suite"] == "otel-test" + assert base64.b64decode(record["traceId"]) == bytes.fromhex("63560bd4d8de74fae7d1e4160f2ee099") + assert base64.b64decode(record["spanId"]) == bytes.fromhex("251484295a9df731") + + +@pytest.mark.parametrize( + "config_file,auth_mode", + [ + ("out_otel_http_logs_oauth2_basic.yaml", "basic"), + ("out_otel_http_logs_oauth2_private_key_jwt.yaml", "private_key_jwt"), + ], + ids=["oauth2_basic", "oauth2_private_key_jwt"], +) +def test_out_opentelemetry_http_logs_oauth2_auth_matrix(config_file, auth_mode): + service = Service(config_file, use_oauth_server=True) + service.start() + configure_oauth_token_response( + status_code=200, + body={"access_token": "oauth-access-token", "token_type": "Bearer", "expires_in": 300}, + ) + + token_requests = service.wait_for_oauth_requests(1) + otlp_requests = service.wait_for_requests(1) + service.stop() + + token_request = next(request for request in token_requests if request["path"] == "/oauth/token") + data_request = next(request for request in otlp_requests if request["path"] == "/custom/logs") + + assert token_request["method"] == "POST" + assert "grant_type=client_credentials" in token_request["raw_data"] + assert data_request["headers"].get("Authorization") == "Bearer oauth-access-token" + + if auth_mode == "basic": + assert "Basic " in token_request["headers"].get("Authorization", "") + assert "scope=logs.write" in token_request["raw_data"] + return + + assert "client_assertion_type=" in token_request["raw_data"] + assert "client_assertion=" in token_request["raw_data"] + assert "client_id=client1" in token_request["raw_data"] + + +@pytest.mark.parametrize( + "config_file,auth_mode", + [ + ("out_otel_grpc_logs_oauth2_basic.yaml", "basic"), + ("out_otel_grpc_logs_oauth2_private_key_jwt.yaml", "private_key_jwt"), + ], + ids=["grpc_oauth2_basic", "grpc_oauth2_private_key_jwt"], +) +def test_out_opentelemetry_grpc_logs_oauth2_auth_matrix(config_file, auth_mode): + service = Service( + config_file, + receiver_mode="grpc", + grpc_methods={"logs": "/custom.logs.v1.Logs/Push"}, + use_oauth_server=True, + ) + service.start() + configure_oauth_token_response( + status_code=200, + body={"access_token": "oauth-access-token", "token_type": "Bearer", "expires_in": 300}, + ) + + token_requests = service.wait_for_oauth_requests(1) + otlp_requests = service.wait_for_requests(1) + service.stop() + + token_request = next(request for request in token_requests if request["path"] == "/oauth/token") + data_request = next(request for request in otlp_requests if request["path"] == "/custom.logs.v1.Logs/Push") + + assert token_request["method"] == "POST" + assert "grant_type=client_credentials" in token_request["raw_data"] + assert data_request["transport"] == "grpc" + assert data_request["headers"].get("authorization") == "Bearer oauth-access-token" + + if auth_mode == "basic": + assert "Basic " in token_request["headers"].get("Authorization", "") + assert "scope=logs.write" in token_request["raw_data"] + return + + assert "client_assertion_type=" in token_request["raw_data"] + assert "client_assertion=" in token_request["raw_data"] + assert "client_id=client1" in token_request["raw_data"] + + +def test_out_opentelemetry_gzip_and_logs_body_key_attributes(): + service = Service("out_otel_http_logs_gzip.yaml") + service.start() + requests_seen = service.wait_for_requests(1) + logs_seen = service.wait_for_signal("logs") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(logs_seen[0])) + record, attributes = next(iter_log_records(output)) + + assert request_seen["headers"]["Content-Encoding"] == "gzip" + assert record["body"]["stringValue"] == "body only" + assert attributes["source"] == "dummy" + assert attributes["level"] == "info" + assert "message" not in attributes + + +def test_out_opentelemetry_zstd_and_logs_body_key_attributes(): + service = Service("out_otel_http_logs_zstd.yaml") + service.start() + requests_seen = service.wait_for_requests(1) + logs_seen = service.wait_for_signal("logs") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(logs_seen[0])) + record, attributes = next(iter_log_records(output)) + + assert request_seen["headers"]["Content-Encoding"] == "zstd" + assert record["body"]["stringValue"] == "zstd body" + assert attributes["source"] == "dummy" + assert "message" not in attributes + + +def test_out_opentelemetry_tls_verification_with_vhost(): + service = Service("out_otel_http_logs_tls.yaml", use_tls=True) + service.start() + requests_seen = service.wait_for_requests(1) + logs_seen = service.wait_for_signal("logs") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(logs_seen[0])) + record = output["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0] + + assert request_seen["transport"] == "http" + assert request_seen["path"] == "/v1/logs" + assert record["body"]["stringValue"] == "hello over tls" + + +def test_out_opentelemetry_grpc_custom_logs_uri(): + service = Service( + "out_otel_grpc_logs.yaml", + receiver_mode="grpc", + grpc_methods={"logs": "/custom.logs.v1.Logs/Push"}, + ) + service.start() + requests_seen = service.wait_for_requests(1) + logs_seen = service.wait_for_signal("logs") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(logs_seen[0])) + record = output["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0] + + assert request_seen["transport"] == "grpc" + assert request_seen["path"] == "/custom.logs.v1.Logs/Push" + assert request_seen["headers"]["x-grpc"] == "otlp-test" + assert record["body"]["stringValue"] == "hello via grpc" + + +def test_out_opentelemetry_metrics_uri_and_add_label(): + service = Service("out_otel_http_metrics.yaml") + service.start() + requests_seen = service.wait_for_requests(1) + metrics_seen = service.wait_for_signal("metrics") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(metrics_seen[0])) + point_attributes = list(iter_metric_attributes(output)) + + assert request_seen["path"] == "/custom/metrics" + assert any(attributes.get("cluster") == "ci" for attributes in point_attributes) + + +def test_out_opentelemetry_grpc_metrics_uri(): + service = Service( + "out_otel_grpc_metrics.yaml", + receiver_mode="grpc", + grpc_methods={"metrics": "/custom.metrics.v1.Metrics/Push"}, + ) + service.start() + requests_seen = service.wait_for_requests(1) + metrics_seen = service.wait_for_signal("metrics") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(metrics_seen[0])) + + assert request_seen["transport"] == "grpc" + assert request_seen["path"] == "/custom.metrics.v1.Metrics/Push" + assert request_seen["headers"]["x-metrics"] == "grpc" + assert output["resourceMetrics"] + + +def test_out_opentelemetry_traces_uri(): + service = Service("out_otel_http_traces.yaml") + service.start() + service.send_json_traces_payload("test_traces_001.in.json") + requests_seen = service.wait_for_requests(1) + traces_seen = service.wait_for_signal("traces") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(traces_seen[0])) + span = output["resourceSpans"][0]["scopeSpans"][0]["spans"][0] + + assert request_seen["path"] == "/custom/traces" + assert span["name"] == "checkout-span" + + +def test_out_opentelemetry_batch_size_splits_log_exports(): + service = Service("out_otel_http_logs_batch_size.yaml") + service.start() + service.send_json_logs_payload("test_logs_001.in.json") + logs_seen = service.wait_for_signal("logs", minimum_count=4, timeout=15) + requests_seen = service.wait_for_requests(4, timeout=15) + service.stop() + + assert len(requests_seen) == 4 + assert {request["path"] for request in requests_seen} == {"/batched/logs"} + + for export_request in logs_seen: + output = json.loads(json_format.MessageToJson(export_request)) + record_count = sum( + len(scope_log.get("logRecords", [])) + for resource_log in output["resourceLogs"] + for scope_log in resource_log.get("scopeLogs", []) + ) + assert record_count == 1 + + +def test_out_opentelemetry_log_severity_message_keys(): + service = Service("out_otel_http_logs_message_keys.yaml") + service.start() + requests_seen = service.wait_for_requests(1) + logs_seen = service.wait_for_signal("logs") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(logs_seen[0])) + record = output["resourceLogs"][0]["scopeLogs"][0]["logRecords"][0] + + assert request_seen["path"] == "/v1/logs" + assert record["body"]["stringValue"] == "message-key body" + assert record["severityText"] == "ERROR" + assert record["severityNumber"] == "SEVERITY_NUMBER_ERROR" + + +def test_out_opentelemetry_custom_metadata_key_accessors(): + service = Service("out_otel_http_metadata_keys.yaml") + service.start() + service.send_json_logs_payload("test_logs_metadata_001.in.json") + requests_seen = service.wait_for_requests(1) + logs_seen = service.wait_for_signal("logs") + service.stop() + + request_seen = requests_seen[0] + output = json.loads(json_format.MessageToJson(logs_seen[0])) + record, attributes = next(iter_log_records(output)) + + assert request_seen["path"] == "/metadata/logs" + assert record["severityText"] == "WARN" + assert record["severityNumber"] == "SEVERITY_NUMBER_WARN" + assert record["timeUnixNano"] == "1650917400000000000" + assert record["observedTimeUnixNano"] == "1650917401000000000" + assert base64.b64decode(record["traceId"]) == bytes.fromhex("63560bd4d8de74fae7d1e4160f2ee099") + assert base64.b64decode(record["spanId"]) == bytes.fromhex("251484295a9df731") + assert record["flags"] == 1 + assert attributes["example_key"] == "example_value" + assert attributes["custom_attr"] == "custom_value" + + +def _wait_for_log_message(service, message, timeout=15): + def _contains_message(): + if not os.path.exists(service.flb.log_file): + return False + with open(service.flb.log_file, encoding="utf-8") as log_file: + return message in log_file.read() + + service.service.wait_for_condition( + _contains_message, + timeout=timeout, + interval=0.5, + description=message, + ) + + +def test_out_opentelemetry_logs_max_resources_enforcement(): + service = Service("out_otel_http_limited_logs.yaml") + service.start() + service.send_json_logs_payload("test_logs_001.in.json") + _wait_for_log_message(service, "max resources limit reached") + service.stop() + + assert data_storage["logs"] == [] + + +def test_out_opentelemetry_logs_max_scopes_enforcement(): + service = Service("out_otel_http_limited_scopes_logs.yaml") + service.start() + service.send_json_logs_payload("test_logs_001.in.json") + logs_seen = service.wait_for_signal("logs", minimum_count=1, timeout=15) + requests_seen = service.wait_for_requests(1, timeout=15) + service.stop() + + assert len(requests_seen) == 1 + + output = json.loads(json_format.MessageToJson(logs_seen[0])) + assert len(output["resourceLogs"]) == 4 + assert all(len(resource_log["scopeLogs"]) == 1 for resource_log in output["resourceLogs"]) diff --git a/tests/integration/scenarios/out_prometheus_exporter/config/out_prometheus_exporter.yaml b/tests/integration/scenarios/out_prometheus_exporter/config/out_prometheus_exporter.yaml new file mode 100644 index 00000000000..2f5ebbd07b3 --- /dev/null +++ b/tests/integration/scenarios/out_prometheus_exporter/config/out_prometheus_exporter.yaml @@ -0,0 +1,18 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: fluentbit_metrics + scrape_interval: 1 + scrape_on_start: true + + outputs: + - name: prometheus_exporter + match: "*" + host: 127.0.0.1 + port: ${EXPORTER_PORT} diff --git a/tests/integration/scenarios/out_prometheus_exporter/tests/test_out_prometheus_exporter_001.py b/tests/integration/scenarios/out_prometheus_exporter/tests/test_out_prometheus_exporter_001.py new file mode 100644 index 00000000000..843471995c8 --- /dev/null +++ b/tests/integration/scenarios/out_prometheus_exporter/tests/test_out_prometheus_exporter_001.py @@ -0,0 +1,102 @@ +import os +import subprocess + +import pytest +from utils.http_matrix import curl_supports_http2, run_curl_request +from utils.test_service import FluentBitTestService + + +def _headers_map(headers_raw): + headers = {} + for line in headers_raw.splitlines(): + if ":" not in line or line.startswith("HTTP/"): + continue + key, value = line.split(":", 1) + headers[key.strip().lower()] = value.strip() + return headers + + +class Service: + def __init__(self): + self.config_file = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../config/out_prometheus_exporter.yaml") + ) + self.service = FluentBitTestService(self.config_file, pre_start=self._pre_start) + + def _pre_start(self, service): + self.exporter_port = service.allocate_port_env("EXPORTER_PORT") + + def start(self): + self.service.start() + self.base_url = f"http://127.0.0.1:{self.exporter_port}" + + def stop(self): + self.service.stop() + + def request(self, path, *, http_mode="http1.1", include_headers=False): + return run_curl_request( + f"{self.base_url}{path}", + method="GET", + http_mode=http_mode, + include_headers=include_headers, + ) + + +def test_out_prometheus_exporter_scrape_endpoint(): + service = Service() + service.start() + + try: + root = service.request("/") + assert root["status_code"] == 200 + assert "Prometheus Exporter" in root["body"] + + metrics = service.service.wait_for_condition( + lambda: ( + response + if response["status_code"] == 200 and "fluentbit_input_metrics_scrapes_total" in response["body"] + else None + ) if (response := service.request("/metrics", include_headers=True)) else None, + timeout=10, + interval=1, + description="prometheus exporter metrics", + ) + assert metrics["status_code"] == 200 + assert "fluentbit_input_metrics_scrapes_total" in metrics["body"] + + headers = _headers_map(metrics["headers_raw"]) + if headers.get("server") != "Fluent Bit" or headers.get("x-http-engine") != "Monkey heritage": + pytest.skip("Unified exporter headers are not available in this Fluent Bit binary") + assert headers["server"] == "Fluent Bit" + assert headers["x-http-engine"] == "Monkey heritage" + assert headers["content-type"].startswith("text/plain; version=0.0.4") + finally: + service.stop() + + +def test_out_prometheus_exporter_http2_metrics(): + if not curl_supports_http2(): + return + + service = Service() + service.start() + + try: + try: + metrics = service.service.wait_for_condition( + lambda: ( + response + if response["status_code"] == 200 and "fluentbit_input_metrics_scrapes_total" in response["body"] + else None + ) if (response := service.request("/metrics", http_mode="http2-prior-knowledge")) else None, + timeout=10, + interval=1, + description="prometheus exporter http2 metrics", + ) + except subprocess.CalledProcessError: + pytest.skip("Prometheus exporter does not support HTTP/2 prior knowledge in this Fluent Bit binary") + assert metrics["status_code"] == 200 + assert metrics["http_version"] == "2" + assert "fluentbit_input_metrics_scrapes_total" in metrics["body"] + finally: + service.stop() diff --git a/tests/integration/scenarios/out_s3/config/out_s3_basic.yaml b/tests/integration/scenarios/out_s3/config/out_s3_basic.yaml new file mode 100644 index 00000000000..d7841a5133d --- /dev/null +++ b/tests/integration/scenarios/out_s3/config/out_s3_basic.yaml @@ -0,0 +1,26 @@ +service: + flush: 1 + grace: 3 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_s3 + dummy: '{"message":"hello from out_s3","source":"dummy"}' + samples: 1 + + outputs: + - name: s3 + match: out_s3 + bucket: test-bucket + region: us-east-1 + endpoint: http://127.0.0.1:${TEST_SUITE_HTTP_PORT} + use_put_object: true + total_file_size: 1M + upload_timeout: 2s + s3_key_format: /payloads/$TAG/$UUID + content_type: application/x-ndjson + store_dir: /tmp/fluent-bit-test-suite-s3-basic diff --git a/tests/integration/scenarios/out_s3/config/out_s3_format_json.yaml b/tests/integration/scenarios/out_s3/config/out_s3_format_json.yaml new file mode 100644 index 00000000000..f7d3d422835 --- /dev/null +++ b/tests/integration/scenarios/out_s3/config/out_s3_format_json.yaml @@ -0,0 +1,27 @@ +service: + flush: 1 + grace: 3 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_s3 + dummy: '{"message":"hello from out_s3 format json","source":"dummy"}' + samples: 1 + + outputs: + - name: s3 + match: out_s3 + bucket: test-bucket + region: us-east-1 + endpoint: http://127.0.0.1:${TEST_SUITE_HTTP_PORT} + use_put_object: true + total_file_size: 1M + upload_timeout: 2s + s3_key_format: /payloads/$TAG/$UUID + format: json + content_type: application/x-ndjson + store_dir: /tmp/fluent-bit-test-suite-s3-format-json diff --git a/tests/integration/scenarios/out_s3/config/out_s3_gzip.yaml b/tests/integration/scenarios/out_s3/config/out_s3_gzip.yaml new file mode 100644 index 00000000000..a970acf3300 --- /dev/null +++ b/tests/integration/scenarios/out_s3/config/out_s3_gzip.yaml @@ -0,0 +1,27 @@ +service: + flush: 1 + grace: 3 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: out_s3 + dummy: '{"message":"hello gzip s3","source":"dummy"}' + samples: 1 + + outputs: + - name: s3 + match: out_s3 + bucket: test-bucket + region: us-east-1 + endpoint: http://127.0.0.1:${TEST_SUITE_HTTP_PORT} + use_put_object: true + total_file_size: 1M + upload_timeout: 2s + s3_key_format: /payloads/$TAG/$UUID + content_type: application/x-ndjson + compression: gzip + store_dir: /tmp/fluent-bit-test-suite-s3-gzip diff --git a/tests/integration/scenarios/out_s3/config/out_s3_otlp_json.yaml b/tests/integration/scenarios/out_s3/config/out_s3_otlp_json.yaml new file mode 100644 index 00000000000..bde96e58020 --- /dev/null +++ b/tests/integration/scenarios/out_s3/config/out_s3_otlp_json.yaml @@ -0,0 +1,26 @@ +service: + flush: 1 + grace: 3 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + listen: 127.0.0.1 + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: s3 + match: "*" + bucket: test-bucket + region: us-east-1 + endpoint: http://127.0.0.1:${TEST_SUITE_HTTP_PORT} + use_put_object: true + total_file_size: 1M + upload_timeout: 2s + s3_key_format: /payloads/$TAG/$UUID + format: otlp_json + content_type: application/json + store_dir: /tmp/fluent-bit-test-suite-s3-otlp-json diff --git a/tests/integration/scenarios/out_s3/tests/test_out_s3_001.py b/tests/integration/scenarios/out_s3/tests/test_out_s3_001.py new file mode 100644 index 00000000000..59a69a57604 --- /dev/null +++ b/tests/integration/scenarios/out_s3/tests/test_out_s3_001.py @@ -0,0 +1,215 @@ +import gzip +import json +import os + +import requests +import pytest +from google.protobuf import json_format +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from server.s3_server import data_storage, s3_server_run, s3_server_stop +from utils.data_utils import read_json_file +from utils.fluent_bit_manager import FluentBitStartupError +from utils.test_service import FluentBitTestService + + +class Service: + def __init__(self, config_file): + self.config_file = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../config", config_file) + ) + self.service = FluentBitTestService( + self.config_file, + data_storage=data_storage, + data_keys=["requests"], + extra_env={ + "AWS_ACCESS_KEY_ID": "test-access-key", + "AWS_SECRET_ACCESS_KEY": "test-secret-key", + "AWS_EC2_METADATA_DISABLED": "true", + }, + pre_start=self._start_receiver, + post_stop=self._stop_receiver, + ) + + def _start_receiver(self, service): + self.s3_port = service.allocate_port_env("TEST_SUITE_HTTP_PORT") + s3_server_run(self.s3_port) + + def _stop_receiver(self, service): + s3_server_stop() + + def start(self): + self.service.start() + self.flb_listener_port = self.service.flb_listener_port + + def stop(self): + self.service.stop() + + def wait_for_request(self, index=0): + return self.service.wait_for_condition( + lambda: data_storage["requests"][index] if len(data_storage["requests"]) > index else None, + timeout=15, + interval=0.5, + description=f"S3 upload request {index}", + ) + + def _resolve_json_fixture(self, json_file): + return os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../../in_opentelemetry/tests/data_files", + json_file, + ) + ) + + def _build_signal_payload(self, json_file, signal_type): + messages = { + "logs": ExportLogsServiceRequest(), + "metrics": ExportMetricsServiceRequest(), + "traces": ExportTraceServiceRequest(), + } + return json_format.Parse( + json.dumps(read_json_file(self._resolve_json_fixture(json_file))), + messages[signal_type], + ) + + def send_logs_payload(self, json_file): + payload = self._build_signal_payload(json_file, "logs") + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/logs", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def send_metrics_payload(self, json_file): + payload = self._build_signal_payload(json_file, "metrics") + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/metrics", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def send_traces_payload(self, json_file): + payload = self._build_signal_payload(json_file, "traces") + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/traces", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + +def _parse_json_lines(body): + lines = [line for line in body.decode("utf-8").splitlines() if line.strip()] + return [json.loads(line) for line in lines] + + +def _parse_single_json_payload(body): + return json.loads(body.decode("utf-8").strip()) + + +def _send_otlp_signal(service, signal_type, json_file): + if signal_type == "logs": + service.send_logs_payload(json_file) + elif signal_type == "metrics": + service.send_metrics_payload(json_file) + else: + service.send_traces_payload(json_file) + + +def _start_or_skip_unsupported_s3_format(service, format_name): + try: + service.start() + except FluentBitStartupError: + log_contents = "" + if service.service.flb and service.service.flb.log_file: + with open(service.service.flb.log_file, "r", encoding="utf-8", errors="replace") as file: + log_contents = file.read() + if f"unknown configuration property '{format_name}'" in log_contents: + pytest.skip(f"s3.{format_name} is not supported by this Fluent Bit binary") + raise + + +def test_out_s3_put_object_uploads_json_lines_payload(): + service = Service("out_s3_basic.yaml") + service.start() + request = service.wait_for_request() + service.stop() + + assert request["method"] == "PUT" + assert request["path"].startswith("/test-bucket/payloads/out_s3/") + assert request["headers"]["Content-Type"] == "application/x-ndjson" + + payload = _parse_json_lines(request["body"]) + assert len(payload) == 1 + assert payload[0]["message"] == "hello from out_s3" + assert payload[0]["source"] == "dummy" + assert "date" in payload[0] + + +def test_out_s3_format_json_uploads_logs_only_as_json_lines(): + service = Service("out_s3_format_json.yaml") + _start_or_skip_unsupported_s3_format(service, "format") + request = service.wait_for_request() + service.stop() + + assert request["method"] == "PUT" + assert request["path"].startswith("/test-bucket/payloads/out_s3/") + assert request["headers"]["Content-Type"] == "application/x-ndjson" + + payload = _parse_json_lines(request["body"]) + assert len(payload) == 1 + assert payload[0]["message"] == "hello from out_s3 format json" + assert payload[0]["source"] == "dummy" + assert "date" in payload[0] + + +def test_out_s3_put_object_gzip_upload_sets_encoding_and_compresses_payload(): + service = Service("out_s3_gzip.yaml") + service.start() + request = service.wait_for_request() + service.stop() + + assert request["method"] == "PUT" + assert request["path"].startswith("/test-bucket/payloads/out_s3/") + assert request["headers"]["Content-Type"] == "application/x-ndjson" + assert request["headers"]["Content-Encoding"] == "gzip" + + payload = _parse_json_lines(gzip.decompress(request["body"])) + assert len(payload) == 1 + assert payload[0]["message"] == "hello gzip s3" + assert payload[0]["source"] == "dummy" + assert "date" in payload[0] + + +@pytest.mark.parametrize( + ("signal_type", "json_file", "root_key", "expected_value"), + [ + ("logs", "test_logs_001.in.json", "resourceLogs", "This is an example log message."), + ("metrics", "test_metrics_001.in.json", "resourceMetrics", "requests_total"), + ("traces", "test_traces_001.in.json", "resourceSpans", "checkout-span"), + ], +) +def test_out_s3_otlp_json_uploads_signal_payloads(signal_type, json_file, root_key, expected_value): + service = Service("out_s3_otlp_json.yaml") + _start_or_skip_unsupported_s3_format(service, "format") + _send_otlp_signal(service, signal_type, json_file) + request = service.wait_for_request() + service.stop() + + assert request["method"] == "PUT" + assert request["path"].startswith("/test-bucket/payloads/") + assert request["headers"]["Content-Type"] == "application/json" + + payload = _parse_single_json_payload(request["body"]) + assert root_key in payload + + rendered = json.dumps(payload) + assert expected_value in rendered diff --git a/tests/integration/scenarios/out_stdout/config/out_stdout_basic.yaml b/tests/integration/scenarios/out_stdout/config/out_stdout_basic.yaml new file mode 100644 index 00000000000..d1c7cc40465 --- /dev/null +++ b/tests/integration/scenarios/out_stdout/config/out_stdout_basic.yaml @@ -0,0 +1,17 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: stdout.logs + dummy: '{"message":"hello from out_stdout","source":"dummy"}' + samples: 1 + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/out_stdout/config/out_stdout_json_lines.yaml b/tests/integration/scenarios/out_stdout/config/out_stdout_json_lines.yaml new file mode 100644 index 00000000000..7ea8bde105e --- /dev/null +++ b/tests/integration/scenarios/out_stdout/config/out_stdout_json_lines.yaml @@ -0,0 +1,20 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: stdout.json + dummy: '{"message":"hello json lines","source":"dummy"}' + samples: 1 + + outputs: + - name: stdout + match: '*' + format: json_lines + json_date_key: timestamp + json_date_format: iso8601 diff --git a/tests/integration/scenarios/out_stdout/config/out_stdout_metrics.yaml b/tests/integration/scenarios/out_stdout/config/out_stdout_metrics.yaml new file mode 100644 index 00000000000..d9c8fd2e75c --- /dev/null +++ b/tests/integration/scenarios/out_stdout/config/out_stdout_metrics.yaml @@ -0,0 +1,15 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/out_stdout/config/out_stdout_otel.yaml b/tests/integration/scenarios/out_stdout/config/out_stdout_otel.yaml new file mode 100644 index 00000000000..d9c8fd2e75c --- /dev/null +++ b/tests/integration/scenarios/out_stdout/config/out_stdout_otel.yaml @@ -0,0 +1,15 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/out_stdout/config/out_stdout_traces.yaml b/tests/integration/scenarios/out_stdout/config/out_stdout_traces.yaml new file mode 100644 index 00000000000..d9c8fd2e75c --- /dev/null +++ b/tests/integration/scenarios/out_stdout/config/out_stdout_traces.yaml @@ -0,0 +1,15 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: opentelemetry + port: ${FLUENT_BIT_TEST_LISTENER_PORT} + + outputs: + - name: stdout + match: '*' diff --git a/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py b/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py new file mode 100644 index 00000000000..a8d444b0e9f --- /dev/null +++ b/tests/integration/scenarios/out_stdout/tests/test_out_stdout_001.py @@ -0,0 +1,204 @@ +import json +import os + +import requests +from google.protobuf import json_format +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ExportMetricsServiceRequest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest + +from utils.data_utils import read_json_file, read_file +from utils.test_service import FluentBitTestService + + +class Service: + def __init__(self, config_file): + self.config_file = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../config", config_file) + ) + self.service = FluentBitTestService(self.config_file) + + def start(self): + self.service.start() + self.flb = self.service.flb + self.flb_listener_port = self.service.flb_listener_port + + def stop(self): + self.service.stop() + + def wait_for_log_contains(self, text, timeout=10): + return self.service.wait_for_condition( + lambda: read_file(self.flb.log_file) if text in read_file(self.flb.log_file) else None, + timeout=timeout, + interval=0.5, + description=f"log text {text!r}", + ) + + def read_log(self): + return read_file(self.flb.log_file) + + def _resolve_json_fixture(self, json_file): + return os.path.abspath( + os.path.join( + os.path.dirname(__file__), + "../../in_opentelemetry/tests/data_files", + json_file, + ) + ) + + def _build_metrics_payload(self, json_file): + return json_format.Parse( + json.dumps(read_json_file(self._resolve_json_fixture(json_file))), + ExportMetricsServiceRequest(), + ) + + def _build_traces_payload(self, json_file): + return json_format.Parse( + json.dumps(read_json_file(self._resolve_json_fixture(json_file))), + ExportTraceServiceRequest(), + ) + + def _build_logs_payload(self, json_file): + return json_format.Parse( + json.dumps(read_json_file(self._resolve_json_fixture(json_file))), + ExportLogsServiceRequest(), + ) + + def send_json_metrics_payload(self, json_file): + payload = self._build_metrics_payload(json_file) + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/metrics", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def send_json_traces_payload(self, json_file): + payload = self._build_traces_payload(json_file) + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/traces", + data=payload.SerializeToString(), + headers={"Content-Type": "application/x-protobuf"}, + timeout=5, + ) + response.raise_for_status() + + def send_logs_json_payload(self, json_file): + payload = self._build_logs_payload(json_file) + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/logs", + data=json_format.MessageToJson(payload), + headers={"Content-Type": "application/json"}, + timeout=5, + ) + response.raise_for_status() + + def send_metrics_json_payload(self, json_file): + payload = self._build_metrics_payload(json_file) + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/metrics", + data=json_format.MessageToJson(payload), + headers={"Content-Type": "application/json"}, + timeout=5, + ) + response.raise_for_status() + + def send_traces_json_payload(self, json_file): + payload = self._build_traces_payload(json_file) + response = requests.post( + f"http://127.0.0.1:{self.flb_listener_port}/v1/traces", + data=json_format.MessageToJson(payload), + headers={"Content-Type": "application/json"}, + timeout=5, + ) + response.raise_for_status() + + +def _find_json_line(log_text, needle): + for line in log_text.splitlines(): + if needle in line and line.lstrip().startswith("{"): + return json.loads(line) + raise AssertionError(f"Could not find JSON line containing {needle!r}") + + +def test_out_stdout_default_format_emits_tagged_log_line(): + service = Service("out_stdout_basic.yaml") + service.start() + log_text = service.wait_for_log_contains("hello from out_stdout") + service.stop() + + assert "[0] stdout.logs:" in log_text + assert "hello from out_stdout" in log_text + assert "\"source\"=>\"dummy\"" in log_text or '"source"' in log_text + + +def test_out_stdout_json_lines_honors_date_key_and_json_format(): + service = Service("out_stdout_json_lines.yaml") + service.start() + log_text = service.wait_for_log_contains("hello json lines") + service.stop() + + payload = _find_json_line(log_text, "hello json lines") + assert payload["message"] == "hello json lines" + assert payload["source"] == "dummy" + assert "timestamp" in payload + assert "T" in payload["timestamp"] + + +def test_out_stdout_metrics_emits_text_representation(): + service = Service("out_stdout_metrics.yaml") + service.start() + service.send_json_metrics_payload("test_metrics_001.in.json") + log_text = service.wait_for_log_contains("requests_total") + service.stop() + + assert "requests_total" in log_text + assert "service.name=\"checkout\"" in log_text + assert "= 42" in log_text + + +def test_out_stdout_traces_emits_text_representation(): + service = Service("out_stdout_traces.yaml") + service.start() + service.send_json_traces_payload("test_traces_001.in.json") + log_text = service.wait_for_log_contains("checkout-span") + service.stop() + + assert "checkout-span" in log_text + assert "trace-scope" in log_text + assert "service.name" in log_text or "checkout" in log_text + + +def test_out_stdout_logs_accepts_otlp_json_ingestion(): + service = Service("out_stdout_otel.yaml") + service.start() + service.send_logs_json_payload("test_logs_001.in.json") + log_text = service.wait_for_log_contains("This is an example log message.") + service.stop() + + assert "This is an example log message." in log_text + assert "This is another example log message." in log_text + + +def test_out_stdout_metrics_accepts_otlp_json_ingestion(): + service = Service("out_stdout_otel.yaml") + service.start() + service.send_metrics_json_payload("test_metrics_001.in.json") + log_text = service.wait_for_log_contains("requests_total") + service.stop() + + assert "requests_total" in log_text + assert "service.name=\"checkout\"" in log_text + assert "= 42" in log_text + + +def test_out_stdout_traces_accepts_otlp_json_ingestion(): + service = Service("out_stdout_otel.yaml") + service.start() + service.send_traces_json_payload("test_traces_001.in.json") + log_text = service.wait_for_log_contains("checkout-span") + service.stop() + + assert "checkout-span" in log_text + assert "trace-scope" in log_text diff --git a/tests/integration/scenarios/out_vivo_exporter/config/out_vivo_exporter.yaml b/tests/integration/scenarios/out_vivo_exporter/config/out_vivo_exporter.yaml new file mode 100644 index 00000000000..5abc2dfc385 --- /dev/null +++ b/tests/integration/scenarios/out_vivo_exporter/config/out_vivo_exporter.yaml @@ -0,0 +1,24 @@ +service: + flush: 1 + grace: 1 + log_level: info + http_server: on + http_port: ${FLUENT_BIT_HTTP_MONITORING_PORT} + +pipeline: + inputs: + - name: dummy + tag: dummy.logs + dummy: '{"message":"vivo"}' + samples: 1 + + - name: fluentbit_metrics + scrape_interval: 1 + scrape_on_start: true + + outputs: + - name: vivo_exporter + match: "*" + host: 127.0.0.1 + port: ${EXPORTER_PORT} + http_cors_allow_origin: "*" diff --git a/tests/integration/scenarios/out_vivo_exporter/tests/test_out_vivo_exporter_001.py b/tests/integration/scenarios/out_vivo_exporter/tests/test_out_vivo_exporter_001.py new file mode 100644 index 00000000000..34831913968 --- /dev/null +++ b/tests/integration/scenarios/out_vivo_exporter/tests/test_out_vivo_exporter_001.py @@ -0,0 +1,112 @@ +import os +import subprocess + +import pytest +from utils.http_matrix import curl_supports_http2, run_curl_request +from utils.test_service import FluentBitTestService + + +def _headers_map(headers_raw): + headers = {} + for line in headers_raw.splitlines(): + if ":" not in line or line.startswith("HTTP/"): + continue + key, value = line.split(":", 1) + headers[key.strip().lower()] = value.strip() + return headers + + +class Service: + def __init__(self): + self.config_file = os.path.abspath( + os.path.join(os.path.dirname(__file__), "../config/out_vivo_exporter.yaml") + ) + self.service = FluentBitTestService(self.config_file, pre_start=self._pre_start) + + def _pre_start(self, service): + self.exporter_port = service.allocate_port_env("EXPORTER_PORT") + + def start(self): + self.service.start() + self.base_url = f"http://127.0.0.1:{self.exporter_port}" + + def stop(self): + self.service.stop() + + def request(self, path, *, http_mode="http1.1", include_headers=False): + return run_curl_request( + f"{self.base_url}{path}", + method="GET", + http_mode=http_mode, + include_headers=include_headers, + ) + + +def test_out_vivo_exporter_endpoints(): + service = Service() + service.start() + + try: + root = service.request("/") + assert root["status_code"] == 200 + assert "Vivo Exporter" in root["body"] + + logs = service.service.wait_for_condition( + lambda: ( + response + if response["status_code"] == 200 and '"message":"vivo"' in response["body"] + else None + ) if (response := service.request("/api/v1/logs?from=0&limit=1", include_headers=True)) else None, + timeout=10, + interval=1, + description="vivo exporter logs", + ) + assert logs["status_code"] == 200 + assert '"message":"vivo"' in logs["body"] + + headers = _headers_map(logs["headers_raw"]) + if headers.get("server") != "Fluent Bit" or headers.get("x-http-engine") != "Monkey heritage": + pytest.skip("Unified exporter headers are not available in this Fluent Bit binary") + assert headers["server"] == "Fluent Bit" + assert headers["x-http-engine"] == "Monkey heritage" + assert headers["content-type"].startswith("application/json") + assert headers["access-control-allow-origin"] == "*" + assert "vivo-stream-next-id" in headers + + metrics = service.request("/api/v1/metrics") + assert metrics["status_code"] == 200 + assert '"name":"scrapes_total"' in metrics["body"] + + internal_metrics = service.request("/api/v1/internal/metrics") + assert internal_metrics["status_code"] == 200 + assert '"input"' in internal_metrics["body"] + finally: + service.stop() + + +def test_out_vivo_exporter_http2_logs(): + if not curl_supports_http2(): + return + + service = Service() + service.start() + + try: + try: + logs = service.service.wait_for_condition( + lambda: ( + response + if response["status_code"] == 200 and '"message":"vivo"' in response["body"] + else None + ) if (response := service.request("/api/v1/logs?from=0&limit=1", http_mode="http2-prior-knowledge")) else None, + timeout=10, + interval=1, + description="vivo exporter http2 logs", + ) + except subprocess.CalledProcessError: + pytest.skip("Vivo exporter does not support HTTP/2 prior knowledge in this Fluent Bit binary") + assert logs["status_code"] == 200 + assert logs["http_version"] == "2" + assert '"message":"vivo"' in logs["body"] + finally: + service.stop() diff --git a/tests/integration/setup-venv.sh b/tests/integration/setup-venv.sh new file mode 100755 index 00000000000..a326db08761 --- /dev/null +++ b/tests/integration/setup-venv.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +VENV_DIR="${SCRIPT_DIR}/.venv" +PYTHON_BIN="${PYTHON:-python3}" + +if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then + echo "error: ${PYTHON_BIN} was not found in PATH" >&2 + exit 1 +fi + +if [[ ! -d "${VENV_DIR}" ]]; then + "${PYTHON_BIN}" -m venv "${VENV_DIR}" +fi + +"${VENV_DIR}/bin/python3" -m pip install --upgrade pip +"${VENV_DIR}/bin/python3" -m pip install -r "${SCRIPT_DIR}/requirements.txt" + +cat < len(data): + raise IncompleteBuffer() + return end + + +def _unpack_array(data, offset, size): + result = [] + for _ in range(size): + value, offset = _unpack_obj(data, offset) + result.append(value) + return result, offset + + +def _unpack_map(data, offset, size): + result = {} + for _ in range(size): + key, offset = _unpack_obj(data, offset) + value, offset = _unpack_obj(data, offset) + result[key] = value + return result, offset + + +def _unpack_ext(data, offset, payload_size): + end = _require_bytes(data, offset, 1 + payload_size) + ext_type = int.from_bytes(data[offset:offset + 1], "big", signed=True) + payload_offset = offset + 1 + return {"__ext__": {"type": ext_type, "data": data[payload_offset:end]}}, end + + +def _unpack_obj(data, offset): + if offset >= len(data): + raise IncompleteBuffer() + + first = data[offset] + offset += 1 + + if first <= 0x7F: + return first, offset + if first >= 0xE0: + return first - 0x100, offset + if 0x80 <= first <= 0x8F: + return _unpack_map(data, offset, first & 0x0F) + if 0x90 <= first <= 0x9F: + return _unpack_array(data, offset, first & 0x0F) + if 0xA0 <= first <= 0xBF: + size = first & 0x1F + end = _require_bytes(data, offset, size) + raw = data[offset:end] + return _decode_str_like(raw), end + if first == 0xC0: + return None, offset + if first == 0xC2: + return False, offset + if first == 0xC3: + return True, offset + if first == 0xC4: + _require_bytes(data, offset, 1) + size = data[offset] + offset += 1 + end = _require_bytes(data, offset, size) + return data[offset:end], end + if first == 0xC5: + _require_bytes(data, offset, 2) + size = int.from_bytes(data[offset:offset + 2], "big") + offset += 2 + end = _require_bytes(data, offset, size) + return data[offset:end], end + if first == 0xC6: + _require_bytes(data, offset, 4) + size = int.from_bytes(data[offset:offset + 4], "big") + offset += 4 + end = _require_bytes(data, offset, size) + return data[offset:end], end + if first == 0xCA: + _require_bytes(data, offset, 4) + return struct.unpack(">f", data[offset:offset + 4])[0], offset + 4 + if first == 0xCB: + _require_bytes(data, offset, 8) + return struct.unpack(">d", data[offset:offset + 8])[0], offset + 8 + if first == 0xCC: + _require_bytes(data, offset, 1) + return data[offset], offset + 1 + if first == 0xCD: + _require_bytes(data, offset, 2) + return int.from_bytes(data[offset:offset + 2], "big"), offset + 2 + if first == 0xCE: + _require_bytes(data, offset, 4) + return int.from_bytes(data[offset:offset + 4], "big"), offset + 4 + if first == 0xCF: + _require_bytes(data, offset, 8) + return int.from_bytes(data[offset:offset + 8], "big"), offset + 8 + if first == 0xD0: + _require_bytes(data, offset, 1) + return int.from_bytes(data[offset:offset + 1], "big", signed=True), offset + 1 + if first == 0xD1: + _require_bytes(data, offset, 2) + return int.from_bytes(data[offset:offset + 2], "big", signed=True), offset + 2 + if first == 0xD2: + _require_bytes(data, offset, 4) + return int.from_bytes(data[offset:offset + 4], "big", signed=True), offset + 4 + if first == 0xD3: + _require_bytes(data, offset, 8) + return int.from_bytes(data[offset:offset + 8], "big", signed=True), offset + 8 + if first == 0xD4: + return _unpack_ext(data, offset, 1) + if first == 0xD5: + return _unpack_ext(data, offset, 2) + if first == 0xD6: + return _unpack_ext(data, offset, 4) + if first == 0xD7: + return _unpack_ext(data, offset, 8) + if first == 0xD8: + return _unpack_ext(data, offset, 16) + if first == 0xD9: + _require_bytes(data, offset, 1) + size = data[offset] + offset += 1 + end = _require_bytes(data, offset, size) + return _decode_str_like(data[offset:end]), end + if first == 0xDA: + _require_bytes(data, offset, 2) + size = int.from_bytes(data[offset:offset + 2], "big") + offset += 2 + end = _require_bytes(data, offset, size) + return _decode_str_like(data[offset:end]), end + if first == 0xDB: + _require_bytes(data, offset, 4) + size = int.from_bytes(data[offset:offset + 4], "big") + offset += 4 + end = _require_bytes(data, offset, size) + return _decode_str_like(data[offset:end]), end + if first == 0xDC: + _require_bytes(data, offset, 2) + size = int.from_bytes(data[offset:offset + 2], "big") + return _unpack_array(data, offset + 2, size) + if first == 0xDD: + _require_bytes(data, offset, 4) + size = int.from_bytes(data[offset:offset + 4], "big") + return _unpack_array(data, offset + 4, size) + if first == 0xDE: + _require_bytes(data, offset, 2) + size = int.from_bytes(data[offset:offset + 2], "big") + return _unpack_map(data, offset + 2, size) + if first == 0xDF: + _require_bytes(data, offset, 4) + size = int.from_bytes(data[offset:offset + 4], "big") + return _unpack_map(data, offset + 4, size) + + raise ValueError(f"Unsupported MessagePack type 0x{first:02x}") + + +def _pack_str(value): + data = value.encode() + length = len(data) + if length <= 31: + return bytes([0xA0 | length]) + data + if length <= 0xFF: + return b"\xD9" + bytes([length]) + data + return b"\xDA" + length.to_bytes(2, "big") + data + + +def _pack_map(mapping): + items = list(mapping.items()) + prefix = bytes([0x80 | len(items)]) + encoded = [] + for key, value in items: + encoded.append(_pack_obj(key)) + encoded.append(_pack_obj(value)) + return prefix + b"".join(encoded) + + +def _pack_obj(value): + if isinstance(value, str): + return _pack_str(value) + if isinstance(value, bytes): + length = len(value) + if length <= 0xFF: + return b"\xC4" + bytes([length]) + value + return b"\xC5" + length.to_bytes(2, "big") + value + if isinstance(value, dict): + return _pack_map(value) + raise TypeError(f"Unsupported pack type {type(value)!r}") + + +def _send_ack(sock, chunk): + sock.sendall(_pack_obj({"ack": chunk})) + + +def _decode_packed_entries(entry, options): + payload = entry + if options.get("compressed") == "gzip": + payload = gzip.decompress(payload) + elif options.get("compressed") == "zstd": + result = subprocess.run( + ["zstd", "-d", "-c"], + input=payload, + capture_output=True, + check=True, + ) + payload = result.stdout + + records = [] + offset = 0 + while offset < len(payload): + value, offset = _unpack_obj(payload, offset) + records.append(_normalize_forward_record(value)) + return records + + +def _normalize_forward_record(entry): + if not isinstance(entry, list): + return {"raw": entry} + + if ( + len(entry) == 2 and + isinstance(entry[0], list) and + len(entry[0]) == 2 + ): + return { + "timestamp": entry[0][0], + "metadata": entry[0][1], + "body": entry[1], + "raw": entry, + } + + if len(entry) == 2: + return {"timestamp": entry[0], "body": entry[1], "metadata": None, "raw": entry} + if len(entry) == 3: + return {"timestamp": entry[0], "metadata": entry[1], "body": entry[2], "raw": entry} + + return {"raw": entry} + + +def _classify_message(root): + tag = root[0] if isinstance(root, list) and len(root) > 0 else None + entry = root[1] if isinstance(root, list) and len(root) > 1 else None + options = root[2] if isinstance(root, list) and len(root) > 2 and isinstance(root[2], dict) else {} + + message = { + "raw": root, + "tag": tag, + "options": options, + "records": [], + "mode": "unknown", + } + + if isinstance(entry, list) and entry and isinstance(entry[0], list): + message["mode"] = "forward" + message["records"] = [_normalize_forward_record(item) for item in entry] + elif isinstance(entry, (bytes, bytearray)): + message["mode"] = "packed_forward" + message["records"] = _decode_packed_entries(bytes(entry), options) + elif len(root) >= 3: + message["mode"] = "message" + message["records"] = [_normalize_forward_record(root[1:4])] + + return message + + +def _handle_client(conn, address): + data_storage["connections"].append({"peer": address}) + buffer = b"" + conn.settimeout(0.5) + + while not server_stop_event.is_set(): + try: + chunk = conn.recv(4096) + except socket.timeout: + continue + except OSError: + break + + if not chunk: + break + + buffer += chunk + + while buffer: + try: + message, offset = _unpack_obj(buffer, 0) + except IncompleteBuffer: + break + + buffer = buffer[offset:] + + if isinstance(message, list): + decoded = _classify_message(message) + data_storage["messages"].append(decoded) + + chunk_id = decoded["options"].get("chunk") + if chunk_id is not None: + _send_ack(conn, chunk_id) + else: + data_storage["messages"].append({"raw": message, "mode": "unknown", "tag": None, "options": {}, "records": []}) + + +def run_forward_server(port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as server: + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", port)) + server.listen() + server.settimeout(0.5) + logger.info("Starting forward server on port %s", port) + + while not server_stop_event.is_set(): + try: + conn, address = server.accept() + except socket.timeout: + continue + except OSError: + break + + with conn: + _handle_client(conn, address) + + +def forward_server_run(port): + global server_thread, server_port + + reset_forward_server_state() + server_port = port + server_thread = threading.Thread(target=run_forward_server, args=(port,), daemon=True) + server_thread.start() + deadline = time.time() + 5 + while time.time() < deadline: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + if sock.connect_ex(("127.0.0.1", port)) == 0: + return server_thread + time.sleep(0.1) + + raise TimeoutError(f"Timed out waiting for forward server on port {port}") + + +def forward_server_stop(): + server_stop_event.set() + if server_port is not None: + try: + with socket.create_connection(("127.0.0.1", server_port), timeout=0.2): + pass + except Exception: + pass diff --git a/tests/integration/src/server/http_server.py b/tests/integration/src/server/http_server.py new file mode 100644 index 00000000000..de3d7972256 --- /dev/null +++ b/tests/integration/src/server/http_server.py @@ -0,0 +1,222 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2024 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import gzip +import threading +import time + +from flask import Flask, request, jsonify, Response +from werkzeug.serving import make_server + +app = Flask(__name__) +data_storage = {"payloads": [], "requests": []} +MOCK_JWKS_BODY = { + "keys": [ + { + "kty": "RSA", + "kid": "test", + "n": "xCUx72fXOyrjUZiiPJZIa7HtYHdQo_LAAkYG3yAcl1mwmh8pXrXB71xSDBI5SZDtKW4g6FEzYmP0jv3xwBdrZO2HQYwdxpCLhiMKEF0neC5w4NsjFlZKpnO53GN5W_c95bEhlVbh7O2q3PZVDhF5x9bdjlDS84NA0CY2l10UbSvIz12XR8uXqt6w9WVznrCe7ucSex3YPBTwll8Tm5H1rs1tPSx_9D0CJtZvxhKfgJtDyJJmV9syI6hlRgXnAsOonycOGSLryaIBtttxKUwy6QQkA-qSLZe2EcG2XoeBy10geOZ4WKGRiGubuuDpB1yFFy4mXQULJF6anO2osE31SQ", + "e": "AQAB", + } + ] +} +response_config = { + "status_code": 200, + "body": {"status": "received"}, + "content_type": "application/json", + "delay_seconds": 0, +} +oauth_token_response = { + "status_code": 200, + "body": { + "access_token": "oauth-access-token", + "token_type": "Bearer", + "expires_in": 300, + }, +} +logger = logging.getLogger(__name__) +server_thread = None +server_instances = [] + + +def reset_http_server_state(): + data_storage["payloads"] = [] + data_storage["requests"] = [] + server_instances.clear() + response_config.update( + { + "status_code": 200, + "body": {"status": "received"}, + "content_type": "application/json", + "delay_seconds": 0, + } + ) + oauth_token_response.update( + { + "status_code": 200, + "body": { + "access_token": "oauth-access-token", + "token_type": "Bearer", + "expires_in": 300, + }, + } + ) + + +def configure_http_response(*, status_code=None, body=None, content_type=None, delay_seconds=None): + if status_code is not None: + response_config["status_code"] = status_code + if body is not None: + response_config["body"] = body + if content_type is not None: + response_config["content_type"] = content_type + if delay_seconds is not None: + response_config["delay_seconds"] = delay_seconds + + +def configure_oauth_token_response(*, status_code=None, body=None): + if status_code is not None: + oauth_token_response["status_code"] = status_code + if body is not None: + oauth_token_response["body"] = body + + +def _build_response(): + if response_config["delay_seconds"]: + time.sleep(response_config["delay_seconds"]) + + body = response_config["body"] + if isinstance(body, (dict, list)): + return jsonify(body), response_config["status_code"] + + return Response( + body, + status=response_config["status_code"], + content_type=response_config["content_type"], + ) + + +def _record_request(): + raw_payload = request.get_data(cache=True) + decoded_payload = _decode_payload(raw_payload) + data = _decode_json_payload(decoded_payload) + raw_data = raw_payload.decode("utf-8", errors="replace") + decoded_data = decoded_payload.decode("utf-8", errors="replace") + + data_storage["payloads"].append(data) + data_storage["requests"].append( + { + "path": request.path, + "query_string": request.query_string.decode("utf-8", errors="replace"), + "method": request.method, + "headers": dict(request.headers), + "raw_data": raw_data, + "decoded_data": decoded_data, + "json": data, + } + ) + + +def _decode_payload(raw_payload): + if request.headers.get("Content-Encoding", "").lower() == "gzip": + return gzip.decompress(raw_payload) + + return raw_payload + + +def _decode_json_payload(decoded_payload): + if not decoded_payload: + return None + + try: + return json.loads(decoded_payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + + +@app.route('/data', methods=['POST']) +@app.route('/shared', methods=['POST']) +@app.route('/solo', methods=['POST']) +@app.route('/dataCollectionRules/', methods=['POST']) +def receive_data(subpath=None): + _record_request() + return _build_response() + + +@app.route('/services/collector', methods=['POST']) +@app.route('/services/collector/event', methods=['POST']) +@app.route('/services/collector/raw', methods=['POST']) +def receive_splunk_hec(): + _record_request() + return _build_response() + + +@app.route('/jwks', methods=['GET']) +def jwks(): + return jsonify(MOCK_JWKS_BODY), 200 + + +@app.route('/oauth/token', methods=['POST']) +def oauth_token(): + _record_request() + return jsonify(oauth_token_response["body"]), oauth_token_response["status_code"] + + +@app.route('/ping', methods=['GET']) +def ping(): + return jsonify({"status": "pong"}), 200 + + +@app.route('/shutdown', methods=['POST']) +def shutdown(): + logger.info("HTTP server shutdown requested") + for server_instance in list(server_instances): + threading.Thread(target=server_instance.shutdown, daemon=True).start() + return jsonify({"status": "shutting down"}), 200 + + +def run_server(port=60000, *, use_tls=False, tls_crt_file=None, tls_key_file=None): + ssl_context = None + if use_tls: + ssl_context = (tls_crt_file, tls_key_file) + + server_instance = make_server("0.0.0.0", port, app, ssl_context=ssl_context) + server_instances.append(server_instance) + server_instance.serve_forever() + + +def http_server_run(port=60000, *, use_tls=False, tls_crt_file=None, tls_key_file=None, + reset_state=True): + global server_thread + + if reset_state: + reset_http_server_state() + + logger.info("Starting HTTP server on port %s", port) + server_thread = threading.Thread( + target=run_server, + kwargs={ + "port": port, + "use_tls": use_tls, + "tls_crt_file": tls_crt_file, + "tls_key_file": tls_key_file, + }, + daemon=True, + ) + server_thread.start() + return server_thread diff --git a/tests/integration/src/server/kafka_server.py b/tests/integration/src/server/kafka_server.py new file mode 100644 index 00000000000..896042d5e31 --- /dev/null +++ b/tests/integration/src/server/kafka_server.py @@ -0,0 +1,416 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2026 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import socket +import struct +import threading + + +logger = logging.getLogger(__name__) + +API_KEY_PRODUCE = 0 +API_KEY_METADATA = 3 +API_KEY_API_VERSIONS = 18 + +BROKER_NODE_ID = 1 + +data_storage = { + "connections": [], + "requests": [], + "messages": [], +} + +server_thread = None +server_socket = None +server_port = None +server_stop_event = threading.Event() +server_ready_event = threading.Event() + + +def reset_kafka_server_state(): + data_storage["connections"] = [] + data_storage["requests"] = [] + data_storage["messages"] = [] + server_stop_event.clear() + server_ready_event.clear() + + +def _recv_exact(sock, size): + chunks = [] + remaining = size + + while remaining > 0: + chunk = sock.recv(remaining) + if not chunk: + return None + chunks.append(chunk) + remaining -= len(chunk) + + return b"".join(chunks) + + +def _read_int16(data, offset): + return struct.unpack_from(">h", data, offset)[0], offset + 2 + + +def _read_int32(data, offset): + return struct.unpack_from(">i", data, offset)[0], offset + 4 + + +def _read_int64(data, offset): + return struct.unpack_from(">q", data, offset)[0], offset + 8 + + +def _read_string(data, offset): + length, offset = _read_int16(data, offset) + if length < 0: + return None, offset + end = offset + length + return data[offset:end].decode("utf-8", errors="replace"), end + + +def _read_bytes(data, offset): + length, offset = _read_int32(data, offset) + if length < 0: + return None, offset + end = offset + length + return data[offset:end], end + + +def _read_array_length(data, offset): + return _read_int32(data, offset) + + +def _parse_request_header(payload): + api_key, offset = _read_int16(payload, 0) + api_version, offset = _read_int16(payload, offset) + correlation_id, offset = _read_int32(payload, offset) + client_id, offset = _read_string(payload, offset) + + return { + "api_key": api_key, + "api_version": api_version, + "correlation_id": correlation_id, + "client_id": client_id, + "body": payload[offset:], + } + + +def _encode_string(value): + encoded = value.encode("utf-8") + return struct.pack(">h", len(encoded)) + encoded + + +def _encode_response(correlation_id, body): + frame = struct.pack(">i", correlation_id) + body + return struct.pack(">i", len(frame)) + frame + + +def _encode_metadata_response(topics, host, port): + body = [] + body.append(struct.pack(">i", 1)) + body.append(struct.pack(">i", BROKER_NODE_ID)) + body.append(_encode_string(host)) + body.append(struct.pack(">i", port)) + + body.append(struct.pack(">i", len(topics))) + for topic in topics: + body.append(struct.pack(">h", 0)) + body.append(_encode_string(topic)) + body.append(struct.pack(">i", 1)) + body.append(struct.pack(">h", 0)) + body.append(struct.pack(">i", 0)) + body.append(struct.pack(">i", BROKER_NODE_ID)) + body.append(struct.pack(">i", 1)) + body.append(struct.pack(">i", BROKER_NODE_ID)) + body.append(struct.pack(">i", 1)) + body.append(struct.pack(">i", BROKER_NODE_ID)) + + return b"".join(body) + + +def _encode_produce_response(topic, partition=0, base_offset=0): + return b"".join( + [ + struct.pack(">i", 1), + _encode_string(topic), + struct.pack(">i", 1), + struct.pack(">i", partition), + struct.pack(">h", 0), + struct.pack(">q", base_offset), + ] + ) + + +def _encode_api_versions_response(api_version): + api_versions = [ + (API_KEY_PRODUCE, 0, 0), + (API_KEY_METADATA, 0, 0), + (API_KEY_API_VERSIONS, 0, 3), + ] + + if api_version >= 3: + entries = [] + for api_key, min_version, max_version in api_versions: + entries.append( + struct.pack(">h", api_key) + + struct.pack(">h", min_version) + + struct.pack(">h", max_version) + + b"\x00" + ) + + return b"".join( + [ + struct.pack(">h", 0), + bytes([len(api_versions) + 1]), + b"".join(entries), + struct.pack(">i", 0), + b"\x00", + ] + ) + + body = [struct.pack(">h", 0), struct.pack(">i", len(api_versions))] + for api_key, min_version, max_version in api_versions: + body.append(struct.pack(">h", api_key)) + body.append(struct.pack(">h", min_version)) + body.append(struct.pack(">h", max_version)) + if api_version >= 1: + body.append(struct.pack(">i", 0)) + return b"".join(body) + + +def _parse_metadata_request(body): + topic_count, offset = _read_array_length(body, 0) + topics = [] + + for _ in range(topic_count): + topic, offset = _read_string(body, offset) + topics.append(topic) + + return topics + + +def _parse_message_set(data): + messages = [] + offset = 0 + + while offset < len(data): + if len(data) - offset < 12: + break + + message_offset, offset = _read_int64(data, offset) + message_size, offset = _read_int32(data, offset) + end = offset + message_size + + if end > len(data): + break + + _, cursor = _read_int32(data, offset) + magic = data[cursor] + cursor += 1 + attributes = data[cursor] + cursor += 1 + key, cursor = _read_bytes(data, cursor) + value, cursor = _read_bytes(data, cursor) + + if magic == 1 and cursor + 8 <= end: + cursor += 8 + + if attributes & 0x07: + raise ValueError("Compressed Kafka message sets are not supported by the fake broker") + + messages.append( + { + "offset": message_offset, + "magic": magic, + "attributes": attributes, + "key": key, + "value": value, + } + ) + offset = end + + return messages + + +def _parse_produce_request(body): + required_acks, offset = _read_int16(body, 0) + timeout_ms, offset = _read_int32(body, offset) + topic_count, offset = _read_array_length(body, offset) + produced_topics = [] + + for _ in range(topic_count): + topic, offset = _read_string(body, offset) + partition_count, offset = _read_array_length(body, offset) + partitions = [] + + for _ in range(partition_count): + partition, offset = _read_int32(body, offset) + message_set, offset = _read_bytes(body, offset) + records = _parse_message_set(message_set or b"") + partitions.append( + { + "partition": partition, + "records": records, + } + ) + + produced_topics.append( + { + "topic": topic, + "partitions": partitions, + } + ) + + return { + "required_acks": required_acks, + "timeout_ms": timeout_ms, + "topics": produced_topics, + } + + +def _handle_request(sock, request, host, port): + data_storage["requests"].append( + { + "api_key": request["api_key"], + "api_version": request["api_version"], + "correlation_id": request["correlation_id"], + "client_id": request["client_id"], + } + ) + + if request["api_key"] == API_KEY_API_VERSIONS: + sock.sendall( + _encode_response( + request["correlation_id"], + _encode_api_versions_response(request["api_version"]), + ) + ) + return + + if request["api_key"] == API_KEY_METADATA: + topics = _parse_metadata_request(request["body"]) + sock.sendall( + _encode_response( + request["correlation_id"], + _encode_metadata_response(topics, host, port), + ) + ) + return + + if request["api_key"] == API_KEY_PRODUCE: + produced = _parse_produce_request(request["body"]) + for topic_data in produced["topics"]: + for partition_data in topic_data["partitions"]: + for record in partition_data["records"]: + data_storage["messages"].append( + { + "topic": topic_data["topic"], + "partition": partition_data["partition"], + "key": record["key"], + "value": record["value"], + "magic": record["magic"], + "attributes": record["attributes"], + "client_id": request["client_id"], + } + ) + + first_topic = produced["topics"][0]["topic"] if produced["topics"] else "test" + sock.sendall( + _encode_response( + request["correlation_id"], + _encode_produce_response(first_topic), + ) + ) + return + + logger.warning("Unsupported Kafka API key %s", request["api_key"]) + + +def _connection_loop(client, address, host, port): + data_storage["connections"].append({"address": address}) + + with client: + while not server_stop_event.is_set(): + header = _recv_exact(client, 4) + if not header: + return + + frame_size = struct.unpack(">i", header)[0] + payload = _recv_exact(client, frame_size) + if payload is None: + return + + request = _parse_request_header(payload) + _handle_request(client, request, host, port) + + +def _server_loop(host, port): + global server_socket, server_port + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((host, port)) + sock.listen(16) + sock.settimeout(0.2) + + server_socket = sock + server_port = port + server_ready_event.set() + logger.info("Starting fake Kafka broker on %s:%s", host, port) + + try: + while not server_stop_event.is_set(): + try: + client, address = sock.accept() + except socket.timeout: + continue + except OSError: + break + + worker = threading.Thread( + target=_connection_loop, + args=(client, address, host, port), + daemon=True, + ) + worker.start() + finally: + try: + sock.close() + except OSError: + pass + server_socket = None + + +def kafka_server_run(port, host="127.0.0.1"): + global server_thread + + reset_kafka_server_state() + server_thread = threading.Thread(target=_server_loop, args=(host, port), daemon=True) + server_thread.start() + server_ready_event.wait(timeout=5) + return server_thread + + +def kafka_server_stop(): + server_stop_event.set() + + if server_socket is not None: + try: + server_socket.close() + except OSError: + pass diff --git a/tests/integration/src/server/otlp_server.py b/tests/integration/src/server/otlp_server.py new file mode 100644 index 00000000000..7bcdf1ef7b0 --- /dev/null +++ b/tests/integration/src/server/otlp_server.py @@ -0,0 +1,311 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2026 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gzip +import logging +import subprocess +import threading +import time +from concurrent import futures + +import grpc +from flask import Flask, Response, jsonify, request +from google.protobuf.message import DecodeError +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ( + ExportLogsServiceRequest, + ExportLogsServiceResponse, +) +from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import ( + ExportMetricsServiceRequest, + ExportMetricsServiceResponse, +) +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ExportTraceServiceResponse, +) +from werkzeug.serving import make_server + +app = Flask(__name__) +data_storage = {"traces": [], "metrics": [], "logs": [], "requests": []} +response_config = { + "status_code": 200, + "body": {"status": "received"}, + "content_type": "application/json", + "delay_seconds": 0, +} +grpc_method_paths = { + "logs": "/opentelemetry.proto.collector.logs.v1.LogsService/Export", + "metrics": "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export", + "traces": "/opentelemetry.proto.collector.trace.v1.TraceService/Export", +} +logger = logging.getLogger(__name__) + +server_thread = None +http_server_instance = None +grpc_server_instance = None +shutdown_flag = threading.Event() + + +def reset_otlp_server_state(): + for key in data_storage: + data_storage[key] = [] + response_config.update( + { + "status_code": 200, + "body": {"status": "received"}, + "content_type": "application/json", + "delay_seconds": 0, + } + ) + grpc_method_paths.update( + { + "logs": "/opentelemetry.proto.collector.logs.v1.LogsService/Export", + "metrics": "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export", + "traces": "/opentelemetry.proto.collector.trace.v1.TraceService/Export", + } + ) + shutdown_flag.clear() + + +def configure_otlp_response(*, status_code=None, body=None, content_type=None, delay_seconds=None): + if status_code is not None: + response_config["status_code"] = status_code + if body is not None: + response_config["body"] = body + if content_type is not None: + response_config["content_type"] = content_type + if delay_seconds is not None: + response_config["delay_seconds"] = delay_seconds + + +def configure_otlp_grpc_methods(*, logs=None, metrics=None, traces=None): + if logs is not None: + grpc_method_paths["logs"] = logs + if metrics is not None: + grpc_method_paths["metrics"] = metrics + if traces is not None: + grpc_method_paths["traces"] = traces + + +def _build_response(): + if response_config["delay_seconds"]: + time.sleep(response_config["delay_seconds"]) + + body = response_config["body"] + if isinstance(body, (dict, list)): + return jsonify(body), response_config["status_code"] + + return Response( + body, + status=response_config["status_code"], + content_type=response_config["content_type"], + ) + + +def _record_request(*, path, headers, raw_payload, transport): + data_storage["requests"].append( + { + "path": path, + "headers": dict(headers), + "raw_size": len(raw_payload), + "raw_payload": raw_payload, + "transport": transport, + } + ) + + +def _decode_payload(raw_payload, headers): + if headers.get("Content-Encoding", "").lower() == "gzip": + return gzip.decompress(raw_payload) + if headers.get("Content-Encoding", "").lower() == "zstd": + result = subprocess.run( + ["zstd", "-d", "-c"], + input=raw_payload, + capture_output=True, + check=True, + ) + return result.stdout + return raw_payload + + +def _parse_http_signal(signal_name, message_type): + try: + decoded_payload = _decode_payload(request.data, request.headers) + otlp_request = message_type() + otlp_request.ParseFromString(decoded_payload) + data_storage[signal_name].append(otlp_request) + _record_request( + path=request.path, + headers=request.headers, + raw_payload=request.data, + transport="http", + ) + return _build_response() + except DecodeError: + return jsonify({"status": "invalid protobuf"}), 400 + + +def _guess_http_signal(path): + lowered_path = path.lower() + + if "metrics" in lowered_path: + return "metrics", ExportMetricsServiceRequest + if "traces" in lowered_path: + return "traces", ExportTraceServiceRequest + + return "logs", ExportLogsServiceRequest + + +@app.route("/shutdown", methods=["POST"]) +def shutdown(): + shutdown_flag.set() + logger.info("OTLP Server is shutting down...") + + if http_server_instance is not None: + threading.Thread(target=http_server_instance.shutdown, daemon=True).start() + + return jsonify({"status": "shutting down"}), 200 + + +@app.route("/v1/traces", methods=["POST"]) +def traces(): + return _parse_http_signal("traces", ExportTraceServiceRequest) + + +@app.route("/v1/metrics", methods=["POST"]) +def metrics(): + return _parse_http_signal("metrics", ExportMetricsServiceRequest) + + +@app.route("/v1/logs", methods=["POST"]) +def logs(): + return _parse_http_signal("logs", ExportLogsServiceRequest) + + +@app.route("/", defaults={"dynamic_path": ""}, methods=["POST"]) +@app.route("/", methods=["POST"]) +def dynamic_signal(dynamic_path): + signal_name, message_type = _guess_http_signal(f"/{dynamic_path}") + return _parse_http_signal(signal_name, message_type) + + +@app.route("/ping", methods=["GET"]) +def ping(): + return jsonify({"status": "pong"}), 200 + + +def run_server(port=4317, *, use_tls=False, tls_crt_file=None, tls_key_file=None): + global http_server_instance + + ssl_context = None + if use_tls: + ssl_context = (tls_crt_file, tls_key_file) + + http_server_instance = make_server("0.0.0.0", port, app, ssl_context=ssl_context) + http_server_instance.serve_forever() + + +def _build_grpc_handler(signal_name, message_type, response_type): + def _handler(request_message, context): + data_storage[signal_name].append(request_message) + _record_request( + path=context._rpc_event.call_details.method.decode(), + headers=context.invocation_metadata(), + raw_payload=request_message.SerializeToString(), + transport="grpc", + ) + return response_type() + + return grpc.unary_unary_rpc_method_handler( + _handler, + request_deserializer=message_type.FromString, + response_serializer=response_type.SerializeToString, + ) + + +class DynamicOtlpGrpcHandler(grpc.GenericRpcHandler): + def service(self, handler_call_details): + method = handler_call_details.method + + if method == grpc_method_paths["logs"]: + return _build_grpc_handler("logs", ExportLogsServiceRequest, ExportLogsServiceResponse) + if method == grpc_method_paths["metrics"]: + return _build_grpc_handler("metrics", ExportMetricsServiceRequest, ExportMetricsServiceResponse) + if method == grpc_method_paths["traces"]: + return _build_grpc_handler("traces", ExportTraceServiceRequest, ExportTraceServiceResponse) + if "logs" in method.lower(): + return _build_grpc_handler("logs", ExportLogsServiceRequest, ExportLogsServiceResponse) + if "metrics" in method.lower(): + return _build_grpc_handler("metrics", ExportMetricsServiceRequest, ExportMetricsServiceResponse) + if "traces" in method.lower(): + return _build_grpc_handler("traces", ExportTraceServiceRequest, ExportTraceServiceResponse) + + return None + + +def run_grpc_server(port=4317, *, use_tls=False, tls_crt_file=None, tls_key_file=None): + global grpc_server_instance + + grpc_server_instance = grpc.server(futures.ThreadPoolExecutor(max_workers=4)) + grpc_server_instance.add_generic_rpc_handlers((DynamicOtlpGrpcHandler(),)) + + bind_address = f"0.0.0.0:{port}" + if use_tls: + with open(tls_key_file, "rb") as key_file: + private_key = key_file.read() + with open(tls_crt_file, "rb") as cert_file: + certificate_chain = cert_file.read() + credentials = grpc.ssl_server_credentials(((private_key, certificate_chain),)) + grpc_server_instance.add_secure_port(bind_address, credentials) + else: + grpc_server_instance.add_insecure_port(bind_address) + + grpc_server_instance.start() + grpc_server_instance.wait_for_termination() + + +def stop_otlp_server(): + global grpc_server_instance + global http_server_instance + + shutdown_flag.set() + + if http_server_instance is not None: + http_server_instance.shutdown() + http_server_instance = None + + if grpc_server_instance is not None: + grpc_server_instance.stop(grace=0) + grpc_server_instance = None + + +def otlp_server_run(port, *, use_tls=False, tls_crt_file=None, tls_key_file=None, use_grpc=False): + global server_thread + + reset_otlp_server_state() + logger.info("Starting OTLP server on port %s", port) + server_thread = threading.Thread( + target=run_grpc_server if use_grpc else run_server, + kwargs={ + "port": port, + "use_tls": use_tls, + "tls_crt_file": tls_crt_file, + "tls_key_file": tls_key_file, + }, + daemon=True, + ) + server_thread.start() + return server_thread diff --git a/tests/integration/src/server/s3_server.py b/tests/integration/src/server/s3_server.py new file mode 100644 index 00000000000..7fdf0273c8a --- /dev/null +++ b/tests/integration/src/server/s3_server.py @@ -0,0 +1,104 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2026 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + + +logger = logging.getLogger(__name__) + +data_storage = { + "requests": [], +} + +server_thread = None +server_instance = None + + +def reset_s3_server_state(): + data_storage["requests"] = [] + + +class _S3RequestHandler(BaseHTTPRequestHandler): + server_version = "FakeS3/1.0" + protocol_version = "HTTP/1.1" + + def _record_request(self): + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) if length > 0 else b"" + data_storage["requests"].append( + { + "method": self.command, + "path": self.path, + "headers": dict(self.headers), + "body": body, + } + ) + + def do_PUT(self): + self._record_request() + self.send_response(200) + self.send_header("ETag", '"fake-s3-etag"') + self.send_header("Content-Length", "0") + self.end_headers() + + def do_POST(self): + self._record_request() + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def do_GET(self): + if self.path == "/ping": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", "18") + self.end_headers() + self.wfile.write(b'{"status":"pong"}') + return + + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, format, *args): + logger.debug("Fake S3 server: %s", format % args) + + +def s3_server_run(port): + global server_thread + global server_instance + + reset_s3_server_state() + server_instance = ThreadingHTTPServer(("0.0.0.0", port), _S3RequestHandler) + server_thread = threading.Thread(target=server_instance.serve_forever, daemon=True) + server_thread.start() + return server_thread + + +def s3_server_stop(): + global server_instance + global server_thread + + if server_instance is not None: + server_instance.shutdown() + server_instance.server_close() + server_instance = None + + if server_thread is not None: + server_thread.join(timeout=5) + server_thread = None diff --git a/tests/integration/src/server/splunk_server.py b/tests/integration/src/server/splunk_server.py new file mode 100644 index 00000000000..6d29c8f4e35 --- /dev/null +++ b/tests/integration/src/server/splunk_server.py @@ -0,0 +1,13 @@ +from flask import Flask, request, jsonify + +app = Flask(__name__) +data_storage = {"events": []} + +@app.route('/services/collector/event', methods=['POST']) +def splunk_event(): + data = request.json + data_storage["events"].append(data) + return jsonify({"text": "Success", "code": 0}), 200 + +if __name__ == '__main__': + app.run(port=8088) diff --git a/tests/integration/src/utils/data_utils.py b/tests/integration/src/utils/data_utils.py new file mode 100644 index 00000000000..e29918fe5a2 --- /dev/null +++ b/tests/integration/src/utils/data_utils.py @@ -0,0 +1,14 @@ +import os +import json + +def read_output(output_path): + with open(output_path, 'r') as file: + return json.load(file) + +def read_json_file(file_path): + with open(file_path, 'r') as file: + return json.load(file) + +def read_file(file_path): + with open(file_path, 'r') as file: + return file.read() diff --git a/tests/integration/src/utils/fluent_bit_manager.py b/tests/integration/src/utils/fluent_bit_manager.py new file mode 100644 index 00000000000..1040efa5be3 --- /dev/null +++ b/tests/integration/src/utils/fluent_bit_manager.py @@ -0,0 +1,209 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2024 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import logging +import os +from pathlib import Path +import shutil +import subprocess +import time + +import requests + +from utils.network import find_available_port, wait_for_port_to_be_free +from utils.valgrind import assert_valgrind_clean + +ENV_FLB_HTTP_MONITORING_PORT = "FLUENT_BIT_HTTP_MONITORING_PORT" + +logger = logging.getLogger(__name__) + +class FluentBitStartupError(RuntimeError): + pass + + +def _default_binary_path(): + repo_root = Path(__file__).resolve().parents[4] + local_binary = repo_root / "build" / "bin" / "fluent-bit" + if local_binary.is_file(): + return str(local_binary) + + binary_from_path = shutil.which("fluent-bit") + if binary_from_path: + return binary_from_path + + return str(local_binary) + + +class FluentBitManager: + def __init__(self, config_path=None, binary_path=None): + logger.info(f"config path {config_path}") + self.config_path = config_path + self.binary_path = binary_path or os.environ.get("FLUENT_BIT_BINARY") or _default_binary_path() + self.binary_absolute_path = os.path.abspath(self.binary_path) + self.process = None + self.http_monitoring_port = None + self.results_dir = None + self.log_file = None + self.valgrind_log_file = None + self.output_handle = None + + def set_http_monitoring_port(self, env_var_name, starting_port=0): + port = find_available_port(starting_port) + os.environ[env_var_name] = str(port) + self.http_monitoring_port = str(port) + + def start(self): + if not self.config_path or not os.path.exists(self.config_path): + raise FileNotFoundError(f"Config file {self.config_path} does not exist") + if not os.path.isfile(self.binary_absolute_path): + raise FileNotFoundError( + f"Fluent Bit binary {self.binary_absolute_path} does not exist. " + "Set FLUENT_BIT_BINARY or build build/bin/fluent-bit." + ) + if not os.access(self.binary_absolute_path, os.X_OK): + raise PermissionError(f"Fluent Bit binary {self.binary_absolute_path} is not executable") + + # create temporary directory for logs + out_dir = self.create_results_directory() + self.results_dir = out_dir + self.log_file = os.path.join(out_dir, "fluent_bit.log") + self.valgrind_log_file = os.path.join(out_dir, "valgrind.log") + self.set_http_monitoring_port(ENV_FLB_HTTP_MONITORING_PORT) + + version, commit = self.get_version_info() + logger.info(f'Fluent Bit info') + logger.info(f' version : {version}') + logger.info(f' path : {self.binary_absolute_path}') + logger.info(f" config file: {self.config_path}") + logger.info(f" logfile : {self.log_file}") + logger.info(f" http port : {self.http_monitoring_port}") + logger.info(f" commit : {commit}") + if self.valgrind_log_file: + logger.info(f" valgrind : {self.valgrind_log_file}") + + command = [ + self.binary_absolute_path, + "-c", self.config_path, + "-l", self.log_file + ] + + valgrind = os.environ.get('VALGRIND', False) + if valgrind: + command = [ + "valgrind", + f"--log-file={self.valgrind_log_file}", + "--leak-check=full", + "--show-leak-kinds=all" + ] + command + + + logger.info(f"Running command {command}") + + self.output_handle = open(self.log_file, "a", encoding="utf-8") + self.process = subprocess.Popen( + command, + stdout=self.output_handle, + stderr=subprocess.STDOUT, + text=True, + ) + logger.info(f"Fluent Bit started (pid: {self.process.pid})") + + # wait for Fluent Bit to start + self.wait_for_fluent_bit() + + def stop(self): + if not self.process: + return + + pid = self.process.pid + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + self.process = None + + if self.output_handle: + self.output_handle.close() + self.output_handle = None + + if self.http_monitoring_port: + wait_for_port_to_be_free(int(self.http_monitoring_port), timeout=10 if os.environ.get("VALGRIND") else 5) + + if os.environ.get("VALGRIND") and os.environ.get("VALGRIND_STRICT"): + assert_valgrind_clean(self.valgrind_log_file) + + logger.info(f"Fluent Bit stopped (pid: {pid})") + + def get_version_info(self): + try: + result = subprocess.run( + [self.binary_absolute_path, '--version'], + capture_output=True, + text=True, + check=True, + ) + output = result.stdout.strip().split('\n') + version = output[0].replace('Fluent Bit ', '').strip() + commit = output[1].strip().replace('Git commit: ', '') if len(output) > 1 else "unknown" + return version, commit + except (subprocess.CalledProcessError, FileNotFoundError) as e: + logger.error("Error running Fluent Bit: %s", e) + raise FluentBitStartupError(f"Unable to execute Fluent Bit binary {self.binary_absolute_path}") from e + + def create_results_directory(self, base_dir=None): + if base_dir is None: + suite_root = Path(__file__).resolve().parents[2] + base_dir = suite_root / "results" + timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + results_dir = os.path.join(base_dir, f"fluent_bit_results_{timestamp}") + os.makedirs(results_dir, exist_ok=True) + return results_dir + + # Check if Fluent Bit is running by trying to reach the uptime endpoint, it waits until + # the value of `uptime_sec` is greater than 1 + def wait_for_fluent_bit(self, timeout=None): + if timeout is None: + timeout = 30 if os.environ.get("VALGRIND") else 10 + url = f"http://127.0.0.1:{self.http_monitoring_port}/api/v1/uptime" + start_time = time.time() + while time.time() - start_time < timeout: + if self.process and self.process.poll() is not None: + raise FluentBitStartupError( + f"Fluent Bit exited early with code {self.process.returncode}. " + f"See log file {self.log_file}" + ) + try: + response = requests.get(url, timeout=0.5) + logger.info(f"Fluent Bit health check: {response.status_code}") + + if response.status_code == 200: + uptime = response.json().get('uptime_sec', 0) + if uptime > 1: + logger.info("Fluent Bit is running, health check OK") + return True + except requests.ConnectionError: + # it's ok to fail, we are testing + pass + + time.sleep(1) + + raise FluentBitStartupError( + f"Fluent Bit did not start within {timeout} seconds. See log file {self.log_file}" + ) diff --git a/tests/integration/src/utils/http_matrix.py b/tests/integration/src/utils/http_matrix.py new file mode 100644 index 00000000000..7fcdf77fda1 --- /dev/null +++ b/tests/integration/src/utils/http_matrix.py @@ -0,0 +1,151 @@ +import os +import subprocess +import tempfile + + +PROTOCOL_CASES = [ + { + "id": "http1_cleartext", + "config_key": "http1_cleartext", + "http_mode": "http1.1", + "use_tls": False, + "expected_http_version": "1.1", + }, + { + "id": "http2_cleartext_prior_knowledge", + "config_key": "http2_cleartext", + "http_mode": "http2-prior-knowledge", + "use_tls": False, + "expected_http_version": "2", + }, + { + "id": "http2_cleartext_upgrade_attempt", + "config_key": "http2_cleartext", + "http_mode": "http2", + "use_tls": False, + "expected_http_version": "1.1", + }, + { + "id": "http2_cleartext_upgrade_fallback_http1", + "config_key": "http1_cleartext", + "http_mode": "http2", + "use_tls": False, + "expected_http_version": "1.1", + }, + { + "id": "http1_tls", + "config_key": "http1_tls", + "http_mode": "http1.1", + "use_tls": True, + "expected_http_version": "1.1", + }, + { + "id": "http2_tls_alpn", + "config_key": "http2_tls", + "http_mode": "http2", + "use_tls": True, + "expected_http_version": "2", + }, + { + "id": "http2_tls_fallback_http1", + "config_key": "http1_tls", + "http_mode": "http2", + "use_tls": True, + "expected_http_version": "1.1", + }, +] + + +def curl_supports_http2(): + result = subprocess.run( + ["curl", "--version"], + capture_output=True, + text=True, + check=True, + ) + first_line = result.stdout.splitlines()[0] if result.stdout else "" + return "HTTP2" in result.stdout or "HTTP2" in first_line + + +def run_curl_request( + url, + payload=None, + *, + method="POST", + headers=None, + http_mode, + insecure_tls=False, + ca_cert_path=None, + include_headers=False, + extra_args=None, +): + command = [ + "curl", + "--silent", + "--show-error", + "--output", + "-", + "--write-out", + "\n__META__%{http_code} %{http_version}", + "--max-time", + "10", + "-X", + method, + ] + + header_file = None + if include_headers: + header_file = tempfile.NamedTemporaryFile(mode="w+b", delete=False) + header_file.close() + command.extend(["--dump-header", header_file.name]) + + for header in headers or []: + command.extend(["-H", header]) + + stdin_payload = None + if payload is not None: + command.extend(["--data-binary", "@-"]) + stdin_payload = payload if isinstance(payload, bytes) else payload.encode() + + if http_mode == "http1.1": + command.append("--http1.1") + elif http_mode == "http2": + command.append("--http2") + elif http_mode == "http2-prior-knowledge": + command.append("--http2-prior-knowledge") + else: + raise ValueError(f"Unsupported HTTP mode {http_mode}") + + if ca_cert_path: + command.extend(["--cacert", ca_cert_path]) + elif insecure_tls: + command.append("--insecure") + + if extra_args: + command.extend(extra_args) + + command.append(url) + + try: + result = subprocess.run(command, input=stdin_payload, capture_output=True, check=True) + output = result.stdout.decode() + body, _, meta = output.rpartition("\n__META__") + status_code, http_version = meta.strip().split(" ", 1) + + response = { + "body": body, + "status_code": int(status_code), + "http_version": http_version, + } + + if include_headers and header_file: + with open(header_file.name, "r", encoding="utf-8", errors="replace") as file: + response["headers_raw"] = file.read() + + return response + finally: + if header_file: + try: + os.unlink(header_file.name) + except FileNotFoundError: + pass diff --git a/tests/integration/src/utils/network.py b/tests/integration/src/utils/network.py new file mode 100644 index 00000000000..0e7771a8be3 --- /dev/null +++ b/tests/integration/src/utils/network.py @@ -0,0 +1,49 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2024 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +import time + +def find_available_port(starting_port=0): + if starting_port in (None, 0): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + for port in range(starting_port, 65535): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(("127.0.0.1", port)) + return port + except OSError: + continue + + raise RuntimeError("No available ports found in the specified range") + + +def wait_for_port_to_be_free(port, *, host="127.0.0.1", timeout=5, interval=0.1): + deadline = time.time() + timeout + while time.time() < deadline: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind((host, port)) + return True + except OSError: + time.sleep(interval) + return False diff --git a/tests/integration/src/utils/test_service.py b/tests/integration/src/utils/test_service.py new file mode 100644 index 00000000000..17a6d4591d6 --- /dev/null +++ b/tests/integration/src/utils/test_service.py @@ -0,0 +1,97 @@ +import os +import time + +import requests + +from utils.fluent_bit_manager import FluentBitManager +from utils.network import find_available_port + + +class FluentBitTestService: + def __init__( + self, + config_path, + *, + data_storage=None, + data_keys=None, + extra_env=None, + pre_start=None, + post_stop=None, + ): + self.config_path = config_path + self.data_storage = data_storage + self.data_keys = data_keys or [] + self.extra_env = extra_env or {} + self.pre_start = pre_start + self.post_stop = post_stop + self.flb = None + self._previous_env = {} + + def _reset_storage(self): + if not self.data_storage: + return + for key in self.data_keys: + self.data_storage[key] = [] + + def _set_env(self, key, value): + self._previous_env.setdefault(key, os.environ.get(key)) + os.environ[key] = value + + def allocate_port_env(self, key, *, starting_port=0): + port = find_available_port(starting_port) + self._set_env(key, str(port)) + return port + + def _restore_env(self): + for key, value in self._previous_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + self._previous_env.clear() + + def start(self): + self._reset_storage() + self.flb = FluentBitManager(self.config_path) + self.flb_listener_port = find_available_port() + self.test_suite_http_port = find_available_port() + self._set_env("FLUENT_BIT_TEST_LISTENER_PORT", str(self.flb_listener_port)) + self._set_env("TEST_SUITE_HTTP_PORT", str(self.test_suite_http_port)) + + for key, value in self.extra_env.items(): + self._set_env(key, str(value)) + + if self.pre_start: + self.pre_start(self) + + self.flb.start() + + def stop(self): + try: + if self.flb: + self.flb.stop() + finally: + if self.post_stop: + self.post_stop(self) + self._restore_env() + + def wait_for_http_endpoint(self, url, *, timeout=10, interval=0.5): + deadline = time.time() + timeout + while time.time() < deadline: + try: + response = requests.get(url, timeout=interval) + if response.status_code == 200: + return + except requests.RequestException: + pass + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for endpoint {url}") + + def wait_for_condition(self, predicate, *, timeout=10, interval=0.5, description="condition"): + deadline = time.time() + timeout + while time.time() < deadline: + value = predicate() + if value: + return value + time.sleep(interval) + raise TimeoutError(f"Timed out waiting for {description}") diff --git a/tests/integration/src/utils/valgrind.py b/tests/integration/src/utils/valgrind.py new file mode 100644 index 00000000000..506f4fe2620 --- /dev/null +++ b/tests/integration/src/utils/valgrind.py @@ -0,0 +1,84 @@ +import re +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class ValgrindSummary: + definitely_lost: int = 0 + indirectly_lost: int = 0 + possibly_lost: int = 0 + still_reachable: int = 0 + suppressed: int = 0 + error_count: int = 0 + context_count: int = 0 + invalid_read_count: int = 0 + invalid_write_count: int = 0 + invalid_free_count: int = 0 + uninitialised_count: int = 0 + + @property + def has_leaks(self): + return any( + [ + self.definitely_lost, + self.indirectly_lost, + self.possibly_lost, + ] + ) + + @property + def has_errors(self): + return self.error_count > 0 + + +def _parse_bytes(text, label): + pattern = rf"{label}:\s*([0-9,]+) bytes" + match = re.search(pattern, text) + if not match: + return 0 + return int(match.group(1).replace(",", "")) + + +def parse_valgrind_log(log_path): + text = Path(log_path).read_text(encoding="utf-8") + summary = ValgrindSummary( + definitely_lost=_parse_bytes(text, "definitely lost"), + indirectly_lost=_parse_bytes(text, "indirectly lost"), + possibly_lost=_parse_bytes(text, "possibly lost"), + still_reachable=_parse_bytes(text, "still reachable"), + suppressed=_parse_bytes(text, "suppressed"), + ) + + error_match = re.search(r"ERROR SUMMARY:\s*([0-9,]+) errors from ([0-9,]+) contexts", text) + if error_match: + summary.error_count = int(error_match.group(1).replace(",", "")) + summary.context_count = int(error_match.group(2).replace(",", "")) + + summary.invalid_read_count = len(re.findall(r"Invalid read", text)) + summary.invalid_write_count = len(re.findall(r"Invalid write", text)) + summary.invalid_free_count = len(re.findall(r"Invalid free", text)) + summary.uninitialised_count = len(re.findall(r"uninitialised|Uninitialised|Use of uninitialised", text)) + + return summary + + +def assert_valgrind_clean(log_path, *, allow_definitely_lost=0, allow_error_count=0): + summary = parse_valgrind_log(log_path) + problems = [] + + if summary.definitely_lost > allow_definitely_lost: + problems.append(f"definitely lost={summary.definitely_lost}") + if summary.indirectly_lost: + problems.append(f"indirectly lost={summary.indirectly_lost}") + if summary.possibly_lost: + problems.append(f"possibly lost={summary.possibly_lost}") + if summary.error_count > allow_error_count: + problems.append(f"errors={summary.error_count}") + + if problems: + raise AssertionError( + f"Valgrind issues found in {log_path}: " + ", ".join(problems) + ) + + return summary diff --git a/tests/integration/src/validators/http_validator.py b/tests/integration/src/validators/http_validator.py new file mode 100644 index 00000000000..406ba31f3bc --- /dev/null +++ b/tests/integration/src/validators/http_validator.py @@ -0,0 +1,5 @@ +from parsers.http_parser import parse_http_payload + +def validate_http_payload(payload, expected_payload): + payload_dict = parse_http_payload(payload) + return payload_dict == expected_payload diff --git a/tests/integration/src/validators/otlp_trace_validator.py b/tests/integration/src/validators/otlp_trace_validator.py new file mode 100644 index 00000000000..b7ff4a27dae --- /dev/null +++ b/tests/integration/src/validators/otlp_trace_validator.py @@ -0,0 +1,6 @@ +from parsers.otlp_parser import parse_trace_request + +def validate_trace_data(trace_request, expected_span_name): + trace_data_dict = parse_trace_request(trace_request) + spans = trace_data_dict["resourceSpans"][0]["instrumentationLibrarySpans"][0]["spans"] + return any(span["name"] == expected_span_name for span in spans) diff --git a/tests/integration/src/validators/splunk_validator.py b/tests/integration/src/validators/splunk_validator.py new file mode 100644 index 00000000000..4f9c14f08f3 --- /dev/null +++ b/tests/integration/src/validators/splunk_validator.py @@ -0,0 +1,5 @@ +from parsers.splunk_parser import parse_splunk_event + +def validate_splunk_event(event, expected_event): + event_dict = parse_splunk_event(event) + return event_dict == expected_event diff --git a/tests/integration/test_valgrind_utils.py b/tests/integration/test_valgrind_utils.py new file mode 100644 index 00000000000..833d04b388d --- /dev/null +++ b/tests/integration/test_valgrind_utils.py @@ -0,0 +1,47 @@ +from pathlib import Path + +import pytest + +from utils.valgrind import assert_valgrind_clean, parse_valgrind_log + + +def test_parse_valgrind_log_detects_clean_run(tmp_path): + log_path = tmp_path / "valgrind.log" + log_path.write_text( + "\n".join( + [ + "==1== HEAP SUMMARY:", + "==1== in use at exit: 0 bytes in 0 blocks", + "==1== total heap usage: 10 allocs, 10 frees, 100 bytes allocated", + "==1== All heap blocks were freed -- no leaks are possible", + "==1== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0)", + ] + ), + encoding="utf-8", + ) + + summary = parse_valgrind_log(log_path) + + assert summary.definitely_lost == 0 + assert summary.error_count == 0 + assert summary.has_leaks is False + + +def test_assert_valgrind_clean_rejects_leaks(tmp_path): + log_path = tmp_path / "valgrind.log" + log_path.write_text( + "\n".join( + [ + "==1== definitely lost: 45 bytes in 2 blocks", + "==1== indirectly lost: 0 bytes in 0 blocks", + "==1== possibly lost: 0 bytes in 0 blocks", + "==1== still reachable: 0 bytes in 0 blocks", + "==1== ERROR SUMMARY: 2 errors from 2 contexts (suppressed: 0 from 0)", + ] + ), + encoding="utf-8", + ) + + with pytest.raises(AssertionError): + assert_valgrind_clean(log_path) + diff --git a/tests/integration/tests/test_fluent_bit_manager.py b/tests/integration/tests/test_fluent_bit_manager.py new file mode 100644 index 00000000000..f9de3620574 --- /dev/null +++ b/tests/integration/tests/test_fluent_bit_manager.py @@ -0,0 +1,169 @@ +# Fluent Bit +# ========== +# Copyright (C) 2015-2024 The Fluent Bit Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import signal +from unittest.mock import Mock + +import pytest +import requests + +from src.utils import fluent_bit_manager as manager_module +from src.utils.fluent_bit_manager import ENV_FLB_BINARY_PATH +from src.utils.fluent_bit_manager import FluentBitManager + + +def test_binary_path_uses_environment_override(monkeypatch): + monkeypatch.setenv(ENV_FLB_BINARY_PATH, "/opt/fluent-bit/bin/fluent-bit") + monkeypatch.setattr(manager_module.shutil, "which", lambda path: f"/resolved/{path}") + + manager = FluentBitManager("/tmp/fluent-bit.yaml") + + assert manager.binary_path == "/opt/fluent-bit/bin/fluent-bit" + assert manager.binary_absolute_path == "/resolved//opt/fluent-bit/bin/fluent-bit" + + +def test_send_signal_raises_when_process_is_missing(): + manager = FluentBitManager("/tmp/fluent-bit.yaml") + + with pytest.raises(RuntimeError, match="not running"): + manager.send_signal(signal.SIGHUP) + + +def test_send_sighup_forwards_signal_to_process(): + manager = FluentBitManager("/tmp/fluent-bit.yaml") + manager.process = Mock() + + manager.send_sighup() + + manager.process.send_signal.assert_called_once_with(signal.SIGHUP) + + +def test_trigger_http_reload_posts_to_reload_endpoint(monkeypatch): + response = Mock() + response.json.return_value = {"reload": "done"} + response.raise_for_status.return_value = None + + def fake_post(url): + assert url == "http://127.0.0.1:2020/api/v2/reload" + return response + + monkeypatch.setattr(manager_module.requests, "post", fake_post) + + manager = FluentBitManager("/tmp/fluent-bit.yaml") + manager.http_monitoring_port = "2020" + + assert manager.trigger_http_reload() == {"reload": "done"} + + +def test_start_uses_unique_valgrind_log_path(monkeypatch, tmp_path): + config_path = tmp_path / "fluent-bit.yaml" + config_path.write_text("service:\n flush: 1\n", encoding="utf-8") + popen_calls = [] + + monkeypatch.setenv("VALGRIND", "1") + monkeypatch.setattr(manager_module.os.path, "exists", lambda path: True) + monkeypatch.setattr(manager_module, "find_available_port", lambda starting_port: 40200) + monkeypatch.setattr(manager_module.requests, "get", lambda url: Mock(status_code=200, json=lambda: {"uptime_sec": 2})) + + created_dirs = iter([ + str(tmp_path / "run-1"), + ]) + + def fake_create_results_directory(self, base_dir='results'): + path = next(created_dirs) + manager_module.os.makedirs(path, exist_ok=True) + return path + + popen_result = Mock() + popen_result.pid = 1234 + + monkeypatch.setattr(FluentBitManager, "create_results_directory", fake_create_results_directory) + monkeypatch.setattr(FluentBitManager, "get_version_info", lambda self: ("vtest", "commit")) + + def fake_popen(command, stdout=None, stderr=None): + popen_calls.append(command) + return popen_result + + monkeypatch.setattr(manager_module.subprocess, "Popen", fake_popen) + + manager = FluentBitManager(str(config_path), "/usr/bin/fluent-bit") + manager.start() + + assert manager.results_dir == str(tmp_path / "run-1") + assert manager.valgrind_log_file == str(tmp_path / "run-1" / "valgrind.log") + assert popen_calls == [[ + "valgrind", + f"--log-file={manager.valgrind_log_file}", + "--leak-check=full", + "/usr/bin/fluent-bit", + "-c", str(config_path), + "-l", str(tmp_path / "run-1" / "fluent_bit.log") + ]] + + +def test_wait_for_hot_reload_count_returns_when_expected_count_is_reached(monkeypatch): + manager = FluentBitManager("/tmp/fluent-bit.yaml") + manager.http_monitoring_port = "2020" + + payloads = iter([ + {"hot_reload_count": 0}, + {"hot_reload_count": 1}, + {"hot_reload_count": 2}, + ]) + + monkeypatch.setattr(manager, "get_reload_status", lambda: next(payloads)) + monkeypatch.setattr(manager_module.time, "sleep", lambda _: None) + + payload = manager.wait_for_hot_reload_count(2, timeout=5) + + assert payload["hot_reload_count"] == 2 + + +def test_wait_for_hot_reload_count_ignores_request_errors(monkeypatch): + manager = FluentBitManager("/tmp/fluent-bit.yaml") + manager.http_monitoring_port = "2020" + + values = iter([ + requests.RequestException("boom"), + {"hot_reload_count": 1}, + ]) + + def fake_get_reload_status(): + value = next(values) + if isinstance(value, Exception): + raise value + return value + + monkeypatch.setattr(manager, "get_reload_status", fake_get_reload_status) + monkeypatch.setattr(manager_module.time, "sleep", lambda _: None) + + payload = manager.wait_for_hot_reload_count(1, timeout=5) + + assert payload["hot_reload_count"] == 1 + + +def test_wait_for_hot_reload_count_times_out(monkeypatch): + manager = FluentBitManager("/tmp/fluent-bit.yaml") + manager.http_monitoring_port = "2020" + + timestamps = iter([0.0, 0.1, 0.2, 0.3]) + + monkeypatch.setattr(manager, "get_reload_status", lambda: {"hot_reload_count": 0}) + monkeypatch.setattr(manager_module.time, "time", lambda: next(timestamps)) + monkeypatch.setattr(manager_module.time, "sleep", lambda _: None) + + with pytest.raises(TimeoutError, match="Timed out waiting for hot reload count 1"): + manager.wait_for_hot_reload_count(1, timeout=0.25)