tests: integration: introduce in-tree integration test suite

.github/scripts/commit_prefix_check.py

@@ -21,7 +21,7 @@
 repo = Repo(".")
 
 # Regex patterns
-PREFIX_RE = re.compile(r"^([a-z0-9_]+:)\s+\S", re.IGNORECASE)
+PREFIX_RE = re.compile(r"^((?:[a-z0-9_]+:\s+)+)\S", re.IGNORECASE)
 SIGNED_OFF_RE = re.compile(r"Signed-off-by:", re.IGNORECASE)
 FENCED_BLOCK_RE = re.compile(
     r"""
@@ -33,6 +33,15 @@
     re.DOTALL | re.VERBOSE,
 )
 
+
+def extract_subject_prefix(line: str):
+    match = PREFIX_RE.match(line)
+
+    if not match:
+        return None
+
+    return match.group(1).rstrip()
+
 def strip_fenced_code_blocks(text: str) -> str:
     """
     Remove fenced code blocks (``` or ~~~) from commit message body.
@@ -78,6 +87,8 @@ def infer_prefix_from_paths(paths):
                 if name:
                     component_prefixes.add(f"{name}:")
                     component_prefixes.add("tests:")
+                    if p.startswith("tests/integration/"):
+                        component_prefixes.add("tests: integration:")
             else:
                 component_prefixes.add("tests:")
 
@@ -226,12 +237,10 @@ def is_version_bump(commit):
     body = strip_fenced_code_blocks(body)
 
     # Subject must start with a prefix
-    subject_prefix_match = PREFIX_RE.match(first_line)
-    if not subject_prefix_match:
+    subject_prefix = extract_subject_prefix(first_line)
+    if not subject_prefix:
         return False, f"Missing prefix in commit subject: '{first_line}'"
 
-    subject_prefix = subject_prefix_match.group(1)
-
     # Run squash detection (but ignore multi-signoff errors)
     bad_squash, reason = detect_bad_squash(body)
 
@@ -313,7 +322,7 @@ def is_version_bump(commit):
     }
 
     # Prefixes that are allowed to cover multiple subcomponents
-    umbrella_prefixes = {"lib:", "tests:", "http_server:"}
+    umbrella_prefixes = {"lib:", "tests:", "tests: integration:", "http_server:"}
 
     # If more than one non-build prefix is inferred AND the subject is not an umbrella
     # prefix, check if the subject prefix is in the expected list. If it is, allow it
@@ -341,6 +350,15 @@ def is_version_bump(commit):
                         f"Expected one of: {expected_str}"
                     )
 
+            elif subj_lower == "tests: integration:":
+                if not all(p.startswith("tests/integration/") for p in norm_paths):
+                    expected_list = sorted(expected)
+                    expected_str = ", ".join(expected_list)
+                    return False, (
+                        f"Subject prefix '{subject_prefix}' does not match files changed.\n"
+                        f"Expected one of: {expected_str}"
+                    )
+
             elif subj_lower == "http_server:":
                 if not all(is_http_server_interface_path(p) for p in norm_paths):
                     expected_list = sorted(expected)

.gitignore

@@ -41,3 +41,7 @@ workflow/
 examples/wasi_serde_json/target/
 # WASM test data
 tests/runtime/wasm/go/*.wasm
+tests/integration/.venv/
+tests/integration/.pytest_cache/
+tests/integration/**/__pycache__/
+tests/integration/results/

AGENTS.md

@@ -8,6 +8,10 @@
   when the affected area is known, because the full enabled suite can be slow.
 - Run a focused integration test with
   `ctest --test-dir build -R flb-it-opentelemetry --output-on-failure`
+- Run the in-tree Python integration suite with:
+  `cd tests/integration && ./setup-venv.sh && ./run_tests.py`
+- List available Python integration scenarios with:
+  `cd tests/integration && ./run_tests.py --list`
 - Run locally with `./build/bin/fluent-bit -c conf/fluent-bit.conf`
 
 ## Project Structure & Module Organization
@@ -18,6 +22,9 @@ Fluent Bit is a C/C++ monorepo built with CMake.
 - `plugins/`: input/filter/processor/output plugins (`in_*`, `filter_*`, `processor_*`, `out_*`).
 - `lib/`: bundled libraries (e.g., `cprofiles`, `ctraces`, `cmetrics`, `chunkio`).
 - `tests/`: integration/runtime tests and fixtures.
+- `tests/integration/`: in-tree Python integration test suite for end-to-end
+  plugin and protocol validation; introduced from the original
+  `github.com/fluent/fluent-bit-test-suite` project.
 - `conf/`: sample configurations for local validation.
 
 Keep changes scoped: plugin logic in its plugin directory, shared behavior in `src/` or `lib/`.
@@ -27,6 +34,14 @@ Keep changes scoped: plugin logic in its plugin directory, shared behavior in `s
 - `cmake --build build -j8`: compile Fluent Bit and tests.
 - `ctest --test-dir build --output-on-failure`: run enabled tests.
 - `ctest --test-dir build -R flb-it-opentelemetry --output-on-failure`: run a focused integration test.
+- `cd tests/integration && ./setup-venv.sh`: create the local virtualenv for
+  the Python integration suite.
+- `cd tests/integration && ./run_tests.py --list`: list available Python
+  integration scenarios.
+- `cd tests/integration && ./run_tests.py`: run the full Python integration
+  suite against `build/bin/fluent-bit`.
+- `cd tests/integration && FLUENT_BIT_BINARY=/path/to/fluent-bit ./run_tests.py`:
+  run the Python integration suite against a specific binary.
 - `./build/bin/fluent-bit -c conf/fluent-bit.conf`: run locally with a config.
 
 ## Coding Style & Naming Conventions
@@ -48,9 +63,17 @@ Keep changes scoped: plugin logic in its plugin directory, shared behavior in `s
 - Add or update tests for behavior changes, especially protocol parsing and encoder/decoder paths.
 - Prefer targeted tests close to the changed module (`tests/internal`, plugin runtime tests).
 - Prefer focused `ctest -R ...` runs or specific test binaries when the touched area is known.
+- Use `tests/integration` when validating end-to-end plugin behavior, network
+  protocols, downstream request generation, or local fake-server interactions
+  that are awkward to cover in `ctest` binaries alone.
+- The Python integration suite is not part of the default CMake `ctest` targets;
+  run it explicitly from `tests/integration`.
 - Run broader test coverage when changing shared lifecycle, routing, storage, or accounting code.
 - Validate both success and failure paths (invalid payloads, boundary sizes, null/missing fields).
 - You can also run specific binaries from `build/bin` (e.g., `./bin/flb-it-opentelemetry`).
+- Keep generated integration artifacts out of git. Do not commit
+  `.venv/`, `.pytest_cache/`, `results/`, or `__pycache__/` under
+  `tests/integration`.
 
 ## Commit & Pull Request Guidelines
 - Prefix commit subjects with the component/plugin name in lowercase, e.g.:

tests/integration/.gitignore

@@ -0,0 +1,7 @@
+__pycache__
+.pytest_cache
+results/*
+valgrind.log
+env
+
+

tests/integration/CONTRIBUTING.md

@@ -0,0 +1,88 @@
+# Contributing to Fluent Bit Test Suite
+
+> note: this guide is still work in process
+
+The way to contribute to this project is through the official Github Repository  [fluent/fluent-bit-test-suite](https://github.com/fluent/fluent-bit-test-suite). All contributions must adhere to this guidelines that aims to make easier it maintenance and high quality over time.
+
+## Why contributing to this project ?
+
+Fluent Bit is one of the widely deployed Telemetry Agent around the globe, millions of new deployments happens every single day. Contributing to this project aims to extend the testing of different areas under different complex configurations.
+
+Our goal is to avoid regressions and making sure Fluent Bit can continue to grow in a healthy way.
+
+## Guidelines
+
+All the code in this project is based on Python 3.x and we have a few requirements:
+
+### Code Style
+
+Avoid using camelCase in variables, functions and method names, use the underscore (`_`) instead.
+
+### GIT Commits
+
+In open source project maintenance, having a clear history is key for us, for hence we expect full clarity in the commits. Clarity must be in the following places:
+
+- commit prefix
+- commit description
+
+#### Commit Prefix and Descriptions
+
+Commit _prefix_ __must not exceed 80__ characters in length. This help to readibility in the GIT log viewers. Make it short and straight to the point.
+
+In _commit description_, make sure each line do not exceed __120 characters__. Here you have more flexibility to write more about the changes, just try to be inside the length limit of each line.
+
+The project have specific components, we enforce that every commit that touches an interface or component be prefixed with that name. As of today we register the following components:
+
+__scenarios__
+
+An scenario defines a main type of pipeline or Fluent Bit component being tested, an example is:
+
+- in-opentelemetry: changes applicable to OTLP input and OTLP round-trip tests
+- in-splunk: changes applicable to Splunk input protocol tests
+- in-elasticsearch: changes applicable to Elasticsearch input compatibility tests
+- in-http: changes applicable to HTTP input end-to-end tests
+
+For any code change that is happening inside [scenarios/in_opentelemetry](https://github.com/fluent/fluent-bit-test-suite/tree/main/scenarios/in_opentelemetry) the commit must be prefixed like this:
+
+``` 
+scenarios: in-opentelemetry: description of the change
+```
+
+Optionally you can add a third component that represents another file interface (without the extension `.py`) .
+
+__server__
+
+In the server components, we have helpers to implement 'fake servers' who mimic other projects that we use as receivers, e.g:
+
+| server | description |
+|--|--|
+| [http](https://github.com/fluent/fluent-bit-test-suite/blob/main/src/server/http_server.py) | Simple HTTP server |
+| [otlp](https://github.com/fluent/fluent-bit-test-suite/blob/main/src/server/otlp_server.py) | OpenTelemetry HTTP Server |
+| [splunk](https://github.com/fluent/fluent-bit-test-suite/blob/main/src/server/splunk_server.py) | Splunk HTTP Server |
+
+When modifying any of those servers or adding new ones, the commits must be prefixed like this:
+
+```
+server: http: some example descripition
+```
+
+### Naming
+
+Prefer Fluent Bit internal plugin naming in test names and descriptions:
+
+- `in_splunk`
+- `in_elasticsearch`
+- `in_http`
+
+Scenario directory names may be broader for now, but new tests and updated assertions should follow the internal plugin names in function names, docs, and commit prefixes whenever practical.
+Scenario directory names are now expected to follow the internal plugin names for maintained scenarios whenever possible.
+
+
+##### Others
+
+Commits should not modify files outside of the scope defined in the prefix, while there might be cases for exceptions we will handle those in the Pull Request review process.
+
+
+### License
+
+All code contributed to this project is under the terms of the Apache v2 License. All commits must be signed (DCO).