From 0ba3ebc1b28a7fdf006fba1dabee5716a914a7d4 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Sat, 7 Sep 2024 17:54:54 +0800 Subject: [PATCH 1/2] feat: just integration-otel to see traces Signed-off-by: Adrian Cole --- CONTRIBUTING.md | 33 ++++++++++++++++++++++++++++----- justfile | 3 +++ pyproject.toml | 10 +++++++++- 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 39fa06a..2912493 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,27 +13,50 @@ We provide a shortcut to standard commands using [just][just] in our `justfile`. Now that you have a local environment, you can make edits and run our tests. -``` +```bash uv run pytest tests -m "not integration" ``` or, as a shortcut, -``` +```bash just test ``` Generally if you are not developing a new provider, you can test most functionality through mocking and the normal test suite. -However to ensure the providers work, we also have integration tests which actually require a credential and connect +However, to ensure the providers work, we also have integration tests which actually require a credential and connect to the provider endpoints. Those can be run with -``` +```bash uv run pytest tests -m integration -# or `just integration` +# or `just integration` +``` + +### Integration tests with OpenTelemetry + +Exchange primarily uses http to access model providers. If you are receiving failures, it can be helpful to see traces +of the underlying HTTP requests. For example, a 404 could be indicative of an incorrect URL or a missing model. + +First, ensure you have an OpenTelemetry compatible collector listening on port 4318, such as +[otel-tui](https://github.com/ymtdzzz/otel-tui). + +```bash +brew install ymtdzzz/tap/otel-tui +otel-tui +``` + +Then, trace your integration tests like this: +```bash +uv run dotenv run -- opentelemetry-instrument pytest tests -m integration +# or `just integration-otel` ``` +Now, you can see failure details like this: + +otel-tui + ## Pull Requests When opening a pull request, please ensure that your PR title adheres to the [Conventional Commits specification](https://www.conventionalcommits.org/). diff --git a/justfile b/justfile index 933d2f8..a47f3a5 100644 --- a/justfile +++ b/justfile @@ -9,6 +9,9 @@ test *FLAGS: integration *FLAGS: uv run pytest tests -m integration {{FLAGS}} +integration-otel *FLAGS: + uv run dotenv run -- opentelemetry-instrument pytest tests -m integration {{FLAGS}} + format: ruff check . --fix diff --git a/pyproject.toml b/pyproject.toml index 5504ce2..2211e5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,15 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.uv] -dev-dependencies = ["pytest>=8.3.2", "codecov>=2.1.13"] +dev-dependencies = [ + "pytest>=8.3.2", + "codecov>=2.1.13", + "opentelemetry-sdk>=1.27.0", + "opentelemetry-exporter-otlp-proto-http>=1.27.0", + "opentelemetry-distro>=0.48b0", + "opentelemetry-instrumentation-httpx>=0.48b0", + "python-dotenv[cli]>=1.0.1" +] [project.entry-points."exchange.provider"] openai = "exchange.providers.openai:OpenAiProvider" From eb5046ac8d2e51b85082f7376809e95957ed83bd Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Sat, 7 Sep 2024 20:04:52 +0800 Subject: [PATCH 2/2] fix dotenv Signed-off-by: Adrian Cole --- CONTRIBUTING.md | 2 +- justfile | 2 +- tests/otel.env | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 tests/otel.env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2912493..7b09ef4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,7 +49,7 @@ otel-tui Then, trace your integration tests like this: ```bash -uv run dotenv run -- opentelemetry-instrument pytest tests -m integration +uv run dotenv -f ./tests/otel.env run -- opentelemetry-instrument pytest tests -m integration # or `just integration-otel` ``` diff --git a/justfile b/justfile index a47f3a5..3d45ae7 100644 --- a/justfile +++ b/justfile @@ -10,7 +10,7 @@ integration *FLAGS: uv run pytest tests -m integration {{FLAGS}} integration-otel *FLAGS: - uv run dotenv run -- opentelemetry-instrument pytest tests -m integration {{FLAGS}} + uv run dotenv -f ./tests/otel.env run -- opentelemetry-instrument pytest tests -m integration {{FLAGS}} format: ruff check . --fix diff --git a/tests/otel.env b/tests/otel.env new file mode 100644 index 0000000..75091c8 --- /dev/null +++ b/tests/otel.env @@ -0,0 +1,5 @@ +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 +OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf +OTEL_RESOURCE_ATTRIBUTES=service.name=exchange-ai +OTEL_METRIC_EXPORT_INTERVAL=100 +OTEL_BSP_SCHEDULE_DELAY=100