From d25fc992ff912b86f5600002a7c2a615ea0dcaf4 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 3 Oct 2025 17:25:27 -0400 Subject: [PATCH 1/8] feat(ourlog): Add vercel log drain endpoint --- CHANGELOG.md | 1 + relay-ourlogs/src/lib.rs | 2 +- relay-ourlogs/src/vercel_to_sentry.rs | 2 +- .../src/endpoints/integrations/mod.rs | 1 + .../src/endpoints/integrations/vercel.rs | 43 +++ relay-server/src/endpoints/mod.rs | 1 + relay-server/src/envelope/item.rs | 4 + relay-server/src/integrations/mod.rs | 5 + .../src/processing/logs/integrations/mod.rs | 2 + .../processing/logs/integrations/vercel.rs | 47 +++ tests/integration/fixtures/__init__.py | 18 + tests/integration/test_vercel_logs.py | 352 ++++++++++++++++++ 12 files changed, 476 insertions(+), 2 deletions(-) create mode 100644 relay-server/src/endpoints/integrations/vercel.rs create mode 100644 relay-server/src/processing/logs/integrations/vercel.rs create mode 100644 tests/integration/test_vercel_logs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 24f24fa4c0c..7951b77f15a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - Upgrade sqlparser and improve SQL parsing for span grouping. ([#5211](https://github.com/getsentry/relay/pull/5211)) - Maps `unknown_error` span status to `internal_error` ([#5202](https://github.com/getsentry/relay/pull/5202)) - Add event merging logic for Playstation crashes. ([#5228](https://github.com/getsentry/relay/pull/5228)) +- Add vercel log drain endpoint. ([#5212](https://github.com/getsentry/relay/pull/5212)) **Bug Fixes**: diff --git a/relay-ourlogs/src/lib.rs b/relay-ourlogs/src/lib.rs index a3358bf7006..aa48b3d0887 100644 --- a/relay-ourlogs/src/lib.rs +++ b/relay-ourlogs/src/lib.rs @@ -12,6 +12,6 @@ mod vercel_to_sentry; pub use self::otel_to_sentry::otel_to_sentry_log; pub use self::size::calculate_size; -pub use self::vercel_to_sentry::vercel_log_to_sentry_log; +pub use self::vercel_to_sentry::{VercelLog, vercel_log_to_sentry_log}; pub use opentelemetry_proto::tonic::logs::v1 as otel_logs; diff --git a/relay-ourlogs/src/vercel_to_sentry.rs b/relay-ourlogs/src/vercel_to_sentry.rs index 585934e373e..b75b0a0ebb2 100644 --- a/relay-ourlogs/src/vercel_to_sentry.rs +++ b/relay-ourlogs/src/vercel_to_sentry.rs @@ -24,7 +24,7 @@ pub struct VercelLog { pub timestamp: i64, /// Identifier for the Vercel project. pub project_id: String, - // Log severity level. + /// Log severity level. pub level: VercelLogLevel, /// Log message content (may be truncated if over 256 KB). pub message: Option, diff --git a/relay-server/src/endpoints/integrations/mod.rs b/relay-server/src/endpoints/integrations/mod.rs index 95eb89aa4d7..f102f5ac487 100644 --- a/relay-server/src/endpoints/integrations/mod.rs +++ b/relay-server/src/endpoints/integrations/mod.rs @@ -1 +1,2 @@ pub mod otlp; +pub mod vercel; diff --git a/relay-server/src/endpoints/integrations/vercel.rs b/relay-server/src/endpoints/integrations/vercel.rs new file mode 100644 index 00000000000..fb4d003620f --- /dev/null +++ b/relay-server/src/endpoints/integrations/vercel.rs @@ -0,0 +1,43 @@ +use axum::extract::DefaultBodyLimit; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use axum::routing::{MethodRouter, post}; +use relay_config::Config; +use relay_dynamic_config::Feature; + +use crate::endpoints::common; +use crate::extractors::IntegrationBuilder; +use crate::integrations::LogsIntegration; +use crate::service::ServiceState; + +/// All routes configured for the Vercel integration. +/// +/// The integration currently supports the following endpoints: +/// - Vercel Log Drain +pub fn routes(config: &Config) -> axum::Router { + axum::Router::new() + .route("/logs", logs::route(config)) + .route("/logs/", logs::route(config)) +} + +mod logs { + use super::*; + + async fn handle( + state: ServiceState, + builder: IntegrationBuilder, + ) -> axum::response::Result { + let envelope = builder + .with_type(LogsIntegration::VercelDrainLog) + .with_required_feature(Feature::VercelLogDrainEndpoint) + .build(); + + common::handle_envelope(&state, envelope).await?; + + Ok(StatusCode::ACCEPTED) + } + + pub fn route(config: &Config) -> MethodRouter { + post(handle).route_layer(DefaultBodyLimit::max(config.max_envelope_size())) + } +} diff --git a/relay-server/src/endpoints/mod.rs b/relay-server/src/endpoints/mod.rs index 22f08ef64e8..41a9b91128f 100644 --- a/relay-server/src/endpoints/mod.rs +++ b/relay-server/src/endpoints/mod.rs @@ -87,6 +87,7 @@ pub fn routes(config: &Config) -> Router{ // configured by users or protocols may force a specific variant. let integration_routes = Router::new() .nest("/api/{project_id}/integration/otlp", integrations::otlp::routes(config)) + .nest("/api/{project_id}/integration/vercel", integrations::vercel::routes(config)) .route_layer(middlewares::cors()); // NOTE: If you add a new (non-experimental) route here, please also list it in diff --git a/relay-server/src/envelope/item.rs b/relay-server/src/envelope/item.rs index ce9b7632883..5b5b255c804 100644 --- a/relay-server/src/envelope/item.rs +++ b/relay-server/src/envelope/item.rs @@ -151,6 +151,10 @@ impl Item { (DataCategory::LogByte, self.len().max(1)), (DataCategory::LogItem, item_count), ], + Some(Integration::Logs(LogsIntegration::VercelDrainLog)) => smallvec![ + (DataCategory::LogByte, self.len().max(1)), + (DataCategory::LogItem, item_count), + ], Some(Integration::Spans(SpansIntegration::OtelV1 { .. })) => { smallvec![(DataCategory::Span, item_count)] } diff --git a/relay-server/src/integrations/mod.rs b/relay-server/src/integrations/mod.rs index 29521b17d51..8fab3c99937 100644 --- a/relay-server/src/integrations/mod.rs +++ b/relay-server/src/integrations/mod.rs @@ -42,6 +42,7 @@ define_integrations!( "application/vnd.sentry.integration.otel.logs+protobuf" => Integration::Logs(LogsIntegration::OtelV1 { format: OtelFormat::Protobuf }), "application/vnd.sentry.integration.otel.spans+json" => Integration::Spans(SpansIntegration::OtelV1 { format: OtelFormat::Json }), "application/vnd.sentry.integration.otel.spans+protobuf" => Integration::Spans(SpansIntegration::OtelV1 { format: OtelFormat::Protobuf }), + "application/vnd.sentry.integration.vercel.logs+json" => Integration::Logs(LogsIntegration::VercelDrainLog), ); /// An exhaustive list of all integrations supported by Relay. @@ -74,6 +75,10 @@ pub enum LogsIntegration { /// /// Supports OTeL's [`LogsData`](opentelemetry_proto::tonic::logs::v1::LogsData). OtelV1 { format: OtelFormat }, + /// The Vercel Log Drain integration. + /// + /// Supports the [`relay_ourlogs::VercelLog`] format. + VercelDrainLog, } /// All span integrations supported by Relay. diff --git a/relay-server/src/processing/logs/integrations/mod.rs b/relay-server/src/processing/logs/integrations/mod.rs index 81368cb0bbd..c3f405960ce 100644 --- a/relay-server/src/processing/logs/integrations/mod.rs +++ b/relay-server/src/processing/logs/integrations/mod.rs @@ -7,6 +7,7 @@ use crate::integrations::{Integration, LogsIntegration}; use crate::managed::RecordKeeper; mod otel; +mod vercel; /// Expands a list of [`Integration`] items into `result`. /// @@ -45,6 +46,7 @@ pub fn expand_into( let result = match integration { LogsIntegration::OtelV1 { format } => otel::expand(format, payload, produce), + LogsIntegration::VercelDrainLog => vercel::expand(payload, produce), }; match result { diff --git a/relay-server/src/processing/logs/integrations/vercel.rs b/relay-server/src/processing/logs/integrations/vercel.rs new file mode 100644 index 00000000000..c1319a0b2dd --- /dev/null +++ b/relay-server/src/processing/logs/integrations/vercel.rs @@ -0,0 +1,47 @@ +use relay_event_schema::protocol::OurLog; +use relay_ourlogs::VercelLog; + +use crate::processing::logs::{Error, Result}; +use crate::services::outcome::DiscardReason; + +/// Expands Vercel logs into the [`OurLog`] format. +pub fn expand(payload: &[u8], mut produce: F) -> Result<()> +where + F: FnMut(OurLog), +{ + let logs = parse_logs_data(payload)?; + + for log in logs { + let ourlog = relay_ourlogs::vercel_log_to_sentry_log(log); + produce(ourlog) + } + + Ok(()) +} + +fn parse_logs_data(payload: &[u8]) -> Result> { + // Try parsing as JSON array first + if let Ok(logs) = serde_json::from_slice::>(payload) { + return Ok(logs); + } + + // Fall back to NDJSON parsing + let payload_str = std::str::from_utf8(payload).map_err(|e| { + relay_log::debug!( + error = &e as &dyn std::error::Error, + "Failed to parse logs data as UTF-8" + ); + Error::Invalid(DiscardReason::InvalidJson) + })?; + + let logs: Vec = payload_str + .lines() + .filter_map(|line| serde_json::from_str::(line.trim()).ok()) + .collect(); + + if logs.is_empty() { + relay_log::debug!("Failed to parse any logs from vercel log drain payload"); + } + + Ok(logs) +} diff --git a/tests/integration/fixtures/__init__.py b/tests/integration/fixtures/__init__.py index d9f6dd13b4b..cda2b3b2caa 100644 --- a/tests/integration/fixtures/__init__.py +++ b/tests/integration/fixtures/__init__.py @@ -257,6 +257,24 @@ def send_otel_logs( response.raise_for_status() + def send_vercel_logs( + self, + project_id, + data=None, + headers=None, + dsn_key_idx=0, + dsn_key=None, + ): + + if dsn_key is None: + dsn_key = self.get_dsn_public_key(project_id, dsn_key_idx) + + url = f"/api/{project_id}/integration/vercel/logs?sentry_key={dsn_key}" + + response = self.post(url, headers=headers, data=data) + + response.raise_for_status() + def send_options(self, project_id, headers=None, dsn_key_idx=0): headers = { "X-Sentry-Auth": self.get_auth_header(project_id, dsn_key_idx), diff --git a/tests/integration/test_vercel_logs.py b/tests/integration/test_vercel_logs.py new file mode 100644 index 00000000000..14087e94111 --- /dev/null +++ b/tests/integration/test_vercel_logs.py @@ -0,0 +1,352 @@ +from unittest import mock +import json + +from sentry_relay.consts import DataCategory + +TEST_CONFIG = { + "outcomes": { + "emit_outcomes": True, + "batch_size": 1, + "batch_interval": 1, + "aggregator": { + "bucket_interval": 1, + "flush_interval": 1, + }, + }, + "aggregator": { + "bucket_interval": 1, + "initial_delay": 0, + }, +} + +# From Vercel Log Drain Docs: https://vercel.com/docs/drains/reference/logs#format +VERCEL_LOG_1 = { + "id": "1573817187330377061717300000", + "deploymentId": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD", + "source": "build", + "host": "my-app-abc123.vercel.app", + "timestamp": 1573817187330, + "projectId": "gdufoJxB6b9b1fEqr1jUtFkyavUU", + "level": "info", + "message": "Build completed successfully", + "buildId": "bld_cotnkcr76", + "type": "stdout", + "projectName": "my-app", +} + +# From Vercel Log Drain Docs: https://vercel.com/docs/drains/reference/logs#format +VERCEL_LOG_2 = { + "id": "1573817250283254651097202070", + "deploymentId": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD", + "source": "lambda", + "host": "my-app-abc123.vercel.app", + "timestamp": 1573817250283, + "projectId": "gdufoJxB6b9b1fEqr1jUtFkyavUU", + "level": "info", + "message": "API request processed", + "entrypoint": "api/index.js", + "requestId": "643af4e3-975a-4cc7-9e7a-1eda11539d90", + "statusCode": 200, + "path": "/api/users", + "executionRegion": "sfo1", + "environment": "production", + "traceId": "1b02cd14bb8642fd092bc23f54c7ffcd", + "spanId": "f24e8631bd11faa7", + "trace.id": "1b02cd14bb8642fd092bc23f54c7ffcd", + "span.id": "f24e8631bd11faa7", + "proxy": { + "timestamp": 1573817250172, + "method": "GET", + "host": "my-app.vercel.app", + "path": "/api/users?page=1", + "userAgent": ["Mozilla/5.0..."], + "referer": "https://my-app.vercel.app", + "region": "sfo1", + "statusCode": 200, + "clientIp": "120.75.16.101", + "scheme": "https", + "vercelCache": "MISS", + }, +} + + +def test_vercel_logs_json_array( + mini_sentry, relay, relay_with_processing, outcomes_consumer, items_consumer +): + """Test Vercel logs ingestion with JSON array format.""" + items_consumer = items_consumer() + outcomes_consumer = outcomes_consumer() + project_id = 42 + project_config = mini_sentry.add_full_project_config(project_id) + project_config["config"]["features"] = [ + "organizations:ourlogs-ingestion", + "organizations:relay-vercel-log-drain-endpoint", + ] + project_config["config"]["retentions"] = { + "log": {"standard": 30, "downsampled": 13 * 30}, + } + + relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) + + vercel_logs_payload = [ + VERCEL_LOG_1, + VERCEL_LOG_2, + ] + + relay.send_vercel_logs( + project_id, + data=json.dumps(vercel_logs_payload), + headers={"Content-Type": "application/json"}, + ) + + # Check that the items are properly processed via items_consumer + items = items_consumer.get_items(n=2) + assert items == [ + { + "attributes": { + "sentry._meta.fields.trace_id": {"stringValue": mock.ANY}, + "sentry.body": {"stringValue": "Build completed successfully"}, + "sentry.browser.name": {"stringValue": "Python Requests"}, + "sentry.browser.version": {"stringValue": "2.32"}, + "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, + "sentry.payload_size_bytes": {"intValue": mock.ANY}, + "sentry.severity_text": {"stringValue": "info"}, + "sentry.timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.timestamp_precise": {"intValue": mock.ANY}, + "server.address": {"stringValue": "my-app-abc123.vercel.app"}, + "vercel.build_id": {"stringValue": "bld_cotnkcr76"}, + "vercel.deployment_id": { + "stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD" + }, + "vercel.id": {"stringValue": "1573817187330377061717300000"}, + "vercel.log_type": {"stringValue": "stdout"}, + "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, + "vercel.project_name": {"stringValue": "my-app"}, + "vercel.source": {"stringValue": "build"}, + }, + "clientSampleRate": 1.0, + "downsampledRetentionDays": 390, + "itemId": mock.ANY, + "itemType": "TRACE_ITEM_TYPE_LOG", + "organizationId": "1", + "projectId": "42", + "received": mock.ANY, + "retentionDays": 30, + "serverSampleRate": 1.0, + "timestamp": mock.ANY, + "traceId": mock.ANY, + }, + { + "attributes": { + "sentry.environment": {"stringValue": "production"}, + "sentry.body": {"stringValue": "API request processed"}, + "sentry.browser.name": {"stringValue": "Python Requests"}, + "sentry.browser.version": {"stringValue": "2.32"}, + "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, + "sentry.payload_size_bytes": {"intValue": mock.ANY}, + "sentry.severity_text": {"stringValue": "info"}, + "sentry.span_id": {"stringValue": "f24e8631bd11faa7"}, + "sentry.timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.timestamp_precise": {"intValue": mock.ANY}, + "server.address": {"stringValue": "my-app-abc123.vercel.app"}, + "url.path": {"stringValue": "/api/users"}, + "vercel.deployment_id": { + "stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD" + }, + "vercel.entrypoint": {"stringValue": "api/index.js"}, + "vercel.execution_region": {"stringValue": "sfo1"}, + "vercel.id": {"stringValue": "1573817250283254651097202070"}, + "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, + "vercel.proxy.client_ip": {"stringValue": "120.75.16.101"}, + "vercel.proxy.host": {"stringValue": "my-app.vercel.app"}, + "vercel.proxy.method": {"stringValue": "GET"}, + "vercel.proxy.path": {"stringValue": "/api/users?page=1"}, + "vercel.proxy.referer": {"stringValue": "https://my-app.vercel.app"}, + "vercel.proxy.region": {"stringValue": "sfo1"}, + "vercel.proxy.scheme": {"stringValue": "https"}, + "vercel.proxy.status_code": {"intValue": "200"}, + "vercel.proxy.timestamp": {"intValue": "1573817250172"}, + "vercel.proxy.user_agent": {"stringValue": '["Mozilla/5.0..."]'}, + "vercel.proxy.vercel_cache": {"stringValue": "MISS"}, + "vercel.request_id": { + "stringValue": "643af4e3-975a-4cc7-9e7a-1eda11539d90" + }, + "vercel.source": {"stringValue": "lambda"}, + "vercel.status_code": {"intValue": "200"}, + }, + "clientSampleRate": 1.0, + "downsampledRetentionDays": 390, + "itemId": mock.ANY, + "itemType": "TRACE_ITEM_TYPE_LOG", + "organizationId": "1", + "projectId": "42", + "received": mock.ANY, + "retentionDays": 30, + "serverSampleRate": 1.0, + "timestamp": mock.ANY, + "traceId": "1b02cd14bb8642fd092bc23f54c7ffcd", + }, + ] + + # Check outcomes + outcomes = outcomes_consumer.get_aggregated_outcomes(n=2) + assert outcomes == [ + { + "category": DataCategory.LOG_ITEM.value, + "key_id": 123, + "org_id": 1, + "outcome": 0, + "project_id": 42, + "quantity": 2, + }, + { + "category": DataCategory.LOG_BYTE.value, + "key_id": 123, + "org_id": 1, + "outcome": 0, + "project_id": 42, + "quantity": mock.ANY, + }, + ] + + +def test_vercel_logs_ndjson( + mini_sentry, relay, relay_with_processing, outcomes_consumer, items_consumer +): + """Test Vercel logs ingestion with NDJSON format.""" + items_consumer = items_consumer() + outcomes_consumer = outcomes_consumer() + project_id = 42 + project_config = mini_sentry.add_full_project_config(project_id) + project_config["config"]["features"] = [ + "organizations:ourlogs-ingestion", + "organizations:relay-vercel-log-drain-endpoint", + ] + project_config["config"]["retentions"] = { + "log": {"standard": 30, "downsampled": 13 * 30}, + } + + relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) + + # Format as NDJSON + ndjson_payload = json.dumps(VERCEL_LOG_1) + "\n" + json.dumps(VERCEL_LOG_2) + "\n" + + relay.send_vercel_logs( + project_id, + data=ndjson_payload, + headers={"Content-Type": "application/x-ndjson"}, + ) + + # Check that the items are properly processed via items_consumer + items = items_consumer.get_items(n=2) + assert items == [ + { + "attributes": { + "sentry._meta.fields.trace_id": {"stringValue": mock.ANY}, + "sentry.body": {"stringValue": "Build completed successfully"}, + "sentry.browser.name": {"stringValue": "Python Requests"}, + "sentry.browser.version": {"stringValue": "2.32"}, + "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, + "sentry.payload_size_bytes": {"intValue": mock.ANY}, + "sentry.severity_text": {"stringValue": "info"}, + "sentry.timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.timestamp_precise": {"intValue": mock.ANY}, + "server.address": {"stringValue": "my-app-abc123.vercel.app"}, + "vercel.build_id": {"stringValue": "bld_cotnkcr76"}, + "vercel.deployment_id": { + "stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD" + }, + "vercel.id": {"stringValue": "1573817187330377061717300000"}, + "vercel.log_type": {"stringValue": "stdout"}, + "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, + "vercel.project_name": {"stringValue": "my-app"}, + "vercel.source": {"stringValue": "build"}, + }, + "clientSampleRate": 1.0, + "downsampledRetentionDays": 390, + "itemId": mock.ANY, + "itemType": "TRACE_ITEM_TYPE_LOG", + "organizationId": "1", + "projectId": "42", + "received": mock.ANY, + "retentionDays": 30, + "serverSampleRate": 1.0, + "timestamp": mock.ANY, + "traceId": mock.ANY, + }, + { + "attributes": { + "sentry.environment": {"stringValue": "production"}, + "sentry.body": {"stringValue": "API request processed"}, + "sentry.browser.name": {"stringValue": "Python Requests"}, + "sentry.browser.version": {"stringValue": "2.32"}, + "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, + "sentry.payload_size_bytes": {"intValue": mock.ANY}, + "sentry.severity_text": {"stringValue": "info"}, + "sentry.span_id": {"stringValue": "f24e8631bd11faa7"}, + "sentry.timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.timestamp_precise": {"intValue": mock.ANY}, + "server.address": {"stringValue": "my-app-abc123.vercel.app"}, + "url.path": {"stringValue": "/api/users"}, + "vercel.deployment_id": { + "stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD" + }, + "vercel.entrypoint": {"stringValue": "api/index.js"}, + "vercel.execution_region": {"stringValue": "sfo1"}, + "vercel.id": {"stringValue": "1573817250283254651097202070"}, + "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, + "vercel.proxy.client_ip": {"stringValue": "120.75.16.101"}, + "vercel.proxy.host": {"stringValue": "my-app.vercel.app"}, + "vercel.proxy.method": {"stringValue": "GET"}, + "vercel.proxy.path": {"stringValue": "/api/users?page=1"}, + "vercel.proxy.referer": {"stringValue": "https://my-app.vercel.app"}, + "vercel.proxy.region": {"stringValue": "sfo1"}, + "vercel.proxy.scheme": {"stringValue": "https"}, + "vercel.proxy.status_code": {"intValue": "200"}, + "vercel.proxy.timestamp": {"intValue": "1573817250172"}, + "vercel.proxy.user_agent": {"stringValue": '["Mozilla/5.0..."]'}, + "vercel.proxy.vercel_cache": {"stringValue": "MISS"}, + "vercel.request_id": { + "stringValue": "643af4e3-975a-4cc7-9e7a-1eda11539d90" + }, + "vercel.source": {"stringValue": "lambda"}, + "vercel.status_code": {"intValue": "200"}, + }, + "clientSampleRate": 1.0, + "downsampledRetentionDays": 390, + "itemId": mock.ANY, + "itemType": "TRACE_ITEM_TYPE_LOG", + "organizationId": "1", + "projectId": "42", + "received": mock.ANY, + "retentionDays": 30, + "serverSampleRate": 1.0, + "timestamp": mock.ANY, + "traceId": "1b02cd14bb8642fd092bc23f54c7ffcd", + }, + ] + + # Check outcomes + outcomes = outcomes_consumer.get_aggregated_outcomes(n=2) + assert outcomes == [ + { + "category": DataCategory.LOG_ITEM.value, + "key_id": 123, + "org_id": 1, + "outcome": 0, + "project_id": 42, + "quantity": 2, + }, + { + "category": DataCategory.LOG_BYTE.value, + "key_id": 123, + "org_id": 1, + "outcome": 0, + "project_id": 42, + "quantity": mock.ANY, + }, + ] From a3db2139f361a1d47f8a041db61c8d8d1f30b163 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 10 Oct 2025 13:53:41 -0700 Subject: [PATCH 2/8] store and use format for vercel logs --- .../src/endpoints/integrations/vercel.rs | 14 +++- relay-server/src/envelope/content_type.rs | 5 ++ relay-server/src/envelope/item.rs | 2 +- relay-server/src/integrations/mod.rs | 13 +++- .../src/processing/logs/integrations/mod.rs | 2 +- .../processing/logs/integrations/vercel.rs | 73 ++++++++++++------- 6 files changed, 77 insertions(+), 32 deletions(-) diff --git a/relay-server/src/endpoints/integrations/vercel.rs b/relay-server/src/endpoints/integrations/vercel.rs index fb4d003620f..142ccad5dd1 100644 --- a/relay-server/src/endpoints/integrations/vercel.rs +++ b/relay-server/src/endpoints/integrations/vercel.rs @@ -6,8 +6,9 @@ use relay_config::Config; use relay_dynamic_config::Feature; use crate::endpoints::common; -use crate::extractors::IntegrationBuilder; -use crate::integrations::LogsIntegration; +use crate::envelope::ContentType; +use crate::extractors::{IntegrationBuilder, RawContentType}; +use crate::integrations::{LogsIntegration, VercelLogDrainFormat}; use crate::service::ServiceState; /// All routes configured for the Vercel integration. @@ -24,11 +25,18 @@ mod logs { use super::*; async fn handle( + content_type: RawContentType, state: ServiceState, builder: IntegrationBuilder, ) -> axum::response::Result { + let format = match ContentType::from(content_type.as_ref()) { + ContentType::Json => VercelLogDrainFormat::Json, + ContentType::NDJson => VercelLogDrainFormat::NDJson, + _ => return Ok(StatusCode::UNSUPPORTED_MEDIA_TYPE), + }; + let envelope = builder - .with_type(LogsIntegration::VercelDrainLog) + .with_type(LogsIntegration::VercelDrainLog { format }) .with_required_feature(Feature::VercelLogDrainEndpoint) .build(); diff --git a/relay-server/src/envelope/content_type.rs b/relay-server/src/envelope/content_type.rs index 33d6fff3cc1..79181511423 100644 --- a/relay-server/src/envelope/content_type.rs +++ b/relay-server/src/envelope/content_type.rs @@ -15,6 +15,8 @@ pub enum ContentType { Text, /// `application/json` Json, + /// `application/x-ndjson` + NDJson, /// `application/x-msgpack` MsgPack, /// `application/octet-stream` @@ -45,6 +47,7 @@ impl ContentType { match self { Self::Text => "text/plain", Self::Json => "application/json", + Self::NDJson => "application/x-ndjson", Self::MsgPack => "application/x-msgpack", Self::OctetStream => "application/octet-stream", Self::Minidump => "application/x-dmp", @@ -74,6 +77,8 @@ impl ContentType { Some(Self::Text) } else if ct.eq_ignore_ascii_case(Self::Json.as_str()) { Some(Self::Json) + } else if ct.eq_ignore_ascii_case(Self::NDJson.as_str()) { + Some(Self::NDJson) } else if ct.eq_ignore_ascii_case(Self::MsgPack.as_str()) { Some(Self::MsgPack) } else if ct.eq_ignore_ascii_case(Self::OctetStream.as_str()) { diff --git a/relay-server/src/envelope/item.rs b/relay-server/src/envelope/item.rs index 5b5b255c804..6316708be52 100644 --- a/relay-server/src/envelope/item.rs +++ b/relay-server/src/envelope/item.rs @@ -151,7 +151,7 @@ impl Item { (DataCategory::LogByte, self.len().max(1)), (DataCategory::LogItem, item_count), ], - Some(Integration::Logs(LogsIntegration::VercelDrainLog)) => smallvec![ + Some(Integration::Logs(LogsIntegration::VercelDrainLog { .. })) => smallvec![ (DataCategory::LogByte, self.len().max(1)), (DataCategory::LogItem, item_count), ], diff --git a/relay-server/src/integrations/mod.rs b/relay-server/src/integrations/mod.rs index 8fab3c99937..cc0ff04246f 100644 --- a/relay-server/src/integrations/mod.rs +++ b/relay-server/src/integrations/mod.rs @@ -42,7 +42,8 @@ define_integrations!( "application/vnd.sentry.integration.otel.logs+protobuf" => Integration::Logs(LogsIntegration::OtelV1 { format: OtelFormat::Protobuf }), "application/vnd.sentry.integration.otel.spans+json" => Integration::Spans(SpansIntegration::OtelV1 { format: OtelFormat::Json }), "application/vnd.sentry.integration.otel.spans+protobuf" => Integration::Spans(SpansIntegration::OtelV1 { format: OtelFormat::Protobuf }), - "application/vnd.sentry.integration.vercel.logs+json" => Integration::Logs(LogsIntegration::VercelDrainLog), + "application/vnd.sentry.integration.vercel.logs+json" => Integration::Logs(LogsIntegration::VercelDrainLog { format: VercelLogDrainFormat::Json }), + "application/vnd.sentry.integration.vercel.logs+ndjson" => Integration::Logs(LogsIntegration::VercelDrainLog { format: VercelLogDrainFormat::NDJson }), ); /// An exhaustive list of all integrations supported by Relay. @@ -78,7 +79,7 @@ pub enum LogsIntegration { /// The Vercel Log Drain integration. /// /// Supports the [`relay_ourlogs::VercelLog`] format. - VercelDrainLog, + VercelDrainLog { format: VercelLogDrainFormat }, } /// All span integrations supported by Relay. @@ -98,3 +99,11 @@ pub enum OtelFormat { /// OTeL data in a JSON container. Json, } + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub enum VercelLogDrainFormat { + // Vercel Log Drain data in a json array payload + Json, + // Vercel Log Drain data in a newline delimited json payload + NDJson, +} diff --git a/relay-server/src/processing/logs/integrations/mod.rs b/relay-server/src/processing/logs/integrations/mod.rs index c3f405960ce..6fe60541156 100644 --- a/relay-server/src/processing/logs/integrations/mod.rs +++ b/relay-server/src/processing/logs/integrations/mod.rs @@ -46,7 +46,7 @@ pub fn expand_into( let result = match integration { LogsIntegration::OtelV1 { format } => otel::expand(format, payload, produce), - LogsIntegration::VercelDrainLog => vercel::expand(payload, produce), + LogsIntegration::VercelDrainLog { format } => vercel::expand(format, payload, produce), }; match result { diff --git a/relay-server/src/processing/logs/integrations/vercel.rs b/relay-server/src/processing/logs/integrations/vercel.rs index c1319a0b2dd..e93e1695299 100644 --- a/relay-server/src/processing/logs/integrations/vercel.rs +++ b/relay-server/src/processing/logs/integrations/vercel.rs @@ -1,15 +1,16 @@ use relay_event_schema::protocol::OurLog; use relay_ourlogs::VercelLog; +use crate::integrations::VercelLogDrainFormat; use crate::processing::logs::{Error, Result}; use crate::services::outcome::DiscardReason; /// Expands Vercel logs into the [`OurLog`] format. -pub fn expand(payload: &[u8], mut produce: F) -> Result<()> +pub fn expand(format: VercelLogDrainFormat, payload: &[u8], mut produce: F) -> Result<()> where F: FnMut(OurLog), { - let logs = parse_logs_data(payload)?; + let logs = parse_logs_data(format, payload)?; for log in logs { let ourlog = relay_ourlogs::vercel_log_to_sentry_log(log); @@ -19,29 +20,51 @@ where Ok(()) } -fn parse_logs_data(payload: &[u8]) -> Result> { - // Try parsing as JSON array first - if let Ok(logs) = serde_json::from_slice::>(payload) { - return Ok(logs); - } +fn parse_logs_data(format: VercelLogDrainFormat, payload: &[u8]) -> Result> { + match format { + VercelLogDrainFormat::Json => { + serde_json::from_slice::>(payload).map_err(|e| { + relay_log::debug!( + error = &e as &dyn std::error::Error, + "Failed to parse logs data as JSON" + ); + Error::Invalid(DiscardReason::InvalidJson) + }) + } + VercelLogDrainFormat::NDJson => { + let payload_str = std::str::from_utf8(payload).map_err(|e| { + relay_log::debug!( + error = &e as &dyn std::error::Error, + "Failed to parse logs data as UTF-8" + ); + Error::Invalid(DiscardReason::InvalidJson) + })?; - // Fall back to NDJSON parsing - let payload_str = std::str::from_utf8(payload).map_err(|e| { - relay_log::debug!( - error = &e as &dyn std::error::Error, - "Failed to parse logs data as UTF-8" - ); - Error::Invalid(DiscardReason::InvalidJson) - })?; - - let logs: Vec = payload_str - .lines() - .filter_map(|line| serde_json::from_str::(line.trim()).ok()) - .collect(); - - if logs.is_empty() { - relay_log::debug!("Failed to parse any logs from vercel log drain payload"); - } + let logs: Vec = payload_str + .lines() + .filter_map(|line| { + let trimmed = line.trim(); + if trimmed.is_empty() { + return None; + } + serde_json::from_str::(trimmed) + .map_err(|e| { + relay_log::debug!( + error = &e as &dyn std::error::Error, + line = trimmed, + "Failed to parse NDJSON line" + ); + }) + .ok() + }) + .collect(); - Ok(logs) + if logs.is_empty() { + relay_log::debug!("Failed to parse any logs from vercel log drain payload"); + return Err(Error::Invalid(DiscardReason::InvalidJson)); + } + + Ok(logs) + } + } } From 887920167d56e602f5cc637a7916abe52b596af4 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 10 Oct 2025 13:59:51 -0700 Subject: [PATCH 3/8] clean up expand for vercel logs --- .../processing/logs/integrations/vercel.rs | 70 +++++++------------ 1 file changed, 26 insertions(+), 44 deletions(-) diff --git a/relay-server/src/processing/logs/integrations/vercel.rs b/relay-server/src/processing/logs/integrations/vercel.rs index e93e1695299..b7ed8cad6d2 100644 --- a/relay-server/src/processing/logs/integrations/vercel.rs +++ b/relay-server/src/processing/logs/integrations/vercel.rs @@ -10,61 +10,43 @@ pub fn expand(format: VercelLogDrainFormat, payload: &[u8], mut produce: F) - where F: FnMut(OurLog), { - let logs = parse_logs_data(format, payload)?; + let mut count: i32 = 0; - for log in logs { - let ourlog = relay_ourlogs::vercel_log_to_sentry_log(log); - produce(ourlog) - } - - Ok(()) -} - -fn parse_logs_data(format: VercelLogDrainFormat, payload: &[u8]) -> Result> { match format { VercelLogDrainFormat::Json => { - serde_json::from_slice::>(payload).map_err(|e| { + let logs = serde_json::from_slice::>(payload).map_err(|e| { relay_log::debug!( error = &e as &dyn std::error::Error, "Failed to parse logs data as JSON" ); Error::Invalid(DiscardReason::InvalidJson) - }) - } - VercelLogDrainFormat::NDJson => { - let payload_str = std::str::from_utf8(payload).map_err(|e| { - relay_log::debug!( - error = &e as &dyn std::error::Error, - "Failed to parse logs data as UTF-8" - ); - Error::Invalid(DiscardReason::InvalidJson) })?; - let logs: Vec = payload_str - .lines() - .filter_map(|line| { - let trimmed = line.trim(); - if trimmed.is_empty() { - return None; - } - serde_json::from_str::(trimmed) - .map_err(|e| { - relay_log::debug!( - error = &e as &dyn std::error::Error, - line = trimmed, - "Failed to parse NDJSON line" - ); - }) - .ok() - }) - .collect(); - - if logs.is_empty() { - relay_log::debug!("Failed to parse any logs from vercel log drain payload"); - return Err(Error::Invalid(DiscardReason::InvalidJson)); + for log in logs { + count += 1; + let ourlog = relay_ourlogs::vercel_log_to_sentry_log(log); + produce(ourlog) + } + } + VercelLogDrainFormat::NDJson => { + for line in payload.split(|&b| b == b'\n') { + if line.is_empty() { + continue; + } + + if let Ok(log) = serde_json::from_slice::(line) { + count += 1; + let ourlog = relay_ourlogs::vercel_log_to_sentry_log(log); + produce(ourlog); + } } - - Ok(logs) } } + + if count == 0 { + relay_log::debug!("Failed to parse any logs from vercel log drain payload"); + return Err(Error::Invalid(DiscardReason::InvalidJson)); + } + + Ok(()) } From 488f6252dea7b672964a747a77f91553e2649c1b Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Fri, 10 Oct 2025 15:39:53 -0700 Subject: [PATCH 4/8] clean up integration tests --- tests/integration/fixtures/__init__.py | 1 + tests/integration/test_vercel_logs.py | 279 ++++++++----------------- 2 files changed, 92 insertions(+), 188 deletions(-) diff --git a/tests/integration/fixtures/__init__.py b/tests/integration/fixtures/__init__.py index cda2b3b2caa..1e396b8debd 100644 --- a/tests/integration/fixtures/__init__.py +++ b/tests/integration/fixtures/__init__.py @@ -274,6 +274,7 @@ def send_vercel_logs( response = self.post(url, headers=headers, data=data) response.raise_for_status() + return response def send_options(self, project_id, headers=None, dsn_key_idx=0): headers = { diff --git a/tests/integration/test_vercel_logs.py b/tests/integration/test_vercel_logs.py index 14087e94111..0047cfcd2e7 100644 --- a/tests/integration/test_vercel_logs.py +++ b/tests/integration/test_vercel_logs.py @@ -69,6 +69,93 @@ }, } +EXPECTED_ITEMS = [ + { + "organizationId": "1", + "projectId": "42", + "traceId": mock.ANY, + "itemId": mock.ANY, + "itemType": "TRACE_ITEM_TYPE_LOG", + "timestamp": mock.ANY, + "attributes": { + "vercel.id": {"stringValue": "1573817187330377061717300000"}, + "sentry.browser.version": {"stringValue": "2.32"}, + "sentry.timestamp_nanos": {"stringValue": "1573817187330000000"}, + "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, + "server.address": {"stringValue": "my-app-abc123.vercel.app"}, + "vercel.source": {"stringValue": "build"}, + "vercel.deployment_id": {"stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD"}, + "vercel.log_type": {"stringValue": "stdout"}, + "sentry.body": {"stringValue": "Build completed successfully"}, + "vercel.project_name": {"stringValue": "my-app"}, + "sentry.severity_text": {"stringValue": "info"}, + "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.timestamp_precise": {"intValue": "1573817187330000000"}, + "vercel.build_id": {"stringValue": "bld_cotnkcr76"}, + "sentry.payload_size_bytes": {"intValue": "436"}, + "sentry.browser.name": {"stringValue": "Python Requests"}, + "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, + "sentry._meta.fields.trace_id": { + "stringValue": '{"meta":{"":{"rem":[["trace_id.missing","s"]]}}}' + }, + }, + "clientSampleRate": 1.0, + "serverSampleRate": 1.0, + "retentionDays": 90, + "received": mock.ANY, + "downsampledRetentionDays": 90, + }, + { + "organizationId": "1", + "projectId": "42", + "traceId": "1b02cd14bb8642fd092bc23f54c7ffcd", + "itemId": mock.ANY, + "itemType": "TRACE_ITEM_TYPE_LOG", + "timestamp": mock.ANY, + "attributes": { + "url.path": {"stringValue": "/api/users"}, + "sentry.browser.version": {"stringValue": "2.32"}, + "vercel.proxy.scheme": {"stringValue": "https"}, + "vercel.entrypoint": {"stringValue": "api/index.js"}, + "vercel.proxy.user_agent": {"stringValue": '["Mozilla/5.0..."]'}, + "vercel.proxy.client_ip": {"stringValue": "120.75.16.101"}, + "server.address": {"stringValue": "my-app-abc123.vercel.app"}, + "vercel.proxy.path": {"stringValue": "/api/users?page=1"}, + "vercel.status_code": {"intValue": "200"}, + "vercel.deployment_id": {"stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD"}, + "vercel.proxy.method": {"stringValue": "GET"}, + "vercel.execution_region": {"stringValue": "sfo1"}, + "sentry.severity_text": {"stringValue": "info"}, + "sentry.span_id": {"stringValue": "f24e8631bd11faa7"}, + "sentry.browser.name": {"stringValue": "Python Requests"}, + "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, + "vercel.request_id": { + "stringValue": "643af4e3-975a-4cc7-9e7a-1eda11539d90" + }, + "vercel.proxy.host": {"stringValue": "my-app.vercel.app"}, + "vercel.proxy.referer": {"stringValue": "https://my-app.vercel.app"}, + "vercel.id": {"stringValue": "1573817250283254651097202070"}, + "sentry.environment": {"stringValue": "production"}, + "vercel.proxy.vercel_cache": {"stringValue": "MISS"}, + "sentry.timestamp_nanos": {"stringValue": "1573817250283000000"}, + "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, + "vercel.source": {"stringValue": "lambda"}, + "vercel.proxy.timestamp": {"intValue": "1573817250172"}, + "sentry.body": {"stringValue": "API request processed"}, + "vercel.proxy.status_code": {"intValue": "200"}, + "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, + "sentry.timestamp_precise": {"intValue": "1573817250283000000"}, + "sentry.payload_size_bytes": {"intValue": "886"}, + "vercel.proxy.region": {"stringValue": "sfo1"}, + }, + "clientSampleRate": 1.0, + "serverSampleRate": 1.0, + "retentionDays": 90, + "received": mock.ANY, + "downsampledRetentionDays": 90, + }, +] + def test_vercel_logs_json_array( mini_sentry, relay, relay_with_processing, outcomes_consumer, items_consumer @@ -82,9 +169,6 @@ def test_vercel_logs_json_array( "organizations:ourlogs-ingestion", "organizations:relay-vercel-log-drain-endpoint", ] - project_config["config"]["retentions"] = { - "log": {"standard": 30, "downsampled": 13 * 30}, - } relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) @@ -99,98 +183,9 @@ def test_vercel_logs_json_array( headers={"Content-Type": "application/json"}, ) - # Check that the items are properly processed via items_consumer items = items_consumer.get_items(n=2) - assert items == [ - { - "attributes": { - "sentry._meta.fields.trace_id": {"stringValue": mock.ANY}, - "sentry.body": {"stringValue": "Build completed successfully"}, - "sentry.browser.name": {"stringValue": "Python Requests"}, - "sentry.browser.version": {"stringValue": "2.32"}, - "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, - "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, - "sentry.payload_size_bytes": {"intValue": mock.ANY}, - "sentry.severity_text": {"stringValue": "info"}, - "sentry.timestamp_nanos": {"stringValue": mock.ANY}, - "sentry.timestamp_precise": {"intValue": mock.ANY}, - "server.address": {"stringValue": "my-app-abc123.vercel.app"}, - "vercel.build_id": {"stringValue": "bld_cotnkcr76"}, - "vercel.deployment_id": { - "stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD" - }, - "vercel.id": {"stringValue": "1573817187330377061717300000"}, - "vercel.log_type": {"stringValue": "stdout"}, - "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, - "vercel.project_name": {"stringValue": "my-app"}, - "vercel.source": {"stringValue": "build"}, - }, - "clientSampleRate": 1.0, - "downsampledRetentionDays": 390, - "itemId": mock.ANY, - "itemType": "TRACE_ITEM_TYPE_LOG", - "organizationId": "1", - "projectId": "42", - "received": mock.ANY, - "retentionDays": 30, - "serverSampleRate": 1.0, - "timestamp": mock.ANY, - "traceId": mock.ANY, - }, - { - "attributes": { - "sentry.environment": {"stringValue": "production"}, - "sentry.body": {"stringValue": "API request processed"}, - "sentry.browser.name": {"stringValue": "Python Requests"}, - "sentry.browser.version": {"stringValue": "2.32"}, - "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, - "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, - "sentry.payload_size_bytes": {"intValue": mock.ANY}, - "sentry.severity_text": {"stringValue": "info"}, - "sentry.span_id": {"stringValue": "f24e8631bd11faa7"}, - "sentry.timestamp_nanos": {"stringValue": mock.ANY}, - "sentry.timestamp_precise": {"intValue": mock.ANY}, - "server.address": {"stringValue": "my-app-abc123.vercel.app"}, - "url.path": {"stringValue": "/api/users"}, - "vercel.deployment_id": { - "stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD" - }, - "vercel.entrypoint": {"stringValue": "api/index.js"}, - "vercel.execution_region": {"stringValue": "sfo1"}, - "vercel.id": {"stringValue": "1573817250283254651097202070"}, - "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, - "vercel.proxy.client_ip": {"stringValue": "120.75.16.101"}, - "vercel.proxy.host": {"stringValue": "my-app.vercel.app"}, - "vercel.proxy.method": {"stringValue": "GET"}, - "vercel.proxy.path": {"stringValue": "/api/users?page=1"}, - "vercel.proxy.referer": {"stringValue": "https://my-app.vercel.app"}, - "vercel.proxy.region": {"stringValue": "sfo1"}, - "vercel.proxy.scheme": {"stringValue": "https"}, - "vercel.proxy.status_code": {"intValue": "200"}, - "vercel.proxy.timestamp": {"intValue": "1573817250172"}, - "vercel.proxy.user_agent": {"stringValue": '["Mozilla/5.0..."]'}, - "vercel.proxy.vercel_cache": {"stringValue": "MISS"}, - "vercel.request_id": { - "stringValue": "643af4e3-975a-4cc7-9e7a-1eda11539d90" - }, - "vercel.source": {"stringValue": "lambda"}, - "vercel.status_code": {"intValue": "200"}, - }, - "clientSampleRate": 1.0, - "downsampledRetentionDays": 390, - "itemId": mock.ANY, - "itemType": "TRACE_ITEM_TYPE_LOG", - "organizationId": "1", - "projectId": "42", - "received": mock.ANY, - "retentionDays": 30, - "serverSampleRate": 1.0, - "timestamp": mock.ANY, - "traceId": "1b02cd14bb8642fd092bc23f54c7ffcd", - }, - ] + assert items == EXPECTED_ITEMS - # Check outcomes outcomes = outcomes_consumer.get_aggregated_outcomes(n=2) assert outcomes == [ { @@ -207,7 +202,7 @@ def test_vercel_logs_json_array( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": mock.ANY, + "quantity": 1322, }, ] @@ -224,9 +219,6 @@ def test_vercel_logs_ndjson( "organizations:ourlogs-ingestion", "organizations:relay-vercel-log-drain-endpoint", ] - project_config["config"]["retentions"] = { - "log": {"standard": 30, "downsampled": 13 * 30}, - } relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) @@ -239,98 +231,9 @@ def test_vercel_logs_ndjson( headers={"Content-Type": "application/x-ndjson"}, ) - # Check that the items are properly processed via items_consumer items = items_consumer.get_items(n=2) - assert items == [ - { - "attributes": { - "sentry._meta.fields.trace_id": {"stringValue": mock.ANY}, - "sentry.body": {"stringValue": "Build completed successfully"}, - "sentry.browser.name": {"stringValue": "Python Requests"}, - "sentry.browser.version": {"stringValue": "2.32"}, - "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, - "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, - "sentry.payload_size_bytes": {"intValue": mock.ANY}, - "sentry.severity_text": {"stringValue": "info"}, - "sentry.timestamp_nanos": {"stringValue": mock.ANY}, - "sentry.timestamp_precise": {"intValue": mock.ANY}, - "server.address": {"stringValue": "my-app-abc123.vercel.app"}, - "vercel.build_id": {"stringValue": "bld_cotnkcr76"}, - "vercel.deployment_id": { - "stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD" - }, - "vercel.id": {"stringValue": "1573817187330377061717300000"}, - "vercel.log_type": {"stringValue": "stdout"}, - "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, - "vercel.project_name": {"stringValue": "my-app"}, - "vercel.source": {"stringValue": "build"}, - }, - "clientSampleRate": 1.0, - "downsampledRetentionDays": 390, - "itemId": mock.ANY, - "itemType": "TRACE_ITEM_TYPE_LOG", - "organizationId": "1", - "projectId": "42", - "received": mock.ANY, - "retentionDays": 30, - "serverSampleRate": 1.0, - "timestamp": mock.ANY, - "traceId": mock.ANY, - }, - { - "attributes": { - "sentry.environment": {"stringValue": "production"}, - "sentry.body": {"stringValue": "API request processed"}, - "sentry.browser.name": {"stringValue": "Python Requests"}, - "sentry.browser.version": {"stringValue": "2.32"}, - "sentry.observed_timestamp_nanos": {"stringValue": mock.ANY}, - "sentry.origin": {"stringValue": "auto.log_drain.vercel"}, - "sentry.payload_size_bytes": {"intValue": mock.ANY}, - "sentry.severity_text": {"stringValue": "info"}, - "sentry.span_id": {"stringValue": "f24e8631bd11faa7"}, - "sentry.timestamp_nanos": {"stringValue": mock.ANY}, - "sentry.timestamp_precise": {"intValue": mock.ANY}, - "server.address": {"stringValue": "my-app-abc123.vercel.app"}, - "url.path": {"stringValue": "/api/users"}, - "vercel.deployment_id": { - "stringValue": "dpl_233NRGRjVZX1caZrXWtz5g1TAksD" - }, - "vercel.entrypoint": {"stringValue": "api/index.js"}, - "vercel.execution_region": {"stringValue": "sfo1"}, - "vercel.id": {"stringValue": "1573817250283254651097202070"}, - "vercel.project_id": {"stringValue": "gdufoJxB6b9b1fEqr1jUtFkyavUU"}, - "vercel.proxy.client_ip": {"stringValue": "120.75.16.101"}, - "vercel.proxy.host": {"stringValue": "my-app.vercel.app"}, - "vercel.proxy.method": {"stringValue": "GET"}, - "vercel.proxy.path": {"stringValue": "/api/users?page=1"}, - "vercel.proxy.referer": {"stringValue": "https://my-app.vercel.app"}, - "vercel.proxy.region": {"stringValue": "sfo1"}, - "vercel.proxy.scheme": {"stringValue": "https"}, - "vercel.proxy.status_code": {"intValue": "200"}, - "vercel.proxy.timestamp": {"intValue": "1573817250172"}, - "vercel.proxy.user_agent": {"stringValue": '["Mozilla/5.0..."]'}, - "vercel.proxy.vercel_cache": {"stringValue": "MISS"}, - "vercel.request_id": { - "stringValue": "643af4e3-975a-4cc7-9e7a-1eda11539d90" - }, - "vercel.source": {"stringValue": "lambda"}, - "vercel.status_code": {"intValue": "200"}, - }, - "clientSampleRate": 1.0, - "downsampledRetentionDays": 390, - "itemId": mock.ANY, - "itemType": "TRACE_ITEM_TYPE_LOG", - "organizationId": "1", - "projectId": "42", - "received": mock.ANY, - "retentionDays": 30, - "serverSampleRate": 1.0, - "timestamp": mock.ANY, - "traceId": "1b02cd14bb8642fd092bc23f54c7ffcd", - }, - ] + assert items == EXPECTED_ITEMS - # Check outcomes outcomes = outcomes_consumer.get_aggregated_outcomes(n=2) assert outcomes == [ { @@ -347,6 +250,6 @@ def test_vercel_logs_ndjson( "org_id": 1, "outcome": 0, "project_id": 42, - "quantity": mock.ANY, + "quantity": 1322, }, ] From 04d7f39aaff4a901917836d10ff11dae81f54521 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 14 Oct 2025 10:36:01 -0400 Subject: [PATCH 5/8] use correct NdJson spelling --- relay-server/src/endpoints/integrations/vercel.rs | 2 +- relay-server/src/envelope/content_type.rs | 8 ++++---- relay-server/src/integrations/mod.rs | 4 ++-- relay-server/src/processing/logs/integrations/vercel.rs | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/relay-server/src/endpoints/integrations/vercel.rs b/relay-server/src/endpoints/integrations/vercel.rs index 142ccad5dd1..7e8372142b5 100644 --- a/relay-server/src/endpoints/integrations/vercel.rs +++ b/relay-server/src/endpoints/integrations/vercel.rs @@ -31,7 +31,7 @@ mod logs { ) -> axum::response::Result { let format = match ContentType::from(content_type.as_ref()) { ContentType::Json => VercelLogDrainFormat::Json, - ContentType::NDJson => VercelLogDrainFormat::NDJson, + ContentType::NdJson => VercelLogDrainFormat::NdJson, _ => return Ok(StatusCode::UNSUPPORTED_MEDIA_TYPE), }; diff --git a/relay-server/src/envelope/content_type.rs b/relay-server/src/envelope/content_type.rs index 79181511423..bd081fae019 100644 --- a/relay-server/src/envelope/content_type.rs +++ b/relay-server/src/envelope/content_type.rs @@ -16,7 +16,7 @@ pub enum ContentType { /// `application/json` Json, /// `application/x-ndjson` - NDJson, + NdJson, /// `application/x-msgpack` MsgPack, /// `application/octet-stream` @@ -47,7 +47,7 @@ impl ContentType { match self { Self::Text => "text/plain", Self::Json => "application/json", - Self::NDJson => "application/x-ndjson", + Self::NdJson => "application/x-ndjson", Self::MsgPack => "application/x-msgpack", Self::OctetStream => "application/octet-stream", Self::Minidump => "application/x-dmp", @@ -77,8 +77,8 @@ impl ContentType { Some(Self::Text) } else if ct.eq_ignore_ascii_case(Self::Json.as_str()) { Some(Self::Json) - } else if ct.eq_ignore_ascii_case(Self::NDJson.as_str()) { - Some(Self::NDJson) + } else if ct.eq_ignore_ascii_case(Self::NdJson.as_str()) { + Some(Self::NdJson) } else if ct.eq_ignore_ascii_case(Self::MsgPack.as_str()) { Some(Self::MsgPack) } else if ct.eq_ignore_ascii_case(Self::OctetStream.as_str()) { diff --git a/relay-server/src/integrations/mod.rs b/relay-server/src/integrations/mod.rs index cc0ff04246f..d76ecf3bd10 100644 --- a/relay-server/src/integrations/mod.rs +++ b/relay-server/src/integrations/mod.rs @@ -43,7 +43,7 @@ define_integrations!( "application/vnd.sentry.integration.otel.spans+json" => Integration::Spans(SpansIntegration::OtelV1 { format: OtelFormat::Json }), "application/vnd.sentry.integration.otel.spans+protobuf" => Integration::Spans(SpansIntegration::OtelV1 { format: OtelFormat::Protobuf }), "application/vnd.sentry.integration.vercel.logs+json" => Integration::Logs(LogsIntegration::VercelDrainLog { format: VercelLogDrainFormat::Json }), - "application/vnd.sentry.integration.vercel.logs+ndjson" => Integration::Logs(LogsIntegration::VercelDrainLog { format: VercelLogDrainFormat::NDJson }), + "application/vnd.sentry.integration.vercel.logs+ndjson" => Integration::Logs(LogsIntegration::VercelDrainLog { format: VercelLogDrainFormat::NdJson }), ); /// An exhaustive list of all integrations supported by Relay. @@ -105,5 +105,5 @@ pub enum VercelLogDrainFormat { // Vercel Log Drain data in a json array payload Json, // Vercel Log Drain data in a newline delimited json payload - NDJson, + NdJson, } diff --git a/relay-server/src/processing/logs/integrations/vercel.rs b/relay-server/src/processing/logs/integrations/vercel.rs index b7ed8cad6d2..0d530164b99 100644 --- a/relay-server/src/processing/logs/integrations/vercel.rs +++ b/relay-server/src/processing/logs/integrations/vercel.rs @@ -28,7 +28,7 @@ where produce(ourlog) } } - VercelLogDrainFormat::NDJson => { + VercelLogDrainFormat::NdJson => { for line in payload.split(|&b| b == b'\n') { if line.is_empty() { continue; From a5c492f04fe1fa47f7417f9a01a2cfe4598e60f1 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 14 Oct 2025 11:56:18 -0400 Subject: [PATCH 6/8] add reference --- relay-server/src/processing/logs/integrations/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay-server/src/processing/logs/integrations/mod.rs b/relay-server/src/processing/logs/integrations/mod.rs index b30c450b2f9..6ff3712608b 100644 --- a/relay-server/src/processing/logs/integrations/mod.rs +++ b/relay-server/src/processing/logs/integrations/mod.rs @@ -44,7 +44,7 @@ pub fn expand_into( let result = match integration { LogsIntegration::OtelV1 { format } => otel::expand(format, &payload, produce), - LogsIntegration::VercelDrainLog { format } => vercel::expand(format, payload, produce), + LogsIntegration::VercelDrainLog { format } => vercel::expand(format, &payload, produce), }; match result { From 84b22248446239d4e4cb99b2b62940f051621024 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 14 Oct 2025 12:08:20 -0400 Subject: [PATCH 7/8] remove default options --- tests/integration/test_vercel_logs.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/tests/integration/test_vercel_logs.py b/tests/integration/test_vercel_logs.py index 0047cfcd2e7..57cceb3b9bd 100644 --- a/tests/integration/test_vercel_logs.py +++ b/tests/integration/test_vercel_logs.py @@ -3,22 +3,6 @@ from sentry_relay.consts import DataCategory -TEST_CONFIG = { - "outcomes": { - "emit_outcomes": True, - "batch_size": 1, - "batch_interval": 1, - "aggregator": { - "bucket_interval": 1, - "flush_interval": 1, - }, - }, - "aggregator": { - "bucket_interval": 1, - "initial_delay": 0, - }, -} - # From Vercel Log Drain Docs: https://vercel.com/docs/drains/reference/logs#format VERCEL_LOG_1 = { "id": "1573817187330377061717300000", @@ -170,7 +154,7 @@ def test_vercel_logs_json_array( "organizations:relay-vercel-log-drain-endpoint", ] - relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) + relay = relay(relay_with_processing()) vercel_logs_payload = [ VERCEL_LOG_1, @@ -186,7 +170,7 @@ def test_vercel_logs_json_array( items = items_consumer.get_items(n=2) assert items == EXPECTED_ITEMS - outcomes = outcomes_consumer.get_aggregated_outcomes(n=2) + outcomes = outcomes_consumer.get_aggregated_outcomes(n=4) assert outcomes == [ { "category": DataCategory.LOG_ITEM.value, @@ -220,7 +204,7 @@ def test_vercel_logs_ndjson( "organizations:relay-vercel-log-drain-endpoint", ] - relay = relay(relay_with_processing(options=TEST_CONFIG), options=TEST_CONFIG) + relay = relay(relay_with_processing()) # Format as NDJSON ndjson_payload = json.dumps(VERCEL_LOG_1) + "\n" + json.dumps(VERCEL_LOG_2) + "\n" @@ -234,7 +218,7 @@ def test_vercel_logs_ndjson( items = items_consumer.get_items(n=2) assert items == EXPECTED_ITEMS - outcomes = outcomes_consumer.get_aggregated_outcomes(n=2) + outcomes = outcomes_consumer.get_aggregated_outcomes(n=4) assert outcomes == [ { "category": DataCategory.LOG_ITEM.value, From 797693a6058603f448cbb11fd13c7762fc88b6a1 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 15 Oct 2025 09:12:29 -0400 Subject: [PATCH 8/8] pr review --- relay-server/src/integrations/mod.rs | 4 ++-- relay-server/src/processing/logs/integrations/vercel.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/relay-server/src/integrations/mod.rs b/relay-server/src/integrations/mod.rs index d76ecf3bd10..e83d88ac394 100644 --- a/relay-server/src/integrations/mod.rs +++ b/relay-server/src/integrations/mod.rs @@ -102,8 +102,8 @@ pub enum OtelFormat { #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub enum VercelLogDrainFormat { - // Vercel Log Drain data in a json array payload + // Vercel Log Drain data in a JSON array payload Json, - // Vercel Log Drain data in a newline delimited json payload + // Vercel Log Drain data in a newline delimited JSON payload NdJson, } diff --git a/relay-server/src/processing/logs/integrations/vercel.rs b/relay-server/src/processing/logs/integrations/vercel.rs index 0d530164b99..fc50aa9f03f 100644 --- a/relay-server/src/processing/logs/integrations/vercel.rs +++ b/relay-server/src/processing/logs/integrations/vercel.rs @@ -25,7 +25,7 @@ where for log in logs { count += 1; let ourlog = relay_ourlogs::vercel_log_to_sentry_log(log); - produce(ourlog) + produce(ourlog); } } VercelLogDrainFormat::NdJson => {