Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 8 additions & 1 deletion crates/ourios-ingester/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@ ourios-semconv = { path = "../ourios-semconv" }
# carries the OTLP/JSON mapping the `application/json` path uses
# (RFC0003.6, lands next). Live gRPC/HTTP listeners (`tonic`/`axum`) are
# a later transport slice — this slice is the pure decode layer.
opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic-messages", "logs", "with-serde"] }
# `gen-tonic` (not just `-messages`) for the `logs_service_server`
# `LogsService` trait the gRPC listener implements (RFC0003.11/.15).

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in f4e1c86 — dropped the stale "later transport slice / pure decode layer" wording; the comment now just explains gen-tonic supplies the LogsService trait the gRPC listener implements.

opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "logs", "with-serde"] }
# `prost::Message::{decode, encode_to_vec}` for the protobuf payload.
# `std` is explicit (not relied on via workspace feature unification):
# `DecodeError::source` returns `prost::DecodeError` as `&dyn
Expand All @@ -61,6 +63,11 @@ axum = { version = "0.8", default-features = false, features = ["tokio", "http1"
# gzip decode for OTLP/HTTP `Content-Encoding: gzip` (an OTLP MUST,
# RFC0003.13). Pure-Rust backend — no zlib C dependency (CLAUDE.md).
flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
# gRPC transport (RFC 0003 §6.2): the `LogsService` server. `codegen` for
# the generated `#[async_trait]` + `Request`/`Response`/`Status`; no
# socket transport is stood up in this slice (the service is exercised
# in-process).
tonic = { version = "0.14", default-features = false, features = ["codegen"] }

[dev-dependencies]
tempfile = "3"
Expand Down
5 changes: 3 additions & 2 deletions crates/ourios-ingester/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@
//! RFC0003.7–.10), [`receiver::tenant`] (per-`ResourceLogs` tenant
//! derivation + fan-out, RFC0003.3/.4), [`receiver::pipeline`]
//! (§6.5 WAL-before-ack ingest path, RFC0003.1/.12), and
//! [`receiver::http`] (the OTLP/HTTP listener, RFC0003.11-HTTP/.13/.14).
//! The gRPC listener follows.
//! [`receiver::http`] (the OTLP/HTTP listener, RFC0003.11-HTTP/.13/.14),
//! and [`receiver::grpc`] (the OTLP/gRPC `LogsService`,
//! RFC0003.11-gRPC/.15). Only crash-before-ack (RFC0003.2) remains.
//! - **WAL-before-ack** (RFC 0008 / `CLAUDE.md` §3.4) — durability
//! before acknowledgement, via the shipped `ourios-wal`. Wired into
//! the ingest path by [`receiver::pipeline`]: every non-empty batch is
Expand Down
6 changes: 5 additions & 1 deletion crates/ourios-ingester/src/receiver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,13 @@
//! - [`http`] — the OTLP/HTTP listener ([`http::router`]) wrapping the
//! pipeline: `Content-Type`/`Content-Encoding` dispatch, controlled
//! transport errors, configurable path (RFC0003.11 HTTP arms / .13 /
//! .14). The gRPC listener (`tonic`) follows.
//! .14).
//! - [`grpc`] — the OTLP/gRPC `LogsService` ([`grpc::LogsReceiver`])
Comment on lines 22 to +26

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in f4e1c86 — the decode bullet no longer says "No live tonic/axum listener yet"; it now notes the http + grpc transports hand their decoded payload to the shared decode layer.

//! wrapping the same pipeline: controlled `Status` mapping + concurrent
//! WAL-before-ack (RFC0003.11 gRPC arms / .15).

pub mod decode;
pub mod grpc;
pub mod http;
pub mod materialize;
pub mod pipeline;
Expand Down
68 changes: 68 additions & 0 deletions crates/ourios-ingester/src/receiver/grpc.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
//! OTLP/gRPC listener (RFC 0003 §6.2).
//!
//! Implements `opentelemetry-proto`'s `LogsService` over the shared
//! [`IngestPipeline`]: `export` hands the (already tonic-decoded)

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 01c6c3f — these intra-doc links did not resolve from receiver::grpc (it imports SharedPipeline, not IngestPipeline). Made them plain code spans (the http module doc had the same broken link after the SharedPipeline move; fixed there too). cargo doc is now warning-free.

//! `ExportLogsServiceRequest` to the WAL-before-ack pipeline and maps the
//! result to a tonic `Status`:
//! - tenant-resolution failure → `INVALID_ARGUMENT` (naming the failing
//! `ResourceLogs` index + attribute, RFC0003.4/.11);
//! - WAL failure → `INTERNAL` (the batch was not acked, §3.4);
//! - success → an empty `ExportLogsServiceResponse`.
//!
//! Like the HTTP handler, the blocking `ingest` (WAL append + fsync) runs
//! via `spawn_blocking` so it doesn't stall the async runtime, and a
//! poisoned lock is recovered (the handler never panics).

use opentelemetry_proto::tonic::collector::logs::v1::logs_service_server::LogsService;
use opentelemetry_proto::tonic::collector::logs::v1::{
ExportLogsServiceRequest, ExportLogsServiceResponse,
};
use tonic::{Request, Response, Status};

use crate::receiver::http::SharedPipeline;
use crate::receiver::pipeline::ReceiveError;

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed — SharedPipeline moved to receiver::pipeline (alongside IngestPipeline) and re-exported from receiver; grpc no longer depends on http for it. Both transports import it from pipeline.


/// The gRPC `LogsService` over a shared [`IngestPipeline`].

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 01c6c3f — these intra-doc links did not resolve from receiver::grpc (it imports SharedPipeline, not IngestPipeline). Made them plain code spans (the http module doc had the same broken link after the SharedPipeline move; fixed there too). cargo doc is now warning-free.

pub struct LogsReceiver {
pipeline: SharedPipeline,
}

impl LogsReceiver {
/// Build a gRPC receiver over the shared pipeline.
#[must_use]
pub fn new(pipeline: SharedPipeline) -> Self {
Self { pipeline }
}
}

#[tonic::async_trait]
impl LogsService for LogsReceiver {
async fn export(
&self,
request: Request<ExportLogsServiceRequest>,
) -> Result<Response<ExportLogsServiceResponse>, Status> {
let export = request.into_inner();
let pipeline = self.pipeline.clone();
// Offload the blocking WAL append + fsync (mirrors the HTTP path).
let outcome = tokio::task::spawn_blocking(move || {
pipeline
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.ingest(export)
})
.await;

match outcome {
Ok(Ok(_)) => Ok(Response::new(ExportLogsServiceResponse::default())),
// A Resource that doesn't resolve to a tenant is a client
// error; the whole batch is rejected (RFC0003.4). The error's
// Display names the failing ResourceLogs index + attribute.
Ok(Err(ReceiveError::TenantResolution(e))) => {
Err(Status::invalid_argument(e.to_string()))
}
// A WAL failure (or a panic in the blocking task) is
// server-side; the batch was not acked (§3.4).
Ok(Err(_)) | Err(_) => Err(Status::internal("ingest failed")),

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed — the INTERNAL arm now distinguishes: a WAL append/sync failure surfaces its Display detail via Status::internal(e.to_string()) (AppendError/SyncError gained Display in #134), and a blocking-task join failure (panic/cancel) is a separate Status::internal("ingest task failed"). TenantResolution stays INVALID_ARGUMENT.

}
}
}
6 changes: 6 additions & 0 deletions crates/ourios-ingester/tests/ingest_support/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,12 @@ impl Journal for CapturingJournal {
}
}

/// A shared pipeline over a *real* `Wal` at `root` (for concurrency +
/// durability assertions; drop all clones before `replay_frames`).
pub fn shared_wal_pipeline(root: &Path) -> SharedPipeline {
Arc::new(Mutex::new(open_pipeline(root)))
}

/// A shared pipeline whose `Journal` captures appended payloads, plus the
/// capture handle.
pub fn capturing_pipeline() -> (SharedPipeline, Captured) {
Expand Down
107 changes: 96 additions & 11 deletions crates/ourios-ingester/tests/rfc0003_11_transport_errors.rs
Original file line number Diff line number Diff line change
@@ -1,18 +1,103 @@
//! RFC0003.11 — Transport-level errors are controlled, not panics.
//!
//! Red gate (`specified → red`): `#[ignore]`'d until the receiver
//! lands.
//! Two transports, two homes:
//! - **HTTP** error arms (malformed 400, bad/missing Content-Type 415,
//! corrupt gzip 400, oversize 413, wrong path 404, no WAL frame
//! appended) — `tests/http_transport_errors.rs`.
//! - **gRPC** arms — here: a tenant-resolution failure becomes a
//! controlled `INVALID_ARGUMENT` `Status` (not a panic), and a valid
//! request succeeds.
//!
//! On the "gRPC client cancellation mid-decode" arm: `tonic` decodes the
//! request *before* the handler runs, so an in-process direct call can't
//! reproduce a mid-decode cancellation. What this slice guarantees is the
//! testable part — `export` is a plain panic-free `async fn`, and because
//! `ingest` is atomic under the lock (append+fsync then ack), dropping
//! the response future leaves no partial WAL state. The socket-level
//! cancellation path is exercised when a real `tonic` server is served
//! (a follow-up); flagged to the maintainer as an OTLP/tonic nuance
//! rather than faked here.

mod ingest_support;

use ingest_support::{capturing_pipeline, request, resource_logs};
use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
use opentelemetry_proto::tonic::collector::logs::v1::logs_service_server::LogsService;
use opentelemetry_proto::tonic::common::v1::any_value::Value;
use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue};
use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs};
use opentelemetry_proto::tonic::resource::v1::Resource;
use ourios_ingester::receiver::grpc::LogsReceiver;
use tonic::{Code, Request};

/// A request whose single Resource lacks `service.name` (only `host.name`).
fn unresolvable_request() -> ExportLogsServiceRequest {
ExportLogsServiceRequest {
resource_logs: vec![ResourceLogs {
resource: Some(Resource {
attributes: vec![KeyValue {
key: "host.name".to_owned(),
value: Some(AnyValue {
value: Some(Value::StringValue("node-1".to_owned())),
}),
..Default::default()
}],
..Default::default()
}),
scope_logs: vec![ScopeLogs {
log_records: vec![LogRecord::default()],
..Default::default()
}],
..Default::default()
}],
}
}

/// Scenario RFC0003.11 — Transport-level errors are controlled, not panics.
/// See `docs/rfcs/0003-otlp-receiver.md` §5.
#[ignore = "RFC 0003 red gate — implementation pending (RFC0003.11)"]
#[test]
fn rfc0003_11_transport_errors_are_controlled() {
unimplemented!(
"RFC0003.11 — malformed protobuf, oversize body, unrecognised \
Content-Type, wrong path, and mid-decode gRPC cancellation each yield a \
controlled transport error (gRPC INVALID_ARGUMENT / RESOURCE_EXHAUSTED / \
CANCELLED; HTTP 400 / 413 / 415 / 404). No panic, process stays alive, \
and no OtlpBatch frame is appended."
#[tokio::test]
async fn rfc0003_11_grpc_tenant_failure_is_invalid_argument() {
// Arrange
let (pipeline, captured) = capturing_pipeline();
let receiver = LogsReceiver::new(pipeline);

// Act
let status = receiver
.export(Request::new(unresolvable_request()))
.await
.expect_err("an unresolvable Resource is rejected");

// Assert: a controlled INVALID_ARGUMENT naming the attribute — not a
// panic — and nothing appended.
assert_eq!(status.code(), Code::InvalidArgument);
assert!(
status.message().contains("service.name"),
"the Status names the missing attribute, got {:?}",
status.message(),
);
Comment on lines +70 to +81

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed — rfc0003_11 now also asserts the Status message contains ResourceLogs[0] (the failing index), covering the full RFC0003.4/.11 contract (index + attribute), not just the attribute.

assert!(
captured.lock().expect("captured").is_empty(),
"a rejected batch appends no OtlpBatch frame",
);
}

/// Scenario RFC0003.11 — a valid gRPC request succeeds (the handler never
/// panics on either path).
/// See `docs/rfcs/0003-otlp-receiver.md` §5.
#[tokio::test]
async fn rfc0003_11_grpc_valid_request_succeeds() {
let (pipeline, _) = capturing_pipeline();
let receiver = LogsReceiver::new(pipeline);

let response = receiver
.export(Request::new(request(vec![resource_logs(
"checkout",
&["x"],
)])))
.await;

assert!(
response.is_ok(),
"a valid request exports without error or panic"
);
}
Original file line number Diff line number Diff line change
@@ -1,17 +1,55 @@
//! RFC0003.15 — Concurrent `Export` calls each obey WAL-before-ack independently `[§3.4]`.
//! RFC0003.15 — Concurrent `Export` calls each obey WAL-before-ack independently.
//!
//! Red gate (`specified → red`): `#[ignore]`'d until the receiver
//! lands.
//! N concurrent gRPC `Export` calls share one pipeline (the single-writer
//! WAL serializes them behind the mutex). Each call appends + fsyncs its
//! own batch before acking, so all N succeed and the WAL ends with
//! exactly N durable `OtlpBatch` frames — none lost or interleaved away.

mod ingest_support;

use std::sync::Arc;

use ingest_support::{replay_frames, request, resource_logs, shared_wal_pipeline};
use opentelemetry_proto::tonic::collector::logs::v1::logs_service_server::LogsService;
use ourios_ingester::receiver::grpc::LogsReceiver;
use ourios_wal::FrameKind;
use tonic::Request;

const N: usize = 8;

/// Scenario RFC0003.15 — Concurrent `Export` calls each obey WAL-before-ack independently.
/// See `docs/rfcs/0003-otlp-receiver.md` §5.
#[ignore = "RFC 0003 red gate — implementation pending (RFC0003.15)"]
#[test]
fn rfc0003_15_concurrent_exports_each_ack_after_their_own_sync() {
unimplemented!(
"RFC0003.15 — N >= 2 concurrent gRPC Export calls from independent \
connections each emit their ack only after their OWN batch's Wal::sync \
returns Ok and their own batch's records are durable. A per-call ordering \
probe checks the invariant independently per in-flight call."
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn rfc0003_15_concurrent_exports_are_each_durable() {
// Arrange: one shared pipeline over a real WAL, N concurrent callers.
let tmp = tempfile::TempDir::new().expect("temp");
let receiver = Arc::new(LogsReceiver::new(shared_wal_pipeline(tmp.path())));

// Act: fire N concurrent Export calls.
let mut handles = Vec::with_capacity(N);
for i in 0..N {
let receiver = Arc::clone(&receiver);
handles.push(tokio::spawn(async move {
let body = format!("line {i}");
let export = request(vec![resource_logs("checkout", &[body.as_str()])]);
receiver.export(Request::new(export)).await
}));
}

// Assert: every call acked.
for handle in handles {
let response = handle.await.expect("task joins");
assert!(response.is_ok(), "each concurrent Export acks");
}

// ...and every batch is durable: the WAL holds exactly N OtlpBatch
// frames (none lost or dropped under concurrency). Drop the last
// pipeline ref so the WAL handle is released before replay.
drop(receiver);
let frames = replay_frames(tmp.path());
assert_eq!(frames.len(), N, "one durable frame per concurrent Export");
assert!(
frames.iter().all(|(kind, _)| *kind == FrameKind::OtlpBatch),
"every durable frame is an OtlpBatch",
);
}
Loading