diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore index 2a57ae800..d3f042269 100644 --- a/benchmarks/.gitignore +++ b/benchmarks/.gitignore @@ -2,3 +2,4 @@ src/gen src/gen-protobufjs node_modules dist +bench-results.json diff --git a/benchmarks/README.md b/benchmarks/README.md index 50931a49e..68f8023c7 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -24,6 +24,7 @@ npm run bench:create-toBinary -w @bufbuild/protobuf-benchmarks npm run bench:fromBinary -w @bufbuild/protobuf-benchmarks npm run bench:fromJson-path -w @bufbuild/protobuf-benchmarks npm run bench:comparison -w @bufbuild/protobuf-benchmarks +npm run bench:matrix -w @bufbuild/protobuf-benchmarks npm run bench:memory -w @bufbuild/protobuf-benchmarks ``` @@ -37,6 +38,7 @@ npm run bench:memory -w @bufbuild/protobuf-benchmarks | `bench-fromBinary.ts` | Cost of `fromBinary(Schema, bytes)` on pre-encoded payloads — reflective decoder walk in isolation | | `bench-fromJson-path.ts` | `fromJsonString + toBinary` and `fromJson + toBinary` paths on the same fixture. The first one is the #6221 regression shape; the second is the partial-fix midpoint | | `bench-comparison-protobufjs.ts` | Cross-library comparison: protobuf-es vs `protobufjs` (pbjs static codegen) on the same `.proto` fixture. Covers full roundtrip, encode-only, decode-only | +| `bench-matrix.ts` | `toBinary` + `fromBinary` across the full realistic-fixture matrix (OTel traces/metrics/logs, K8s Pod list, GraphQL request/response, RPC envelope, stress). Emits a JSON summary on stdout for CI diffing | | `bench-memory.ts` | Heap allocations per operation (`heapUsed` delta after forced GC) for both libraries. Requires `--expose-gc` | ## Methodology @@ -51,9 +53,76 @@ npm run bench:memory -w @bufbuild/protobuf-benchmarks ## Fixtures -The fixture in `proto/nested.proto` is a simplified subset of the [OTLP `ExportTraceServiceRequest`](https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/collector/trace/v1/trace_service.proto) — enough shape (bytes fields, fixed64 timestamps, repeated nested `KeyValue`, two levels of grouping) to be representative of a real export hot path without dragging in the full OpenTelemetry proto dependency graph. The default payload is 100 spans, each with 10 attributes. - -`proto/small.proto` is a 3-scalar-field message that isolates per-call overhead (create/toBinary) without allocation noise. +The suite runs across a matrix of payload shapes so a regression can be +attributed to a class of workload rather than lumped into a single "encoder +is slower" result. All fixtures live under `proto/` and are built by +helpers in `src/fixtures.ts`. + +| Fixture | `.proto` | Shape | Typical encoded size | Notes | +|---------|----------|-------|---------------------:|-------| +| `SimpleMessage` | `small.proto` | 3 scalar fields | ~19 B | per-call overhead baseline | +| `ExportTraceRequest` | `nested.proto` | OTel traces: 100 spans × 10 attrs, fixed64 timestamps, bytes IDs | ~35 KB | repro of [open-telemetry/opentelemetry-js#6221](https://github.com/open-telemetry/opentelemetry-js/issues/6221) | +| `ExportMetricsRequest` | `otel-metrics.proto` | OTel metrics: 50 series with Gauge/Sum/Histogram mix, explicit bucket bounds | ~17 KB | exercises the `oneof data` dispatch + repeated doubles/uint64s | +| `ExportLogsRequest` | `otel-logs.proto` | OTel logs: 100 LogRecords, severity, string body, trace/span IDs | ~21 KB | string-heavy with attribute maps | +| `K8sPodList` | `k8s-pod.proto` | 20 Pods with labels/annotations, 2 containers each, ports + env + resource limits | ~29 KB | map-dominant config payload | +| `GraphQLRequest` | `graphql.proto` | Long query string + JSON-encoded variables map | ~0.6 KB | mixes a large string with `map` | +| `GraphQLResponse` | `graphql.proto` | JSON-encoded `data` + structured errors | ~1.4 KB | bytes + repeated messages with string paths | +| `RpcRequest` | `rpc-simple.proto` | Routing fields + header map + 256-byte payload | ~0.5 KB | baseline RPC envelope | +| `RpcResponse` | `rpc-simple.proto` | Status + header map + 512-byte payload | ~0.6 KB | baseline RPC response | +| `StressMessage` | `stress.proto` | Depth-8 self-nested + 200-wide int32/string/attr arrays + 4KB blob + every scalar type | ~13 KB | synthetic — surfaces per-scalar-type regressions | + +### Design notes + +- **Map-heavy vs. list-heavy.** Kubernetes payloads stress `map` + encode paths; OTel payloads stress repeated nested messages. Both show up in + production consumers and the encoder walks differ. +- **Deep nesting.** The stress fixture recurses through `StressMessage.child` + eight levels deep. The encoder pays a length-prefix per level (fork buffer + + measure + prefix), so depth is a distinct failure mode from total size. +- **All scalar types exactly once.** `StressMessage` declares each proto3 + scalar in a fixed slot so a regression specific to `sfixed64` or `sint32` + varint zig-zag is visible in this fixture but not in realistic ones. +- **GraphQL/RPC payloads use `bytes` for opaque data** rather than structured + sub-messages because real clients carry JSON-encoded variables and + opaque RPC payloads as bytes on the wire; the benchmark reflects that. + +### Future work + +- `bench-matrix` currently measures `toBinary` + `fromBinary` on pre-built + messages. A follow-up pass should add `create + toBinary` (full roundtrip) + and `fromJsonString + toBinary` paths across the matrix to catch + regressions in the JSON-input code paths that the existing + `bench-fromJson-path` only exercises on the OTLP traces fixture. +- GraphQL variables are currently modelled as `map` with JSON + blobs per value. A richer fixture using a `google.protobuf.Struct`-like + shape would exercise the same code paths the `@bufbuild/protobuf/wkt` + `Value` type uses in real services. + +## Report snapshot + +Generated by `npm run bench:report -w @bufbuild/protobuf-benchmarks`. The +script writes `bench-results.json`, `chart.svg`, and the table below. See +`src/report.ts` for the generator and `src/report-helpers.ts` for the +rendering helpers. + +![chart](./chart.svg) + + + +| Fixture | Bytes | toBinary | toBinaryFast | protobufjs | Best | +| ---------------------------------- | -----: | -------: | -----------: | ---------: | -------------------- | +| SimpleMessage | 19 | 1.06M | 1.23M | - | toBinaryFast (1.16x) | +| ExportTraceRequest (100 spans) | 32,926 | 1,778 | 3,218 | 2,676 | toBinaryFast (1.20x) | +| ExportMetricsRequest (50 series) | 17,696 | 3,316 | 6,429 | - | toBinaryFast (1.94x) | +| ExportLogsRequest (100 records) | 21,319 | 3,257 | 6,032 | - | toBinaryFast (1.85x) | +| K8sPodList (20 pods) | 28,900 | 3,458 | 5,444 | - | toBinaryFast (1.57x) | +| GraphQLRequest | 624 | 256,431 | 312,765 | - | toBinaryFast (1.22x) | +| GraphQLResponse | 1,366 | 293,549 | 402,902 | - | toBinaryFast (1.37x) | +| RpcRequest | 501 | 379,472 | 455,529 | - | toBinaryFast (1.20x) | +| RpcResponse | 602 | 579,153 | 766,432 | - | toBinaryFast (1.32x) | +| StressMessage (depth=8, width=200) | 12,868 | 10,848 | 18,034 | - | toBinaryFast (1.66x) | + + ## Current results diff --git a/benchmarks/chart.svg b/benchmarks/chart.svg new file mode 100644 index 000000000..383423f26 --- /dev/null +++ b/benchmarks/chart.svg @@ -0,0 +1,132 @@ + + + + Encoder throughput by fixture (ops/sec, log scale) + + ops/sec (log10) + + + + 10 + + 100 + + 1K + + 10K + + 100K + + 1M + + 10M + + + toBinary: 1,064,677 ops/sec (SimpleMessage) + + + toBinaryFast: 1,230,371 ops/sec (SimpleMessage) + + + SimpleMessage + + + toBinary: 1,778 ops/sec (ExportTraceRequest (100 spans)) + + + toBinaryFast: 3,218 ops/sec (ExportTraceRequest (100 spans)) + + + protobufjs: 2,676 ops/sec (ExportTraceRequest (100 spans)) + + + ExportTraceRequest (100 spans) + + + toBinary: 3,316 ops/sec (ExportMetricsRequest (50 series)) + + + toBinaryFast: 6,429 ops/sec (ExportMetricsRequest (50 series)) + + + ExportMetricsRequest (50 series) + + + toBinary: 3,257 ops/sec (ExportLogsRequest (100 records)) + + + toBinaryFast: 6,032 ops/sec (ExportLogsRequest (100 records)) + + + ExportLogsRequest (100 records) + + + toBinary: 3,458 ops/sec (K8sPodList (20 pods)) + + + toBinaryFast: 5,444 ops/sec (K8sPodList (20 pods)) + + + K8sPodList (20 pods) + + + toBinary: 256,431 ops/sec (GraphQLRequest) + + + toBinaryFast: 312,765 ops/sec (GraphQLRequest) + + + GraphQLRequest + + + toBinary: 293,549 ops/sec (GraphQLResponse) + + + toBinaryFast: 402,902 ops/sec (GraphQLResponse) + + + GraphQLResponse + + + toBinary: 379,472 ops/sec (RpcRequest) + + + toBinaryFast: 455,529 ops/sec (RpcRequest) + + + RpcRequest + + + toBinary: 579,153 ops/sec (RpcResponse) + + + toBinaryFast: 766,432 ops/sec (RpcResponse) + + + RpcResponse + + + toBinary: 10,848 ops/sec (StressMessage (depth=8, width=200)) + + + toBinaryFast: 18,034 ops/sec (StressMessage (depth=8, width=200)) + + + StressMessage (depth=8, width=200) + + + + toBinary + + toBinaryFast + + protobufjs + + diff --git a/benchmarks/package.json b/benchmarks/package.json index a7aa80b56..55efeb1f6 100644 --- a/benchmarks/package.json +++ b/benchmarks/package.json @@ -10,10 +10,12 @@ "bench:fromJson-path": "tsx src/bench-fromJson-path.ts", "bench:fromBinary": "tsx src/bench-fromBinary.ts", "bench:comparison": "tsx src/bench-comparison-protobufjs.ts", + "bench:matrix": "tsx src/bench-matrix.ts", "bench:memory": "node --expose-gc --import tsx src/bench-memory.ts", + "bench:report": "tsx src/report.ts", "build": "../node_modules/typescript/bin/tsc --noEmit", "generate": "buf generate && npm run generate:protobufjs", - "generate:protobufjs": "pbjs -t static-module -w commonjs --force-long -o src/gen-protobufjs/nested.cjs proto/nested.proto", + "generate:protobufjs": "mkdir -p src/gen-protobufjs && pbjs -t static-module -w commonjs --force-long -o src/gen-protobufjs/nested.cjs proto/nested.proto", "format": "biome format --write", "license-header": "license-header", "lint": "biome lint --error-on-warnings" diff --git a/benchmarks/proto/graphql.proto b/benchmarks/proto/graphql.proto new file mode 100644 index 000000000..659daed93 --- /dev/null +++ b/benchmarks/proto/graphql.proto @@ -0,0 +1,57 @@ +// Copyright 2021-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package bench.v1; + +// GraphQL request/response envelope wrapped in protobuf. This is the shape +// of a GraphQL-over-gRPC gateway: `query` + `operation_name` + variables +// (stored as JSON-encoded bytes so the schema can carry arbitrary value +// types without leaking the GraphQL scalar system into protobuf). +// +// Benchmark value: large string payloads (query text), map +// (variables), and a nested error array with paths (repeated string). + +message GraphQLSourceLocation { + uint32 line = 1; + uint32 column = 2; +} + +message GraphQLError { + string message = 1; + repeated GraphQLSourceLocation locations = 2; + // GraphQL errors carry a JSON `path` that alternates string keys and + // int indices. We flatten to repeated string here (numeric indices are + // stringified) because protobuf doesn't have a heterogeneous list type + // and this matches what a typed client would serialize over the wire. + repeated string path = 3; + map extensions = 4; +} + +message GraphQLRequest { + string query = 1; + string operation_name = 2; + // Variables are JSON-encoded per key so that numbers, strings, nulls, + // and nested objects all fit without inventing a sum type. This + // matches the pattern used by apollo-rs and graphql-go gateways. + map variables = 3; + map extensions = 4; +} + +message GraphQLResponse { + // `data` is the top-level GraphQL `data` field, JSON-encoded. + bytes data = 1; + repeated GraphQLError errors = 2; + map extensions = 3; +} diff --git a/benchmarks/proto/k8s-pod.proto b/benchmarks/proto/k8s-pod.proto new file mode 100644 index 000000000..ae53e0ec2 --- /dev/null +++ b/benchmarks/proto/k8s-pod.proto @@ -0,0 +1,103 @@ +// Copyright 2021-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package bench.v1; + +// Subset of the Kubernetes core/v1 Pod object. Field numbers are NOT the +// upstream JSON-generated numbers (the canonical k8s API serializes as +// JSON, not protobuf); this is a plausible protobuf-shaped representation +// used to exercise map-heavy configuration payloads with the same shape +// kube-apiserver/kubelet would traffic over gRPC-equivalent channels. +// +// Why this shape matters for the benchmark: k8s payloads are map-dominant +// (labels, annotations, limits, requests), carry multiple small nested +// messages (containers, ports, env), and are a common integration target +// for protobuf-based RPC inside clusters. + +message K8sObjectMeta { + string name = 1; + string namespace = 2; + string uid = 5; + string resource_version = 6; + int64 generation = 17; + map labels = 11; + map annotations = 12; + fixed64 creation_timestamp_unix_nano = 8; +} + +message K8sEnvVar { + string name = 1; + string value = 2; +} + +message K8sPort { + string name = 1; + int32 container_port = 3; + string protocol = 4; +} + +message K8sResourceRequirements { + map limits = 1; + map requests = 2; +} + +message K8sContainer { + string name = 1; + string image = 2; + repeated string command = 3; + repeated string args = 4; + repeated K8sEnvVar env = 7; + repeated K8sPort ports = 6; + K8sResourceRequirements resources = 8; + string image_pull_policy = 14; +} + +message K8sPodSpec { + repeated K8sContainer containers = 2; + string restart_policy = 3; + string node_name = 10; + string service_account_name = 9; + int64 termination_grace_period_seconds = 4; +} + +message K8sContainerStatus { + string name = 1; + bool ready = 4; + int32 restart_count = 5; + string image = 6; + string image_id = 7; + string container_id = 8; + bool started = 10; +} + +message K8sPodStatus { + string phase = 1; + string pod_ip = 6; + string host_ip = 5; + fixed64 start_time_unix_nano = 7; + repeated K8sContainerStatus container_statuses = 8; +} + +message K8sPod { + K8sObjectMeta metadata = 1; + K8sPodSpec spec = 2; + K8sPodStatus status = 3; +} + +// Convenience list wrapper — exercises the repeated-of-complex-message +// path at the outer level. Matches how a PodList would be serialized. +message K8sPodList { + repeated K8sPod items = 2; +} diff --git a/benchmarks/proto/otel-logs.proto b/benchmarks/proto/otel-logs.proto new file mode 100644 index 000000000..1a984c75b --- /dev/null +++ b/benchmarks/proto/otel-logs.proto @@ -0,0 +1,78 @@ +// Copyright 2021-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package bench.v1; + +// Simplified subset of the OTLP logs export request. Modelled after +// opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest. Local +// AnyValue/KeyValue/Resource/Scope prefixed with `Log` to avoid name +// collision with nested.proto when compiled together. + +message LogAnyValue { + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + bytes bytes_value = 7; + } +} + +message LogKeyValue { + string key = 1; + LogAnyValue value = 2; +} + +message LogInstrumentationScope { + string name = 1; + string version = 2; + repeated LogKeyValue attributes = 3; +} + +message LogResource { + repeated LogKeyValue attributes = 1; +} + +// SeverityNumber replicates the OTLP severity enum as plain int32 — we +// avoid proto3 enums in the fixture because the fast path + protobufjs +// handling of unknown enum values complicates the comparison. +message LogRecord { + fixed64 time_unix_nano = 1; + fixed64 observed_time_unix_nano = 11; + int32 severity_number = 2; + string severity_text = 3; + LogAnyValue body = 5; + repeated LogKeyValue attributes = 6; + uint32 dropped_attributes_count = 7; + fixed32 flags = 8; + bytes trace_id = 9; + bytes span_id = 10; +} + +message ScopeLogs { + LogInstrumentationScope scope = 1; + repeated LogRecord log_records = 2; + string schema_url = 3; +} + +message ResourceLogs { + LogResource resource = 1; + repeated ScopeLogs scope_logs = 2; + string schema_url = 3; +} + +message ExportLogsRequest { + repeated ResourceLogs resource_logs = 1; +} diff --git a/benchmarks/proto/otel-metrics.proto b/benchmarks/proto/otel-metrics.proto new file mode 100644 index 000000000..2587397dd --- /dev/null +++ b/benchmarks/proto/otel-metrics.proto @@ -0,0 +1,116 @@ +// Copyright 2021-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package bench.v1; + +// Simplified subset of the OTLP metrics export request. Intentionally shaped +// after opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest +// to be representative of a real batched metric export call without pulling +// in the full dependency graph. Shares AnyValue / KeyValue / Resource / +// InstrumentationScope with nested.proto by re-declaring them locally — buf +// keeps each .proto self-contained here (the per-file `bench.v1` scope is +// preserved but message names are prefixed with `Metric` to avoid a +// collision with nested.proto when compiled together). + +message MetricAnyValue { + oneof value { + string string_value = 1; + bool bool_value = 2; + int64 int_value = 3; + double double_value = 4; + bytes bytes_value = 7; + } +} + +message MetricKeyValue { + string key = 1; + MetricAnyValue value = 2; +} + +message MetricInstrumentationScope { + string name = 1; + string version = 2; + repeated MetricKeyValue attributes = 3; +} + +message MetricResource { + repeated MetricKeyValue attributes = 1; +} + +// NumberDataPoint is the leaf type shared by Gauge and Sum. Matches the +// OTLP shape with a scalar oneof (`as_double` / `as_int`). +message NumberDataPoint { + repeated MetricKeyValue attributes = 7; + fixed64 start_time_unix_nano = 2; + fixed64 time_unix_nano = 3; + oneof value { + double as_double = 4; + int64 as_int = 6; + } +} + +// HistogramDataPoint mirrors the shape used by Prometheus-style histograms +// exported via OTLP — explicit bucket bounds + per-bucket counts. +message HistogramDataPoint { + repeated MetricKeyValue attributes = 9; + fixed64 start_time_unix_nano = 2; + fixed64 time_unix_nano = 3; + uint64 count = 4; + double sum = 5; + repeated uint64 bucket_counts = 6; + repeated double explicit_bounds = 7; + double min = 11; + double max = 12; +} + +message Gauge { + repeated NumberDataPoint data_points = 1; +} + +message Sum { + repeated NumberDataPoint data_points = 1; + int32 aggregation_temporality = 2; + bool is_monotonic = 3; +} + +message Histogram { + repeated HistogramDataPoint data_points = 1; + int32 aggregation_temporality = 2; +} + +message Metric { + string name = 1; + string description = 2; + string unit = 3; + oneof data { + Gauge gauge = 5; + Sum sum = 7; + Histogram histogram = 9; + } +} + +message ScopeMetrics { + MetricInstrumentationScope scope = 1; + repeated Metric metrics = 2; +} + +message ResourceMetrics { + MetricResource resource = 1; + repeated ScopeMetrics scope_metrics = 2; +} + +message ExportMetricsRequest { + repeated ResourceMetrics resource_metrics = 1; +} diff --git a/benchmarks/proto/rpc-simple.proto b/benchmarks/proto/rpc-simple.proto new file mode 100644 index 000000000..a6e3dd5b6 --- /dev/null +++ b/benchmarks/proto/rpc-simple.proto @@ -0,0 +1,40 @@ +// Copyright 2021-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package bench.v1; + +// RpcRequest/RpcResponse is the most common RPC envelope shape seen in +// production — a handful of routing fields + string-keyed headers + an +// opaque payload carried as bytes. This fixture measures the baseline +// per-call overhead: no deep nesting, a small map, a moderate-size +// bytes blob. The ratio of this vs. the OTLP workload tells us how much +// of the slowdown comes from per-message fixed costs vs. per-field walk. + +message RpcRequest { + string service = 1; + string method = 2; + map headers = 3; + bytes payload = 4; + fixed64 request_id = 5; + int64 deadline_ms = 6; +} + +message RpcResponse { + fixed64 request_id = 1; + int32 status_code = 2; + map headers = 3; + bytes payload = 4; + string error_message = 5; +} diff --git a/benchmarks/proto/stress.proto b/benchmarks/proto/stress.proto new file mode 100644 index 000000000..21c82b0f0 --- /dev/null +++ b/benchmarks/proto/stress.proto @@ -0,0 +1,68 @@ +// Copyright 2021-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; +package bench.v1; + +// Synthetic stress payload. Exists to exercise encoder edge cases that +// realistic fixtures don't always hit in the same message: +// - deep recursive nesting via `child` +// - wide repeated of scalars (int32) and strings +// - map-ish attribute list via repeated StressKeyValue +// - a large opaque string (`blob`) and bytes (`blob_bytes`) +// - every scalar type exactly once so we can attribute per-type cost +// +// Intentionally NOT representative of any production schema. Useful to +// detect type-specific regressions (e.g. a change that makes sfixed64 10x +// slower would show up here but not on OTLP-only benchmarks). + +message StressKeyValue { + string key = 1; + string value = 2; +} + +message StressMessage { + // Recursive nesting — fixture builder caps the depth. The encoder + // length-prefix walk pays O(depth) per iteration so deep nesting is + // an interesting failure mode. + StressMessage child = 1; + + // Wide repeated — scalar + message arrays. Numbers > 100 items are + // where packed-encoding path becomes dominant. + repeated int32 ids = 2; + repeated string tags = 3; + repeated StressKeyValue attrs = 4; + + // Large contiguous strings/bytes — exercise TextEncoder + raw copy. + string blob = 5; + bytes blob_bytes = 20; + + // Every proto3 scalar exactly once. Field numbers spaced to keep + // varint tag widths consistent (single-byte tags for the common + // hot-path fields 1-15, multi-byte for 16+). + int32 i32 = 10; + int64 i64 = 11; + uint32 u32 = 12; + uint64 u64 = 13; + sint32 s32 = 14; + sint64 s64 = 15; + bool b = 16; + float f32 = 17; + double f64 = 18; + string str = 19; + fixed32 fx32 = 21; + fixed64 fx64 = 22; + sfixed32 sfx32 = 23; + sfixed64 sfx64 = 24; +} diff --git a/benchmarks/src/bench-fromBinary.ts b/benchmarks/src/bench-fromBinary.ts index 8f17972ed..0bf787b4b 100644 --- a/benchmarks/src/bench-fromBinary.ts +++ b/benchmarks/src/bench-fromBinary.ts @@ -44,12 +44,9 @@ export async function runFromBinaryBench() { buildExportTraceRequest(), ); - bench.add( - `fromBinary() SimpleMessage (${smallBytes.byteLength} B)`, - () => { - fromBinary(SimpleMessageSchema, smallBytes); - }, - ); + bench.add(`fromBinary() SimpleMessage (${smallBytes.byteLength} B)`, () => { + fromBinary(SimpleMessageSchema, smallBytes); + }); bench.add( `fromBinary() ExportTraceRequest (${SPAN_COUNT} spans, ${traceBytes.byteLength} B)`, diff --git a/benchmarks/src/bench-matrix.ts b/benchmarks/src/bench-matrix.ts new file mode 100644 index 000000000..6266d6d78 --- /dev/null +++ b/benchmarks/src/bench-matrix.ts @@ -0,0 +1,220 @@ +// Copyright 2021-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Matrix benchmark runner. +// +// Runs each realistic fixture through the `toBinary` + `fromBinary` path +// and reports a combined table. Unlike bench-toBinary / bench-fromBinary +// which focus on a single shape in depth, bench-matrix is the "spread" +// view — useful for spotting whether a regression lands on one class of +// payloads (e.g. map-heavy k8s) vs another (e.g. deep nesting stress). +// +// Fixtures covered (see fixtures.ts and proto/ for the shapes): +// - SimpleMessage — 3 scalar fields, baseline per-call cost +// - ExportTraceRequest — OTel traces (existing fixture) +// - ExportMetricsRequest — OTel metrics: Gauge/Sum/Histogram mix +// - ExportLogsRequest — OTel logs: LogRecord batch +// - K8sPodList — Kubernetes: map-heavy configuration payload +// - GraphQLRequest — GraphQL query + variables (JSON-in-bytes) +// - GraphQLResponse — GraphQL response payload +// - RpcRequest/RpcResponse — baseline RPC envelope +// - StressMessage — synthetic: deep nesting + all scalar types +// +// Output format: tinybench table + JSON dump to stdout (for CI). Each row +// is a ` / ` pair so downstream tooling can diff across runs. + +import { Bench } from "tinybench"; +import { toBinary, fromBinary } from "@bufbuild/protobuf"; + +import { SimpleMessageSchema } from "./gen/small_pb.js"; +import { ExportTraceRequestSchema } from "./gen/nested_pb.js"; +import { ExportMetricsRequestSchema } from "./gen/otel-metrics_pb.js"; +import { ExportLogsRequestSchema } from "./gen/otel-logs_pb.js"; +import { K8sPodListSchema } from "./gen/k8s-pod_pb.js"; +import { + GraphQLRequestSchema, + GraphQLResponseSchema, +} from "./gen/graphql_pb.js"; +import { RpcRequestSchema, RpcResponseSchema } from "./gen/rpc-simple_pb.js"; +import { StressMessageSchema } from "./gen/stress_pb.js"; + +import { + buildSmallMessage, + buildExportTraceRequest, + buildExportMetricsRequest, + buildExportLogsRequest, + buildK8sPodList, + buildGraphQLRequest, + buildGraphQLResponse, + buildRpcRequest, + buildRpcResponse, + buildStressMessage, + SPAN_COUNT, + METRICS_SERIES_COUNT, + LOGS_RECORD_COUNT, + K8S_POD_COUNT, + STRESS_DEPTH, + STRESS_ARRAY_WIDTH, +} from "./fixtures.js"; + +// GenMessage is how protobuf-es ties a runtime schema to a message type. +// We accept `unknown` here and rely on tinybench to just call into the +// right callback; there's no actual type relationship that survives the +// matrix dispatch, so the inner cast is load-bearing. +// biome-ignore lint/suspicious/noExplicitAny: matrix dispatch is intentionally loose +type AnySchema = any; +// biome-ignore lint/suspicious/noExplicitAny: matrix dispatch is intentionally loose +type AnyMsg = any; + +interface MatrixCase { + name: string; + schema: AnySchema; + build: () => AnyMsg; + /** + * Short description of what makes this fixture distinctive — shown in + * the summary so downstream readers can correlate a regression row with + * the payload class without having to open fixtures.ts. + */ + shape: string; +} + +const cases: MatrixCase[] = [ + { + name: "SimpleMessage", + schema: SimpleMessageSchema, + build: () => buildSmallMessage(), + shape: "3 scalar fields, baseline per-call cost", + }, + { + name: `ExportTraceRequest (${SPAN_COUNT} spans)`, + schema: ExportTraceRequestSchema, + build: () => buildExportTraceRequest(), + shape: "OTel traces: repeated nested KeyValue + fixed64 timestamps", + }, + { + name: `ExportMetricsRequest (${METRICS_SERIES_COUNT} series)`, + schema: ExportMetricsRequestSchema, + build: () => buildExportMetricsRequest(), + shape: "OTel metrics: Gauge/Sum/Histogram oneof + buckets + bounds", + }, + { + name: `ExportLogsRequest (${LOGS_RECORD_COUNT} records)`, + schema: ExportLogsRequestSchema, + build: () => buildExportLogsRequest(), + shape: "OTel logs: LogRecord batch, string body, trace/span IDs", + }, + { + name: `K8sPodList (${K8S_POD_COUNT} pods)`, + schema: K8sPodListSchema, + build: () => buildK8sPodList(), + shape: + "map-heavy: labels, annotations, limits, requests + repeated containers", + }, + { + name: "GraphQLRequest", + schema: GraphQLRequestSchema, + build: () => buildGraphQLRequest(), + shape: "long query string + map variables", + }, + { + name: "GraphQLResponse", + schema: GraphQLResponseSchema, + build: () => buildGraphQLResponse(), + shape: "JSON-in-bytes data + errors with paths", + }, + { + name: "RpcRequest", + schema: RpcRequestSchema, + build: () => buildRpcRequest(), + shape: "baseline RPC envelope: small map + 256-byte payload", + }, + { + name: "RpcResponse", + schema: RpcResponseSchema, + build: () => buildRpcResponse(), + shape: "baseline RPC response: small map + 512-byte payload", + }, + { + name: `StressMessage (depth=${STRESS_DEPTH}, width=${STRESS_ARRAY_WIDTH})`, + schema: StressMessageSchema, + build: () => buildStressMessage(), + shape: "synthetic: deep nesting + all scalar types + 4KB blob", + }, +]; + +export async function runMatrixBench() { + const bench = new Bench({ time: 1000, warmupTime: 200 }); + + // Pre-build every message + pre-encoded bytes outside the measurement + // loop so the encoder/decoder benchmarks reflect the encode/decode walk + // in isolation. + const prepared = cases.map((c) => { + const msg = c.build(); + const bytes = toBinary(c.schema, msg); + return { ...c, msg, bytes }; + }); + + for (const p of prepared) { + bench.add(`${p.name} :: toBinary (pre-built, ${p.bytes.length} B)`, () => { + toBinary(p.schema, p.msg); + }); + } + for (const p of prepared) { + bench.add(`${p.name} :: fromBinary (${p.bytes.length} B)`, () => { + fromBinary(p.schema, p.bytes); + }); + } + + await bench.run(); + return { bench, prepared }; +} + +function summarize( + bench: Bench, +): Array<{ name: string; opsPerSec: number; rme: number; samples: number }> { + return bench.tasks.map((t) => ({ + name: t.name, + // tinybench exposes hz (ops/sec) + rme (%) + samples count on the task + // result. We serialize this to JSON so CI can diff runs deterministically. + opsPerSec: t.result?.hz ?? 0, + rme: t.result?.rme ?? 0, + samples: t.result?.samples.length ?? 0, + })); +} + +// Run standalone: `tsx src/bench-matrix.ts` +if (import.meta.url === `file://${process.argv[1]}`) { + const { bench, prepared } = await runMatrixBench(); + console.log("\n=== Matrix: encoded sizes ==="); + console.table( + prepared.map((p) => ({ + fixture: p.name, + bytes: p.bytes.length, + shape: p.shape, + })), + ); + console.log("\n=== Matrix: toBinary + fromBinary across fixtures ==="); + console.table(bench.table()); + + // Emit machine-readable JSON on the last line — consumable by CI diff + // tooling without having to scrape the tinybench table. + const payload = { + node: process.version, + platform: `${process.platform}/${process.arch}`, + timestamp: new Date().toISOString(), + results: summarize(bench), + }; + console.log("\n=== Matrix JSON ==="); + console.log(JSON.stringify(payload)); +} diff --git a/benchmarks/src/bench-memory.ts b/benchmarks/src/bench-memory.ts index 7e82d8811..0ab8588af 100644 --- a/benchmarks/src/bench-memory.ts +++ b/benchmarks/src/bench-memory.ts @@ -182,23 +182,17 @@ async function main() { const samples: MemSample[] = []; samples.push( - measure( - `protobuf-es: create + toBinary (${SPAN_COUNT} spans)`, - () => { - const msg = create(ExportTraceRequestSchema, initEs); - toBinary(ExportTraceRequestSchema, msg); - }, - ), + measure(`protobuf-es: create + toBinary (${SPAN_COUNT} spans)`, () => { + const msg = create(ExportTraceRequestSchema, initEs); + toBinary(ExportTraceRequestSchema, msg); + }), ); samples.push( - measure( - `protobuf-es: create + toBinaryFast (${SPAN_COUNT} spans)`, - () => { - const msg = create(ExportTraceRequestSchema, initEs); - toBinaryFast(ExportTraceRequestSchema, msg); - }, - ), + measure(`protobuf-es: create + toBinaryFast (${SPAN_COUNT} spans)`, () => { + const msg = create(ExportTraceRequestSchema, initEs); + toBinaryFast(ExportTraceRequestSchema, msg); + }), ); samples.push( diff --git a/benchmarks/src/fixtures.ts b/benchmarks/src/fixtures.ts index 01ed36c0c..402a15062 100644 --- a/benchmarks/src/fixtures.ts +++ b/benchmarks/src/fixtures.ts @@ -25,6 +25,65 @@ import { ResourceSchema, InstrumentationScopeSchema, } from "./gen/nested_pb.js"; +import { + ExportMetricsRequestSchema, + type ExportMetricsRequest, + GaugeSchema, + HistogramDataPointSchema, + HistogramSchema, + MetricAnyValueSchema, + MetricInstrumentationScopeSchema, + MetricKeyValueSchema, + MetricResourceSchema, + MetricSchema, + NumberDataPointSchema, + ResourceMetricsSchema, + ScopeMetricsSchema, + SumSchema, +} from "./gen/otel-metrics_pb.js"; +import { + ExportLogsRequestSchema, + type ExportLogsRequest, + LogAnyValueSchema, + LogInstrumentationScopeSchema, + LogKeyValueSchema, + LogRecordSchema, + LogResourceSchema, + ResourceLogsSchema, + ScopeLogsSchema, +} from "./gen/otel-logs_pb.js"; +import { + K8sContainerSchema, + K8sContainerStatusSchema, + K8sEnvVarSchema, + K8sObjectMetaSchema, + K8sPodListSchema, + type K8sPodList, + K8sPodSchema, + K8sPodSpecSchema, + K8sPodStatusSchema, + K8sPortSchema, + K8sResourceRequirementsSchema, +} from "./gen/k8s-pod_pb.js"; +import { + GraphQLErrorSchema, + GraphQLRequestSchema, + type GraphQLRequest, + GraphQLResponseSchema, + type GraphQLResponse, + GraphQLSourceLocationSchema, +} from "./gen/graphql_pb.js"; +import { + RpcRequestSchema, + type RpcRequest, + RpcResponseSchema, + type RpcResponse, +} from "./gen/rpc-simple_pb.js"; +import { + StressKeyValueSchema, + StressMessageSchema, + type StressMessage, +} from "./gen/stress_pb.js"; // Shared fixture construction. Kept deterministic so benchmark runs are // comparable: string lengths, attribute counts, and span counts are fixed. @@ -237,3 +296,499 @@ export function buildExportTraceRequestJsonShape() { ], }; } + +// --------------------------------------------------------------------------- +// Matrix fixtures — each builder is parameterized by a scale `n` so that the +// same shape can be measured at a few realistic cardinalities. Defaults are +// chosen to match what we see in production traffic for each payload class. +// --------------------------------------------------------------------------- + +// ---- OTel metrics --------------------------------------------------------- +// +// A batched metrics export: one Resource, one Scope, a mix of Gauge/Sum and +// a Histogram with explicit bucket bounds. `n` controls the number of +// distinct metric series; each series contributes a handful of data points. + +export const METRICS_SERIES_COUNT = 50; + +function buildMetricAttributes(metricIdx: number) { + const keys = [ + "service.name", + "service.version", + "deployment.environment", + "host.name", + "cloud.region", + ]; + return keys.map((key, i) => + create(MetricKeyValueSchema, { + key, + value: create(MetricAnyValueSchema, { + value: { case: "stringValue", value: `v-${metricIdx}-${i}` }, + }), + }), + ); +} + +function buildNumberDataPoint(idx: number, asDouble: boolean) { + return create(NumberDataPointSchema, { + attributes: buildMetricAttributes(idx), + startTimeUnixNano: 1_700_000_000_000_000_000n, + timeUnixNano: 1_700_000_000_000_001_000n + BigInt(idx) * 1000n, + value: asDouble + ? { case: "asDouble", value: 1.0 + idx * 0.125 } + : { case: "asInt", value: BigInt(idx * 17) }, + }); +} + +function buildHistogramDataPoint(idx: number) { + const buckets = [0, 1, 5, 10, 50, 100]; + return create(HistogramDataPointSchema, { + attributes: buildMetricAttributes(idx), + startTimeUnixNano: 1_700_000_000_000_000_000n, + timeUnixNano: 1_700_000_000_000_001_000n + BigInt(idx) * 1000n, + count: BigInt(100 + idx), + sum: 123.456 * idx, + bucketCounts: buckets.map((b) => BigInt(b + idx)), + explicitBounds: [0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0], + min: 0, + max: 9999.0, + }); +} + +export function buildExportMetricsRequest( + n: number = METRICS_SERIES_COUNT, +): ExportMetricsRequest { + const metrics = [] as ReturnType>[]; + for (let i = 0; i < n; i++) { + const kind = i % 3; + if (kind === 0) { + metrics.push( + create(MetricSchema, { + name: `metric.gauge.${i}`, + description: "gauge metric", + unit: "1", + data: { + case: "gauge", + value: create(GaugeSchema, { + dataPoints: [ + buildNumberDataPoint(i, true), + buildNumberDataPoint(i + 1, true), + ], + }), + }, + }), + ); + } else if (kind === 1) { + metrics.push( + create(MetricSchema, { + name: `metric.sum.${i}`, + description: "monotonic counter", + unit: "By", + data: { + case: "sum", + value: create(SumSchema, { + aggregationTemporality: 2, + isMonotonic: true, + dataPoints: [ + buildNumberDataPoint(i, false), + buildNumberDataPoint(i + 1, false), + ], + }), + }, + }), + ); + } else { + metrics.push( + create(MetricSchema, { + name: `metric.histogram.${i}`, + description: "request duration", + unit: "s", + data: { + case: "histogram", + value: create(HistogramSchema, { + aggregationTemporality: 2, + dataPoints: [buildHistogramDataPoint(i)], + }), + }, + }), + ); + } + } + const scope = create(MetricInstrumentationScopeSchema, { + name: "@example/metrics", + version: "1.0.0", + }); + const resource = create(MetricResourceSchema, { + attributes: buildMetricAttributes(0), + }); + return create(ExportMetricsRequestSchema, { + resourceMetrics: [ + create(ResourceMetricsSchema, { + resource, + scopeMetrics: [create(ScopeMetricsSchema, { scope, metrics })], + }), + ], + }); +} + +// ---- OTel logs ------------------------------------------------------------ +// +// Batched logs export with string body, severity, attributes, and trace +// correlation IDs. `n` controls the number of log records in the batch. + +export const LOGS_RECORD_COUNT = 100; + +function buildLogAttributes(recordIdx: number) { + const keys = ["code.namespace", "code.function", "thread.id", "log.source"]; + return keys.map((key, i) => + create(LogKeyValueSchema, { + key, + value: create(LogAnyValueSchema, { + value: { case: "stringValue", value: `v-${recordIdx}-${i}` }, + }), + }), + ); +} + +export function buildExportLogsRequest( + n: number = LOGS_RECORD_COUNT, +): ExportLogsRequest { + const records = [] as ReturnType>[]; + for (let i = 0; i < n; i++) { + records.push( + create(LogRecordSchema, { + timeUnixNano: 1_700_000_000_000_000_000n + BigInt(i) * 1000n, + observedTimeUnixNano: 1_700_000_000_000_001_000n + BigInt(i) * 1000n, + severityNumber: 9 + (i % 4), + severityText: ["INFO", "WARN", "ERROR", "DEBUG"][i % 4], + body: create(LogAnyValueSchema, { + value: { + case: "stringValue", + value: `log message #${i}: operation completed in ${i % 100}ms`, + }, + }), + attributes: buildLogAttributes(i), + droppedAttributesCount: 0, + flags: i & 0xff, + traceId: bytes(i, 16), + spanId: bytes(i + 1, 8), + }), + ); + } + const scope = create(LogInstrumentationScopeSchema, { + name: "@example/logger", + version: "1.0.0", + }); + const resource = create(LogResourceSchema, { + attributes: buildLogAttributes(0), + }); + return create(ExportLogsRequestSchema, { + resourceLogs: [ + create(ResourceLogsSchema, { + resource, + scopeLogs: [ + create(ScopeLogsSchema, { + scope, + logRecords: records, + schemaUrl: "", + }), + ], + schemaUrl: "", + }), + ], + }); +} + +// ---- K8s Pod list --------------------------------------------------------- +// +// Representative payload for a kubelet → apiserver listing call. Maps +// (labels, annotations, limits, requests) dominate; each pod has 2 +// containers with env vars, ports, resource requirements, and a few +// container statuses. + +export const K8S_POD_COUNT = 20; + +function buildK8sContainer(podIdx: number, containerIdx: number) { + return create(K8sContainerSchema, { + name: `container-${containerIdx}`, + image: `ghcr.io/example/app:v1.${podIdx}.${containerIdx}`, + command: ["/bin/app", "--config=/etc/app/config.yaml"], + args: ["--log-level=info", `--instance=pod-${podIdx}`], + env: [ + create(K8sEnvVarSchema, { name: "NODE_ENV", value: "production" }), + create(K8sEnvVarSchema, { name: "PORT", value: "8080" }), + create(K8sEnvVarSchema, { + name: "POD_NAME", + value: `example-pod-${podIdx}`, + }), + create(K8sEnvVarSchema, { name: "POD_NAMESPACE", value: "default" }), + ], + ports: [ + create(K8sPortSchema, { + name: "http", + containerPort: 8080, + protocol: "TCP", + }), + create(K8sPortSchema, { + name: "metrics", + containerPort: 9090, + protocol: "TCP", + }), + ], + resources: create(K8sResourceRequirementsSchema, { + limits: { cpu: "1000m", memory: "512Mi" }, + requests: { cpu: "100m", memory: "128Mi" }, + }), + imagePullPolicy: "IfNotPresent", + }); +} + +function buildK8sPod(i: number) { + const meta = create(K8sObjectMetaSchema, { + name: `example-pod-${i}`, + namespace: "default", + uid: `uid-${i.toString().padStart(8, "0")}`, + resourceVersion: `${100000 + i}`, + generation: BigInt(1), + labels: { + app: "example", + component: "api", + "app.kubernetes.io/name": "example", + "app.kubernetes.io/instance": `instance-${i}`, + tier: "backend", + }, + annotations: { + "prometheus.io/scrape": "true", + "prometheus.io/port": "9090", + "kubectl.kubernetes.io/last-applied-configuration": "{}", + }, + creationTimestampUnixNano: 1_700_000_000_000_000_000n + BigInt(i) * 1000n, + }); + const spec = create(K8sPodSpecSchema, { + containers: [buildK8sContainer(i, 0), buildK8sContainer(i, 1)], + restartPolicy: "Always", + nodeName: `node-${i % 5}.cluster.local`, + serviceAccountName: "default", + terminationGracePeriodSeconds: BigInt(30), + }); + const status = create(K8sPodStatusSchema, { + phase: "Running", + podIp: `10.0.${(i >> 8) & 0xff}.${i & 0xff}`, + hostIp: `10.1.0.${i % 255}`, + startTimeUnixNano: 1_700_000_000_000_000_000n + BigInt(i) * 1000n, + containerStatuses: [ + create(K8sContainerStatusSchema, { + name: "container-0", + ready: true, + restartCount: 0, + image: `ghcr.io/example/app:v1.${i}.0`, + imageId: `sha256:${"a".repeat(64)}`, + containerId: `containerd://${"b".repeat(64)}`, + started: true, + }), + create(K8sContainerStatusSchema, { + name: "container-1", + ready: true, + restartCount: 0, + image: `ghcr.io/example/app:v1.${i}.1`, + imageId: `sha256:${"c".repeat(64)}`, + containerId: `containerd://${"d".repeat(64)}`, + started: true, + }), + ], + }); + return create(K8sPodSchema, { metadata: meta, spec, status }); +} + +export function buildK8sPodList(n: number = K8S_POD_COUNT): K8sPodList { + const items = [] as ReturnType[]; + for (let i = 0; i < n; i++) { + items.push(buildK8sPod(i)); + } + return create(K8sPodListSchema, { items }); +} + +// ---- GraphQL -------------------------------------------------------------- +// +// A medium-size GraphQL request/response pair: long query string, several +// variables (JSON-encoded bytes), small response payload. Mirrors a typical +// authenticated GraphQL API call. + +const GRAPHQL_QUERY = ` + query GetUser($id: ID!, $includePosts: Boolean!, $postLimit: Int!) { + user(id: $id) { + id + email + displayName + avatarUrl + createdAt + lastSeenAt + posts(limit: $postLimit) @include(if: $includePosts) { + id + title + body + createdAt + tags + author { id displayName } + } + followers(first: 10) { + edges { node { id displayName } } + pageInfo { hasNextPage endCursor } + } + } + } +`.trim(); + +export function buildGraphQLRequest(): GraphQLRequest { + const encoder = new TextEncoder(); + return create(GraphQLRequestSchema, { + query: GRAPHQL_QUERY, + operationName: "GetUser", + variables: { + id: encoder.encode('"user-42"'), + includePosts: encoder.encode("true"), + postLimit: encoder.encode("25"), + }, + extensions: { + traceId: "00000000000000000000000000000000", + "x-client-version": "web/1.2.3", + }, + }); +} + +export function buildGraphQLResponse(): GraphQLResponse { + const encoder = new TextEncoder(); + // Smallish JSON response body — 2 KB-ish. + const data = encoder.encode( + JSON.stringify({ + user: { + id: "user-42", + email: "alice@example.com", + displayName: "Alice", + avatarUrl: "https://cdn.example.com/avatars/42.png", + createdAt: "2024-01-15T10:00:00Z", + lastSeenAt: "2026-04-19T09:30:00Z", + posts: Array.from({ length: 5 }, (_, i) => ({ + id: `post-${i}`, + title: `Example post ${i}`, + body: `Lorem ipsum dolor sit amet, consectetur ${i}`, + createdAt: "2025-01-01T00:00:00Z", + tags: ["news", "update"], + author: { id: "user-42", displayName: "Alice" }, + })), + }, + }), + ); + return create(GraphQLResponseSchema, { + data, + errors: [ + create(GraphQLErrorSchema, { + message: "deprecated field 'lastSeenAt' will be removed in v2", + locations: [ + create(GraphQLSourceLocationSchema, { line: 7, column: 7 }), + ], + path: ["user", "lastSeenAt"], + extensions: { + code: encoder.encode('"DEPRECATED_FIELD"'), + }, + }), + ], + extensions: { + traceId: encoder.encode('"00000000000000000000000000000000"'), + }, + }); +} + +// ---- RPC simple ----------------------------------------------------------- +// +// Baseline lightweight RPC envelope. Small headers map, modest payload, +// routing fields. This is the shape of a typical gRPC unary call; useful as +// a lower bound on per-call overhead. + +export function buildRpcRequest(): RpcRequest { + return create(RpcRequestSchema, { + service: "example.api.v1.UserService", + method: "GetUser", + headers: { + "x-request-id": "req-00000000-0000-0000-0000-000000000000", + authorization: "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", + "content-type": "application/grpc", + "grpc-accept-encoding": "identity,gzip", + }, + payload: new Uint8Array(256).fill(0xab), + requestId: 0x0123456789abcdefn, + deadlineMs: BigInt(5_000), + }); +} + +export function buildRpcResponse(): RpcResponse { + return create(RpcResponseSchema, { + requestId: 0x0123456789abcdefn, + statusCode: 0, + headers: { + "content-type": "application/grpc", + "grpc-status": "0", + "x-response-time-ms": "12", + }, + payload: new Uint8Array(512).fill(0xcd), + errorMessage: "", + }); +} + +// ---- Stress --------------------------------------------------------------- +// +// Synthetic payload with deep nesting, wide repeated fields, every scalar +// type exercised once, and a large opaque blob. Used to surface +// type-specific encoder regressions. + +export const STRESS_DEPTH = 8; +export const STRESS_ARRAY_WIDTH = 200; +export const STRESS_BLOB_SIZE = 4096; + +export function buildStressMessage( + depth: number = STRESS_DEPTH, + width: number = STRESS_ARRAY_WIDTH, + blobSize: number = STRESS_BLOB_SIZE, +): StressMessage { + const ids = new Array(width); + const tags = new Array(width); + const attrs = [] as ReturnType>[]; + for (let i = 0; i < width; i++) { + ids[i] = i; + tags[i] = `tag-${i}`; + attrs.push(create(StressKeyValueSchema, { key: `k${i}`, value: `v${i}` })); + } + const blob = "x".repeat(blobSize); + const blobBytes = new Uint8Array(blobSize).fill(0x42); + + // Build from the leaf up so we don't recurse through `create()` closures. + let current = create(StressMessageSchema, { + ids, + tags, + attrs, + blob, + blobBytes, + i32: -1, + i64: -1n, + u32: 0xffffffff >>> 0, + u64: 0xffffffffffffffffn, + s32: -2, + s64: -2n, + b: true, + f32: 3.14, + f64: Math.E, + str: "stress", + fx32: 0xdeadbeef >>> 0, + fx64: 0xcafebabedeadbeefn, + sfx32: -3, + sfx64: -3n, + }); + for (let i = 1; i < depth; i++) { + current = create(StressMessageSchema, { + child: current, + // Leave all other fields at defaults at intermediate levels to keep + // the message size growth bounded by depth rather than exploding. + }); + } + return current; +} diff --git a/benchmarks/src/report-helpers.ts b/benchmarks/src/report-helpers.ts new file mode 100644 index 000000000..48b4ff048 --- /dev/null +++ b/benchmarks/src/report-helpers.ts @@ -0,0 +1,399 @@ +// Copyright 2021-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Report helpers: markdown table generator, SVG chart builder, and README +// marker-based injector. Pattern lifted from packages/bundle-size/src/util.ts +// so two reports in this repo share a look-and-feel, but the inputs differ: +// bundle-size plots bytes vs file count as line series; we plot ops/sec as +// grouped bar charts keyed on fixture, with one bar per encoder variant. +// +// SVG is produced as a raw string, no external charting dependency. We +// stay at a fixed viewBox and compute x/y positions per bar so the output +// is stable under re-runs (barring genuine benchmark variance). + +import { readFileSync, writeFileSync } from "node:fs"; + +/** + * Single benchmark measurement. One row per (fixture × encoder) pair. + * `opsPerSec` is the median throughput reported by tinybench; `bytesPerOp` + * is the encoded size divided by one (we do not currently measure heap per + * op in this report — see bench-memory.ts for that). + */ +export interface BenchmarkResult { + fixture: string; + encoder: string; + opsPerSec: number; + bytesPerOp?: number; + encodedSize: number; +} + +/** + * Encoders we plot. Order matters — it drives the legend and bar ordering + * within a fixture group. Kept small and fixed so the chart is legible; + * when a new encoder is added, extend this array and the colors map. + */ +export const ENCODERS = ["toBinary", "toBinaryFast", "protobufjs"] as const; +export type Encoder = (typeof ENCODERS)[number]; + +export const ENCODER_COLORS: Record = { + toBinary: "#8b8b8b", + toBinaryFast: "#ffa600", + protobufjs: "#347fc4", +}; + +// --- Markdown table -------------------------------------------------------- + +/** + * Format an ops/sec number the way the tinybench tables in README.md do: + * three significant digits for the common ~100..10M range, thousand + * separators on the integer part. "-" for missing (encoder not applicable + * to this fixture). + */ +function formatOps(ops: number | undefined): string { + if (ops === undefined || ops === 0 || !Number.isFinite(ops)) return "-"; + if (ops >= 1_000_000) return `${(ops / 1_000_000).toFixed(2)}M`; + if (ops >= 10_000) + return new Intl.NumberFormat("en-US").format(Math.round(ops)); + if (ops >= 1000) + return new Intl.NumberFormat("en-US").format(Math.round(ops)); + return Math.round(ops).toString(); +} + +function formatBytes(bytes: number | undefined): string { + if (bytes === undefined || bytes === 0) return "-"; + return new Intl.NumberFormat("en-US").format(bytes); +} + +/** + * Best-ratio column summary. For each fixture we pick the fastest encoder + * and report " x vs ". Useful at-a-glance + * signal: if toBinary is the winner on every row, the fast path isn't + * helping; if protobufjs always wins, we still have ground to cover. + */ +function bestEncoderRatio(row: Record) { + const entries = ENCODERS.map( + (enc) => [enc, row[enc]?.opsPerSec ?? 0] as const, + ) + .filter(([, ops]) => ops > 0) + .sort((a, b) => b[1] - a[1]); + if (entries.length < 2) return "-"; + const [winner, winnerOps] = entries[0]; + const [, runnerUpOps] = entries[1]; + const ratio = winnerOps / runnerUpOps; + return `${winner} (${ratio.toFixed(2)}x)`; +} + +/** + * Group the flat result list by fixture, pick one row per encoder. Fixtures + * keep the order they were first encountered — the matrix preserves a + * meaningful layout (simple → complex → synthetic) that we do not want to + * re-sort alphabetically. + */ +function groupByFixture(results: BenchmarkResult[]): Array<{ + fixture: string; + encodedSize: number; + perEncoder: Record; +}> { + const order: string[] = []; + const groups = new Map< + string, + { + fixture: string; + encodedSize: number; + perEncoder: Record; + } + >(); + for (const r of results) { + let g = groups.get(r.fixture); + if (!g) { + g = { + fixture: r.fixture, + encodedSize: r.encodedSize, + perEncoder: { + toBinary: undefined, + toBinaryFast: undefined, + protobufjs: undefined, + }, + }; + groups.set(r.fixture, g); + order.push(r.fixture); + } + if ((ENCODERS as readonly string[]).includes(r.encoder)) { + g.perEncoder[r.encoder as Encoder] = r; + } + // Keep the largest observed encoded size — encoders produce wire- + // identical bytes for the same input, but if a future variant ever + // diverges this prevents an accidental zero. + if (r.encodedSize > g.encodedSize) g.encodedSize = r.encodedSize; + } + return order.flatMap((n) => { + const group = groups.get(n); + return group ? [group] : []; + }); +} + +/** + * Generate a markdown table with one row per fixture and one column per + * encoder (ops/sec). Extra columns: encoded bytes and the best-encoder + * ratio. Emitted between the README markers by `injectTable`. + */ +export function generateBenchmarkMarkdownTable( + results: BenchmarkResult[], +): string { + const groups = groupByFixture(results); + const header = [ + "Fixture", + "Bytes", + "toBinary", + "toBinaryFast", + "protobufjs", + "Best", + ]; + const rows: string[][] = groups.map((g) => [ + g.fixture, + formatBytes(g.encodedSize), + formatOps(g.perEncoder.toBinary?.opsPerSec), + formatOps(g.perEncoder.toBinaryFast?.opsPerSec), + formatOps(g.perEncoder.protobufjs?.opsPerSec), + bestEncoderRatio(g.perEncoder), + ]); + + const colWidths = header.map((h, i) => + Math.max(h.length, ...rows.map((r) => r[i].length)), + ); + + // Alignment: fixture left, everything else right — ops/sec and bytes + // read better right-aligned because they are variable-width numbers. + const align = ["left", "right", "right", "right", "right", "left"] as const; + + const pad = (s: string, w: number, a: (typeof align)[number]) => + a === "left" ? s.padEnd(w) : s.padStart(w); + + const sep = (w: number, a: (typeof align)[number]) => { + if (a === "left") return "-".repeat(w); + return `${"-".repeat(w - 1)}:`; + }; + + const lines: string[] = []; + lines.push( + `| ${header.map((h, i) => pad(h, colWidths[i], align[i])).join(" | ")} |`, + ); + lines.push(`| ${colWidths.map((w, i) => sep(w, align[i])).join(" | ")} |`); + for (const r of rows) { + lines.push( + `| ${r.map((c, i) => pad(c, colWidths[i], align[i])).join(" | ")} |`, + ); + } + return `\n${lines.join("\n")}\n\n`; +} + +// --- README injector ------------------------------------------------------- + +const TABLE_START = "\n"; +const TABLE_END = ""; + +/** + * Replace the content between the `` and + * `` markers with the provided table. If the + * markers are missing the function inserts a new section right after the + * top-level title so the first run of `bench:report` on a freshly + * authored README still works. + */ +export function injectTable(filePath: string, table: string): void { + const fileContent = readFileSync(filePath, "utf-8"); + const iStart = fileContent.indexOf(TABLE_START); + const iEnd = fileContent.indexOf(TABLE_END); + if (iStart < 0 || iEnd < 0) { + // Markers missing — append a new section so the README remains the + // canonical home for the table without a manual editing step. + const section = `\n## Report output\n\n${TABLE_START}${table}${TABLE_END}\n`; + writeFileSync(filePath, fileContent + section); + return; + } + const head = fileContent.substring(0, iStart + TABLE_START.length); + const foot = fileContent.substring(iEnd); + const newContent = head + table + foot; + if (newContent !== fileContent) { + writeFileSync(filePath, newContent); + } +} + +// --- SVG chart ------------------------------------------------------------- + +/** + * Log-base-10 of an ops/sec value, clamped at 0 so a missing / zero + * measurement does not blow up the axis or produce a negative bar. The + * report spans ~100..2M ops/sec across fixtures, so a log scale is the + * only way to keep SimpleMessage and ExportTraceRequest readable on the + * same chart. + */ +function log10Ops(ops: number): number { + if (!Number.isFinite(ops) || ops <= 1) return 0; + return Math.log10(ops); +} + +/** + * Escape characters that would break out of a node or attribute + * value. Fixtures can contain `&` via future naming — keep this resilient. + */ +function escapeXml(s: string): string { + return s + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +/** + * Render a grouped bar chart to SVG as a plain string. Each fixture is a + * group on the X axis; each encoder is a colored bar within the group. The + * Y axis is log(ops/sec). Dimensions are generous enough to fit long + * fixture names rotated 35 degrees without overlapping adjacent groups. + */ +export function generateBenchmarkChart(results: BenchmarkResult[]): string { + const groups = groupByFixture(results); + const n = groups.length; + + // Layout constants. `barWidth` is per-encoder; `groupWidth` is all + // three encoders plus a gap before the next fixture group. Changing + // any of these propagates downstream — they are the only magic numbers + // in this function. + const barWidth = 20; + const groupGap = 18; + const groupWidth = ENCODERS.length * barWidth + groupGap; + const marginLeft = 90; + const marginRight = 20; + const marginTop = 60; + const marginBottom = 150; + const chartHeight = 320; + const chartWidth = n * groupWidth; + const totalWidth = marginLeft + chartWidth + marginRight; + const totalHeight = marginTop + chartHeight + marginBottom; + + // Y axis: pick the smallest decade below the fastest encoder across all + // fixtures so the tallest bar reaches ~95% of the chart area. log10 + // grid lines every decade. + const maxOps = Math.max( + 1, + ...results.map((r) => (Number.isFinite(r.opsPerSec) ? r.opsPerSec : 0)), + ); + const yMaxLog = Math.ceil(log10Ops(maxOps)); + const yMinLog = 1; // 10 ops/sec floor — anything slower is not in scope. + const yRange = yMaxLog - yMinLog; + + const yToPixel = (opsLog: number) => { + const clamped = Math.max(yMinLog, Math.min(yMaxLog, opsLog)); + return ( + marginTop + chartHeight - ((clamped - yMinLog) / yRange) * chartHeight + ); + }; + + const parts: string[] = []; + parts.push( + `\n` + + `\n` + + ` \n`, + ); + + // Title + Y axis label (rotated so it runs vertically along the axis). + parts.push( + ` ` + + `Encoder throughput by fixture (ops/sec, log scale)` + + `\n`, + ); + parts.push( + ` \n` + + ` ops/sec (log10)\n` + + ` \n`, + ); + + // Y axis line + decade grid lines + labels. + parts.push( + ` \n`, + ); + for (let tick = yMinLog; tick <= yMaxLog; tick++) { + const y = yToPixel(tick); + const value = 10 ** tick; + const label = + value >= 1_000_000 + ? `${value / 1_000_000}M` + : value >= 1000 + ? `${value / 1000}K` + : `${value}`; + parts.push( + ` \n` + + ` ${label}\n`, + ); + } + + // X axis line. + parts.push( + ` \n`, + ); + + // Bars + fixture labels (rotated 35 degrees to keep long names legible). + for (let i = 0; i < groups.length; i++) { + const g = groups[i]; + const groupX = marginLeft + i * groupWidth + groupGap / 2; + for (let j = 0; j < ENCODERS.length; j++) { + const enc = ENCODERS[j]; + const r = g.perEncoder[enc]; + if (!r || !Number.isFinite(r.opsPerSec) || r.opsPerSec <= 0) continue; + const barX = groupX + j * barWidth; + const barTop = yToPixel(log10Ops(r.opsPerSec)); + const barH = marginTop + chartHeight - barTop; + const color = ENCODER_COLORS[enc]; + parts.push( + ` \n` + + ` ${escapeXml(enc)}: ${Math.round(r.opsPerSec).toLocaleString("en-US")} ops/sec (${g.fixture})\n` + + ` \n`, + ); + } + // Fixture label rotated to avoid clipping against neighbours. + const labelX = groupX + (ENCODERS.length * barWidth) / 2; + const labelY = marginTop + chartHeight + 12; + parts.push( + ` \n` + + ` ${escapeXml(g.fixture)}\n` + + ` \n`, + ); + } + + // Legend: a row of (swatch, label) pairs placed horizontally near the + // top of the chart area. Kept on one line — three encoders fit. + const legendY = marginTop - 24; + let legendX = marginLeft; + parts.push(` \n`); + for (const enc of ENCODERS) { + const color = ENCODER_COLORS[enc]; + parts.push( + ` \n` + + ` ${enc}\n`, + ); + legendX += 18 + enc.length * 7 + 16; + } + parts.push(` \n`); + + parts.push(`\n`); + return parts.join(""); +} diff --git a/benchmarks/src/report.ts b/benchmarks/src/report.ts new file mode 100644 index 000000000..4badd37b7 --- /dev/null +++ b/benchmarks/src/report.ts @@ -0,0 +1,324 @@ +// Copyright 2021-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Benchmark report generator. +// +// Runs a multi-encoder matrix (toBinary, toBinaryFast, protobufjs-where- +// generated) across the fixture set exposed by bench-matrix.ts, then emits: +// +// 1. bench-results.json — machine-readable raw data for CI diffing. +// 2. chart.svg — grouped-bar SVG chart (log ops/sec per fixture). +// 3. README.md — markdown table injected between the +// markers. +// +// Inspired by the packages/bundle-size/src/report.ts pattern: read the raw +// stats, pass them through table+chart generators, write files next to the +// README. Unlike bundle-size which bundles TypeScript via esbuild on every +// run, we run tinybench — so this script takes tens of seconds even at the +// reduced 600ms-per-case setting below. +// +// Usage: +// npm run bench:report -w @bufbuild/protobuf-benchmarks +// +// To re-render from an existing bench-results.json without re-running the +// benchmark (useful for iterating on the chart layout), set +// `BENCH_REPORT_READ_ONLY=1`. + +import { readFileSync, writeFileSync, existsSync } from "node:fs"; +import { createRequire } from "node:module"; +import { Bench } from "tinybench"; +import { toBinary, toBinaryFast } from "@bufbuild/protobuf"; + +import { SimpleMessageSchema } from "./gen/small_pb.js"; +import { ExportTraceRequestSchema } from "./gen/nested_pb.js"; +import { ExportMetricsRequestSchema } from "./gen/otel-metrics_pb.js"; +import { ExportLogsRequestSchema } from "./gen/otel-logs_pb.js"; +import { K8sPodListSchema } from "./gen/k8s-pod_pb.js"; +import { + GraphQLRequestSchema, + GraphQLResponseSchema, +} from "./gen/graphql_pb.js"; +import { RpcRequestSchema, RpcResponseSchema } from "./gen/rpc-simple_pb.js"; +import { StressMessageSchema } from "./gen/stress_pb.js"; + +import { + buildSmallMessage, + buildExportTraceRequest, + buildExportMetricsRequest, + buildExportLogsRequest, + buildK8sPodList, + buildGraphQLRequest, + buildGraphQLResponse, + buildRpcRequest, + buildRpcResponse, + buildStressMessage, + SPAN_COUNT, + METRICS_SERIES_COUNT, + LOGS_RECORD_COUNT, + K8S_POD_COUNT, + STRESS_DEPTH, + STRESS_ARRAY_WIDTH, +} from "./fixtures.js"; + +import { + type BenchmarkResult, + generateBenchmarkChart, + generateBenchmarkMarkdownTable, + injectTable, +} from "./report-helpers.js"; + +// biome-ignore lint/suspicious/noExplicitAny: matrix dispatch is loose by design +type AnySchema = any; +// biome-ignore lint/suspicious/noExplicitAny: matrix dispatch is loose by design +type AnyMsg = any; + +interface FixtureCase { + name: string; + schema: AnySchema; + build: () => AnyMsg; +} + +const cases: FixtureCase[] = [ + { + name: "SimpleMessage", + schema: SimpleMessageSchema, + build: buildSmallMessage, + }, + { + name: `ExportTraceRequest (${SPAN_COUNT} spans)`, + schema: ExportTraceRequestSchema, + build: buildExportTraceRequest, + }, + { + name: `ExportMetricsRequest (${METRICS_SERIES_COUNT} series)`, + schema: ExportMetricsRequestSchema, + build: buildExportMetricsRequest, + }, + { + name: `ExportLogsRequest (${LOGS_RECORD_COUNT} records)`, + schema: ExportLogsRequestSchema, + build: buildExportLogsRequest, + }, + { + name: `K8sPodList (${K8S_POD_COUNT} pods)`, + schema: K8sPodListSchema, + build: buildK8sPodList, + }, + { + name: "GraphQLRequest", + schema: GraphQLRequestSchema, + build: buildGraphQLRequest, + }, + { + name: "GraphQLResponse", + schema: GraphQLResponseSchema, + build: buildGraphQLResponse, + }, + { + name: "RpcRequest", + schema: RpcRequestSchema, + build: buildRpcRequest, + }, + { + name: "RpcResponse", + schema: RpcResponseSchema, + build: buildRpcResponse, + }, + { + name: `StressMessage (depth=${STRESS_DEPTH}, width=${STRESS_ARRAY_WIDTH})`, + schema: StressMessageSchema, + build: buildStressMessage, + }, +]; + +// protobufjs is only generated for nested.proto (the OTel traces shape), +// via the `generate:protobufjs` script. We still want it on the chart +// because it is the external baseline referenced in #6221; other fixtures +// simply leave the protobufjs bar missing, which the chart + table handle. +interface PbjsCtor { + create(properties: Record): Record; + encode(message: Record): { finish(): Uint8Array }; +} + +function loadPbjsExportTraceRequest(): PbjsCtor | null { + try { + const require = createRequire(import.meta.url); + // biome-ignore lint/suspicious/noExplicitAny: generated pbjs has dynamic shape + const mod = require("./gen-protobufjs/nested.cjs") as any; + return mod.bench.v1.ExportTraceRequest as PbjsCtor; + } catch { + // Missing codegen is non-fatal for the report — we just skip the + // protobufjs bar. Running `npm run generate:protobufjs` remedies this. + return null; + } +} + +/** + * Construct a plain-JS init object for protobufjs's `ExportTraceRequest`. + * Mirrors the init shape used in bench-comparison-protobufjs.ts, because + * protobufjs accepts oneof fields on the parent message directly rather + * than via the `{ case, value }` ADT protobuf-es uses — passing a + * protobuf-es init object into pbjs silently produces an empty encode. + */ +function buildPbjsOtelInit(): Record { + const spans: unknown[] = []; + for (let i = 0; i < SPAN_COUNT; i++) { + const attributes: unknown[] = []; + for (let j = 0; j < 10; j++) { + let anyValue: Record; + if (j === 2 || j === 5) { + anyValue = { intValue: (200 + j).toString() }; + } else if (j === 8) { + anyValue = { boolValue: (i + j) % 7 === 0 }; + } else { + anyValue = { stringValue: `v${i}-${j}` }; + } + attributes.push({ key: `k${j}`, value: anyValue }); + } + spans.push({ + traceId: new Uint8Array(16), + spanId: new Uint8Array(8), + name: `span-${i}`, + startTimeUnixNano: "1700000000000000000", + endTimeUnixNano: "1700000000000001000", + attributes, + }); + } + return { + resourceSpans: [ + { + resource: { + attributes: [], + labels: { + env: "production", + region: "us-east-1", + cluster: "bench-cluster", + }, + }, + scopeSpans: [ + { + scope: { name: "@example/tracer", version: "1.0.0" }, + spans, + }, + ], + }, + ], + }; +} + +/** + * Run the matrix and return flat per-(fixture × encoder) rows. + * + * tinybench `time`/`warmupTime` here are intentionally tighter than the + * main bench-matrix.ts (1000ms / 200ms) because the report runs 2–3x as + * many cases as the matrix and we want it to finish in a single + * development cycle. The noise floor is correspondingly higher; consumers + * of the raw numbers should use bench-matrix.ts, not the report file. + */ +async function runReportBench(): Promise { + const bench = new Bench({ time: 600, warmupTime: 150 }); + + const prepared = cases.map((c) => { + const msg = c.build(); + const bytes = toBinary(c.schema, msg); + return { ...c, msg, bytes }; + }); + + // protobuf-es encoders. We never change the schema/message references + // inside the benchmark function body — that would pull allocation cost + // into the measurement. Everything is captured in the closure once. + for (const p of prepared) { + bench.add(`${p.name} :: toBinary`, () => { + toBinary(p.schema, p.msg); + }); + bench.add(`${p.name} :: toBinaryFast`, () => { + toBinaryFast(p.schema, p.msg); + }); + } + + // protobufjs is only available for the OTel traces fixture. If the + // CommonJS module is missing we quietly skip, which is handled + // gracefully in the report output (table: "-", chart: no bar). + const pbjs = loadPbjsExportTraceRequest(); + const pbjsFixtureName = `ExportTraceRequest (${SPAN_COUNT} spans)`; + if (pbjs) { + const init = buildPbjsOtelInit(); + const preBuilt = pbjs.create(init); + bench.add(`${pbjsFixtureName} :: protobufjs`, () => { + pbjs.encode(preBuilt).finish(); + }); + } + + await bench.run(); + + // Flatten tinybench tasks into the BenchmarkResult shape. Task names + // use the " :: " separator we constructed above — any future formatting + // change must stay in sync here, which is why the split is load-bearing. + const results: BenchmarkResult[] = []; + for (const task of bench.tasks) { + const separator = " :: "; + const sepIdx = task.name.lastIndexOf(separator); + if (sepIdx < 0) continue; + const fixture = task.name.substring(0, sepIdx); + const encoder = task.name.substring(sepIdx + separator.length); + const prep = prepared.find((p) => p.name === fixture); + const encodedSize = prep ? prep.bytes.length : 0; + results.push({ + fixture, + encoder, + opsPerSec: task.result?.hz ?? 0, + encodedSize, + }); + } + return results; +} + +// --- main ------------------------------------------------------------------ + +const outDir = new URL("../", import.meta.url).pathname; +const resultsPath = `${outDir}bench-results.json`; +const chartPath = `${outDir}chart.svg`; +const readmePath = `${outDir}README.md`; + +let results: BenchmarkResult[]; +if (process.env.BENCH_REPORT_READ_ONLY === "1" && existsSync(resultsPath)) { + // Re-render mode: useful while iterating on chart / table layout so the + // author does not pay the ~30s benchmark cost for each rendering tweak. + const raw = JSON.parse(readFileSync(resultsPath, "utf-8")) as { + results: BenchmarkResult[]; + }; + results = raw.results; + console.log(`Loaded ${results.length} results from ${resultsPath}`); +} else { + console.log("Running benchmark matrix for report (this takes ~30s)..."); + results = await runReportBench(); + const payload = { + node: process.version, + platform: `${process.platform}/${process.arch}`, + timestamp: new Date().toISOString(), + results, + }; + writeFileSync(resultsPath, `${JSON.stringify(payload, null, 2)}\n`); + console.log(`Wrote ${resultsPath}`); +} + +// Build outputs. The chart and table see identical inputs, so any +// divergence between them is a layout bug in one of the generators. +const table = generateBenchmarkMarkdownTable(results); +injectTable(readmePath, table); +console.log(`Injected table into ${readmePath}`); + +const chart = generateBenchmarkChart(results); +writeFileSync(chartPath, chart); +console.log(`Wrote ${chartPath}`); diff --git a/benchmarks/src/verify-correctness.ts b/benchmarks/src/verify-correctness.ts index aa819e971..78287b56a 100644 --- a/benchmarks/src/verify-correctness.ts +++ b/benchmarks/src/verify-correctness.ts @@ -28,7 +28,8 @@ import { SimpleMessageSchema } from "./gen/small_pb.js"; import { buildExportTraceRequest, buildSmallMessage } from "./fixtures.js"; function summarize(label: string, slow: Uint8Array, fast: Uint8Array): void { - const byteMatch = slow.length === fast.length && slow.every((b, i) => b === fast[i]); + const byteMatch = + slow.length === fast.length && slow.every((b, i) => b === fast[i]); console.log( `[${label}] slow=${slow.length}B fast=${fast.length}B bytesIdentical=${byteMatch}`, ); diff --git a/packages/protobuf-test/src/to-binary-fast.test.ts b/packages/protobuf-test/src/to-binary-fast.test.ts index 4666f0377..9bb48630e 100644 --- a/packages/protobuf-test/src/to-binary-fast.test.ts +++ b/packages/protobuf-test/src/to-binary-fast.test.ts @@ -27,10 +27,7 @@ import { fromBinary, protoInt64, } from "@bufbuild/protobuf"; -import { - MapsMessageSchema, - MapsEnum, -} from "./gen/ts/extra/msg-maps_pb.js"; +import { MapsMessageSchema, MapsEnum } from "./gen/ts/extra/msg-maps_pb.js"; import { OneofMessageSchema, OneofMessageFooSchema, diff --git a/packages/protobuf/src/to-binary-fast.ts b/packages/protobuf/src/to-binary-fast.ts index 4ea562247..9709e5c48 100644 --- a/packages/protobuf/src/to-binary-fast.ts +++ b/packages/protobuf/src/to-binary-fast.ts @@ -410,7 +410,8 @@ function estimateMapEntryBody( ): { body: number; valueSubSize: number } { // Entry key is always field number 1. const keySize = - tagSize(1, scalarWireType(field.mapKey)) + scalarSize(field.mapKey, keyTyped); + tagSize(1, scalarWireType(field.mapKey)) + + scalarSize(field.mapKey, keyTyped); let valSize: number; let valueSubSize = 0; switch (field.mapKind) { @@ -913,12 +914,7 @@ function writeMessageInto( | Record | undefined; if (!obj || Object.keys(obj).length === 0) continue; - writeMapField( - c, - field as DescField & { fieldKind: "map" }, - obj, - sizes, - ); + writeMapField(c, field as DescField & { fieldKind: "map" }, obj, sizes); continue; }