prototype(protobuf): two-pass size estimator (+6.28x encode, -54% memory)

benchmarks/src/bench-comparison-protobufjs.ts

@@ -27,7 +27,7 @@
 // future protobuf-es changes can be tracked against a stable baseline.
 
 import { Bench } from "tinybench";
-import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
+import { create, toBinary, toBinaryFast, fromBinary } from "@bufbuild/protobuf";
 import { ExportTraceRequestSchema } from "./gen/nested_pb.js";
 import { SPAN_COUNT } from "./fixtures.js";
 
@@ -119,6 +119,14 @@ export async function runComparisonBench() {
     },
   );
 
+  bench.add(
+    `protobuf-es: create+toBinaryFast (${SPAN_COUNT} spans, OTel-like)`,
+    () => {
+      const msg = create(ExportTraceRequestSchema, initEs);
+      toBinaryFast(ExportTraceRequestSchema, msg);
+    },
+  );
+
   bench.add(
     `protobufjs: create+encode (${SPAN_COUNT} spans, OTel-like)`,
     () => {
@@ -135,6 +143,10 @@ export async function runComparisonBench() {
     toBinary(ExportTraceRequestSchema, esPrebuilt);
   });
 
+  bench.add(`protobuf-es: toBinaryFast pre-built (${SPAN_COUNT} spans)`, () => {
+    toBinaryFast(ExportTraceRequestSchema, esPrebuilt);
+  });
+
   bench.add(`protobufjs: encode pre-built (${SPAN_COUNT} spans)`, () => {
     ExportTraceRequestJs.encode(pbjsPrebuilt).finish();
   });

benchmarks/src/bench-create-toBinary.ts

@@ -18,7 +18,7 @@
 // of an OTLP trace export call made once per batch.
 
 import { Bench } from "tinybench";
-import { create, toBinary } from "@bufbuild/protobuf";
+import { create, toBinary, toBinaryFast } from "@bufbuild/protobuf";
 import { SimpleMessageSchema } from "./gen/small_pb.js";
 import {
   ExportTraceRequestSchema,
@@ -43,6 +43,15 @@ export async function runCreateToBinaryBench() {
     toBinary(SimpleMessageSchema, m);
   });
 
+  bench.add("create() + toBinaryFast() SimpleMessage", () => {
+    const m = create(SimpleMessageSchema, {
+      name: "bench-message",
+      value: 42,
+      enabled: true,
+    });
+    toBinaryFast(SimpleMessageSchema, m);
+  });
+
   bench.add(
     `create() + toBinary() ExportTraceRequest (${SPAN_COUNT} spans, OTel-like)`,
     () => {
@@ -85,6 +94,48 @@ export async function runCreateToBinaryBench() {
     },
   );
 
+  bench.add(
+    `create() + toBinaryFast() ExportTraceRequest (${SPAN_COUNT} spans, OTel-like)`,
+    () => {
+      const spans = [] as ReturnType<typeof create<typeof SpanSchema>>[];
+      for (let i = 0; i < SPAN_COUNT; i++) {
+        const attrs = [] as ReturnType<typeof create<typeof KeyValueSchema>>[];
+        for (let j = 0; j < 10; j++) {
+          attrs.push(
+            create(KeyValueSchema, {
+              key: `k${j}`,
+              stringValue: `v${i}-${j}`,
+            }),
+          );
+        }
+        spans.push(
+          create(SpanSchema, {
+            traceId: new Uint8Array(16),
+            spanId: new Uint8Array(8),
+            name: `span-${i}`,
+            startTimeUnixNano: 1_700_000_000_000_000_000n,
+            endTimeUnixNano: 1_700_000_000_000_001_000n,
+            attributes: attrs,
+          }),
+        );
+      }
+      const scope = create(InstrumentationScopeSchema, {
+        name: "@example/tracer",
+        version: "1.0.0",
+      });
+      const resource = create(ResourceSchema, { attributes: [] });
+      const req = create(ExportTraceRequestSchema, {
+        resourceSpans: [
+          create(ResourceSpansSchema, {
+            resource,
+            scopeSpans: [create(ScopeSpansSchema, { scope, spans })],
+          }),
+        ],
+      });
+      toBinaryFast(ExportTraceRequestSchema, req);
+    },
+  );
+
   await bench.run();
   return bench;
 }

benchmarks/src/bench-memory.ts

@@ -26,7 +26,7 @@
 //   (or)
 //   npm run bench:memory  (package.json wires --expose-gc)
 
-import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
+import { create, toBinary, toBinaryFast, fromBinary } from "@bufbuild/protobuf";
 import { ExportTraceRequestSchema } from "./gen/nested_pb.js";
 import { SPAN_COUNT } from "./fixtures.js";
 
@@ -152,6 +152,16 @@ async function main() {
     ),
   );
 
+  samples.push(
+    measure(
+      `protobuf-es: create + toBinaryFast (${SPAN_COUNT} spans)`,
+      () => {
+        const msg = create(ExportTraceRequestSchema, initEs);
+        toBinaryFast(ExportTraceRequestSchema, msg);
+      },
+    ),
+  );
+
   samples.push(
     measure(`protobufjs: create + encode (${SPAN_COUNT} spans)`, () => {
       const msg = ExportTraceRequestJs.create(initPbjs);

benchmarks/src/bench-toBinary.ts

@@ -17,7 +17,7 @@
 // reflects the reflective binary encoder cost in isolation.
 
 import { Bench } from "tinybench";
-import { toBinary } from "@bufbuild/protobuf";
+import { toBinary, toBinaryFast } from "@bufbuild/protobuf";
 import { SimpleMessageSchema } from "./gen/small_pb.js";
 import { ExportTraceRequestSchema } from "./gen/nested_pb.js";
 import {
@@ -43,6 +43,17 @@ export async function runToBinaryBench() {
     },
   );
 
+  bench.add("toBinaryFast() SimpleMessage (pre-built)", () => {
+    toBinaryFast(SimpleMessageSchema, small);
+  });
+
+  bench.add(
+    `toBinaryFast() ExportTraceRequest (pre-built, ${SPAN_COUNT} spans)`,
+    () => {
+      toBinaryFast(ExportTraceRequestSchema, traceRequest);
+    },
+  );
+
   await bench.run();
   return bench;
 }

benchmarks/src/verify-correctness.ts

@@ -0,0 +1,65 @@
+// Copyright 2021-2026 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Correctness check for the experimental `toBinaryFast` encoder.
+//
+// We don't claim byte-identical output against `toBinary` — repeated
+// scalar ordering and presence-zero handling could legitimately differ
+// on future descriptors. The load-bearing claim is *semantic* round-trip
+// equivalence: decoding either encoding produces structurally-equal
+// messages. This file exercises that on the OTel-shaped fixture used by
+// the benchmarks.
+
+import assert from "node:assert/strict";
+import { toBinary, toBinaryFast, fromBinary } from "@bufbuild/protobuf";
+import { ExportTraceRequestSchema } from "./gen/nested_pb.js";
+import { SimpleMessageSchema } from "./gen/small_pb.js";
+import { buildExportTraceRequest, buildSmallMessage } from "./fixtures.js";
+
+function summarize(label: string, slow: Uint8Array, fast: Uint8Array): void {
+  const byteMatch = slow.length === fast.length && slow.every((b, i) => b === fast[i]);
+  console.log(
+    `[${label}] slow=${slow.length}B fast=${fast.length}B bytesIdentical=${byteMatch}`,
+  );
+}
+
+// OTel-shaped ExportTraceRequest with 100 spans.
+{
+  const msg = buildExportTraceRequest();
+  const slow = toBinary(ExportTraceRequestSchema, msg);
+  const fast = toBinaryFast(ExportTraceRequestSchema, msg);
+
+  // Decode both — require structural equality of the resulting messages.
+  const decodedSlow = fromBinary(ExportTraceRequestSchema, slow);
+  const decodedFast = fromBinary(ExportTraceRequestSchema, fast);
+  assert.deepStrictEqual(
+    decodedFast,
+    decodedSlow,
+    "toBinaryFast produced a payload that decodes differently than toBinary",
+  );
+  summarize("ExportTraceRequest", slow, fast);
+}
+
+// SimpleMessage (scalars only): ensures the flat-scalar path works.
+{
+  const msg = buildSmallMessage();
+  const slow = toBinary(SimpleMessageSchema, msg);
+  const fast = toBinaryFast(SimpleMessageSchema, msg);
+  const decodedSlow = fromBinary(SimpleMessageSchema, slow);
+  const decodedFast = fromBinary(SimpleMessageSchema, fast);
+  assert.deepStrictEqual(decodedFast, decodedSlow);
+  summarize("SimpleMessage", slow, fast);
+}
+
+console.log("\nOK — semantic round-trip verified for all fixtures");

packages/protobuf/src/index.ts

@@ -23,6 +23,7 @@ export * from "./registry.js";
 export type { JsonValue, JsonObject } from "./json-value.js";
 export { toBinary } from "./to-binary.js";
 export type { BinaryWriteOptions } from "./to-binary.js";
+export { toBinaryFast } from "./to-binary-fast.js";
 export { fromBinary, mergeFromBinary } from "./from-binary.js";
 export type { BinaryReadOptions } from "./from-binary.js";
 export * from "./to-json.js";