diff --git a/experimental/CHANGELOG.md b/experimental/CHANGELOG.md index 914d6dc597e..b7ae0cec1a5 100644 --- a/experimental/CHANGELOG.md +++ b/experimental/CHANGELOG.md @@ -135,6 +135,7 @@ For notes on migrating to 2.x / 0.200.x see [the upgrade guide](doc/upgrade-to-2 * feat(opentelemetry-sdk-node): set instrumentation and propagators for experimental start [#6148](https://github.com/open-telemetry/opentelemetry-js/pull/6148) @maryliag * refactor(configuration): set console exporter as empty object [#6164](https://github.com/open-telemetry/opentelemetry-js/pull/6164) @maryliag * feat(instrumentation-http, instrumentation-fetch, instrumentation-xml-http-request): support "QUERY" as a known HTTP method +* feat(otlp-transformer): add custom protobuf logs serializer [#6228](https://github.com/open-telemetry/opentelemetry-js/pull/6228) @pichlermarc ### :bug: Bug Fixes diff --git a/experimental/packages/otlp-transformer/.eslintignore b/experimental/packages/otlp-transformer/.eslintignore index 345f1a599ed..9d6db6b9222 100644 --- a/experimental/packages/otlp-transformer/.eslintignore +++ b/experimental/packages/otlp-transformer/.eslintignore @@ -1,2 +1,3 @@ build src/generated +test/generated diff --git a/experimental/packages/otlp-transformer/.gitignore b/experimental/packages/otlp-transformer/.gitignore index 2cded78f303..782ee3b989f 100644 --- a/experimental/packages/otlp-transformer/.gitignore +++ b/experimental/packages/otlp-transformer/.gitignore @@ -1,3 +1,5 @@ src/generated/* +test/generated/* !src/generated/.gitkeep +!test/generated/.gitkeep !src/logs diff --git a/experimental/packages/otlp-transformer/package.json b/experimental/packages/otlp-transformer/package.json index 16278607658..fc8838a17ec 100644 --- a/experimental/packages/otlp-transformer/package.json +++ b/experimental/packages/otlp-transformer/package.json @@ -17,8 +17,8 @@ "compile": "tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json", "clean": "tsc --build --clean tsconfig.json tsconfig.esm.json tsconfig.esnext.json", "protos": "npm run submodule && npm run protos:generate", - "protos:generate:js": "pbjs -t static-module -p ./protos -w commonjs --null-defaults -o ./src/generated/root.js ./protos/opentelemetry/proto/common/v1/common.proto ./protos/opentelemetry/proto/resource/v1/resource.proto ./protos/opentelemetry/proto/trace/v1/trace.proto ./protos/opentelemetry/proto/collector/trace/v1/trace_service.proto ./protos/opentelemetry/proto/metrics/v1/metrics.proto ./protos/opentelemetry/proto/collector/metrics/v1/metrics_service.proto ./protos/opentelemetry/proto/logs/v1/logs.proto ./protos/opentelemetry/proto/collector/logs/v1/logs_service.proto", - "protos:generate:ts": "pbts -o ./src/generated/root.d.ts ./src/generated/root.js", + "protos:generate:js": "pbjs -t static-module -p ./protos -w commonjs --null-defaults -o ./src/generated/root.js ./protos/opentelemetry/proto/common/v1/common.proto ./protos/opentelemetry/proto/resource/v1/resource.proto ./protos/opentelemetry/proto/trace/v1/trace.proto ./protos/opentelemetry/proto/collector/trace/v1/trace_service.proto ./protos/opentelemetry/proto/metrics/v1/metrics.proto ./protos/opentelemetry/proto/collector/metrics/v1/metrics_service.proto ./protos/opentelemetry/proto/logs/v1/logs.proto ./protos/opentelemetry/proto/collector/logs/v1/logs_service.proto && pbjs -t static-module -p ./test/fixtures -w commonjs --null-defaults -o ./test/generated/testbed.js ./test/fixtures/testbed.proto", + "protos:generate:ts": "pbts -o ./src/generated/root.d.ts ./src/generated/root.js && pbts -o ./test/generated/testbed.d.ts ./test/generated/testbed.js", "protos:generate": "npm run protos:generate:js && npm run protos:generate:ts", "lint": "eslint . --ext .ts", "lint:fix": "eslint . --ext .ts --fix", @@ -65,6 +65,7 @@ "devDependencies": { "@opentelemetry/api": "1.9.0", "@types/mocha": "10.0.10", + "@types/sinon": "^17.0.4", "@types/webpack-env": "1.16.3", "babel-plugin-istanbul": "7.0.1", "karma": "6.4.4", diff --git a/experimental/packages/otlp-transformer/src/common/protobuf/common-serializer.ts b/experimental/packages/otlp-transformer/src/common/protobuf/common-serializer.ts new file mode 100644 index 00000000000..e3b0ca176fc --- /dev/null +++ b/experimental/packages/otlp-transformer/src/common/protobuf/common-serializer.ts @@ -0,0 +1,189 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ +import type { Attributes, HrTime } from '@opentelemetry/api'; +import type { AnyValue, LogAttributes } from '@opentelemetry/api-logs'; +import type { IProtobufWriter } from './i-protobuf-writer'; + +/** + * Write HrTime [seconds, nanoseconds] directly as fixed64 to the serializer. + * Converts to nanoseconds and writes as 64-bit little-endian integer without allocations. + * + * HrTime represents: total_nanos = seconds * 1_000_000_000 + nanoseconds + * We need to split this into low (bits 0-31) and high (bits 32-63). + * + * @param serializer - The protobuf writer + * @param hrTime - HrTime tuple [seconds, nanoseconds] + */ +export function writeHrTimeAsFixed64( + serializer: IProtobufWriter, + hrTime: HrTime +): void { + const seconds = hrTime[0]; + const nanos = hrTime[1]; + + // Calculate total nanoseconds (seconds * 1_000_000_000 + nanos) split into [low32, high32]. + // + // We cannot use `seconds * 1e9` directly because for Unix timestamps + // (seconds > ~9_007_199) the product exceeds Number.MAX_SAFE_INTEGER and loses + // integer precision in IEEE 754 double arithmetic. + // + // So we split `seconds` into its lower 16 bits and remaining upper bits so + // every multiplication stays + // within the safe-integer range (< 2^53): + // secondsLower16Bits * 1e9 <= 65535 * 1e9 ≈ 6.55e13 < 2^53 + // secondsUpperBits * 1e9 <= 65535 * 1e9 ≈ 6.55e13 < 2^53 + // + // Then recombine: + // seconds * 1e9 = + // secondsUpperBits * 1e9 * 2^16 + + // secondsLower16Bits * 1e9 + + const nanosPerSecond = 1_000_000_000; + // Split `seconds` into low 16 bits and the remaining upper bits. + // This avoids the signed 32-bit seconds limit behind the Year 2038 problem: + // the practical limit here is the encoded fixed64 range. Callers must keep + // `seconds * 1e9 + nanos` within uint64, i.e. up to + // [18_446_744_073, 709_551_615]. Beyond that, the serialized value wraps. + const secondsLower16Bits = seconds & 0xffff; // bits 0-15 of seconds + const secondsUpperBits = (seconds / 0x10000) >>> 0; // bits 16+ of seconds + + const nanosFromLower16Bits = secondsLower16Bits * nanosPerSecond; // exact integer, < 2^53 + const nanosFromUpperBits = secondsUpperBits * nanosPerSecond; // exact integer, < 2^53 + + // Split the lower-16-bit contribution into [low32, high32]. + const lower16ContributionLow32 = nanosFromLower16Bits >>> 0; + const lower16ContributionHigh32 = Math.floor( + nanosFromLower16Bits / 0x100000000 + ); + + // The upper-bits contribution is shifted left by 16 bits when recombined. + const upperBitsContributionLow32 = + ((nanosFromUpperBits & 0xffff) * 0x10000) >>> 0; + const upperBitsContributionHigh32 = (nanosFromUpperBits / 0x10000) >>> 0; + + // Add the two contributions plus the sub-second nanoseconds with carry propagation. + const low32WithCarry = + lower16ContributionLow32 + upperBitsContributionLow32 + nanos; + const totalLow = low32WithCarry >>> 0; + const carry = Math.floor(low32WithCarry / 0x100000000); // 0, 1, or 2 + const totalHigh = + (lower16ContributionHigh32 + upperBitsContributionHigh32 + carry) >>> 0; + + serializer.writeFixed64(totalLow, totalHigh); +} + +/** + * Write Attributes directly to protobuf as repeated KeyValue + */ +export function writeAttributes( + writer: IProtobufWriter, + attributes: Attributes | LogAttributes, + fieldNumber: number +): void { + for (const key in attributes) { + if (!Object.prototype.hasOwnProperty.call(attributes, key)) { + continue; + } + const value = attributes[key]; + writer.writeTag(fieldNumber, 2); // repeated KeyValue attributes (field varies, length-delimited) + const kvStart = writer.startLengthDelimited(); + const startPos = writer.pos; + writeKeyValue(writer, key, value); + writer.finishLengthDelimited(kvStart, writer.pos - startPos); + } +} + +/** + * Write a KeyValue pair directly to protobuf + */ +export function writeKeyValue( + writer: IProtobufWriter, + key: string, + value: AnyValue +): void { + writer.writeTag(1, 2); // KeyValue.key (field 1, string, length-delimited) + writer.writeString(key); + writer.writeTag(2, 2); // KeyValue.value (field 2, AnyValue, length-delimited) + const valueStart = writer.startLengthDelimited(); + const startPos = writer.pos; + writeAnyValue(writer, value); + writer.finishLengthDelimited(valueStart, writer.pos - startPos); +} + +// int64 range bounds for int_value vs double_value encoding. +// -(2^63) is the signed int64 minimum; 2^63 is one past the maximum. +// Both are exact IEEE 754 doubles (powers of two), so comparisons are precise. +const MIN_64_BIT_INT = -(2 ** 63); +const MAX_64_BIT_INT = 2 ** 63; + +/** + * Write an AnyValue directly from raw attribute value to protobuf + */ +export function writeAnyValue(writer: IProtobufWriter, value: AnyValue): void { + const t = typeof value; + if (t === 'string') { + writer.writeTag(1, 2); // AnyValue.string_value (field 1, length-delimited) + writer.writeString(value as string); + } else if (t === 'boolean') { + writer.writeTag(2, 0); // AnyValue.bool_value (field 2, varint) + writer.writeVarint((value as boolean) ? 1 : 0); + } else if (t === 'number') { + const numValue = value as number; + // Encode as int_value (int64) when the number is an integer within the int64 + // range. Number.isSafeInteger() would be too conservative — integers beyond + // 2^53-1 that are exactly representable in IEEE 754 (e.g. powers of two) + // still fit in int64 and must not be downgraded to double_value. + if ( + Number.isInteger(numValue) && + numValue >= MIN_64_BIT_INT && + numValue < MAX_64_BIT_INT + ) { + writer.writeTag(3, 0); // AnyValue.int_value (field 3, varint) + writer.writeVarint(numValue); + } else { + writer.writeTag(4, 1); // AnyValue.double_value (field 4, fixed64) + writer.writeDouble(numValue); + } + } else if (value instanceof Uint8Array) { + writer.writeTag(7, 2); // AnyValue.bytes_value (field 7, length-delimited) + writer.writeBytes(value); + } else if (Array.isArray(value)) { + writer.writeTag(5, 2); // AnyValue.array_value (field 5, ArrayValue, length-delimited) + const arrayStart = writer.startLengthDelimited(); + const arrayStartPos = writer.pos; + for (const item of value) { + writer.writeTag(1, 2); // ArrayValue.values (field 1, AnyValue, length-delimited) + const itemStart = writer.startLengthDelimited(); + const itemStartPos = writer.pos; + writeAnyValue(writer, item); + writer.finishLengthDelimited(itemStart, writer.pos - itemStartPos); + } + writer.finishLengthDelimited(arrayStart, writer.pos - arrayStartPos); + } else if (t === 'object' && value != null) { + writer.writeTag(6, 2); // AnyValue.kvlist_value (field 6, KeyValueList, length-delimited) + const kvlistStart = writer.startLengthDelimited(); + const kvlistStartPos = writer.pos; + const obj = value as Record; + for (const k in obj) { + if (!Object.prototype.hasOwnProperty.call(obj, k)) { + continue; + } + const v = obj[k]; + writer.writeTag(1, 2); // KeyValueList.values (field 1, KeyValue, length-delimited) + const kvStart = writer.startLengthDelimited(); + const kvStartPos = writer.pos; + writer.writeTag(1, 2); // KeyValue.key (field 1, string, length-delimited) + writer.writeString(k); + writer.writeTag(2, 2); // KeyValue.value (field 2, AnyValue, length-delimited) + const valueStart = writer.startLengthDelimited(); + const valueStartPos = writer.pos; + writeAnyValue(writer, v); + writer.finishLengthDelimited(valueStart, writer.pos - valueStartPos); + writer.finishLengthDelimited(kvStart, writer.pos - kvStartPos); + } + writer.finishLengthDelimited(kvlistStart, writer.pos - kvlistStartPos); + } + // Else: unsupported type, write nothing +} diff --git a/experimental/packages/otlp-transformer/src/common/protobuf/i-protobuf-writer.ts b/experimental/packages/otlp-transformer/src/common/protobuf/i-protobuf-writer.ts new file mode 100644 index 00000000000..eb8a497e1d2 --- /dev/null +++ b/experimental/packages/otlp-transformer/src/common/protobuf/i-protobuf-writer.ts @@ -0,0 +1,17 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +export interface IProtobufWriter { + pos: number; + writeTag(fieldNumber: number, wireType: number): void; + writeVarint(value: number): void; + writeFixed32(value: number): void; + writeFixed64(low: number, high: number): void; + writeBytes(bytes: Uint8Array): void; + writeString(str: string): void; + writeDouble(value: number): void; + startLengthDelimited(): number; + finishLengthDelimited(pos: number, length: number): void; +} diff --git a/experimental/packages/otlp-transformer/src/common/protobuf/protobuf-size-estimator.ts b/experimental/packages/otlp-transformer/src/common/protobuf/protobuf-size-estimator.ts new file mode 100644 index 00000000000..eb846b72cad --- /dev/null +++ b/experimental/packages/otlp-transformer/src/common/protobuf/protobuf-size-estimator.ts @@ -0,0 +1,80 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { IProtobufWriter } from './i-protobuf-writer'; +import { estimateVarintSize } from './utils'; + +/** + * Calculate UTF-8 byte length without encoding + * @param str valid UTF-16 string + */ +function utf8ByteLength(str: string): number { + // No quick path for ASCII, since we need to loop though all chars anyway. + const len = str.length; + let byteLen = 0; + for (let i = 0; i < len; i++) { + const code = str.charCodeAt(i); + if (code < 0x80) { + byteLen += 1; + } else if (code < 0x800) { + byteLen += 2; + } else if (code < 0xd800 || code >= 0xe000) { + byteLen += 3; + } else { + // Surrogate pair + i++; // Skip the next character + byteLen += 4; + } + } + return byteLen; +} + +/** + * Size estimator for protobuf messages. + * Implements the same interface as ProtobufWriter but only counts bytes without allocating a buffer. + * @internal + */ +export class ProtobufSizeEstimator implements IProtobufWriter { + public pos: number = 0; + + startLengthDelimited(): number { + return this.pos; + } + + finishLengthDelimited(_: number, length: number): void { + this.pos += estimateVarintSize(length); + } + + writeVarint(value: number): void { + this.pos += estimateVarintSize(value); + } + + writeFixed32(_value: number): void { + this.pos += 4; + } + + writeFixed64(_low: number, _high: number): void { + this.pos += 8; + } + + writeBytes(bytes: Uint8Array): void { + this.pos += estimateVarintSize(bytes.length); + this.pos += bytes.length; + } + + writeTag(fieldNumber: number, wireType: number): void { + this.writeVarint((fieldNumber << 3) | wireType); + } + + writeDouble(_value: number): void { + this.pos += 8; + } + + writeString(str: string): void { + const byteLen = utf8ByteLength(str); + this.pos += estimateVarintSize(byteLen); + this.pos += byteLen; + } +} diff --git a/experimental/packages/otlp-transformer/src/common/protobuf/protobuf-writer.ts b/experimental/packages/otlp-transformer/src/common/protobuf/protobuf-writer.ts new file mode 100644 index 00000000000..e9b91085b1b --- /dev/null +++ b/experimental/packages/otlp-transformer/src/common/protobuf/protobuf-writer.ts @@ -0,0 +1,266 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ +import { diag } from '@opentelemetry/api'; +import type { IProtobufWriter } from './i-protobuf-writer'; +import { estimateVarintSize } from './utils'; + +export const GROWING_BUFFER_DEBUG_MESSAGE = + 'ProtobufWriter: estimated size was too small, growing buffer.'; + +/** + * bytes reserved for length in length-delimited fields + * using 1 to assume most length-delimited fields are small + */ +const RESERVED_LENGTH_BYTES = 1; + +/** + * Primitive protobuf writer, optimized to avoid small object allocations. + * Grows buffer dynamically if initial size is exceeded. + */ +export class ProtobufWriter implements IProtobufWriter { + private _buffer: Uint8Array; + // Avoid using TextEncoder type. While the global is there on all supported runtimes, types may differ. + private readonly _textEncoder: { encode: (str: string) => Uint8Array }; + private _dataView: DataView; + + public pos: number = 0; + + constructor(estimatedSize = 65536) { + this._buffer = new Uint8Array(estimatedSize); + this._textEncoder = new TextEncoder(); + this._dataView = new DataView(this._buffer.buffer, this._buffer.byteOffset); + } + + /** + * Ensure buffer has capacity for at least size more bytes + */ + private _ensureCapacity(size: number): void { + const needed = this.pos + size; + if (needed <= this._buffer.length) { + return; + } + + // It is safe to grow the buffer, but we assume that estimation is correct. + // Getting to this point indicates incorrect estimation or over-reservation + // of space in this writer and can lead to poor memory performance. + diag.debug(GROWING_BUFFER_DEBUG_MESSAGE); + + // Double buffer size until we have enough space + let newSize = this._buffer.length * 2; + while (newSize < needed) { + newSize *= 2; + } + + const newBuffer = new Uint8Array(newSize); + newBuffer.set(this._buffer); + this._buffer = newBuffer; + // Recreate DataView for the new buffer + this._dataView = new DataView(this._buffer.buffer, this._buffer.byteOffset); + } + + /** + * Get the written bytes as a Uint8Array + */ + finish(): Uint8Array { + return this._buffer.subarray(0, this.pos); + } + + /** + * Insert placeholder for length. Update later with {@link finishLengthDelimited} + * Returns the position where to write the length. + */ + startLengthDelimited(): number { + const lengthPos = this.pos; + // Reserve bytes for the length varint (RESERVED_LENGTH_BYTES should be fit to the common case) + this._ensureCapacity(RESERVED_LENGTH_BYTES); + this.pos += RESERVED_LENGTH_BYTES; + return lengthPos; + } + + /** + * Write length varint at placeholder position and shift content forward if needed. + * Most messages are small (< 128 bytes), so we reserve 1 byte and only shift + * when the length needs more bytes. + */ + finishLengthDelimited(pos: number, length: number): void { + // Calculate varint size needed for this length + const v = length >>> 0; + + // Shift content forward if we need more bytes than reserved + const varintSize = estimateVarintSize(v); + if (varintSize > RESERVED_LENGTH_BYTES) { + const additionalBytes = varintSize - RESERVED_LENGTH_BYTES; + this._ensureCapacity(additionalBytes); + this._buffer.copyWithin( + pos + varintSize, + pos + RESERVED_LENGTH_BYTES, + this.pos + ); + this.pos += additionalBytes; + } + + // Write the varint at the placeholder position, inlined to avoid unnecessary checks. + let writePos = pos; + if (v < 0x80) { + this._buffer[writePos] = v; + } else if (v < 0x4000) { + this._buffer[writePos++] = (v & 0x7f) | 0x80; + this._buffer[writePos] = v >>> 7; + } else if (v < 0x200000) { + this._buffer[writePos++] = (v & 0x7f) | 0x80; + this._buffer[writePos++] = ((v >>> 7) & 0x7f) | 0x80; + this._buffer[writePos] = v >>> 14; + } else if (v < 0x10000000) { + this._buffer[writePos++] = (v & 0x7f) | 0x80; + this._buffer[writePos++] = ((v >>> 7) & 0x7f) | 0x80; + this._buffer[writePos++] = ((v >>> 14) & 0x7f) | 0x80; + this._buffer[writePos] = v >>> 21; + } else { + this._buffer[writePos++] = (v & 0x7f) | 0x80; + this._buffer[writePos++] = ((v >>> 7) & 0x7f) | 0x80; + this._buffer[writePos++] = ((v >>> 14) & 0x7f) | 0x80; + this._buffer[writePos++] = ((v >>> 21) & 0x7f) | 0x80; + this._buffer[writePos] = v >>> 28; + } + } + + /** + * Write a varint (variable-length integer) + */ + writeVarint(value: number): void { + this._ensureCapacity(estimateVarintSize(value)); + // Check if value fits in 32-bit range + if (value >= 0 && value <= 0xffffffff) { + // 32-bit or small integer + let v = value >>> 0; // Convert to unsigned 32-bit + while (v > 0x7f) { + this._buffer[this.pos++] = (v & 0x7f) | 0x80; + v >>>= 7; + } + this._buffer[this.pos++] = v; + } else { + // Needs 64-bit handling - convert to [low, high] + let low: number; + let high: number; + + if (value >= 0) { + // Positive number + low = value >>> 0; + high = (value / 0x100000000) >>> 0; + } else { + // Negative number - use two's complement + const abs = Math.abs(value); + low = abs >>> 0; + high = (abs / 0x100000000) >>> 0; + + // Two's complement: invert bits and add 1 + low = ~low >>> 0; + high = ~high >>> 0; + low = (low + 1) >>> 0; + if (low === 0) { + high = (high + 1) >>> 0; + } + } + + // Write as 64-bit varint + while (high > 0 || low > 0x7f) { + this._buffer[this.pos++] = (low & 0x7f) | 0x80; + low = ((low >>> 7) | (high << 25)) >>> 0; + high >>>= 7; + } + this._buffer[this.pos++] = low & 0x7f; + } + } + + /** + * Write a 32-bit fixed integer (little-endian) + */ + writeFixed32(value: number): void { + this._ensureCapacity(4); + const v = value >>> 0; + this._buffer[this.pos++] = v & 0xff; + this._buffer[this.pos++] = (v >>> 8) & 0xff; + this._buffer[this.pos++] = (v >>> 16) & 0xff; + this._buffer[this.pos++] = (v >>> 24) & 0xff; + } + + /** + * Write a 64-bit fixed integer (little-endian) + * @param low - Low 32 bits + * @param high - High 32 bits + */ + writeFixed64(low: number, high: number): void { + this._ensureCapacity(8); + const l = low >>> 0; + const h = high >>> 0; + + // Write low 32 bits + this._buffer[this.pos++] = l & 0xff; + this._buffer[this.pos++] = (l >>> 8) & 0xff; + this._buffer[this.pos++] = (l >>> 16) & 0xff; + this._buffer[this.pos++] = (l >>> 24) & 0xff; + + // Write high 32 bits + this._buffer[this.pos++] = h & 0xff; + this._buffer[this.pos++] = (h >>> 8) & 0xff; + this._buffer[this.pos++] = (h >>> 16) & 0xff; + this._buffer[this.pos++] = (h >>> 24) & 0xff; + } + + /** + * Write length-delimited data (varint length + bytes) + */ + writeBytes(bytes: Uint8Array): void { + this.writeVarint(bytes.length); + this._ensureCapacity(bytes.length); + this._buffer.set(bytes, this.pos); + this.pos += bytes.length; + } + + /** + * Write a field key (field number + wire type) + */ + writeTag(fieldNumber: number, wireType: number): void { + this.writeVarint((fieldNumber << 3) | wireType); + } + + /** + * Write a double (64-bit IEEE 754) + */ + writeDouble(value: number): void { + this._ensureCapacity(8); + this._dataView.setFloat64(this.pos, value, true); // true = little-endian + this.pos += 8; + } + + /** + * Write a string as UTF-8 bytes (length-delimited) + */ + writeString(str: string): void { + // Fast path for ASCII strings (most common case) + let isAscii = true; + const len = str.length; + for (let i = 0; i < len; i++) { + if (str.charCodeAt(i) > 127) { + isAscii = false; + break; + } + } + + if (isAscii) { + // Write length varint + this.writeVarint(len); + this._ensureCapacity(len); + // Write ASCII bytes directly + for (let i = 0; i < len; i++) { + this._buffer[this.pos++] = str.charCodeAt(i); + } + } else { + // Use TextEncoder for non-ASCII strings + const bytes = this._textEncoder.encode(str); + this.writeBytes(bytes); + } + } +} diff --git a/experimental/packages/otlp-transformer/src/common/protobuf/utils.ts b/experimental/packages/otlp-transformer/src/common/protobuf/utils.ts new file mode 100644 index 00000000000..38156576592 --- /dev/null +++ b/experimental/packages/otlp-transformer/src/common/protobuf/utils.ts @@ -0,0 +1,22 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Estimate size of a number encoded as varint. + * @param v value to calculate size for + * @returns size in bytes of the varint encoding of the value + */ +export function estimateVarintSize(v: number): number { + if (v < 0) return 10; + if (v < 0x80) return 1; + if (v < 0x4000) return 2; + if (v < 0x200000) return 3; + if (v < 0x10000000) return 4; + if (v < 0x800000000) return 5; + if (v < 0x40000000000) return 6; + if (v < 0x2000000000000) return 7; + if (v < 0x100000000000000) return 8; + return 9; +} diff --git a/experimental/packages/otlp-transformer/src/logs/protobuf/logs-serializer.ts b/experimental/packages/otlp-transformer/src/logs/protobuf/logs-serializer.ts new file mode 100644 index 00000000000..5f21a3fe4be --- /dev/null +++ b/experimental/packages/otlp-transformer/src/logs/protobuf/logs-serializer.ts @@ -0,0 +1,248 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ +import type { ReadableLogRecord } from '@opentelemetry/sdk-logs'; +import { ProtobufWriter } from '../../common/protobuf/protobuf-writer'; +import { hexToBinary } from '../../common/hex-to-binary'; +import type { Resource } from '@opentelemetry/resources'; +import type { InstrumentationScope } from '@opentelemetry/core'; +import { SeverityNumber } from '@opentelemetry/api-logs'; +import { + writeAnyValue, + writeAttributes, + writeHrTimeAsFixed64, +} from '../../common/protobuf/common-serializer'; +import type { IProtobufWriter } from '../../common/protobuf/i-protobuf-writer'; +import { ProtobufSizeEstimator } from '../../common/protobuf/protobuf-size-estimator'; + +/** + * Serialize a single LogRecord directly from ReadableLogRecord + */ +function serializeLogRecord( + writer: IProtobufWriter, + logRecord: ReadableLogRecord +): void { + const logStart = writer.startLengthDelimited(); + const logStartPos = writer.pos; + + // time_unix_nano (field 1, fixed64) + writer.writeTag(1, 1); // wire type 1 (fixed64) + writeHrTimeAsFixed64(writer, logRecord.hrTime); + + // severity_number (field 2, enum/varint) - skip if unspecified + if ( + logRecord.severityNumber !== undefined && + logRecord.severityNumber !== SeverityNumber.UNSPECIFIED + ) { + writer.writeTag(2, 0); + writer.writeVarint(logRecord.severityNumber); + } + + // severity_text (field 3, string) - skip if empty + if (logRecord.severityText) { + writer.writeTag(3, 2); + writer.writeString(logRecord.severityText); + } + + // body (field 5, AnyValue) - skip if undefined + if (logRecord.body !== undefined) { + writer.writeTag(5, 2); + const bodyStart = writer.startLengthDelimited(); + const bodyStartPos = writer.pos; + writeAnyValue(writer, logRecord.body); + writer.finishLengthDelimited(bodyStart, writer.pos - bodyStartPos); + } + + // attributes (field 6, repeated KeyValue) + if (logRecord.attributes) { + writeAttributes(writer, logRecord.attributes, 6); + } + + // dropped_attributes_count (field 7, uint32) + writer.writeTag(7, 0); + writer.writeVarint(logRecord.droppedAttributesCount); + + // flags (field 8, fixed32) - skip if 0 or undefined + if (logRecord.spanContext?.traceFlags) { + writer.writeTag(8, 5); // wire type 5 (fixed32) + writer.writeFixed32(logRecord.spanContext.traceFlags); + } + + // trace_id (field 9, bytes) - skip if empty + if (logRecord.spanContext?.traceId) { + writer.writeTag(9, 2); + writer.writeBytes(hexToBinary(logRecord.spanContext.traceId)); + } + + // span_id (field 10, bytes) - skip if empty + if (logRecord.spanContext?.spanId) { + writer.writeTag(10, 2); + writer.writeBytes(hexToBinary(logRecord.spanContext.spanId)); + } + + // observed_time_unix_nano (field 11, fixed64) + writer.writeTag(11, 1); // wire type 1 (fixed64) + writeHrTimeAsFixed64(writer, logRecord.hrTimeObserved); + + // event_name (field 12, string) - skip if empty + if (logRecord.eventName) { + writer.writeTag(12, 2); + writer.writeString(logRecord.eventName); + } + + writer.finishLengthDelimited(logStart, writer.pos - logStartPos); +} + +/** + * Serialize ScopeLogs directly from SDK types + */ +function serializeScopeLogs( + writer: IProtobufWriter, + scope: InstrumentationScope, + logRecords: ReadableLogRecord[] +): void { + const scopeLogsStart = writer.startLengthDelimited(); + const scopeLogsStartPos = writer.pos; + + // scope (field 1, InstrumentationScope) + writer.writeTag(1, 2); + const scopeStart = writer.startLengthDelimited(); + const scopeStartPos = writer.pos; + + // Write InstrumentationScope fields directly + writer.writeTag(1, 2); + writer.writeString(scope.name); + + if (scope.version) { + writer.writeTag(2, 2); + writer.writeString(scope.version); + } + + writer.finishLengthDelimited(scopeStart, writer.pos - scopeStartPos); + + // log_records (field 2, repeated LogRecord) + for (const logRecord of logRecords) { + writer.writeTag(2, 2); + serializeLogRecord(writer, logRecord); + } + + // schema_url (field 3, string) - skip if empty + if (scope.schemaUrl) { + writer.writeTag(3, 2); + writer.writeString(scope.schemaUrl); + } + + writer.finishLengthDelimited(scopeLogsStart, writer.pos - scopeLogsStartPos); +} + +function serializeResource( + writer: IProtobufWriter, + resource: Resource, + fieldNumber: number +) { + writer.writeTag(fieldNumber, 2); + const resourceStart = writer.startLengthDelimited(); + const resourceStartPos = writer.pos; + + // Write Resource attributes directly + if (resource.attributes) { + writeAttributes(writer, resource.attributes, 1); + } + + // dropped_attributes_count (field 2, uint32) - set to 0 as we don't track this + writer.writeTag(2, 0); + writer.writeVarint(0); + + writer.finishLengthDelimited(resourceStart, writer.pos - resourceStartPos); +} + +/** + * Serialize ResourceLogs directly from SDK Resource type + */ +function serializeResourceLogs( + writer: IProtobufWriter, + resource: Resource, + scopeMap: Map +): void { + const resourceLogsStart = writer.startLengthDelimited(); + const resourceLogsStartPos = writer.pos; + + // resource (field 1, Resource) + serializeResource(writer, resource, 1); + + // scope_logs (field 2, repeated ScopeLogs) + for (const scopeLogs of scopeMap.values()) { + writer.writeTag(2, 2); + const scope = scopeLogs[0].instrumentationScope; + serializeScopeLogs(writer, scope, scopeLogs); + } + + // schema_url (field 3, string) - skip if empty + if (resource.schemaUrl) { + writer.writeTag(3, 2); + writer.writeString(resource.schemaUrl); + } + + writer.finishLengthDelimited( + resourceLogsStart, + writer.pos - resourceLogsStartPos + ); +} + +/** + * Group log records by resource and instrumentation scope + */ +function createResourceMap( + logRecords: ReadableLogRecord[] +): Map> { + const resourceMap: Map< + Resource, + Map + > = new Map(); + + for (const record of logRecords) { + const resource = record.resource; + const scope = record.instrumentationScope; + + let ismMap: Map | undefined = + resourceMap.get(resource); + if (!ismMap) { + ismMap = new Map(); + resourceMap.set(resource, ismMap); + } + + let records = ismMap.get(scope); + if (!records) { + records = []; + ismMap.set(scope, records); + } + records.push(record); + } + return resourceMap; +} + +/** + * Serialize ExportLogsServiceRequest directly from ReadableLogRecord[] + */ +export function serializeLogsExportRequest( + logRecords: ReadableLogRecord[] +): Uint8Array { + const resourceMap = createResourceMap(logRecords); + + // First pass: estimate size + const estimator = new ProtobufSizeEstimator(); + for (const [resource, scopeMap] of resourceMap) { + estimator.writeTag(1, 2); + serializeResourceLogs(estimator, resource, scopeMap); + } + + // Second pass: write with estimated size + const writer = new ProtobufWriter(estimator.pos); + for (const [resource, scopeMap] of resourceMap) { + writer.writeTag(1, 2); + serializeResourceLogs(writer, resource, scopeMap); + } + + return writer.finish(); +} diff --git a/experimental/packages/otlp-transformer/src/logs/protobuf/logs.ts b/experimental/packages/otlp-transformer/src/logs/protobuf/logs.ts index 2d0d2c0a788..139db40c080 100644 --- a/experimental/packages/otlp-transformer/src/logs/protobuf/logs.ts +++ b/experimental/packages/otlp-transformer/src/logs/protobuf/logs.ts @@ -4,21 +4,16 @@ */ import * as root from '../../generated/root'; -import type { IExportLogsServiceRequest } from '../internal-types'; import type { IExportLogsServiceResponse } from '../export-response'; -import { createExportLogsServiceRequest } from '../internal'; import type { ReadableLogRecord } from '@opentelemetry/sdk-logs'; import type { ExportType } from '../../common/protobuf/protobuf-export-type'; import type { ISerializer } from '../../i-serializer'; -import { PROTOBUF_ENCODER } from '../../common/utils'; +import { serializeLogsExportRequest } from './logs-serializer'; const logsResponseType = root.opentelemetry.proto.collector.logs.v1 .ExportLogsServiceResponse as ExportType; -const logsRequestType = root.opentelemetry.proto.collector.logs.v1 - .ExportLogsServiceRequest as ExportType; - /* * @experimental this serializer may receive breaking changes in minor versions, pin this package's version when using this constant */ @@ -27,8 +22,7 @@ export const ProtobufLogsSerializer: ISerializer< IExportLogsServiceResponse > = { serializeRequest: (arg: ReadableLogRecord[]) => { - const request = createExportLogsServiceRequest(arg, PROTOBUF_ENCODER); - return logsRequestType.encode(request).finish(); + return serializeLogsExportRequest(arg); }, deserializeResponse: (arg: Uint8Array) => { return logsResponseType.decode(arg); diff --git a/experimental/packages/otlp-transformer/test/fixtures/testbed.proto b/experimental/packages/otlp-transformer/test/fixtures/testbed.proto new file mode 100644 index 00000000000..5865ace0e1d --- /dev/null +++ b/experimental/packages/otlp-transformer/test/fixtures/testbed.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +message TestMessage { + int32 field1 = 1; + string field2 = 2; + + message Nested { + int32 field1 = 1; + string field2 = 2; + } + + Nested field3 = 3; + + // Repeated fields to exercise different wire types and repeated code-paths + repeated int32 field4 = 4; + repeated string field5 = 5; + repeated bytes field6 = 6; + repeated Nested field7 = 7; + repeated fixed32 field8 = 8; + repeated fixed64 field9 = 9; + repeated double field10 = 10; +} diff --git a/experimental/packages/otlp-transformer/test/generated/.gitkeep b/experimental/packages/otlp-transformer/test/generated/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d diff --git a/experimental/packages/otlp-transformer/test/logs.test.ts b/experimental/packages/otlp-transformer/test/logs.test.ts index e71c8221b39..483ca77fe07 100644 --- a/experimental/packages/otlp-transformer/test/logs.test.ts +++ b/experimental/packages/otlp-transformer/test/logs.test.ts @@ -3,11 +3,12 @@ * SPDX-License-Identifier: Apache-2.0 */ import type { HrTime } from '@opentelemetry/api'; -import { TraceFlags } from '@opentelemetry/api'; +import { diag, TraceFlags } from '@opentelemetry/api'; import type { InstrumentationScope } from '@opentelemetry/core'; import type { Resource } from '@opentelemetry/resources'; import { resourceFromAttributes } from '@opentelemetry/resources'; import * as assert from 'assert'; +import * as sinon from 'sinon'; import type { ReadableLogRecord } from '@opentelemetry/sdk-logs'; import { SeverityNumber } from '@opentelemetry/api-logs'; import type { Encoder } from '../src/common/utils'; @@ -19,6 +20,7 @@ import { ESeverityNumber } from '../src/logs/internal-types'; import { createExportLogsServiceRequest } from '../src/logs/internal'; import { ProtobufLogsSerializer } from '../src/logs/protobuf'; import { JsonLogsSerializer } from '../src/logs/json'; +import { GROWING_BUFFER_DEBUG_MESSAGE } from '../src/common/protobuf/protobuf-writer'; function createExpectedLogJson(encoder: Encoder): IExportLogsServiceRequest { const timeUnixNano = encoder.encodeHrTime([1680253513, 123241635]); @@ -68,6 +70,10 @@ function createExpectedLogJson(encoder: Encoder): IExportLogsServiceRequest { key: 'bytes-attribute', value: { bytesValue: bytesValue }, }, + { + key: 'double-attribute', + value: { doubleValue: 1.23 }, + }, ], droppedAttributesCount: 0, flags: 1, @@ -125,6 +131,10 @@ function createExpectedLogProtobuf(): IExportLogsServiceRequest { key: 'bytes-attribute', value: { bytesValue: bytesValue }, }, + { + key: 'double-attribute', + value: { doubleValue: 1.23 }, + }, ], droppedAttributesCount: 0, flags: 1, @@ -149,6 +159,7 @@ const DEFAULT_LOG_FRAGMENT: Omit< attributes: { 'some-attribute': 'some attribute value', 'bytes-attribute': new Uint8Array([1, 2, 3, 4, 5]), + 'double-attribute': 1.23, }, droppedAttributesCount: 0, severityNumber: SeverityNumber.ERROR, @@ -335,7 +346,17 @@ describe('Logs', () => { }); describe('ProtobufLogsSerializer', function () { - it('serializes an export request', () => { + let diagStub: sinon.SinonStub; + + beforeEach(function () { + diagStub = sinon.stub(diag, 'debug'); + }); + + afterEach(function () { + sinon.restore(); + }); + + it('serializes an export request', function () { const serialized = ProtobufLogsSerializer.serializeRequest([log_1_1_1]); assert.ok(serialized, 'serialized response is undefined'); const decoded = @@ -359,6 +380,7 @@ describe('Logs', () => { ); assert.deepStrictEqual(decodedObj, expected); + sinon.assert.neverCalledWith(diagStub, GROWING_BUFFER_DEBUG_MESSAGE); }); it('deserializes a response', () => { @@ -387,10 +409,11 @@ describe('Logs', () => { ); }); - it('does not throw when deserializing an empty response', () => { + it('does not throw when deserializing an empty response', function () { assert.doesNotThrow(() => ProtobufLogsSerializer.deserializeResponse(new Uint8Array([])) ); + sinon.assert.neverCalledWith(diagStub, GROWING_BUFFER_DEBUG_MESSAGE); }); }); diff --git a/experimental/packages/otlp-transformer/test/protobuf/common-serializer.test.ts b/experimental/packages/otlp-transformer/test/protobuf/common-serializer.test.ts new file mode 100644 index 00000000000..0d6edaeda3e --- /dev/null +++ b/experimental/packages/otlp-transformer/test/protobuf/common-serializer.test.ts @@ -0,0 +1,696 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ +import * as assert from 'assert'; +import { ProtobufWriter } from '../../src/common/protobuf/protobuf-writer'; +import { + writeAnyValue, + writeKeyValue, + writeHrTimeAsFixed64, +} from '../../src/common/protobuf/common-serializer'; +import * as root from '../../src/generated/root'; +import type { AnyValue } from '@opentelemetry/api-logs'; +import type { HrTime } from '@opentelemetry/api'; +import { uint8ArrayToBase64 } from '../utils'; + +/** + * Helper function to serialize an AnyValue and decode it using generated protobuf code. + * Uses longs: Number, bytes: String as results differ across platforms otherwise. + */ +function serializeAndDecodeAnyValue(value: AnyValue): any { + const writer = new ProtobufWriter(); + writeAnyValue(writer, value); + const buffer = writer.finish(); + + const decoded = root.opentelemetry.proto.common.v1.AnyValue.decode(buffer); + return root.opentelemetry.proto.common.v1.AnyValue.toObject(decoded, { + longs: Number, + bytes: String, + }); +} + +/** + * Helper function to serialize a KeyValue and decode it using generated protobuf code. + * Uses longs: Number, bytes: String as results differ across platforms otherwise. + */ +function serializeAndDecodeKeyValue(key: string, value: AnyValue): any { + const writer = new ProtobufWriter(); + writeKeyValue(writer, key, value); + const buffer = writer.finish(); + + const decoded = root.opentelemetry.proto.common.v1.KeyValue.decode(buffer); + return root.opentelemetry.proto.common.v1.KeyValue.toObject(decoded, { + longs: Number, + bytes: String, + }); +} + +describe('common-serializer', function () { + describe('writeAnyValue', function () { + describe('string values', function () { + it('serializes a string value', function () { + const obj = serializeAndDecodeAnyValue('hello world'); + assert.strictEqual(obj.stringValue, 'hello world'); + }); + + it('serializes an empty string', function () { + const obj = serializeAndDecodeAnyValue(''); + assert.strictEqual(obj.stringValue, ''); + }); + + it('serializes a string with special characters', function () { + const obj = serializeAndDecodeAnyValue('hello\nworld\t"test"'); + assert.strictEqual(obj.stringValue, 'hello\nworld\t"test"'); + }); + + it('serializes a string with unicode characters', function () { + const obj = serializeAndDecodeAnyValue('你好世界 🌍'); + assert.strictEqual(obj.stringValue, '你好世界 🌍'); + }); + }); + + describe('boolean values', function () { + it('serializes true', function () { + const obj = serializeAndDecodeAnyValue(true); + assert.strictEqual(obj.boolValue, true); + }); + + it('serializes false', function () { + const obj = serializeAndDecodeAnyValue(false); + assert.strictEqual(obj.boolValue, false); + }); + }); + + describe('integer values', function () { + it('serializes zero', function () { + const obj = serializeAndDecodeAnyValue(0); + assert.strictEqual(obj.intValue, 0); + }); + + it('serializes positive integers', function () { + const obj = serializeAndDecodeAnyValue(42); + assert.strictEqual(obj.intValue, 42); + }); + + it('serializes negative integers', function () { + const obj = serializeAndDecodeAnyValue(-42); + assert.strictEqual(obj.intValue, -42); + }); + + it('serializes large positive integers', function () { + const obj = serializeAndDecodeAnyValue(1000000); + assert.strictEqual(obj.intValue, 1000000); + }); + + it('serializes Number.MAX_SAFE_INTEGER - 1', function () { + const obj = serializeAndDecodeAnyValue(Number.MAX_SAFE_INTEGER - 1); + assert.strictEqual(obj.intValue, Number.MAX_SAFE_INTEGER - 1); + }); + + it('serializes Number.MAX_SAFE_INTEGER', function () { + const obj = serializeAndDecodeAnyValue(Number.MAX_SAFE_INTEGER); + assert.strictEqual(obj.intValue, Number.MAX_SAFE_INTEGER); + }); + + it('serializes Number.MIN_SAFE_INTEGER', function () { + const obj = serializeAndDecodeAnyValue(Number.MIN_SAFE_INTEGER); + assert.strictEqual(obj.intValue, Number.MIN_SAFE_INTEGER); + }); + + it('serializes integer value that requires carry logic', function () { + const obj = serializeAndDecodeAnyValue(-4294967296 /* -2^32 */); + assert.strictEqual(obj.intValue, -4294967296 /* -2^32 */); + }); + + // 2^53 is beyond MAX_SAFE_INTEGER but exactly representable in IEEE 754 and + // still within the int64 range, so it must be encoded as int_value, not double. + it('serializes integers beyond MAX_SAFE_INTEGER but within int64 range as int_value', function () { + const largeInt = Number.MAX_SAFE_INTEGER + 1; // 2^53, exactly representable + const obj = serializeAndDecodeAnyValue(largeInt); + assert.strictEqual(obj.intValue, largeInt); + }); + + // -(2^63) is the int64 minimum and exactly representable as a double (power of two). + it('serializes -(2^63) as int_value (int64 minimum, exactly representable)', function () { + const int64Min = -(2 ** 63); + const obj = serializeAndDecodeAnyValue(int64Min); + assert.strictEqual(obj.intValue, int64Min); + }); + + // 2^63 exceeds the int64 maximum. 2^63 - 1 is not representable in IEEE 754 + // (it rounds up to 2^63), making 2^63 the first double past the int64 range. + it('serializes 2^63 as double_value (exceeds int64 maximum)', function () { + const overInt64Max = 2 ** 63; + const obj = serializeAndDecodeAnyValue(overInt64Max); + assert.strictEqual(obj.doubleValue, overInt64Max); + assert.strictEqual(obj.intValue, undefined); + }); + + it('serializes -(2^64) as double_value (below int64 minimum)', function () { + const belowInt64Min = -(2 ** 64); + const obj = serializeAndDecodeAnyValue(belowInt64Min); + assert.strictEqual(obj.doubleValue, belowInt64Min); + assert.strictEqual(obj.intValue, undefined); + }); + + // 1e100 satisfies Number.isInteger() but is far outside the int64 range. + it('serializes 1e100 as double_value (integer but exceeds int64 range)', function () { + const obj = serializeAndDecodeAnyValue(1e100); + assert.strictEqual(obj.doubleValue, 1e100); + assert.strictEqual(obj.intValue, undefined); + }); + + it('serializes -1e100 as double_value (integer but exceeds int64 range)', function () { + const obj = serializeAndDecodeAnyValue(-1e100); + assert.strictEqual(obj.doubleValue, -1e100); + assert.strictEqual(obj.intValue, undefined); + }); + }); + + describe('double values', function () { + it('serializes floating point numbers', function () { + const obj = serializeAndDecodeAnyValue(3.14); + assert.strictEqual(obj.doubleValue, 3.14); + }); + + it('serializes small decimal numbers', function () { + const obj = serializeAndDecodeAnyValue(0.1); + assert.strictEqual(obj.doubleValue, 0.1); + }); + + it('serializes negative decimal numbers', function () { + const obj = serializeAndDecodeAnyValue(-2.5); + assert.strictEqual(obj.doubleValue, -2.5); + }); + + it('serializes very small numbers', function () { + const obj = serializeAndDecodeAnyValue(1e-10); + assert.strictEqual(obj.doubleValue, 1e-10); + }); + + it('serializes Number.MAX_VALUE', function () { + const obj = serializeAndDecodeAnyValue(Number.MAX_VALUE); + // Number.MAX_VALUE is larger than a 64-bit integer, so it should serialize as double + assert.strictEqual(obj.doubleValue, Number.MAX_VALUE); + }); + + it('serializes Number.MIN_VALUE', function () { + const obj = serializeAndDecodeAnyValue(Number.MIN_VALUE); + assert.strictEqual(obj.doubleValue, Number.MIN_VALUE); + }); + + it('serializes Infinity', function () { + const obj = serializeAndDecodeAnyValue(Infinity); + assert.strictEqual(obj.doubleValue, Infinity); + }); + + it('serializes -Infinity', function () { + const obj = serializeAndDecodeAnyValue(-Infinity); + assert.strictEqual(obj.doubleValue, -Infinity); + }); + + it('serializes NaN', function () { + const obj = serializeAndDecodeAnyValue(NaN); + assert.strictEqual(obj.doubleValue, NaN); + }); + }); + + describe('bytes values', function () { + it('serializes a Uint8Array', function () { + const bytes = new Uint8Array([0, 1, 2, 3, 4]); + const obj = serializeAndDecodeAnyValue(bytes); + + const expectedBase64 = uint8ArrayToBase64(bytes); + assert.strictEqual(obj.bytesValue, expectedBase64); + }); + + it('serializes an empty Uint8Array', function () { + const bytes = new Uint8Array([]); + const obj = serializeAndDecodeAnyValue(bytes); + + const expectedBase64 = uint8ArrayToBase64(bytes); + assert.strictEqual(obj.bytesValue, expectedBase64); + }); + + it('serializes a Uint8Array with all byte values', function () { + const bytes = new Uint8Array(256); + for (let i = 0; i < 256; i++) { + bytes[i] = i; + } + const obj = serializeAndDecodeAnyValue(bytes); + + const expectedBase64 = uint8ArrayToBase64(bytes); + assert.strictEqual(obj.bytesValue, expectedBase64); + }); + }); + + describe('array values', function () { + it('serializes an empty array', function () { + const obj = serializeAndDecodeAnyValue([]); + + // Empty arrays serialize with an arrayValue, but the values field is omitted when empty + assert.deepStrictEqual(obj.arrayValue, {}); + }); + + it('serializes an array with mixed types', function () { + const obj = serializeAndDecodeAnyValue([ + 1, + 'two', + false, + 2.5, + new Uint8Array([0, 1, 2]), + ]); + + const expectedBase64 = uint8ArrayToBase64(new Uint8Array([0, 1, 2])); + + assert.deepStrictEqual(obj.arrayValue.values, [ + { intValue: 1 }, + { stringValue: 'two' }, + { boolValue: false }, + { doubleValue: 2.5 }, + { bytesValue: expectedBase64 }, + ]); + }); + + it('serializes nested arrays', function () { + const obj = serializeAndDecodeAnyValue([1, [2, 3], [[4]]]); + + assert.deepStrictEqual(obj.arrayValue.values, [ + { intValue: 1 }, + { + arrayValue: { + values: [{ intValue: 2 }, { intValue: 3 }], + }, + }, + { + arrayValue: { + values: [ + { + arrayValue: { + values: [{ intValue: 4 }], + }, + }, + ], + }, + }, + ]); + }); + + it('serializes arrays with objects', function () { + const obj = serializeAndDecodeAnyValue([ + { key: 'value' }, + { nested: { key: 'value' } }, + ]); + + assert.deepStrictEqual(obj.arrayValue.values, [ + { + kvlistValue: { + values: [ + { + key: 'key', + value: { stringValue: 'value' }, + }, + ], + }, + }, + { + kvlistValue: { + values: [ + { + key: 'nested', + value: { + kvlistValue: { + values: [ + { + key: 'key', + value: { stringValue: 'value' }, + }, + ], + }, + }, + }, + ], + }, + }, + ]); + }); + }); + + describe('object/kvlist values', function () { + it('serializes an empty object', function () { + const obj = serializeAndDecodeAnyValue({}); + + // Empty objects serialize with a kvlistValue, but the values field is omitted when empty + assert.deepStrictEqual(obj.kvlistValue, {}); + }); + + it('serializes a simple object', function () { + const obj = serializeAndDecodeAnyValue({ key: 'value', number: 42 }); + + assert.deepStrictEqual(obj.kvlistValue.values, [ + { + key: 'key', + value: { stringValue: 'value' }, + }, + { + key: 'number', + value: { intValue: 42 }, + }, + ]); + }); + + it('serializes nested objects', function () { + const obj = serializeAndDecodeAnyValue({ + outer: { + inner: { + deepKey: 'deepValue', + }, + }, + }); + + assert.deepStrictEqual(obj.kvlistValue.values, [ + { + key: 'outer', + value: { + kvlistValue: { + values: [ + { + key: 'inner', + value: { + kvlistValue: { + values: [ + { + key: 'deepKey', + value: { stringValue: 'deepValue' }, + }, + ], + }, + }, + }, + ], + }, + }, + }, + ]); + }); + + it('serializes objects with special characters in keys', function () { + const obj = serializeAndDecodeAnyValue({ + 'key-with-dashes': 'value1', + 'key.with.dots': 'value2', + 'key with spaces': 'value3', + key_with_underscores: 'value4', + }); + + assert.deepStrictEqual(obj.kvlistValue.values, [ + { key: 'key-with-dashes', value: { stringValue: 'value1' } }, + { key: 'key.with.dots', value: { stringValue: 'value2' } }, + { key: 'key with spaces', value: { stringValue: 'value3' } }, + { key: 'key_with_underscores', value: { stringValue: 'value4' } }, + ]); + }); + + it('serializes objects with mixed value types', function () { + const obj = serializeAndDecodeAnyValue({ + stringVal: 'text', + intVal: 123, + doubleVal: 3.14, + boolVal: true, + arrayVal: [1, 2, 3], + objectVal: { nested: 'value' }, + }); + + assert.deepStrictEqual(obj.kvlistValue.values, [ + { + key: 'stringVal', + value: { stringValue: 'text' }, + }, + { + key: 'intVal', + value: { intValue: 123 }, + }, + { + key: 'doubleVal', + value: { doubleValue: 3.14 }, + }, + { + key: 'boolVal', + value: { boolValue: true }, + }, + { + key: 'arrayVal', + value: { + arrayValue: { + values: [{ intValue: 1 }, { intValue: 2 }, { intValue: 3 }], + }, + }, + }, + { + key: 'objectVal', + value: { + kvlistValue: { + values: [ + { + key: 'nested', + value: { stringValue: 'value' }, + }, + ], + }, + }, + }, + ]); + }); + }); + + describe('null and undefined handling', function () { + it('handles null by writing nothing', function () { + const writer = new ProtobufWriter(); + writeAnyValue(writer, null); + const buffer = writer.finish(); + + // Empty buffer or minimal message structure + assert.ok(buffer.length === 0 || buffer.length < 5); + }); + + it('handles undefined by writing nothing', function () { + const writer = new ProtobufWriter(); + writeAnyValue(writer, undefined); + const buffer = writer.finish(); + + // Empty buffer or minimal message structure + assert.ok(buffer.length === 0 || buffer.length < 5); + }); + + it('handles null in arrays by writing nothing for that element', function () { + const obj = serializeAndDecodeAnyValue([1, null, 3]); + + // Middle element should have no value set (empty AnyValue) + assert.deepStrictEqual(obj.arrayValue.values, [ + { intValue: 1 }, + {}, // Empty AnyValue for null + { intValue: 3 }, + ]); + }); + + it('handles undefined in objects by writing nothing for that value', function () { + const obj = serializeAndDecodeAnyValue({ + key1: 'value1', + key2: undefined, + key3: 'value3', + }); + + assert.deepStrictEqual(obj.kvlistValue.values, [ + { + key: 'key1', + value: { stringValue: 'value1' }, + }, + { + key: 'key2', + value: {}, // Empty AnyValue for undefined + }, + { + key: 'key3', + value: { stringValue: 'value3' }, + }, + ]); + }); + }); + }); + + describe('writeKeyValue', function () { + it('serializes a key-value pair with string value', function () { + const obj = serializeAndDecodeKeyValue('myKey', 'myValue'); + + assert.deepStrictEqual(obj, { + key: 'myKey', + value: { stringValue: 'myValue' }, + }); + }); + + it('serializes a key-value pair with integer value', function () { + const obj = serializeAndDecodeKeyValue('count', 42); + + assert.deepStrictEqual(obj, { + key: 'count', + value: { intValue: 42 }, + }); + }); + + it('serializes a key-value pair with double value', function () { + const obj = serializeAndDecodeKeyValue('ratio', 3.14); + + assert.deepStrictEqual(obj, { + key: 'ratio', + value: { doubleValue: 3.14 }, + }); + }); + + it('serializes a key-value pair with boolean value', function () { + const obj = serializeAndDecodeKeyValue('enabled', true); + + assert.deepStrictEqual(obj, { + key: 'enabled', + value: { boolValue: true }, + }); + }); + + it('serializes a key-value pair with array value', function () { + const obj = serializeAndDecodeKeyValue('items', [1, 2, 3]); + + assert.deepStrictEqual(obj, { + key: 'items', + value: { + arrayValue: { + values: [{ intValue: 1 }, { intValue: 2 }, { intValue: 3 }], + }, + }, + }); + }); + + it('serializes a key-value pair with object value', function () { + const obj = serializeAndDecodeKeyValue('metadata', { version: '1.0' }); + + assert.deepStrictEqual(obj, { + key: 'metadata', + value: { + kvlistValue: { + values: [ + { + key: 'version', + value: { stringValue: '1.0' }, + }, + ], + }, + }, + }); + }); + }); + + describe('writeHrTimeAsFixed64', function () { + /** + * Reference implementation using BigInt for correctness verification. + * Computes the fixed64 payload from total_nanos = seconds * 1e9 + nanos + * as [low32, high32], wrapping modulo 2^64 like protobuf fixed64 does. + */ + function expectedFixed64(hrTime: HrTime): [number, number] { + const total = + (BigInt(hrTime[0]) * 1_000_000_000n + BigInt(hrTime[1])) & + 0xffff_ffff_ffff_ffffn; + return [ + Number(total & 0xffff_ffffn), + Number((total >> 32n) & 0xffff_ffffn), + ]; + } + + function serializeHrTime(hrTime: HrTime): [number, number] { + const writer = new ProtobufWriter(16); + writeHrTimeAsFixed64(writer, hrTime); + const buf = writer.finish(); + // fixed64 is 8 bytes, little-endian: bytes 0-3 = low32, bytes 4-7 = high32 + const low = buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24); + const high = buf[4] | (buf[5] << 8) | (buf[6] << 16) | (buf[7] << 24); + return [low >>> 0, high >>> 0]; + } + + it('encodes [0, 0] (epoch)', function () { + const hrTime: HrTime = [0, 0]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('encodes [1, 0] (exactly 1 second)', function () { + const hrTime: HrTime = [1, 0]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('encodes [0, 1] (1 nanosecond)', function () { + const hrTime: HrTime = [0, 1]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('encodes [1, 999_999_999] (max nanos in a second)', function () { + const hrTime: HrTime = [1, 999_999_999]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('encodes a typical 2023 Unix timestamp [1_700_000_000, 0]', function () { + const hrTime: HrTime = [1_700_000_000, 0]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('encodes [1_700_000_000, 500_000_000]', function () { + const hrTime: HrTime = [1_700_000_000, 500_000_000]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('encodes [1_700_000_001, 999_999_999] (nanos cause carry into high word)', function () { + const hrTime: HrTime = [1_700_000_001, 999_999_999]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + // Regression: values above ~9_007_199 seconds previously lost precision + // because `seconds * 1e9` exceeded Number.MAX_SAFE_INTEGER. + it('regression: encodes [5_100_000_153, 1] without precision loss', function () { + const hrTime: HrTime = [5_100_000_153, 1]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('regression: encodes [5_300_000_159, 999_999_999] without precision loss', function () { + const hrTime: HrTime = [5_300_000_159, 999_999_999]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('regression: encodes [9_007_200, 0] (just above old precision boundary)', function () { + // 9_007_199 * 1e9 ≈ Number.MAX_SAFE_INTEGER; the old code lost precision here + const hrTime: HrTime = [9_007_200, 0]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('regression: encodes [9_100_000_273, 0] without precision loss', function () { + const hrTime: HrTime = [9_100_000_273, 0]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('encodes the Year 2038 boundary [2_147_483_647, 999_999_999]', function () { + const hrTime: HrTime = [2_147_483_647, 999_999_999]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('encodes a large timestamp [10_000_000_000, 999_999_999]', function () { + const hrTime: HrTime = [10_000_000_000, 999_999_999]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('encodes the largest HrTime that fits in fixed64 [18_446_744_073, 709_551_615]', function () { + // July 4, 2554, at 23:59:59.999999999 UTC - the largest timestamp that can be represented in fixed64 nanoseconds since Unix epoch + const hrTime: HrTime = [18_446_744_073, 709_551_615]; + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + + it('wraps without throwing once total nanoseconds exceeds fixed64 range [18_446_744_073, 709_551_616]', function () { + const hrTime: HrTime = [18_446_744_073, 709_551_616]; + assert.deepStrictEqual(serializeHrTime(hrTime), [0, 0]); + assert.deepStrictEqual(serializeHrTime(hrTime), expectedFixed64(hrTime)); + }); + }); +}); diff --git a/experimental/packages/otlp-transformer/test/protobuf/protobuf-size-estimator.test.ts b/experimental/packages/otlp-transformer/test/protobuf/protobuf-size-estimator.test.ts new file mode 100644 index 00000000000..f2bdbe79f31 --- /dev/null +++ b/experimental/packages/otlp-transformer/test/protobuf/protobuf-size-estimator.test.ts @@ -0,0 +1,304 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as assert from 'assert'; +import { ProtobufSizeEstimator } from '../../src/common/protobuf/protobuf-size-estimator'; +import { ProtobufWriter } from '../../src/common/protobuf/protobuf-writer'; + +describe('ProtobufSizeEstimator', function () { + describe('size estimation accuracy', function () { + it('should match ProtobufWriter size for writeVarint', function () { + const testValues = [ + 0, // min 1-byte varint + 1, // 1-byte varint + 127, // max 1-byte varint + 128, // min 2-byte varint + 300, // 2-byte varint + 16383, // max 2-byte varint + 16384, // min 3-byte varint + 2097151, // max 3-byte varint + 2097152, // min 4-byte varint + 268435455, // max 4-byte varint + 268435456, // min 5-byte varint + 4294967295, // max 5-byte varint (32-bit unsigned) + -1, // 10-byte varint (negative) + -100, // 10-byte varint (negative) + -2147483648, // 10-byte varint (min 32-bit signed) + 4294967296, // min 6-byte varint (2^32) + 8589934591, // 6-byte varint + 34359738367, // 7-byte varint + 4398046511103, // 8-byte varint + 562949953421311, // 9-byte varint + // we're just testing if size estimation works here, precision loss does not matter in this case. + // eslint-disable-next-line no-loss-of-precision + 72057594037927935, // 10-byte varint + ]; + + for (const value of testValues) { + const writer = new ProtobufWriter(1024); + const estimator = new ProtobufSizeEstimator(); + + writer.writeVarint(value); + estimator.writeVarint(value); + + assert.strictEqual( + estimator.pos, + writer.pos, + `Unexpected size mismatch for value ${value}: expected ${writer.pos} but got ${estimator.pos}` + ); + } + }); + + it('should match ProtobufWriter size for writeFixed32', function () { + const testValues = [0, 1, 255, 65535, 0x12345678, 0xffffffff]; + + for (const value of testValues) { + const writer = new ProtobufWriter(1024); + const estimator = new ProtobufSizeEstimator(); + + writer.writeFixed32(value); + estimator.writeFixed32(value); + + assert.strictEqual(estimator.pos, writer.pos); + } + }); + + it('should match ProtobufWriter size for writeFixed64', function () { + const testValues = [ + { low: 0, high: 0 }, + { low: 1, high: 0 }, + { low: 0xffffffff, high: 0 }, + { low: 0, high: 0xffffffff }, + { low: 0xffffffff, high: 0xffffffff }, + { low: 0x12345678, high: 0x9abcdef0 }, + ]; + + for (const { low, high } of testValues) { + const writer = new ProtobufWriter(1024); + const estimator = new ProtobufSizeEstimator(); + + writer.writeFixed64(low, high); + estimator.writeFixed64(low, high); + + assert.strictEqual(estimator.pos, writer.pos); + } + }); + + it('should match ProtobufWriter size for writeDouble', function () { + const testValues = [ + 0.0, + 1.0, + -1.0, + 3.14159, + -3.14159, + Number.MAX_VALUE, + Number.MIN_VALUE, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ]; + + for (const value of testValues) { + const writer = new ProtobufWriter(1024); + const estimator = new ProtobufSizeEstimator(); + + writer.writeDouble(value); + estimator.writeDouble(value); + + assert.strictEqual(estimator.pos, writer.pos); + } + }); + + it('should match ProtobufWriter size for writeBytes', function () { + const testValues = [ + new Uint8Array([]), + new Uint8Array([0]), + new Uint8Array([1, 2, 3]), + new Uint8Array(127), // max 1-byte length varint + new Uint8Array(128), // min 2-byte length varint + new Uint8Array(255), + new Uint8Array([0xff, 0xfe, 0xfd, 0xfc]), + ]; + + for (const bytes of testValues) { + const writer = new ProtobufWriter(1024); + const estimator = new ProtobufSizeEstimator(); + + writer.writeBytes(bytes); + estimator.writeBytes(bytes); + + assert.strictEqual(estimator.pos, writer.pos); + } + }); + + it('should match ProtobufWriter size for writeString with ASCII', function () { + const testValues = [ + '', + 'a', + 'hello', + 'Hello, World!', + 'a'.repeat(127), // max 1-byte length varint + 'a'.repeat(128), // min 2-byte length varint + 'The quick brown fox jumps over the lazy dog', + ]; + + for (const str of testValues) { + const writer = new ProtobufWriter(1024); + const estimator = new ProtobufSizeEstimator(); + + writer.writeString(str); + estimator.writeString(str); + + assert.strictEqual(estimator.pos, writer.pos); + } + }); + + it('should match ProtobufWriter size for writeString with UTF-8', function () { + const testValues = [ + 'Hello 世界', // Mix of ASCII and multi-byte + '日本語', // 3-byte UTF-8 chars + '🚀', // 4-byte emoji + '😀����😂', // Multiple emojis + 'Ñoño', // 2-byte UTF-8 chars + '€100', // Euro sign (3 bytes) + 'Test 测试 🧪', // Mixed content + '\u0001\u007f\u0080\u07ff\u0800\uffff', // Various UTF-8 ranges + ]; + + for (const str of testValues) { + const writer = new ProtobufWriter(1024); + const estimator = new ProtobufSizeEstimator(); + + writer.writeString(str); + estimator.writeString(str); + + assert.strictEqual(estimator.pos, writer.pos); + } + }); + + it('should match ProtobufWriter size for writeTag', function () { + const testValues = [ + { fieldNumber: 1, wireType: 0 }, + { fieldNumber: 1, wireType: 1 }, + { fieldNumber: 1, wireType: 2 }, + { fieldNumber: 1, wireType: 5 }, + { fieldNumber: 15, wireType: 0 }, // max 1-byte tag + { fieldNumber: 16, wireType: 0 }, // min 2-byte tag + { fieldNumber: 100, wireType: 2 }, + { fieldNumber: 2047, wireType: 0 }, // max 2-byte tag + { fieldNumber: 2048, wireType: 0 }, // min 3-byte tag + ]; + + for (const { fieldNumber, wireType } of testValues) { + const writer = new ProtobufWriter(1024); + const estimator = new ProtobufSizeEstimator(); + + writer.writeTag(fieldNumber, wireType); + estimator.writeTag(fieldNumber, wireType); + + assert.strictEqual(estimator.pos, writer.pos); + } + }); + + it('should match ProtobufWriter size for length-delimited fields', function () { + const testValues = [ + 0, // empty + 1, // small + 127, // max 1-byte length + 128, // min 2-byte length + 300, + 16383, // max 2-byte length + 16384, // min 3-byte length + ]; + + for (const length of testValues) { + const writer = new ProtobufWriter(32768); + const estimator = new ProtobufSizeEstimator(); + + const writerStartPos = writer.startLengthDelimited(); + const estimatorStartPos = estimator.startLengthDelimited(); + + // Write some dummy data + const dummyData = new Uint8Array(length); + writer.writeBytes(dummyData); + estimator.writeBytes(dummyData); + + const writerContentLength = writer.pos - writerStartPos - 1; + const estimatorContentLength = estimator.pos - estimatorStartPos; + + writer.finishLengthDelimited(writerStartPos, writerContentLength); + estimator.finishLengthDelimited( + estimatorStartPos, + estimatorContentLength + ); + + assert.strictEqual(estimator.pos, writer.pos); + } + }); + + it('should match ProtobufWriter size for complex message', function () { + const writer = new ProtobufWriter(1024); + const estimator = new ProtobufSizeEstimator(); + + // Simulate a complex protobuf message with various field types + // field 1: varint + writer.writeTag(1, 0); + writer.writeVarint(12345); + estimator.writeTag(1, 0); + estimator.writeVarint(12345); + + // field 2: fixed64 + writer.writeTag(2, 1); + writer.writeFixed64(0x12345678, 0x9abcdef0); + estimator.writeTag(2, 1); + estimator.writeFixed64(0x12345678, 0x9abcdef0); + + // field 3: length-delimited (string) + writer.writeTag(3, 2); + writer.writeString('Hello, World! 🌍'); + estimator.writeTag(3, 2); + estimator.writeString('Hello, World! 🌍'); + + // field 4: nested message + writer.writeTag(4, 2); + const writerNestedStart = writer.startLengthDelimited(); + estimator.writeTag(4, 2); + const estimatorNestedStart = estimator.startLengthDelimited(); + + // nested field 1: varint + writer.writeTag(1, 0); + writer.writeVarint(42); + estimator.writeTag(1, 0); + estimator.writeVarint(42); + + // nested field 2: bytes + writer.writeTag(2, 2); + writer.writeBytes(new Uint8Array([1, 2, 3, 4, 5])); + estimator.writeTag(2, 2); + estimator.writeBytes(new Uint8Array([1, 2, 3, 4, 5])); + + const writerNestedLength = writer.pos - writerNestedStart - 1; + const estimatorNestedLength = estimator.pos - estimatorNestedStart; + writer.finishLengthDelimited(writerNestedStart, writerNestedLength); + estimator.finishLengthDelimited( + estimatorNestedStart, + estimatorNestedLength + ); + + // field 5: double + writer.writeTag(5, 1); + writer.writeDouble(3.14159); + estimator.writeTag(5, 1); + estimator.writeDouble(3.14159); + + // field 6: fixed32 + writer.writeTag(6, 5); + writer.writeFixed32(0xdeadbeef); + estimator.writeTag(6, 5); + estimator.writeFixed32(0xdeadbeef); + + assert.strictEqual(estimator.pos, writer.pos); + }); + }); +}); diff --git a/experimental/packages/otlp-transformer/test/protobuf/protobuf-writer.test.ts b/experimental/packages/otlp-transformer/test/protobuf/protobuf-writer.test.ts new file mode 100644 index 00000000000..34b54a7dff6 --- /dev/null +++ b/experimental/packages/otlp-transformer/test/protobuf/protobuf-writer.test.ts @@ -0,0 +1,336 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import { diag } from '@opentelemetry/api'; +import { + ProtobufWriter, + GROWING_BUFFER_DEBUG_MESSAGE, +} from '../../src/common/protobuf/protobuf-writer'; + +// Use pbjs-generated test helper. +import * as testbed from '../generated/testbed'; +import { toLongBits } from '../../src/common/utils'; + +describe('ProtobufWriter', function () { + describe('round-trip with protobuf.js', function () { + it('should write and read back complex message', function () { + const writer = new ProtobufWriter(1024); + + // Note: one large test, to avoid having to have many .proto files + // field 1: varint + writer.writeTag(1, 0); + writer.writeVarint(42); + + // field 2: string + writer.writeTag(2, 2); + writer.writeString('test'); + + // field 3: nested message + writer.writeTag(3, 2); + const nestedPos = writer.startLengthDelimited(); + const nestedStartPos = writer.pos; + writer.writeTag(1, 0); + writer.writeVarint(100); + writer.finishLengthDelimited(nestedPos, writer.pos - nestedStartPos); + + // Repeated int32 field4 (varint, multiple occurrences) + const field4Values = [1, 2, 150, 0, -1, 2147483647, -2147483648]; + for (const v of field4Values) { + writer.writeTag(4, 0); + writer.writeVarint(v); + } + + // Repeated string field5 (length-delimited) + const field5Values = ['a', 'bb', 'ccc', 'x'.repeat(1000), '😀']; + for (const s of field5Values) { + writer.writeTag(5, 2); + writer.writeString(s); + } + + // Repeated bytes field6 + const field6Values = [ + new Uint8Array([1, 2]), + new Uint8Array([3]), + new Uint8Array(200).fill(0x7f), + new Uint8Array([]), + ]; + for (const b of field6Values) { + writer.writeTag(6, 2); + writer.writeBytes(b); + } + + // Repeated nested messages field7 + const field7Values = [ + { field1: 10 }, + { field1: 20, field2: 'y'.repeat(300) }, // add long string to test length varint expansion + { field1: 2147483647 }, + { field1: -2147483648 }, + ]; + field7Values.forEach(value => { + writer.writeTag(7, 2); + const nestedValueStart = writer.startLengthDelimited(); + const nestedValueStartPos = writer.pos; + writer.writeTag(1, 0); + writer.writeVarint(value.field1); + if (value.field2 != null) { + writer.writeTag(2, 2); + writer.writeString(value.field2); + } + writer.finishLengthDelimited( + nestedValueStart, + writer.pos - nestedValueStartPos + ); + }); + + // Repeated fixed32 field8 (wire type 5) + const field8Values = [0x11223344, 0x55667788, 0xffffffff, 0x00000000]; + for (const f32 of field8Values) { + writer.writeTag(8, 5); + writer.writeFixed32(f32); + } + + // Repeated fixed64 field9 (wire type 1) + const field9Expected = [ + 361984551007945476n, // non-zero low and high words + 1013046106550635533n, // crosses 2^32 boundary + 42n, // fits in low 32 bits + 0n, // zero value + ]; + + const field9ExpectedWithProtobufJsPrecisionLoss = field9Expected.map( + val => { + // force precision loss that happens with protobuf.js in the browser. + return Number(val); + } + ); + + for (const b of field9Expected) { + const bits = toLongBits(BigInt(b)); + writer.writeTag(9, 1); + writer.writeFixed64(bits.low, bits.high); + } + + // Repeated double field10 (wire type 1 - 64-bit IEEE 754) + const field10Values = [ + 0.0, + -0.0, + 1.5, + -1234.5678, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ]; + for (const d of field10Values) { + writer.writeTag(10, 1); + writer.writeDouble(d); + } + + const buffer = writer.finish(); + + // Decode using generated protobuf types + const decoded = testbed.TestMessage.toObject( + testbed.TestMessage.decode(buffer), + { + // This is a protobuf.js quirk and incurs some precision loss that's taken into account in the expected data. + // Using String here will incur the same precision loss on browser only, using Number to prevent having to + // have different assertions for browser and Node.js. + // We may have to change the expected values to match the input exactly if we ever switch + // to another protobuf implementation for testing. + longs: Number, + } + ) as testbed.ITestMessage; + + // field 1 + assert.strictEqual(decoded.field1, 42); + + // field 2 + assert.strictEqual(decoded.field2, 'test'); + + // field 3 nested + assert.ok(decoded.field3); + assert.strictEqual(decoded.field3.field1, 100); + + // field4 repeated + assert.deepStrictEqual(decoded.field4, field4Values); + + // field5 repeated + assert.deepStrictEqual(decoded.field5, field5Values); + + // field6 repeated - decoded bytes may be Buffer/Uint8Array; compare contents + assert.deepStrictEqual( + decoded.field6?.map(item => new Uint8Array(item)), // protobuf.js loves to give us Buffers on Node.js, so convert to Uint8Array + field6Values + ); + + // field7 nested repeated + assert.deepStrictEqual(decoded.field7, field7Values); + + // field8 fixed32 repeated + assert.deepStrictEqual(decoded.field8, field8Values); + + // field10 repeated double + assert.deepStrictEqual(decoded.field10, field10Values); + + // field9 fixed64 repeated. + assert.deepStrictEqual( + decoded.field9, + field9ExpectedWithProtobufJsPrecisionLoss + ); + }); + }); + + describe('buffer growth', function () { + let sandbox: sinon.SinonSandbox; + + beforeEach(function () { + sandbox = sinon.createSandbox(); + }); + + afterEach(function () { + sandbox.restore(); + }); + + it('should grow buffer when capacity is exceeded', function () { + const diagStub = sandbox.stub(diag, 'debug'); + + const writer = new ProtobufWriter(4); + // Write more than 4 bytes + writer.writeVarint(1); + writer.writeVarint(2); + writer.writeVarint(3); + writer.writeVarint(4); + writer.writeVarint(5); + + const buffer = writer.finish(); + assert.strictEqual(buffer.length, 5); + + sinon.assert.calledWith(diagStub, GROWING_BUFFER_DEBUG_MESSAGE); + }); + + it('should handle buffer growth during writeBytes', function () { + const diagStub = sandbox.stub(diag, 'debug'); + + const writer = new ProtobufWriter(4); + const largeData = new Uint8Array(100); + writer.writeBytes(largeData); + + const buffer = writer.finish(); + assert.strictEqual(buffer.length, 101); + + sinon.assert.calledWith(diagStub, GROWING_BUFFER_DEBUG_MESSAGE); + }); + + it('should handle buffer growth during writeString', function () { + const diagStub = sandbox.stub(diag, 'debug'); + + const writer = new ProtobufWriter(4); + const longString = 'a'.repeat(100); + writer.writeString(longString); + + const buffer = writer.finish(); + assert.strictEqual(buffer.length, 101); + + sinon.assert.calledWith(diagStub, GROWING_BUFFER_DEBUG_MESSAGE); + }); + }); + + describe('large sub-messages', function () { + it('should not shift message on 1-byte varint length prefix', function () { + const writer = new ProtobufWriter(4); + const messageStart = writer.startLengthDelimited(); // reserve space + writer.writeVarint(1); + writer.writeVarint(2); + writer.writeVarint(3); + writer.finishLengthDelimited(messageStart, 3); // accurate length that fits in 1 byte, so no shifting needed + const result = writer.finish(); + + assert.deepStrictEqual( + result, + new Uint8Array([ + ...[3] /* varint encoding of 3 */, + ...[1, 2, 3] /* message contents remain in-place */, + ]) + ); + }); + + // Note: the following tests pretend that the message is larger than it actually is + // to avoid actually having to use such large messages. This keeps memory use acceptable + // as otherwise we'd be using ~2GiB of memory for the largest test, which is not practical. + + it('should shift message on 2-byte varint length prefix', function () { + const writer = new ProtobufWriter(4); + const messageStart = writer.startLengthDelimited(); // reserve space + writer.writeVarint(1); + writer.writeVarint(2); + writer.writeVarint(3); + writer.finishLengthDelimited(messageStart, 128); // pretend message is 128 bytes to force 2-byte varint length + const result = writer.finish(); + + assert.deepStrictEqual( + result, + new Uint8Array([ + ...[128, 1] /* varint encoding of 128 */, + ...[1, 2, 3] /* message contents properly shifted */, + ]) + ); + }); + + it('should shift message on 3-byte varint length prefix', function () { + const writer = new ProtobufWriter(4); + const messageStart = writer.startLengthDelimited(); // reserve space + writer.writeVarint(1); + writer.writeVarint(2); + writer.writeVarint(3); + writer.finishLengthDelimited(messageStart, Math.pow(2, 21) - 1); // pretend message is 2^21-1 bytes to force 3-byte varint length + const result = writer.finish(); + + assert.deepStrictEqual( + result, + new Uint8Array([ + ...[255, 255, 127] /* varint encoding of 2^21-1 */, + ...[1, 2, 3] /* message contents properly shifted */, + ]) + ); + }); + + it('should shift message on 4-byte varint length prefix', function () { + const writer = new ProtobufWriter(4); + const messageStart = writer.startLengthDelimited(); // reserve space + writer.writeVarint(1); + writer.writeVarint(2); + writer.writeVarint(3); + writer.finishLengthDelimited(messageStart, Math.pow(2, 28) - 1); // pretend message is 2^28-1 bytes to force 4-byte varint length + const result = writer.finish(); + + assert.deepStrictEqual( + result, + new Uint8Array([ + ...[255, 255, 255, 127] /* varint encoding of 2^28-1 */, + ...[1, 2, 3] /* message contents properly shifted */, + ]) + ); + }); + + it('should shift message on 5-byte varint length prefix', function () { + const writer = new ProtobufWriter(4); + const messageStart = writer.startLengthDelimited(); // reserve space + writer.writeVarint(1); + writer.writeVarint(2); + writer.writeVarint(3); + writer.finishLengthDelimited(messageStart, Math.pow(2, 31) - 1); // pretend message is 2^31-1 bytes to force 5-byte varint length + const result = writer.finish(); + + assert.deepStrictEqual( + result, + new Uint8Array([ + ...[255, 255, 255, 255, 7] /* varint encoding of 2^31-1 */, + ...[1, 2, 3] /* message contents properly shifted */, + ]) + ); + }); + }); +}); diff --git a/experimental/packages/otlp-transformer/test/utils.ts b/experimental/packages/otlp-transformer/test/utils.ts index 7a51a753b06..3de50b3b76c 100644 --- a/experimental/packages/otlp-transformer/test/utils.ts +++ b/experimental/packages/otlp-transformer/test/utils.ts @@ -14,3 +14,11 @@ export function toBase64(hexStr: string) { const decoder = new TextDecoder('utf8'); return btoa(decoder.decode(hexToBinary(hexStr))); } + +/** + * Cross-platform utility function to convert a Uint8Array to a base64 string. + * This works in both Node.js and browsers so that we can avoid using Buffer in tests. + */ +export function uint8ArrayToBase64(bytes: Uint8Array): string { + return btoa(String.fromCharCode(...bytes)); +} diff --git a/experimental/packages/sdk-logs/src/export/ReadableLogRecord.ts b/experimental/packages/sdk-logs/src/export/ReadableLogRecord.ts index 4be7738f677..7e1f6046a20 100644 --- a/experimental/packages/sdk-logs/src/export/ReadableLogRecord.ts +++ b/experimental/packages/sdk-logs/src/export/ReadableLogRecord.ts @@ -21,6 +21,11 @@ export interface ReadableLogRecord { readonly body?: LogBody; readonly eventName?: string; readonly resource: Resource; + /** + * The instrumentation scope associated with this log record. Identity of this object + * MUST be stable across identical scopes, as it is intended be used for efficient scope-based + * filtering and grouping. + */ readonly instrumentationScope: InstrumentationScope; readonly attributes: LogAttributes; readonly droppedAttributesCount: number;