test(o11ylogsdb): unit test suite (82 tests, 8 files)

packages/o11ylogsdb/test/chunk.test.ts

@@ -0,0 +1,180 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  CHUNK_VERSION,
+  ChunkBuilder,
+  type ChunkPolicy,
+  chunkWireSize,
+  DefaultChunkPolicy,
+  deserializeChunk,
+  readRecords,
+  serializeChunk,
+} from "../src/chunk.js";
+import { defaultRegistry } from "../src/codec-baseline.js";
+import type { InstrumentationScope, LogRecord, Resource } from "../src/types.js";
+
+const resource: Resource = { attributes: [{ key: "service.name", value: "test" }] };
+const scope: InstrumentationScope = { name: "test-scope" };
+const registry = defaultRegistry();
+
+function rec(partial: Partial<LogRecord> & { timeUnixNano: bigint }): LogRecord {
+  return {
+    severityNumber: 9,
+    severityText: "INFO",
+    body: "hello",
+    attributes: [],
+    ...partial,
+  };
+}
+
+describe("ChunkBuilder", () => {
+  it("freezes an empty chunk with sentinel severity range and zero time", () => {
+    const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy(), registry);
+    const chunk = builder.freeze();
+    expect(chunk.header.nLogs).toBe(0);
+    expect(chunk.header.severityRange).toEqual({ min: 1, max: 24 });
+    expect(chunk.header.timeRange).toEqual({ minNano: "0", maxNano: "0" });
+  });
+
+  it("computes time range from first/last record", () => {
+    const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy(), registry);
+    builder.append(rec({ timeUnixNano: 100n }));
+    builder.append(rec({ timeUnixNano: 200n }));
+    builder.append(rec({ timeUnixNano: 300n }));
+    const chunk = builder.freeze();
+    expect(chunk.header.timeRange).toEqual({ minNano: "100", maxNano: "300" });
+  });
+
+  it("computes severity range across all records", () => {
+    const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy(), registry);
+    builder.append(rec({ timeUnixNano: 1n, severityNumber: 17 })); // ERROR
+    builder.append(rec({ timeUnixNano: 2n, severityNumber: 5 })); // DEBUG
+    builder.append(rec({ timeUnixNano: 3n, severityNumber: 9 })); // INFO
+    const chunk = builder.freeze();
+    expect(chunk.header.severityRange).toEqual({ min: 5, max: 17 });
+  });
+
+  it("hoists resource and scope into the header", () => {
+    const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy(), registry);
+    builder.append(rec({ timeUnixNano: 1n }));
+    const chunk = builder.freeze();
+    expect(chunk.header.resource).toEqual(resource);
+    expect(chunk.header.scope).toEqual(scope);
+  });
+
+  it("size() reflects appended records and reset() clears them", () => {
+    const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy(), registry);
+    expect(builder.size()).toBe(0);
+    builder.append(rec({ timeUnixNano: 1n }));
+    builder.append(rec({ timeUnixNano: 2n }));
+    expect(builder.size()).toBe(2);
+    builder.reset();
+    expect(builder.size()).toBe(0);
+  });
+});
+
+describe("chunk wire format", () => {
+  it("round-trips a chunk through serialize/deserialize", () => {
+    const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy(), registry);
+    builder.append(rec({ timeUnixNano: 1n, body: "first" }));
+    builder.append(rec({ timeUnixNano: 2n, body: "second" }));
+    const chunk = builder.freeze();
+    const bytes = serializeChunk(chunk);
+    const parsed = deserializeChunk(bytes);
+    expect(parsed.header).toEqual(chunk.header);
+    expect(Array.from(parsed.payload)).toEqual(Array.from(chunk.payload));
+  });
+
+  it("rejects chunks with bad magic bytes", () => {
+    const bytes = new Uint8Array([0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00]);
+    expect(() => deserializeChunk(bytes)).toThrow(/bad chunk magic/);
+  });
+
+  it("schemaVersion in the header equals CHUNK_VERSION", () => {
+    const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy(), registry);
+    const chunk = builder.freeze();
+    expect(chunk.header.schemaVersion).toBe(CHUNK_VERSION);
+  });
+});
+
+describe("chunkWireSize", () => {
+  it("matches serializeChunk(c).length without materializing the buffer", () => {
+    const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy(), registry);
+    for (let i = 0; i < 10; i++) {
+      builder.append(rec({ timeUnixNano: BigInt(i), body: `record-${i}` }));
+    }
+    const chunk = builder.freeze();
+    expect(chunkWireSize(chunk)).toBe(serializeChunk(chunk).length);
+  });
+});
+
+describe("readRecords with default policy", () => {
+  it("round-trips records through NDJSON encode/decode", () => {
+    const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy(), registry);
+    const inputs = [
+      rec({ timeUnixNano: 100n, body: "alpha", severityNumber: 9 }),
+      rec({ timeUnixNano: 200n, body: "beta", severityNumber: 13 }),
+    ];
+    for (const r of inputs) builder.append(r);
+    const chunk = builder.freeze();
+    const records = readRecords(chunk, registry);
+    expect(records.length).toBe(2);
+    expect(records[0]?.body).toBe("alpha");
+    expect(records[0]?.timeUnixNano).toBe(100n);
+    expect(records[1]?.body).toBe("beta");
+    expect(records[1]?.severityNumber).toBe(13);
+  });
+});
+
+describe("ChunkPolicy hooks", () => {
+  it("preEncode/postDecode round-trips a meta blob", () => {
+    const seenMeta: unknown[] = [];
+    const policy: ChunkPolicy = {
+      bodyCodec: () => "zstd-19",
+      preEncode: (records) => ({ records, meta: { tag: "preEncodeMeta" } }),
+      postDecode: (records, meta) => {
+        seenMeta.push(meta);
+        return records;
+      },
+    };
+    const builder = new ChunkBuilder(resource, scope, policy, registry);
+    builder.append(rec({ timeUnixNano: 1n }));
+    const chunk = builder.freeze();
+    expect(chunk.header.codecMeta).toEqual({ tag: "preEncodeMeta" });
+    readRecords(chunk, registry, policy);
+    expect(seenMeta).toEqual([{ tag: "preEncodeMeta" }]);
+  });
+
+  it("encodePayload/decodePayload bypasses NDJSON entirely", () => {
+    let encodeCalls = 0;
+    let decodeCalls = 0;
+    const policy: ChunkPolicy = {
+      bodyCodec: () => "raw",
+      encodePayload: (records) => {
+        encodeCalls++;
+        const json = JSON.stringify(records.map((r) => r.body));
+        return { payload: new TextEncoder().encode(json), meta: { kind: "binary" } };
+      },
+      decodePayload: (buf, _n, meta) => {
+        decodeCalls++;
+        expect(meta).toEqual({ kind: "binary" });
+        const bodies = JSON.parse(new TextDecoder().decode(buf)) as string[];
+        return bodies.map((body, i) => ({
+          timeUnixNano: BigInt(i),
+          severityNumber: 9,
+          severityText: "INFO",
+          body,
+          attributes: [],
+        }));
+      },
+    };
+    const builder = new ChunkBuilder(resource, scope, policy, registry);
+    builder.append(rec({ timeUnixNano: 1n, body: "x" }));
+    builder.append(rec({ timeUnixNano: 2n, body: "y" }));
+    const chunk = builder.freeze();
+    expect(encodeCalls).toBe(1);
+    const records = readRecords(chunk, registry, policy);
+    expect(decodeCalls).toBe(1);
+    expect(records.map((r) => r.body)).toEqual(["x", "y"]);
+  });
+});

packages/o11ylogsdb/test/compact.test.ts

@@ -0,0 +1,68 @@
+import { describe, expect, it } from "vitest";
+
+import { ChunkBuilder, DefaultChunkPolicy, readRecords } from "../src/chunk.js";
+import { defaultRegistry } from "../src/codec-baseline.js";
+import { compactChunk } from "../src/compact.js";
+import type { InstrumentationScope, LogRecord, Resource } from "../src/types.js";
+
+const resource: Resource = { attributes: [{ key: "service.name", value: "test" }] };
+const scope: InstrumentationScope = { name: "test-scope" };
+const registry = defaultRegistry();
+
+function buildZ19Chunk(n: number) {
+  const builder = new ChunkBuilder(resource, scope, new DefaultChunkPolicy("zstd-19"), registry);
+  for (let i = 0; i < n; i++) {
+    const r: LogRecord = {
+      timeUnixNano: BigInt(i),
+      severityNumber: 9,
+      severityText: "INFO",
+      body: `record ${i} payload-payload-payload-payload`,
+      attributes: [],
+    };
+    builder.append(r);
+  }
+  return builder.freeze();
+}
+
+describe("compactChunk", () => {
+  it("re-encodes the payload under a new codec and preserves records", () => {
+    const z19 = buildZ19Chunk(50);
+    const { chunk: z3 } = compactChunk(z19, registry, "zstd-3");
+    expect(z3.header.codecName).toBe("zstd-3");
+    expect(z3.header.payloadBytes).toBe(z3.payload.length);
+    // Round-trip equivalence — same record sequence.
+    const before = readRecords(z19, registry);
+    const after = readRecords(z3, registry);
+    expect(after.length).toBe(before.length);
+    for (let i = 0; i < before.length; i++) {
+      expect(after[i]?.body).toBe(before[i]?.body);
+      expect(after[i]?.timeUnixNano).toBe(before[i]?.timeUnixNano);
+    }
+  });
+
+  it("reports stats matching the new and old payload sizes", () => {
+    const z19 = buildZ19Chunk(50);
+    const { chunk: z3, stats } = compactChunk(z19, registry, "zstd-3");
+    expect(stats.inputBytes).toBe(z19.payload.length);
+    expect(stats.outputBytes).toBe(z3.payload.length);
+    expect(stats.decodeMillis).toBeGreaterThanOrEqual(0);
+    expect(stats.encodeMillis).toBeGreaterThanOrEqual(0);
+  });
+
+  it("is a no-op when target codec equals current codec", () => {
+    const z19 = buildZ19Chunk(10);
+    const { chunk, stats } = compactChunk(z19, registry, "zstd-19");
+    expect(chunk).toBe(z19);
+    expect(stats.decodeMillis).toBe(0);
+    expect(stats.encodeMillis).toBe(0);
+  });
+
+  it("does not mutate the input chunk", () => {
+    const z19 = buildZ19Chunk(10);
+    const originalCodec = z19.header.codecName;
+    const originalPayloadLen = z19.payload.length;
+    compactChunk(z19, registry, "zstd-3");
+    expect(z19.header.codecName).toBe(originalCodec);
+    expect(z19.payload.length).toBe(originalPayloadLen);
+  });
+});

packages/o11ylogsdb/test/drain.test.ts

@@ -0,0 +1,138 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  DRAIN_DEFAULT_CONFIG,
+  Drain,
+  mergeTemplate,
+  PARAM_STR,
+  similarity,
+  tokenize,
+} from "../src/drain.js";
+
+describe("tokenize", () => {
+  it("splits on whitespace runs", () => {
+    expect(tokenize("foo   bar  baz")).toEqual(["foo", "bar", "baz"]);
+  });
+
+  it("returns an empty array for empty/whitespace-only input", () => {
+    expect(tokenize("")).toEqual([]);
+    expect(tokenize("   ")).toEqual([]);
+  });
+});
+
+describe("similarity", () => {
+  it("returns 1.0 for identical token lists with no params", () => {
+    const [sim, paramCount] = similarity(["a", "b", "c"], ["a", "b", "c"]);
+    expect(sim).toBe(1);
+    expect(paramCount).toBe(0);
+  });
+
+  it("treats wildcard positions as non-matches in the numerator", () => {
+    const [sim, paramCount] = similarity([PARAM_STR, "b", "c"], ["a", "b", "c"]);
+    expect(sim).toBeCloseTo(2 / 3);
+    expect(paramCount).toBe(1);
+  });
+
+  it("returns 0 when no positions match", () => {
+    const [sim] = similarity(["a", "b", "c"], ["x", "y", "z"]);
+    expect(sim).toBe(0);
+  });
+});
+
+describe("mergeTemplate", () => {
+  it("replaces mismatched positions with PARAM_STR and reports change", () => {
+    const tpl = ["GET", "/api/users/123", "200"];
+    const changed = mergeTemplate(tpl, ["GET", "/api/users/456", "200"]);
+    expect(tpl).toEqual(["GET", PARAM_STR, "200"]);
+    expect(changed).toBe(true);
+  });
+
+  it("returns false when nothing changes", () => {
+    const tpl = ["GET", "/health", "200"];
+    const changed = mergeTemplate(tpl, ["GET", "/health", "200"]);
+    expect(changed).toBe(false);
+  });
+});
+
+describe("Drain.matchOrAdd", () => {
+  it("creates a new cluster for the first line of a shape", () => {
+    const drain = new Drain();
+    const r = drain.matchOrAdd("user 42 logged in");
+    expect(r.isNew).toBe(true);
+    expect(r.templateId).toBe(1);
+    expect(drain.templateCount()).toBe(1);
+  });
+
+  it("matches a similar second line and merges variable positions", () => {
+    const drain = new Drain();
+    const a = drain.matchOrAdd("user 42 logged in");
+    const b = drain.matchOrAdd("user 99 logged in");
+    expect(b.isNew).toBe(false);
+    expect(b.templateId).toBe(a.templateId);
+    expect(b.vars).toEqual(["99"]);
+    expect(drain.templateCount()).toBe(1);
+  });
+
+  it("creates a separate cluster for a different token-count shape", () => {
+    const drain = new Drain();
+    drain.matchOrAdd("user 42 logged in");
+    drain.matchOrAdd("connection lost");
+    expect(drain.templateCount()).toBe(2);
+  });
+
+  it("emits stable, sequential cluster ids", () => {
+    const drain = new Drain();
+    expect(drain.matchOrAdd("a x b").templateId).toBe(1);
+    expect(drain.matchOrAdd("c y d").templateId).toBe(2);
+    expect(drain.matchOrAdd("a z b").templateId).toBe(1);
+  });
+});
+
+describe("Drain.matchTemplate", () => {
+  it("returns undefined for a never-seen line", () => {
+    const drain = new Drain();
+    expect(drain.matchTemplate("nothing here")).toBeUndefined();
+  });
+
+  it("does not mutate state", () => {
+    const drain = new Drain();
+    drain.matchOrAdd("user 42 logged in");
+    const before = drain.templateCount();
+    drain.matchTemplate("user 99 logged in");
+    drain.matchTemplate("brand new line never seen");
+    expect(drain.templateCount()).toBe(before);
+  });
+});
+
+describe("Drain.reconstruct", () => {
+  it("produces a single-space-joined line that round-trips simple templates", () => {
+    const drain = new Drain();
+    drain.matchOrAdd("user 42 logged in");
+    const r = drain.matchOrAdd("user 99 logged in");
+    const cluster = [...drain.templates()][0];
+    expect(cluster).toBeDefined();
+    const tokens = (cluster?.template ?? "").split(" ");
+    expect(Drain.reconstruct(tokens, r.vars)).toBe("user 99 logged in");
+  });
+
+  it("normalizes runs of whitespace to single spaces (the documented contract)", () => {
+    const drain = new Drain();
+    drain.matchOrAdd("user 42 logged in");
+    const r = drain.matchOrAdd("user   99   logged   in");
+    const cluster = [...drain.templates()][0];
+    expect(cluster).toBeDefined();
+    const tokens = (cluster?.template ?? "").split(" ");
+    expect(Drain.reconstruct(tokens, r.vars)).toBe("user 99 logged in");
+  });
+});
+
+describe("DRAIN_DEFAULT_CONFIG", () => {
+  it("matches the published reference defaults", () => {
+    expect(DRAIN_DEFAULT_CONFIG).toEqual({
+      depth: 4,
+      simTh: 0.4,
+      maxChildren: 100,
+      parametrizeNumericTokens: true,
+    });
+  });
+});