fix(fizarrita): classify codecs by size-preservation, not by compressor name

fizarrita/src/get-worker.ts

@@ -12,6 +12,7 @@
 import type { WorkerPool, WorkerPoolTask } from "@fideus-labs/worker-pool"
 import type {
   Chunk,
+  CodecMetadata,
   DataType,
   Readable,
   Scalar,
@@ -286,14 +287,54 @@ export function readBloscFrameContentSize(
   return nbytes
 }
 
+/**
+ * Codecs whose encoded output is exactly as long as the array bytes they encode,
+ * so that a raw chunk's `byteLength` IS its decoded byte length.
+ *
+ * The list is short because the property is strict. `transpose` reorders and
+ * `bytes` reinterprets, neither changing the count. Everything else changes it
+ * one way or another: compressors shrink, `crc32c` appends a checksum,
+ * `scale_offset` and `cast_value` re-type the values, `vlen-utf8` and `json2`
+ * are variable-length, and `sharding_indexed` wraps a whole index plus inner
+ * chunks that are usually compressed themselves.
+ *
+ * v2-style `numcodecs.` prefixes are stripped before lookup, so
+ * `numcodecs.transpose` matches.
+ */
+const SIZE_PRESERVING_CODECS = new Set(["bytes", "transpose"])
+
+/**
+ * Whether any codec in the chain makes the raw chunk length differ from the
+ * decoded length.
+ *
+ * Deliberately answered by allowlisting the codecs known to preserve size, not
+ * by listing the compressors: an unrecognised codec must count as size-changing.
+ * The two mistakes are not symmetric — treating a size-changing codec as
+ * size-preserving returns the *compressed* length as the decompressed one, a
+ * silently wrong number that {@link inferChunkShape} then takes as fact, while
+ * treating a size-preserving codec as size-changing only costs one decode via
+ * the fallback and still returns the right answer.
+ *
+ * Naming compressors instead put every codec outside a fixed list —
+ * e.g. HTJ2K — on the silently-wrong side.
+ */
+export function hasSizeChangingCodec(
+  codecs: readonly Pick<CodecMetadata, "name">[],
+): boolean {
+  return codecs.some((codec) => {
+    const name = codec.name.toLowerCase().replace(/^numcodecs\./, "")
+    return !SIZE_PRESERVING_CODECS.has(name)
+  })
+}
+
 /**
  * Try to determine the decompressed byte size of a raw chunk without full decoding.
  *
  * Hybrid strategy (cheapest first):
  *  1. Zstd frame header — read FCS field (zero-cost, no decompression)
  *  2. Blosc header — read nbytes field (zero-cost, no decompression)
- *  3. Uncompressed check — if raw byte count matches a plausible element count,
- *     the chunk may be uncompressed (bytes codec only)
+ *  3. Size-preserving check — if every codec in the chain preserves byte count,
+ *     the raw byte count IS the decompressed size
  *  4. Full decode — decode chunk c/0/0/0 using the codec pipeline and count elements
  *
  * Returns the decompressed byte size, or null if detection failed.
@@ -311,28 +352,10 @@ async function probeDecompressedSize<D extends DataType>(
   const bloscSize = readBloscFrameContentSize(rawBytes)
   if (bloscSize != null) return bloscSize
 
-  // 3. Check if the raw bytes could be an uncompressed chunk.
-  //    For the bytes codec (no compression), rawBytes.byteLength IS the
-  //    decompressed size. We check whether the codec chain is bytes-only
-  //    (no bytes_to_bytes compression codecs).
-  const hasCompression = codecMeta.codecs.some((c) => {
-    const name = c.name.toLowerCase()
-    // array_to_array codecs (transpose, etc.) don't change byte size
-    // array_to_bytes codecs (bytes, etc.) don't compress
-    // bytes_to_bytes codecs are the compressors
-    return (
-      name === "gzip" ||
-      name === "zlib" ||
-      name === "blosc" ||
-      name === "zstd" ||
-      name === "lz4" ||
-      name === "bz2" ||
-      name === "lzma" ||
-      name === "snappy"
-    )
-  })
-  if (!hasCompression) {
-    // No compression codec — raw bytes are the decompressed data
+  // 3. Check whether the codec chain preserves byte count end to end — a
+  //    transpose + bytes chain does, not just a bare bytes one. When it does,
+  //    rawBytes.byteLength IS the decompressed size.
+  if (!hasSizeChangingCodec(codecMeta.codecs)) {
     return rawBytes.byteLength
   }
 

fizarrita/src/index.ts

@@ -14,6 +14,7 @@ export {
   DEFAULT_WORKER_URL,
   getStoreId,
   getWorker,
+  hasSizeChangingCodec,
   inferChunkShape,
   probeActualChunkShape,
   readArrayMetadata,

test/app/main.ts

@@ -67,7 +67,7 @@ import type {
 // ---------------------------------------------------------------------------
 
 import * as zarr from 'zarrita'
-import { getWorker, setWorker, readZstdFrameContentSize, readBloscFrameContentSize, inferChunkShape } from '../../fizarrita/src/index.js'
+import { getWorker, setWorker, readZstdFrameContentSize, readBloscFrameContentSize, inferChunkShape, hasSizeChangingCodec } from '../../fizarrita/src/index.js'
 import type { GetWorkerOptions, SetWorkerOptions, ChunkCache } from '../../fizarrita/src/index.js'
 
 // Expose helpers on the window so Playwright tests can call them.
@@ -94,6 +94,7 @@ declare global {
     readZstdFrameContentSize: typeof readZstdFrameContentSize
     readBloscFrameContentSize: typeof readBloscFrameContentSize
     inferChunkShape: typeof inferChunkShape
+    hasSizeChangingCodec: typeof hasSizeChangingCodec
   }
 }
 
@@ -120,3 +121,4 @@ window.setWorker = setWorker
 window.readZstdFrameContentSize = readZstdFrameContentSize
 window.readBloscFrameContentSize = readBloscFrameContentSize
 window.inferChunkShape = inferChunkShape
+window.hasSizeChangingCodec = hasSizeChangingCodec

test/browser/chunk-shape-inference.spec.ts

@@ -744,3 +744,66 @@ test.describe('getWorker — chunk shape auto-detection integration', () => {
     ])
   })
 })
+
+test.describe('hasSizeChangingCodec', () => {
+  test.beforeEach(async ({ page }) => {
+    await page.goto('/')
+    await page.waitForFunction(() => typeof window.hasSizeChangingCodec === 'function')
+  })
+
+  const classify = (page: import('@playwright/test').Page, names: string[]) =>
+    page.evaluate(
+      (codecNames) => window.hasSizeChangingCodec(codecNames.map((name) => ({ name }))),
+      names,
+    )
+
+  test('treats a bytes-only chain as size-preserving', async ({ page }) => {
+    expect(await classify(page, ['bytes'])).toBe(false)
+  })
+
+  test('treats transpose + bytes as size-preserving', async ({ page }) => {
+    expect(await classify(page, ['transpose', 'bytes'])).toBe(false)
+  })
+
+  test('strips the numcodecs. prefix before matching', async ({ page }) => {
+    expect(await classify(page, ['numcodecs.transpose', 'bytes'])).toBe(false)
+  })
+
+  test('is case-insensitive', async ({ page }) => {
+    expect(await classify(page, ['Bytes', 'TRANSPOSE'])).toBe(false)
+  })
+
+  test('flags the known compressors', async ({ page }) => {
+    for (const name of ['gzip', 'zlib', 'blosc', 'zstd', 'lz4']) {
+      expect(await classify(page, ['bytes', name])).toBe(true)
+    }
+  })
+
+  // The regression this function exists for. A JPEG 2000 chunk is compressed by
+  // an order of magnitude or more, but its codec name is on no fixed list of
+  // compressors — so a compressor-allowlist reported the *compressed* length as
+  // the decompressed one and fed that to inferChunkShape as fact.
+  test('flags codecs it does not recognise, including JPEG 2000 and HTJ2K', async ({ page }) => {
+    for (const name of ['imagecodecs_jpeg2k', 'htj2k', 'jpeg2k', 'imagecodecs_jpegxl']) {
+      expect(await classify(page, ['bytes', name])).toBe(true)
+    }
+  })
+
+  // Not compression, but not size-preserving either: crc32c appends a 4-byte
+  // checksum, and scale_offset/cast_value re-type the values.
+  test('flags codecs that resize without compressing', async ({ page }) => {
+    for (const name of ['crc32c', 'scale_offset', 'cast_value', 'vlen-utf8', 'json2']) {
+      expect(await classify(page, ['bytes', name])).toBe(true)
+    }
+  })
+
+  // Sharding wraps an index plus inner chunks that are usually compressed
+  // themselves, so a shard's raw length is never its decoded length.
+  test('flags sharding', async ({ page }) => {
+    expect(await classify(page, ['sharding_indexed'])).toBe(true)
+  })
+
+  test('treats an empty codec chain as size-preserving', async ({ page }) => {
+    expect(await classify(page, [])).toBe(false)
+  })
+})