From 725f070d0c288ca4d49d691714f4bc0cb6c4d8f6 Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Tue, 28 Jul 2026 15:20:37 +0100 Subject: [PATCH 1/3] Read AnnData's nullable-encoded columns in the JS reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A nullable column is stored as a *group* of `values` + `mask`, not an array, so opening it as an array fails outright. Because `obs/_index` and `var/_index` are themselves columns, the visible symptom was missing variable names rather than a missing value — the browser fell back to `varN`. This is not confined to stores our writer has touched: AnnData writes nullable encodings by default from 0.13, and `spatialdata` inherits that, so freshly written stores carry them too. Handles all three encodings sharing the layout — `nullable-string-array`, `nullable-integer`, `nullable-boolean` — since they differ only in the dtype of `values`. The mask is applied rather than discarded, so a missing entry reads as `null` and stays distinguishable from an empty string or a real zero. Both read paths are covered: `_loadColumn`, which dispatches on `encoding-type`, and `getFlatArrDecompressed`, which index reads reach directly and which now resolves the node before assuming it is an array. A group with any other encoding still fails, with a message naming what was found. Not routed through `anndata.js`: it dispatches none of the nullable encodings (in the published 0.0.2 and on main), and it pins zarrita 0.5.1, which is the subject of the still-open upstream #48. Our index and column reads already deliberately bypass it for exactly this reason. Co-Authored-By: Claude Opus 5 --- packages/core/src/models/VAnnDataSource.ts | 20 ++- packages/core/src/models/nullableArrays.ts | 63 +++++++ .../core/tests/nullableStringArray.spec.ts | 163 ++++++++++++++++++ 3 files changed, 244 insertions(+), 2 deletions(-) create mode 100644 packages/core/src/models/nullableArrays.ts create mode 100644 packages/core/tests/nullableStringArray.spec.ts diff --git a/packages/core/src/models/VAnnDataSource.ts b/packages/core/src/models/VAnnDataSource.ts index 07ea1e2a..e3f17158 100644 --- a/packages/core/src/models/VAnnDataSource.ts +++ b/packages/core/src/models/VAnnDataSource.ts @@ -1,7 +1,9 @@ +import * as zarr from 'zarrita'; import { get as zarrGet, open as zarrOpen } from 'zarrita'; import type { TableColumnData } from '../types'; import type { DataSourceParams } from '../Vutils'; import { dirname } from '../Vutils'; +import { isNullableEncoding, readNullableArray } from './nullableArrays'; import ZarrDataSource from './VZarrDataSource'; function prependSlash(path: string) { @@ -107,6 +109,8 @@ export default class AnnDataSource extends ZarrDataSource { codesPath = `${path}/codes`; } else if (encodingType === 'string-array') { return this.getFlatArrDecompressed(path); + } else if (isNullableEncoding(encodingType)) { + return readNullableArray(storeRoot.resolve(path)); } else { const { dtype } = await zarrOpen(storeRoot.resolve(path), { kind: 'array' }); if (dtype === 'v2:object') { @@ -168,9 +172,21 @@ export default class AnnDataSource extends ZarrDataSource { */ async getFlatArrDecompressed(path: string) { const { storeRoot } = this; - const arr = await zarrOpen(storeRoot.resolve(path), { kind: 'array' }); + const location = storeRoot.resolve(path); + // A nullable column is a group, so opening it as an array throws. Resolve the + // node first rather than assuming, since index paths reach here directly. + const node = await zarrOpen(location); + if (node instanceof zarr.Group) { + if (!isNullableEncoding(node.attrs['encoding-type'])) { + throw new Error( + `Expected an array at ${path}, but found a group encoded as ` + + `${String(node.attrs['encoding-type'] ?? 'unknown')}.` + ); + } + return (await readNullableArray(location)) as string[]; + } // Zarrita supports decoding vlen-utf8-encoded string arrays. - const data = await zarrGet(arr); + const data = await zarrGet(node); if (data.data?.[Symbol.iterator]) { return Array.from(data.data) as string[]; } diff --git a/packages/core/src/models/nullableArrays.ts b/packages/core/src/models/nullableArrays.ts new file mode 100644 index 00000000..77a21201 --- /dev/null +++ b/packages/core/src/models/nullableArrays.ts @@ -0,0 +1,63 @@ +import { type Location, type Readable, get as zarrGet, open as zarrOpen } from 'zarrita'; +import type { TableValue } from '../types'; + +/** + * AnnData's nullable encodings, which store a column as a *group* rather than an + * array: a `values` array plus a boolean `mask` marking the null positions. + * + * Readers that open a column path as an array fail outright on these — the + * symptom is a missing index rather than a missing value, because + * `obs/_index` and `var/_index` are themselves ordinary columns. AnnData writes + * this layout by default from 0.13 onwards (and `spatialdata` inherits it), so + * it turns up in freshly written stores, not only in ones that have been + * rewritten. + * + * See the AnnData on-disk specification, "Nullable integers, booleans and + * strings" — all three share this group layout at encoding-version 0.1.0. + */ +export const NULLABLE_ENCODING_TYPES = new Set([ + 'nullable-string-array', + 'nullable-integer', + 'nullable-boolean', +]); + +export function isNullableEncoding(encodingType: unknown): boolean { + return typeof encodingType === 'string' && NULLABLE_ENCODING_TYPES.has(encodingType); +} + +/** + * Read a nullable-encoded column into a flat array, with `null` at masked + * positions. + * + * The mask is read alongside the values rather than ignored: a masked entry is + * absent, not empty, and callers that render it (tooltips, legends) need to be + * able to tell those apart. + */ +export async function readNullableArray(location: Location): Promise { + const valuesArray = await zarrOpen(location.resolve('values'), { kind: 'array' }); + const values = await zarrGet(valuesArray); + + // The specification requires a mask, but a values-only group is still + // unambiguous, so read what is there rather than failing on a missing mask. + let mask: boolean[] | undefined; + try { + const maskArray = await zarrOpen(location.resolve('mask'), { kind: 'array' }); + mask = toArray((await zarrGet(maskArray)).data).map(Boolean); + } catch { + mask = undefined; + } + + return toArray(values.data).map((value, index) => (mask?.[index] ? null : value)); +} + +/** + * Materialise a zarrita chunk as a plain array. + * + * Zarrita returns several backing types — native TypedArrays plus its own + * `BoolArray`/`ByteStringArray`/`UnicodeStringArray`. They are all iterable but + * do not share an indexable interface, so the union has no common supertype to + * narrow to; iterating is the one operation defined across all of them. + */ +function toArray(data: unknown): TableValue[] { + return Array.from(data as Iterable); +} diff --git a/packages/core/tests/nullableStringArray.spec.ts b/packages/core/tests/nullableStringArray.spec.ts new file mode 100644 index 00000000..5e0eb59c --- /dev/null +++ b/packages/core/tests/nullableStringArray.spec.ts @@ -0,0 +1,163 @@ +import { execSync } from 'node:child_process'; +import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import SpatialDataTableSource from '../src/models/VTableSource.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const writerRoot = join(__dirname, '../../../python/spatialdata-js-util'); + +/** + * Write a SpatialData store whose table uses AnnData's nullable encodings. + * + * These are groups of `values` + `mask`, not arrays. AnnData writes them by + * default from 0.13 onwards, so this is what a freshly written store looks like + * — a reader that opens `var/_index` as an array fails on it and has no variable + * names to show. + * + * Note that only the index ends up as `nullable-string-array`: AnnData's + * `strings_to_categoricals` turns string *columns* into categoricals on write, + * so the nullable string case is reached through `obs/_index` and `var/_index`. + * A nullable *integer* column exercises the same group layout with a mask that + * actually has a bit set. + */ +function writeNullableTableFixture(storeRoot: string) { + execSync( + `uv run --extra write python - <<'PY' +import anndata as ad +import numpy as np +import pandas as pd +import scipy.sparse as sp +import spatialdata as sd +from pathlib import Path +from spatialdata.models import TableModel + +root = Path(${JSON.stringify(storeRoot)}) + +adata = ad.AnnData(X=sp.random(10, 6, density=0.5, format="csc", random_state=0, dtype=np.float32)) +adata.var.index = pd.array([f"GENE{i}" for i in range(6)], dtype="string") +adata.obs.index = pd.array([f"cell{i}" for i in range(10)], dtype="string") +adata.obs["region"] = pd.Categorical(["img"] * 10) +adata.obs["instance_id"] = np.arange(10) +# A genuine missing value, so the mask is exercised rather than only the +# all-present case. +adata.var["measured"] = pd.array([1, 2, None, 4, 5, 6], dtype="Int64") + +table = TableModel.parse( + adata, region="img", region_key="region", instance_key="instance_id" +) + +ad.settings.zarr_write_format = 3 +ad.settings.auto_shard_zarr_v3 = False +ad.settings.allow_write_nullable_strings = True +sd.SpatialData(tables={"table": table}).write(root, overwrite=True) + +# Fail loudly here rather than letting the test assert against the wrong layout. +encoding = __import__("json").loads( + (root / "tables" / "table" / "var" / "_index" / "zarr.json").read_text() +)["attributes"]["encoding-type"] +assert encoding == "nullable-string-array", f"fixture wrote {encoding!r}" +PY`, + { cwd: writerRoot, stdio: 'pipe' } + ); +} + +function createFilesystemStore(root: string) { + const readStoreBytes = async (relativePath: string): Promise => { + const fullPath = join(root, relativePath); + try { + const info = await stat(fullPath); + if (info.isDirectory()) { + return null; + } + return await readFile(fullPath); + } catch { + return null; + } + }; + + return { + async get(path: string) { + const relativePath = path.startsWith('/') ? path.slice(1) : path; + return readStoreBytes(relativePath); + }, + async getRange( + path: string, + range: { offset?: number; length?: number; suffixLength?: number } + ) { + const relativePath = path.startsWith('/') ? path.slice(1) : path; + const bytes = await readStoreBytes(relativePath); + if (!bytes) { + return null; + } + if (range.suffixLength != null) { + return bytes.subarray(bytes.length - range.suffixLength); + } + const offset = range.offset ?? 0; + const length = range.length ?? bytes.length - offset; + return bytes.subarray(offset, offset + length); + }, + }; +} + +describe('nullable-encoded AnnData columns', () => { + let fixtureRoot: string; + let source: SpatialDataTableSource; + + beforeAll(async () => { + fixtureRoot = await mkdtemp(join(tmpdir(), 'nullable-anndata-')); + const storeRoot = join(fixtureRoot, 'store.zarr'); + writeNullableTableFixture(storeRoot); + source = new SpatialDataTableSource({ + store: createFilesystemStore(storeRoot), + fileType: '.zarr', + }); + }, 300_000); + + afterAll(async () => { + await rm(fixtureRoot, { recursive: true, force: true }); + }); + + it('reads var names from a nullable-string-array index', async () => { + const names = await source.loadVarIndex('tables/table'); + expect(names).toEqual(['GENE0', 'GENE1', 'GENE2', 'GENE3', 'GENE4', 'GENE5']); + }); + + it('reads a nullable-string-array obs index', async () => { + const [ids] = await source.loadObsColumns(['tables/table/obs/_index']); + expect(Array.from(ids ?? [])).toEqual([ + 'cell0', + 'cell1', + 'cell2', + 'cell3', + 'cell4', + 'cell5', + 'cell6', + 'cell7', + 'cell8', + 'cell9', + ]); + }); + + it('still prefers instance_key over the obs index when the table declares one', async () => { + // Not a nullable-encoding concern, but it shares the code path — the new + // branch must not divert reads that were already resolving correctly. + const ids = await source.loadObsIndex('tables/table'); + expect(ids).toEqual(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']); + }); + + it('reads a nullable integer column, preserving the missing entry as null', async () => { + const [measured] = await source.loadVarColumns(['tables/table/var/measured']); + // Masked entries must stay distinguishable from a real zero. + expect(Array.from(measured ?? [], (v) => (v === null ? null : Number(v)))).toEqual([ + 1, + 2, + null, + 4, + 5, + 6, + ]); + }); +}); From 6480681d63ad507f50b788119756082288f4f816 Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Tue, 28 Jul 2026 15:40:05 +0100 Subject: [PATCH 2/3] Decode zarr v3 categorical columns to labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A categorical column's values were only decoded when its categories array had dtype `v2:object`. A zarr v3 store writes them as `string`, so the check failed and the column resolved to its raw integer codes — plausible looking numbers rather than an error, and wrong wherever a label was expected. v2 fixed-width unicode (`v2:U*`) had the same problem. Test against the type rather than one spelling of it: zarrita's `is()` already knows that `string`, `v2:U*` and `v2:S*` are all text, so the check becomes `is('string') || is('object')`. Also map pandas' -1 "missing" code to null instead of indexing off the end of the categories array and yielding undefined. Co-Authored-By: Claude Opus 5 --- packages/core/src/models/VAnnDataSource.ts | 33 ++++++++++++++----- .../core/tests/nullableStringArray.spec.ts | 8 +++++ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/packages/core/src/models/VAnnDataSource.ts b/packages/core/src/models/VAnnDataSource.ts index e3f17158..5309ae85 100644 --- a/packages/core/src/models/VAnnDataSource.ts +++ b/packages/core/src/models/VAnnDataSource.ts @@ -95,16 +95,14 @@ export default class AnnDataSource extends ZarrDataSource { let categoriesValues: string[] | undefined; let codesPath: string | undefined; if (categories) { - const { dtype } = await zarrOpen(storeRoot.resolve(`${prefix}/${categories}`), { - kind: 'array', - }); - if (dtype === 'v2:object') { - categoriesValues = await this.getFlatArrDecompressed(`${prefix}/${categories}`); + const categoriesPath = `${prefix}/${categories}`; + if (await this.hasTextValues(categoriesPath)) { + categoriesValues = await this.getFlatArrDecompressed(categoriesPath); } } else if (encodingType === 'categorical') { - const { dtype } = await zarrOpen(storeRoot.resolve(`${path}/categories`), { kind: 'array' }); - if (dtype === 'v2:object') { - categoriesValues = await this.getFlatArrDecompressed(`${path}/categories`); + const categoriesPath = `${path}/categories`; + if (await this.hasTextValues(categoriesPath)) { + categoriesValues = await this.getFlatArrDecompressed(categoriesPath); } codesPath = `${path}/codes`; } else if (encodingType === 'string-array') { @@ -123,7 +121,24 @@ export default class AnnDataSource extends ZarrDataSource { if (!categoriesValues) { return data as TableColumnData; } - return Array.from(data, (i) => categoriesValues[i as number]); + // Pandas encodes a missing categorical as code -1, which indexes nothing. + return Array.from(data, (code) => { + const index = Number(code); + return index < 0 ? null : categoriesValues[index]; + }); + } + + /** + * Whether the array at *path* holds text, and so needs decoding to strings. + * + * Covers zarr v3 `string` arrays as well as v2's `object` and fixed-width + * unicode. Testing for `v2:object` alone silently leaves categorical columns + * resolving to their raw integer codes, which look like plausible data rather + * than an error. + */ + private async hasTextValues(path: string): Promise { + const arr = await zarrOpen(this.storeRoot.resolve(path), { kind: 'array' }); + return arr.is('string') || arr.is('object'); } /** diff --git a/packages/core/tests/nullableStringArray.spec.ts b/packages/core/tests/nullableStringArray.spec.ts index 5e0eb59c..97443095 100644 --- a/packages/core/tests/nullableStringArray.spec.ts +++ b/packages/core/tests/nullableStringArray.spec.ts @@ -44,6 +44,9 @@ adata.obs["instance_id"] = np.arange(10) # A genuine missing value, so the mask is exercised rather than only the # all-present case. adata.var["measured"] = pd.array([1, 2, None, 4, 5, 6], dtype="Int64") +# A categorical with a missing entry. Written as a zarr v3 \`string\` categories +# array, which is the case a v2-only dtype check silently mis-reads as codes. +adata.var["family"] = pd.Categorical(["kinase", "kinase", None, "gpcr", "gpcr", "kinase"]) table = TableModel.parse( adata, region="img", region_key="region", instance_key="instance_id" @@ -148,6 +151,11 @@ describe('nullable-encoded AnnData columns', () => { expect(ids).toEqual(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']); }); + it('decodes a zarr v3 categorical to labels rather than raw codes', async () => { + const [family] = await source.loadVarColumns(['tables/table/var/family']); + expect(Array.from(family ?? [])).toEqual(['kinase', 'kinase', null, 'gpcr', 'gpcr', 'kinase']); + }); + it('reads a nullable integer column, preserving the missing entry as null', async () => { const [measured] = await source.loadVarColumns(['tables/table/var/measured']); // Masked entries must stay distinguishable from a real zero. From 22285df1a5d61751be6c378228e7cbacd692d052 Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Fri, 31 Jul 2026 14:51:53 +0100 Subject: [PATCH 3/3] Report the declared kind of nullable obs columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `classifyObsColumnNode` settles a column's kind from its `encoding-type` where one is decisive, and otherwise falls back to the dtype in the node's array metadata. A nullable column is a group of `values` + `mask`, so it has no array metadata of its own, and its encoding was not in the lookup — every one of them came back `undefined`, which sends `'auto'` mode back to sniffing decoded values. That is not a rare shape. AnnData 0.13 defaults to zarr v3 and writes string columns as `nullable-string-array` there, so on a freshly written store the columns arriving without a declared kind are most of the text ones — the case the lookup exists to avoid. The three encoding names now live in `nullableArrays` alongside the reader, mapped to the kind of their `values`, because reading them and classifying them have to stay in step and are in different modules. Also switches `getObsColumnKinds` to the `getObsGroup()` helper. It was still reaching for `this.parsed.obs` behind a `typeof === 'object'` test, which admits a `LazyZarrArray` — the narrowing a08fd37 introduced for the two accessors either side of it. Co-Authored-By: Claude Opus 5 --- .changeset/nullable-anndata-columns.md | 30 ++++++++ packages/core/src/models/index.ts | 18 ++++- packages/core/src/models/nullableArrays.ts | 20 +++-- .../core/tests/nullableStringArray.spec.ts | 73 +++++++++++++++++-- packages/core/tests/tableElement.spec.ts | 24 ++++++ 5 files changed, 150 insertions(+), 15 deletions(-) create mode 100644 .changeset/nullable-anndata-columns.md diff --git a/.changeset/nullable-anndata-columns.md b/.changeset/nullable-anndata-columns.md new file mode 100644 index 00000000..af5505c6 --- /dev/null +++ b/.changeset/nullable-anndata-columns.md @@ -0,0 +1,30 @@ +--- +'@spatialdata/core': minor +--- + +Read AnnData's nullable-encoded columns, and report their kind. + +A nullable column is a **group** of `values` + `mask`, not an array, so opening its +path as an array fails outright. The visible symptom is usually a missing *index* +rather than a missing value, because `obs/_index` and `var/_index` are ordinary +columns — a table would load with `varN` in place of gene names, or with no row ids. +This is not a legacy shape to tolerate: AnnData 0.13 defaults to zarr v3 and writes +string columns this way by default, `_index` included, so it is what a freshly +written `spatialdata` store looks like. + +All three nullable encodings are read (`nullable-string-array`, `nullable-integer`, +`nullable-boolean`), with the mask honoured — a masked entry decodes as `null`, which +the missing-value handling already treats as absent, so it stays distinguishable from +a real `0` or `''` rather than being silently rendered as one. + +`getObsColumnKinds` recognises the same three. Without this the kind lookup fell +through to array metadata that a group does not have and returned `undefined`, +sending `'auto'` mode back to sniffing decoded values for exactly the columns AnnData +now writes by default — the case that lookup exists to avoid. + +Also fixes zarr v3 categoricals decoding to their raw integer codes. The categories +array is written as `string` on v3, and the text check tested for one v2 spelling of +that dtype (`v2:object`), so the column resolved to codes — plausible-looking numbers +rather than an error. The check now asks zarrita whether the dtype is text, which +covers v3 `string` and v2 `U`/`S` alike, and pandas' `-1` missing code maps to `null` +instead of indexing off the end of the categories. diff --git a/packages/core/src/models/index.ts b/packages/core/src/models/index.ts index bb4a12f4..76d099f3 100644 --- a/packages/core/src/models/index.ts +++ b/packages/core/src/models/index.ts @@ -29,6 +29,7 @@ import type { ZarrTree, } from '../types'; import { ATTRS_KEY, Err, Ok, ZARRAY_KEY } from '../types'; +import { NULLABLE_ENCODING_KINDS } from './nullableArrays'; import SpatialDataPointsSource from './VPointsSource'; import SpatialDataShapesSource from './VShapesSource'; import SpatialDataTableSource from './VTableSource'; @@ -211,10 +212,20 @@ export type TableKeys = { instanceKey: string; }; -/** AnnData `encoding-type` values that settle the kind on their own. */ +/** + * AnnData `encoding-type` values that settle the kind on their own. + * + * The nullable encodings have to be listed here rather than left to the dtype + * branch below: they are groups of `values` + `mask`, so there is no array + * metadata on the node itself to fall back to. Leaving them out is not a + * missing edge case — AnnData 0.13 writes string columns this way by default on + * zarr v3, `obs/_index` included, so the columns that most need a declared kind + * are exactly the ones that would arrive without one. + */ const OBS_KIND_BY_ENCODING: Record = { categorical: 'categorical', 'string-array': 'string', + ...NULLABLE_ENCODING_KINDS, }; /** @@ -415,9 +426,8 @@ export class TableElement extends AbstractElement<'tables'> { * `undefined` for a column that is absent or whose node we do not recognise. */ getObsColumnKinds(columnNames: string[]): Array { - const node = this.parsed as ZarrTree; - const obsNode = node.obs as ZarrTree | undefined; - if (!obsNode || typeof obsNode !== 'object') { + const obsNode = this.getObsGroup(); + if (!obsNode) { return columnNames.map(() => undefined); } return columnNames.map((columnName) => classifyObsColumnNode(obsNode[columnName])); diff --git a/packages/core/src/models/nullableArrays.ts b/packages/core/src/models/nullableArrays.ts index 77a21201..711f14aa 100644 --- a/packages/core/src/models/nullableArrays.ts +++ b/packages/core/src/models/nullableArrays.ts @@ -1,5 +1,5 @@ import { type Location, type Readable, get as zarrGet, open as zarrOpen } from 'zarrita'; -import type { TableValue } from '../types'; +import type { TableColumnKind, TableValue } from '../types'; /** * AnnData's nullable encodings, which store a column as a *group* rather than an @@ -14,12 +14,20 @@ import type { TableValue } from '../types'; * * See the AnnData on-disk specification, "Nullable integers, booleans and * strings" — all three share this group layout at encoding-version 0.1.0. + * + * The mapped value is the kind of the column's `values`, which is what the + * encoding name already tells us — the mask changes which entries are present, + * never what type they are. Declared here rather than at the classifier so the + * three encoding names are listed once: reading them and classifying them have + * to stay in step, and they live in different modules. */ -export const NULLABLE_ENCODING_TYPES = new Set([ - 'nullable-string-array', - 'nullable-integer', - 'nullable-boolean', -]); +export const NULLABLE_ENCODING_KINDS: Record = { + 'nullable-string-array': 'string', + 'nullable-integer': 'numeric', + 'nullable-boolean': 'boolean', +}; + +export const NULLABLE_ENCODING_TYPES = new Set(Object.keys(NULLABLE_ENCODING_KINDS)); export function isNullableEncoding(encodingType: unknown): boolean { return typeof encodingType === 'string' && NULLABLE_ENCODING_TYPES.has(encodingType); diff --git a/packages/core/tests/nullableStringArray.spec.ts b/packages/core/tests/nullableStringArray.spec.ts index 97443095..e6507ae7 100644 --- a/packages/core/tests/nullableStringArray.spec.ts +++ b/packages/core/tests/nullableStringArray.spec.ts @@ -3,8 +3,10 @@ import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { FileSystemStore } from '@zarrita/storage'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import SpatialDataTableSource from '../src/models/VTableSource.js'; +import { readZarr } from '../src/store/index.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); const writerRoot = join(__dirname, '../../../python/spatialdata-js-util'); @@ -41,6 +43,12 @@ adata.var.index = pd.array([f"GENE{i}" for i in range(6)], dtype="string") adata.obs.index = pd.array([f"cell{i}" for i in range(10)], dtype="string") adata.obs["region"] = pd.Categorical(["img"] * 10) adata.obs["instance_id"] = np.arange(10) +# Nullable obs columns, so the kind lookup is exercised on columns and not only +# on the index (which \`getObsColumnNames\` filters out). +adata.obs["qc_count"] = pd.array([1, None, 3, 4, 5, 6, 7, 8, 9, 10], dtype="Int64") +adata.obs["passes_qc"] = pd.array( + [True, None, False, True, True, False, True, True, False, True], dtype="boolean" +) # A genuine missing value, so the mask is exercised rather than only the # all-present case. adata.var["measured"] = pd.array([1, 2, None, 4, 5, 6], dtype="Int64") @@ -58,10 +66,21 @@ ad.settings.allow_write_nullable_strings = True sd.SpatialData(tables={"table": table}).write(root, overwrite=True) # Fail loudly here rather than letting the test assert against the wrong layout. -encoding = __import__("json").loads( - (root / "tables" / "table" / "var" / "_index" / "zarr.json").read_text() -)["attributes"]["encoding-type"] -assert encoding == "nullable-string-array", f"fixture wrote {encoding!r}" +json = __import__("json") + + +def encoding_of(*parts): + path = root.joinpath("tables", "table", *parts, "zarr.json") + return json.loads(path.read_text())["attributes"]["encoding-type"] + + +for parts, expected in [ + (("var", "_index"), "nullable-string-array"), + (("obs", "qc_count"), "nullable-integer"), + (("obs", "passes_qc"), "nullable-boolean"), +]: + actual = encoding_of(*parts) + assert actual == expected, f"fixture wrote {actual!r} for {'/'.join(parts)}" PY`, { cwd: writerRoot, stdio: 'pipe' } ); @@ -107,11 +126,12 @@ function createFilesystemStore(root: string) { describe('nullable-encoded AnnData columns', () => { let fixtureRoot: string; + let storeRoot: string; let source: SpatialDataTableSource; beforeAll(async () => { fixtureRoot = await mkdtemp(join(tmpdir(), 'nullable-anndata-')); - const storeRoot = join(fixtureRoot, 'store.zarr'); + storeRoot = join(fixtureRoot, 'store.zarr'); writeNullableTableFixture(storeRoot); source = new SpatialDataTableSource({ store: createFilesystemStore(storeRoot), @@ -168,4 +188,47 @@ describe('nullable-encoded AnnData columns', () => { 6, ]); }); + + describe('declared column kinds', () => { + /** + * The kind lookup runs on the consolidated metadata rather than on decoded + * values, so it is only meaningful against a tree opened from a real store — + * a mock tree would assert the shape we chose to write in the mock. + */ + async function openTable() { + const sdata = await readZarr(new FileSystemStore(storeRoot)); + const table = sdata.tables?.table; + if (!table) { + throw new Error('fixture store has no `table`'); + } + return table; + } + + it('reports the kind of a nullable column instead of leaving it undefined', async () => { + const table = await openTable(); + const kinds = Object.fromEntries( + ['qc_count', 'passes_qc', 'region', 'instance_id'].map((name) => [ + name, + table.getObsColumnKinds([name])[0], + ]) + ); + // The nullable pair is the point: they are groups, so there is no array + // metadata to fall back on and an unlisted encoding yields `undefined`, + // which sends callers back to sniffing decoded values. + expect(kinds).toEqual({ + qc_count: 'numeric', + passes_qc: 'boolean', + region: 'categorical', + instance_id: 'numeric', + }); + }); + + it('still excludes the index from the obs columns when it is nullable-encoded', async () => { + const table = await openTable(); + expect(table.getObsColumnNames()).not.toContain('_index'); + expect(table.getObsColumnNames()).toEqual( + expect.arrayContaining(['region', 'instance_id', 'qc_count', 'passes_qc']) + ); + }); + }); }); diff --git a/packages/core/tests/tableElement.spec.ts b/packages/core/tests/tableElement.spec.ts index 5b337068..a12c665f 100644 --- a/packages/core/tests/tableElement.spec.ts +++ b/packages/core/tests/tableElement.spec.ts @@ -173,6 +173,30 @@ describe('obs column kinds from consolidated metadata', () => { ).toEqual(['numeric', 'numeric', 'boolean', 'string', 'categorical']); }); + it('classifies nullable columns from the encoding alone', () => { + // A nullable column is a GROUP of `values` + `mask`, so the node carries no + // array metadata of its own — the encoding name is the only thing on it that + // says what the column holds. AnnData 0.13 writes string columns this way by + // default on zarr v3, so this is the common shape, not an exotic one. + const nullableNode = (encodingType: string, valuesDataType: string) => ({ + [ATTRS_KEY]: { 'encoding-type': encodingType, 'encoding-version': '0.1.0' }, + values: arrayNode({ data_type: valuesDataType }), + mask: arrayNode({ data_type: 'bool' }), + }); + + const table = tableWithObs({ + barcode: nullableNode('nullable-string-array', 'string'), + qc_count: nullableNode('nullable-integer', 'int64'), + passes_qc: nullableNode('nullable-boolean', 'bool'), + }); + + expect(table.getObsColumnKinds(['barcode', 'qc_count', 'passes_qc'])).toEqual([ + 'string', + 'numeric', + 'boolean', + ]); + }); + it('returns undefined for absent or unrecognised columns', () => { const table = tableWithObs({ mystery: { [ATTRS_KEY]: { 'encoding-type': 'something-new' } },