diff --git a/.changeset/points-feature-scan-and-name-selection.md b/.changeset/points-feature-scan-and-name-selection.md
new file mode 100644
index 00000000..59ece2f0
--- /dev/null
+++ b/.changeset/points-feature-scan-and-name-selection.md
@@ -0,0 +1,39 @@
+---
+"@spatialdata/core": patch
+"@spatialdata/layers": patch
+"@spatialdata/vis": patch
+---
+
+Points: feature selection now works on large elements, and persists by name.
+
+**Selections persist as feature names.** `PointsLayerConfig.featureNames` is the
+durable, serializable form and what the UI writes. Codes are app-assigned for a
+dictionary-only element (a Xenium `transcripts` has `feature_name` and no code
+column), so a stored code could silently come back meaning a different feature.
+`featureCodes` still works and still takes effect at runtime, but names win when
+both are present. `resolveFeatureSelectionCodes` / `featureNamesForCodes` are
+exported from `@spatialdata/core` for converting between the two.
+
+**The feature scan now completes on large elements.** Previously, selecting a
+feature on a multi-million-row element frequently never resolved — the scan
+plateaued part-way through and the layer sat there. It now reads through
+`ParquetFile.stream({ columns, rowGroups })`, which fetches per column chunk, so
+the projection reaches the network instead of pulling whole row groups — all 12
+columns of a Xenium `transcripts` to use three. The scan runs in the points
+worker, keeping the parquet decode off the main thread. Selecting one gene from a 12.1M-row element now
+settles in ~1.0s, with main-thread time roughly a third of what the pre-streaming
+path cost.
+
+Also fixed along the way: a full-dataset catalog scan being silently cancelled by
+the resident preview settling underneath it (leaving counts stuck and colours
+mismatched); row-group chunks handing out the cached footer buffer, which the
+worker transfer detached (`DataCloneError`, dropping the element onto whole-file
+reads); parquet part layout being re-probed on every call; a server that answers
+a directory path with 500 rather than 404 wedging part traversal; and point size
+not accounting for an element's transform scale.
+
+Known limitation: for a dictionary-only element the fallback catalog path cannot
+tally per-feature counts, and it settles *successfully* without them — so the
+retry path does not repair it and counts stay absent for the session. Names and
+selection are unaffected. Treat a missing count as unknown, not zero, and do not
+read the presence of counts as a signal that the scan completed.
diff --git a/.claude/launch.json b/.claude/launch.json
index cf42fafc..409bbc0b 100644
--- a/.claude/launch.json
+++ b/.claude/launch.json
@@ -8,6 +8,12 @@
"autoPort": true,
"port": 5173
},
+ {
+ "name": "vis-demo-alt",
+ "runtimeExecutable": "pnpm",
+ "runtimeArgs": ["--filter", "@spatialdata/vis", "dev:demo", "--port", "5180", "--strictPort"],
+ "port": 5180
+ },
{
"name": "docs",
"runtimeExecutable": "pnpm",
diff --git a/docs/docs/vis/headless-viewer.mdx b/docs/docs/vis/headless-viewer.mdx
index 8cec98da..9028f4b0 100644
--- a/docs/docs/vis/headless-viewer.mdx
+++ b/docs/docs/vis/headless-viewer.mdx
@@ -253,6 +253,104 @@ Row alignment comes from `@spatialdata/core` (`createFeatureTableAlignment`);
colour encoding from `@spatialdata/layers` (`buildShapeFillColorByFeatureId`).
`SpatialCanvasViewer` wires these when `fillColorByColumn` is set.
+## Points: feature selection and colour
+
+Points styling is driven the same way as shapes — by updating the stack entry
+props, not by reaching into vis internals. The serializable fields on a points
+entry:
+
+```ts
+entry.props = {
+ ...entry.props,
+ // Which features are drawn. Omit for "all features".
+ featureNames: ['EPCAM', 'MALL'],
+ // Per-feature colour, keyed by feature name. Absent features keep the default
+ // categorical colour.
+ featureColorOverrides: { EPCAM: [220, 30, 30] },
+ // Categorical colour-by-feature. ON by default; pass false for a flat colour.
+ colorByFeature: true,
+ // Radius in the ELEMENT's own coordinate units (the layer folds in the
+ // element's transform scale, so the same value means the same apparent size
+ // across elements whose transforms differ).
+ pointSize: 0.1,
+ // Max rows retained in memory for the resident window. Raising it draws more
+ // points at the cost of memory and decode time.
+ pointsMemoryCap: 4_000_000,
+};
+```
+
+### Selections persist as names, not codes
+
+`featureNames` is the durable form and the one to serialize. There is also a
+`featureCodes: number[]`, which is retained for runtime use and for configs
+written before names existed — **do not persist it**.
+
+The reason is that a points element often has no feature-code column in the file.
+A Xenium `transcripts` carries `feature_name` and no codes; the same is true of a
+merfish `cell_type`. For those, codes are *assigned by the application* as a
+first-seen index while building the feature catalog, so the same gene is not
+guaranteed the same number between the instant resident-subset catalog and the
+full one, between the two catalog-building paths (which are chosen by row count),
+or between servers that differ in HTTP range support. A persisted code can
+therefore come back meaning a different feature, with nothing to signal it.
+
+Names are resolved to whatever codes the current catalog uses, at render time.
+Names the element does not have are dropped rather than coerced, so a config
+saved against one dataset can be applied to another without inventing features.
+`featureNames` takes precedence when both fields are present.
+
+If you need to convert in either direction yourself, `@spatialdata/core` exports
+`resolveFeatureSelectionCodes(selection, catalog)` and `featureNamesForCodes`.
+
+### Reading feature state headlessly
+
+To build your own feature UI, read the engine directly. `pointsEngine` and
+`resolvePointsTarget` come off the renderer-hook result; wrap a subtree in the
+provider and consume the hook:
+
+```tsx
+import { PointsFeatureStateProvider, usePointsFeatureState } from '@spatialdata/vis';
+
+const { pointsEngine, resolvePointsTarget } = useSpatialCanvasRenderer(/* … */);
+
+
+
+;
+
+function MyFeatureList({ config }) {
+ const {
+ catalog, // { featureKey, entries: [{ code, name, count? }] } | null
+ catalogLoading, // no catalog yet, one is on its way
+ catalogRefining, // full scan running behind an instant preview
+ residentCodes, // features present in the resident window
+ loadedMatchingCodes, // features currently on screen via the last scan
+ supportsOnDemandLoad,// a whole-dataset scan can reach beyond the window
+ matchingLoadState, // progress of the scan for this selection
+ residentFeatureCounts,
+ requestCatalog, // idempotent; upgrades the preview to the full list
+ setHighlightedFeature,
+ } = usePointsFeatureState(config);
+}
+```
+
+Pass the layer config (the hook resolves `featureNames` internally against the
+catalog it is already reading). An array of already-resolved codes is also
+accepted.
+
+Two things worth designing around:
+
+- The catalog arrives in two stages. An instant preview covering just the
+ resident window is published first, then the authoritative full-dataset list
+ supersedes it; `catalogRefining` distinguishes them. Counts may lag the names —
+ and for a dictionary-only element they may never arrive at all, because the
+ fallback that builds the catalog cannot tally them. That is a *successful*
+ settle, so nothing retries it. Treat counts as optional: sort and label from the
+ names, and do not read a missing count as zero or as "still loading".
+- Selecting a feature whose points are outside the resident window triggers a
+ whole-dataset scan when `supportsOnDemandLoad` is true. `matchingLoadState`
+ reports its progress so the UI can show that points are still arriving rather
+ than appearing to be complete.
+
## Auto-fit and view state
- Pass `viewState={null}` on first render to let the viewer compute an initial
@@ -314,6 +412,7 @@ Before treating the API as stable for MDV:
- [x] Controlled `coordinateSystem`, `renderStack`, `viewState`
- [x] Host overlay descriptors with `hostLayerResolver`; `deckLayers` / `deckProps` passthrough remains for compatibility
- [x] `renderTooltip={false}` for external tooltip ownership
+- [x] Points feature selection persisted by NAME (`featureNames`), with `featureColorOverrides` keyed by name
- [x] `demo/headless` route with local `blobs.zarr` fixture
- [ ] Additional `demo/headless-*` variants (Leva controls, custom deck layers)
- [ ] Tooltip/pick row resolution fully on shared `FeatureTableAlignment` (in progress)
diff --git a/docs/docs/vis/mdv-release-checklist.mdx b/docs/docs/vis/mdv-release-checklist.mdx
index 7f4c8b98..4599ad5c 100644
--- a/docs/docs/vis/mdv-release-checklist.mdx
+++ b/docs/docs/vis/mdv-release-checklist.mdx
@@ -63,7 +63,7 @@ geometry/images and maps **MDV’s render stack** to Viv/deck output.
| Images | Yes (Viv) |
| Shapes | Yes — MDV drives style/filter via `layers` state |
| Labels | Render when configured; segmentation vs shapes experiment is not a release gate |
-| Points | v1.1 |
+| Points | v1.1 — feature selection and colour are config-driven and serializable; tiling/index strategy still in progress |
### Custom deck layers (additive, not a replacement for state)
@@ -95,6 +95,42 @@ Current local state:
Optional later: MDV-only `PolygonLayer` built from exported geometry helpers; not required if config-driven styling is sufficient.
+### Points v1.1: feature selection and colour via `layers` state
+
+Points support the same config-driven model as shapes — MDV updates the layer
+config, vis owns loading and rendering:
+
+- [x] `PointsLayerConfig` accepts a serializable selection (`featureNames`), per-feature colour (`featureColorOverrides`), `colorByFeature`, `pointSize`, `pointsMemoryCap`
+- [x] Selections persist by feature NAME, not code, so a saved config survives catalog renumbering ([Headless viewer guide](./headless-viewer))
+- [x] Colour overrides keyed by feature name for the same reason
+- [x] Headless feature UI supported without vis internals: `PointsFeatureStateProvider` + `usePointsFeatureState`
+- [ ] Tiling / index strategy for very large elements (follow-up: benchmarking index strategies, Python writer utilities)
+
+The serialization detail that matters for saved MDV charts: a points element
+frequently has **no feature-code column in the file** — a Xenium `transcripts`
+carries `feature_name` only — so codes are assigned by the application while
+building the feature catalog and are not stable across catalog upgrades, catalog
+paths, or servers. `featureNames` is therefore the durable field; `featureCodes`
+exists for runtime and backwards compatibility and should not be persisted.
+
+Current local state:
+
+- `@spatialdata/core` exposes `resolveFeatureSelectionCodes()` and
+ `featureNamesForCodes()` for converting between the durable and runtime forms
+ against a given catalog.
+- The feature catalog is two-stage: an instant resident-subset preview, then the
+ authoritative full-dataset list. A selection made against either resolves to
+ the same features, which is what the name form buys.
+- Remaining release risk: a selection whose points fall outside the resident
+ window needs a whole-dataset scan, whose cost scales with element size — this
+ is what the tiling/index follow-up addresses. `matchingLoadState` exposes the
+ progress meanwhile.
+- Remaining release risk: for a **dictionary-only** element, the fallback catalog
+ path cannot tally counts, and it settles *successfully* without them — so the
+ retry path does not repair it and the counts stay absent for the session. Do not
+ treat count ordering or the presence of counts as a signal that the scan
+ finished; use the names, and render a missing count as unknown rather than zero.
+
### Bespoke layer extension path
- Primary extension for **state-owned** SpatialData layers: MDV-controlled `renderStack.entries`
diff --git a/docs/plans/points-redesign-punchlist.md b/docs/plans/points-redesign-punchlist.md
index 1450a5d5..2673012b 100644
--- a/docs/plans/points-redesign-punchlist.md
+++ b/docs/plans/points-redesign-punchlist.md
@@ -42,6 +42,51 @@ state model, not by patching mutations here.
---
+## Known open — feature counts can settle permanently absent
+
+Observed intermittently on a 12.1M-row Xenium `transcripts`: the feature panel
+sticks on "sorted by count so far" and never reaches authoritative counts.
+**Remounting the panel does not clear it**, which distinguishes it from the
+preview-vs-full supersession bug fixed in `46d5b89`.
+
+Mechanism (read from code; not yet reproduced deterministically):
+
+1. `listPointsFeatures` tries `listPointsFeaturesByStreamingScan` first, which
+ tallies counts as it scans, and falls back on any failure.
+2. The fallback does not tally. `listPointsFeaturesWithCounts` then calls
+ `loadFeatureCounts`, which needs an integer code column — so for a
+ **dictionary-only** element (Xenium `transcripts`, merfish `cell_type`) it
+ returns an empty map and the catalog settles with no counts.
+3. `PointsResolver.ensureFeatureCatalog` short-circuits on
+ `slot.settledKey === 'full'`, so that countless catalog is never re-requested.
+ The failure is therefore permanent for the session, and remount-immune.
+
+The non-determinism plausibly comes from `serverSupportsStreamingRanges`, a live
+two-request probe memoised per ORIGIN in a static map: if it loses a race or is
+throttled once, the whole origin is marked unservable for the session and the
+countless path is taken. Suspected to have become more likely when `be64b65` put
+the feature scan on that same probe, so it now runs earlier and more often —
+**unverified**.
+
+**Update (`8d1a875`).** Review on #89 identified the concrete mechanism, and it
+is the one suspected above: the probe cached a *thrown* fetch as if it were the
+server's answer, so a single failed request demoted the origin for the life of
+the page. The probe now caches only definitive answers (a 416, or a 200 that
+ignored `Range`); a thrown fetch evicts. That removes the most likely trigger,
+but it does **not** close this item — it makes the countless path rarer without
+making it recoverable. The permanence below is untouched, and any other route to
+the fallback still produces the same stuck panel.
+
+Two independent fixes, either of which removes the permanence:
+
+- Do not settle `'full'` for a catalog missing counts it should have — settle it
+ under an upgradable phase so a retry is possible. Makes it self-healing.
+- Make the fallback path tally counts for dict-only elements, so falling back is
+ a performance difference rather than a correctness one.
+
+Deliberately not fixed blind: forcing `serverSupportsStreamingRanges` to false
+should give a deterministic reproduction to fix against first.
+
## Defer-to-redesign
Each notes *why* it's coupled to the state-model / decode rework.
@@ -51,6 +96,30 @@ Each notes *why* it's coupled to the state-model / decode rework.
about this ambient stateful thing` (onProgress), `// given ongoing problems with
agent debugging, inclined to more purity. Might consider using
Effect?`, `// there will be various mutating side-effects on entry…`.
+
+ **Effect remains an open question, not a closed one.** Where an ADR or plan
+ reads as having ruled it out, that is "not now", not "decided against" —
+ revisit it on its merits when this item is picked up.
+
+ Evidence from the #89 review, which is the argument for this item stated in
+ defects rather than in taste. Four independent findings, one shape: *state
+ arriving out of step with the thing it describes.*
+
+ | Finding | The step it fell out of |
+ | --- | --- |
+ | Range probe cached a thrown fetch (`8d1a875`) | a failure outliving the request that caused it |
+ | Worker `fromUrl` cache kept a rejection (`8d1a875`) | same, one layer down |
+ | Matched batch drawn unfiltered without row codes (`615c926`) | codes vs the batch they align to |
+ | `loadAll()` landing after its loader was replaced (`57f77fd`) | a read vs the resource it read from |
+ | `rowCodes` readiness gate ignored the cap (this entry's sibling) | codes vs the window they mask |
+
+ Each was individually cheap to fix and none were found by the type system,
+ because in every case the stale value is the *correct type* — the cache, slot
+ or state field simply has no way to say "this is no longer about the thing you
+ are asking about". That is the property a principled effect/resource model
+ makes structural instead of a per-site discipline, and it is why the fixes
+ above are guards rather than a design change: five guards is evidence for D1,
+ not a substitute for it.
- **D2 — Break up `useLayerData`.** The monolith the engine threads through; also
the reason for the `'use no memo'` hatches (`PointsFeatureFilterPanel`,
`ShowMatchingPoints`). A properly reactive state layer retires the hatches.
@@ -70,15 +139,22 @@ Each notes *why* it's coupled to the state-model / decode rework.
one worker; the engine keys by element and assumes single-demand-per-element.
Multi-layer sharing / a work queue belongs with the engine redesign.
- **D7 — GeoArrow encoding.** Unexplored; a decode-path spike, not this PR.
-- **D8 — Streaming cancellation semantics.** The generators have no `AbortSignal`
- threaded to the worker, and an abandoned manual `.next()` loop won't clean up.
- Fine while consumers drain; design it with the new state layer.
+- **D8 — Streaming cancellation semantics.** *Partly addressed on the Track A
+ branch:* an `AbortSignal` is threaded to the scan generator, so supersede/evict
+ abort it between chunks. What remains is the general case this entry was written
+ about — the signal does not reach the WORKER, and an abandoned manual `.next()`
+ loop still won't clean up. Design the rest with the new state layer.
- **D9 — Remove `'use no memo'` hatches (stable-snapshot option).** Give the
engine stable-identity snapshot accessors so `useSyncExternalStore` tracks the
value directly and the compiler stops needing an opt-out. Part of D1/D2.
-- **D10 — Progressive-overlay visibility logic + flashing.** F1 fixed the
+- **D10 — Progressive-overlay visibility logic + flashing.** *The flashing is
+ fixed on the Track A branch* — a scan-stable partial resource plus
+ `resourceRevision` means the overlay updates in place instead of being torn down
+ per chunk. The rest of this entry stands: the stable-growing-GPU-buffer work
+ below is still the destination, and it is shared with D3. Historical description
+ of the flash follows. F1 fixed the
deselected-feature-lingering slice, but *which* points show during a partial
- load still has logic problems, and it **flashes badly**: every notify rebuilds
+ load still has logic problems, and it **flashed badly**: every notify rebuilt
the partial buffer into a fresh `PointsRenderResource` (new identity each
chunk), so deck tears down and recreates the `__partial` layer per step instead
of updating it in place. The real fix is a stable growing GPU buffer (preallocate
diff --git a/docs/plans/resource-resolver-handoff.md b/docs/plans/resource-resolver-handoff.md
index cce3d5ce..b6b7707f 100644
--- a/docs/plans/resource-resolver-handoff.md
+++ b/docs/plans/resource-resolver-handoff.md
@@ -1,7 +1,8 @@
# Resource Resolver — implementation handoff
-**Status:** Step 0 + Step 1 **landed** — see [Progress](#progress-2026-07-15). Step 2
-(Tracks A / B / C) and Step 3 (Renderer Adapter cleanup) remain.
+**Status:** Step 0 + Step 1 **landed**; **Step 2 Track A (points state model) landed**
+— see [Progress](#progress-2026-07-15). Step 2 Tracks B / C and Step 3 (Renderer
+Adapter cleanup) remain.
**Decisions:** [ADR 0004 — Resource Resolver Owned By Core](../adr/0004-resource-resolver-owned-by-core.md), [ADR 0005 — Memory Accounting Before Management](../adr/0005-memory-accounting-before-management.md)
**Supersedes:** [layer-data-engine-decomposition.md](layer-data-engine-decomposition.md)
**Vocabulary:** [CONTEXT.md](../../CONTEXT.md) — *Resource Resolver, Renderer Adapter, Spatial Entry, Resolution, Spatial Entry Error, Entry Notice, Encoded/Decoded Tier, Resource Ceiling*
@@ -29,10 +30,29 @@ Read the two ADRs first. This document is sequencing, not rationale.
`pointsEngine.ensureMatchingFeaturesLoaded` / `ensureRowFeatureCodes` calls in
`getLayers` stay put (they migrate into `plan()` under Track A), so the points
reconcile context carries only the memory cap.
-- **Remaining:** Step 2 Tracks A (points state model / `RequestSlot` / races R1–R5),
- B (shapes loader seam + tooltip ping-pong), C (memory, ADR 0005 rungs 1–3); Step 3
- (Renderer Adapter `project()`/`render()`, `'use no memo'` removal, dead-surface
- cleanup). None started.
+- **Step 2 Track A — points state model: landed** (branch
+ `claude/points-implementation-stages`, commits A1–A8). All four points resources —
+ `preload`, `rowCodes`, `catalog`, `matching` — are now `RequestSlot`s (one tested
+ dedup/supersede/settle primitive, keyed so everything a request depends on is in the
+ key). **Races R1/R2/R3/R5 closed** with fail-before/pass-after tests. Failures are
+ structured, **retryable** `SpatialEntryError`s with a `retry()` API (the stuck
+ full-catalog-scan fix) — retryable meaning a request that FAILED. A catalog that
+ settles successfully but without counts (the dict-only fallback) is not a failure
+ and `retry()` does not repair it; see the "feature counts can settle permanently
+ absent" entry in the punchlist. Cancellation is threaded to the scan generator (D8;
+ supersede/evict abort it between chunks). The render-phase engine kicks are
+ **migrated into `plan()`** — `getLayers` is now pure reads, driven by the reconcile
+ effect. The streaming-overlay **flash is fixed** (D10) via a scan-stable partial
+ resource + `resourceRevision`. The Effect-vs-plain spike ran and **plain won**
+ (recorded above). `PointsResolver`'s public surface is unchanged; the 855-line
+ `pointsDataEngine` regression net stays green (a few failure/cap-alignment cases
+ flipped to the new behaviour). 573 tests green repo-wide.
+ - **Pending browser verification:** the D10 no-flash behaviour is proven headlessly
+ (adapter resource-identity + revision) but its *visual* confirmation needs a
+ running app with a large streaming Xenium `transcripts` scan.
+- **Remaining:** Step 2 Tracks B (shapes loader seam + tooltip ping-pong), C (memory,
+ ADR 0005 rungs 1–3); Step 3 (Renderer Adapter `project()`/`render()`, `'use no memo'`
+ removal, dead-surface cleanup; retire the `PointsDataEngine` facade). Not started.
---
@@ -176,6 +196,27 @@ unless it wins on *all three* of — supersession correctness under two concurre
scans; interruption that actually reaches the worker; fewer lines to set up a race in
a test. A tie means the plain slot wins.
+> **Spike outcome (Track A step A8): plain wins. Effect not adopted.** Run
+> empirically — `effect@3.22` added as a `core` devDependency, both slots implemented
+> and driven through the same two-concurrent-scan supersession race
+> (`packages/core/tests/matchingSlotEffectSpike.spec.ts`), then both removed per
+> "delete the loser". Against the three criteria:
+> 1. **Supersession correctness — tie.** Both drop the superseded scan's result; the
+> plain slot by record identity (`this.current !== record`), Effect by
+> `Fiber.interrupt`.
+> 2. **Interruption reaches the worker — tie.** Both abort the scan's `AbortSignal`
+> (the seam A5 threads to the generator). The plain slot aborts its own
+> `AbortController` *synchronously*; Effect runs the `Effect.async` canceler via
+> `runFork(Fiber.interrupt(...))`, needing a runtime tick to propagate.
+> 3. **Lines to set up a race — plain wins.** Plain: two synchronous `request()`
+> calls, assert immediately. Effect: an `Effect.async` + canceler + `runFork` slot
+> (~15 extra lines) *and* an awaited runtime tick before every assertion; plus a
+> multi-second import cost on a package whose ethos is zero runtime deps.
+>
+> Effect ties two and loses one, so by the agreed rule the plain `RequestSlot` wins.
+> The `RequestSlot` seam is deliberately shaped so a future reconsideration (e.g. if
+> `tgpu-htj2k`'s dependency-free stance is renegotiated) is a swap, not a rewrite.
+
---
#### Track B — Shapes
diff --git a/packages/core/src/engine/PointsResolver.ts b/packages/core/src/engine/PointsResolver.ts
index b86f84a2..1c26925b 100644
--- a/packages/core/src/engine/PointsResolver.ts
+++ b/packages/core/src/engine/PointsResolver.ts
@@ -4,6 +4,7 @@ import { DEFAULT_POINTS_MEMORY_CAP } from '../pointsLimits.js';
import type { PointsLoadProgress, PointsLoadResult } from '../pointsLoadOptions.js';
import type { PointsFeatureCatalog } from '../pointsTiling.js';
import type { EntryNotice } from './errors.js';
+import { RequestSlot } from './RequestSlot.js';
import { Resolution } from './resolution.js';
import type { EntryResources, ResolveContext, ResolveTask, ResourceResolver } from './resolver.js';
import { SnapshotCache } from './snapshotCache.js';
@@ -42,11 +43,30 @@ import { SnapshotCache } from './snapshotCache.js';
* rows in *file order*, so index i in the codes array names the feature of point i
* in the batch.
*
- * **The memory cap must reach both calls identically or the filter mask is
- * misaligned.** It currently does not: `ensureRowFeatureCodes` takes no cap and so
- * falls back to the 4M default while `ensureLoaded` honours the user's. That is
- * race R5, and it is Track A's to fix — this commit is a re-housing, and
- * deliberately preserves the behaviour, bug and all.
+ * **The memory cap reaches both calls identically** — that is what keeps the mask
+ * aligned. The `preload` and `rowCodes` slots are both keyed on the memory cap, and
+ * `ensureRowFeatureCodes` reads the codes at the preload's cap (its slot key). This
+ * closes race R5 (Track A): the old `ensureRowFeatureCodes` took no cap and fell back
+ * to the 4M default while `ensureLoaded` honoured the user's, misaligning the mask
+ * against an 8M resident batch.
+ *
+ * Keying the slot only makes the misalignment *representable*; {@link plan} is what
+ * acts on it. It gates on {@link hasRowFeatureCodesAtCap}, not on readiness — codes
+ * settled at 4M stay "ready" through a raise to 8M, so a readiness gate leaves a
+ * stale mask in place over the bigger batch and R5 survives in the one place that
+ * decides whether to fix it.
+ *
+ * ## State model (Track A)
+ *
+ * All four resources — `preload`, `rowCodes`, `catalog`, `matching` — are
+ * {@link RequestSlot}s: one tested dedup/supersede/settle primitive, keyed so that
+ * everything a request depends on is in the key. Supersession is by record identity,
+ * never value — a superseded load cannot write anything. The keys ARE the race fixes:
+ * `preload`/`rowCodes` on the memory cap (R1, R5); `matching` on
+ * `` `${signature}#${cap}` `` (R2 dedups a re-selected covered scan, R3 supersedes on
+ * a cap raise). A failed slot holds a structured, **retryable** `SpatialEntryError`
+ * that {@link retry} re-runs — which is what unsticks the previously-permanent
+ * full-catalog-scan failure.
*/
export type PointsLoadStatus = 'idle' | 'loading' | 'ready' | 'error';
@@ -72,50 +92,75 @@ export interface PointsResolveConfig {
}
interface PointsEntry {
- data?: PointsLoadResult;
- /** Memory cap (max resident rows) the current `data`/`loading` was requested
- * with. A change means the resident window must reload — see `ensureLoaded`. */
- memoryCap?: number;
- /** Aborts the in-flight preload when it is superseded (a cap change), so a
- * stale load doesn't run its expensive main-thread fallback to completion. */
- loadAbort?: AbortController;
- status: PointsLoadStatus;
- loading?: Promise;
- /** Feature catalog: `undefined` while unloaded, `null` once settled for an
- * element with no `feature_key`, else the catalog. `catalogLoaded` disambiguates
- * "not yet requested" from "settled as null". */
- catalog?: PointsFeatureCatalog | null;
- catalogLoaded?: boolean;
- catalogLoading?: Promise;
- /** True once the full-dataset catalog scan (`listFeaturesWithCounts`) has
- * replaced any resident-subset preview. */
- catalogComplete?: boolean;
- /** Per-row feature codes aligned to the resident batch (see class doc). */
- rowCodes?: ArrayLike;
- rowCodesLoaded?: boolean;
- rowCodesLoading?: Promise;
- /** The catalog whose code space {@link rowCodes} are expressed in. */
+ /**
+ * Resident geometry preload, keyed by memory cap. The key IS the cap: a cap
+ * change supersedes (reload), an identical cap dedups, and a lowered cap is served
+ * by an in-memory shed (`settle`) rather than a fetch. Record-identity
+ * supersession is what closes R1 (a superseded reload can no longer wipe the live
+ * one's markers). Its `stale` retention is the atomic swap — the previous batch
+ * stays on screen until the larger one settles.
+ */
+ preload: RequestSlot;
+ /**
+ * Per-row feature codes aligned to the resident batch (see class doc), **keyed by
+ * memory cap**. Keying on the cap is the R5 fix: the codes are read at the same
+ * window as the geometry, so index i in the codes names the feature of point i in
+ * the batch. `V` is `ArrayLike | undefined` because an element with no
+ * codes settles `ready(undefined)` — a settled fact, not an absence.
+ */
+ rowCodes: RequestSlot | undefined>;
+ /**
+ * Feature catalog, two-phase, as a {@link RequestSlot} keyed `'preview' | 'full'`.
+ * The resident-subset **preview** falls out of the geometry preload's decode
+ * (`settle('preview', …)`); the authoritative **full** scan
+ * (`listFeaturesWithCounts`) supersedes it (`request('full', …)`), retaining the
+ * preview as `stale` so it keeps showing while the full list loads. A settled value
+ * is the catalog, or `null` for an element with no `feature_key` — a fact, not an
+ * absence. A failed full scan is `failed` + **retryable** (Track A step A4): it no
+ * longer settles permanently, so {@link retry} can re-run it.
+ */
+ catalog: RequestSlot;
+ /** The catalog whose code space the {@link rowCodes} value is expressed in. */
rowCodesCatalog?: PointsFeatureCatalog;
+ /**
+ * The resident-subset preview from the last geometry decode, held OUTSIDE the
+ * slot so it can be offered as a fallback without cancelling anything.
+ *
+ * `settle` aborts the in-flight request, so settling a preview on top of a
+ * running full scan destroys it. Keeping the preview here lets
+ * {@link PointsResolver.getFeatureCatalog} still show it instantly while the scan
+ * runs (or after one fails), which is the whole point of the preview, without the
+ * write that killed the scan.
+ */
+ previewCatalog?: PointsFeatureCatalog;
/** True when the element has a file-backed feature code column (authoritative
* codes; a real feature index). False for dictionary-only feature columns. */
featureCodeColumn?: boolean;
- /** Memoized distinct codes in {@link rowCodes}, invalidated by identity. Note
- * this is a DATA memo (a Set), not a render resource — it stays in core. */
+ /** Memoized distinct codes in the resident {@link rowCodes}, invalidated by
+ * identity. A DATA memo (a Set), not a render resource — it stays in core. */
residentCodes?: ReadonlySet;
residentCodesSource?: ArrayLike;
- /** Whole-dataset points for the active selection, keyed by the selected-codes
- * `signature` so a selection change rebuilds it. */
- matching?: { signature: string; result: PointsLoadResult };
- /** In-flight feature-index scan, with progressive counts from `onProgress`. */
- matchingLoading?: {
- signature: string;
- promise: Promise;
- matchedRows: number;
- scannedRows: number;
- partialResult?: PointsLoadResult;
- };
+ /**
+ * Whole-dataset points for the active selection — the feature-index scan — as a
+ * {@link RequestSlot}. Keyed by `` `${signature}#${cap}` ``: the selected-codes
+ * signature closes R2 (re-selecting a covered selection dedups to the live scan),
+ * and the cap closes R3 (raising the cap supersedes rather than reusing the smaller
+ * scan). The value carries its `signature` so coverage checks can read it, and the
+ * streaming `partial` is the scan's growing buffer.
+ */
+ matching: RequestSlot;
}
+/** A settled or in-flight matched batch, tagged with the selection it covers. */
+interface MatchingValue {
+ readonly signature: string;
+ readonly result: PointsLoadResult;
+}
+
+/** The two catalog phases: the instant resident-subset preview, then the
+ * authoritative full-dataset scan that supersedes it. */
+type CatalogPhase = 'preview' | 'full';
+
/** Public snapshot of a selection's feature-index load, for the filter panel. */
export interface PointsMatchingLoadState {
loading: boolean;
@@ -142,6 +187,63 @@ export class PointsResolver implements ResourceResolver this.notify();
+ entry = {
+ preload: new RequestSlot({
+ context: {
+ elementKey: key,
+ kind: 'points',
+ resource: 'preload',
+ fallback: 'load-failed',
+ },
+ onChange,
+ // The resident batch stays on screen through a reload (stale retention),
+ // so only its settle is a re-render — matching the pre-slot notify count.
+ notifyOnLoading: false,
+ }),
+ rowCodes: new RequestSlot | undefined>({
+ context: {
+ elementKey: key,
+ kind: 'points',
+ resource: 'rowCodes',
+ fallback: 'decode-failed',
+ },
+ onChange,
+ notifyOnLoading: false,
+ }),
+ matching: new RequestSlot({
+ context: {
+ elementKey: key,
+ kind: 'points',
+ resource: 'matching',
+ fallback: 'decode-failed',
+ },
+ onChange,
+ // The scan reports progress and a growing partial the panel/overlay draw,
+ // so its loading transitions and streamed partials ARE re-renders.
+ notifyOnLoading: true,
+ }),
+ catalog: new RequestSlot({
+ context: {
+ elementKey: key,
+ kind: 'points',
+ resource: 'catalog',
+ fallback: 'decode-failed',
+ },
+ onChange,
+ // The full-list scan shows a spinner; its loading transition is a re-render.
+ notifyOnLoading: true,
+ }),
+ };
+ this.entries.set(key, entry);
+ }
+ return entry;
+ }
+
// --- ResourceResolver -------------------------------------------------------
/**
@@ -168,13 +270,41 @@ export class PointsResolver implements ResourceResolver 0;
// Was `void engine.ensureRowFeatureCodes(...)` at useLayerData.ts:1425.
- const needsRowCodes = selectionActive || config.colorByFeature === true;
- if (needsRowCodes && !this.hasRowFeatureCodes(key)) {
- tasks.push({ id: `${key}#rowCodes`, resource: 'rowCodes' });
+ // Colour-by-feature is ON BY DEFAULT in the renderer (opt-out via
+ // `colorByFeature: false`), so the per-row codes must load whenever colour is not
+ // explicitly disabled — not only on an active selection. Gating on
+ // `=== true` left the "all features" view (no selection, no explicit flag) with no
+ // codes, so it drew flat. A dataset with a code column carries codes on the batch
+ // regardless, but the dict-only fallback settles the codes through THIS task, so
+ // the gate is what made dict-only "all features" render flat.
+ const needsRowCodes = selectionActive || config.colorByFeature !== false;
+ // Codes are only a valid mask for the batch they were read at THE SAME CAP as
+ // (R5) — index i names point i only then. `isReady` does not say that: codes
+ // settled at 4M stay ready after a raise to 8M, so this gate never re-requested
+ // them and the mask silently addressed the wrong rows against the bigger batch.
+ // Ask at the cap they would actually be loaded at, and put it in the task id so
+ // a cap change re-dispatches instead of deduping.
+ const rowCodesCap = this.rowCodesCap(key);
+ // While a preload is in flight, hold off: its decode settles the codes at its
+ // own cap for free whenever the element carries a code column, and asking now
+ // would race a second full read of the feature column against it. If it settles
+ // WITHOUT codes (the dict-only fallback), the next plan pass sees them
+ // misaligned and asks then. A first load — no codes at all — never waits.
+ const preloadInFlight = this.entries.get(key)?.preload.isLoading === true;
+ const deferToPreload = this.hasRowFeatureCodes(key) && preloadInFlight;
+ if (needsRowCodes && !this.hasRowFeatureCodesAtCap(key, rowCodesCap) && !deferToPreload) {
+ tasks.push({ id: `${key}#rowCodes:${rowCodesCap}`, resource: 'rowCodes' });
}
// Was `void engine.ensureMatchingFeaturesLoaded(...)` at useLayerData.ts:1375.
- if (selectionActive && this.supportsFeatureScan(key)) {
+ //
+ // Only worth scanning when the resident batch might be MISSING matching rows.
+ // A complete (untruncated) preload already holds every row in the dataset, so
+ // the render path's in-memory filter returns exactly what a whole-dataset scan
+ // would — instantly, with no I/O. Scanning anyway re-read the entire file and
+ // showed "Loading selected features… 0 points so far" for a selection whose
+ // points were already in memory.
+ if (selectionActive && this.supportsFeatureScan(key) && !this.isResidentComplete(key)) {
const signature = PointsResolver.matchingSignature(selection);
tasks.push({
id: `${key}#matching:${signature}:${cap}`,
@@ -252,51 +382,48 @@ export class PointsResolver implements ResourceResolver {
- const entry = this.entries.get(key);
- if (!entry) return Resolution.idle();
- switch (entry.status) {
- case 'ready':
- return entry.data ? Resolution.ready(entry.data) : Resolution.idle();
- case 'loading':
- // `stale` is what keeps the old batch on screen through a cap raise —
- // the atomic swap the old engine described as "no blank".
- return Resolution.loading(entry.data !== undefined ? { stale: entry.data } : {});
- case 'error':
- // Step 1 preserves today's behaviour: the engine console.errors and leaves
- // status 'error' with no structured error value. Wiring SpatialEntryError
- // through these paths is Track A's `retryable` work.
- return Resolution.idle();
- default:
- return Resolution.idle();
- }
+ // The slot IS the resolution — built at mutation time, returned by identity.
+ // A `loading` carries the previous batch as `stale` (the atomic swap, no blank);
+ // a rejected load is now a structured `failed` (Track A wired the error through).
+ return this.entries.get(key)?.preload.resolution ?? Resolution.idle();
}
private catalogResolution(key: string): Resolution {
- const entry = this.entries.get(key);
- if (!entry) return Resolution.idle();
- if (entry.catalogLoaded) return Resolution.ready(entry.catalog ?? null);
- return this.isFeatureCatalogLoading(key) ? Resolution.loading() : Resolution.idle();
+ const slot = this.entries.get(key)?.catalog;
+ if (!slot) return Resolution.idle();
+ // The catalog rides the geometry preload, so surface a running preload as the
+ // catalog loading too — a spinner, not an "idle" gap before the preview arrives.
+ if (slot.resolution.status === 'idle' && this.entries.get(key)?.preload.isLoading) {
+ return Resolution.loading();
+ }
+ return slot.resolution;
}
private rowCodesResolution(key: string): Resolution | undefined> {
- const entry = this.entries.get(key);
- if (!entry) return Resolution.idle();
- if (entry.rowCodesLoaded) return Resolution.ready(entry.rowCodes);
- return entry.rowCodesLoading ? Resolution.loading() : Resolution.idle();
+ return this.entries.get(key)?.rowCodes.resolution ?? Resolution.idle();
}
private matchingResolution(key: string): Resolution {
- const entry = this.entries.get(key);
- if (!entry) return Resolution.idle();
- const loading = entry.matchingLoading;
- if (loading) {
- return Resolution.loading({
- ...(loading.partialResult !== undefined ? { partial: loading.partialResult } : {}),
- ...(entry.matching !== undefined ? { stale: entry.matching.result } : {}),
- progress: { done: loading.matchedRows, scanned: loading.scannedRows },
- });
+ // Unwrap the slot's Resolution into Resolution
+ // — the resource surface is the batch, the signature is internal bookkeeping.
+ // Built on a snapshot-cache miss (once per version), so a fresh identity is fine.
+ const slot = this.entries.get(key)?.matching;
+ if (!slot) return Resolution.idle();
+ const r = slot.resolution;
+ switch (r.status) {
+ case 'ready':
+ return Resolution.ready(r.value.result);
+ case 'loading':
+ return Resolution.loading({
+ ...(r.partial !== undefined ? { partial: r.partial.result } : {}),
+ ...(r.stale !== undefined ? { stale: r.stale.result } : {}),
+ ...(r.progress !== undefined ? { progress: r.progress } : {}),
+ });
+ case 'failed':
+ return Resolution.failed(r.error, r.stale?.result);
+ default:
+ return Resolution.idle();
}
- return entry.matching ? Resolution.ready(entry.matching.result) : Resolution.idle();
}
private notices(key: string, featureCodes: readonly number[] | undefined): EntryNotice[] {
@@ -332,7 +459,10 @@ export class PointsResolver implements ResourceResolver | undefined {
+ const entry = this.entries.get(key);
+ return entry?.preload.partial?.featureCodeCounts ?? entry?.preload.lastGood?.featureCodeCounts;
+ }
+
+ /**
+ * The key (`${signature}#${cap}`) of the in-flight scan whose partial is streaming,
+ * or `undefined` when no scan is loading. The Renderer Adapter uses it to tell a
+ * *growing* partial (same scan, keep the resource identity, bump a revision) from a
+ * *new* scan (fresh resource) — the D10 flash fix.
+ */
+ getPartialScanKey(key: string): string | undefined {
+ const slot = this.entries.get(key)?.matching;
+ return slot?.isLoading ? slot.pendingKey : undefined;
}
getStatus(key: string): PointsLoadStatus {
- return this.entries.get(key)?.status ?? 'idle';
+ const resolution = this.entries.get(key)?.preload.resolution;
+ switch (resolution?.status) {
+ case 'loading':
+ return 'loading';
+ case 'ready':
+ return 'ready';
+ case 'failed':
+ return 'error';
+ default:
+ return 'idle';
+ }
}
/** Order-independent cache key for a selected-codes set. */
@@ -369,6 +545,17 @@ export class PointsResolver implements ResourceResolver left - right).join(',');
}
+ /** The matching slot key — signature AND cap, so both R2 and R3 are decided by it. */
+ private static matchingKey(signature: string, memoryCap: number): string {
+ return `${signature}#${memoryCap}`;
+ }
+
+ /** Split a matching slot key back into its signature and cap. */
+ private static parseMatchingKey(key: string): { signature: string; memoryCap: number } {
+ const hash = key.lastIndexOf('#');
+ return { signature: key.slice(0, hash), memoryCap: Number(key.slice(hash + 1)) };
+ }
+
/** Feature codes a matched batch/scan covers, parsed from its signature. */
private static coveredCodes(signature: string): Set {
if (signature === '') {
@@ -419,22 +606,32 @@ export class PointsResolver implements ResourceResolver 0 && entry.matching) {
- const covered = PointsResolver.coveredCodes(entry.matching.signature);
+ const matched = entry.matching.lastGood;
+ if (featureCodes && featureCodes.length > 0 && matched) {
+ const covered = PointsResolver.coveredCodes(matched.signature);
if (covered.size > 0 && featureCodes.every((code) => covered.has(code))) {
- const result = entry.matching.result;
+ const result = matched.result;
return {
truncated: result.preloadTruncated === true,
loaded: result.shape[1] ?? 0,
@@ -485,91 +683,115 @@ export class PointsResolver implements ResourceResolver {
const { key, layerId, element } = target;
- const existing = this.entries.get(key);
- // (1) Existing data covers this cap without a reload.
- if (
- existing?.data !== undefined &&
- PointsResolver.batchAdequateForCap(existing.data, memoryCap)
- ) {
- if (existing.loading) {
- existing.loadAbort?.abort();
- existing.loading = undefined;
- existing.loadAbort = undefined;
- }
- existing.memoryCap = memoryCap;
- // Cap lowered below what's resident → shed the excess in memory (no re-fetch).
- if ((existing.data.shape[1] ?? 0) > memoryCap) {
- existing.data = PointsResolver.sliceResidentBatch(existing.data, memoryCap);
- existing.residentCodes = undefined;
- existing.residentCodesSource = undefined;
- if (existing.rowCodes && existing.rowCodes.length > memoryCap) {
- existing.rowCodes = Array.prototype.slice.call(existing.rowCodes, 0, memoryCap);
- }
- this.notify();
- }
- return Promise.resolve();
- }
- // (2) A load for this exact cap is already in flight → dedup.
- if (existing?.loading && existing.memoryCap === memoryCap) {
- return existing.loading;
- }
-
- const entry: PointsEntry = existing ?? { status: 'idle' };
- entry.loadAbort?.abort();
- entry.memoryCap = memoryCap;
- const abort = new AbortController();
- entry.loadAbort = abort;
- entry.status = 'loading';
- this.entries.set(key, entry);
- this.callbacks.onStatus?.(layerId, 'loading');
-
- const loading = (async () => {
- try {
- // Read the feature column with the geometry so the filter's catalog and
- // per-row codes come from this one decode. The catalog here reflects only
- // the *resident* batch — an instant preview the full-dataset scan may
- // still supersede.
- const data = await element.loadPoints({
- includeFeatureCodes: true,
- memoryCap,
- signal: abort.signal,
- });
- if (abort.signal.aborted || entry.memoryCap !== memoryCap) {
- return;
- }
- entry.data = data;
+ const entry = this.ensureEntry(key);
+ const slot = entry.preload;
+ const resident = slot.lastGood;
+
+ // (1) The resident batch already covers this cap — no reload.
+ if (resident !== undefined && PointsResolver.batchAdequateForCap(resident, memoryCap)) {
+ if ((resident.shape[1] ?? 0) > memoryCap) {
+ // Cap lowered below what's resident → shed the excess IN MEMORY (no re-fetch),
+ // to a new key so a later raise supersedes. `settle` also cancels any
+ // in-flight reload for a different cap.
+ slot.settle(memoryCap, PointsResolver.sliceResidentBatch(resident, memoryCap));
entry.residentCodes = undefined;
entry.residentCodesSource = undefined;
- entry.status = 'ready';
- entry.featureCodeColumn = data.hasFeatureCodeColumn === true;
- if (data.featureCatalog !== undefined && !entry.catalogComplete) {
- entry.catalog = data.featureCatalog;
- entry.catalogLoaded = true;
- }
- if (data.featureCodes !== undefined) {
- entry.rowCodes = data.featureCodes;
- entry.rowCodesLoaded = true;
- entry.rowCodesCatalog = data.featureCatalog;
- this.reconcileRowCodes(entry);
+ const codes = entry.rowCodes.value;
+ // Deliberately conditional, and NOT re-keyed unconditionally on a shed.
+ // Codes shorter than the new window are already misaligned, and re-keying
+ // them to this cap would assert an alignment they do not have — the exact
+ // lie the slot key exists to prevent. Leaving the key stale is what makes
+ // the planning gate re-request them.
+ //
+ // `>` rather than `>=` is not a gap: codes are `min(rows, theirCap)` long, so
+ // whenever their key differs from their length the resident batch is at most
+ // that length too, and a shed below it makes the comparison strict anyway.
+ if (codes && codes.length > memoryCap) {
+ entry.rowCodes.settle(memoryCap, Array.prototype.slice.call(codes, 0, memoryCap));
}
- this.callbacks.onStatus?.(layerId, 'ready');
- } catch (error) {
- // Aborted (cap changed) or superseded → not a real error; stay quiet.
- if (abort.signal.aborted || entry.memoryCap !== memoryCap) {
- return;
- }
- entry.status = 'error';
- this.callbacks.onStatus?.(layerId, 'error');
- console.error(`Failed to load points for ${layerId}:`, error);
- } finally {
- if (entry.memoryCap === memoryCap) {
- entry.loading = undefined;
- entry.loadAbort = undefined;
+ } else if (slot.isLoading) {
+ // Resident already adequate but a reload for another cap is running → cancel
+ // it and keep the resident batch.
+ slot.settle(memoryCap, resident);
+ }
+ return slot.pending ?? Promise.resolve();
+ }
+
+ // (2)(3) Reload at this cap. The slot dedups an identical in-flight request and
+ // supersedes one for a different cap (R1: a superseded reload cannot write the
+ // live one's state). The previous batch stays on screen as `stale` until the new
+ // one settles — the atomic swap.
+ const before = slot.pending;
+ // Repaint granularity for the progressive preload: emitting every row group would
+ // re-render far more often than the eye needs on a multi-million-row load.
+ const PRELOAD_NOTIFY_STEP = 250_000;
+ let lastNotifiedRows = 0;
+ const loading = slot.request(memoryCap, async ({ emit, signal }) => {
+ // Read the feature column with the geometry so the filter's catalog and per-row
+ // codes come from this one decode. The catalog here reflects only the *resident*
+ // batch — an instant preview the full-dataset scan may still supersede.
+ const data = await element.loadPoints({
+ includeFeatureCodes: true,
+ memoryCap,
+ signal,
+ // Progressive preload (D3): publish the growing geometry so the base layer
+ // paints points as they decode instead of staying blank until the whole
+ // window lands. `emit` is inert once this request is superseded.
+ onProgress: (progress) => {
+ const silent = progress.matchedRows - lastNotifiedRows < PRELOAD_NOTIFY_STEP;
+ if (!silent) {
+ lastNotifiedRows = progress.matchedRows;
+ }
+ emit(
+ progress.partialResult,
+ { done: progress.matchedRows, scanned: progress.scannedRows },
+ { silent }
+ );
+ },
+ });
+ // Superseded mid-flight (a newer cap won): drop the derived cross-slot writes
+ // and let the slot ignore the return. Writing catalog/row codes from a stale
+ // load is exactly the corruption R1/R5 were.
+ if (signal.aborted) return data;
+ entry.residentCodes = undefined;
+ entry.residentCodesSource = undefined;
+ entry.featureCodeColumn = data.hasFeatureCodeColumn === true;
+ if (data.featureCatalog !== undefined) {
+ entry.previewCatalog = data.featureCatalog;
+ // Instant resident-subset preview; the full-dataset scan may supersede it.
+ // But `settle` ABORTS whatever the slot is running, so writing the preview
+ // while the full scan is in flight silently kills it — and nothing re-requests
+ // a catalog (`plan()` never emits a catalog task; only the panel's mount effect
+ // does), so the list stayed on partial "≥" counts until the panel remounted.
+ // The preview is a strict downgrade of a scan already under way; it stays
+ // available through `previewCatalog` instead.
+ const fullPending = entry.catalog.isLoading && entry.catalog.pendingKey === 'full';
+ if (entry.catalog.settledKey !== 'full' && !fullPending) {
+ entry.catalog.settle('preview', data.featureCatalog);
}
- this.notify();
}
- })();
- entry.loading = loading;
+ if (data.featureCodes !== undefined) {
+ // Row codes fall out of this decode, aligned to the batch at exactly this cap.
+ entry.rowCodes.settle(memoryCap, data.featureCodes);
+ entry.rowCodesCatalog = data.featureCatalog;
+ this.reconcileRowCodes(entry, this.getFeatureCatalog(key));
+ }
+ return data;
+ });
+
+ // Mirror the old onStatus contract precisely: 'loading' when a NEW load starts
+ // (not on dedup), then 'ready'/'error' for the load that actually settles this
+ // cap. A superseded or aborted load reports nothing.
+ if (loading !== before) {
+ this.callbacks.onStatus?.(layerId, 'loading');
+ void loading.then(() => {
+ if (slot.isReady && Object.is(slot.settledKey, memoryCap)) {
+ this.callbacks.onStatus?.(layerId, 'ready');
+ } else if (slot.isFailed) {
+ this.callbacks.onStatus?.(layerId, 'error');
+ }
+ });
+ }
return loading;
}
@@ -581,176 +803,185 @@ export class PointsResolver implements ResourceResolver {
const { key, element } = target;
- const entry = this.entries.get(key) ?? { status: 'idle' as PointsLoadStatus };
- this.entries.set(key, entry);
+ const entry = this.ensureEntry(key);
+ const slot = entry.matching;
const signature = PointsResolver.matchingSignature(featureCodes);
const isCoveredBy = (sig: string): boolean => {
const covered = PointsResolver.coveredCodes(sig);
return featureCodes.every((code) => covered.has(code));
};
- // A loaded batch already covers this selection AND still satisfies the cap →
- // reuse it; the layer filters down. No scan.
+
+ // (1) A last-good batch already covers this selection AND still satisfies the
+ // cap → reuse it; the layer filters down in memory. No scan. Coverage is a
+ // subset relation, richer than the slot's exact-key dedup, so it stays here.
+ const lastGood = slot.lastGood;
if (
- entry.matching &&
- isCoveredBy(entry.matching.signature) &&
- PointsResolver.batchAdequateForCap(entry.matching.result, memoryCap)
+ lastGood &&
+ isCoveredBy(lastGood.signature) &&
+ PointsResolver.batchAdequateForCap(lastGood.result, memoryCap)
) {
- entry.matchingLoading = undefined;
- return Promise.resolve();
+ // A now-unneeded scan may be in flight (the selection just shrank) → cancel it
+ // and keep the covering batch resident.
+ if (slot.isLoading) {
+ slot.settle(PointsResolver.matchingKey(lastGood.signature, memoryCap), lastGood);
+ }
+ return slot.pending ?? Promise.resolve();
}
- // An in-flight scan will cover this selection once it settles → wait for it.
- if (entry.matchingLoading && isCoveredBy(entry.matchingLoading.signature)) {
- return entry.matchingLoading.promise;
+
+ // (2) An in-flight scan at this cap will cover this selection once it settles →
+ // wait for it. This is R2: re-selecting a covered selection mid-scan must not
+ // start a second scan corrupting the first.
+ if (slot.isLoading && slot.pendingKey !== undefined) {
+ const pending = PointsResolver.parseMatchingKey(slot.pendingKey);
+ if (pending.memoryCap === memoryCap && isCoveredBy(pending.signature)) {
+ return slot.pending ?? Promise.resolve();
+ }
}
+ // (3) A new scan. The key carries the cap, so raising it supersedes rather than
+ // being served by the smaller scan (R3).
+ const scanKey = PointsResolver.matchingKey(signature, memoryCap);
const PROGRESS_NOTIFY_STEP = 5_000;
let lastNotifiedMatched = 0;
- const onProgress = (progress: PointsLoadProgress): void => {
- const loading = entry.matchingLoading;
- if (!loading || loading.signature !== signature) {
- return;
- }
- loading.matchedRows = progress.matchedRows;
- loading.scannedRows = progress.scannedRows;
- loading.partialResult = progress.partialResult;
- if (progress.matchedRows - lastNotifiedMatched >= PROGRESS_NOTIFY_STEP) {
- lastNotifiedMatched = progress.matchedRows;
- this.notify(); // runs during the async scan, not render — safe to notify sync
- }
- };
-
- const promise = (async () => {
- try {
- // Dict-only elements have no file-backed code column, so the scan must
- // resolve each row's feature_name against the same catalog the selection
- // was made in. The core call ignores this for indexed elements.
- const featureCodeByName =
- entry.featureCodeColumn === true ? undefined : featureCodeMapFromCatalog(entry.catalog);
- const result = await element.loadPointsMatchingFeatureCodes({
- featureCodes,
- memoryCap,
- onProgress,
- ...(featureCodeByName ? { featureCodeByName } : {}),
- });
- // Apply only if this is still the latest requested scan. Keeping the
- // previous `matching` batch until the current one is ready is what lets
- // the render keep showing the prior selection instead of blanking.
- if (entry.matchingLoading?.signature === signature) {
- entry.matching = { signature, result };
+ return slot.request(scanKey, async ({ emit, signal }) => {
+ // Dict-only elements have no file-backed code column, so the scan must resolve
+ // each row's feature_name against the same catalog the selection was made in.
+ // The core call ignores this for indexed elements.
+ const featureCodeByName =
+ entry.featureCodeColumn === true
+ ? undefined
+ : featureCodeMapFromCatalog(this.getFeatureCatalog(key));
+ const onProgress = (progress: PointsLoadProgress): void => {
+ // Keep the partial buffer fresh on EVERY tick (its identity drives the
+ // overlay resource), but only NOTIFY every PROGRESS_NOTIFY_STEP matched rows
+ // — the render granularity the old engine used. `emit` is dropped by the slot
+ // once this scan is superseded.
+ const silent = progress.matchedRows - lastNotifiedMatched < PROGRESS_NOTIFY_STEP;
+ if (!silent) {
+ lastNotifiedMatched = progress.matchedRows;
}
- } catch (error) {
- console.error(`Failed feature-index scan for ${target.layerId}:`, error);
- } finally {
- if (entry.matchingLoading?.signature === signature) {
- entry.matchingLoading = undefined;
- }
- this.notify();
- }
- })();
- entry.matchingLoading = { signature, promise, matchedRows: 0, scannedRows: 0 };
- // No queueMicrotask here, and none needed: nothing kicks a scan from render
- // any more. `plan()` is pure and returns a task; the store calls `load()` from
- // a commit-phase effect. The old engine's `queueMicrotask(() => this.notify())`
- // existed solely to defend against a synchronous notify during render, and the
- // phase separation makes that unreachable by construction.
- this.notify();
- return promise;
+ emit(
+ { signature, result: progress.partialResult },
+ { done: progress.matchedRows, scanned: progress.scannedRows },
+ { silent }
+ );
+ };
+ const result = await element.loadPointsMatchingFeatureCodes({
+ featureCodes,
+ memoryCap,
+ onProgress,
+ signal, // superseded scan aborts between row-group chunks
+ ...(featureCodeByName ? { featureCodeByName } : {}),
+ });
+ return { signature, result };
+ });
}
/** Whether the feature-index scan for this exact selection is in flight. */
isMatchingLoading(key: string, featureCodes: readonly number[]): boolean {
- const entry = this.entries.get(key);
- return entry?.matchingLoading?.signature === PointsResolver.matchingSignature(featureCodes);
+ const slot = this.entries.get(key)?.matching;
+ if (!slot?.isLoading || slot.pendingKey === undefined) {
+ return false;
+ }
+ return (
+ PointsResolver.parseMatchingKey(slot.pendingKey).signature ===
+ PointsResolver.matchingSignature(featureCodes)
+ );
}
getMatchingLoadState(
key: string,
featureCodes: readonly number[]
): PointsMatchingLoadState | undefined {
- const entry = this.entries.get(key);
- if (!entry) {
+ const slot = this.entries.get(key)?.matching;
+ if (!slot) {
return undefined;
}
const signature = PointsResolver.matchingSignature(featureCodes);
- const loading = entry.matchingLoading;
- if (loading?.signature === signature) {
- return {
- loading: true,
- matchedRows: loading.matchedRows,
- scannedRows: loading.scannedRows,
- settled: false,
- };
+
+ // A scan for exactly this selection is in flight.
+ if (slot.isLoading && slot.pendingKey !== undefined) {
+ const pending = PointsResolver.parseMatchingKey(slot.pendingKey);
+ if (pending.signature === signature) {
+ const progress =
+ slot.resolution.status === 'loading' ? slot.resolution.progress : undefined;
+ return {
+ loading: true,
+ matchedRows: progress?.done ?? 0,
+ scannedRows: progress?.scanned ?? 0,
+ settled: false,
+ };
+ }
}
- const matching = entry.matching;
- if (!matching) {
+
+ const matched = slot.lastGood;
+ if (!matched) {
return undefined;
}
- if (matching.signature === signature) {
- return {
- loading: false,
- matchedRows: matching.result.shape[1] ?? 0,
- scannedRows: matching.result.shape[1] ?? 0,
- settled: true,
- };
+ const rows = matched.result.shape[1] ?? 0;
+ if (matched.signature === signature) {
+ return { loading: false, matchedRows: rows, scannedRows: rows, settled: true };
}
// A larger loaded batch covers this selection — served from memory, no scan.
- const covered = PointsResolver.coveredCodes(matching.signature);
+ const covered = PointsResolver.coveredCodes(matched.signature);
if (covered.size > 0 && featureCodes.every((code) => covered.has(code))) {
- return {
- loading: false,
- matchedRows: matching.result.shape[1] ?? 0,
- scannedRows: matching.result.shape[1] ?? 0,
- settled: true,
- covered: true,
- };
+ return { loading: false, matchedRows: rows, scannedRows: rows, settled: true, covered: true };
}
return undefined;
}
- /** The feature codes the settled matched batch covers. */
+ /** The feature codes the last-good matched batch covers. */
getLoadedMatchingFeatureCodes(key: string): ReadonlySet | undefined {
- const matching = this.entries.get(key)?.matching;
- if (!matching) {
+ const matched = this.entries.get(key)?.matching.lastGood;
+ if (!matched) {
return undefined;
}
- return PointsResolver.coveredCodes(matching.signature);
+ return PointsResolver.coveredCodes(matched.signature);
}
- /** Per-row feature codes of the settled matched batch, row-aligned with it. */
+ /** Per-row feature codes of the last-good matched batch, row-aligned with it. */
getMatchingRowFeatureCodes(key: string): ArrayLike | undefined {
- return this.entries.get(key)?.matching?.result.featureCodes;
+ return this.entries.get(key)?.matching.lastGood?.result.featureCodes;
}
/** Per-row feature codes of the in-flight scan's partial buffer. */
getMatchingPartialRowFeatureCodes(key: string): ArrayLike | undefined {
- return this.entries.get(key)?.matchingLoading?.partialResult?.featureCodes;
+ return this.entries.get(key)?.matching.partial?.result.featureCodes;
}
// --- Feature catalog --------------------------------------------------------
getFeatureCatalog(key: string): PointsFeatureCatalog | null | undefined {
const entry = this.entries.get(key);
- return entry?.catalogLoaded ? (entry.catalog ?? null) : undefined;
+ const slot = entry?.catalog;
+ if (!slot) return undefined;
+ // Settled (preview or full) → the value (a catalog, or null for no feature_key).
+ if (slot.isReady) return slot.value ?? null;
+ // Loading: prefer the in-flight PARTIAL (the full names/codes list, published
+ // before the slow counts scan) over an older preview — it is the more complete
+ // list, just without counts yet. Then a preview, or a failed full-scan that
+ // retained one; last, a preview the preload produced *underneath* a running scan,
+ // which is deliberately not settled into the slot (see `previewCatalog`).
+ // Nothing at all → undefined (not loaded).
+ return slot.partial ?? slot.lastGood ?? entry?.previewCatalog ?? undefined;
}
+ /** True while a settled catalog does not yet exist AND one is on its way (either
+ * the full-list scan or the geometry preload that carries the preview). */
isFeatureCatalogLoading(key: string): boolean {
const entry = this.entries.get(key);
- if (!entry || entry.catalogLoaded) {
- return false;
- }
- // The catalog rides the geometry preload, so a running geometry load counts as
- // the catalog loading too — a spinner, not a premature "load feature list" prompt.
- return entry.catalogLoading !== undefined || entry.loading !== undefined;
+ if (!entry) return false;
+ const slot = entry.catalog;
+ // A settled preview or full catalog is "loaded"; the full scan behind a preview
+ // is *refining*, not loading (see isFeatureCatalogRefining).
+ if (slot.isReady || slot.lastGood !== undefined) return false;
+ return slot.isLoading || entry.preload.isLoading;
}
/** True while the full-dataset scan runs behind an instant resident-subset preview. */
isFeatureCatalogRefining(key: string): boolean {
- const entry = this.entries.get(key);
- return (
- entry?.catalogLoaded === true &&
- entry.catalogComplete !== true &&
- entry.catalogLoading !== undefined
- );
+ const slot = this.entries.get(key)?.catalog;
+ return slot?.isLoading === true && slot.pendingKey === 'full' && slot.lastGood !== undefined;
}
/** True when the element has a file-backed feature code column (globally authoritative). */
@@ -769,24 +1000,33 @@ export class PointsResolver implements ResourceResolver | undefined {
const entry = this.entries.get(key);
- const rowCodes = entry?.rowCodes;
+ const rowCodes = entry?.rowCodes.value;
if (!entry || rowCodes === undefined) {
return undefined;
}
@@ -821,91 +1061,137 @@ export class PointsResolver implements ResourceResolver {
const { key, element } = target;
- const entry = this.entries.get(key) ?? { status: 'idle' as PointsLoadStatus };
- this.entries.set(key, entry);
- if (entry.catalogComplete) {
+ const entry = this.ensureEntry(key);
+ const slot = entry.catalog;
+ // Already the authoritative full catalog, or a full scan already in flight → done.
+ if (slot.settledKey === 'full') {
return Promise.resolve();
}
- if (entry.catalogLoading) {
- return entry.catalogLoading;
- }
-
- const loading = (async () => {
- try {
- const fullCatalog = await element.listFeaturesWithCounts();
- entry.catalog = fullCatalog;
- // The full-dataset catalog is authoritative. Re-express any resident row
- // codes in its code space so the render's per-row codes match the panel's
- // selection and swatches.
- this.reconcileRowCodes(entry);
- } catch (error) {
- // Keep any resident preview catalog on failure rather than blanking it.
- if (!entry.catalogLoaded) entry.catalog = null;
- console.error(`Failed to build points feature catalog for ${target.layerId}:`, error);
- } finally {
- entry.catalogLoaded = true;
- entry.catalogComplete = true;
- entry.catalogLoading = undefined;
- this.notify();
- }
- })();
- entry.catalogLoading = loading;
- this.notify(); // surface the loading transition to the panel
- return loading;
+ if (slot.isLoading && slot.pendingKey === 'full') {
+ return slot.pending ?? Promise.resolve();
+ }
+ // Request 'full' — supersedes any 'preview', retaining it as `stale` so the
+ // preview keeps showing while the full list loads. A rejection becomes a
+ // `failed` (retryable) resolution, NOT a permanent null-settle: that is what
+ // A4's retry() unsticks. The preview, if any, survives as the failed `stale`.
+ return slot.request('full', async ({ emit, signal }) => {
+ const fullCatalog = await element.listFeaturesWithCounts({
+ // Publish the names-only catalog the moment it is known, so the panel can
+ // list features (and colour them) while the per-feature counts scan — which
+ // walks every row group — is still running. `emit` is inert once superseded.
+ onPartialCatalog: (partial) => {
+ emit(partial);
+ },
+ });
+ // R1: a superseded load must not write anything, least of all CROSS-SLOT state.
+ // The slot drops this return value, but `reconcileRowCodes` writes straight to
+ // `entry.rowCodes` — and `listFeaturesWithCounts` takes no signal, so a
+ // superseded scan runs to completion and lands here regardless. Remapping the
+ // rows into a catalog nobody is showing is precisely the split that drew every
+ // point in another gene's colour.
+ if (signal.aborted) return fullCatalog;
+ // The full-dataset catalog is authoritative. Re-express any resident row codes
+ // in its space so the render's per-row codes match the panel's selection.
+ this.reconcileRowCodes(entry, fullCatalog);
+ return fullCatalog;
+ });
}
// --- Row feature codes ------------------------------------------------------
getRowFeatureCodes(key: string): ArrayLike | undefined {
- return this.entries.get(key)?.rowCodes;
+ return this.entries.get(key)?.rowCodes.value;
}
- /** True once row codes have settled (even if the element has none). */
+ /** True once row codes have settled (even if the element has none). Says nothing
+ * about WHICH cap they were read at — see {@link hasRowFeatureCodesAtCap}. */
hasRowFeatureCodes(key: string): boolean {
- return this.entries.get(key)?.rowCodesLoaded === true;
+ return this.entries.get(key)?.rowCodes.isReady === true;
+ }
+
+ /**
+ * The cap {@link ensureRowFeatureCodes} would read the codes at — i.e. the
+ * resident batch's window. Kept beside it so the planning gate and the loader
+ * cannot disagree about which cap "aligned" means.
+ */
+ private rowCodesCap(key: string): number {
+ const preload = this.entries.get(key)?.preload;
+ return preload?.settledKey ?? preload?.pendingKey ?? DEFAULT_POINTS_MEMORY_CAP;
+ }
+
+ /** True once row codes have settled AT `memoryCap` — the only state in which they
+ * are a valid row-aligned mask for the resident batch at that cap. */
+ hasRowFeatureCodesAtCap(key: string, memoryCap: number): boolean {
+ const slot = this.entries.get(key)?.rowCodes;
+ return slot?.isReady === true && Object.is(slot.settledKey, memoryCap);
}
/**
* Idempotently load the row feature codes for the resident batch.
*
- * NOTE (R5): this takes no memory cap, so core falls back to the 4M default
- * while `ensureLoaded` honours the user's — misaligning the filter mask against
- * an 8M resident batch. Preserved verbatim here; it is Track A's to fix.
+ * **R5 fix:** the codes are read at the resident preload's cap — its slot key — so
+ * index i in the codes names the feature of point i in the batch. Reading them at a
+ * different window (the old 4M default while the preload honoured an 8M cap) is
+ * exactly the mask misalignment R5 was. Normally the codes fall out of the geometry
+ * decode (`ensureLoaded`) and this is a no-op; it is the fallback for a codeless
+ * preload or a filter toggled before the codes were resident.
*/
ensureRowFeatureCodes(target: PointsLoadTarget): Promise {
const { key, element } = target;
- const entry = this.entries.get(key) ?? { status: 'idle' as PointsLoadStatus };
- this.entries.set(key, entry);
- if (entry.rowCodesLoaded) {
- return Promise.resolve();
+ const entry = this.ensureEntry(key);
+ const slot = entry.rowCodes;
+ const cap = entry.preload.settledKey ?? entry.preload.pendingKey ?? DEFAULT_POINTS_MEMORY_CAP;
+ // Already aligned at this cap (typically settled by the preload decode) → no-op.
+ if (slot.isReady && Object.is(slot.settledKey, cap)) {
+ return slot.pending ?? Promise.resolve();
}
- if (entry.rowCodesLoading) {
- return entry.rowCodesLoading;
- }
-
- const loading = (async () => {
- try {
- const catalog = this.getFeatureCatalog(key);
- entry.rowCodes = await element.loadRowFeatureCodes({ featureCatalog: catalog });
- entry.rowCodesCatalog = catalog ?? undefined;
- this.reconcileRowCodes(entry);
- } catch (error) {
- entry.rowCodes = undefined;
- console.error(`Failed to load points row feature codes for ${target.layerId}:`, error);
- } finally {
- entry.rowCodesLoaded = true;
- entry.rowCodesLoading = undefined;
- this.notify();
- }
- })();
- entry.rowCodesLoading = loading;
- return loading;
+ return slot.request(cap, async ({ signal }) => {
+ const catalog = this.getFeatureCatalog(key);
+ const codes = await element.loadRowFeatureCodes({
+ featureCatalog: catalog,
+ memoryCap: cap,
+ signal,
+ });
+ if (signal.aborted) return codes;
+ // These codes were just built against `catalog`, so their code space IS the
+ // current one — no remap here. A *later* catalog upgrade re-expresses them via
+ // `ensureFeatureCatalog` → `reconcileRowCodes`, which reads the settled value.
+ entry.rowCodesCatalog = catalog ?? undefined;
+ return codes;
+ });
+ }
+
+ // --- Retry ------------------------------------------------------------------
+
+ /**
+ * Re-run any **failed** resources of an element. This is what unsticks the
+ * permanently-settled catalog scan (ADR 0004 §3): a failed full-catalog scan is a
+ * `failed` slot, not a null-settle, so `retry()` re-runs its loader. Idle/loading/
+ * ready slots are untouched. Returns once every retried load settles.
+ */
+ retry(key: string): Promise {
+ const entry = this.entries.get(key);
+ if (!entry) return Promise.resolve();
+ const pending = [entry.preload, entry.catalog, entry.rowCodes, entry.matching]
+ .filter((slot) => slot.isFailed)
+ .map((slot) => slot.retry())
+ .filter((promise): promise is Promise => promise !== undefined);
+ return Promise.all(pending).then(() => undefined);
}
// --- Lifecycle --------------------------------------------------------------
/** Drop an element from the cache. Catalog and row codes live in the same entry. */
evict(key: string): void {
+ const entry = this.entries.get(key);
+ if (entry) {
+ // Abort any in-flight load so a superseded/evicted scan stops decoding rather
+ // than running to completion into a dropped result.
+ entry.preload.reset();
+ entry.rowCodes.reset();
+ entry.catalog.reset();
+ entry.matching.reset();
+ }
const existed = this.entries.delete(key);
this.snapshots.evictByElement(key);
// Notify so external-store consumers drop the now-stale snapshot immediately,
@@ -914,6 +1200,12 @@ export class PointsResolver implements ResourceResolver` — one in-flight-or-settled resource, as a value.
+ *
+ * This is the primitive Track A (ADR 0004 §Step 2) uses to replace the four
+ * hand-rolled dedup/supersede/settle implementations inside `PointsResolver`'s
+ * `PointsEntry` (`ensureLoaded`, `ensureFeatureCatalog`, `ensureRowFeatureCodes`,
+ * `ensureMatchingFeaturesLoaded`). Each of those grew its own ad-hoc mutable
+ * bookkeeping — a `loading` promise here, a `signature` guard there, a `finally`
+ * that clears markers only `if (entry.memoryCap === memoryCap)` — and every one of
+ * the four known points races (R1, R2, R3, R5) is a bug in that bookkeeping. A slot
+ * makes the bookkeeping one tested thing.
+ *
+ * ## The two rules that make it correct
+ *
+ * 1. **Supersession by record identity, never by value.** Each `request` allocates
+ * an `InFlight` record. Every async continuation — success, failure, and each
+ * streamed `emit` — first checks `if (this.current !== record) return`. A load
+ * that has been superseded cannot write anything: not its result, not its error,
+ * not a late progress tick. This is the exact discipline `SpatialEntryStore`
+ * already uses for its per-task `AbortController`s, lifted to the value level.
+ * R1 and R2 are two live loads with equal keys clobbering each other; this rule
+ * is what forbids it.
+ *
+ * 2. **Everything the request depends on lives in `K`.** Dedup is `equals(key,
+ * currentKey)`; anything not in the key cannot supersede. R3 (a cap raise served
+ * by the smaller scan) and R5 (row codes loaded at the wrong cap) are both keys
+ * that omit a dimension — so callers put the memory cap *in* the key, exactly as
+ * `ResolveTask.id` already carries it.
+ *
+ * ## Failure is a state
+ *
+ * A rejected loader becomes `Resolution.failed(error, stale)` via
+ * `toSpatialEntryError` — not a `console.error` and a dead status. `retry()` re-runs
+ * the last request's loader, which is what unsticks a permanently-settled catalog
+ * scan (ADR 0004 §3). Cancellation is checked first and is a non-event: a
+ * superseded or aborted load reverts to the last good value, it does not paint an
+ * error.
+ *
+ * ## Identity discipline
+ *
+ * `resolution` is constructed at mutation time and returned by reference — never
+ * rebuilt on read. A resolver's `snapshot()` reads `slot.resolution` straight
+ * through, so a fresh identity per render would be a deck teardown per frame (see
+ * `resolution.ts` "the identity rule"). The only values that change identity are
+ * the ones a load actually produces.
+ */
+
+import { isCancellation, type SpatialEntryErrorContext, toSpatialEntryError } from './errors.js';
+import { Resolution, type ResolutionProgress } from './resolution.js';
+
+/** What a slot's loader is handed: the abort signal the slot owns, and a way to
+ * publish an in-flight partial. Both are inert once the request is superseded. */
+export interface SlotLoadContext {
+ /** Aborted when this request is superseded (or the slot is reset/disposed). */
+ readonly signal: AbortSignal;
+ /**
+ * Publish the streaming load's growing value (+ progress). Dropped silently once
+ * this request is no longer current — a late tick from a superseded scan cannot
+ * repaint the live one.
+ *
+ * `options.silent` updates the value **without** firing `onChange`: use it to keep
+ * the partial data fresh on every producer tick while throttling how often the
+ * host re-renders. The value's identity still changes, so a getter reading it sees
+ * the update; only the notification is suppressed.
+ */
+ emit(partial: V, progress?: ResolutionProgress, options?: { silent?: boolean }): void;
+}
+
+/** Runs one request. Returns the settled value; may `emit` partials along the way. */
+export type SlotLoader = (ctx: SlotLoadContext) => Promise;
+
+export interface RequestSlotOptions {
+ /** Classifier context for failures this slot's loaders throw. `elementKey`/`kind`
+ * are fixed per slot; `resource`/`fallback` name what the loader was doing. */
+ readonly context: SpatialEntryErrorContext;
+ /** Key equality. Defaults to `Object.is`; keep `K` a value with meaningful `===`
+ * (a primitive, or an interned string like `` `${signature}#${cap}` ``). */
+ readonly equals?: (a: K, b: K) => boolean;
+ /** Invoked after any resolution change, so the owner can bump a version / notify. */
+ readonly onChange?: () => void;
+ /**
+ * Whether a transition *into or within* the `loading` state fires `onChange`.
+ * Default `true`. Set `false` for a slot whose loading-start (and streamed
+ * partials) should NOT trigger a host re-render because the drawable value — the
+ * retained `stale` — has not changed: the resident preload and its row codes keep
+ * the previous batch on screen through a reload, so only their *settle* is a
+ * re-render (a catalog spinner, by contrast, wants the loading transition). This
+ * is what keeps `notify()` counts identical to the pre-slot engine.
+ */
+ readonly notifyOnLoading?: boolean;
+}
+
+interface InFlight {
+ readonly key: K;
+ readonly controller: AbortController;
+ /** Assigned synchronously right after construction; never observed before then. */
+ promise: Promise;
+}
+
+export class RequestSlot {
+ private readonly context: SpatialEntryErrorContext;
+ private readonly equals: (a: K, b: K) => boolean;
+ private readonly onChange: (() => void) | undefined;
+ private readonly notifyOnLoading: boolean;
+
+ private current: InFlight | undefined;
+ private _resolution: Resolution = Resolution.idle();
+ /** Key of the last `ready` value — drives the "already satisfied" dedup and is
+ * the base `retry()` re-runs. Meaningful only while `status === 'ready'`. */
+ private readyKey: K | undefined;
+ /** The last request, retained so `retry()` can re-run it verbatim. */
+ private lastRequest: { key: K; loader: SlotLoader } | undefined;
+
+ constructor(options: RequestSlotOptions) {
+ this.context = options.context;
+ this.equals = options.equals ?? Object.is;
+ this.onChange = options.onChange;
+ this.notifyOnLoading = options.notifyOnLoading ?? true;
+ }
+
+ // --- Reads ------------------------------------------------------------------
+
+ /** The slot's state as a value. Identity-stable between mutations. */
+ get resolution(): Resolution {
+ return this._resolution;
+ }
+
+ /** The settled value, and only that — `undefined` while loading, even with a stale. */
+ get value(): V | undefined {
+ return Resolution.readyValue(this._resolution);
+ }
+
+ /** The newest drawable value: `ready`, else a retained `stale`. */
+ get lastGood(): V | undefined {
+ return Resolution.lastGood(this._resolution);
+ }
+
+ /** The in-flight load's growing buffer, if it has produced one. */
+ get partial(): V | undefined {
+ return Resolution.partialValue(this._resolution);
+ }
+
+ get isLoading(): boolean {
+ return this._resolution.status === 'loading';
+ }
+
+ get isReady(): boolean {
+ return this._resolution.status === 'ready';
+ }
+
+ get isFailed(): boolean {
+ return this._resolution.status === 'failed';
+ }
+
+ /** The key currently loading, if any — for adequacy checks against a new request. */
+ get pendingKey(): K | undefined {
+ return this.current?.key;
+ }
+
+ /** The key of the settled value, if `ready`. */
+ get settledKey(): K | undefined {
+ return this._resolution.status === 'ready' ? this.readyKey : undefined;
+ }
+
+ /** The in-flight promise, if any — for awaiting or returning from a dedup. */
+ get pending(): Promise | undefined {
+ return this.current?.promise;
+ }
+
+ // --- Mutations --------------------------------------------------------------
+
+ /**
+ * Ask the slot for `key`. **Dedups** an in-flight or already-settled request for
+ * the same key; otherwise **supersedes**: aborts the previous load, retains its
+ * last good value as `stale`, and runs `loader`.
+ *
+ * Returns the promise of whichever load now serves this key (the existing one on
+ * a dedup, the new one on a supersede).
+ */
+ request(key: K, loader: SlotLoader): Promise {
+ this.lastRequest = { key, loader };
+
+ const current = this.current;
+ if (current && this.equals(current.key, key)) {
+ return current.promise; // same key, still in flight → dedup
+ }
+ if (
+ !current &&
+ this._resolution.status === 'ready' &&
+ this.readyKey !== undefined &&
+ this.equals(this.readyKey, key)
+ ) {
+ return Promise.resolve(); // already satisfied for this key → no-op
+ }
+
+ // Supersede. Capturing `stale` now chains it across repeated supersessions,
+ // because `lastGood` already reads through a prior `loading.stale`.
+ current?.controller.abort();
+ const stale = Resolution.lastGood(this._resolution);
+ const controller = new AbortController();
+ const record: InFlight = {
+ key,
+ controller,
+ promise: undefined as unknown as Promise,
+ };
+ this.current = record;
+ this.set(Resolution.loading(stale !== undefined ? { stale } : {}));
+
+ const run = async (): Promise => {
+ try {
+ const value = await loader({
+ signal: controller.signal,
+ emit: (partial, progress, options) => {
+ if (this.current !== record) return; // superseded → drop the tick
+ // Update the partial value (identity changes so a getter sees it); notify
+ // unless silent — the caller throttles re-renders while keeping data fresh.
+ this._resolution = Resolution.loading({
+ ...(stale !== undefined ? { stale } : {}),
+ partial,
+ ...(progress !== undefined ? { progress } : {}),
+ });
+ if (!options?.silent) this.onChange?.();
+ },
+ });
+ if (this.current !== record) return; // superseded → drop the result
+ this.current = undefined;
+ this.readyKey = key;
+ this.set(Resolution.ready(value));
+ } catch (cause) {
+ if (this.current !== record) return; // superseded → not our failure
+ this.current = undefined;
+ if (isCancellation(cause) || controller.signal.aborted) {
+ // A non-event: fall back to the last good value rather than paint an error.
+ this.set(stale !== undefined ? Resolution.ready(stale) : Resolution.idle());
+ if (stale === undefined) this.readyKey = undefined;
+ } else {
+ this.set(Resolution.failed(toSpatialEntryError(cause, this.context), stale));
+ }
+ }
+ };
+ record.promise = run();
+ return record.promise;
+ }
+
+ /**
+ * Set `ready(value)` for `key` directly, cancelling any in-flight load. For
+ * values produced *outside* the slot's own loader — the resident-preview catalog
+ * and row codes that fall out of the geometry preload's single decode, and the
+ * in-memory cap shed (a lower cap slices the resident batch to a new key without
+ * re-fetching).
+ */
+ settle(key: K, value: V): void {
+ this.current?.controller.abort();
+ this.current = undefined;
+ this.readyKey = key;
+ this.set(Resolution.ready(value));
+ }
+
+ /** Re-run the last request's loader. The retry affordance behind a failed state. */
+ retry(): Promise | undefined {
+ if (!this.lastRequest) return undefined;
+ if (this.current) return this.current.promise; // already (re)loading
+ // Clear the ready-key short-circuit so an unchanged key still re-runs.
+ this.readyKey = undefined;
+ return this.request(this.lastRequest.key, this.lastRequest.loader);
+ }
+
+ /** Abort any in-flight load and return to `idle`. For eviction / disposal. */
+ reset(): void {
+ this.current?.controller.abort();
+ this.current = undefined;
+ this.readyKey = undefined;
+ this.lastRequest = undefined;
+ this.set(Resolution.idle());
+ }
+
+ private set(next: Resolution): void {
+ this._resolution = next;
+ if (next.status === 'loading' && !this.notifyOnLoading) return;
+ this.onChange?.();
+ }
+}
diff --git a/packages/core/src/engine/index.ts b/packages/core/src/engine/index.ts
index 29affa45..9a74ae7a 100644
--- a/packages/core/src/engine/index.ts
+++ b/packages/core/src/engine/index.ts
@@ -21,6 +21,8 @@ export type {
PointsResolverCallbacks,
} from './PointsResolver.js';
export { PointsResolver } from './PointsResolver.js';
+export type { RequestSlotOptions, SlotLoadContext, SlotLoader } from './RequestSlot.js';
+export { RequestSlot } from './RequestSlot.js';
export type { ResolutionProgress } from './resolution.js';
// `Resolution` is both the type and its constructor namespace — one export carries both.
export { fromResult, Resolution } from './resolution.js';
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 3e767a57..88aaeb37 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -15,8 +15,10 @@ export {
export { tableToIndexColumnName } from './models/VTableSource.js';
export {
featureCodeMapFromCatalog,
+ featureNamesForCodes,
mergeFeatureCountsIntoCatalog,
remapRowFeatureCodes,
+ resolveFeatureSelectionCodes,
} from './pointsFeatures.js';
export {
applyRenderCapToColumnar,
diff --git a/packages/core/src/models/VPointsSource.ts b/packages/core/src/models/VPointsSource.ts
index e175e1dc..1d910005 100644
--- a/packages/core/src/models/VPointsSource.ts
+++ b/packages/core/src/models/VPointsSource.ts
@@ -1,3 +1,4 @@
+import type { Vector } from 'apache-arrow';
import { decodeIntStat, parseParquetFileMetaData } from '../parquetFooterStats.js';
import {
buildFeatureCatalogFromColumns,
@@ -209,6 +210,7 @@ import {
extractSentinelBoundingBox,
featureCodeAllowSet,
filterPointsToBounds,
+ isMortonSentinelValue,
MORTON_CODE_2D_COLUMN,
mortonIntervalsForBounds,
type PointsFeatureCatalog,
@@ -263,6 +265,209 @@ export function normalizeAxes(axes: Axis[]) {
const pointsElementRegex = /^points\/([^/]*)$/;
const pointsSubElementRegex = /^points\/([^/]*)\/(.*)$/;
+/**
+ * Rows per batch when streaming the feature column for the catalog.
+ *
+ * Larger than the upstream 1024 default: each batch costs an IPC round-trip, and
+ * at 1024 that overhead dominated (60 batches for 60k rows). 16k keeps partials
+ * frequent enough to look progressive while amortising the per-batch cost.
+ */
+const FEATURE_STREAM_BATCH_ROWS = 16_384;
+
+/**
+ * Rows per batch when streaming geometry + colour for the preload.
+ *
+ * Bigger than the catalog scan's batch: this path copies axis values per batch, so
+ * fewer, larger batches amortise that better, while still being small enough that
+ * the first coloured points land in tens of milliseconds.
+ */
+const PRELOAD_STREAM_BATCH_ROWS = 65_536;
+
+/**
+ * Assign a code to each row of one feature-column batch, appending into `codeBuffer`
+ * at `offset` and tallying into `codeCounts`.
+ *
+ * Codes are allocated on first sight via the shared `nameToCode`, so they stay
+ * stable for the whole stream — a gene coloured in batch 1 keeps its code in batch
+ * 62, and `codeToName` always describes the codes actually written. For a
+ * dictionary chunk that means dictionary order, not row order; the caller documents
+ * why that is fine.
+ *
+ * A DICTIONARY column resolves its values once per chunk and maps raw indices;
+ * asking the vector per row would materialise a JS string per point (~59s for 4M
+ * rows) instead of once per distinct feature. See `resolveRowFeatureCodesFromTable`.
+ */
+/**
+ * Add this batch's per-feature row counts into `counts`, resolving names through
+ * the already-populated `nameToCode`.
+ *
+ * Same dictionary-first approach as {@link appendFeatureCodesFromColumn}: read each
+ * chunk's distinct values once rather than materialising a string per row.
+ */
+/** Exported for test only — not re-exported from the package index. */
+export function tallyFeatureCodesFromColumn(
+ column: Vector,
+ rows: number,
+ nameToCode: ReadonlyMap,
+ counts: Map,
+ /** The element's morton column when it is tiled. Every catalog builder this is
+ * paired with is told to skip the sentinel rows; walking them here instead would
+ * let the counts and the entries of the SAME catalog disagree. Only the first
+ * four rows can be sentinels, so this costs four boxed reads. */
+ mortonColumn?: Vector | null
+): void {
+ const isSentinelRow = (rowIndex: number): boolean =>
+ mortonColumn != null && rowIndex < 4 && isMortonSentinelValue(mortonColumn.get(rowIndex));
+ let seen = 0;
+ let chunkStart = 0;
+ for (const chunk of column.data) {
+ if (seen >= rows) {
+ break;
+ }
+ const take = Math.min(chunk.length, rows - seen);
+ const dictionary = chunk.dictionary;
+ if (dictionary && chunk.nullCount === 0) {
+ const codeByIndex = new Int32Array(dictionary.length);
+ for (let index = 0; index < dictionary.length; index += 1) {
+ const name = dictionary.get(index);
+ codeByIndex[index] = name == null ? -1 : (nameToCode.get(String(name)) ?? -1);
+ }
+ const indices = chunk.values as ArrayLike;
+ for (let row = 0; row < take; row += 1) {
+ if (isSentinelRow(chunkStart + row)) {
+ continue;
+ }
+ const index = indices[row];
+ const code = index >= 0 && index < codeByIndex.length ? codeByIndex[index] : -1;
+ if (code >= 0) {
+ counts.set(code, (counts.get(code) ?? 0) + 1);
+ }
+ }
+ } else {
+ for (let row = 0; row < take; row += 1) {
+ if (isSentinelRow(chunkStart + row)) {
+ continue;
+ }
+ const value = column.get(chunkStart + row);
+ const code = value == null ? -1 : (nameToCode.get(String(value)) ?? -1);
+ if (code >= 0) {
+ counts.set(code, (counts.get(code) ?? 0) + 1);
+ }
+ }
+ }
+ seen += take;
+ chunkStart += chunk.length;
+ }
+}
+
+function appendFeatureCodesFromColumn(
+ column: Vector,
+ rows: number,
+ codeToName: Map,
+ nameToCode: Map,
+ codeBuffer: Int32Array,
+ codeCounts: Map,
+ offset: number
+): void {
+ const codeFor = (name: string): number => {
+ let code = nameToCode.get(name);
+ if (code === undefined) {
+ code = nameToCode.size;
+ nameToCode.set(name, code);
+ codeToName.set(code, name);
+ }
+ return code;
+ };
+ let written = 0;
+ let chunkStart = 0;
+ for (const chunk of column.data) {
+ if (written >= rows) {
+ break;
+ }
+ const take = Math.min(chunk.length, rows - written);
+ const dictionary = chunk.dictionary;
+ if (dictionary && chunk.nullCount === 0) {
+ const codeByIndex = new Int32Array(dictionary.length);
+ for (let index = 0; index < dictionary.length; index += 1) {
+ const name = dictionary.get(index);
+ codeByIndex[index] = name == null ? -1 : codeFor(String(name));
+ }
+ const indices = chunk.values as ArrayLike;
+ for (let row = 0; row < take; row += 1) {
+ const index = indices[row];
+ const code = index >= 0 && index < codeByIndex.length ? codeByIndex[index] : -1;
+ codeBuffer[offset + written + row] = code;
+ codeCounts.set(code, (codeCounts.get(code) ?? 0) + 1);
+ }
+ } else {
+ for (let row = 0; row < take; row += 1) {
+ const value = column.get(chunkStart + row);
+ const code = value == null ? -1 : codeFor(String(value));
+ codeBuffer[offset + written + row] = code;
+ codeCounts.set(code, (codeCounts.get(code) ?? 0) + 1);
+ }
+ }
+ written += take;
+ chunkStart += chunk.length;
+ }
+}
+
+/**
+ * How long a single streamed batch may take before the scan gives up and falls
+ * back to the byte-oriented path.
+ *
+ * The streaming reader has two failure modes that cannot be caught normally: a
+ * failed range request mid-stream leaves `read()` pending forever (no error, no
+ * rejection), and a wasm panic surfaces as an async `RuntimeError` that escapes
+ * try/catch, again leaving the read unsettled. Neither is recoverable by
+ * inspection, so progress is bounded instead: no batch within this window means
+ * abandon the stream rather than strand every consumer waiting on the catalog.
+ *
+ * Generous because a legitimate first batch over a slow link has been measured
+ * near 3s; this is an abnormal-condition backstop, not a latency budget.
+ */
+const FEATURE_STREAM_STALL_TIMEOUT_MS = 15_000;
+
+/**
+ * A no-progress watchdog for the whole streaming attempt.
+ *
+ * It must cover opening the file and the stream, not only the reads: an HTTP
+ * error mid-stream panics the reader during setup as readily as during a read,
+ * and in both cases the promise simply never settles. Guarding `read()` alone
+ * leaves `fromUrl()`/`stream()` able to hang forever.
+ *
+ * `progress()` is called whenever a batch lands, so a legitimately long scan
+ * keeps running while a truly stalled one is abandoned.
+ */
+function createStallGuard(timeoutMs: number) {
+ let timer: ReturnType | undefined;
+ let reject: ((error: Error) => void) | undefined;
+ let settled = false;
+ const promise = new Promise((_resolve, rejectFn) => {
+ reject = rejectFn;
+ });
+ const arm = () => {
+ clearTimeout(timer);
+ if (settled) {
+ return;
+ }
+ timer = setTimeout(() => {
+ settled = true;
+ reject?.(new Error(`Parquet feature stream made no progress for ${timeoutMs}ms`));
+ }, timeoutMs);
+ };
+ arm();
+ return {
+ /** Rejects once the watchdog trips; never resolves. */
+ promise,
+ progress: arm,
+ dispose: () => {
+ settled = true;
+ clearTimeout(timer);
+ },
+ };
+}
+
function getPointsElementPath(arrPath?: string) {
if (arrPath) {
const matches = arrPath.match(pointsSubElementRegex);
@@ -378,6 +583,311 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
* shape: [number, number],
* }>} A promise for a zarr array containing the data.
*/
+ /**
+ * Progressive geometry preload (D3): decode the capped window ONE ROW GROUP AT A
+ * TIME, emitting a growing partial after each, so points appear while the rest
+ * decodes instead of only after a single multi-second whole-part decode. This is
+ * the fix for "wild-type transcripts show nothing for ages".
+ *
+ * Only the axes (and an authoritative INTEGER feature-code column, when the
+ * dataset has one) are read here. That restriction is the whole design:
+ * `readParquetRowGroup` mis-decodes DICTIONARY-encoded columns, so the
+ * `feature_name` dict column can never be read this way — but plain float axes and
+ * a plain int code column are safe. So a dataset WITH a code column streams fully
+ * COLOURED from the first chunk, while a dict-only dataset streams flat and has
+ * its codes/catalog settled afterwards by the one-shot whole-part decode.
+ *
+ * The accumulator is preallocated at `maxRows` and appended at an offset cursor;
+ * each partial exposes the filled prefix as `subarray` VIEWS, so a progress tick
+ * is free — no re-concatenation (contrast {@link pointsScanChunkProgress}, which
+ * re-copies the whole buffer per chunk and is O(chunks²)).
+ *
+ * Returns null when streaming isn't possible (no dataset metadata, no range
+ * reads, worker disabled, nothing decoded) so the caller falls back to one-shot.
+ */
+ /**
+ * Progressive preload that streams geometry AND colour together.
+ *
+ * The row-group preload below cannot read a DICTIONARY-typed `feature_name`, so
+ * for the common Xenium-style element (dict feature column, no integer code
+ * column) it streams geometry and publishes no codes at all — points appear
+ * quickly but stay flat until a whole separate decode settles the codes. This
+ * path removes that split: `ParquetFile.stream()` decodes dictionary columns
+ * correctly, so x/y and the feature column arrive in the same batches and every
+ * partial is already coloured.
+ *
+ * The returned `featureCodes` and `featureCatalog` are built from one shared
+ * name→code map, so they are consistent WITH EACH OTHER — which is the contract
+ * the resolver needs. They are NOT in the same code space as the full-dataset
+ * scan: codes here follow each chunk's dictionary order, while the scan assigns
+ * in row order, so the same gene generally gets a different number. That is
+ * expected for a dictionary-only element and is what `reconcileRowCodes` /
+ * `remapRowFeatureCodes` exist for — the resolver re-expresses these codes
+ * against the authoritative catalog once it settles, keyed off the catalog
+ * returned here. Returning the matching catalog is therefore load-bearing.
+ *
+ * Reading codes from the dictionary rather than per row also means the catalog is
+ * complete from the first batch (all 541 genes of a wild-type element at ~620ms),
+ * at the cost of possibly listing a gene whose rows are all beyond the cap.
+ *
+ * Measured on a 4M-row wild-type transcripts element: first coloured batch at
+ * ~40ms and all 4M rows coloured in ~350ms, against a row-group preload that
+ * reaches full geometry at ~11s and never colours.
+ *
+ * Returns null when the fast path does not apply (non-URL store, no suffix-range
+ * support, no streaming reader, nothing decoded) so the caller falls through.
+ */
+ private async streamPointsWithFeaturesByUrl(
+ parquetPath: string,
+ options: {
+ axisNames: string[];
+ featureKey: string;
+ maxRows: number;
+ totalRowCount: number;
+ preloadTruncated: boolean;
+ hasFeatureCodeColumn: boolean;
+ onProgress?: (progress: PointsLoadProgress) => void;
+ signal?: AbortSignal;
+ }
+ ): Promise {
+ if (!(await this.canStreamParquetByUrl())) {
+ return null;
+ }
+ const dataset = await this.loadParquetDatasetMetadata(parquetPath);
+ const partPaths = dataset?.parts.map((part) => part.path);
+ if (!partPaths || partPaths.length === 0) {
+ return null;
+ }
+ const partUrls: string[] = [];
+ for (const partPath of partPaths) {
+ const url = this.resolveStoreUrl(partPath);
+ if (!url || !(await this.serverSupportsStreamingRanges(url))) {
+ return null;
+ }
+ partUrls.push(url);
+ }
+ const { ParquetFile } = await SpatialDataTableSource.parquetModulePromise;
+ if (!ParquetFile) {
+ return null;
+ }
+
+ const { axisNames, featureKey, maxRows } = options;
+ const axisCount = axisNames.length;
+ const { tableFromIPC } = await import('apache-arrow');
+ const { featureCatalogFromCodeMap } = await import('../pointsFeatures.js');
+
+ // Preallocate once and append at a cursor; partials expose the filled prefix as
+ // subarray VIEWS so emitting progress stays O(1).
+ const axisBuffers = Array.from({ length: axisCount }, () => new Float32Array(maxRows));
+ const codeBuffer = new Int32Array(maxRows);
+ const codeToName = new Map();
+ const nameToCode = new Map();
+ const codeCounts = new Map();
+ let filled = 0;
+
+ const snapshot = (): PointsLoadResult => ({
+ shape: [axisCount, filled] as [number, number],
+ data: axisBuffers.map((buffer) => buffer.subarray(0, filled)),
+ totalRowCount: options.totalRowCount,
+ preloadTruncated: options.preloadTruncated,
+ hasFeatureCodeColumn: options.hasFeatureCodeColumn,
+ featureCodes: codeBuffer.subarray(0, filled),
+ featureCatalog: featureCatalogFromCodeMap(featureKey, codeToName),
+ featureCodeCounts: new Map(codeCounts),
+ });
+
+ for (const [partIndex, url] of partUrls.entries()) {
+ if (filled >= maxRows) {
+ break;
+ }
+ const file = await ParquetFile.fromUrl(url);
+ const stream = await file.stream({
+ columns: [...axisNames, featureKey],
+ batchSize: PRELOAD_STREAM_BATCH_ROWS,
+ });
+ const reader = stream.getReader();
+ try {
+ for (;;) {
+ checkAbort(options.signal); // superseded → stop before the next batch
+ if (filled >= maxRows) {
+ break;
+ }
+ const { done, value } = await reader.read();
+ if (done) {
+ break;
+ }
+ const table = tableFromIPC(value.intoIPCStream());
+ const rows = Math.min(table.numRows, maxRows - filled);
+ if (rows <= 0) {
+ continue;
+ }
+ for (let axis = 0; axis < axisCount; axis += 1) {
+ const column = table.getChild(axisNames[axis]);
+ if (!column) {
+ continue;
+ }
+ const values = column.toArray() as ArrayLike;
+ for (let row = 0; row < rows; row += 1) {
+ axisBuffers[axis][filled + row] = values[row];
+ }
+ }
+ const featureColumn = table.getChild(featureKey);
+ if (!featureColumn) {
+ // No feature column in the projection means this path cannot colour
+ // anything; hand back what we have and let the caller settle codes.
+ return filled > 0 ? snapshot() : null;
+ }
+ appendFeatureCodesFromColumn(
+ featureColumn,
+ rows,
+ codeToName,
+ nameToCode,
+ codeBuffer,
+ codeCounts,
+ filled
+ );
+ filled += rows;
+ options.onProgress?.({
+ // Unfiltered preload: every decoded row is kept, so scanned === matched.
+ scannedRows: filled,
+ matchedRows: filled,
+ partIndex,
+ partCount: partUrls.length,
+ partialResult: snapshot(),
+ });
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ }
+ return filled > 0 ? snapshot() : null;
+ }
+
+ private async streamPointsGeometryByRowGroup(
+ parquetPath: string,
+ options: {
+ axisNames: string[];
+ columns: string[];
+ maxRows: number;
+ totalRowCount: number;
+ preloadTruncated: boolean;
+ /** Required alongside {@link featureCodeColumnName} for the decode to emit
+ * per-row codes at all — the worker gates code extraction on `featureKey`. */
+ featureKey?: string;
+ featureCodeColumnName?: string;
+ onProgress?: (progress: PointsLoadProgress) => void;
+ signal?: AbortSignal;
+ }
+ ): Promise {
+ const dataset = await this.loadParquetDatasetMetadata(parquetPath);
+ if (!dataset || dataset.totalNumRowGroups <= 0) {
+ return null;
+ }
+ const { axisNames, maxRows, featureCodeColumnName } = options;
+ const axisCount = axisNames.length;
+ const axisBuffers = Array.from({ length: axisCount }, () => new Float32Array(maxRows));
+ const codeBuffer = featureCodeColumnName ? new Int32Array(maxRows) : undefined;
+ let filled = 0;
+ // Goes false the moment any chunk fails to supply one code PER ROW. Observed in
+ // practice: a per-row-group read of the code column can come back with just the
+ // column's distinct values (e.g. 4 codes for a 100k-row group in a feature-sorted
+ // file), because the row-group path mis-handles dictionary encoding — the same
+ // constraint that keeps `feature_name` off this path. Once false the stream
+ // publishes NO codes, so the caller falls through to the one-shot decode.
+ let codesComplete = codeBuffer !== undefined;
+ // Running per-feature tally. Free in I/O terms — the codes are already decoded —
+ // and O(rows) once overall, so a panel can show per-feature stats long before the
+ // whole-dataset counts scan finishes. Counts cover the streamed prefix only.
+ const codeCounts = new Map();
+
+ // Views over the filled prefix — no copy, so emitting a partial is O(1). The
+ // tally is passed by reference and keeps growing; consumers read it per tick.
+ const snapshot = (): PointsLoadResult => ({
+ shape: [axisCount, filled] as [number, number],
+ data: axisBuffers.map((buffer) => buffer.subarray(0, filled)),
+ totalRowCount: options.totalRowCount,
+ preloadTruncated: options.preloadTruncated,
+ hasFeatureCodeColumn: featureCodeColumnName !== undefined,
+ ...(codeBuffer && codesComplete
+ ? { featureCodes: codeBuffer.subarray(0, filled), featureCodeCounts: new Map(codeCounts) }
+ : {}),
+ });
+
+ for (let rowGroupIndex = 0; rowGroupIndex < dataset.totalNumRowGroups; rowGroupIndex += 1) {
+ checkAbort(options.signal); // superseded → stop before the next range read
+ if (filled >= maxRows) {
+ break;
+ }
+ const chunk = await this.readParquetRowGroupBytesByGroupIndex(parquetPath, rowGroupIndex);
+ if (!chunk) {
+ return filled > 0 ? snapshot() : null;
+ }
+ const decoded = await decodeParquetGeometryCappedInWorker({
+ rowGroups: [chunk],
+ axisNames,
+ columns: options.columns,
+ maxRows: maxRows - filled,
+ // BOTH are required: the worker gates code extraction on `featureKey`, and
+ // `resolveRowFeatureCodesFromTable` then returns the code column directly —
+ // it never touches the (unprojected, dict-encoded) name column. Passing only
+ // `featureCodeColumnName` silently yields NO codes, which is a colourless
+ // element rather than a loud failure.
+ ...(featureCodeColumnName && options.featureKey
+ ? { featureCodeColumnName, featureKey: options.featureKey }
+ : {}),
+ });
+ if (!decoded) {
+ // Worker unavailable: with nothing decoded yet the caller can still take the
+ // one-shot path; mid-stream we keep what we have rather than discard it.
+ return filled > 0 ? snapshot() : null;
+ }
+ const decodedRows = decoded.shape[1] ?? decoded.data[0]?.length ?? 0;
+ const rows = Math.min(decodedRows, maxRows - filled);
+ if (rows <= 0) {
+ continue;
+ }
+ for (let axis = 0; axis < axisCount; axis += 1) {
+ const column = decoded.data[axis];
+ if (!column) {
+ continue;
+ }
+ const values =
+ column instanceof Float32Array
+ ? column.subarray(0, rows)
+ : Float32Array.from(Array.prototype.slice.call(column, 0, rows));
+ axisBuffers[axis].set(values, filled);
+ }
+ if (codeBuffer) {
+ // A SHORT codes array is unusable, not partially usable: writing it would
+ // leave the remaining rows at 0 — a VALID feature code — so those points
+ // would be confidently mis-coloured rather than left uncoloured. Demand one
+ // code per row or discard codes for the whole stream.
+ const chunkCodes = decoded.featureCodes;
+ if (chunkCodes && chunkCodes.length >= rows) {
+ codeBuffer.set(chunkCodes.subarray(0, rows), filled);
+ // Tally this chunk while its codes are hot, rather than re-walking the
+ // whole prefix on every progress tick.
+ for (let row = 0; row < rows; row += 1) {
+ const code = chunkCodes[row];
+ codeCounts.set(code, (codeCounts.get(code) ?? 0) + 1);
+ }
+ } else {
+ codesComplete = false;
+ }
+ }
+ filled += rows;
+ options.onProgress?.({
+ // Unfiltered preload: every decoded row is kept, so scanned === matched.
+ scannedRows: filled,
+ matchedRows: filled,
+ partIndex: rowGroupIndex,
+ partCount: dataset.totalNumRowGroups,
+ partialResult: snapshot(),
+ });
+ }
+ return filled > 0 ? snapshot() : null;
+ }
+
async loadPoints(
elementPath: string,
options: PointsLoadOptions = {}
@@ -427,8 +937,76 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
}
}
+ // Preferred progressive preload: stream geometry AND the feature column in the
+ // same batches, so points paint already coloured. Tried before the row-group
+ // path because that one cannot read a dictionary feature column at all — the
+ // case where colour otherwise waits for a whole separate decode. Needs no
+ // worker: the per-batch work is a dictionary lookup and a typed-array copy.
+ if (options.onProgress && featureKey) {
+ try {
+ const streamed = await this.streamPointsWithFeaturesByUrl(parquetPath, {
+ axisNames,
+ featureKey,
+ maxRows,
+ totalRowCount: rowCount,
+ preloadTruncated: truncatePreload,
+ hasFeatureCodeColumn: featureCodeColumnName !== undefined,
+ onProgress: options.onProgress,
+ ...(options.signal ? { signal: options.signal } : {}),
+ });
+ if (streamed?.featureCodes !== undefined) {
+ return streamed;
+ }
+ } catch (error) {
+ if (error instanceof DOMException && error.name === 'AbortError') {
+ throw error;
+ }
+ console.warn(`Streaming points preload failed for ${elementPath}; falling back.`, error);
+ }
+ }
+
ensurePointsWorker();
if (isPointsWorkerEnabled()) {
+ // Progressive preload (D3), when the caller asked for progress and the store
+ // supports row-group range reads. Streams the axes — plus an authoritative
+ // integer code column when the dataset has one, so those datasets stream
+ // COLOURED rather than colour-later.
+ if (options.onProgress && (await this.canLoadParquetRowGroups())) {
+ try {
+ const streamed = await this.streamPointsGeometryByRowGroup(parquetPath, {
+ axisNames,
+ columns: [...axisNames, ...(featureCodeColumnName ? [featureCodeColumnName] : [])],
+ maxRows,
+ totalRowCount: rowCount,
+ preloadTruncated: truncatePreload,
+ ...(featureKey ? { featureKey } : {}),
+ ...(featureCodeColumnName ? { featureCodeColumnName } : {}),
+ onProgress: options.onProgress,
+ ...(options.signal ? { signal: options.signal } : {}),
+ });
+ // The streamed batch is the FINAL result only when nothing more is needed
+ // from the dictionary column: either the element has no feature key at all,
+ // or an authoritative code column ACTUALLY produced per-row codes. Checking
+ // `streamed.featureCodes` rather than merely "a code column exists" is
+ // deliberate: if the codes ever fail to come back, we degrade to the slower
+ // one-shot decode (correct, just not streamed) instead of settling a
+ // permanently colourless batch — the failure mode this guard exists for.
+ // A dict-only element always falls through, its early paint already banked.
+ const streamedIsComplete =
+ streamed !== null && (!featureKey || streamed.featureCodes !== undefined);
+ if (streamedIsComplete) {
+ return streamed;
+ }
+ } catch (error) {
+ if (error instanceof DOMException && error.name === 'AbortError') {
+ throw error;
+ }
+ console.warn(
+ `Progressive points preload failed for ${elementPath}; falling back to a single decode.`,
+ error
+ );
+ }
+ }
try {
if (featureKey) {
// Off-thread the codes-with-geometry decode: fetch whole row-group (or
@@ -565,6 +1143,244 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
return parts.length > 0 ? { parts } : null;
}
+ /**
+ * Whether the URL-streaming scan applies: a URL-backed store whose server
+ * actually serves the range shapes the reader needs, for every part.
+ */
+ private async canStreamMatchingScan(
+ parquetPath: string
+ ): Promise<{ urls: string[]; rowGroupCounts: number[] } | null> {
+ if (!(await this.canStreamParquetByUrl())) {
+ return null;
+ }
+ const dataset = await this.loadParquetDatasetMetadata(parquetPath);
+ const partPaths = dataset?.parts.map((part) => part.path);
+ if (!partPaths || partPaths.length === 0) {
+ return null;
+ }
+ const urls: string[] = [];
+ for (const partPath of partPaths) {
+ const url = this.resolveStoreUrl(partPath);
+ if (!url || !(await this.serverSupportsStreamingRanges(url))) {
+ return null;
+ }
+ urls.push(url);
+ }
+ return { urls, rowGroupCounts: dataset?.numRowGroupsByPart ?? [] };
+ }
+
+ /**
+ * Feature scan over `ParquetFile.stream({ columns, rowGroups })`.
+ *
+ * The row-group path this replaces range-reads `rowGroup.fileOffset()` for
+ * `compressedSize()` — the WHOLE row group, every column — because parquet-wasm
+ * cannot fetch individual column chunks (docs/parquet-wasm-limitations.md);
+ * column projection only happens later, at decode time, once the bytes are
+ * already down the wire. On a Xenium `transcripts` element that means fetching
+ * and decompressing all 12 columns (cell_id, transcript_id, fov_name, qv, …) for
+ * every one of 12.1M rows to use three of them: x, y and the feature column.
+ * Profiling a single-gene selection put ~30% of the wall clock in those reads and
+ * ~50% in the WASM decode of them.
+ *
+ * `stream()` issues its own ranged fetches per COLUMN CHUNK, so the projection
+ * reaches the network. This is the same reader the preload already uses
+ * ({@link streamPointsWithFeaturesByUrl}); the scan differs only in filtering
+ * each batch and yielding progress.
+ *
+ * Decoding moves to this thread rather than the worker — `stream()` is
+ * browser-main-thread only today (`supportsParquetStreaming` requires `window`).
+ * That is tolerable because the work arrives in `PRELOAD_STREAM_BATCH_ROWS`
+ * batches, so it is many short tasks rather than one long one; the byte-oriented
+ * worker path remains for stores this cannot serve.
+ */
+ private async *streamMatchingFeatureCodesByChunk(
+ partUrls: string[],
+ rowGroupCounts: number[],
+ options: {
+ axisNames: string[];
+ axisCount: number;
+ featureKey: string;
+ featureCodeColumnName?: string;
+ featureCodes: readonly number[];
+ featureCodeByName?: ReadonlyMap;
+ columnNames: string[];
+ memoryCap: number;
+ totalRowCount: number;
+ abort?: AbortSignal;
+ }
+ ) {
+ const { ParquetFile } = await SpatialDataTableSource.parquetModulePromise;
+ if (!ParquetFile) {
+ // `canStreamMatchingScan` already checked for this; belt and braces, since
+ // silently yielding nothing here would look like "the gene has no points".
+ throw new Error('ParquetFile.stream is unavailable for the feature scan.');
+ }
+ const { tableFromIPC } = await import('apache-arrow');
+ const { Float32PointBuffer, Int32PointBuffer, scanTableByFeatureCodes } = await import(
+ '../workers/pointsWorkerScan.js'
+ );
+
+ let matchedRows = 0;
+ let scannedRows = 0;
+ const accumulatedChunks: ColumnarPointsChunk[] = [];
+ // Batches are small (65k rows), so flushing one chunk per batch would push
+ // hundreds of chunks through `pointsScanChunkProgress`, whose re-concat is
+ // quadratic in CHUNK COUNT — fine for the ~9 the row-group path produces, not
+ // for hundreds. Accumulate across batches and flush on this stride instead.
+ const FLUSH_SCANNED_ROWS = 1_000_000;
+ let xs = new Float32PointBuffer();
+ let ys = new Float32PointBuffer();
+ let zs = new Float32PointBuffer();
+ let codes = new Int32PointBuffer();
+ let pendingMatched = 0;
+ let scannedAtLastFlush = 0;
+
+ const hasZ = options.axisNames.includes('z');
+ const flush = (partIndex: number) => {
+ const data: ArrayLike[] = hasZ
+ ? [xs.toArray(), ys.toArray(), zs.toArray()]
+ : [xs.toArray(), ys.toArray()];
+ const chunk = toColumnarPointsChunk(
+ { shape: [options.axisCount, pendingMatched], data, featureCodes: codes.toArray() },
+ options.axisCount
+ );
+ accumulatedChunks.push(chunk);
+ xs = new Float32PointBuffer();
+ ys = new Float32PointBuffer();
+ zs = new Float32PointBuffer();
+ codes = new Int32PointBuffer();
+ pendingMatched = 0;
+ scannedAtLastFlush = scannedRows;
+ return pointsScanChunkProgress(accumulatedChunks, chunk, {
+ scannedRows,
+ matchedRows,
+ totalRowCount: options.totalRowCount,
+ memoryCap: options.memoryCap,
+ partIndex,
+ partCount: partUrls.length,
+ });
+ };
+
+ // Preferred: stream IN THE WORKER, so the decode stays off the main thread.
+ // One request per row-group window keeps progress granular without the worker
+ // protocol needing streamed responses — each response is simply a chunk.
+ if (isPointsWorkerEnabled() && rowGroupCounts.length === partUrls.length) {
+ const featureCodeEntries = options.featureCodeByName
+ ? [...options.featureCodeByName].map(([name, code]) => ({ name, code }))
+ : undefined;
+ const ROW_GROUPS_PER_REQUEST = 2;
+ for (const [partIndex, url] of partUrls.entries()) {
+ const groupCount = rowGroupCounts[partIndex] ?? 0;
+ for (let start = 0; start < groupCount; start += ROW_GROUPS_PER_REQUEST) {
+ checkAbort(options.abort); // superseded → stop before the next request
+ if (matchedRows >= options.memoryCap) {
+ break;
+ }
+ const window: number[] = [];
+ for (let g = start; g < Math.min(start + ROW_GROUPS_PER_REQUEST, groupCount); g += 1) {
+ window.push(g);
+ }
+ const partial = await scanParquetByFeatureCodesInWorker({
+ streamUrl: url,
+ streamRowGroups: window,
+ streamColumns: options.columnNames,
+ axisNames: options.axisNames,
+ featureKey: options.featureKey,
+ ...(options.featureCodeColumnName
+ ? { featureCodeColumnName: options.featureCodeColumnName }
+ : {}),
+ featureCodes: options.featureCodes,
+ memoryCap: options.memoryCap - matchedRows,
+ ...(featureCodeEntries ? { featureCodeEntries } : {}),
+ });
+ if (!partial) {
+ // Worker went away mid-scan: keep what matched rather than discard it.
+ break;
+ }
+ scannedRows += partial.scannedRows;
+ if (partial.matchedRows > 0) {
+ matchedRows += partial.matchedRows;
+ const chunk = toColumnarPointsChunk(partial.data, options.axisCount);
+ accumulatedChunks.push(chunk);
+ yield pointsScanChunkProgress(accumulatedChunks, chunk, {
+ scannedRows,
+ matchedRows,
+ totalRowCount: options.totalRowCount,
+ memoryCap: options.memoryCap,
+ partIndex,
+ partCount: partUrls.length,
+ });
+ }
+ }
+ }
+ return {
+ totalRowCount: options.totalRowCount,
+ axisNames: options.axisNames,
+ scannedRows,
+ matchedRows,
+ };
+ }
+
+ for (const [partIndex, url] of partUrls.entries()) {
+ if (matchedRows >= options.memoryCap) {
+ break;
+ }
+ const file = await ParquetFile.fromUrl(url);
+ const stream = await file.stream({
+ columns: options.columnNames,
+ batchSize: PRELOAD_STREAM_BATCH_ROWS,
+ });
+ const reader = stream.getReader();
+ try {
+ for (;;) {
+ checkAbort(options.abort); // superseded → stop before the next batch
+ if (matchedRows >= options.memoryCap) {
+ break;
+ }
+ const { done, value } = await reader.read();
+ if (done) {
+ break;
+ }
+ const table = tableFromIPC(value.intoIPCStream());
+ scannedRows += table.numRows;
+ const before = matchedRows;
+ matchedRows = scanTableByFeatureCodes({
+ table,
+ axisNames: options.axisNames,
+ featureKey: options.featureKey,
+ ...(options.featureCodeColumnName
+ ? { featureCodeColumnName: options.featureCodeColumnName }
+ : {}),
+ featureCodes: options.featureCodes,
+ memoryCap: options.memoryCap,
+ matchedRows,
+ xs,
+ ys,
+ zs,
+ codes,
+ ...(options.featureCodeByName ? { featureCodeByName: options.featureCodeByName } : {}),
+ });
+ pendingMatched += matchedRows - before;
+ if (pendingMatched > 0 && scannedRows - scannedAtLastFlush >= FLUSH_SCANNED_ROWS) {
+ yield flush(partIndex);
+ }
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ if (pendingMatched > 0) {
+ yield flush(partIndex); // part boundary: don't carry matches across files
+ }
+ }
+
+ return {
+ totalRowCount: options.totalRowCount,
+ axisNames: options.axisNames,
+ scannedRows,
+ matchedRows,
+ };
+ }
+
async *loadPointsMatchingFeatureCodesByChunk(
elementPath: string,
options: {
@@ -581,9 +1397,9 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
}
) {
ensurePointsWorker();
+ checkAbort(options.abort);
const parquetPath = getParquetPath(elementPath);
const zattrs = await this.loadSpatialDataElementAttrs(elementPath);
- // if (options.abort?.aborted) return;
const { axes, spatialdata_attrs: spatialDataAttrs } = zattrs;
const normAxes = normalizeAxes(axes);
const axisNames = normAxes.map((axis: { name: string }) => axis.name);
@@ -631,6 +1447,30 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
? [...options.featureCodeByName].map(([name, code]) => ({ name, code }))
: undefined;
+ // Preferred path: let the reader fetch only the projected columns (see
+ // `streamMatchingFeatureCodesByChunk`). Falls through to the byte-oriented
+ // worker path below for stores it cannot serve — non-URL stores, and servers
+ // that do not answer the range shapes the reader needs.
+ const streamablePartUrls = await this.canStreamMatchingScan(parquetPath);
+ if (streamablePartUrls) {
+ return yield* this.streamMatchingFeatureCodesByChunk(
+ streamablePartUrls.urls,
+ streamablePartUrls.rowGroupCounts,
+ {
+ axisNames,
+ axisCount,
+ featureKey,
+ ...(featureCodeColumnName ? { featureCodeColumnName } : {}),
+ featureCodes: options.featureCodes,
+ ...(options.featureCodeByName ? { featureCodeByName: options.featureCodeByName } : {}),
+ columnNames,
+ memoryCap: options.memoryCap,
+ totalRowCount,
+ ...(options.abort ? { abort: options.abort } : {}),
+ }
+ );
+ }
+
let matchedRows = 0;
let scannedRows = 0;
// Growing buffer of every matched chunk so far — `pointsScanChunkProgress`
@@ -665,6 +1505,10 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
const canSkipRowGroups = rowGroupExtents.length === datasetRowGroups;
for (let rowGroupIndex = 0; rowGroupIndex < datasetRowGroups; rowGroupIndex += 1) {
+ // A superseded scan aborts here, between row groups — the at-most-one-chunk
+ // bound on wasted decode. Throws AbortError, which the resolver's slot reads
+ // as a non-event.
+ checkAbort(options.abort);
if (matchedRows >= options.memoryCap) {
break;
}
@@ -678,9 +1522,12 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
if (!chunk) {
continue;
}
- // the memoryCap could work by the consumer choosing not to exhaust the stream
- // (although that wouldn't help to pass last worker invocation a smaller chunk size)
- // we should be passing abort to worker
+ // Cancellation is enforced between chunks (checkAbort above), not inside the
+ // worker: each worker call decodes ONE row group — a single, uninterruptible
+ // WASM decode — so an abort can at most skip the NEXT chunk, which is what the
+ // loop-top check does. There is no queue of pending worker requests to drain
+ // (chunks are awaited serially), so a worker-side cancel message would buy
+ // nothing here; revisit only if one request ever spans many row groups.
const partial = await scanParquetByFeatureCodesInWorker({
rowGroups: [chunk],
axisNames,
@@ -717,6 +1564,7 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
}
for (let partIndex = 0; partIndex < partPaths.length; partIndex += 1) {
+ checkAbort(options.abort); // superseded → stop before the next part's decode
const partPath = partPaths[partIndex];
if (matchedRows >= options.memoryCap) {
break;
@@ -766,15 +1614,21 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
* code space the selection was made in. When absent for a dict-only
* element the scan cannot match by name and returns nothing. */
featureCodeByName?: ReadonlyMap;
+ /** Aborts the scan between row-group chunks when it is superseded. */
+ signal?: AbortSignal;
}
): Promise {
- const chunkGenerator = this.loadPointsMatchingFeatureCodesByChunk(elementPath, options);
+ const chunkGenerator = this.loadPointsMatchingFeatureCodesByChunk(elementPath, {
+ ...options,
+ abort: options.signal,
+ });
// Each `progress.partialResult` is already the full accumulated buffer, so the
// last one IS the whole matched batch — no need to re-accumulate/concat here.
// Final totals come from the generator's return value (authoritative: it also
// counts rows scanned after the last match, which the last partial can't see).
let latest: PointsLoadResult | undefined;
while (true) {
+ checkAbort(options.signal);
const next = await chunkGenerator.next();
if (next.done) {
const { totalRowCount, scannedRows, matchedRows, axisNames } = next.value;
@@ -904,11 +1758,39 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
return countFeatureCodesHistogram(rowCodes);
}
- async listPointsFeaturesWithCounts(elementPath: string): Promise {
- const catalog = await this.listPointsFeatures(elementPath);
+ /**
+ * The authoritative feature catalog, in two steps: the NAME/CODE list (cheap) and
+ * then per-feature counts (a scan of every row group — the slow part).
+ *
+ * `onPartialCatalog` is called with the names-only catalog as soon as it is known,
+ * before the counts scan starts. That is what lets the feature panel list features
+ * immediately instead of showing "Loading features…" for the whole scan: the names
+ * are what the list, swatches and selection need, and only the count column has to
+ * wait.
+ *
+ * When the streaming scan applies it fires repeatedly *during* the name scan too,
+ * each time with a longer list, so the panel fills in rather than appearing at
+ * once. Codes are stable across those partials (see
+ * `listPointsFeaturesByStreamingScan`), so earlier entries never move.
+ */
+ async listPointsFeaturesWithCounts(
+ elementPath: string,
+ options?: { onPartialCatalog?: (catalog: PointsFeatureCatalog) => void }
+ ): Promise {
+ const catalog = await this.listPointsFeatures(elementPath, {
+ onPartialCatalog: options?.onPartialCatalog,
+ });
if (!catalog) {
return null;
}
+ options?.onPartialCatalog?.(catalog);
+ // The streaming scan counts as it goes, so a catalog can arrive complete. Don't
+ // then run the counts scan: for a dictionary-only element it would return an
+ // empty map (it needs an integer code column) and merging that back would drop
+ // the counts we already have.
+ if (catalog.entries.some((entry) => entry.count !== undefined)) {
+ return catalog;
+ }
try {
const counts = await this.loadFeatureCounts(elementPath);
return mergeFeatureCountsIntoCatalog(catalog, counts);
@@ -929,8 +1811,10 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
options: {
memoryCap?: number;
featureCatalog?: PointsFeatureCatalog | null;
+ signal?: AbortSignal;
} = {}
): Promise | undefined> {
+ checkAbort(options.signal);
const parquetPath = getParquetPath(elementPath);
const zattrs = await this.loadSpatialDataElementAttrs(elementPath);
const { spatialdata_attrs: spatialDataAttrs } = zattrs;
@@ -1013,7 +1897,10 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
return this.resolveParquetRowCount(parquetPath);
}
- async listPointsFeatures(elementPath: string): Promise {
+ async listPointsFeatures(
+ elementPath: string,
+ options?: { onPartialCatalog?: (catalog: PointsFeatureCatalog) => void }
+ ): Promise {
const zattrs = await this.loadSpatialDataElementAttrs(elementPath);
const featureKey = zattrs.spatialdata_attrs?.feature_key;
if (typeof featureKey !== 'string' || featureKey.length === 0) {
@@ -1046,7 +1933,8 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
parquetPath,
featureKey,
featureCodeColumnName,
- hasMortonColumn
+ hasMortonColumn,
+ options?.onPartialCatalog
);
}
@@ -1059,13 +1947,177 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
return null;
}
- return buildFeatureCatalogFromColumns(
+ const catalog = buildFeatureCatalogFromColumns(
featureKey,
nameColumn,
codeColumn,
mortonColumn,
arrowTable.numRows
);
+ // Count here too, for the same reason the streaming scan does: a
+ // dictionary-only element gets nothing from `loadFeatureCounts`, and this
+ // whole-table read has already decoded every row, so the counts are exact and
+ // free. Without this, an under-the-cap element like a 3.7M-row MERFISH
+ // `single_molecule` showed a feature list whose stats never arrived.
+ const counts = new Map();
+ tallyFeatureCodesFromColumn(
+ nameColumn,
+ arrowTable.numRows,
+ // `?? new Map()` is not dead: `featureCodeMapFromCatalog` is typed
+ // `Map | undefined` for its null-catalog callers, and nothing narrows it here.
+ featureCodeMapFromCatalog(catalog) ?? new Map(),
+ counts,
+ mortonColumn
+ );
+ return mergeFeatureCountsIntoCatalog(catalog, counts);
+ }
+
+ /**
+ * Build the feature catalog by streaming ONLY the feature column(s) over range
+ * reads, publishing the catalog as it grows.
+ *
+ * This is the fast path for the oversized-dataset scan. The feature column is a
+ * tiny fraction of a points parquet (~4KB of an 8.8MB Xenium-style file, the
+ * rest being geometry and an unused dask index), so projecting it turns a
+ * whole-file download into a handful of range requests.
+ *
+ * Codes stay compatible with every other catalog build: both assign codes in
+ * first-seen row order, so a streamed prefix agrees with the whole-file scan on
+ * every code it has assigned so far. That is what makes the partial catalogs
+ * safe to render — a feature's code never changes as more rows arrive, so
+ * selections and swatches made against a partial stay valid.
+ *
+ * Returns null (rather than throwing) whenever the fast path does not apply, so
+ * the caller falls through to the byte-oriented scan.
+ */
+ private async listPointsFeaturesByStreamingScan(
+ parquetPath: string,
+ featureKey: string,
+ featureCodeColumnName: string | undefined,
+ hasMortonColumn: boolean,
+ columnNames: string[],
+ onPartialCatalog?: (catalog: PointsFeatureCatalog) => void
+ ): Promise {
+ if (!(await this.canStreamParquetByUrl())) {
+ return null;
+ }
+ // Use the discovered parts rather than guessing paths: these are known to
+ // exist and are in row order, which the code assignment depends on.
+ const datasetMetadata = await this.loadParquetDatasetMetadata(parquetPath);
+ const partPaths = datasetMetadata?.parts.map((part) => part.path);
+ if (!partPaths || partPaths.length === 0) {
+ return null;
+ }
+ const partUrls: string[] = [];
+ for (const partPath of partPaths) {
+ const url = this.resolveStoreUrl(partPath);
+ if (!url) {
+ return null;
+ }
+ partUrls.push(url);
+ }
+
+ // Decline before the reader ever sees the URL: a server that refuses one of
+ // the range shapes it needs makes it panic unrecoverably rather than error.
+ if (!(await this.serverSupportsStreamingRanges(partUrls[0]))) {
+ return null;
+ }
+
+ const { ParquetFile } = await SpatialDataTableSource.parquetModulePromise;
+ if (!ParquetFile) {
+ return null;
+ }
+
+ const { tableFromIPC } = await import('apache-arrow');
+ const {
+ accumulateFeatureCatalogFromTable,
+ featureCatalogFromCodeMap,
+ featureCatalogNeedsParquetFallback,
+ } = await import('../pointsFeatures.js');
+ const codeToName = new Map();
+ const nameToCode = new Map();
+ // Tally per feature while the column is already decoded. This is the ONLY place
+ // a dictionary-only element can get counts: `loadFeatureCounts` needs an integer
+ // code column and returns an empty map without one, which is why such elements
+ // showed feature stats that never settled. Costs no extra I/O — the rows are in
+ // hand — and covers the whole dataset because this scan reads every row.
+ const counts = new Map();
+ let publishedFeatureCount = 0;
+
+ // The watchdog spans opening AND draining every part: a failed range request
+ // can panic the reader at any of those points, and the panic never settles
+ // the promise it came from.
+ const stall = createStallGuard(FEATURE_STREAM_STALL_TIMEOUT_MS);
+ const scanAllParts = async () => {
+ for (const url of partUrls) {
+ const file = await ParquetFile.fromUrl(url);
+ const stream = await file.stream({
+ columns: columnNames,
+ batchSize: FEATURE_STREAM_BATCH_ROWS,
+ });
+ const reader = stream.getReader();
+ try {
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) {
+ break;
+ }
+ stall.progress();
+ const batch = tableFromIPC(value.intoIPCStream());
+ accumulateFeatureCatalogFromTable(
+ codeToName,
+ nameToCode,
+ batch,
+ featureKey,
+ featureCodeColumnName,
+ { skipMortonSentinels: hasMortonColumn }
+ );
+ // After the accumulate above, every name in this batch has a code.
+ const featureColumn = batch.getChild(featureKey);
+ if (featureColumn) {
+ tallyFeatureCodesFromColumn(
+ featureColumn,
+ batch.numRows,
+ nameToCode,
+ counts,
+ // Same flag the accumulate above uses, so counts and entries agree.
+ hasMortonColumn ? batch.getChild(MORTON_CODE_2D_COLUMN) : null
+ );
+ }
+ // Only republish when the list actually grew; most batches add
+ // nothing once the common features have been seen.
+ if (onPartialCatalog && codeToName.size > publishedFeatureCount) {
+ publishedFeatureCount = codeToName.size;
+ onPartialCatalog(featureCatalogFromCodeMap(featureKey, codeToName));
+ }
+ }
+ } finally {
+ // Best-effort: after a wasm panic the reader may itself be unusable,
+ // and cancel() can hang exactly like read() does — never await it.
+ void reader.cancel().catch(() => {});
+ reader.releaseLock();
+ }
+ }
+ };
+
+ try {
+ await Promise.race([scanAllParts(), stall.promise]);
+ } catch (error) {
+ // Includes the watchdog tripping. Returning null keeps the contract with
+ // the caller: fall through to the byte-oriented scan rather than surface
+ // a half-built catalog.
+ console.warn(`Streaming feature catalog scan abandoned for ${parquetPath}:`, error);
+ return null;
+ } finally {
+ stall.dispose();
+ }
+
+ if (featureCatalogNeedsParquetFallback(codeToName)) {
+ return null;
+ }
+ // Counts ride along: this scan read every row, so they are dataset-wide, and
+ // for a dictionary-only element nothing else can produce them.
+ return mergeFeatureCountsIntoCatalog(featureCatalogFromCodeMap(featureKey, codeToName), counts);
}
/**
@@ -1076,7 +2128,8 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
parquetPath: string,
featureKey: string,
featureCodeColumnName: string | undefined,
- hasMortonColumn: boolean
+ hasMortonColumn: boolean,
+ onPartialCatalog?: (catalog: PointsFeatureCatalog) => void
): Promise {
const columnNames = [featureKey];
if (featureCodeColumnName) {
@@ -1086,13 +2139,41 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
columnNames.push(MORTON_CODE_2D_COLUMN);
}
+ // Fast path first: streaming reads only the feature column and can publish
+ // the list while it scans. Falls through on any non-applicable store/runtime.
+ try {
+ const streamed = await this.listPointsFeaturesByStreamingScan(
+ parquetPath,
+ featureKey,
+ featureCodeColumnName,
+ hasMortonColumn,
+ columnNames,
+ onPartialCatalog
+ );
+ if (streamed) {
+ return streamed;
+ }
+ } catch (error) {
+ console.warn(
+ `Streaming feature catalog scan failed for ${parquetPath}; falling back.`,
+ error
+ );
+ }
+
ensurePointsWorker();
if (isPointsWorkerEnabled()) {
try {
const payload = await this.readParquetWorkerPayload(parquetPath, {
maxRows: Number.POSITIVE_INFINITY,
fullPartsForFallback: true,
- includeRowGroups: true,
+ // Row groups are only usable here with an integer code column — the
+ // dictionary name column cannot be read that way. Asking for them
+ // anyway on a dict-only element fetched every row group AND every
+ // part: the whole dataset downloaded twice to use half of it.
+ includeRowGroups: featureCodeColumnName !== undefined,
+ // Both are handed to the worker: the row-group decode can still come
+ // back unusable, and parts are the fallback.
+ partsAlongsideRowGroups: true,
});
const catalog = await scanParquetFeatureCatalogInWorker({
rowGroups:
@@ -1442,9 +2523,14 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
return null;
}
- const xs: number[] = [];
- const ys: number[] = [];
- const zs: number[] = [];
+ // Dynamic, like the call site below: keeps the worker scan module out of the
+ // eager main-thread bundle. Hoisted above the loop so the buffers can be built.
+ const { Float32PointBuffer, scanMortonTableInBounds } = await import(
+ '../workers/pointsWorkerScan.js'
+ );
+ const xs = new Float32PointBuffer();
+ const ys = new Float32PointBuffer();
+ const zs = new Float32PointBuffer();
const hasZ = metadata.axisNames.includes('z');
const filterByFeature = allowedFeatureCodes !== null;
const featureCodeColumnName =
@@ -1508,7 +2594,6 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
if (!table) {
continue;
}
- const { scanMortonTableInBounds } = await import('../workers/pointsWorkerScan.js');
scanMortonTableInBounds({
table,
rowGroupIndex: rowGroup,
@@ -1528,9 +2613,7 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource {
}
return {
- data: hasZ
- ? [new Float32Array(xs), new Float32Array(ys), new Float32Array(zs)]
- : [new Float32Array(xs), new Float32Array(ys)],
+ data: hasZ ? [xs.toArray(), ys.toArray(), zs.toArray()] : [xs.toArray(), ys.toArray()],
shape: [hasZ ? 3 : 2, xs.length],
bounds: options.bounds,
loadMode: 'row-groups',
diff --git a/packages/core/src/models/VTableSource.ts b/packages/core/src/models/VTableSource.ts
index a0f970dc..efa18201 100644
--- a/packages/core/src/models/VTableSource.ts
+++ b/packages/core/src/models/VTableSource.ts
@@ -6,6 +6,7 @@ import {
type ParquetModule,
type ParquetRowGroupReadOptions,
type ParquetWasmMetadata,
+ supportsParquetStreaming,
} from '../parquetWasmLoader.js';
import type { TableColumnData } from '../types';
import type { DataSourceParams } from '../Vutils';
@@ -184,6 +185,18 @@ export default class SpatialDataTableSource extends AnnDataSource {
* `loadPolygonShapes` all target the same file).
*/
parquetTableCache: Record>;
+ /**
+ * Remembers parquet part layout per path — single file vs. `part.N.parquet`
+ * directory, and the per-part metadata — so the probe sequence (see
+ * {@link loadParquetDatasetMetadata}) runs once, not on every one of the ~20
+ * calls a single points load makes. Only real datasets are held; nulls/failures
+ * are evicted.
+ */
+ parquetDatasetMetadataCache: Map>;
+ /** Part paths discovered by whole-file probing — the transport-independent
+ * fallback used when a store has no range support, so the layout is still
+ * resolved once rather than per call. */
+ parquetPartPathsCache: Map>;
/** Morton min/max per row group — avoids re-decoding row groups during bisect. */
rowGroupColumnExtentCache: Map;
obsIndices: Record>;
@@ -206,6 +219,8 @@ export default class SpatialDataTableSource extends AnnDataSource {
// TODO: change to column-specific storage.
this.parquetTableBytes = {};
this.parquetTableCache = {};
+ this.parquetDatasetMetadataCache = new Map();
+ this.parquetPartPathsCache = new Map();
this.rowGroupColumnExtentCache = new Map();
// Table-specific properties
@@ -280,13 +295,42 @@ export default class SpatialDataTableSource extends AnnDataSource {
* relative to the store root.
* @returns The parquet file bytes.
*/
+ /**
+ * The candidate paths to try for "the parquet file at `parquetPath`", best
+ * first.
+ *
+ * Blind, the order has to be [directory, part.0] — you cannot know which it is
+ * without asking. But once the layout IS known, leading with the directory means
+ * a guaranteed-useless request every time: an HTML listing to be rejected by the
+ * magic check on a static server, a 500 on MDV. A PEEK at the resolved layout
+ * (never resolving it — see {@link loadParquetSchemaBytes}) lets a known
+ * multipart element go straight to its first real part.
+ */
+ private async orderedParquetCandidatePaths(parquetPath: string): Promise {
+ const pending = this.parquetDatasetMetadataCache.get(parquetPath);
+ if (pending) {
+ try {
+ const firstPartPath = (await pending)?.parts[0]?.path;
+ if (firstPartPath && firstPartPath !== parquetPath) {
+ return [firstPartPath];
+ }
+ } catch {
+ // A peek must never be load-bearing. The cached resolution rejects (and
+ // evicts itself) on a transient failure; that means "layout unknown", not
+ // "this path is broken" — fall back to the blind candidate order rather
+ // than throwing out of the caller's probe loop.
+ }
+ }
+ return getParquetCandidatePaths(parquetPath);
+ }
+
async loadParquetBytes(parquetPath: string) {
if (this.parquetTableBytes[parquetPath]) {
// Return the cached bytes.
return this.parquetTableBytes[parquetPath];
}
- for (const candidatePath of getParquetCandidatePaths(parquetPath)) {
+ for (const candidatePath of await this.orderedParquetCandidatePaths(parquetPath)) {
try {
// Some servers return an HTML directory listing for multipart parquet
// directories, so validate the bytes before caching or parsing them.
@@ -323,9 +367,32 @@ export default class SpatialDataTableSource extends AnnDataSource {
async loadParquetSchemaBytes(parquetPath: string) {
const { store } = this.storeRoot;
if (store.getRange) {
+ // An ALREADY-resolved layout carries each part's footer bytes — exactly what
+ // this returns. Reuse them rather than re-walking the candidate paths: that
+ // walk probes the DIRECTORY path first every time, and this runs on every
+ // schema/column resolution, so it kept re-issuing the directory read (and
+ // MDV's 500) long after the layout was known.
+ //
+ // Deliberately a PEEK, not a call: resolving the layout from here would probe
+ // the whole part sequence, so an element whose footer `readMetadata` cannot
+ // parse (and which therefore has no layout) would pay for both walks instead
+ // of just this one.
+ const pending = this.parquetDatasetMetadataCache.get(parquetPath);
+ if (pending) {
+ try {
+ const schemaBytes = (await pending)?.parts[0]?.schemaBytes;
+ if (schemaBytes) {
+ return schemaBytes;
+ }
+ } catch {
+ // A rejected layout means "unknown", not "unresolvable": fall through to
+ // the candidate walk below rather than failing the whole resolution.
+ }
+ }
+
let lastError: Error | null = null;
- for (const candidatePath of getParquetCandidatePaths(parquetPath)) {
+ for (const candidatePath of await this.orderedParquetCandidatePaths(parquetPath)) {
try {
const footerBytes = await this.loadParquetFooterBytesForPath(candidatePath);
if (footerBytes) return footerBytes;
@@ -435,7 +502,10 @@ export default class SpatialDataTableSource extends AnnDataSource {
}
protected async resolveParquetRowCount(parquetPath: string): Promise {
- // may be better to cache this? we get e.g. a lot of 404 requests for `points.parquet/points.4.parquet`
+ // The cached layout answers this outright on any range-capable store; the
+ // whole-file paths below are the fallback for stores without range support.
+ // (This is where the "lots of 404s for part.N" TODO lived — the enumeration is
+ // now shared and memoized rather than repeated per call.)
const datasetMetadata = await this.loadParquetDatasetMetadata(parquetPath);
if (datasetMetadata?.totalNumRows) {
return datasetMetadata.totalNumRows;
@@ -446,23 +516,26 @@ export default class SpatialDataTableSource extends AnnDataSource {
return directPart.metadata.fileMetadata().numRows();
}
+ // Same part set the loop here used to rediscover: every probe involved goes
+ // through the magic-checked `loadParquetFileBytesAtPath`, so sharing the
+ // memoized discovery finds exactly what enumerating inline did.
+ const partPaths = await this.discoverMultipartPartPaths(parquetPath);
let totalRows = 0;
let foundPart = false;
- for (let partIndex = 0; ; partIndex += 1) {
- const partPath = `${parquetPath}/part.${partIndex}.parquet`;
+ for (const partPath of partPaths) {
const part = await this.loadParquetPartMetadataFromFullFile(partPath);
if (part) {
foundPart = true;
totalRows += part.metadata.fileMetadata().numRows();
continue;
}
+ // A part whose footer will not parse still contributes rows; fall back to
+ // counting a single column.
const columnCount = await this.countRowsFromFullParquetFile(partPath);
if (columnCount > 0) {
foundPart = true;
totalRows += columnCount;
- continue;
}
- break;
}
if (foundPart) {
return totalRows;
@@ -489,20 +562,103 @@ export default class SpatialDataTableSource extends AnnDataSource {
};
}
- async loadParquetDatasetMetadata(parquetPath: string): Promise {
+ /**
+ * Probe one path as a single parquet file, resolving to `null` — never
+ * throwing — when it is not one.
+ *
+ * `parquetPath` for a points/shapes element is a DIRECTORY of `part.N.parquet`
+ * files as often as it is a single file, and servers disagree on how they
+ * answer a range read of a directory: a static server 404s (the store maps that
+ * to `undefined` → `null`), but MDV's Flask returns **500** `[Errno 21] Is a
+ * directory`, S3 can 403, and some return a `200` body that fails the parquet
+ * magic check. The zarrita store throws on every non-2xx that is not 404, so
+ * without this guard a single misbehaving directory response escaped all the way
+ * out of {@link loadParquetDatasetMetadata} — the direct probe threw before the
+ * `part.N` enumeration ever ran, and the element wedged. Every one of those
+ * responses means the same thing here: "not a single parquet file at this path",
+ * so enumerate parts instead. This mirrors the already-tolerant
+ * {@link loadParquetFileBytesAtPath} and {@link loadParquetSchemaBytes}.
+ */
+ private async probeParquetPartMetadata(path: string): Promise {
+ try {
+ return await this.loadParquetPartMetadata(path);
+ } catch {
+ return null;
+ }
+ }
+
+ /**
+ * Resolve whether `parquetPath` is a single parquet file or a directory of
+ * `part.N.parquet` files, plus the per-part metadata — remembering the answer.
+ *
+ * Part discovery costs real requests every time it runs: a probe of the
+ * directory path itself (which servers answer inconsistently — a 404, or the
+ * MDV 500, see {@link probeParquetPartMetadata}), plus one 404 past the final
+ * part to find the end of the sequence. For a read-only store the layout never
+ * changes, yet a single points load calls this ~20 times (row counts, tiling,
+ * row-group extents, the streaming reader, …), so uncached it repeated that
+ * whole probe sequence over and over — the trailing 404s and repeated directory
+ * 500s visible in the network tab.
+ *
+ * The promise is cached so concurrent callers share one probe, but only a real
+ * dataset is remembered: a `null` resolution or a rejection is evicted. Now that
+ * {@link probeParquetPartMetadata} turns a failed probe into `null` rather than a
+ * throw, a transient all-probes-failed must not be allowed to stick and
+ * permanently mark a real dataset as absent — and a genuine not-a-dataset path
+ * is cheap to re-probe.
+ */
+ loadParquetDatasetMetadata(parquetPath: string): Promise {
+ const cached = this.parquetDatasetMetadataCache.get(parquetPath);
+ if (cached) {
+ return cached;
+ }
+ const promise = this.loadParquetDatasetMetadataUncached(parquetPath).then(
+ (result) => {
+ if (result === null) {
+ this.evictIfCurrent(this.parquetDatasetMetadataCache, parquetPath, promise);
+ }
+ return result;
+ },
+ (error) => {
+ this.evictIfCurrent(this.parquetDatasetMetadataCache, parquetPath, promise);
+ throw error;
+ }
+ );
+ this.parquetDatasetMetadataCache.set(parquetPath, promise);
+ return promise;
+ }
+
+ /** Drop a cache entry only if it still holds `promise` — a later call that
+ * superseded it (e.g. after an eviction) must not be clobbered by an earlier
+ * promise's late resolution. */
+ private evictIfCurrent(
+ cache: Map>,
+ key: string,
+ promise: Promise
+ ): void {
+ if (cache.get(key) === promise) {
+ cache.delete(key);
+ }
+ }
+
+ private async loadParquetDatasetMetadataUncached(
+ parquetPath: string
+ ): Promise {
const { readMetadata } = await SpatialDataTableSource.parquetModulePromise;
const { store } = this.storeRoot;
if (!readMetadata || !store.getRange) {
return null;
}
- const directPart = await this.loadParquetPartMetadata(parquetPath);
+ const directPart = await this.probeParquetPartMetadata(parquetPath);
const parts: ParquetPartMetadata[] = [];
if (directPart) {
parts.push(directPart);
} else {
for (let partIndex = 0; ; partIndex++) {
- const part = await this.loadParquetPartMetadata(`${parquetPath}/part.${partIndex}.parquet`);
+ const part = await this.probeParquetPartMetadata(
+ `${parquetPath}/part.${partIndex}.parquet`
+ );
if (!part) {
break;
}
@@ -543,6 +699,117 @@ export default class SpatialDataTableSource extends AnnDataSource {
);
}
+ /**
+ * Absolute http(s) URL for a store-relative path, or null when the store is
+ * not URL-backed.
+ *
+ * The streaming parquet reader fetches on its own rather than through the
+ * store, so it only applies when the store resolves to a plain URL — the same
+ * capability-check shape as `store.getRange`. Custom, prefixed and in-memory
+ * stores return null here and keep the byte-oriented path.
+ */
+ protected resolveStoreUrl(path: string): string | null {
+ const base = (this.storeRoot.store as { url?: string | URL }).url;
+ if (base === undefined || base === null) {
+ return null;
+ }
+ try {
+ const baseHref = typeof base === 'string' ? base : base.href;
+ const rootHref = baseHref.endsWith('/') ? baseHref : `${baseHref}/`;
+ const resolved = new URL(path.replace(/^\/+/, ''), rootHref);
+ // The reader fetches directly; anything the browser will not range-read
+ // (file:, blob:, custom schemes) has to fall back.
+ return resolved.protocol === 'http:' || resolved.protocol === 'https:' ? resolved.href : null;
+ } catch {
+ return null;
+ }
+ }
+
+ /** Whether the URL-backed streaming reader is usable for this store+runtime. */
+ protected async canStreamParquetByUrl(): Promise {
+ if (!supportsParquetStreaming()) {
+ return false;
+ }
+ const module = await SpatialDataTableSource.parquetModulePromise;
+ return typeof module.ParquetFile?.fromUrl === 'function';
+ }
+
+ /**
+ * Verify a server actually serves the range shapes the streaming reader needs,
+ * before handing it the URL.
+ *
+ * The reader fetches on its own and treats a refused range as unreachable: it
+ * panics with `RuntimeError: unreachable` AND leaves its promise unsettled, so
+ * the failure can be neither caught nor awaited. Probing first is the only way
+ * to decline cleanly instead of relying on the stall watchdog.
+ *
+ * It needs two shapes (observed by logging a real scan):
+ * - `bytes=-N` suffix ranges, to read the footer
+ * - `bytes=A-B` bounded ranges, to read column chunks
+ *
+ * Suffix ranges are the fragile one — plenty of static servers answer 416.
+ * The rest of this class tolerates that by falling back to whole-file reads
+ * (see `loadParquetFooterBytesForPath`), which is why such a server otherwise
+ * looks healthy.
+ *
+ * Cached per origin: the answer is a property of the server, not the file.
+ */
+ private static readonly rangeProbeByOrigin = new Map>();
+
+ protected serverSupportsStreamingRanges(url: string): Promise {
+ let origin: string;
+ try {
+ origin = new URL(url).origin;
+ } catch {
+ return Promise.resolve(false);
+ }
+ const cached = SpatialDataTableSource.rangeProbeByOrigin.get(origin);
+ if (cached) {
+ return cached;
+ }
+ // A *definitive* answer — the server replied and we read its status — is a
+ // property of the server and caches for the session. A thrown fetch is not
+ // an answer: caching it would let one blip (a dropped connection, a reload
+ // race, a momentarily unreachable server) demote every element on the origin
+ // to the whole-file path for the rest of the page's life, which reads as a
+ // non-deterministic "sometimes points/counts never settle".
+ let definitive = true;
+ const probe = (async () => {
+ try {
+ // `no-store` is essential, not a nicety. These files are served with a
+ // long max-age, so once any whole-file read has populated the HTTP
+ // cache the browser answers suffix ranges itself and a cached probe
+ // reports success for a server that actually returns 416. The reader
+ // then works only until the entry is evicted, and panics after that.
+ // Probe the server so the decision is a property of the server alone.
+ const [suffix, bounded] = await Promise.all([
+ fetch(url, { headers: { Range: 'bytes=-8' }, cache: 'no-store' }),
+ fetch(url, { headers: { Range: 'bytes=0-7' }, cache: 'no-store' }),
+ ]);
+ // A 200 means the server ignored Range and sent the whole body; the
+ // reader would then compute offsets against the wrong window.
+ if (suffix.status !== 206 || bounded.status !== 206) {
+ return false;
+ }
+ const [suffixBytes, boundedBytes] = await Promise.all([
+ suffix.arrayBuffer(),
+ bounded.arrayBuffer(),
+ ]);
+ return suffixBytes.byteLength === 8 && boundedBytes.byteLength === 8;
+ } catch {
+ definitive = false;
+ return false;
+ }
+ })();
+ probe.then(() => {
+ if (!definitive && SpatialDataTableSource.rangeProbeByOrigin.get(origin) === probe) {
+ SpatialDataTableSource.rangeProbeByOrigin.delete(origin);
+ }
+ });
+ SpatialDataTableSource.rangeProbeByOrigin.set(origin, probe);
+ return probe;
+ }
+
/**
* Fetch compressed row-group bytes via range read (no parquet decode on the caller thread).
*/
@@ -581,7 +848,15 @@ export default class SpatialDataTableSource extends AnnDataSource {
return null;
}
return {
- schemaBytes: part.schemaBytes,
+ // A COPY, deliberately. These chunks are posted to the points worker with
+ // their buffers TRANSFERRED, which detaches them here — so handing out the
+ // cached metadata's own footer buffer would detach the cache on first use,
+ // and every later row group would post an already-detached buffer
+ // (`DataCloneError`, killing the progressive preload and dropping the
+ // element onto whole-file reads). The chunk owns its bytes; the cache keeps
+ // its own. Footers are small next to the row-group payload, and structured
+ // -cloning instead of transferring would copy just the same.
+ schemaBytes: new Uint8Array(part.schemaBytes),
rowGroupBytes,
rowGroupIndex: relativeRowGroupIndex,
globalRowGroupIndex: rowGroupIndex,
@@ -632,6 +907,15 @@ export default class SpatialDataTableSource extends AnnDataSource {
/**
* Row-group and part byte payloads for worker-side parquet decode.
+ *
+ * Part bytes are WHOLE FILES — for a points element that is the entire dataset
+ * (100MB+), so fetching them when the caller will not read them is the single
+ * most expensive mistake this class can make. Most callers pass either the row
+ * groups or the parts to the worker, never both, so parts are skipped by default
+ * once row groups are in hand. Only a caller that genuinely needs parts AS A
+ * FALLBACK ALONGSIDE row groups — i.e. it hands both to the worker because the
+ * row-group decode may come back unusable, i.e. a dictionary column — sets
+ * {@link partsAlongsideRowGroups}.
*/
protected async readParquetWorkerPayload(
parquetPath: string,
@@ -640,6 +924,13 @@ export default class SpatialDataTableSource extends AnnDataSource {
fullPartsForFallback?: boolean;
/** When false (default), only part bytes are fetched for worker decode. */
includeRowGroups?: boolean;
+ /**
+ * Fetch whole-part bytes EVEN WHEN row groups were returned, because the
+ * caller passes both to the worker and cannot know in advance which it will
+ * need. Costs a full-dataset download — set it only when the row-group
+ * decode can genuinely fail to produce what the caller wants.
+ */
+ partsAlongsideRowGroups?: boolean;
}
): Promise<{
rowGroups: Array<{
@@ -654,6 +945,9 @@ export default class SpatialDataTableSource extends AnnDataSource {
const rowGroups = canUseRowGroups
? await this.readParquetRowGroupsBytesCapped(parquetPath, options.maxRows)
: [];
+ if (rowGroups.length > 0 && options.partsAlongsideRowGroups !== true) {
+ return { rowGroups, parts: [] };
+ }
const partsMaxRows = options.fullPartsForFallback ? Number.POSITIVE_INFINITY : options.maxRows;
const { parts } = await this.readParquetDatasetBytesCapped(parquetPath, partsMaxRows);
return { rowGroups, parts };
@@ -764,17 +1058,63 @@ export default class SpatialDataTableSource extends AnnDataSource {
return tablePromise;
}
+ /**
+ * The `part.N.parquet` paths of a multipart directory, or `[]` for a single
+ * parquet file.
+ *
+ * This is the SECOND way the codebase derives "how many parts are there" —
+ * {@link loadParquetDatasetMetadata} is the first. They differ only in transport:
+ * that one range-reads footers (needs `store.getRange`), this one fetches whole
+ * files (works on any store), which is why both exist. But they answer the same
+ * question, so when the metadata already knows the layout this must not re-probe:
+ * it was enumerating `part.0`, `part.1`, … again — over WHOLE-FILE gets — even for
+ * an element the metadata had already identified as a single file, which is where
+ * the duplicate trailing 404s came from.
+ *
+ * Falls back to probing only when the metadata is unavailable (a store without
+ * range support), and memoizes that fallback too. An empty result is not cached,
+ * for the same reason a null dataset is not: it is indistinguishable from a
+ * transient failure to read part 0.
+ */
private async discoverMultipartPartPaths(parquetPath: string): Promise {
- const partPaths: string[] = [];
- for (let partIndex = 0; ; partIndex += 1) {
- const partPath = `${parquetPath}/part.${partIndex}.parquet`;
- const bytes = await this.loadParquetFileBytesAtPath(partPath);
- if (!bytes) {
- break;
- }
- partPaths.push(partPath);
+ // Cached, so this is free once any caller has resolved the layout.
+ const dataset = await this.loadParquetDatasetMetadata(parquetPath);
+ if (dataset?.parts.length) {
+ // A single part AT the requested path means "not a directory" — the same
+ // answer the probe loop below reaches by finding no `part.0.parquet`.
+ const isSingleFile = dataset.parts.length === 1 && dataset.parts[0]?.path === parquetPath;
+ return isSingleFile ? [] : dataset.parts.map((part) => part.path);
+ }
+
+ const cached = this.parquetPartPathsCache.get(parquetPath);
+ if (cached) {
+ return cached;
}
- return partPaths;
+ const promise = (async () => {
+ const partPaths: string[] = [];
+ for (let partIndex = 0; ; partIndex += 1) {
+ const partPath = `${parquetPath}/part.${partIndex}.parquet`;
+ const bytes = await this.loadParquetFileBytesAtPath(partPath);
+ if (!bytes) {
+ break;
+ }
+ partPaths.push(partPath);
+ }
+ return partPaths;
+ })().then(
+ (partPaths) => {
+ if (partPaths.length === 0) {
+ this.evictIfCurrent(this.parquetPartPathsCache, parquetPath, promise);
+ }
+ return partPaths;
+ },
+ (error) => {
+ this.evictIfCurrent(this.parquetPartPathsCache, parquetPath, promise);
+ throw error;
+ }
+ );
+ this.parquetPartPathsCache.set(parquetPath, promise);
+ return promise;
}
private async loadMultipartParquetTableFromPartPaths(
diff --git a/packages/core/src/models/index.ts b/packages/core/src/models/index.ts
index 06fcfee9..78762988 100644
--- a/packages/core/src/models/index.ts
+++ b/packages/core/src/models/index.ts
@@ -536,6 +536,7 @@ export class PointsElement extends AbstractSpatialElement<'points', PointsAttrs>
async loadRowFeatureCodes(options?: {
memoryCap?: number;
featureCatalog?: PointsFeatureCatalog | null;
+ signal?: AbortSignal;
}) {
return this.vPoints.loadPointsRowFeatureCodes(`points/${this.key}`, options);
}
@@ -551,8 +552,9 @@ export class PointsElement extends AbstractSpatialElement<'points', PointsAttrs>
featureCodes: readonly number[];
onProgress?: (progress: PointsLoadProgress) => void;
featureCodeByName?: ReadonlyMap;
+ /** Aborts the scan between row-group chunks when it is superseded. */
+ signal?: AbortSignal;
}) {
- //todo generator version of this.
return this.vPoints.loadPointsMatchingFeatureCodes(`points/${this.key}`, options);
}
@@ -560,8 +562,12 @@ export class PointsElement extends AbstractSpatialElement<'points', PointsAttrs>
return this.vPoints.loadFeatureCounts(`points/${this.key}`);
}
- async listFeaturesWithCounts() {
- return this.vPoints.listPointsFeaturesWithCounts(`points/${this.key}`);
+ async listFeaturesWithCounts(options?: {
+ /** Called with the names-only catalog before the (slow) counts scan, so a panel
+ * can list features while counts are still loading. */
+ onPartialCatalog?: (catalog: PointsFeatureCatalog) => void;
+ }) {
+ return this.vPoints.listPointsFeaturesWithCounts(`points/${this.key}`, options);
}
async getPointsTilingMetadata() {
diff --git a/packages/core/src/parquetWasmLoader.ts b/packages/core/src/parquetWasmLoader.ts
index 9082317a..54cbee86 100644
--- a/packages/core/src/parquetWasmLoader.ts
+++ b/packages/core/src/parquetWasmLoader.ts
@@ -24,6 +24,33 @@ export interface ParquetRowGroupReadOptions {
offset?: number;
}
+export interface ParquetStreamOptions extends ParquetRowGroupReadOptions {
+ /** Rows per emitted record batch (upstream default 1024). */
+ batchSize?: number;
+ /** Restrict the stream to these row-group indexes. */
+ rowGroups?: number[];
+ /** Concurrent range requests the reader may have in flight. */
+ concurrency?: number;
+}
+
+/**
+ * A URL-backed parquet reader that issues its own range requests.
+ *
+ * Unlike {@link ParquetModule.readParquetRowGroup}, this decodes DICTIONARY-typed
+ * columns correctly, and it yields batches *within* a row group rather than only
+ * at row-group boundaries. It is browser-only (see {@link supportsParquetStreaming})
+ * and needs a fetchable URL, so it is a fast path, not a replacement for the
+ * byte-oriented APIs that work against any `zarr.Readable`.
+ */
+export interface ParquetWasmFile {
+ metadata(): ParquetWasmMetadata;
+ stream(options?: ParquetStreamOptions): Promise>;
+}
+
+export interface ParquetWasmFileConstructor {
+ fromUrl(url: string): Promise;
+}
+
export interface ParquetModule {
readParquet: (bytes: Uint8Array, options?: ParquetRowGroupReadOptions) => ParquetWasmTableLike;
readSchema: (bytes: Uint8Array) => ParquetWasmTableLike;
@@ -34,6 +61,32 @@ export interface ParquetModule {
rowGroupIndex: number,
options?: ParquetRowGroupReadOptions
) => ParquetWasmTableLike;
+ ParquetFile?: ParquetWasmFileConstructor;
+}
+
+/**
+ * Whether {@link ParquetWasmFile.stream} may be used in this runtime.
+ *
+ * The streaming reader's async fetch path panics under Node (`RuntimeError:
+ * unreachable`) and the panic escapes try/catch, so it cannot be probed
+ * defensively — it must be gated on the runtime up front. Tests and any SSR
+ * path therefore keep the byte-oriented reads.
+ *
+ * The reader itself needs only `fetch` and WASM, both of which a Worker has.
+ * `window` was standing in for "is a browser" and so excluded workers by
+ * accident, which forced the streaming scan to decode on the main thread. Accept
+ * a worker scope explicitly instead: `WorkerGlobalScope` is the reliable probe
+ * across classic and module workers (`importScripts` is absent from the latter).
+ */
+export function supportsParquetStreaming(): boolean {
+ if (typeof fetch !== 'function') {
+ return false;
+ }
+ if (typeof process !== 'undefined' && process.versions?.node != null) {
+ return false;
+ }
+ const scope = globalThis as { WorkerGlobalScope?: unknown };
+ return typeof window !== 'undefined' || scope.WorkerGlobalScope !== undefined;
}
function normalizeParquetModule(module: unknown): ParquetModule {
@@ -43,11 +96,19 @@ function normalizeParquetModule(module: unknown): ParquetModule {
// External WASM builds have drifted API surfaces and incomplete declarations;
// keep the boundary narrow and capability-check every optional method.
const candidate = module as Record;
- const { readParquet, readSchema, readMetadata, readParquetRowGroup } = candidate;
+ const { readParquet, readSchema, readMetadata, readParquetRowGroup, ParquetFile } = candidate;
if (typeof readParquet !== 'function' || typeof readSchema !== 'function') {
throw new Error('parquet-wasm module is missing required readParquet/readSchema APIs');
}
+ // `ParquetFile` is a wasm-bindgen class; probe the static factory on the raw
+ // value before narrowing, since the declared interface is not callable.
+ const parquetFileIsUsable =
+ typeof ParquetFile === 'function' &&
+ typeof (ParquetFile as { fromUrl?: unknown }).fromUrl === 'function';
return {
+ ParquetFile: parquetFileIsUsable
+ ? (ParquetFile as unknown as ParquetWasmFileConstructor)
+ : undefined,
readParquet: readParquet as ParquetModule['readParquet'],
readSchema: readSchema as ParquetModule['readSchema'],
readMetadata:
diff --git a/packages/core/src/pointsFeatures.ts b/packages/core/src/pointsFeatures.ts
index 68d310fb..479c40d0 100644
--- a/packages/core/src/pointsFeatures.ts
+++ b/packages/core/src/pointsFeatures.ts
@@ -1,4 +1,4 @@
-import type { Table, Vector } from 'apache-arrow';
+import type { Data, Table, Vector } from 'apache-arrow';
import { Type } from 'apache-arrow';
import {
isMortonSentinelValue,
@@ -289,20 +289,61 @@ export function resolveRowFeatureCodesFromTable(
if (!nameColumn) {
return undefined;
}
- const dictionary = dictionaryStrings(nameColumn);
-
if (!featureCodeByName) {
return undefined;
}
const out = new Int32Array(table.numRows);
- for (let rowIndex = 0; rowIndex < table.numRows; rowIndex += 1) {
- const name = resolveFeatureName(nameColumn.get(rowIndex), dictionary);
- out[rowIndex] = featureCodeByName.get(name) ?? -1;
+ let offset = 0;
+ for (const chunk of nameColumn.data) {
+ writeChunkFeatureCodes(chunk, nameColumn, featureCodeByName, out, offset);
+ offset += chunk.length;
}
return out;
}
+/**
+ * Fill `out[offset …]` with the feature code of every row in one column chunk.
+ *
+ * A DICTIONARY-typed column resolves its distinct values once per chunk and then
+ * maps raw indices, rather than asking the vector for a value per row. That
+ * matters enormously at points scale: Arrow materialises a fresh JS string for
+ * every `get()` on a dictionary vector, so the per-row form pays 4M UTF-8
+ * decodes and 4M string hashes to resolve a few hundred distinct genes. Measured
+ * on a 4M-row Xenium transcripts column: 59.3s per-row vs 40ms here.
+ *
+ * Chunks are handled individually because each parquet column chunk carries its
+ * own dictionary — indices from one chunk do not address another's values.
+ */
+function writeChunkFeatureCodes(
+ chunk: Data,
+ column: Vector,
+ featureCodeByName: ReadonlyMap,
+ out: Int32Array,
+ offset: number
+): void {
+ const dictionary = chunk.dictionary;
+ // Nulls need the vector's own null handling, so leave those chunks to the
+ // general path rather than second-guessing the validity bitmap here.
+ if (dictionary && chunk.nullCount === 0) {
+ const codeByIndex = new Int32Array(dictionary.length);
+ for (let index = 0; index < dictionary.length; index += 1) {
+ const name = dictionary.get(index);
+ codeByIndex[index] = name == null ? -1 : (featureCodeByName.get(String(name)) ?? -1);
+ }
+ const indices = chunk.values as ArrayLike;
+ for (let row = 0; row < chunk.length; row += 1) {
+ const index = indices[row];
+ out[offset + row] = index >= 0 && index < codeByIndex.length ? codeByIndex[index] : -1;
+ }
+ return;
+ }
+ for (let row = 0; row < chunk.length; row += 1) {
+ const value = column.get(offset + row);
+ out[offset + row] = value == null ? -1 : (featureCodeByName.get(String(value)) ?? -1);
+ }
+}
+
export function featureFilterNeedsRowCodes(
featureCodes: readonly number[] | undefined,
featureCodeColumnName: string | undefined,
@@ -345,3 +386,72 @@ export function mergeFeatureCountsIntoCatalog(
})),
};
}
+
+/**
+ * The effective feature-code selection for a points layer.
+ *
+ * Selections persist as NAMES, not codes. For an element with a file-backed
+ * `*_codes` column the codes are authoritative and either form would do, but for
+ * a dictionary-only element (the common case — a Xenium `transcripts` carries
+ * `feature_name` and no code column) codes are APP-ASSIGNED: a first-seen index
+ * from whichever catalog scan ran. They are not guaranteed stable across the
+ * preview→full catalog upgrade, across the row-count threshold that picks between
+ * catalog paths, or across servers that differ in range support. A stored
+ * `featureCodes` can therefore come back meaning a different gene, silently.
+ * Names are also simply readable in a saved config.
+ *
+ * `featureNames` wins when present; `featureCodes` remains for runtime use and for
+ * configs written before this existed.
+ *
+ * Returns `undefined` for "no filter — draw everything". Names that are not in the
+ * catalog are dropped: a config may name genes this element does not have.
+ *
+ * When names are present but no catalog has loaded yet, the result is an empty
+ * selection rather than `undefined` — deliberately. Resolving to "everything"
+ * would flash the whole dataset before the catalog settles, which is both wrong
+ * and expensive; an empty selection draws nothing and self-corrects on the next
+ * notify.
+ */
+export function resolveFeatureSelectionCodes(
+ selection: {
+ featureNames?: readonly string[] | undefined;
+ featureCodes?: readonly number[] | undefined;
+ },
+ catalog: PointsFeatureCatalog | null | undefined
+): number[] | undefined {
+ const { featureNames, featureCodes } = selection;
+ if (featureNames === undefined) {
+ return featureCodes ? [...featureCodes] : undefined;
+ }
+ if (!catalog) {
+ return [];
+ }
+ const codeByName = featureCodeMapFromCatalog(catalog);
+ const codes: number[] = [];
+ for (const name of featureNames) {
+ const code = codeByName?.get(name);
+ if (code !== undefined) {
+ codes.push(code);
+ }
+ }
+ return codes.sort((left, right) => left - right);
+}
+
+/** The names for a set of codes, for writing a selection back as durable names. */
+export function featureNamesForCodes(
+ codes: Iterable,
+ catalog: PointsFeatureCatalog | null | undefined
+): string[] {
+ if (!catalog) {
+ return [];
+ }
+ const nameByCode = new Map(catalog.entries.map((entry) => [entry.code, entry.name]));
+ const names: string[] = [];
+ for (const code of codes) {
+ const name = nameByCode.get(code);
+ if (name !== undefined) {
+ names.push(name);
+ }
+ }
+ return names.sort();
+}
diff --git a/packages/core/src/pointsLoadOptions.ts b/packages/core/src/pointsLoadOptions.ts
index 43feeafb..8fcda285 100644
--- a/packages/core/src/pointsLoadOptions.ts
+++ b/packages/core/src/pointsLoadOptions.ts
@@ -51,6 +51,14 @@ export interface PointsLoadResult {
* {@link PointsLoadOptions.includeFeatureCodes}. Gates the whole-dataset
* feature-index scan (only worthwhile / correct when codes are authoritative). */
hasFeatureCodeColumn?: boolean;
+ /**
+ * Per-feature point counts WITHIN THIS BATCH (`code → rows`), accumulated as the
+ * batch was decoded. Not the dataset-wide totals — those need the full counts scan
+ * — but a running tally that costs nothing extra, because the codes are already in
+ * hand while streaming. Lets a panel show per-feature stats long before the
+ * whole-dataset scan lands. Absent when the batch carries no per-row codes.
+ */
+ featureCodeCounts?: ReadonlyMap;
totalRowCount?: number;
preloadTruncated?: boolean;
/** Rows scanned when loading with an active feature filter. */
diff --git a/packages/core/src/workers/points-worker.ts b/packages/core/src/workers/points-worker.ts
index 3fa7b7ac..b65e8c5b 100644
--- a/packages/core/src/workers/points-worker.ts
+++ b/packages/core/src/workers/points-worker.ts
@@ -1,5 +1,9 @@
import { tableFromIPC, tableToIPC } from 'apache-arrow';
-import { getParquetModule, type ParquetModule } from '../parquetWasmLoader.js';
+import {
+ getParquetModule,
+ type ParquetModule,
+ type ParquetWasmFile,
+} from '../parquetWasmLoader.js';
import { buildFeatureCatalogFromColumns } from '../pointsFeatures.js';
import { filterColumnarByFeatureCodes } from '../pointsTiling.js';
import { decodeShapesGeometryFlat } from '../shapesGeometryDecode.js';
@@ -16,7 +20,9 @@ import {
decodeParquetPayloadToTable,
extractGeometryColumnar,
extractRowFeatureCodesFromTable,
+ Float32PointBuffer,
histogramToSortedArrays,
+ Int32PointBuffer,
scanFeatureCatalogFromPayload,
scanMortonTableInBounds,
scanTableByFeatureCodes,
@@ -254,19 +260,112 @@ async function handleScanParquetFeatureCounts(
};
}
+/** Cached per URL: `fromUrl` reads the footer, and one scan issues several
+ * requests against the same file as it walks row-group windows. */
+const streamFilesByUrl = new Map>();
+
+/**
+ * Stream variant of the feature scan, running IN THE WORKER.
+ *
+ * The byte-oriented path has the caller range-read whole row groups — every
+ * column — because parquet-wasm cannot fetch individual column chunks. This asks
+ * `ParquetFile.stream` for just the projected columns, so the projection reaches
+ * the network, and keeps the decode off the main thread (which the URL-streaming
+ * scan could not do while `supportsParquetStreaming` required `window`).
+ *
+ * One request covers a row-group window chosen by the caller, so progress stays
+ * granular without the protocol needing streamed responses.
+ */
+async function scanStreamByFeatureCodes(
+ request: Extract,
+ input: {
+ matchedRows: number;
+ xs: Float32PointBuffer;
+ ys: Float32PointBuffer;
+ zs: Float32PointBuffer;
+ codes: Int32PointBuffer;
+ scannedRows: number;
+ }
+): Promise<{ matchedRows: number; scannedRows: number }> {
+ const url = request.streamUrl as string;
+ const { ParquetFile } = await getParquetModule();
+ if (!ParquetFile) {
+ throw new Error('ParquetFile.stream is unavailable in the points worker');
+ }
+ let filePromise = streamFilesByUrl.get(url);
+ if (!filePromise) {
+ filePromise = ParquetFile.fromUrl(url);
+ // Cache the attempt, not the failure. Without this a single transient
+ // footer read poisons the URL for the life of the worker: every later scan
+ // awaits the same rejected promise and can never retry.
+ filePromise.catch(() => {
+ if (streamFilesByUrl.get(url) === filePromise) {
+ streamFilesByUrl.delete(url);
+ }
+ });
+ streamFilesByUrl.set(url, filePromise);
+ }
+ const file = await filePromise;
+ const featureCodeByName = request.featureCodeEntries
+ ? new Map(request.featureCodeEntries.map((entry) => [entry.name, entry.code]))
+ : undefined;
+
+ const stream = await file.stream({
+ ...(request.streamColumns?.length ? { columns: request.streamColumns } : {}),
+ ...(request.streamRowGroups?.length ? { rowGroups: request.streamRowGroups } : {}),
+ batchSize: 65_536,
+ });
+ const reader = stream.getReader();
+ try {
+ for (;;) {
+ if (input.matchedRows >= request.memoryCap) {
+ break;
+ }
+ const { done, value } = await reader.read();
+ if (done) {
+ break;
+ }
+ const table = tableFromIPC(value.intoIPCStream());
+ input.scannedRows += table.numRows;
+ input.matchedRows = scanTableByFeatureCodes({
+ table,
+ axisNames: request.axisNames,
+ featureKey: request.featureKey,
+ ...(request.featureCodeColumnName
+ ? { featureCodeColumnName: request.featureCodeColumnName }
+ : {}),
+ featureCodes: request.featureCodes,
+ memoryCap: request.memoryCap,
+ matchedRows: input.matchedRows,
+ xs: input.xs,
+ ys: input.ys,
+ zs: input.zs,
+ codes: input.codes,
+ ...(featureCodeByName ? { featureCodeByName } : {}),
+ });
+ }
+ } finally {
+ reader.releaseLock();
+ }
+ return { matchedRows: input.matchedRows, scannedRows: input.scannedRows };
+}
+
async function scanPayloadByFeatureCodes(
parquetModule: ParquetModule,
request: Extract,
input: {
matchedRows: number;
- xs: number[];
- ys: number[];
- zs: number[];
- codes: number[];
+ xs: Float32PointBuffer;
+ ys: Float32PointBuffer;
+ zs: Float32PointBuffer;
+ codes: Int32PointBuffer;
scannedRows: number;
}
): Promise<{ matchedRows: number; scannedRows: number }> {
const _hasZ = request.axisNames.includes('z');
+ if (request.streamUrl) {
+ return scanStreamByFeatureCodes(request, input);
+ }
const columns = [
...request.axisNames,
request.featureKey,
@@ -336,10 +435,13 @@ async function handleScanParquetByFeatureCodes(
): Promise {
const parquetModule = await getParquetModule();
const hasZ = request.axisNames.includes('z');
- const xs: number[] = [];
- const ys: number[] = [];
- const zs: number[] = [];
- const codes: number[] = [];
+ // Typed accumulators, reserved per chunk against an exact upper bound inside the
+ // scan (see `TypedPointBuffer`): no boxing, no growth copies, and no final
+ // `Float32Array.from` of a `number[]` holding both representations at once.
+ const xs = new Float32PointBuffer();
+ const ys = new Float32PointBuffer();
+ const zs = new Float32PointBuffer();
+ const codes = new Int32PointBuffer();
const { matchedRows, scannedRows } = await scanPayloadByFeatureCodes(parquetModule, request, {
matchedRows: 0,
xs,
@@ -348,10 +450,10 @@ async function handleScanParquetByFeatureCodes(
codes,
scannedRows: 0,
});
- const outX = Float32Array.from(xs);
- const outY = Float32Array.from(ys);
- const outZ = hasZ ? Float32Array.from(zs) : undefined;
- const outCodes = codes.length > 0 ? Int32Array.from(codes) : undefined;
+ const outX = xs.toArray();
+ const outY = ys.toArray();
+ const outZ = hasZ ? zs.toArray() : undefined;
+ const outCodes = codes.length > 0 ? codes.toArray() : undefined;
const shape = outZ ? [3, outX.length] : [2, outX.length];
return {
ok: true,
@@ -383,9 +485,9 @@ async function handleScanMortonRowGroupsInBounds(
request.mortonCodeColumnName,
...(request.featureCodeColumnName ? [request.featureCodeColumnName] : []),
];
- const xs: number[] = [];
- const ys: number[] = [];
- const zs: number[] = [];
+ const xs = new Float32PointBuffer();
+ const ys = new Float32PointBuffer();
+ const zs = new Float32PointBuffer();
for (const chunk of request.rowGroups) {
const table = tableFromIPC(
parquetModule
@@ -407,9 +509,9 @@ async function handleScanMortonRowGroupsInBounds(
zs,
});
}
- const outX = Float32Array.from(xs);
- const outY = Float32Array.from(ys);
- const outZ = hasZ ? Float32Array.from(zs) : undefined;
+ const outX = xs.toArray();
+ const outY = ys.toArray();
+ const outZ = hasZ ? zs.toArray() : undefined;
const shape = outZ ? [3, outX.length] : [2, outX.length];
return {
ok: true,
diff --git a/packages/core/src/workers/pointsWorkerClient.ts b/packages/core/src/workers/pointsWorkerClient.ts
index 8a60e79c..8ad8b344 100644
--- a/packages/core/src/workers/pointsWorkerClient.ts
+++ b/packages/core/src/workers/pointsWorkerClient.ts
@@ -506,6 +506,11 @@ export async function scanParquetFeatureCountsInWorker(
}
export type ScanParquetByFeatureCodesInput = ParquetWorkerPayload & {
+ /** Stream variant: the worker fetches only the projected columns from this URL
+ * (see the protocol type). Mutually exclusive with `parts`/`rowGroups`. */
+ streamUrl?: string;
+ streamRowGroups?: number[];
+ streamColumns?: string[];
axisNames: string[];
featureKey: string;
featureCodeColumnName?: string;
@@ -526,7 +531,7 @@ export async function scanParquetByFeatureCodesInWorker(
if (!isPointsWorkerEnabled()) {
return null;
}
- if (!input.parts?.length && !input.rowGroups?.length) {
+ if (!input.parts?.length && !input.rowGroups?.length && !input.streamUrl) {
return null;
}
const request: Extract = {
diff --git a/packages/core/src/workers/pointsWorkerProtocol.ts b/packages/core/src/workers/pointsWorkerProtocol.ts
index bd761425..474c7065 100644
--- a/packages/core/src/workers/pointsWorkerProtocol.ts
+++ b/packages/core/src/workers/pointsWorkerProtocol.ts
@@ -101,6 +101,21 @@ export type PointsWorkerRequest =
type: 'scanParquetByFeatureCodes';
parts?: Uint8Array[];
rowGroups?: ParquetRowGroupBytesChunk[];
+ /**
+ * Stream variant: fetch and decode only the projected columns from this URL,
+ * in the worker, instead of the caller shipping whole row-group BYTES.
+ *
+ * `parts`/`rowGroups` carry every column of a row group, because
+ * parquet-wasm cannot fetch individual column chunks; `ParquetFile.stream`
+ * issues its own ranged fetches per column chunk, so the projection reaches
+ * the network. The caller decides whether the URL is servable (see
+ * `canStreamMatchingScan`) and passes a row-group window per request so
+ * progress stays granular without the protocol needing streamed responses.
+ */
+ streamUrl?: string;
+ streamRowGroups?: number[];
+ /** Projected columns for the stream variant: axes + the feature column. */
+ streamColumns?: string[];
axisNames: string[];
featureKey: string;
featureCodeColumnName?: string;
diff --git a/packages/core/src/workers/pointsWorkerScan.ts b/packages/core/src/workers/pointsWorkerScan.ts
index 01652212..f945e9e1 100644
--- a/packages/core/src/workers/pointsWorkerScan.ts
+++ b/packages/core/src/workers/pointsWorkerScan.ts
@@ -1,4 +1,4 @@
-import { type Table, tableFromIPC } from 'apache-arrow';
+import { type Table, tableFromIPC, type Vector } from 'apache-arrow';
import {
accumulateFeatureCatalogFromTable,
buildFeatureCatalogFromColumns,
@@ -288,6 +288,129 @@ export async function scanFeatureCatalogFromPayload(
return featureCatalogFromCodeMap(input.featureKey, codeToName);
}
+/**
+ * Typed, growable output for the scans' matched coordinates.
+ *
+ * The scans used to accumulate into `number[]` and copy into a typed array at the
+ * end. That pays three times: every value is boxed as a double (8 bytes plus V8
+ * overhead against 4), the array reallocates as it grows, and the final
+ * `Float32Array.from` copies the lot again — with both representations live at the
+ * peak. Measured over 3 arrays:
+ *
+ * 646k rows 4M rows
+ * number[] 19ms 116ms
+ * growable 7ms 76ms (doubling copies dominate at 4M)
+ * exact-sized 6ms 17ms
+ *
+ * So the capacity hint, not the typed storage, is what carries the win at scale.
+ * Callers therefore {@link reserve} an exact upper bound before each chunk's loop —
+ * `min(rows in this chunk, remaining cap)` — which is known for free once the chunk
+ * is decoded, so pushes never reallocate. Growth is still handled, because a wrong
+ * or absent hint must stay correct rather than corrupt the output.
+ */
+class TypedPointBuffer {
+ private buffer: T;
+ private count = 0;
+
+ constructor(
+ private readonly make: (length: number) => T,
+ initialCapacity = 0
+ ) {
+ this.buffer = make(Math.max(initialCapacity, 0));
+ }
+
+ get length(): number {
+ return this.count;
+ }
+
+ /** Ensure room for `additional` more values without reallocating. */
+ reserve(additional: number): void {
+ const needed = this.count + Math.max(additional, 0);
+ if (needed <= this.buffer.length) {
+ return;
+ }
+ this.grow(needed);
+ }
+
+ push(value: number): void {
+ if (this.count === this.buffer.length) {
+ // No hint, or the hint was low: double (never from 0, which never grows).
+ this.grow(Math.max(this.buffer.length * 2, 1024));
+ }
+ this.buffer[this.count] = value;
+ this.count += 1;
+ }
+
+ private grow(capacity: number): void {
+ const next = this.make(capacity);
+ next.set(this.buffer.subarray(0, this.count) as never);
+ this.buffer = next;
+ }
+
+ /**
+ * The filled prefix, exactly sized. Zero-copy when the reservation was exact —
+ * the normal case — and otherwise one copy, which is what the old
+ * `Float32Array.from` cost anyway, so this is never worse.
+ */
+ toArray(): T {
+ return (
+ this.count === this.buffer.length ? this.buffer : this.buffer.slice(0, this.count)
+ ) as T;
+ }
+}
+
+export class Float32PointBuffer extends TypedPointBuffer {
+ constructor(initialCapacity = 0) {
+ super((length) => new Float32Array(length), initialCapacity);
+ }
+}
+
+export class Int32PointBuffer extends TypedPointBuffer {
+ constructor(initialCapacity = 0) {
+ super((length) => new Int32Array(length), initialCapacity);
+ }
+}
+
+/**
+ * A numeric Arrow column as ONE indexable typed array.
+ *
+ * `Vector.get(i)` looks like an array read but is not. On a multi-chunk vector —
+ * which is every table assembled from more than one record batch — Arrow swaps in
+ * a prototype whose `get` is `binarySearch(data, offsets, i)`, so each read walks
+ * the chunk offsets. Even the single-chunk fast path is a closure dispatch
+ * returning a boxed value. Measured over 4M rows, per column:
+ *
+ * .get() per row 1 chunk 49ms | 8 chunks 148ms | 64 chunks 244ms
+ * toArray() + index 1 chunk 6ms | 8 chunks 17ms | 64 chunks 5ms
+ *
+ * and the scan loops pay that for x, y and z, over every SCANNED row (the whole
+ * dataset), not just the matched ones. `toArray()` costs 0–2ms — it is zero-copy
+ * for a single chunk and one sequential concat otherwise — so hoisting it out of
+ * the loop is 15–50x on the hot path for a one-line change at each call site.
+ *
+ * A nullable column is materialised through `get()` once, with nulls as NaN, so
+ * callers keep a single indexed loop shape rather than a second slow path. Note
+ * that this makes a non-finite coordinate skipped rather than emitted, which the
+ * old `typeof x !== 'number'` test let through — a point at NaN cannot render.
+ */
+function numericColumnValues(column: Vector | null | undefined): ArrayLike | null {
+ if (!column) {
+ return null;
+ }
+ if (column.nullCount === 0) {
+ const values = column.toArray();
+ if (ArrayBuffer.isView(values)) {
+ return values as unknown as ArrayLike;
+ }
+ }
+ const out = new Float64Array(column.length);
+ for (let index = 0; index < column.length; index += 1) {
+ const value = column.get(index);
+ out[index] = typeof value === 'number' ? value : Number.NaN;
+ }
+ return out;
+}
+
export function scanMortonTableInBounds(input: {
table: Table;
rowGroupIndex: number;
@@ -296,9 +419,9 @@ export function scanMortonTableInBounds(input: {
mortonCodeColumnName: string;
featureCodeColumnName?: string;
featureCodes?: readonly number[];
- xs: number[];
- ys: number[];
- zs: number[];
+ xs: Float32PointBuffer;
+ ys: Float32PointBuffer;
+ zs: Float32PointBuffer;
}): void {
const allowedFeatureCodes = featureCodeAllowSet(input.featureCodes);
const filterByFeature = allowedFeatureCodes !== null;
@@ -313,7 +436,37 @@ export function scanMortonTableInBounds(input: {
if (!xColumn || !yColumn) {
return;
}
- for (let rowIndex = 0; rowIndex < input.table.numRows; rowIndex += 1) {
+ // Hoisted out of the loop: see `numericColumnValues`. The feature-code column
+ // matters most here — it is read for EVERY row, before any bounds rejection.
+ const xValues = numericColumnValues(xColumn);
+ const yValues = numericColumnValues(yColumn);
+ const zValues = numericColumnValues(zColumn);
+ const featureCodeValues = numericColumnValues(featureCodeColumn);
+ if (!xValues || !yValues) {
+ return;
+ }
+ // A filter this scan cannot honour must match NOTHING, not everything. The
+ // predicate below used to carry the column check as a conjunct, so a missing
+ // code column made it false for every row and the caller got the whole chunk
+ // back — one gene requested, 4M points drawn, and no error anywhere. An empty
+ // result is also wrong, but it is wrong visibly.
+ if (filterByFeature && !featureCodeValues) {
+ return;
+ }
+ // Hoisted: `Table.numRows` is not a field but
+ // `data.reduce((n, d) => n + d.length, 0)` — a closure allocation and a walk of
+ // every chunk. As a loop CONDITION that ran per row. See `scanTableByFeatureCodes`.
+ const numRows = input.table.numRows;
+ // Upper bound: at most one match per row. Bounds-rejection usually leaves this
+ // over-reserved, but only transiently, and it removes the growth copies.
+ input.xs.reserve(numRows);
+ input.ys.reserve(numRows);
+ if (zValues) {
+ input.zs.reserve(numRows);
+ }
+ for (let rowIndex = 0; rowIndex < numRows; rowIndex += 1) {
+ // Sentinels only ever occupy the first rows of the first row group, so this
+ // stays on the (rare) boxed read rather than materialising the whole column.
if (
input.rowGroupIndex === 0 &&
rowIndex < 4 &&
@@ -323,14 +476,16 @@ export function scanMortonTableInBounds(input: {
}
if (
filterByFeature &&
- featureCodeColumn &&
- !rowMatchesFeatureCode(featureCodeColumn.get(rowIndex), allowedFeatureCodes)
+ !rowMatchesFeatureCode(
+ (featureCodeValues as ArrayLike)[rowIndex],
+ allowedFeatureCodes
+ )
) {
continue;
}
- const x = xColumn.get(rowIndex);
- const y = yColumn.get(rowIndex);
- if (typeof x !== 'number' || typeof y !== 'number') {
+ const x = xValues[rowIndex];
+ const y = yValues[rowIndex];
+ if (!Number.isFinite(x) || !Number.isFinite(y)) {
continue;
}
if (
@@ -343,9 +498,9 @@ export function scanMortonTableInBounds(input: {
}
input.xs.push(x);
input.ys.push(y);
- if (zColumn) {
- const z = zColumn.get(rowIndex);
- input.zs.push(typeof z === 'number' ? z : 0);
+ if (zValues) {
+ const z = zValues[rowIndex];
+ input.zs.push(Number.isFinite(z) ? z : 0);
}
}
}
@@ -403,11 +558,11 @@ export function scanTableByFeatureCodes(input: {
featureCodes: readonly number[];
memoryCap: number;
matchedRows: number;
- xs: number[];
- ys: number[];
- zs: number[];
+ xs: Float32PointBuffer;
+ ys: Float32PointBuffer;
+ zs: Float32PointBuffer;
/** Optional per-matched-row feature codes, collected for colour-by-feature. */
- codes?: number[];
+ codes?: Int32PointBuffer;
/** Authoritative name→code map for dict-only elements (no code column), so a
* row's feature_name resolves to the same code space the selection uses. */
featureCodeByName?: ReadonlyMap;
@@ -425,27 +580,51 @@ export function scanTableByFeatureCodes(input: {
const xColumn = input.axisNames.includes('x') ? input.table.getChild('x') : null;
const yColumn = input.axisNames.includes('y') ? input.table.getChild('y') : null;
const zColumn = input.axisNames.includes('z') ? input.table.getChild('z') : null;
- if (!xColumn || !yColumn) {
+ // Hoisted out of the loop: see `numericColumnValues` — a per-row `Vector.get`
+ // binary-searches the chunk offsets, and this loop runs over every scanned row.
+ const xValues = numericColumnValues(xColumn);
+ const yValues = numericColumnValues(yColumn);
+ const zValues = numericColumnValues(zColumn);
+ if (!xValues || !yValues) {
return input.matchedRows;
}
let matchedRows = input.matchedRows;
- for (let rowIndex = 0; rowIndex < input.table.numRows; rowIndex += 1) {
+ // Hoisted, and this is the one that dominated. `Table.numRows` is NOT a field:
+ //
+ // get numRows() { return this.data.reduce((n, d) => n + d.length, 0); }
+ //
+ // — a fresh closure plus a walk of every chunk, on EVERY iteration, because it
+ // sat in the loop condition. Measured over 4M rows: 35ms at 1 chunk, 112ms at 8,
+ // 662ms at 64, against 4-7ms hoisted (7x / 30x / 97x). It grows with chunk count,
+ // so it got worse exactly as the table got bigger — and it outweighed the whole
+ // per-row `Vector.get` cost it was sitting next to.
+ const numRows = input.table.numRows;
+ // At most one match per scanned row, and never past the cap: an exact upper
+ // bound, so the pushes below cannot reallocate. See `TypedPointBuffer`.
+ const headroom = Math.min(numRows, Math.max(input.memoryCap - matchedRows, 0));
+ input.xs.reserve(headroom);
+ input.ys.reserve(headroom);
+ if (zValues) {
+ input.zs.reserve(headroom);
+ }
+ input.codes?.reserve(headroom);
+ for (let rowIndex = 0; rowIndex < numRows; rowIndex += 1) {
if (matchedRows >= input.memoryCap) {
break;
}
if (allowed !== null && !rowMatchesFeatureCode(rowCodes[rowIndex], allowed)) {
continue;
}
- const x = xColumn.get(rowIndex);
- const y = yColumn.get(rowIndex);
- if (typeof x !== 'number' || typeof y !== 'number') {
+ const x = xValues[rowIndex];
+ const y = yValues[rowIndex];
+ if (!Number.isFinite(x) || !Number.isFinite(y)) {
continue;
}
input.xs.push(x);
input.ys.push(y);
- if (zColumn) {
- const z = zColumn.get(rowIndex);
- input.zs.push(typeof z === 'number' ? z : 0);
+ if (zValues) {
+ const z = zValues[rowIndex];
+ input.zs.push(Number.isFinite(z) ? z : 0);
}
input.codes?.push(rowCodes[rowIndex] ?? -1);
matchedRows += 1;
diff --git a/packages/core/tests/parquetStreamingScope.spec.ts b/packages/core/tests/parquetStreamingScope.spec.ts
new file mode 100644
index 00000000..2adbae0a
--- /dev/null
+++ b/packages/core/tests/parquetStreamingScope.spec.ts
@@ -0,0 +1,65 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { supportsParquetStreaming } from '../src/parquetWasmLoader.js';
+
+/**
+ * `ParquetFile.stream` needs only `fetch` and WASM, both of which a Worker has —
+ * but this check used `window` as a stand-in for "is a browser", which excluded
+ * workers by accident and so forced the streaming feature scan to decode on the
+ * main thread.
+ *
+ * The Node exclusion is not stylistic: the reader's async fetch path panics there
+ * with `RuntimeError: unreachable`, and the panic escapes try/catch, so it cannot
+ * be probed defensively and must stay gated up front.
+ */
+const g = globalThis as unknown as Record;
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+});
+
+describe('supportsParquetStreaming', () => {
+ it('is false under Node, where the reader panics unrecoverably', () => {
+ // The real test environment: `process.versions.node` is set.
+ expect(supportsParquetStreaming()).toBe(false);
+ });
+
+ it('accepts a worker scope — no window, but WorkerGlobalScope present', () => {
+ vi.stubGlobal('process', undefined);
+ vi.stubGlobal('window', undefined);
+ vi.stubGlobal('WorkerGlobalScope', class {});
+ vi.stubGlobal('fetch', () => Promise.resolve());
+ expect(supportsParquetStreaming()).toBe(true);
+ });
+
+ it('accepts the browser main thread', () => {
+ vi.stubGlobal('process', undefined);
+ vi.stubGlobal('window', {});
+ vi.stubGlobal('fetch', () => Promise.resolve());
+ expect(supportsParquetStreaming()).toBe(true);
+ });
+
+ it('rejects a scope that is neither — no window, no WorkerGlobalScope', () => {
+ vi.stubGlobal('process', undefined);
+ vi.stubGlobal('window', undefined);
+ vi.stubGlobal('fetch', () => Promise.resolve());
+ const had = 'WorkerGlobalScope' in g;
+ if (had) {
+ vi.stubGlobal('WorkerGlobalScope', undefined);
+ }
+ expect(supportsParquetStreaming()).toBe(false);
+ });
+
+ it('rejects any scope without fetch', () => {
+ vi.stubGlobal('process', undefined);
+ vi.stubGlobal('window', {});
+ vi.stubGlobal('fetch', undefined);
+ expect(supportsParquetStreaming()).toBe(false);
+ });
+
+ it('still rejects Node even when a fetch polyfill is present', () => {
+ vi.stubGlobal('window', {});
+ vi.stubGlobal('fetch', () => Promise.resolve());
+ // `process.versions.node` is set by the real environment here.
+ expect(supportsParquetStreaming()).toBe(false);
+ });
+});
diff --git a/packages/core/tests/parquetWorkerPayload.spec.ts b/packages/core/tests/parquetWorkerPayload.spec.ts
new file mode 100644
index 00000000..898b3dc2
--- /dev/null
+++ b/packages/core/tests/parquetWorkerPayload.spec.ts
@@ -0,0 +1,105 @@
+import { describe, expect, it, vi } from 'vitest';
+import SpatialDataTableSource from '../src/models/VTableSource.js';
+
+/**
+ * `readParquetWorkerPayload` decides whether to download WHOLE PARTS — for a points
+ * element that is the entire dataset, often 100MB+. These tests pin when it does,
+ * because an unnecessary parts fetch is invisible in behaviour and only shows up as
+ * a giant request in the network tab.
+ */
+type PayloadOptions = {
+ maxRows: number;
+ fullPartsForFallback?: boolean;
+ includeRowGroups?: boolean;
+ partsAlongsideRowGroups?: boolean;
+};
+
+function harness(options: { rowGroupCount: number; canUseRowGroups?: boolean }) {
+ const source = new SpatialDataTableSource({
+ fileType: '.zarr',
+ store: {
+ async get() {
+ return undefined;
+ },
+ } as never,
+ });
+ const internals = source as unknown as {
+ canLoadParquetRowGroups: () => Promise;
+ readParquetRowGroupsBytesCapped: (path: string, maxRows: number) => Promise;
+ readParquetDatasetBytesCapped: (
+ path: string,
+ maxRows: number
+ ) => Promise<{ parts: Uint8Array[] }>;
+ readParquetWorkerPayload: (
+ path: string,
+ options: PayloadOptions
+ ) => Promise<{ rowGroups: unknown[]; parts: Uint8Array[] }>;
+ };
+ vi.spyOn(internals, 'canLoadParquetRowGroups').mockResolvedValue(
+ options.canUseRowGroups !== false
+ );
+ vi.spyOn(internals, 'readParquetRowGroupsBytesCapped').mockResolvedValue(
+ Array.from({ length: options.rowGroupCount }, (_value, index) => ({
+ schemaBytes: new Uint8Array([1]),
+ rowGroupBytes: new Uint8Array([2]),
+ rowGroupIndex: index,
+ }))
+ );
+ // Stands in for the whole-dataset download.
+ const fetchParts = vi
+ .spyOn(internals, 'readParquetDatasetBytesCapped')
+ .mockResolvedValue({ parts: [new Uint8Array([9, 9, 9])] });
+ return {
+ fetchParts,
+ run: (options2: PayloadOptions) =>
+ internals.readParquetWorkerPayload('points/a/points.parquet', options2),
+ };
+}
+
+describe('readParquetWorkerPayload — whole-part downloads', () => {
+ it('does not download parts when row groups satisfy the caller', async () => {
+ const { fetchParts, run } = harness({ rowGroupCount: 4 });
+
+ const payload = await run({ maxRows: 1000, includeRowGroups: true });
+
+ expect(payload.rowGroups).toHaveLength(4);
+ expect(payload.parts).toEqual([]);
+ expect(fetchParts).not.toHaveBeenCalled();
+ });
+
+ it('downloads parts alongside row groups only when explicitly asked', async () => {
+ const { fetchParts, run } = harness({ rowGroupCount: 4 });
+
+ // The catalog scan hands BOTH to the worker: a row-group decode of a
+ // dictionary column can come back unusable, so parts are the fallback.
+ const payload = await run({
+ maxRows: Number.POSITIVE_INFINITY,
+ includeRowGroups: true,
+ partsAlongsideRowGroups: true,
+ fullPartsForFallback: true,
+ });
+
+ expect(payload.rowGroups).toHaveLength(4);
+ expect(payload.parts).toHaveLength(1);
+ expect(fetchParts).toHaveBeenCalledOnce();
+ });
+
+ it('falls back to parts when no row groups were requested', async () => {
+ const { fetchParts, run } = harness({ rowGroupCount: 0 });
+
+ const payload = await run({ maxRows: 1000 });
+
+ expect(payload.rowGroups).toEqual([]);
+ expect(payload.parts).toHaveLength(1);
+ expect(fetchParts).toHaveBeenCalledOnce();
+ });
+
+ it('falls back to parts when the store cannot do row-group reads', async () => {
+ const { fetchParts, run } = harness({ rowGroupCount: 0, canUseRowGroups: false });
+
+ const payload = await run({ maxRows: 1000, includeRowGroups: true });
+
+ expect(payload.rowGroups).toEqual([]);
+ expect(fetchParts).toHaveBeenCalledOnce();
+ });
+});
diff --git a/packages/core/tests/pointsCatalogSupersession.spec.ts b/packages/core/tests/pointsCatalogSupersession.spec.ts
new file mode 100644
index 00000000..29ac7af1
--- /dev/null
+++ b/packages/core/tests/pointsCatalogSupersession.spec.ts
@@ -0,0 +1,134 @@
+import { Matrix4 } from '@math.gl/core';
+import { describe, expect, it, vi } from 'vitest';
+import {
+ type PointsResolveConfig,
+ PointsResolver,
+ type ResolveContext,
+} from '../src/engine/index.js';
+import type { PointsElement, PointsFeatureCatalog } from '../src/models/index.js';
+import type { PointsLoadResult } from '../src/pointsLoadOptions.js';
+
+/**
+ * The resident preload and the full-dataset catalog scan both write the catalog
+ * slot, and they run CONCURRENTLY: the panel kicks the full scan the moment it
+ * mounts, while a multi-million-row preload is still streaming.
+ *
+ * That overlap is where a merfish element went wrong in two visible ways — the
+ * feature list stuck on partial "≥" counts forever, and colours that no longer
+ * matched the panel (hovering the most abundant gene lit a small unrelated
+ * cluster). Both come from the same window, so they are pinned together.
+ *
+ * A dict-only element (no `feature_key` code column) is the case that matters:
+ * its codes are app-assigned, so the preview and the full scan generally number
+ * the same gene DIFFERENTLY, and the render's per-row codes only agree with the
+ * panel while both come from the same catalog.
+ */
+
+/** Rows are gene A, B, A, B — stated once, in names, so the code space is explicit. */
+const ROW_NAMES = ['A', 'B', 'A', 'B'] as const;
+
+const PREVIEW_CATALOG: PointsFeatureCatalog = {
+ featureKey: 'gene',
+ entries: [
+ { code: 0, name: 'A' },
+ { code: 1, name: 'B' },
+ ],
+};
+
+// The full scan walks rows in file order and assigns codes as it meets each gene,
+// so the same genes come back numbered the other way round — with counts.
+const FULL_CATALOG: PointsFeatureCatalog = {
+ featureKey: 'gene',
+ entries: [
+ { code: 0, name: 'B', count: 2 },
+ { code: 1, name: 'A', count: 2 },
+ ],
+};
+
+function codesIn(catalog: PointsFeatureCatalog, names: readonly string[]): number[] {
+ const byName = new Map(catalog.entries.map((entry) => [entry.name, entry.code]));
+ return names.map((name) => byName.get(name) as number);
+}
+
+function deferred() {
+ let resolve!: (value: T) => void;
+ const promise = new Promise((r) => {
+ resolve = r;
+ });
+ return { promise, resolve };
+}
+
+/** Let every already-queued microtask (and the timer turn behind it) run. */
+const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
+
+function harness() {
+ const fullScan = deferred();
+ const el = {
+ key: 'transcripts',
+ // Dict-only: `hasFeatureCodeColumn` is false, and the decode hands back BOTH the
+ // per-row codes and the preview catalog they are expressed in.
+ loadPoints: vi.fn(
+ async (): Promise => ({
+ shape: [2, ROW_NAMES.length],
+ data: [new Float32Array(ROW_NAMES.length), new Float32Array(ROW_NAMES.length)],
+ featureCodes: Int32Array.from(codesIn(PREVIEW_CATALOG, ROW_NAMES)),
+ featureCatalog: PREVIEW_CATALOG,
+ hasFeatureCodeColumn: false,
+ })
+ ),
+ listFeaturesWithCounts: vi.fn(async () => fullScan.promise),
+ loadRowFeatureCodes: vi.fn(async () => Int32Array.from(codesIn(PREVIEW_CATALOG, ROW_NAMES))),
+ } as unknown as PointsElement;
+
+ const resolver = new PointsResolver();
+ const target = { key: 'transcripts', layerId: 'layer-p', element: el };
+ const context: ResolveContext = {
+ entryId: 'layer-p',
+ elementKey: 'transcripts',
+ kind: 'points',
+ element: el,
+ config: {},
+ transform: new Matrix4(),
+ };
+ return { el, resolver, target, context, fullScan };
+}
+
+describe('catalog scan vs. the preload that finishes underneath it', () => {
+ it('keeps the full catalog when the preload settles its preview mid-scan', async () => {
+ const { resolver, target, fullScan } = harness();
+
+ // The panel mounts and asks for the full list; the preload is still running.
+ const scan = resolver.ensureFeatureCatalog(target);
+ // …and finishes first, carrying its instant resident-subset preview.
+ await resolver.ensureLoaded(target, 1_000);
+ fullScan.resolve(FULL_CATALOG);
+ await scan;
+ await flush();
+
+ // The preview is a strict downgrade of a scan that is already in flight. Losing
+ // the full result here is what left the panel on "sorted by count so far"
+ // permanently — nothing re-requests the catalog, so only a panel remount recovered.
+ expect(resolver.getFeatureCatalog('transcripts')).toEqual(FULL_CATALOG);
+ });
+
+ it('leaves row codes in the same code space as the catalog it reports', async () => {
+ const { resolver, target, fullScan } = harness();
+
+ const scan = resolver.ensureFeatureCatalog(target);
+ await resolver.ensureLoaded(target, 1_000);
+ fullScan.resolve(FULL_CATALOG);
+ await scan;
+ await flush();
+
+ // The invariant the renderer depends on: row code i names the same gene the panel
+ // shows for code i. Break it and every point is drawn in another gene's colour —
+ // and the hover highlight lights an unrelated scatter of points.
+ const catalog = resolver.getFeatureCatalog('transcripts');
+ expect(catalog).not.toBeNull();
+ const rowCodes = resolver.getRowFeatureCodes('transcripts');
+ expect(rowCodes).toBeDefined();
+ expect([...(rowCodes as ArrayLike as Int32Array)]).toEqual(
+ codesIn(catalog as PointsFeatureCatalog, ROW_NAMES)
+ );
+ });
+});
diff --git a/packages/core/tests/pointsFeatureSelection.spec.ts b/packages/core/tests/pointsFeatureSelection.spec.ts
new file mode 100644
index 00000000..ec988b14
--- /dev/null
+++ b/packages/core/tests/pointsFeatureSelection.spec.ts
@@ -0,0 +1,90 @@
+import { describe, expect, it } from 'vitest';
+import { featureNamesForCodes, resolveFeatureSelectionCodes } from '../src/pointsFeatures.js';
+import type { PointsFeatureCatalog } from '../src/pointsTiling.js';
+
+/**
+ * Selections persist as NAMES because for a dictionary-only element the codes are
+ * app-assigned — a first-seen index from whichever catalog scan ran — so the same
+ * gene can be numbered differently between the resident preview and the full
+ * catalog, between the two catalog paths, or between servers. A stored code can
+ * therefore come back meaning a different gene, silently.
+ *
+ * The two catalogs below are the same three genes numbered differently, which is
+ * exactly the situation the name form exists to survive.
+ */
+const PREVIEW: PointsFeatureCatalog = {
+ featureKey: 'feature_name',
+ entries: [
+ { code: 0, name: 'EPCAM' },
+ { code: 1, name: 'MALL' },
+ { code: 2, name: 'TCIM' },
+ ],
+};
+
+const FULL: PointsFeatureCatalog = {
+ featureKey: 'feature_name',
+ entries: [
+ { code: 0, name: 'TCIM' },
+ { code: 1, name: 'EPCAM' },
+ { code: 2, name: 'MALL' },
+ ],
+};
+
+describe('resolveFeatureSelectionCodes', () => {
+ it('resolves names to whichever codes the CURRENT catalog uses', () => {
+ expect(resolveFeatureSelectionCodes({ featureNames: ['EPCAM'] }, PREVIEW)).toEqual([0]);
+ expect(resolveFeatureSelectionCodes({ featureNames: ['EPCAM'] }, FULL)).toEqual([1]);
+ });
+
+ it('survives a catalog renumbering — the whole point of the name form', () => {
+ const names = ['EPCAM', 'TCIM'];
+ // Same genes, different numbering, and the selection still means those genes.
+ const asPreview = resolveFeatureSelectionCodes({ featureNames: names }, PREVIEW);
+ const asFull = resolveFeatureSelectionCodes({ featureNames: names }, FULL);
+ expect(featureNamesForCodes(asPreview as number[], PREVIEW)).toEqual(['EPCAM', 'TCIM']);
+ expect(featureNamesForCodes(asFull as number[], FULL)).toEqual(['EPCAM', 'TCIM']);
+ // …whereas the stored CODES would not have: [0, 2] is EPCAM+TCIM under the
+ // preview but TCIM+MALL under the full catalog. This is the silent bug.
+ expect(featureNamesForCodes([0, 2], PREVIEW)).toEqual(['EPCAM', 'TCIM']);
+ expect(featureNamesForCodes([0, 2], FULL)).toEqual(['MALL', 'TCIM']);
+ });
+
+ it('round-trips a selection through names and back', () => {
+ const codes = [1, 2];
+ const names = featureNamesForCodes(codes, FULL);
+ expect(names).toEqual(['EPCAM', 'MALL']);
+ expect(resolveFeatureSelectionCodes({ featureNames: names }, FULL)).toEqual(codes);
+ });
+
+ it('treats absent names as "no filter" and falls back to legacy codes', () => {
+ expect(resolveFeatureSelectionCodes({}, FULL)).toBeUndefined();
+ expect(resolveFeatureSelectionCodes({ featureCodes: [2] }, FULL)).toEqual([2]);
+ });
+
+ it('lets names win over a stale legacy code list', () => {
+ expect(
+ resolveFeatureSelectionCodes({ featureNames: ['TCIM'], featureCodes: [99] }, FULL)
+ ).toEqual([0]);
+ });
+
+ it('drops names this element does not have, rather than inventing codes', () => {
+ // A config may name genes from another dataset; those must not become -1 or 0.
+ expect(resolveFeatureSelectionCodes({ featureNames: ['EPCAM', 'NOT_HERE'] }, FULL)).toEqual([
+ 1,
+ ]);
+ expect(resolveFeatureSelectionCodes({ featureNames: ['NOT_HERE'] }, FULL)).toEqual([]);
+ });
+
+ it('selects nothing — not everything — while the catalog is still loading', () => {
+ // Resolving to `undefined` here would read as "no filter" and flash the whole
+ // dataset before the catalog settles. Empty draws nothing and self-corrects.
+ expect(resolveFeatureSelectionCodes({ featureNames: ['EPCAM'] }, undefined)).toEqual([]);
+ expect(resolveFeatureSelectionCodes({ featureNames: ['EPCAM'] }, null)).toEqual([]);
+ // But an absent selection is still "everything", catalog or no catalog.
+ expect(resolveFeatureSelectionCodes({}, undefined)).toBeUndefined();
+ });
+
+ it('keeps an explicit empty selection empty', () => {
+ expect(resolveFeatureSelectionCodes({ featureNames: [] }, FULL)).toEqual([]);
+ });
+});
diff --git a/packages/core/tests/pointsFeatureStreamingCatalog.spec.ts b/packages/core/tests/pointsFeatureStreamingCatalog.spec.ts
new file mode 100644
index 00000000..dd9c5d91
--- /dev/null
+++ b/packages/core/tests/pointsFeatureStreamingCatalog.spec.ts
@@ -0,0 +1,107 @@
+import { Dictionary, Int32, tableFromArrays, Utf8, vectorFromArray } from 'apache-arrow';
+import { describe, expect, it } from 'vitest';
+import { supportsParquetStreaming } from '../src/parquetWasmLoader.js';
+import {
+ accumulateFeatureCatalogFromTable,
+ featureCatalogFromCodeMap,
+} from '../src/pointsFeatures.js';
+
+const FEATURE_KEY = 'feature_name';
+
+/** A dictionary-typed feature column — the case row-group reads cannot decode. */
+function dictionaryFeatureTable(names: string[]) {
+ return tableFromArrays({
+ [FEATURE_KEY]: vectorFromArray(names, new Dictionary(new Utf8(), new Int32())),
+ });
+}
+
+function catalogFromWholeTable(names: string[]) {
+ const codeToName = new Map();
+ const nameToCode = new Map();
+ accumulateFeatureCatalogFromTable(
+ codeToName,
+ nameToCode,
+ dictionaryFeatureTable(names),
+ FEATURE_KEY,
+ undefined
+ );
+ return featureCatalogFromCodeMap(FEATURE_KEY, codeToName);
+}
+
+/** Accumulate in batches, capturing the catalog after each — what streaming does. */
+function catalogsFromBatches(names: string[], batchSize: number) {
+ const codeToName = new Map();
+ const nameToCode = new Map();
+ const partials = [];
+ for (let offset = 0; offset < names.length; offset += batchSize) {
+ accumulateFeatureCatalogFromTable(
+ codeToName,
+ nameToCode,
+ dictionaryFeatureTable(names.slice(offset, offset + batchSize)),
+ FEATURE_KEY,
+ undefined
+ );
+ partials.push(featureCatalogFromCodeMap(FEATURE_KEY, codeToName));
+ }
+ return partials;
+}
+
+// Features clustered so later batches introduce genuinely new names — otherwise
+// every batch would see every feature and the prefix property would be trivial.
+const NAMES = [
+ ...Array.from({ length: 40 }, (_v, i) => `early_${i % 4}`),
+ ...Array.from({ length: 40 }, (_v, i) => `mid_${i % 5}`),
+ ...Array.from({ length: 40 }, (_v, i) => `late_${i % 3}`),
+];
+
+describe('streaming feature catalog code space', () => {
+ it('assigns the same codes batch-by-batch as it does whole-table', () => {
+ const whole = catalogFromWholeTable(NAMES);
+ const partials = catalogsFromBatches(NAMES, 16);
+ const final = partials[partials.length - 1];
+
+ expect(final.entries).toEqual(whole.entries);
+ });
+
+ it('never reassigns a code as later batches arrive', () => {
+ const partials = catalogsFromBatches(NAMES, 16);
+
+ // Each partial must be a prefix-consistent view: every entry it publishes
+ // keeps the same code in every later partial. This is what makes it safe for
+ // the panel to render (and the user to select from) a partial catalog.
+ const final = partials[partials.length - 1];
+ const finalByName = new Map(final.entries.map((entry) => [entry.name, entry.code]));
+ for (const partial of partials) {
+ for (const entry of partial.entries) {
+ expect(finalByName.get(entry.name)).toBe(entry.code);
+ }
+ }
+ });
+
+ it('grows monotonically and reaches the full feature set', () => {
+ const partials = catalogsFromBatches(NAMES, 16);
+
+ for (let i = 1; i < partials.length; i += 1) {
+ expect(partials[i].entries.length).toBeGreaterThanOrEqual(partials[i - 1].entries.length);
+ }
+ expect(partials[0].entries.length).toBeLessThan(new Set(NAMES).size);
+ expect(partials[partials.length - 1].entries.length).toBe(new Set(NAMES).size);
+ });
+
+ it('is independent of batch size', () => {
+ const reference = catalogFromWholeTable(NAMES);
+ for (const batchSize of [1, 7, 16, 64, 1000]) {
+ const partials = catalogsFromBatches(NAMES, batchSize);
+ expect(partials[partials.length - 1].entries).toEqual(reference.entries);
+ }
+ });
+});
+
+describe('supportsParquetStreaming', () => {
+ it('is false under Node so tests and SSR keep the byte-oriented reads', () => {
+ // The streaming reader panics under Node with an async `RuntimeError:
+ // unreachable` that escapes try/catch, so this guard cannot be probed
+ // defensively — it has to stay false here.
+ expect(supportsParquetStreaming()).toBe(false);
+ });
+});
diff --git a/packages/core/tests/pointsFeatureTallySentinels.spec.ts b/packages/core/tests/pointsFeatureTallySentinels.spec.ts
new file mode 100644
index 00000000..e096ae9e
--- /dev/null
+++ b/packages/core/tests/pointsFeatureTallySentinels.spec.ts
@@ -0,0 +1,74 @@
+import { Dictionary, Int16, tableFromArrays, Utf8, vectorFromArray } from 'apache-arrow';
+import { describe, expect, it } from 'vitest';
+import { tallyFeatureCodesFromColumn } from '../src/models/VPointsSource.js';
+import { MORTON_CODE_2D_COLUMN, MORTON_CODE_EXTREME_VALUE_INDICATOR } from '../src/pointsTiling.js';
+
+/**
+ * A morton-tiled element carries up to four SENTINEL rows at the head of its first
+ * row group — they encode the dataset bounding box, not real points. Every catalog
+ * builder is told to skip them; the tally that fills in the same catalog's counts
+ * was not, so counts and entries could disagree about the very same catalog.
+ *
+ * Small in magnitude (at most four rows) but not cosmetic: it is a count of points
+ * that are not points, in the one number the feature panel presents as authoritative.
+ */
+
+const FEATURE_KEY = 'feature_name';
+const nameToCode = new Map([
+ ['GENE_A', 0],
+ ['GENE_B', 1],
+]);
+
+/** Two sentinel rows, then four real ones. */
+const NAMES = ['GENE_A', 'GENE_A', 'GENE_A', 'GENE_B', 'GENE_A', 'GENE_B'];
+const MORTON = [
+ MORTON_CODE_EXTREME_VALUE_INDICATOR,
+ MORTON_CODE_EXTREME_VALUE_INDICATOR,
+ 17,
+ 18,
+ 19,
+ 20,
+];
+
+function columns(dictionaryEncoded: boolean) {
+ const table = tableFromArrays({
+ [FEATURE_KEY]: dictionaryEncoded
+ ? (vectorFromArray(NAMES, new Dictionary(new Utf8(), new Int16())) as never)
+ : (NAMES as never),
+ [MORTON_CODE_2D_COLUMN]: Int32Array.from(MORTON) as never,
+ });
+ return {
+ name: table.getChild(FEATURE_KEY)as never,
+ morton: table.getChild(MORTON_CODE_2D_COLUMN) as never,
+ rows: table.numRows,
+ };
+}
+
+function tally(dictionaryEncoded: boolean, withMorton: boolean) {
+ const { name, morton, rows } = columns(dictionaryEncoded);
+ const counts = new Map();
+ tallyFeatureCodesFromColumn(name, rows, nameToCode, counts, withMorton ? morton : null);
+ return counts;
+}
+
+describe('feature count tally — morton sentinels', () => {
+ // Both branches matter: the dictionary fast path is the one a real Xenium
+ // `transcripts` takes, and it indexes the chunk directly rather than via `get`.
+ it.each([
+ ['dictionary-encoded', true],
+ ['plain utf8', false],
+ ])('excludes sentinel rows from the counts (%s)', (_label, dictionaryEncoded) => {
+ const counts = tally(dictionaryEncoded, true);
+ // 4 real rows: GENE_A ×2, GENE_B ×2. The two sentinels also say GENE_A.
+ expect(counts.get(0)).toBe(2);
+ expect(counts.get(1)).toBe(2);
+ expect([...counts.values()].reduce((sum, n) => sum + n, 0)).toBe(4);
+ });
+
+ it('counts every row when the element is not tiled', () => {
+ // No morton column → nothing to skip, and the head rows are ordinary points.
+ const counts = tally(true, false);
+ expect(counts.get(0)).toBe(4);
+ expect(counts.get(1)).toBe(2);
+ });
+});
diff --git a/packages/core/tests/pointsFeatures.spec.ts b/packages/core/tests/pointsFeatures.spec.ts
index df6777f1..69ef2dc4 100644
--- a/packages/core/tests/pointsFeatures.spec.ts
+++ b/packages/core/tests/pointsFeatures.spec.ts
@@ -109,13 +109,15 @@ describe('SpatialDataPointsSource feature catalog', () => {
});
it('lists distinct feature names and codes across multipart parquet', async () => {
+ // Under the preload cap → whole-table read, which tallies as it decodes:
+ // gene_a x2, gene_b x2, gene_c x1 across the two parts.
const catalog = await source.listPointsFeatures('points/transcripts');
expect(catalog).toEqual({
featureKey: 'feature_name',
entries: [
- { code: 0, name: 'gene_a' },
- { code: 1, name: 'gene_b' },
- { code: 2, name: 'gene_c' },
+ { code: 0, name: 'gene_a', count: 2 },
+ { code: 1, name: 'gene_b', count: 2 },
+ { code: 2, name: 'gene_c', count: 1 },
],
});
});
@@ -177,6 +179,8 @@ PY`,
'resolveParquetRowCount' as keyof SpatialDataPointsSource
).mockResolvedValue(5_000_000);
+ // Oversized → the byte-oriented feature-column scan, which does not tally
+ // (only the streaming scan and the whole-table read do).
const catalog = await dictSource.listPointsFeatures('points/dict_large');
expect(catalog?.entries).toEqual([
{ code: 0, name: 'gene_a' },
@@ -252,15 +256,15 @@ PY`,
const catalog = await dictSource.listPointsFeatures('points/dict_with_codes');
expect(catalog?.entries).toEqual([
- { code: 0, name: 'TP53' },
- { code: 1, name: 'ABCC11' },
+ { code: 0, name: 'TP53', count: 1 },
+ { code: 1, name: 'ABCC11', count: 2 },
]);
const featureCodes = await dictSource.loadPointsRowFeatureCodes('points/dict_with_codes');
expect([...featureCodes!]).toEqual([1, 0, 1]);
});
- it('omits counts for dictionary-only feature columns without explicit code mapping', async () => {
+ it('counts dictionary-only feature columns from the catalog build, not loadFeatureCounts', async () => {
const elementDir = join(fixtureRoot, 'points', 'dict_counts_untrusted');
await mkdir(elementDir, { recursive: true });
execSync(
@@ -297,14 +301,21 @@ PY`,
},
});
+ // `loadFeatureCounts` still declines: it derives codes independently of the
+ // catalog, and for a dictionary-only element those codes are app-assigned, so
+ // its counts could be keyed to a DIFFERENT code space than the catalog they
+ // would be merged into. That guard stays.
const counts = await dictSource.loadFeatureCounts('points/dict_counts_untrusted');
expect(counts.size).toBe(0);
+ // The catalog build counts as it decodes instead, keyed by the very map it
+ // assigns codes from, so the counts cannot disagree with the entries they sit
+ // on. Rows are ABCC11, TP53, TP53, EGFR.
const catalog = await dictSource.listPointsFeaturesWithCounts('points/dict_counts_untrusted');
expect(catalog?.entries).toEqual([
- { code: 0, name: 'ABCC11' },
- { code: 1, name: 'TP53' },
- { code: 2, name: 'EGFR' },
+ { code: 0, name: 'ABCC11', count: 1 },
+ { code: 1, name: 'TP53', count: 2 },
+ { code: 2, name: 'EGFR', count: 1 },
]);
});
@@ -398,11 +409,12 @@ PY`,
},
});
+ // Counts follow the row codes below: ABCC11 x2, TP53 x3, EGFR x1.
const catalog = await dictSource.listPointsFeatures('points/dict_local_indices');
expect(catalog?.entries).toEqual([
- { code: 0, name: 'ABCC11' },
- { code: 1, name: 'TP53' },
- { code: 2, name: 'EGFR' },
+ { code: 0, name: 'ABCC11', count: 2 },
+ { code: 1, name: 'TP53', count: 3 },
+ { code: 2, name: 'EGFR', count: 1 },
]);
const featureCodes = await dictSource.loadPointsRowFeatureCodes('points/dict_local_indices');
diff --git a/packages/core/tests/pointsMortonScanFilter.spec.ts b/packages/core/tests/pointsMortonScanFilter.spec.ts
new file mode 100644
index 00000000..8504d44c
--- /dev/null
+++ b/packages/core/tests/pointsMortonScanFilter.spec.ts
@@ -0,0 +1,65 @@
+import { tableFromArrays } from 'apache-arrow';
+import { describe, expect, it } from 'vitest';
+import { Float32PointBuffer, scanMortonTableInBounds } from '../src/workers/pointsWorkerScan.js';
+
+/**
+ * The tiled scan filters by feature code using a column it looks up by name. If
+ * that column is not in the decoded chunk the scan cannot honour the filter at
+ * all — and the failure mode matters, because nothing downstream checks.
+ *
+ * Passing every row means one selected gene renders as the whole dataset, in the
+ * selection's colour, with no error logged. Matching nothing is also wrong, but
+ * it shows up as "my gene has no points" rather than as plausible-looking data.
+ */
+
+function scan(columns: Record, over: Record = {}) {
+ const xs = new Float32PointBuffer();
+ const ys = new Float32PointBuffer();
+ const zs = new Float32PointBuffer();
+ scanMortonTableInBounds({
+ table: tableFromArrays(columns as never),
+ rowGroupIndex: 1, // past the sentinel window, so no rows are skipped for it
+ bounds: { minX: -1e6, minY: -1e6, maxX: 1e6, maxY: 1e6 },
+ axisNames: ['x', 'y'],
+ mortonCodeColumnName: 'morton',
+ xs,
+ ys,
+ zs,
+ ...over,
+ } as never);
+ return Array.from(xs.toArray());
+}
+
+const withCodes = {
+ x: Float32Array.from([0, 1, 2, 3]),
+ y: Float32Array.from([0, 1, 2, 3]),
+ morton: Int32Array.from([10, 11, 12, 13]),
+ feature_name_codes: Int32Array.from([0, 1, 0, 2]),
+};
+const withoutCodes = {
+ x: withCodes.x,
+ y: withCodes.y,
+ morton: withCodes.morton,
+};
+
+describe('morton scan — feature filter', () => {
+ it('keeps only the requested codes when the column is present', () => {
+ expect(
+ scan(withCodes, { featureCodeColumnName: 'feature_name_codes', featureCodes: [0] })
+ ).toEqual([0, 2]);
+ });
+
+ it('matches nothing when the filter cannot be honoured', () => {
+ // Column named but absent from the decoded chunk.
+ expect(
+ scan(withoutCodes, { featureCodeColumnName: 'feature_name_codes', featureCodes: [0] })
+ ).toEqual([]);
+ // Filter requested with no column name at all.
+ expect(scan(withoutCodes, { featureCodes: [0] })).toEqual([]);
+ });
+
+ it('still returns every in-bounds row when no filter was requested', () => {
+ expect(scan(withoutCodes)).toEqual([0, 1, 2, 3]);
+ expect(scan(withCodes, { featureCodeColumnName: 'feature_name_codes' })).toEqual([0, 1, 2, 3]);
+ });
+});
diff --git a/packages/core/tests/pointsPreloadStreaming.spec.ts b/packages/core/tests/pointsPreloadStreaming.spec.ts
new file mode 100644
index 00000000..f951d279
--- /dev/null
+++ b/packages/core/tests/pointsPreloadStreaming.spec.ts
@@ -0,0 +1,71 @@
+import { describe, expect, it } from 'vitest';
+import { remapRowFeatureCodes } from '../src/pointsFeatures.js';
+import type { PointsFeatureCatalog } from '../src/pointsTiling.js';
+
+/**
+ * The streaming preload publishes per-row codes in ITS OWN code space (each chunk's
+ * dictionary order) together with the catalog describing that space. The resolver
+ * then re-expresses those codes against the authoritative catalog via
+ * `remapRowFeatureCodes`. These tests pin that handoff, which is what keeps a
+ * point's colour matching the panel once the full scan lands.
+ */
+
+/** Preload-style catalog: dictionary (alphabetical) order. */
+const PRELOAD_CATALOG: PointsFeatureCatalog = {
+ featureKey: 'feature_name',
+ entries: [
+ { code: 0, name: 'ABCC11' },
+ { code: 1, name: 'ACE2' },
+ { code: 2, name: 'ACKR1' },
+ ],
+};
+
+/** Full-scan catalog: row order — deliberately a different code space. */
+const AUTHORITATIVE_CATALOG: PointsFeatureCatalog = {
+ featureKey: 'feature_name',
+ entries: [
+ { code: 0, name: 'ACKR1' },
+ { code: 1, name: 'ABCC11' },
+ { code: 2, name: 'ACE2' },
+ ],
+};
+
+describe('streaming preload code space', () => {
+ it('remaps preload codes into the authoritative space by name', () => {
+ // rows: ABCC11, ACE2, ACKR1, ABCC11 (preload codes)
+ const preloadCodes = Int32Array.from([0, 1, 2, 0]);
+ const remapped = remapRowFeatureCodes(preloadCodes, PRELOAD_CATALOG, AUTHORITATIVE_CATALOG);
+ // same genes, authoritative codes
+ expect(Array.from(remapped)).toEqual([1, 2, 0, 1]);
+ });
+
+ it('preserves the gene each row refers to across the remap', () => {
+ const preloadCodes = Int32Array.from([0, 1, 2, 2, 1, 0]);
+ const preloadName = new Map(PRELOAD_CATALOG.entries.map((e) => [e.code, e.name]));
+ const authoritativeName = new Map(AUTHORITATIVE_CATALOG.entries.map((e) => [e.code, e.name]));
+
+ const remapped = remapRowFeatureCodes(preloadCodes, PRELOAD_CATALOG, AUTHORITATIVE_CATALOG);
+ for (let row = 0; row < preloadCodes.length; row += 1) {
+ expect(authoritativeName.get(remapped[row])).toBe(preloadName.get(preloadCodes[row]));
+ }
+ });
+
+ it('marks genes missing from the authoritative catalog as -1', () => {
+ const partialAuthoritative: PointsFeatureCatalog = {
+ featureKey: 'feature_name',
+ entries: [{ code: 0, name: 'ACE2' }],
+ };
+ const remapped = remapRowFeatureCodes(
+ Int32Array.from([0, 1, 2]),
+ PRELOAD_CATALOG,
+ partialAuthoritative
+ );
+ expect(Array.from(remapped)).toEqual([-1, 0, -1]);
+ });
+
+ it('is an identity pass when both catalogs already agree', () => {
+ const codes = Int32Array.from([2, 0, 1, 1]);
+ const remapped = remapRowFeatureCodes(codes, PRELOAD_CATALOG, PRELOAD_CATALOG);
+ expect(Array.from(remapped)).toEqual(Array.from(codes));
+ });
+});
diff --git a/packages/core/tests/pointsResolver.spec.ts b/packages/core/tests/pointsResolver.spec.ts
index 3781862e..e8c3a168 100644
--- a/packages/core/tests/pointsResolver.spec.ts
+++ b/packages/core/tests/pointsResolver.spec.ts
@@ -76,10 +76,13 @@ describe('plan() — pure, synchronous, starts nothing', () => {
expect(el.loadPointsMatchingFeatureCodes).not.toHaveBeenCalled();
});
- it('plans a preload for a fresh entry', () => {
+ it('plans a preload for a fresh entry, plus rowCodes (colour is on by default)', () => {
const tasks = new PointsResolver().plan(ctx(element()));
- expect(tasks.map((t) => t.resource)).toEqual(['preload']);
+ // Colour-by-feature is on by default, so the per-row codes are planned alongside
+ // the preload — for a code-column dataset they fall out of the preload decode (the
+ // rowCodes task is then a no-op); for a dict-only dataset the task settles them.
+ expect(tasks.map((t) => t.resource)).toEqual(['preload', 'rowCodes']);
});
it('puts the memory cap IN the task id, so a cap change supersedes rather than dedups', () => {
@@ -94,21 +97,30 @@ describe('plan() — pure, synchronous, starts nothing', () => {
expect(at4m?.id).toContain('4000000');
});
- it('plans rowCodes only when a filter or colour-by-feature needs them', () => {
+ it('plans rowCodes by default (colour is on by default), skipping only when colour is off and nothing is selected', () => {
const resolver = new PointsResolver();
const resources = (config: PointsResolveConfig) =>
resolver.plan(ctx(element(), config)).map((t) => t.resource);
- expect(resources({})).not.toContain('rowCodes');
+ // Colour-by-feature is on by default, so the codes load without any explicit flag.
+ expect(resources({})).toContain('rowCodes');
expect(resources({ colorByFeature: true })).toContain('rowCodes');
expect(resources({ featureCodes: [0] })).toContain('rowCodes');
+ // A live filter still needs the codes even with colour explicitly off.
+ expect(resources({ featureCodes: [0], colorByFeature: false })).toContain('rowCodes');
+ // Colour explicitly off AND nothing selected: no code consumer, so skip the load.
+ expect(resources({ colorByFeature: false })).not.toContain('rowCodes');
// An empty selection is "no filter", not "filter to nothing".
- expect(resources({ featureCodes: [] })).not.toContain('rowCodes');
+ expect(resources({ featureCodes: [], colorByFeature: false })).not.toContain('rowCodes');
});
it('plans a matching scan only once the element is known to support one', async () => {
const resolver = new PointsResolver();
- const el = element();
+ // Truncated: rows exist beyond what is resident, so a scan can actually add
+ // something. (A complete batch is covered by the next test.)
+ const el = element({
+ loadPoints: vi.fn(async () => batch(4, { preloadTruncated: true, totalRowCount: 1_000 })),
+ });
const config: PointsResolveConfig = { featureCodes: [0] };
// Before anything loads we cannot know whether a scan is even possible.
@@ -119,6 +131,21 @@ describe('plan() — pure, synchronous, starts nothing', () => {
expect(resolver.plan(ctx(el, config)).map((t) => t.resource)).toContain('matching');
});
+ it('plans no matching scan when the resident batch holds every row', async () => {
+ // A complete preload already contains every matching row, so the render path's
+ // in-memory filter is exact and a whole-dataset scan is pure waste. Scanning
+ // anyway made a selection on a fully-resident element sit on "Loading selected
+ // features… 0 points so far" while it re-read the entire file.
+ const resolver = new PointsResolver();
+ const el = element(); // batch(4), untruncated
+ const config: PointsResolveConfig = { featureCodes: [0] };
+
+ await resolver.ensureLoaded({ key: 'transcripts', layerId: 'layer-p', element: el });
+
+ expect(resolver.plan(ctx(el, config)).map((t) => t.resource)).not.toContain('matching');
+ expect(el.loadPointsMatchingFeatureCodes).not.toHaveBeenCalled();
+ });
+
it('stops planning a preload once one is resident', async () => {
const resolver = new PointsResolver();
const el = element();
@@ -263,7 +290,13 @@ describe('snapshot() — per-resource resolutions, identity-stable', () => {
const snapshot = resolver.snapshot(ctx(el));
expect(Resolution.isReady(snapshot.resources.preload as never)).toBe(true);
- expect(Resolution.readyValue(snapshot.resources.catalog as never)).toBeNull();
+ // A4: a failed full-catalog scan is a retryable `failed`, not a permanent
+ // null-settle — and it must not blank the healthy preload beside it.
+ const catalog = snapshot.resources.catalog;
+ expect(Resolution.isFailed(catalog as never)).toBe(true);
+ if (catalog.status === 'failed') {
+ expect(catalog.error.retryable).toBe(true);
+ }
});
it('carries `stale` through a cap raise, so the old batch keeps drawing', async () => {
@@ -375,3 +408,281 @@ describe('SpatialEntryStore — the reconcile loop', () => {
expect(s.getVersion()).toBeGreaterThan(before);
});
});
+
+describe('Track A — races closed by the slot keys', () => {
+ /** An element whose preload settlements you control per memory cap. */
+ function deferredPreloadElement() {
+ const release = new Map void>();
+ const loadPoints = vi.fn(
+ (opts: { memoryCap: number; signal?: AbortSignal }) =>
+ new Promise((resolve, reject) => {
+ release.set(opts.memoryCap, resolve);
+ opts.signal?.addEventListener('abort', () =>
+ reject(new DOMException('aborted', 'AbortError'))
+ );
+ })
+ );
+ const loadRowFeatureCodes = vi.fn(async () => new Int32Array([0, 1, 0, 1]));
+ const el = {
+ key: 'transcripts',
+ loadPoints,
+ loadRowFeatureCodes,
+ listFeaturesWithCounts: vi.fn(async () => null),
+ } as unknown as PointsElement;
+ return { el, loadPoints, loadRowFeatureCodes, release };
+ }
+
+ const target = (el: PointsElement) => ({ key: 'transcripts', layerId: 'L', element: el });
+
+ it('R1: a cap drag 4M→8M→4M does not wipe the live load, so a redundant request dedups', async () => {
+ // The old bug: superseding 4M→8M→4M left the *first* 4M load's `finally` to run
+ // with `entry.memoryCap === 4M` (the final cap), so it cleared the LIVE final
+ // load's markers. A subsequent 4M request then failed to dedup and kicked a
+ // SECOND concurrent decode. Record-identity supersession forbids this.
+ const resolver = new PointsResolver();
+ const { el, loadPoints, release } = deferredPreloadElement();
+
+ const p4a = resolver.ensureLoaded(target(el), 4_000_000); // decode #1 (4M)
+ const p8 = resolver.ensureLoaded(target(el), 8_000_000); // decode #2 (8M), aborts #1
+ const p4b = resolver.ensureLoaded(target(el), 4_000_000); // decode #3 (4M), aborts #2
+
+ // Let the superseded first 4M load's rejection + continuation run — this is where
+ // the old `finally` wiped the live load's markers.
+ await p4a;
+
+ // A redundant 4M request must dedup to the live decode #3, NOT start a fourth.
+ const p4c = resolver.ensureLoaded(target(el), 4_000_000);
+ expect(loadPoints).toHaveBeenCalledTimes(3);
+
+ release.get(4_000_000)?.(batch(4));
+ await Promise.all([p4b, p4c]);
+ expect(resolver.getData('transcripts')?.shape[1]).toBe(4);
+ await Promise.allSettled([p8]);
+ });
+
+ it('R5: row codes are read at the resident preload cap, not the 4M default', async () => {
+ // The old bug: `ensureRowFeatureCodes` took no cap, so it read 4M rows while an
+ // 8M preload was resident → index i in the codes named a different row than
+ // point i in the batch → a corrupted filter mask. Keying the rowCodes slot on the
+ // preload's cap is the fix.
+ const resolver = new PointsResolver();
+ const { el, loadRowFeatureCodes, release } = deferredPreloadElement();
+
+ // Preload in flight at 8M (pendingKey = 8M).
+ const preload = resolver.ensureLoaded(target(el), 8_000_000);
+ // Filter toggled mid-preload → the codes must be read at the SAME 8M window.
+ await resolver.ensureRowFeatureCodes(target(el));
+
+ expect(loadRowFeatureCodes).toHaveBeenCalledWith(
+ expect.objectContaining({ memoryCap: 8_000_000 })
+ );
+ release.get(8_000_000)?.(batch(8));
+ await preload;
+ });
+
+ /** An element whose feature-index scans you settle per call. */
+ function deferredScanElement() {
+ const calls: Array<{
+ featureCodes: number[];
+ memoryCap: number;
+ resolve: (result: PointsLoadResult) => void;
+ }> = [];
+ const loadPointsMatchingFeatureCodes = vi.fn(
+ (opts: { featureCodes: readonly number[]; memoryCap: number }) =>
+ new Promise((resolve) => {
+ calls.push({ featureCodes: [...opts.featureCodes], memoryCap: opts.memoryCap, resolve });
+ })
+ );
+ const el = {
+ key: 'transcripts',
+ loadPoints: vi.fn(async () => batch(4)), // hasFeatureCodeColumn: true → authoritative
+ loadPointsMatchingFeatureCodes,
+ } as unknown as PointsElement;
+ return { el, loadPointsMatchingFeatureCodes, calls };
+ }
+
+ it('R2: a superseded scan cannot corrupt the reselected one ({0,1}→{2}→{0,1})', async () => {
+ // The old bug: rapid selection changes left two scans with the SAME signature
+ // running concurrently (the first, and the reselected third), both writing the
+ // one shared matchingLoading marker — so the superseded first scan's `finally`
+ // could clobber the live third's result. Record-identity supersession forbids it.
+ const resolver = new PointsResolver();
+ const { el, loadPointsMatchingFeatureCodes, calls } = deferredScanElement();
+ const t = { key: 'transcripts', layerId: 'L', element: el };
+ await resolver.ensureLoaded(t);
+
+ resolver.ensureMatchingFeaturesLoaded(t, [0, 1]); // scan A
+ resolver.ensureMatchingFeaturesLoaded(t, [2]); // scan B (supersedes A)
+ const pC = resolver.ensureMatchingFeaturesLoaded(t, [0, 1]); // scan C (supersedes B)
+ expect(loadPointsMatchingFeatureCodes).toHaveBeenCalledTimes(3);
+
+ const resultA = batch(9, { featureCodes: new Int32Array([0, 1, 0, 1, 0, 1, 0, 1, 0]) });
+ const resultC = batch(3, { featureCodes: new Int32Array([0, 1, 0]) });
+ // The superseded first scan settles FIRST — in the old engine this is where it
+ // wrote resultA over the live scan's marker.
+ calls[0].resolve(resultA);
+ await Promise.resolve();
+ // The live reselected scan settles.
+ calls[2].resolve(resultC);
+ await pC;
+
+ expect(resolver.getMatchedBatch('transcripts')).toBe(resultC);
+ calls[1].resolve(batch(1)); // drain the superseded {2} scan
+ });
+
+ it('R3: raising the cap during a scan supersedes it, not served by the smaller one', async () => {
+ // The old bug: a cap raise for the same selection was "covered" by the in-flight
+ // smaller scan and deduped to it, so the extra rows were never fetched. The cap
+ // is in the slot key, so it supersedes.
+ const resolver = new PointsResolver();
+ const { el, loadPointsMatchingFeatureCodes, calls } = deferredScanElement();
+ const t = { key: 'transcripts', layerId: 'L', element: el };
+ await resolver.ensureLoaded(t, 4_000_000);
+
+ resolver.ensureMatchingFeaturesLoaded(t, [0], 4_000_000); // scan at 4M
+ const p8 = resolver.ensureMatchingFeaturesLoaded(t, [0], 8_000_000); // raise → supersede
+
+ expect(loadPointsMatchingFeatureCodes).toHaveBeenCalledTimes(2);
+ expect(calls[1]?.memoryCap).toBe(8_000_000);
+
+ calls[1].resolve(batch(6));
+ await p8;
+ calls[0].resolve(batch(3)); // drain the superseded 4M scan
+ });
+});
+
+describe('Track A — retryable failures', () => {
+ it('a failed full-catalog scan is retryable, and retry() re-runs it', async () => {
+ const resolver = new PointsResolver();
+ let attempts = 0;
+ const el = element({
+ listFeaturesWithCounts: vi.fn(async () => {
+ attempts += 1;
+ if (attempts === 1) throw new Error('scan failed');
+ return { featureKey: 'feature_name', entries: [{ code: 0, name: 'GeneA' }] };
+ }),
+ });
+ const t = { key: 'transcripts', layerId: 'L', element: el };
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+
+ await resolver.ensureFeatureCatalog(t);
+ const failed = resolver.snapshot(ctx(el)).resources.catalog;
+ expect(Resolution.isFailed(failed as never)).toBe(true);
+ if (failed.status === 'failed') expect(failed.error.retryable).toBe(true);
+ // The old code marked it permanently complete; here the value is simply not loaded.
+ expect(resolver.getFeatureCatalog('transcripts')).toBeUndefined();
+
+ await resolver.retry('transcripts');
+ expect(resolver.getFeatureCatalog('transcripts')).toEqual({
+ featureKey: 'feature_name',
+ entries: [{ code: 0, name: 'GeneA' }],
+ });
+ expect(Resolution.isReady(resolver.snapshot(ctx(el)).resources.catalog as never)).toBe(true);
+ });
+});
+
+describe('Track A — cancellation reaches the scan (D8)', () => {
+ /** An element whose in-flight scan never settles, capturing the signal it sees. */
+ function neverSettlingScanElement() {
+ const signals: AbortSignal[] = [];
+ const el = {
+ key: 'transcripts',
+ loadPoints: vi.fn(async () => batch(4)),
+ loadPointsMatchingFeatureCodes: vi.fn(
+ (opts: { signal?: AbortSignal }) =>
+ new Promise(() => {
+ if (opts.signal) signals.push(opts.signal);
+ })
+ ),
+ } as unknown as PointsElement;
+ return { el, signals };
+ }
+
+ const target = (el: PointsElement) => ({ key: 'transcripts', layerId: 'L', element: el });
+
+ it('supersede aborts the previous scan’s signal — cancellation reaches the element', async () => {
+ const resolver = new PointsResolver();
+ const { el, signals } = neverSettlingScanElement();
+ await resolver.ensureLoaded(target(el));
+
+ resolver.ensureMatchingFeaturesLoaded(target(el), [0]); // scan A
+ expect(signals[0]?.aborted).toBe(false);
+ resolver.ensureMatchingFeaturesLoaded(target(el), [1]); // scan B supersedes A
+ expect(signals[0]?.aborted).toBe(true);
+ });
+
+ it('evict aborts an in-flight scan', async () => {
+ const resolver = new PointsResolver();
+ const { el, signals } = neverSettlingScanElement();
+ await resolver.ensureLoaded(target(el));
+
+ resolver.ensureMatchingFeaturesLoaded(target(el), [0]);
+ expect(signals[0]?.aborted).toBe(false);
+ resolver.evict('transcripts');
+ expect(signals[0]?.aborted).toBe(true);
+ });
+});
+
+describe('progressive preload (D3)', () => {
+ // The fix for "a cold wild-type transcripts load shows nothing for ages": the
+ // preload publishes its growing geometry so the base can paint while the rest
+ // decodes, instead of only after the whole capped window lands.
+ it('exposes the growing geometry as a preload partial, then settles the full batch', async () => {
+ // An element whose loadPoints streams two chunks before resolving.
+ const el = element({
+ loadPoints: vi.fn(
+ async (options: {
+ onProgress?: (p: {
+ scannedRows: number;
+ matchedRows: number;
+ partIndex: number;
+ partCount: number;
+ partialResult: PointsLoadResult;
+ }) => void;
+ }) => {
+ options.onProgress?.({
+ scannedRows: 2,
+ matchedRows: 2,
+ partIndex: 0,
+ partCount: 2,
+ partialResult: batch(2),
+ });
+ options.onProgress?.({
+ scannedRows: 4,
+ matchedRows: 4,
+ partIndex: 1,
+ partCount: 2,
+ partialResult: batch(4),
+ });
+ return batch(4);
+ }
+ ),
+ });
+ const resolver = new PointsResolver();
+ const seen: number[] = [];
+ resolver.subscribe(() => {
+ const partial = resolver.getPreloadPartialBatch('transcripts');
+ if (partial) seen.push(partial.shape[1] ?? 0);
+ });
+
+ const pending = resolver.ensureLoaded({ key: 'transcripts', layerId: 'l', element: el });
+ // Partials are published while the load is still in flight — that IS the feature.
+ expect(resolver.getPreloadPartialBatch('transcripts')?.shape[1]).toBe(4);
+ await pending;
+
+ // Once settled, the resident batch takes over and equals the one-shot result.
+ expect(resolver.getData('transcripts')?.shape[1]).toBe(4);
+ // At least one growing partial was observed before the settle.
+ expect(seen.length).toBeGreaterThan(0);
+ });
+
+ it('passes an onProgress through to the element so streaming can happen at all', async () => {
+ const el = element();
+ const resolver = new PointsResolver();
+ await resolver.ensureLoaded({ key: 'transcripts', layerId: 'l', element: el });
+
+ expect(el.loadPoints).toHaveBeenCalledWith(
+ expect.objectContaining({ onProgress: expect.any(Function) })
+ );
+ });
+});
diff --git a/packages/core/tests/pointsRowCodesCapAlignment.spec.ts b/packages/core/tests/pointsRowCodesCapAlignment.spec.ts
new file mode 100644
index 00000000..37002d6e
--- /dev/null
+++ b/packages/core/tests/pointsRowCodesCapAlignment.spec.ts
@@ -0,0 +1,170 @@
+import { Matrix4 } from '@math.gl/core';
+import { describe, expect, it, vi } from 'vitest';
+import {
+ type PointsResolveConfig,
+ PointsResolver,
+ type ResolveContext,
+} from '../src/engine/index.js';
+import type { PointsElement } from '../src/models/index.js';
+import type { PointsLoadResult } from '../src/pointsLoadOptions.js';
+
+/**
+ * R5 says the row codes are only a valid mask for the resident batch when both
+ * were read at the SAME memory cap — index i names point i only then. The slot key
+ * carries the cap so that misalignment is representable and therefore checkable.
+ *
+ * The planning gate did not check it. `hasRowFeatureCodes` is `isReady`, which
+ * stays true after a cap raise, so codes settled at the old cap were never
+ * re-requested and silently addressed the wrong rows of the bigger batch. This is
+ * the R5 misalignment surviving in the one place that decides whether to fix it.
+ *
+ * Only reachable when the preload does NOT carry codes itself — a dict-only
+ * element, whose codes come from the `loadRowFeatureCodes` fallback. When the
+ * decode supplies them it re-settles them at its own cap for free.
+ */
+
+const SMALL_CAP = 4;
+const LARGE_CAP = 8;
+const TOTAL_ROWS = 8;
+
+/** A dict-only preload: geometry, no `featureCodes`, truncated so a raise reloads. */
+function codelessBatch(rows: number): PointsLoadResult {
+ return {
+ shape: [2, rows],
+ data: [new Float32Array(rows), new Float32Array(rows)],
+ hasFeatureCodeColumn: false,
+ preloadTruncated: rows < TOTAL_ROWS,
+ totalRowCount: TOTAL_ROWS,
+ };
+}
+
+function dictOnlyElement() {
+ return {
+ key: 'transcripts',
+ loadPoints: vi.fn(async ({ memoryCap }: { memoryCap: number }) =>
+ codelessBatch(Math.min(TOTAL_ROWS, memoryCap))
+ ),
+ listFeaturesWithCounts: vi.fn(async () => null),
+ // Row-aligned with whatever window it is asked for — the alignment contract.
+ loadRowFeatureCodes: vi.fn(
+ async ({ memoryCap }: { memoryCap: number }) =>
+ new Int32Array(Math.min(TOTAL_ROWS, memoryCap))
+ ),
+ loadPointsMatchingFeatureCodes: vi.fn(async () => codelessBatch(1)),
+ } as unknown as PointsElement;
+}
+
+const ctx = (
+ el: PointsElement,
+ config: PointsResolveConfig = {}
+): ResolveContext => ({
+ entryId: 'layer-p',
+ elementKey: 'transcripts',
+ kind: 'points',
+ element: el,
+ config,
+ transform: new Matrix4(),
+});
+
+const signal = () => new AbortController().signal;
+
+const rowCodesTasks = (resolver: PointsResolver, el: PointsElement, cap: number) =>
+ resolver.plan(ctx(el, { pointsMemoryCap: cap })).filter((task) => task.resource === 'rowCodes');
+
+describe('row codes — cap alignment with the resident batch', () => {
+ it('re-plans the codes after a cap raise leaves them at the old window', async () => {
+ const resolver = new PointsResolver();
+ const el = dictOnlyElement();
+
+ await resolver.load(
+ { id: 'p', resource: 'preload', payload: { memoryCap: SMALL_CAP } },
+ ctx(el),
+ signal()
+ );
+ await resolver.load({ id: 'r', resource: 'rowCodes' }, ctx(el), signal());
+
+ expect(resolver.getRowFeatureCodes('transcripts')?.length).toBe(SMALL_CAP);
+ expect(resolver.hasRowFeatureCodesAtCap('transcripts', SMALL_CAP)).toBe(true);
+ // Aligned at the resident cap: nothing to do.
+ expect(rowCodesTasks(resolver, el, SMALL_CAP)).toEqual([]);
+
+ // Raise the cap and let the bigger preload settle. The codes are now a 4-row
+ // mask over an 8-row batch.
+ await resolver.load(
+ { id: 'p2', resource: 'preload', payload: { memoryCap: LARGE_CAP } },
+ ctx(el),
+ signal()
+ );
+ expect(resolver.getData('transcripts')?.shape[1]).toBe(LARGE_CAP);
+ expect(resolver.hasRowFeatureCodesAtCap('transcripts', LARGE_CAP)).toBe(false);
+
+ // The gate must notice. `hasRowFeatureCodes` is still true, which is exactly
+ // why it could not be the gate.
+ expect(resolver.hasRowFeatureCodes('transcripts')).toBe(true);
+ const tasks = rowCodesTasks(resolver, el, LARGE_CAP);
+ expect(tasks).toHaveLength(1);
+ // The cap is in the id, so a cap change re-dispatches instead of deduping
+ // against the task that already ran at the smaller window.
+ expect(tasks[0]?.id).toContain(String(LARGE_CAP));
+
+ await resolver.load(tasks[0] as never, ctx(el, { pointsMemoryCap: LARGE_CAP }), signal());
+ expect(resolver.getRowFeatureCodes('transcripts')?.length).toBe(LARGE_CAP);
+ expect(rowCodesTasks(resolver, el, LARGE_CAP)).toEqual([]);
+ });
+
+ it('waits for an in-flight preload rather than racing it for the same column', async () => {
+ const resolver = new PointsResolver();
+ const el = dictOnlyElement();
+
+ await resolver.load(
+ { id: 'p', resource: 'preload', payload: { memoryCap: SMALL_CAP } },
+ ctx(el),
+ signal()
+ );
+ await resolver.load({ id: 'r', resource: 'rowCodes' }, ctx(el), signal());
+ const readsBefore = (el.loadRowFeatureCodes as ReturnType).mock.calls.length;
+
+ // Start the larger preload WITHOUT awaiting it.
+ let release = (): void => {};
+ const gate = new Promise((resolve) => {
+ release = resolve;
+ });
+ (el.loadPoints as ReturnType).mockImplementationOnce(async () => {
+ await gate;
+ return codelessBatch(LARGE_CAP);
+ });
+ const inFlight = resolver.load(
+ { id: 'p2', resource: 'preload', payload: { memoryCap: LARGE_CAP } },
+ ctx(el),
+ signal()
+ );
+
+ // Codes are stale, but the decode may be about to supply them at the new cap.
+ // Asking now would read the whole feature column a second time in parallel.
+ expect(rowCodesTasks(resolver, el, LARGE_CAP)).toEqual([]);
+
+ release();
+ await inFlight;
+ expect((el.loadRowFeatureCodes as ReturnType).mock.calls.length).toBe(readsBefore);
+ // It did NOT supply them (dict-only), so now the gate asks.
+ expect(rowCodesTasks(resolver, el, LARGE_CAP)).toHaveLength(1);
+ });
+
+ it('still plans the codes on a first load, while the preload is in flight', async () => {
+ // The defer above is only for codes that already exist at a stale cap. With no
+ // codes at all there is nothing to lose by asking, and waiting would delay
+ // colour on every cold load.
+ const resolver = new PointsResolver();
+ const el = dictOnlyElement();
+ (el.loadPoints as ReturnType).mockImplementationOnce(
+ () => new Promise(() => {})
+ );
+ void resolver.load(
+ { id: 'p', resource: 'preload', payload: { memoryCap: SMALL_CAP } },
+ ctx(el),
+ signal()
+ );
+
+ expect(rowCodesTasks(resolver, el, SMALL_CAP)).toHaveLength(1);
+ });
+});
diff --git a/packages/core/tests/pointsRowFeatureCodes.spec.ts b/packages/core/tests/pointsRowFeatureCodes.spec.ts
new file mode 100644
index 00000000..fdc3a69f
--- /dev/null
+++ b/packages/core/tests/pointsRowFeatureCodes.spec.ts
@@ -0,0 +1,120 @@
+import { Dictionary, Int16, tableFromArrays, Utf8, Vector, vectorFromArray } from 'apache-arrow';
+import { describe, expect, it } from 'vitest';
+import { resolveRowFeatureCodesFromTable } from '../src/pointsFeatures.js';
+
+const FEATURE_KEY = 'feature_name';
+
+function dictionaryTable(names: string[]) {
+ return tableFromArrays({
+ [FEATURE_KEY]: vectorFromArray(names, new Dictionary(new Utf8(), new Int16())),
+ });
+}
+
+/** Reference implementation: the per-row form this used to use. */
+function expectedCodes(names: (string | null)[], codeByName: Map) {
+ return names.map((name) => (name == null ? -1 : (codeByName.get(name) ?? -1)));
+}
+
+describe('resolveRowFeatureCodesFromTable', () => {
+ it('maps a dictionary column through its dictionary', () => {
+ const names = ['GENE_A', 'GENE_B', 'GENE_A', 'GENE_C', 'GENE_B'];
+ const codeByName = new Map([
+ ['GENE_A', 0],
+ ['GENE_B', 1],
+ ['GENE_C', 2],
+ ]);
+ const codes = resolveRowFeatureCodesFromTable(
+ dictionaryTable(names),
+ FEATURE_KEY,
+ undefined,
+ codeByName
+ );
+ expect(Array.from(codes as Int32Array)).toEqual(expectedCodes(names, codeByName));
+ });
+
+ it('gives -1 to names absent from the map', () => {
+ const names = ['GENE_A', 'MISSING', 'GENE_B'];
+ const codeByName = new Map([
+ ['GENE_A', 7],
+ ['GENE_B', 9],
+ ]);
+ const codes = resolveRowFeatureCodesFromTable(
+ dictionaryTable(names),
+ FEATURE_KEY,
+ undefined,
+ codeByName
+ );
+ expect(Array.from(codes as Int32Array)).toEqual([7, -1, 9]);
+ });
+
+ it('handles nulls in a dictionary column', () => {
+ const names = ['GENE_A', null, 'GENE_B'];
+ const codeByName = new Map([
+ ['GENE_A', 0],
+ ['GENE_B', 1],
+ ]);
+ const codes = resolveRowFeatureCodesFromTable(
+ // biome-ignore lint/suspicious/noExplicitAny: mixed null/string literal for the fixture
+ dictionaryTable(names as any),
+ FEATURE_KEY,
+ undefined,
+ codeByName
+ );
+ expect(Array.from(codes as Int32Array)).toEqual([0, -1, 1]);
+ });
+
+ it('handles a plain (non-dictionary) utf8 column', () => {
+ const names = ['GENE_A', 'GENE_B', 'GENE_A'];
+ const codeByName = new Map([
+ ['GENE_A', 3],
+ ['GENE_B', 4],
+ ]);
+ const table = tableFromArrays({ [FEATURE_KEY]: names });
+ const codes = resolveRowFeatureCodesFromTable(table, FEATURE_KEY, undefined, codeByName);
+ expect(Array.from(codes as Int32Array)).toEqual([3, 4, 3]);
+ });
+
+ it('prefers an explicit feature-code column when present', () => {
+ const table = tableFromArrays({
+ [FEATURE_KEY]: ['GENE_A', 'GENE_B'],
+ feature_name_codes: Int32Array.from([11, 22]),
+ });
+ const codes = resolveRowFeatureCodesFromTable(
+ table,
+ FEATURE_KEY,
+ 'feature_name_codes',
+ new Map()
+ );
+ expect(Array.from(codes as ArrayLike)).toEqual([11, 22]);
+ });
+
+ it('resolves each chunk against its OWN dictionary', () => {
+ // Parquet gives every column chunk its own dictionary, so index 0 in one
+ // chunk need not be the same gene as index 0 in the next — a real 4M-row
+ // transcripts column arrives as thousands of such chunks. Reading only the
+ // first chunk's dictionary (as the old helper did) mislabels later rows.
+ const chunkA = vectorFromArray(
+ ['GENE_A', 'GENE_B'],
+ new Dictionary(new Utf8(), new Int16(), 0)
+ );
+ const chunkB = vectorFromArray(
+ ['GENE_C', 'GENE_D'],
+ new Dictionary(new Utf8(), new Int16(), 1)
+ );
+ const column = new Vector([...chunkA.data, ...chunkB.data]);
+ expect(column.data.length).toBe(2);
+
+ const codeByName = new Map([
+ ['GENE_A', 0],
+ ['GENE_B', 1],
+ ['GENE_C', 2],
+ ['GENE_D', 3],
+ ]);
+ const table = {
+ numRows: 4,
+ getChild: (name: string) => (name === FEATURE_KEY ? column : null),
+ } as unknown as Parameters[0];
+ const codes = resolveRowFeatureCodesFromTable(table, FEATURE_KEY, undefined, codeByName);
+ expect(Array.from(codes as Int32Array)).toEqual([0, 1, 2, 3]);
+ });
+});
diff --git a/packages/core/tests/pointsScanStreamGate.spec.ts b/packages/core/tests/pointsScanStreamGate.spec.ts
new file mode 100644
index 00000000..07f8da6d
--- /dev/null
+++ b/packages/core/tests/pointsScanStreamGate.spec.ts
@@ -0,0 +1,96 @@
+import { describe, expect, it, vi } from 'vitest';
+import SpatialDataPointsSource from '../src/models/VPointsSource.js';
+
+/**
+ * The feature scan prefers `ParquetFile.stream({ columns, rowGroups })`, whose
+ * reader fetches per COLUMN CHUNK — the point being that the projection reaches
+ * the network instead of pulling whole row groups (all 12 columns of a Xenium
+ * `transcripts`) to use three.
+ *
+ * That reader needs a fetchable URL and a server that answers the range shapes it
+ * expects, so it is a fast path and not a replacement. These pin the GATE: every
+ * store the reader cannot serve has to fall through to the byte-oriented worker
+ * path, because a wrong answer here does not fail loudly — it either fetches
+ * nothing (a gene that renders no points) or silently keeps the slow path forever.
+ */
+function sourceWithStore(store: unknown) {
+ return new SpatialDataPointsSource({ store, fileType: '.zarr' } as never);
+}
+
+type Internals = {
+ canStreamMatchingScan: (
+ path: string
+ ) => Promise<{ urls: string[]; rowGroupCounts: number[] } | null>;
+ canStreamParquetByUrl: () => Promise;
+ loadParquetDatasetMetadata: (path: string) => Promise;
+ resolveStoreUrl: (path: string) => string | null;
+ serverSupportsStreamingRanges: (url: string) => Promise;
+};
+
+const parquetPath = 'points/transcripts/points.parquet';
+const twoParts = {
+ parts: [{ path: `${parquetPath}/part.0.parquet` }, { path: `${parquetPath}/part.1.parquet` }],
+ // Row-group counts ride along so the worker path can window its requests.
+ numRowGroupsByPart: [3, 2],
+};
+
+function harness(over: Partial = {}) {
+ const source = sourceWithStore({ async get() {}, async getRange() {} });
+ const internals = source as unknown as Internals;
+ vi.spyOn(internals, 'canStreamParquetByUrl').mockResolvedValue(true);
+ vi.spyOn(internals, 'loadParquetDatasetMetadata').mockResolvedValue(twoParts);
+ vi.spyOn(internals, 'resolveStoreUrl').mockImplementation(
+ (path: string) => `http://example.test/${path}`
+ );
+ vi.spyOn(internals, 'serverSupportsStreamingRanges').mockResolvedValue(true);
+ for (const [key, value] of Object.entries(over)) {
+ vi.spyOn(internals, key as keyof Internals).mockImplementation(value as never);
+ }
+ return internals;
+}
+
+describe('feature scan — streaming gate', () => {
+ it('streams when every part resolves to a range-capable URL', async () => {
+ const internals = harness();
+ await expect(internals.canStreamMatchingScan(parquetPath)).resolves.toEqual({
+ urls: [
+ `http://example.test/${parquetPath}/part.0.parquet`,
+ `http://example.test/${parquetPath}/part.1.parquet`,
+ ],
+ rowGroupCounts: [3, 2],
+ });
+ });
+
+ it('declines when the reader is unavailable in this runtime', async () => {
+ // `supportsParquetStreaming()` is false off the browser main thread — notably
+ // inside the points worker, and in Node.
+ const internals = harness({ canStreamParquetByUrl: async () => false });
+ await expect(internals.canStreamMatchingScan(parquetPath)).resolves.toBeNull();
+ });
+
+ it('declines for a non-URL store, which the reader cannot fetch at all', async () => {
+ const internals = harness({ resolveStoreUrl: () => null });
+ await expect(internals.canStreamMatchingScan(parquetPath)).resolves.toBeNull();
+ });
+
+ it('declines when the server refuses the range shapes the reader needs', async () => {
+ // A server that 416s suffix ranges makes the reader trap with an unsettleable
+ // promise, so this must be decided BEFORE handing it a URL.
+ const internals = harness({ serverSupportsStreamingRanges: async () => false });
+ await expect(internals.canStreamMatchingScan(parquetPath)).resolves.toBeNull();
+ });
+
+ it('declines when even one part of a multipart element is unservable', async () => {
+ // Mixed capability would otherwise scan some parts and silently skip others —
+ // a partial gene rather than a failure.
+ const internals = harness({
+ serverSupportsStreamingRanges: async (url: string) => !url.endsWith('part.1.parquet'),
+ });
+ await expect(internals.canStreamMatchingScan(parquetPath)).resolves.toBeNull();
+ });
+
+ it('declines when the part layout cannot be resolved', async () => {
+ const internals = harness({ loadParquetDatasetMetadata: async () => null });
+ await expect(internals.canStreamMatchingScan(parquetPath)).resolves.toBeNull();
+ });
+});
diff --git a/packages/core/tests/pointsWorkerScan.spec.ts b/packages/core/tests/pointsWorkerScan.spec.ts
index 0018ff90..88b6dbde 100644
--- a/packages/core/tests/pointsWorkerScan.spec.ts
+++ b/packages/core/tests/pointsWorkerScan.spec.ts
@@ -5,6 +5,8 @@ import {
decodeParquetRowGroupsToTable,
extractGeometryColumnar,
extractRowFeatureCodesFromTable,
+ Float32PointBuffer,
+ Int32PointBuffer,
scanFeatureCatalogFromPayload,
scanTableByFeatureCodes,
} from '../src/workers/pointsWorkerScan.js';
@@ -154,9 +156,9 @@ describe('scanTableByFeatureCodes with featureCodeByName (dict-only)', () => {
['gene_b', 1],
['gene_c', 2],
]);
- const xs: number[] = [];
- const ys: number[] = [];
- const codes: number[] = [];
+ const xs = new Float32PointBuffer();
+ const ys = new Float32PointBuffer();
+ const codes = new Int32PointBuffer();
const matched = scanTableByFeatureCodes({
table,
axisNames: ['x', 'y'],
@@ -167,14 +169,14 @@ describe('scanTableByFeatureCodes with featureCodeByName (dict-only)', () => {
matchedRows: 0,
xs,
ys,
- zs: [],
+ zs: new Float32PointBuffer(),
codes,
featureCodeByName,
});
expect(matched).toBe(2);
- expect(xs).toEqual([11, 13]); // the two gene_c rows
- expect(ys).toEqual([21, 23]);
- expect(codes).toEqual([2, 2]); // authoritative codes retained
+ expect([...xs.toArray()]).toEqual([11, 13]); // the two gene_c rows
+ expect([...ys.toArray()]).toEqual([21, 23]);
+ expect([...codes.toArray()]).toEqual([2, 2]); // authoritative codes retained
});
it('matches nothing when no name→code map is supplied for dict-only data', () => {
@@ -183,7 +185,7 @@ describe('scanTableByFeatureCodes with featureCodeByName (dict-only)', () => {
y: Float32Array.from([20, 21]),
feature_name: ['gene_a', 'gene_c'],
});
- const xs: number[] = [];
+ const xs = new Float32PointBuffer();
const matched = scanTableByFeatureCodes({
table,
axisNames: ['x', 'y'],
@@ -193,11 +195,11 @@ describe('scanTableByFeatureCodes with featureCodeByName (dict-only)', () => {
memoryCap: 1_000,
matchedRows: 0,
xs,
- ys: [],
- zs: [],
+ ys: new Float32PointBuffer(),
+ zs: new Float32PointBuffer(),
});
expect(matched).toBe(0);
- expect(xs).toEqual([]);
+ expect([...xs.toArray()]).toEqual([]);
});
});
@@ -252,3 +254,126 @@ describe('scanFeatureCatalogFromPayload', () => {
]);
});
});
+
+/**
+ * The scan reads coordinates by row index. `Vector.get(i)` makes that look like an
+ * array read, but on a MULTI-CHUNK vector — any table assembled from more than one
+ * record batch, which is the normal case for a multi-row-group or multi-part read —
+ * Arrow swaps in a `binarySearch` over chunk offsets per call. The scan now hoists
+ * each column into one flat typed array instead.
+ *
+ * These pin the thing that rewrite could plausibly break: chunk-boundary indexing.
+ * A row must resolve to the same coordinate whether the table arrived as one chunk
+ * or several, and matches must still be found in the later chunks.
+ */
+describe('scanTableByFeatureCodes over a multi-chunk table', () => {
+ const scan = (table: Parameters[0]['table']) => {
+ const xs = new Float32PointBuffer();
+ const ys = new Float32PointBuffer();
+ const codes = new Int32PointBuffer();
+ const matched = scanTableByFeatureCodes({
+ table,
+ axisNames: ['x', 'y'],
+ featureKey: 'feature_name',
+ featureCodeColumnName: 'feature_name_codes',
+ featureCodes: [1],
+ memoryCap: 1_000,
+ matchedRows: 0,
+ xs,
+ ys,
+ zs: new Float32PointBuffer(),
+ codes,
+ });
+ // Compare as plain arrays: the scan writes into typed buffers now.
+ return { matched, xs: [...xs.toArray()], ys: [...ys.toArray()], codes: [...codes.toArray()] };
+ };
+
+ const rows = (start: number, count: number) => ({
+ x: Float32Array.from({ length: count }, (_v, i) => start + i),
+ y: Float32Array.from({ length: count }, (_v, i) => 100 + start + i),
+ // Alternating codes, so every chunk holds both matches and non-matches.
+ feature_name: Array.from({ length: count }, (_v, i) => `gene_${(start + i) % 2}`),
+ feature_name_codes: Int32Array.from({ length: count }, (_v, i) => (start + i) % 2),
+ });
+
+ it('reads the same rows from a chunked table as from a contiguous one', () => {
+ const whole = tableFromArrays(rows(0, 40) as never);
+ // One record batch per source table — the shape a multi-row-group or
+ // multi-part read produces, and what makes the columns multi-chunk.
+ const chunked = tableFromArrays(rows(0, 20) as never).concat(
+ tableFromArrays(rows(20, 20) as never)
+ );
+ expect(chunked.numRows).toBe(40);
+ // Guard the premise: if this were single-chunk the test would prove nothing.
+ expect(chunked.getChild('x')?.data.length).toBeGreaterThan(1);
+
+ const fromWhole = scan(whole);
+ const fromChunked = scan(chunked);
+
+ expect(fromChunked.matched).toBe(fromWhole.matched);
+ expect(fromChunked.xs).toEqual(fromWhole.xs);
+ expect(fromChunked.ys).toEqual(fromWhole.ys);
+ expect(fromChunked.codes).toEqual(fromWhole.codes);
+ // And the values are actually right, not merely equal to each other: odd x.
+ expect(fromChunked.xs).toEqual(Array.from({ length: 20 }, (_v, i) => i * 2 + 1));
+ // Matches from the SECOND chunk are present — the chunk-offset bug would drop
+ // or misread these.
+ expect(fromChunked.xs.at(-1)).toBe(39);
+ expect(fromChunked.ys.at(-1)).toBe(139);
+ });
+});
+
+/**
+ * The scans reserve an exact upper bound per chunk so pushes never reallocate.
+ * A hint is only a hint though — it can be absent, low, or (via bounds rejection)
+ * far too high — so growth has to stay correct. A buffer that silently truncated
+ * or mis-sized here would drop points with no error anywhere.
+ */
+describe('Float32PointBuffer', () => {
+ it('grows correctly with no reservation at all', () => {
+ const buffer = new Float32PointBuffer();
+ for (let i = 0; i < 5000; i += 1) {
+ buffer.push(i);
+ }
+ const out = buffer.toArray();
+ expect(out.length).toBe(5000);
+ expect(out[0]).toBe(0);
+ expect(out[4999]).toBe(4999);
+ });
+
+ it('grows past an under-estimate without losing earlier values', () => {
+ const buffer = new Float32PointBuffer();
+ buffer.reserve(4);
+ for (let i = 0; i < 100; i += 1) {
+ buffer.push(i);
+ }
+ expect([...buffer.toArray()]).toEqual(Array.from({ length: 100 }, (_v, i) => i));
+ });
+
+ it('trims an over-estimate to the filled prefix', () => {
+ const buffer = new Float32PointBuffer();
+ buffer.reserve(10_000); // e.g. a row group whose rows nearly all fail the bounds test
+ buffer.push(1);
+ buffer.push(2);
+ const out = buffer.toArray();
+ expect(out.length).toBe(2);
+ expect([...out]).toEqual([1, 2]);
+ });
+
+ it('accumulates across successive reservations, as a multi-chunk scan does', () => {
+ const buffer = new Float32PointBuffer();
+ buffer.reserve(3);
+ for (const value of [1, 2, 3]) buffer.push(value);
+ buffer.reserve(3); // next chunk: reserve is ADDITIONAL headroom, not a reset
+ for (const value of [4, 5, 6]) buffer.push(value);
+ expect([...buffer.toArray()]).toEqual([1, 2, 3, 4, 5, 6]);
+ });
+
+ it('keeps Int32 codes exact (no float rounding of large codes)', () => {
+ const buffer = new Int32PointBuffer();
+ buffer.reserve(2);
+ buffer.push(16_777_217); // not representable in float32
+ buffer.push(-1);
+ expect([...buffer.toArray()]).toEqual([16_777_217, -1]);
+ });
+});
diff --git a/packages/core/tests/requestSlot.spec.ts b/packages/core/tests/requestSlot.spec.ts
new file mode 100644
index 00000000..caf769ad
--- /dev/null
+++ b/packages/core/tests/requestSlot.spec.ts
@@ -0,0 +1,262 @@
+import { describe, expect, it, vi } from 'vitest';
+import type { SpatialEntryErrorContext } from '../src/engine/errors.js';
+import { RequestSlot, type SlotLoadContext } from '../src/engine/RequestSlot.js';
+
+/**
+ * `RequestSlot`, driven headless.
+ *
+ * This is the primitive Track A hangs the four points slots off; the two rules it
+ * must uphold — supersession by record identity, and "everything the request
+ * depends on lives in K" — are exactly what closes races R1/R2/R3/R5 once the
+ * slots consume it. Those rules are pinned here, at the primitive, so the per-slot
+ * race specs in A2/A3 only have to prove the *keys* are right.
+ */
+
+const context: SpatialEntryErrorContext = {
+ elementKey: 'transcripts',
+ kind: 'points',
+ resource: 'test',
+ fallback: 'load-failed',
+};
+
+const slot = (
+ over: Partial<{ equals: (a: K, b: K) => boolean; onChange: () => void }> = {}
+) => new RequestSlot({ context, ...over });
+
+/** A loader whose settlement you control, so two loads can be interleaved. */
+function deferred() {
+ let resolve!: (value: V) => void;
+ let reject!: (cause: unknown) => void;
+ const promise = new Promise((res, rej) => {
+ resolve = res;
+ reject = rej;
+ });
+ return { promise, resolve, reject };
+}
+
+describe('initial state', () => {
+ it('is idle, with no value and no pending', () => {
+ const s = slot();
+ expect(s.resolution.status).toBe('idle');
+ expect(s.value).toBeUndefined();
+ expect(s.pending).toBeUndefined();
+ });
+});
+
+describe('request → loading → ready', () => {
+ it('goes loading then ready, and calls the loader once', async () => {
+ const s = slot();
+ const loader = vi.fn(async () => 'v');
+ const p = s.request(4, loader);
+ expect(s.isLoading).toBe(true);
+ await p;
+ expect(s.isReady).toBe(true);
+ expect(s.value).toBe('v');
+ expect(s.settledKey).toBe(4);
+ expect(loader).toHaveBeenCalledTimes(1);
+ });
+
+ it('notifies onChange for each transition', async () => {
+ const onChange = vi.fn();
+ const s = slot({ onChange });
+ await s.request(4, async () => 'v');
+ // at least loading + ready
+ expect(onChange.mock.calls.length).toBeGreaterThanOrEqual(2);
+ });
+
+ it('with notifyOnLoading:false, a clean load notifies once (settle only)', async () => {
+ const onChange = vi.fn();
+ const s = new RequestSlot({ context, notifyOnLoading: false, onChange });
+ await s.request(4, async () => 'v');
+ // loading-start is quiet; only the settle fires. This is what keeps the
+ // preload/rowCodes notify counts identical to the pre-slot engine.
+ expect(onChange).toHaveBeenCalledTimes(1);
+ });
+});
+
+describe('dedup by key', () => {
+ it('a second request with the same key returns the same in-flight promise and does not re-run', () => {
+ const s = slot();
+ const loader = vi.fn(async () => 'v');
+ const first = s.request(4, loader);
+ const second = s.request(4, loader);
+ expect(second).toBe(first);
+ expect(loader).toHaveBeenCalledTimes(1);
+ });
+
+ it('a request for an already-ready key is a no-op', async () => {
+ const s = slot();
+ const loader = vi.fn(async () => 'v');
+ await s.request(4, loader);
+ await s.request(4, loader);
+ expect(loader).toHaveBeenCalledTimes(1);
+ });
+});
+
+describe('supersession by record identity (R1/R2 essence)', () => {
+ it('a superseded load cannot write its result even if it settles last', async () => {
+ const s = slot();
+ const first = deferred();
+ const second = deferred();
+ const firstSignals: AbortSignal[] = [];
+
+ s.request(4, (ctx: SlotLoadContext) => {
+ firstSignals.push(ctx.signal);
+ return first.promise;
+ });
+ // Different key supersedes; the first record is now stale.
+ const p2 = s.request(8, () => second.promise);
+
+ // Second settles first, then the superseded first settles late.
+ second.resolve('eight');
+ await p2;
+ expect(s.value).toBe('eight');
+
+ first.resolve('four'); // must be dropped — this record is not current
+ await Promise.resolve();
+ await Promise.resolve();
+ expect(s.value).toBe('eight');
+ // The superseded load's signal was aborted.
+ expect(firstSignals[0]?.aborted).toBe(true);
+ });
+
+ it('re-requesting the same key while a superseding load is in flight dedups to the live one', async () => {
+ const s = slot();
+ const a = deferred();
+ const b = deferred();
+ s.request('sigA', () => a.promise);
+ const pB = s.request('sigB', () => b.promise); // supersede
+ const pB2 = s.request('sigB', () => b.promise); // R2: same signature → dedup, no 2nd scan
+ expect(pB2).toBe(pB);
+ });
+});
+
+describe('stale retention', () => {
+ it('keeps the previous ready value as stale while a supersede loads', async () => {
+ const s = slot();
+ await s.request(4, async () => 'first');
+ const next = deferred();
+ s.request(8, () => next.promise);
+ expect(s.isLoading).toBe(true);
+ expect(s.value).toBeUndefined(); // ready value gone while loading
+ expect(s.lastGood).toBe('first'); // but still drawable via stale
+ next.resolve('second');
+ await s.pending;
+ expect(s.value).toBe('second');
+ });
+});
+
+describe('failure and retry', () => {
+ it('classifies a thrown error as failed + retryable, keeping stale', async () => {
+ const s = slot();
+ await s.request(4, async () => 'good');
+ await s.request(8, async () => {
+ throw new Error('decode boom');
+ });
+ expect(s.isFailed).toBe(true);
+ const r = s.resolution;
+ if (r.status !== 'failed') throw new Error('expected failed');
+ expect(r.error.retryable).toBe(true);
+ expect(r.error.kind).toBe('load-failed');
+ expect(r.stale).toBe('good'); // previous value retained
+ });
+
+ it('retry() re-runs the last loader and can reach ready', async () => {
+ const s = slot();
+ let attempts = 0;
+ const loader = async () => {
+ attempts += 1;
+ if (attempts === 1) throw new Error('transient');
+ return 'recovered';
+ };
+ await s.request(8, loader);
+ expect(s.isFailed).toBe(true);
+ await s.retry();
+ expect(s.isReady).toBe(true);
+ expect(s.value).toBe('recovered');
+ expect(attempts).toBe(2);
+ });
+});
+
+describe('cancellation is a non-event', () => {
+ it('an AbortError reverts to the last good value, not an error', async () => {
+ const s = slot();
+ await s.request(4, async () => 'good');
+ const gate = deferred();
+ const p = s.request(8, (ctx: SlotLoadContext) => {
+ ctx.signal.addEventListener('abort', () => {
+ gate.reject(new DOMException('aborted', 'AbortError'));
+ });
+ return gate.promise;
+ });
+ // Reset aborts the in-flight load.
+ s.reset();
+ await p.catch(() => undefined);
+ expect(s.isFailed).toBe(false);
+ });
+});
+
+describe('streaming partials', () => {
+ it('emit publishes partial + progress while loading, and is ignored after supersede', async () => {
+ const s = slot();
+ let capturedEmit!: SlotLoadContext['emit'];
+ const gate = deferred();
+ s.request('scan', (ctx) => {
+ capturedEmit = ctx.emit;
+ return gate.promise;
+ });
+ capturedEmit([1, 2], { done: 2, scanned: 10 });
+ expect(s.partial).toEqual([1, 2]);
+ const r = s.resolution;
+ if (r.status !== 'loading') throw new Error('expected loading');
+ expect(r.progress).toEqual({ done: 2, scanned: 10 });
+
+ // Supersede, then a late emit from the old scan must be dropped.
+ s.request('scan2', () => deferred().promise);
+ capturedEmit([1, 2, 3]);
+ expect(s.partial).toBeUndefined(); // new loading has no partial yet
+ });
+
+ it('a silent emit updates the partial without notifying', async () => {
+ const onChange = vi.fn();
+ const s = new RequestSlot({ context, onChange });
+ let emit!: SlotLoadContext['emit'];
+ s.request('scan', (ctx) => {
+ emit = ctx.emit;
+ return deferred().promise;
+ });
+ onChange.mockClear(); // ignore the loading-start notify
+ emit([1], undefined, { silent: true });
+ expect(s.partial).toEqual([1]); // value fresh...
+ expect(onChange).not.toHaveBeenCalled(); // ...but no re-render
+ emit([1, 2]); // a loud tick flushes
+ expect(s.partial).toEqual([1, 2]);
+ expect(onChange).toHaveBeenCalledTimes(1);
+ });
+});
+
+describe('settle', () => {
+ it('sets ready directly and cancels any in-flight load', async () => {
+ const s = slot();
+ const never = deferred();
+ const signals: AbortSignal[] = [];
+ s.request(4, (ctx) => {
+ signals.push(ctx.signal);
+ return never.promise;
+ });
+ s.settle(4, 'direct');
+ expect(s.isReady).toBe(true);
+ expect(s.value).toBe('direct');
+ expect(signals[0]?.aborted).toBe(true);
+ });
+});
+
+describe('reset', () => {
+ it('aborts and returns to idle', async () => {
+ const s = slot();
+ await s.request(4, async () => 'v');
+ s.reset();
+ expect(s.resolution.status).toBe('idle');
+ expect(s.value).toBeUndefined();
+ });
+});
diff --git a/packages/core/tests/vtableDirectoryResponse.spec.ts b/packages/core/tests/vtableDirectoryResponse.spec.ts
new file mode 100644
index 00000000..c44d9f89
--- /dev/null
+++ b/packages/core/tests/vtableDirectoryResponse.spec.ts
@@ -0,0 +1,239 @@
+import { execSync } from 'node:child_process';
+import { mkdtemp, readFile, rm, stat } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import SpatialDataTableSource from '../src/models/VTableSource.js';
+
+/**
+ * A points/shapes `*.parquet` path is a DIRECTORY of `part.N.parquet` files as
+ * often as it is a single file. Servers disagree on how they answer a range read
+ * of a directory, and that disagreement was load-bearing: a static server 404s
+ * (which the store maps to `undefined`), so part enumeration ran and the element
+ * loaded — but MDV's Flask returns **500** `[Errno 21] Is a directory`, the store
+ * throws on any non-2xx that is not 404, and that throw escaped
+ * `loadParquetDatasetMetadata` before it ever probed `part.0.parquet`. The element
+ * wedged.
+ *
+ * This pins the resolver against BOTH server behaviours over the same real
+ * multipart fixture: the only difference between the two stores is what a read of
+ * the directory path does (return null vs. throw), and both must find the parts.
+ */
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const writerRoot = join(__dirname, '../../../python/spatialdata-experimental-writer');
+
+async function writeMultipartParquetFixture(root: string, partRows: [number, number]) {
+ execSync(
+ `uv run python - <<'PY'
+import pyarrow as pa
+import pyarrow.parquet as pq
+from pathlib import Path
+
+root = Path(${JSON.stringify(root)})
+root.mkdir(parents=True, exist_ok=True)
+
+def write_part(path: Path, start: int, count: int) -> None:
+ table = pa.table(
+ {
+ "x": [float(start + i) for i in range(count)],
+ "y": [float(i) for i in range(count)],
+ "feature_name": [f"gene_{i % 3}" for i in range(count)],
+ "feature_name_codes": pa.array([(i % 3) for i in range(count)], type=pa.int32()),
+ }
+ )
+ pq.write_table(table, path)
+
+write_part(root / "part.0.parquet", 0, ${partRows[0]})
+write_part(root / "part.1.parquet", ${partRows[0]}, ${partRows[1]})
+PY`,
+ { cwd: writerRoot, stdio: 'pipe' }
+ );
+}
+
+/**
+ * A store whose read of a directory path THROWS instead of returning null — the
+ * MDV/Flask "[Errno 21] Is a directory" 500, as the zarrita store surfaces it (any
+ * non-2xx that is not 404 becomes a throw).
+ */
+function createDirectoryThrowsStore(root: string) {
+ /** Every path read, in order — the test's stand-in for the network tab. */
+ const reads: string[] = [];
+ const isDirectory = async (relativePath: string): Promise => {
+ try {
+ return (await stat(join(root, relativePath))).isDirectory();
+ } catch {
+ return false;
+ }
+ };
+ const readStoreBytes = async (relativePath: string): Promise => {
+ reads.push(relativePath);
+ if (await isDirectory(relativePath)) {
+ // The server would 500 here; the store turns that into a throw.
+ throw new Error(`Unexpected response status 500 [Errno 21] Is a directory: ${relativePath}`);
+ }
+ try {
+ return await readFile(join(root, relativePath));
+ } catch {
+ return null; // missing file → 404 → null (this is how part enumeration stops)
+ }
+ };
+
+ return {
+ reads,
+ countReadsOf: (path: string) => reads.filter((read) => read === path).length,
+ clearReads: () => {
+ reads.length = 0;
+ },
+ async get(path: string) {
+ return readStoreBytes(path.startsWith('/') ? path.slice(1) : path);
+ },
+ async getRange(
+ path: string,
+ range: { offset?: number; length?: number; suffixLength?: number }
+ ) {
+ const bytes = await readStoreBytes(path.startsWith('/') ? path.slice(1) : path);
+ if (!bytes) {
+ return null;
+ }
+ if (range.suffixLength != null) {
+ return bytes.subarray(bytes.length - range.suffixLength);
+ }
+ const offset = range.offset ?? 0;
+ const length = range.length ?? bytes.length - offset;
+ return bytes.subarray(offset, offset + length);
+ },
+ };
+}
+
+describe('SpatialDataTableSource — directory path returns 500 (MDV/Flask)', () => {
+ let fixtureRoot: string;
+ let source: SpatialDataTableSource;
+ let store: ReturnType;
+ const parquetPath = 'points/transcripts/points.parquet';
+
+ beforeAll(async () => {
+ fixtureRoot = await mkdtemp(join(tmpdir(), 'directory-500-parquet-'));
+ await writeMultipartParquetFixture(join(fixtureRoot, parquetPath), [100, 50]);
+ store = createDirectoryThrowsStore(fixtureRoot);
+ source = new SpatialDataTableSource({ store, fileType: '.zarr' });
+ }, 120_000);
+
+ afterAll(async () => {
+ await rm(fixtureRoot, { recursive: true, force: true });
+ });
+
+ it('traverses part.N.parquet instead of throwing when the directory 500s', async () => {
+ const metadata = await source.loadParquetDatasetMetadata(parquetPath);
+ expect(metadata).not.toBeNull();
+ expect(metadata?.parts.map((part) => part.path)).toEqual([
+ `${parquetPath}/part.0.parquet`,
+ `${parquetPath}/part.1.parquet`,
+ ]);
+ expect(metadata?.totalNumRows).toBe(150);
+ });
+
+ it('reads the full table across parts despite the directory 500', async () => {
+ const table = await source.loadParquetTable(parquetPath);
+ expect(table.numRows).toBe(150);
+ });
+
+ /**
+ * The layout of a read-only store never changes, but a single points load asks
+ * for it ~20 times (row counts, tiling, row-group extents, the streaming reader,
+ * …). Uncached, each of those repeated the whole probe sequence — which is what
+ * put a stream of repeated directory 500s and trailing 404s in the network tab.
+ */
+ describe('remembers the layout', () => {
+ const missingPart = `${parquetPath}/part.2.parquet`;
+
+ it('probes the directory and the end-of-sequence 404 exactly once', async () => {
+ // A COLD source: the shared one is already warm from the tests above (which
+ // is itself the behaviour under test, just not measurable from here).
+ const cold = new SpatialDataTableSource({
+ store: createDirectoryThrowsStore(fixtureRoot),
+ fileType: '.zarr',
+ });
+ const coldStore = (cold as unknown as { storeRoot: { store: typeof store } }).storeRoot.store;
+
+ const first = await cold.loadParquetDatasetMetadata(parquetPath);
+ const probesAfterFirst = coldStore.reads.length;
+ expect(probesAfterFirst).toBeGreaterThan(0);
+ for (let i = 0; i < 5; i += 1) {
+ await cold.loadParquetDatasetMetadata(parquetPath);
+ }
+
+ // The 500-ing directory path and the 404 past the last part: once each, ever.
+ expect(coldStore.countReadsOf(parquetPath)).toBe(1);
+ expect(coldStore.countReadsOf(missingPart)).toBe(1);
+ // Five further calls cost NOTHING — not merely fewer reads.
+ expect(coldStore.reads.length).toBe(probesAfterFirst);
+ expect(first?.parts).toHaveLength(2);
+ });
+
+ it('shares one probe between concurrent callers', async () => {
+ // The real trigger: many call sites fire at once during a points load, so a
+ // cache that only populated on settle would still stampede.
+ const fresh = new SpatialDataTableSource({
+ store: createDirectoryThrowsStore(fixtureRoot),
+ fileType: '.zarr',
+ });
+ const freshStore = (fresh as unknown as { storeRoot: { store: typeof store } }).storeRoot
+ .store;
+
+ const results = await Promise.all(
+ Array.from({ length: 6 }, () => fresh.loadParquetDatasetMetadata(parquetPath))
+ );
+
+ expect(freshStore.countReadsOf(parquetPath)).toBe(1);
+ expect(freshStore.countReadsOf(missingPart)).toBe(1);
+ // All callers get the same resolved layout.
+ for (const result of results) {
+ expect(result?.parts.map((part) => part.path)).toEqual([
+ `${parquetPath}/part.0.parquet`,
+ `${parquetPath}/part.1.parquet`,
+ ]);
+ }
+ });
+
+ it('does not re-probe parts when reading tables, which uses the other enumerator', async () => {
+ // `discoverMultipartPartPaths` derives the same layout by WHOLE-FILE reads
+ // (the fallback for stores without range support). It used to enumerate
+ // part.0, part.1, … independently — even for elements the metadata had
+ // already resolved — which is the second source of repeated 404s.
+ const cold = new SpatialDataTableSource({
+ store: createDirectoryThrowsStore(fixtureRoot),
+ fileType: '.zarr',
+ });
+ const coldStore = (cold as unknown as { storeRoot: { store: typeof store } }).storeRoot.store;
+
+ await cold.loadParquetTable(parquetPath);
+ await cold.loadParquetTable(parquetPath, ['x', 'y']);
+
+ expect(coldStore.countReadsOf(parquetPath)).toBe(1);
+ expect(coldStore.countReadsOf(missingPart)).toBe(1);
+ });
+
+ it('does not cache a miss, so a transient failure cannot mark a real dataset absent', async () => {
+ // `probeParquetPartMetadata` turns a failed probe into null rather than a
+ // throw, so an all-probes-failed run is indistinguishable from "no dataset
+ // here". Caching that would strand a real element behind one network blip.
+ const flaky = new SpatialDataTableSource({
+ store: createDirectoryThrowsStore(fixtureRoot),
+ fileType: '.zarr',
+ });
+ const absent = 'points/absent/points.parquet';
+
+ expect(await flaky.loadParquetDatasetMetadata(absent)).toBeNull();
+ const readsAfterMiss = (
+ flaky as unknown as { storeRoot: { store: typeof store } }
+ ).storeRoot.store.reads.filter((read) => read.startsWith('points/absent')).length;
+ expect(await flaky.loadParquetDatasetMetadata(absent)).toBeNull();
+ const readsAfterSecondMiss = (
+ flaky as unknown as { storeRoot: { store: typeof store } }
+ ).storeRoot.store.reads.filter((read) => read.startsWith('points/absent')).length;
+
+ expect(readsAfterSecondMiss).toBeGreaterThan(readsAfterMiss);
+ });
+ });
+});
diff --git a/packages/core/tests/vtableLayoutPeek.spec.ts b/packages/core/tests/vtableLayoutPeek.spec.ts
new file mode 100644
index 00000000..3c0b9457
--- /dev/null
+++ b/packages/core/tests/vtableLayoutPeek.spec.ts
@@ -0,0 +1,76 @@
+import { describe, expect, it } from 'vitest';
+import SpatialDataTableSource from '../src/models/VTableSource.js';
+
+/**
+ * Two hot paths PEEK at the resolved part layout to skip a known-useless probe
+ * (the directory read that MDV answers with a 500). A peek is an optimisation,
+ * so it must never be load-bearing.
+ *
+ * The cached layout promise can reject — `loadParquetDatasetMetadata` evicts on
+ * rejection precisely because a failure is expected to be transient. Awaiting it
+ * bare means that rejection escapes into the CALLER, and in `loadParquetBytes`
+ * it escapes from the `for…of` header, so it bypasses the per-candidate
+ * `try/catch` that exists to keep probing. One transient footer failure would
+ * then turn a perfectly loadable single-file element into a hard error rather
+ * than a fall back to the blind candidate order.
+ */
+
+const parquetPath = 'points/transcripts/points.parquet';
+
+/** A parquet magic-number header, so `isParquetFileBytes` accepts the fixture. */
+function parquetLikeBytes() {
+ const bytes = new Uint8Array(16);
+ bytes.set([0x50, 0x41, 0x52, 0x31], 0); // 'PAR1'
+ bytes.set([0x50, 0x41, 0x52, 0x31], 12);
+ return bytes;
+}
+
+type Internals = {
+ parquetDatasetMetadataCache: Map>;
+};
+
+function sourceWithRejectedLayout(paths: string[]) {
+ const served = new Set(paths);
+ const requested: string[] = [];
+ const source = new SpatialDataTableSource({
+ store: {
+ async get(path: string) {
+ requested.push(path);
+ if (!served.has(path.replace(/^\//, ''))) {
+ throw new Error(`404 ${path}`);
+ }
+ return parquetLikeBytes();
+ },
+ },
+ fileType: '.zarr',
+ } as never);
+
+ const rejected = Promise.reject(new Error('transient footer read failure'));
+ // The real cache entry evicts itself on rejection; attach the same handler so
+ // this fixture does not trip Node's unhandled-rejection detector.
+ rejected.catch(() => {});
+ (source as unknown as Internals).parquetDatasetMetadataCache.set(parquetPath, rejected);
+
+ return { source, requested };
+}
+
+describe('parquet layout peek — a rejected cached layout', () => {
+ it('falls back to the blind candidate walk instead of failing the read', async () => {
+ // The element is a plain single file, which the blind order finds on the
+ // FIRST candidate — so a failure here can only come from the peek.
+ const { source, requested } = sourceWithRejectedLayout([parquetPath]);
+
+ const bytes = await source.loadParquetBytes(parquetPath);
+
+ expect(bytes).not.toBeNull();
+ expect(requested).toContain(`/${parquetPath}`);
+ });
+
+ it('still walks to a later candidate when the first is unservable', async () => {
+ const { source } = sourceWithRejectedLayout([`${parquetPath}/part.0.parquet`]);
+
+ const bytes = await source.loadParquetBytes(parquetPath);
+
+ expect(bytes).not.toBeNull();
+ });
+});
diff --git a/packages/core/tests/vtableMultipart.spec.ts b/packages/core/tests/vtableMultipart.spec.ts
index a9636a88..ae297195 100644
--- a/packages/core/tests/vtableMultipart.spec.ts
+++ b/packages/core/tests/vtableMultipart.spec.ts
@@ -122,3 +122,61 @@ describe('SpatialDataTableSource multipart parquet reads', () => {
expect(table.getChild('y')?.length).toBe(120);
});
});
+
+/**
+ * Row-group chunks are handed to the points worker with their buffers TRANSFERRED
+ * (zero-copy), which detaches them in this thread. `schemaBytes` used to be a live
+ * reference into the cached dataset metadata, so the first transfer detached the
+ * CACHE — and the next row group posted an already-detached buffer:
+ *
+ * DataCloneError: ArrayBuffer at index 0 is already detached
+ *
+ * which aborted the progressive preload and dropped the element onto the
+ * whole-file fallback. A chunk must therefore own its bytes outright.
+ */
+describe('row-group chunks own their bytes', () => {
+ let fixtureRoot: string;
+ let source: SpatialDataTableSource;
+ const parquetPath = 'points/transcripts/points.parquet';
+
+ beforeAll(async () => {
+ fixtureRoot = await mkdtemp(join(tmpdir(), 'rowgroup-ownership-'));
+ await writeMultipartParquetFixture(join(fixtureRoot, parquetPath), [100, 50]);
+ source = new SpatialDataTableSource({
+ store: createFilesystemStore(fixtureRoot),
+ fileType: '.zarr',
+ });
+ }, 120_000);
+
+ afterAll(async () => {
+ await rm(fixtureRoot, { recursive: true, force: true });
+ });
+
+ it('does not hand out the cached footer buffer, so transferring one chunk cannot detach the next', async () => {
+ const internals = source as unknown as {
+ readParquetRowGroupBytesByGroupIndex: (
+ path: string,
+ index: number
+ ) => Promise<{ schemaBytes: Uint8Array; rowGroupBytes: Uint8Array } | null>;
+ };
+
+ const first = await internals.readParquetRowGroupBytesByGroupIndex(parquetPath, 0);
+ expect(first).not.toBeNull();
+
+ // Exactly what the worker client does: transfer the buffer away.
+ structuredClone(first?.schemaBytes.buffer, {
+ transfer: [first?.schemaBytes.buffer as ArrayBuffer],
+ });
+ expect(first?.schemaBytes.buffer.detached).toBe(true);
+
+ // The next read must still be usable — both for a later row group and for a
+ // repeat of the same one.
+ const second = await internals.readParquetRowGroupBytesByGroupIndex(parquetPath, 0);
+ expect(second?.schemaBytes.buffer.detached).toBe(false);
+ expect(second?.schemaBytes.length).toBeGreaterThan(0);
+
+ // And the underlying metadata is intact for the NEXT consumer too.
+ const dataset = await source.loadParquetDatasetMetadata(parquetPath);
+ expect(dataset?.parts[0]?.schemaBytes.buffer.detached).toBe(false);
+ });
+});
diff --git a/packages/core/tests/vtableRangeProbeCache.spec.ts b/packages/core/tests/vtableRangeProbeCache.spec.ts
new file mode 100644
index 00000000..b653ac7d
--- /dev/null
+++ b/packages/core/tests/vtableRangeProbeCache.spec.ts
@@ -0,0 +1,114 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import SpatialDataPointsSource from '../src/models/VPointsSource.js';
+
+/**
+ * The range probe caches its answer per ORIGIN, for the life of the page, and a
+ * `false` demotes every element on that origin to the whole-file read path.
+ *
+ * That makes the difference between "the server said no" and "the request did
+ * not complete" load-bearing. Caching the second reads, from the outside, as a
+ * non-deterministic failure: one dropped connection during startup and points
+ * or feature counts never settle again until a hard reload — with nothing in the
+ * log to say why, because the fallback path looks healthy.
+ *
+ * These pin that only a DEFINITIVE answer sticks.
+ */
+
+type Internals = {
+ serverSupportsStreamingRanges: (url: string) => Promise;
+};
+
+const url = 'http://example.test/points/transcripts/points.parquet';
+
+function probeSource() {
+ const source = new SpatialDataPointsSource({
+ store: { async get() {}, async getRange() {} },
+ fileType: '.zarr',
+ } as never);
+ return source as unknown as Internals;
+}
+
+/** The static cache outlives any one instance, so each case needs a fresh origin. */
+function freshUrl(name: string) {
+ return `http://${name}.test/points.parquet`;
+}
+
+function partialResponse(byteLength: number) {
+ return {
+ status: 206,
+ arrayBuffer: async () => new ArrayBuffer(byteLength),
+ } as unknown as Response;
+}
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+});
+
+describe('streaming range probe — cache policy', () => {
+ it('caches a successful probe so the second call issues no requests', async () => {
+ const fetchSpy = vi.fn(async () => partialResponse(8));
+ vi.stubGlobal('fetch', fetchSpy);
+ const target = freshUrl('cache-ok');
+
+ await expect(probeSource().serverSupportsStreamingRanges(target)).resolves.toBe(true);
+ expect(fetchSpy).toHaveBeenCalledTimes(2); // suffix + bounded
+
+ // A different instance, same origin: the answer is a property of the server.
+ await expect(probeSource().serverSupportsStreamingRanges(target)).resolves.toBe(true);
+ expect(fetchSpy).toHaveBeenCalledTimes(2);
+ });
+
+ it('caches a definitive refusal — a 416 server should not be re-probed forever', async () => {
+ const fetchSpy = vi.fn(async () => ({ status: 416 }) as unknown as Response);
+ vi.stubGlobal('fetch', fetchSpy);
+ const target = freshUrl('cache-416');
+
+ await expect(probeSource().serverSupportsStreamingRanges(target)).resolves.toBe(false);
+ await expect(probeSource().serverSupportsStreamingRanges(target)).resolves.toBe(false);
+ expect(fetchSpy).toHaveBeenCalledTimes(2);
+ });
+
+ it('does NOT cache a thrown probe, so a transient failure can recover', async () => {
+ let attempt = 0;
+ const fetchSpy = vi.fn(async () => {
+ attempt += 1;
+ // Both requests of the first probe fail; everything after succeeds.
+ if (attempt <= 2) throw new TypeError('Failed to fetch');
+ return partialResponse(8);
+ });
+ vi.stubGlobal('fetch', fetchSpy);
+ const target = freshUrl('transient');
+
+ await expect(probeSource().serverSupportsStreamingRanges(target)).resolves.toBe(false);
+ // Without eviction this stays false for the life of the page.
+ await expect(probeSource().serverSupportsStreamingRanges(target)).resolves.toBe(true);
+ });
+
+ it('still shares one in-flight probe between concurrent callers', async () => {
+ const fetchSpy = vi.fn(async () => partialResponse(8));
+ vi.stubGlobal('fetch', fetchSpy);
+ const target = freshUrl('inflight');
+ const source = probeSource();
+
+ const [first, second] = await Promise.all([
+ source.serverSupportsStreamingRanges(target),
+ source.serverSupportsStreamingRanges(target),
+ ]);
+ expect([first, second]).toEqual([true, true]);
+ expect(fetchSpy).toHaveBeenCalledTimes(2);
+ });
+
+ it('treats a short body as a refusal — the reader would read the wrong window', async () => {
+ vi.stubGlobal('fetch', vi.fn(async () => partialResponse(4)));
+ await expect(probeSource().serverSupportsStreamingRanges(freshUrl('short'))).resolves.toBe(
+ false
+ );
+ });
+
+ it('declines a non-URL target without probing', async () => {
+ const fetchSpy = vi.fn(async () => partialResponse(8));
+ vi.stubGlobal('fetch', fetchSpy);
+ await expect(probeSource().serverSupportsStreamingRanges('not a url')).resolves.toBe(false);
+ expect(fetchSpy).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/layers/src/PointsLayer.ts b/packages/layers/src/PointsLayer.ts
index c60b4859..0663674f 100644
--- a/packages/layers/src/PointsLayer.ts
+++ b/packages/layers/src/PointsLayer.ts
@@ -8,7 +8,11 @@ import {
filterBatchSignature,
hasPreloadedRowFeatureCodes,
} from './pointsFeatureCodes.js';
-import type { ColumnarNdarrayPointsBatch, PointsRenderResource } from './pointsLoader.js';
+import type {
+ ColumnarNdarrayPointsBatch,
+ PointsLoader,
+ PointsRenderResource,
+} from './pointsLoader.js';
import { resolvePointsRenderStrategy } from './pointsRenderStrategies.js';
import {
DEFAULT_POINT_RADIUS_MAX_PIXELS,
@@ -31,9 +35,20 @@ export interface PointsLayerProps {
color?: [number, number, number, number];
/** Colour points by their per-point feature code instead of the flat color. */
colorByFeature?: boolean;
+ /** Number of feature codes the colour LUT must cover (catalog `maxCode + 1`). */
+ featureCodeSpaceSize?: number;
+ /** Per-feature colour overrides (`code → [r,g,b]`); absent codes keep the default. */
+ featureColorOverrides?: import('./pointsFeatureColor.js').FeatureColorOverrides | null;
+ /** Emphasise one feature code: its points keep their colour, others desaturate +
+ * dim. -1 (default) highlights nothing. */
+ highlightFeatureCode?: number;
featureCodes?: readonly number[];
/** Source-side integer codes aligned with the preloaded table rows. */
preloadedFeatureCodes?: ArrayLike;
+ /** Bumps when a stable resource's backing batch grows in place (the streaming
+ * partial overlay, D10). A change re-reads `loader.loadAll()` WITHOUT resetting the
+ * layer, so the overlay fills in without a per-chunk teardown. */
+ resourceRevision?: number;
/** Max rows to draw after feature filtering. */
renderCap?: number;
showTileDebugOverlay?: boolean;
@@ -131,6 +146,18 @@ export class PointsLayer extends CompositeLayer {
return;
}
+ // Same loader, but its stable backing batch was swapped in place (the streaming
+ // partial overlay grows per chunk, D10; the base swaps resident↔matched↔streaming,
+ // P2): re-read `loadAll` for the new buffer and re-filter, WITHOUT the reset above
+ // — that is what keeps it from flashing. Return so the signature-filter pass below
+ // does not run against the STALE `preloadedBatch` with the NEW codes (a swap
+ // changes the batch and its row-aligned codes together); `refreshPreloadedBatch`
+ // re-filters the new batch with the current props.
+ if (props.resourceRevision !== oldProps.resourceRevision) {
+ void this.refreshPreloadedBatch();
+ return;
+ }
+
const signature = filterBatchSignature(
props.featureCodes,
props.preloadedFeatureCodes,
@@ -155,6 +182,30 @@ export class PointsLayer extends CompositeLayer {
}
}
+ /**
+ * What a `loadAll()` read was issued against. BOTH parts matter, for different
+ * races:
+ *
+ * - `revision` — the streaming overlay bumps it per chunk, so several reads can
+ * be in flight at once. Today the adapter's `loadAll` snapshots the holder
+ * synchronously and never awaits, so those resolve in call order and the last
+ * writer is the newest; that is a property of one loader implementation, not
+ * of the {@link PointsLoader} contract this method is written against.
+ * - `loader` — a loader swap (a cap raise) resets the batch state and starts a
+ * fresh read, but an OLD read already in flight still resolves afterwards and
+ * would write the previous loader's batch over it. A revision check alone does
+ * not catch this: revisions are per-holder and can coincide across a swap.
+ */
+ private loadToken(): { loader: PointsLoader; revision: number | undefined } {
+ return { loader: this.props.resource.loader, revision: this.props.resourceRevision };
+ }
+
+ private isStaleLoad(token: { loader: PointsLoader; revision: number | undefined }): boolean {
+ return (
+ this.props.resource.loader !== token.loader || this.props.resourceRevision !== token.revision
+ );
+ }
+
private async ensurePreloadedBatch(): Promise {
const { resource } = this.props;
if (resource.loader.capabilities.kind !== 'preloaded-columnar') {
@@ -164,7 +215,11 @@ export class PointsLayer extends CompositeLayer {
if (existing) {
return;
}
+ const token = this.loadToken();
const batch = await resource.loader.loadAll?.();
+ if (this.isStaleLoad(token)) {
+ return;
+ }
if (batch?.format === 'columnar-ndarray') {
this.setState({ preloadedBatch: batch });
const awaitingRowCodes = featureFilterAwaitingRowCodes(
@@ -184,6 +239,44 @@ export class PointsLayer extends CompositeLayer {
}
}
+ /**
+ * Re-read the (grown) batch from a stable loader whose backing buffer changed in
+ * place — the D10 streaming overlay. Unlike {@link ensurePreloadedBatch} it has no
+ * "already loaded" short-circuit (the whole point is to pick up the growth) and it
+ * does not reset filter state, so the overlay updates without a teardown.
+ */
+ private async refreshPreloadedBatch(): Promise {
+ const { resource } = this.props;
+ if (resource.loader.capabilities.kind !== 'preloaded-columnar') {
+ return;
+ }
+ const token = this.loadToken();
+ const batch = await resource.loader.loadAll?.();
+ // A newer revision (or a different loader) landed while this read was in
+ // flight; that read owns the state. See `loadToken`.
+ if (this.isStaleLoad(token)) {
+ return;
+ }
+ if (batch?.format !== 'columnar-ndarray') {
+ return;
+ }
+ this.setState({ preloadedBatch: batch });
+ const awaitingRowCodes = featureFilterAwaitingRowCodes(
+ this.props.featureCodes,
+ this.props.preloadedFeatureCodes
+ );
+ if (!awaitingRowCodes) {
+ void this.ensureFilteredBatch(
+ batch,
+ filterBatchSignature(
+ this.props.featureCodes,
+ this.props.preloadedFeatureCodes,
+ this.props.renderCap
+ )
+ );
+ }
+ }
+
private async ensureFilteredBatch(
batch: ColumnarNdarrayPointsBatch,
signature: string
diff --git a/packages/layers/src/adapters/PointsRendererAdapter.ts b/packages/layers/src/adapters/PointsRendererAdapter.ts
index 54fa6dde..7492c9e2 100644
--- a/packages/layers/src/adapters/PointsRendererAdapter.ts
+++ b/packages/layers/src/adapters/PointsRendererAdapter.ts
@@ -1,5 +1,9 @@
import type { PointsElement, PointsLoadResult } from '@spatialdata/core';
-import type { PointsRenderResource } from '../pointsLoader.js';
+import {
+ columnarBatchFromPointData,
+ type PointsLoader,
+ type PointsRenderResource,
+} from '../pointsLoader.js';
import {
pointsRenderResourceSignature,
resolvePointsRenderResource,
@@ -48,7 +52,29 @@ interface ResourceMemo {
interface EntryMemos {
resident?: ResourceMemo;
matched?: ResourceMemo;
- partial?: ResourceMemo;
+}
+
+/**
+ * The streaming overlay's resource (D10). Unlike the resident/matched memos — which
+ * key on batch IDENTITY and so mint a new resource whenever the batch changes — the
+ * partial's resource is held **stable for the lifetime of one scan** and its backing
+ * batch is swapped through a mutable holder, with a `revision` counter bumped on each
+ * growth. That is what stops `PointsLayer` tearing the `__partial` sublayer down and
+ * rebuilding it per chunk (the flash): the loader identity never changes mid-scan, so
+ * the composite re-reads the grown buffer on a `resourceRevision` prop change instead
+ * of resetting.
+ */
+interface GrowingPartial {
+ /** The scan this partial belongs to (`${signature}#${cap}`); a change means a new scan. */
+ scanKey: string;
+ /** The element the loader is bound to — see the note on `growingBases`. A scan key
+ * repeats across a dataset swap (same selection, same cap), so it alone does not
+ * catch a replaced element. */
+ element: PointsElement;
+ resource: PointsRenderResource;
+ /** Swapped per chunk; the loader's `loadAll` reads through it. */
+ holder: { current: PointsLoadResult };
+ revision: number;
}
const RESOLVE_OPTIONS = { experimentalOptimizations: 'off' as const };
@@ -58,6 +84,21 @@ const isEmpty = (batch: PointsLoadResult): boolean => (batch.shape[1] ?? 0) ===
export class PointsRendererAdapter {
private readonly memos = new Map();
+ private readonly growingPartials = new Map();
+ /** The base layer's stable resource per element — see {@link getBaseResource}. */
+ private readonly growingBases = new Map<
+ string,
+ {
+ /** The element the resource's loader is bound to. Stable while `spatialData`
+ * is, so this only differs after a real dataset swap — where the resolver
+ * cache is deliberately preserved under the same key, which is exactly how a
+ * stale loader would otherwise survive one. */
+ element: PointsElement;
+ resource: PointsRenderResource;
+ holder: { current: PointsLoadResult };
+ revision: number;
+ }
+ >();
private entry(key: string): EntryMemos {
let memos = this.memos.get(key);
@@ -112,24 +153,152 @@ export class PointsRendererAdapter {
}
/**
- * The in-flight scan's growing buffer, as a resource, so points progressively
- * fill in before the full scan settles. Rebuilds only when a new chunk grows the
- * buffer — not per pan, which is when the user is most likely to be moving.
+ * The in-flight scan's growing buffer, as a resource (D10).
+ *
+ * The resource identity is **held stable for the whole scan** (keyed on `scanKey`,
+ * not the batch): a grown buffer swaps the mutable holder and bumps
+ * {@link getMatchingPartialRevision} instead of minting a new resource. So the
+ * `PointsLayer` composite is NOT torn down per chunk — it re-reads the grown buffer
+ * on a `resourceRevision` prop change. One deck layer per *(entry, selection)*,
+ * zero teardowns per scan. A new scan (`scanKey` change) mints a fresh resource.
*/
getMatchingPartialResource(
+ element: PointsElement,
+ key: string,
+ batch: PointsLoadResult | undefined,
+ scanKey: string | undefined
+ ): PointsRenderResource | null {
+ if (!batch || isEmpty(batch) || scanKey === undefined) {
+ this.growingPartials.delete(key);
+ return null;
+ }
+ let growing = this.growingPartials.get(key);
+ if (!growing || growing.scanKey !== scanKey || growing.element !== element) {
+ // New scan → build ONE resource whose loader reads through a mutable holder.
+ const holder = { current: batch };
+ const resource = this.buildGrowingResource(element, holder);
+ if (!resource) return null;
+ growing = { scanKey, element, resource, holder, revision: 0 };
+ this.growingPartials.set(key, growing);
+ } else if (growing.holder.current !== batch) {
+ // Same scan, grown buffer → swap the holder + bump the revision. SAME resource.
+ growing.holder.current = batch;
+ growing.revision += 1;
+ }
+ return growing.resource;
+ }
+
+ /** The revision of the in-flight partial's growing buffer — a `PointsLayer`
+ * `resourceRevision` prop, bumped each time the buffer grows so the composite
+ * re-reads without a teardown. */
+ getMatchingPartialRevision(key: string): number {
+ return this.growingPartials.get(key)?.revision ?? 0;
+ }
+
+ /**
+ * The **base** layer's render resource — ONE stable resource per element whose
+ * backing batch evolves.
+ *
+ * The base's "current best view" changes over an element's life: the resident
+ * preload (streaming in during initial load), that preload filtered to a selection,
+ * then the whole-dataset matched batch once a scan covers the selection. Each of
+ * those is a *different* batch, and the old code drew them under one `id: layerId`
+ * from two different resources (resident vs matched) — so every transition changed
+ * the loader identity and `PointsLayer` hard-reset (the base flash).
+ *
+ * Here the resource identity is fixed for the element (built once, from the first
+ * batch); a new batch swaps the mutable holder and bumps {@link getBaseRevision},
+ * and `PointsLayer` re-reads `loadAll` on that revision change WITHOUT resetting. No
+ * teardown across resident↔matched↔streaming transitions. Callers choose the batch
+ * (matched-if-covered else resident) and pass the matching `preloadedFeatureCodes`.
+ */
+ getBaseResource(
element: PointsElement,
key: string,
batch: PointsLoadResult | undefined
- ) {
- if (!batch || isEmpty(batch)) return null;
- return this.resolve(this.entry(key), 'partial', element, batch);
+ ): PointsRenderResource | null {
+ if (!batch || isEmpty(batch)) {
+ this.growingBases.delete(key);
+ return null;
+ }
+ let growing = this.growingBases.get(key);
+ if (!growing || growing.element !== element) {
+ const holder = { current: batch };
+ const resource = this.buildGrowingResource(element, holder);
+ if (!resource) return null;
+ growing = { element, resource, holder, revision: 0 };
+ this.growingBases.set(key, growing);
+ } else if (growing.holder.current !== batch) {
+ growing.holder.current = batch;
+ growing.revision += 1;
+ }
+ return growing.resource;
+ }
+
+ /** The base resource's revision — a `PointsLayer` `resourceRevision` prop, bumped
+ * each time the base batch is swapped (resident↔matched↔streaming) so the composite
+ * re-reads without a teardown. */
+ getBaseRevision(key: string): number {
+ return this.growingBases.get(key)?.revision ?? 0;
+ }
+
+ /** A stable render resource whose `loadAll` reads the current holder batch. */
+ private buildGrowingResource(
+ element: PointsElement,
+ holder: { current: PointsLoadResult }
+ ): PointsRenderResource | null {
+ const base = resolvePointsRenderResource(
+ element,
+ { preloaded: holder.current, metadataKnown: false },
+ RESOLVE_OPTIONS
+ );
+ if (!base) return null;
+ // `base` is resolved from the batch the holder held at BUILD time, and the
+ // whole point of the holder is that the batch changes afterwards. `loadAll`
+ // reads through it, so it stays current; a `loadInBounds` bound to the
+ // original `base` would answer viewport queries from the first preload
+ // forever. Re-resolve lazily when the holder has moved — only on demand, and
+ // only once per swap, so the stable-identity `loader` below is untouched.
+ //
+ // No path reaches this today: these resources report `preloaded-columnar`,
+ // whose strategy renders from `loadAll` alone. It is wrong the moment a tiled
+ // strategy is pointed at a growing resource, which is what D5 does.
+ let resolvedFor = holder.current;
+ let active = base;
+ const currentBase = (): PointsRenderResource => {
+ if (resolvedFor !== holder.current) {
+ const next = resolvePointsRenderResource(
+ element,
+ { preloaded: holder.current, metadataKnown: false },
+ RESOLVE_OPTIONS
+ );
+ resolvedFor = holder.current;
+ if (next) active = next;
+ }
+ return active;
+ };
+ const loader: PointsLoader = {
+ capabilities: base.loader.capabilities,
+ loadInBounds: (options) => currentBase().loader.loadInBounds(options),
+ loadAll: async () =>
+ columnarBatchFromPointData({
+ shape: holder.current.shape,
+ data: holder.current.data,
+ ...(holder.current.featureCodes ? { featureCodes: holder.current.featureCodes } : {}),
+ }),
+ };
+ return { element, loader };
}
evict(key: string): void {
this.memos.delete(key);
+ this.growingPartials.delete(key);
+ this.growingBases.delete(key);
}
dispose(): void {
this.memos.clear();
+ this.growingPartials.clear();
+ this.growingBases.clear();
}
}
diff --git a/packages/layers/src/engine/PointsDataEngine.ts b/packages/layers/src/engine/PointsDataEngine.ts
index 977f3090..3d4b3427 100644
--- a/packages/layers/src/engine/PointsDataEngine.ts
+++ b/packages/layers/src/engine/PointsDataEngine.ts
@@ -5,8 +5,16 @@ import {
PointsResolver,
} from '@spatialdata/core';
import { PointsRendererAdapter } from '../adapters/PointsRendererAdapter.js';
+import type { FeatureColorOverrides } from '../pointsFeatureColor.js';
import type { PointsRenderResource } from '../pointsLoader.js';
+/** Per-feature colour overrides as authored in layer config: keyed by feature NAME
+ * (robust to the code remapping between a resident-preview and the full catalog),
+ * resolved to codes at render time by {@link PointsDataEngine.getFeatureColorOverrideMap}. */
+export type FeatureColorOverridesByName = Readonly<
+ Record
+>;
+
/**
* `PointsDataEngine` — now a **facade** over `PointsResolver` (`core`) and
* `PointsRendererAdapter` (`layers`).
@@ -59,6 +67,24 @@ export type PointsDataEngineCallbacks = PointsResolverCallbacks;
export class PointsDataEngine {
private readonly resolver: PointsResolver;
private readonly adapter = new PointsRendererAdapter();
+ /** Memo for {@link getFeatureCodeSpaceSize}, invalidated by catalog identity. */
+ private readonly codeSpaceMemo = new Map<
+ string,
+ { catalog: PointsFeatureCatalog | null | undefined; size: number }
+ >();
+ /** Per-element hover-highlighted feature code (runtime-only UI state), or -1 for
+ * none. Lives here — not in core — because it is a render concern the feature panel
+ * writes and the render path reads through this one shared engine. */
+ private readonly highlightByKey = new Map();
+ /** Memo for {@link getFeatureColorOverrideMap}, invalidated by config + catalog. */
+ private readonly overrideMapMemo = new Map<
+ string,
+ {
+ source: FeatureColorOverridesByName | undefined;
+ catalog: PointsFeatureCatalog | null | undefined;
+ map: FeatureColorOverrides | null;
+ }
+ >();
constructor(callbacks: PointsDataEngineCallbacks = {}) {
this.resolver = new PointsResolver(callbacks);
@@ -97,15 +123,43 @@ export class PointsDataEngine {
return this.adapter.getMatchingResource(element, key, this.resolver.getMatchedBatch(key));
}
- /** Render resource for the in-flight scan's growing partial buffer. */
+ /** Render resource for the in-flight scan's growing partial buffer. Identity is
+ * stable for the scan's lifetime (D10); {@link getMatchingPartialRevision} bumps
+ * as it grows. */
getMatchingPartialResource(element: PointsElement, key: string): PointsRenderResource | null {
return this.adapter.getMatchingPartialResource(
element,
key,
- this.resolver.getPartialBatch(key)
+ this.resolver.getPartialBatch(key),
+ this.resolver.getPartialScanKey(key)
);
}
+ /** The growing partial's revision — a `PointsLayer` `resourceRevision` prop, so the
+ * `__partial` sublayer re-reads the grown buffer without a per-chunk teardown. */
+ getMatchingPartialRevision(key: string): number {
+ return this.adapter.getMatchingPartialRevision(key);
+ }
+
+ /**
+ * The BASE layer's stable render resource for a chosen batch (matched-if-covered
+ * else resident — the caller decides). Identity is fixed for the element; the batch
+ * swaps under it (see {@link getBaseRevision}), so the base never tears down across
+ * resident↔matched↔streaming transitions.
+ */
+ getBaseResource(
+ element: PointsElement,
+ key: string,
+ batch: PointsLoadResult | undefined
+ ): PointsRenderResource | null {
+ return this.adapter.getBaseResource(element, key, batch);
+ }
+
+ /** The base resource's revision — a `PointsLayer` `resourceRevision` prop. */
+ getBaseRevision(key: string): number {
+ return this.adapter.getBaseRevision(key);
+ }
+
// --- Lifecycle (resolver-owned) ---------------------------------------------
ensureLoaded(target: PointsLoadTarget, memoryCap?: number): Promise {
@@ -142,6 +196,23 @@ export class PointsDataEngine {
return this.resolver.getData(key);
}
+ /** The last-good matched-selection batch (whole-dataset scan result). */
+ getMatchedBatch(key: string): PointsLoadResult | undefined {
+ return this.resolver.getMatchedBatch(key);
+ }
+
+ /** The in-flight preload's growing geometry (D3) — drawn before the first full
+ * window settles so a cold load paints progressively. */
+ getPreloadPartialBatch(key: string): PointsLoadResult | undefined {
+ return this.resolver.getPreloadPartialBatch(key);
+ }
+
+ /** Running per-feature counts over the resident window, available while the
+ * whole-dataset counts scan is still running. */
+ getResidentFeatureCounts(key: string): ReadonlyMap | undefined {
+ return this.resolver.getResidentFeatureCounts(key);
+ }
+
getStatus(key: string): PointsLoadStatus {
return this.resolver.getStatus(key);
}
@@ -185,6 +256,98 @@ export class PointsDataEngine {
return this.resolver.getFeatureCatalog(key);
}
+ /**
+ * The feature-code space size — `maxCode + 1` across the catalog, i.e. the width the
+ * colour LUT must cover so every point's code indexes a real texel. 0 until a catalog
+ * loads. Memoised on catalog identity (the resolver replaces it, never mutates), so
+ * this is O(entries) only when the catalog changes — cheap enough for the per-frame
+ * `getLayers`.
+ */
+ getFeatureCodeSpaceSize(key: string): number {
+ const catalog = this.resolver.getFeatureCatalog(key);
+ const cached = this.codeSpaceMemo.get(key);
+ if (cached && cached.catalog === catalog) {
+ return cached.size;
+ }
+ let size = 0;
+ if (catalog) {
+ for (const entry of catalog.entries) {
+ if (entry.code + 1 > size) {
+ size = entry.code + 1;
+ }
+ }
+ }
+ this.codeSpaceMemo.set(key, { catalog, size });
+ return size;
+ }
+
+ /**
+ * Resolve config's by-NAME colour overrides to the `code → rgb` map the LUT builder
+ * wants, using the current catalog's name↔code mapping. Returns null when there are
+ * no overrides (or no catalog yet) — the palette then falls back to all defaults.
+ *
+ * Keyed by name on purpose: a feature's code can differ between the resident-preview
+ * catalog and the authoritative full one, but its name does not, so an override
+ * authored against a name lands on the right feature once the catalog settles.
+ * Memoised on (config identity, catalog identity) so the map — and thus the palette
+ * texture downstream — keeps a stable identity across the per-frame `getLayers`.
+ */
+ getFeatureColorOverrideMap(
+ key: string,
+ overridesByName: FeatureColorOverridesByName | undefined
+ ): FeatureColorOverrides | null {
+ const catalog = this.resolver.getFeatureCatalog(key);
+ const cached = this.overrideMapMemo.get(key);
+ if (cached && cached.source === overridesByName && cached.catalog === catalog) {
+ return cached.map;
+ }
+ let map: Map | null = null;
+ if (overridesByName && catalog) {
+ const codeByName = new Map();
+ for (const entry of catalog.entries) {
+ codeByName.set(entry.name, entry.code);
+ }
+ const resolved = new Map();
+ for (const [name, rgb] of Object.entries(overridesByName)) {
+ const code = codeByName.get(name);
+ if (code !== undefined) {
+ resolved.set(code, rgb);
+ }
+ }
+ // Null (not an empty map) when nothing resolved, so callers fall back to the
+ // all-default palette and the identity check stays meaningful.
+ if (resolved.size > 0) {
+ map = resolved;
+ }
+ }
+ this.overrideMapMemo.set(key, { source: overridesByName, catalog, map });
+ return map;
+ }
+
+ /** The hover-highlighted feature code for an element, or -1 for none. Read by the
+ * render path (drives the `highlightFeatureCode` uniform). */
+ getHighlightedFeature(key: string): number {
+ return this.highlightByKey.get(key) ?? -1;
+ }
+
+ /**
+ * Set (or clear, with null) the hover-highlighted feature for an element and notify
+ * subscribers so the panel and the render path repaint. Called from the feature
+ * list's row hover. A no-op when unchanged, so mousemove churn is cheap.
+ */
+ setHighlightedFeature(key: string, featureCode: number | null): void {
+ const next = featureCode ?? -1;
+ if (this.getHighlightedFeature(key) === next) {
+ return;
+ }
+ if (next < 0) {
+ this.highlightByKey.delete(key);
+ } else {
+ this.highlightByKey.set(key, next);
+ }
+ this.resolver.notify();
+ }
+
isFeatureCatalogLoading(key: string): boolean {
return this.resolver.isFeatureCatalogLoading(key);
}
@@ -213,16 +376,27 @@ export class PointsDataEngine {
return this.resolver.hasRowFeatureCodes(key);
}
+ /** Re-run any failed resources of an element (e.g. a stuck full-catalog scan). */
+ retry(key: string): Promise {
+ return this.resolver.retry(key);
+ }
+
// --- Lifecycle --------------------------------------------------------------
/** Drop an element from both halves — the data AND the resources built from it. */
evict(key: string): void {
this.resolver.evict(key);
this.adapter.evict(key);
+ this.codeSpaceMemo.delete(key);
+ this.overrideMapMemo.delete(key);
+ this.highlightByKey.delete(key);
}
dispose(): void {
this.resolver.dispose();
this.adapter.dispose();
+ this.codeSpaceMemo.clear();
+ this.overrideMapMemo.clear();
+ this.highlightByKey.clear();
}
}
diff --git a/packages/layers/src/index.ts b/packages/layers/src/index.ts
index 3252fee9..42caa814 100644
--- a/packages/layers/src/index.ts
+++ b/packages/layers/src/index.ts
@@ -41,6 +41,10 @@ export type { LabelsLayerProps, LabelsSelection } from './LabelsLayer';
export { LabelsLayer, MAX_LABEL_CHANNELS } from './LabelsLayer';
export type { PointsLayerProps } from './PointsLayer';
export { PointsLayer } from './PointsLayer';
+// Exported so a CALLER can check the same condition `PointsLayer` checks before
+// handing it a filter it would decline to apply. See the strategy's
+// `resolveScatterBatch`: awaiting row codes means the batch is drawn WHOLE.
+export { featureFilterAwaitingRowCodes } from './pointsFeatureCodes.js';
export { featureCodeToCssColor, featureCodeToRgb } from './pointsFeatureColor.js';
export { PointsFeatureColorExtension } from './pointsFeatureColorExtension.js';
export {
diff --git a/packages/layers/src/pointsFeatureColor.ts b/packages/layers/src/pointsFeatureColor.ts
index 69f1f115..c47a066a 100644
--- a/packages/layers/src/pointsFeatureColor.ts
+++ b/packages/layers/src/pointsFeatureColor.ts
@@ -62,3 +62,64 @@ export function featureCodeToCssColor(code: number): string {
const [r, g, b] = featureCodeToRgb(code);
return `rgb(${r}, ${g}, ${b})`;
}
+
+/** Per-feature colour overrides: `code → [r, g, b]` (0–255). Any code absent here
+ * keeps its default {@link featureCodeToRgb} colour. */
+export type FeatureColorOverrides = ReadonlyMap;
+
+/**
+ * Default LUT width, used whenever the catalog is unknown or smaller.
+ *
+ * The colour of a code is a PURE FUNCTION of that code — the catalog is not an input.
+ * Sizing the table from the catalog was a design error with a very visible cost: the
+ * catalog is the LAST thing to load on a big element, so until it landed the palette
+ * was one texel wide and the shader clamped every code to texel 0 — the whole layer
+ * one flat colour for the entire load. Covering a generous code space up front makes
+ * colour correct from the first streamed chunk; the width only grows if a catalog
+ * turns out to be bigger. 4096 texels is 16 KB.
+ */
+export const DEFAULT_FEATURE_PALETTE_WIDTH = 4096;
+
+/** The LUT width for a (possibly unknown) code space. Callers that compare against an
+ * existing texture must use this, so "needed" and "built" agree. */
+export function featurePaletteWidth(codeSpaceSize: number): number {
+ const requested = Number.isFinite(codeSpaceSize) ? Math.floor(codeSpaceSize) : 0;
+ return Math.max(DEFAULT_FEATURE_PALETTE_WIDTH, requested);
+}
+
+/** A colour lookup table indexed by feature code — one RGBA texel per code. Uploaded
+ * to a GPU texture and sampled by {@link pointsFeatureColorExtension} with
+ * `texelFetch(pfcPalette, ivec2(code, 0), 0)`. */
+export interface FeaturePalette {
+ /** RGBA8, row-major: bytes `[4*code .. 4*code+3]` are the colour for `code`. */
+ data: Uint8Array;
+ /** Texture width = number of codes covered (`maxCode + 1`). Always ≥ 1. */
+ width: number;
+}
+
+/**
+ * Build the feature-colour lookup table. Texel `i` is the colour for code `i`:
+ * {@link featureCodeToRgb} by default (so the palette is byte-identical to the
+ * procedural shader it replaced), with `overrides` patching individual codes.
+ *
+ * `codeSpaceSize` is a LOWER bound, not the answer: the table is always at least
+ * {@link DEFAULT_FEATURE_PALETTE_WIDTH} wide so colour works before any catalog
+ * loads. A code beyond the table is clamped to the last texel by the shader, so an
+ * under-sized table mis-colours the tail rather than crashing.
+ */
+export function buildFeaturePalette(
+ codeSpaceSize: number,
+ overrides?: FeatureColorOverrides
+): FeaturePalette {
+ const width = featurePaletteWidth(codeSpaceSize);
+ const data = new Uint8Array(width * 4);
+ for (let code = 0; code < width; code += 1) {
+ const [r, g, b] = overrides?.get(code) ?? featureCodeToRgb(code);
+ const offset = code * 4;
+ data[offset] = r;
+ data[offset + 1] = g;
+ data[offset + 2] = b;
+ data[offset + 3] = 255;
+ }
+ return { data, width };
+}
diff --git a/packages/layers/src/pointsFeatureColorExtension.ts b/packages/layers/src/pointsFeatureColorExtension.ts
index 99f8eeb7..4a38adae 100644
--- a/packages/layers/src/pointsFeatureColorExtension.ts
+++ b/packages/layers/src/pointsFeatureColorExtension.ts
@@ -1,58 +1,92 @@
-import type { Layer } from '@deck.gl/core';
+import type { Layer, UpdateParameters } from '@deck.gl/core';
import { LayerExtension } from '@deck.gl/core';
-import { PFC_CHROMA, PFC_GOLDEN_RATIO_CONJUGATE, PFC_LIGHTNESS } from './pointsFeatureColor.js';
+import {
+ buildFeaturePalette,
+ type FeatureColorOverrides,
+ featurePaletteWidth,
+} from './pointsFeatureColor.js';
-/** Render a JS number as a GLSL float literal (always with a decimal point, so an
- * integer-valued constant doesn't become an `int` in the shader). Lets the shader
- * interpolate the SAME palette constants the JS swatch mirror uses. */
-function glslFloat(value: number): string {
- const text = String(value);
- return text.includes('.') || text.includes('e') ? text : `${text}.0`;
+/** A luma texture, narrowed to the members this extension touches. */
+interface PaletteTexture {
+ width: number;
+ destroy?: () => void;
+ delete?: () => void;
+}
+
+interface DeviceLike {
+ createTexture(descriptor: Record): PaletteTexture;
}
/**
- * Uniform block for the highlight. The stored value is `highlightCode + 1`, so
- * the "no highlight" state is 0 — which is also what an unbound/zeroed UBO
- * reads, making the default safe even if the binding ever fails (feature code 0
- * would otherwise be a valid, and wrongly-highlighted, value).
+ * Uniform block for the colour pass:
+ * - `highlightCode`: the emphasised feature code + 1 (0 = no highlight; also what a
+ * zeroed UBO reads, so the default is safe even if the binding fails — code 0 would
+ * otherwise be a valid, wrongly-highlighted value).
+ * - `paletteWidth`: the LUT width, so the shader can clamp an out-of-range code to
+ * the last texel instead of reading undefined memory.
*/
-const PFC_HIGHLIGHT_MODULE = {
- name: 'pfcHighlight',
+const PFC_COLOR_MODULE = {
+ name: 'pfcColor',
vs: /* glsl */ `
- layout(std140) uniform pfcHighlightUniforms {
- float code;
- } pfcHighlight;
+ uniform sampler2D pfcPalette;
+ layout(std140) uniform pfcColorUniforms {
+ float highlightCode;
+ float paletteWidth;
+ } pfcColor;
`,
- uniformTypes: { code: 'f32' as const },
+ uniformTypes: { highlightCode: 'f32' as const, paletteWidth: 'f32' as const },
};
+/**
+ * Dispose a luma texture across the two method names different versions expose.
+ *
+ * `destroy()` is the luma v9 API; `delete()` is the deprecated alias kept for
+ * backwards compatibility. Prefer the first and fall back — calling both meant a
+ * double free on every version that has them BOTH, which is every version that
+ * has `delete()` at all.
+ */
+function destroyTexture(texture: PaletteTexture | undefined): void {
+ if (typeof texture?.destroy === 'function') {
+ texture.destroy();
+ return;
+ }
+ texture?.delete?.();
+}
+
/**
* Colours scatter points by their per-point feature code, entirely on the GPU.
*
- * The feature code rides along as an instance attribute (`featureCode`, supplied
- * as the binary `getFeatureCode` attribute) and a small vertex-shader hook maps
- * it to a categorical colour, overwriting `vFillColor`. The mapping is
- * procedural (a golden-angle hue from the code) so there is no palette buffer to
- * upload and no CPU colour pass; the code attribute is also the one a future
- * per-code visibility mask will read.
+ * The feature code rides along as an instance attribute (`featureCode`, supplied as
+ * the binary `getFeatureCode` attribute); the vertex shader looks the code up in a
+ * **palette texture** (`pfcPalette`) — a 1-row RGBA LUT, one texel per code — and
+ * writes the result to `vFillColor`. The palette is built on the CPU from
+ * {@link buildFeaturePalette} (defaults matching the old procedural golden-angle
+ * hue, plus any per-feature overrides), so colour is now DATA, not a hard-coded
+ * formula: a feature can be recoloured by patching one texel.
*
- * The extension is attached to EVERY scatter layer, not just when colour is on.
- * This is load-bearing: deck only calls an extension's `initializeState` when
- * the layer first mounts, so attaching it lazily (when colour is toggled on)
- * would never register the `featureCode` attribute — the sublayer already
- * exists and only updates. Colour is instead gated by the attribute value: with
- * no `getFeatureCode` buffer the attribute reads its `-1` default and the shader
- * leaves the flat fill colour untouched.
+ * Why a texture and not the old inline OKLab math: it makes per-feature override
+ * possible at all, gives one source of truth shared with the JS swatches, and keeps
+ * the shader trivial (one `texelFetch`). Hover highlight stays a UNIFORM
+ * (`highlightCode`), not a palette write — it changes every mousemove, and a uniform
+ * is far cheaper than re-uploading a texture per frame.
*
- * Two more deck subtleties that cost a debugging session:
- * - the `in float featureCode` declaration must be in `vs:#decl` (deck does NOT
- * auto-declare it) and the main hook in a TOP-LEVEL `inject` (a module's own
- * `inject` does not apply to the host layer here);
- * - `defaultProps.getFeatureCode` must be declared or deck treats the attribute
- * as constant and never reads the binary buffer (as DataFilterExtension does).
+ * The extension is attached to EVERY scatter layer, not just when colour is on. This
+ * is load-bearing: deck only calls an extension's `initializeState` when the layer
+ * first mounts, so attaching it lazily (when colour is toggled on) would never
+ * register the `featureCode` attribute — the sublayer already exists and only
+ * updates. Colour is instead gated by the attribute value: with no `getFeatureCode`
+ * buffer the attribute reads its `-1` default and the shader leaves the flat fill
+ * colour untouched. A palette texture is ALWAYS bound (a 1×1 fallback until a real
+ * one arrives), because a declared sampler with no binding is a draw error.
*
- * Deliberately the smallest possible deck extension — one attribute, one shader
- * hook — so it is a low-risk first candidate to port to a WebGPU shading model.
+ * Deck subtleties that each cost a debugging session:
+ * - `in float featureCode` must be declared in `vs:#decl` (deck does NOT auto-declare
+ * it) and the colour hook in a TOP-LEVEL `inject` (a module's own `inject` does not
+ * apply to the host layer here);
+ * - `defaultProps.getFeatureCode` must be declared or deck treats the attribute as
+ * constant and never reads the binary buffer (as DataFilterExtension does);
+ * - the `pfcPalette` sampler is bound with `model.setBindings` (mirroring
+ * `LabelsBitmaskTileLayer`), which is separate from the UBO's `setShaderModuleProps`.
*/
export class PointsFeatureColorExtension extends LayerExtension {
static get componentName(): string {
@@ -64,6 +98,11 @@ export class PointsFeatureColorExtension extends LayerExtension {
/** Emphasize one feature code: points of other codes are desaturated + dimmed
* while this is >= 0. -1 (default) highlights nothing. */
highlightFeatureCode: { type: 'number', value: -1 },
+ /** Number of feature codes the palette must cover (the catalog's `maxCode + 1`).
+ * Sizes the LUT texture. */
+ featureCodeSpaceSize: { type: 'number', value: 0 },
+ /** Per-feature colour overrides (`code → [r,g,b]`); absent codes keep the default. */
+ featureColorOverrides: { type: 'object', value: null as FeatureColorOverrides | null },
};
getShaders(this: Layer, extension: this) {
@@ -71,68 +110,31 @@ export class PointsFeatureColorExtension extends LayerExtension {
const shaders = (super.getShaders(extension) ?? {}) as { modules?: unknown[] };
return {
...shaders,
- modules: [...(shaders.modules ?? []), PFC_HIGHLIGHT_MODULE],
+ modules: [...(shaders.modules ?? []), PFC_COLOR_MODULE],
inject: {
'vs:#decl': /* glsl */ `
in float featureCode;
-
- // OKLab → linear sRGB → gamma sRGB. OKLab spaces hues perceptually
- // evenly, so a golden-angle sweep of its hue gives adjacent codes
- // colours that look as distinct as they are numerically (unlike HSV,
- // where big hue arcs — the greens — read as one colour). Out-of-gamut
- // (L,C) combinations are clamped rather than gamut-mapped; fine for
- // categorical swatches at a fixed moderate chroma.
- vec3 pfc_oklab2rgb(vec3 lab) {
- float l_ = lab.x + 0.3963377774 * lab.y + 0.2158037573 * lab.z;
- float m_ = lab.x - 0.1055613458 * lab.y - 0.0638541728 * lab.z;
- float s_ = lab.x - 0.0894841775 * lab.y - 1.2914855480 * lab.z;
- vec3 lms = vec3(l_ * l_ * l_, m_ * m_ * m_, s_ * s_ * s_);
- vec3 rgb = vec3(
- 4.0767416621 * lms.x - 3.3077115913 * lms.y + 0.2309699292 * lms.z,
- -1.2684380046 * lms.x + 2.6097574011 * lms.y - 0.3413193965 * lms.z,
- -0.0041960863 * lms.x - 0.7034186147 * lms.y + 1.7076147010 * lms.z
- );
- vec3 low = rgb * 12.92;
- vec3 high = 1.055 * pow(max(rgb, 0.0), vec3(1.0 / 2.4)) - 0.055;
- return clamp(mix(high, low, step(rgb, vec3(0.0031308))), 0.0, 1.0);
- }
-
- // Golden-angle hue in OKLCh at a fixed lightness/chroma. The lightness,
- // chroma and golden-ratio constants come from pointsFeatureColor.ts so
- // the swatches and the GPU points share one source (tweak them there).
- vec3 pfc_codeToColor(float code) {
- float h = fract(code * ${glslFloat(PFC_GOLDEN_RATIO_CONJUGATE)}) * 6.28318530717958648;
- return pfc_oklab2rgb(vec3(
- ${glslFloat(PFC_LIGHTNESS)},
- ${glslFloat(PFC_CHROMA)} * cos(h),
- ${glslFloat(PFC_CHROMA)} * sin(h)
- ));
- }
`,
'vs:#main-end': /* glsl */ `
if (featureCode >= 0.0) {
- vec3 pfcColor = pfc_codeToColor(featureCode);
- // Highlight: uniform holds highlightCode + 1 (0 = off). Non-matching
- // codes are desaturated toward their luminance and dimmed.
- if (pfcHighlight.code > 0.5 && abs(featureCode - (pfcHighlight.code - 1.0)) > 0.5) {
- float pfcLum = dot(pfcColor, vec3(0.2126, 0.7152, 0.0722));
- pfcColor = mix(vec3(pfcLum), pfcColor, 0.2) * 0.55;
+ // Clamp to the last texel so a code beyond the LUT mis-colours its tail
+ // rather than reading undefined texture memory (texelFetch ignores the
+ // sampler's clamp-to-edge, unlike texture()).
+ int pfcIdx = clamp(int(featureCode + 0.5), 0, int(pfcColor.paletteWidth) - 1);
+ vec3 pfcRgb = texelFetch(pfcPalette, ivec2(pfcIdx, 0), 0).rgb;
+ // Highlight: uniform holds highlightCode + 1 (0 = off). Non-matching codes
+ // are desaturated toward their luminance and dimmed.
+ if (pfcColor.highlightCode > 0.5 && abs(featureCode - (pfcColor.highlightCode - 1.0)) > 0.5) {
+ float pfcLum = dot(pfcRgb, vec3(0.2126, 0.7152, 0.0722));
+ pfcRgb = mix(vec3(pfcLum), pfcRgb, 0.2) * 0.55;
}
- vFillColor = vec4(pfcColor, vFillColor.a);
+ vFillColor = vec4(pfcRgb, vFillColor.a);
}
`,
},
};
}
- draw(this: Layer): void {
- const highlight = (this.props as { highlightFeatureCode?: number }).highlightFeatureCode ?? -1;
- // Store code + 1 so "no highlight" is 0 (safe default; see PFC_HIGHLIGHT_MODULE).
- (this as unknown as { setShaderModuleProps(props: unknown): void }).setShaderModuleProps({
- pfcHighlight: { code: highlight >= 0 ? highlight + 1 : 0 },
- });
- }
-
initializeState(this: Layer): void {
const attributeManager = this.getAttributeManager();
attributeManager?.add({
@@ -144,5 +146,114 @@ export class PointsFeatureColorExtension extends LayerExtension {
defaultValue: -1,
},
});
+ // Build from the ACTUAL props, not a hard-coded 1×1. This sublayer only mounts
+ // once there is a batch to draw, by which time the catalog is often already
+ // loaded — so `featureCodeSpaceSize` arrives at its final value here and never
+ // "changes" again. Seeding a 1×1 and waiting for a change left the palette one
+ // texel wide forever, and the shader clamps every code to texel 0: one flat
+ // colour for the whole layer. (`buildFeaturePalette` floors width at 1, so an
+ // unknown code space still yields a bindable fallback.)
+ const props = this.props as {
+ featureCodeSpaceSize?: number;
+ featureColorOverrides?: FeatureColorOverrides | null;
+ };
+ pfcSetPaletteTexture(
+ this,
+ pfcBuildPaletteTexture(
+ this,
+ props.featureCodeSpaceSize ?? 0,
+ props.featureColorOverrides ?? null
+ )
+ );
+ }
+
+ updateState(this: Layer, params: UpdateParameters): void {
+ const props = params.props as {
+ featureCodeSpaceSize?: number;
+ featureColorOverrides?: FeatureColorOverrides | null;
+ };
+ const oldProps = params.oldProps as typeof props;
+ const state = this.state as { pfcPaletteTexture?: PaletteTexture };
+ // Reconcile against the texture we actually hold rather than against a prop
+ // transition: a width mismatch means the palette cannot colour every code, no
+ // matter which update did or didn't fire. Self-healing, so a missed transition
+ // degrades to a rebuild instead of a permanently wrong palette.
+ const neededWidth = featurePaletteWidth(props.featureCodeSpaceSize ?? 0);
+ if (
+ state.pfcPaletteTexture?.width !== neededWidth ||
+ props.featureColorOverrides !== oldProps.featureColorOverrides
+ ) {
+ pfcSetPaletteTexture(
+ this,
+ pfcBuildPaletteTexture(
+ this,
+ props.featureCodeSpaceSize ?? 0,
+ props.featureColorOverrides ?? null
+ )
+ );
+ }
+ }
+
+ draw(this: Layer): void {
+ const props = this.props as { highlightFeatureCode?: number };
+ const state = this.state as { pfcPaletteTexture?: PaletteTexture; model?: unknown };
+ const highlight = props.highlightFeatureCode ?? -1;
+ const texture = state.pfcPaletteTexture;
+ // Store code + 1 so "no highlight" is 0 (safe default; see PFC_COLOR_MODULE).
+ (this as unknown as { setShaderModuleProps(props: unknown): void }).setShaderModuleProps({
+ pfcColor: {
+ highlightCode: highlight >= 0 ? highlight + 1 : 0,
+ paletteWidth: texture?.width ?? 1,
+ },
+ });
+ if (texture) {
+ (
+ state.model as { setBindings?: (b: Record) => void } | undefined
+ )?.setBindings?.({ pfcPalette: texture });
+ }
+ }
+
+ finalizeState(this: Layer): void {
+ const state = this.state as { pfcPaletteTexture?: PaletteTexture };
+ destroyTexture(state.pfcPaletteTexture);
+ state.pfcPaletteTexture = undefined;
+ }
+}
+
+/** Create the LUT texture for a code space (+ overrides). Falls back to a 1×1 texel
+ * when the code space isn't known yet, so a texture is always available to bind. */
+function pfcBuildPaletteTexture(
+ layer: Layer,
+ codeSpaceSize: number,
+ overrides: FeatureColorOverrides | null
+): PaletteTexture | undefined {
+ const device = (layer.context as { device?: DeviceLike } | undefined)?.device;
+ if (!device) {
+ return undefined;
+ }
+ const palette = buildFeaturePalette(codeSpaceSize, overrides ?? undefined);
+ return device.createTexture({
+ width: palette.width,
+ height: 1,
+ dimension: '2d',
+ data: palette.data,
+ mipmaps: false,
+ format: 'rgba8unorm',
+ sampler: {
+ minFilter: 'nearest',
+ magFilter: 'nearest',
+ addressModeU: 'clamp-to-edge',
+ addressModeV: 'clamp-to-edge',
+ },
+ });
+}
+
+/** Swap the layer's palette texture, disposing the previous one. */
+function pfcSetPaletteTexture(layer: Layer, texture: PaletteTexture | undefined): void {
+ const state = layer.state as { pfcPaletteTexture?: PaletteTexture };
+ if (state.pfcPaletteTexture === texture) {
+ return;
}
+ destroyTexture(state.pfcPaletteTexture);
+ state.pfcPaletteTexture = texture;
}
diff --git a/packages/layers/src/pointsLoader.ts b/packages/layers/src/pointsLoader.ts
index ed0c0aaf..4d88ef96 100644
--- a/packages/layers/src/pointsLoader.ts
+++ b/packages/layers/src/pointsLoader.ts
@@ -85,6 +85,13 @@ export function columnarBatchFromPointData(
bounds: options?.bounds,
loadMode: options?.loadMode,
pointCount,
+ // Both `PointData` and the batch declare row-aligned codes, and the batch's
+ // doc says they are carried through — so silently dropping them here made
+ // every caller that passed them (see `buildGrowingResource`) look correct
+ // while the field vanished. Inert today only because the render path
+ // re-supplies codes from props; a caller relying on the declared contract
+ // would get flat colour with nothing to point at.
+ ...(data.featureCodes ? { featureCodes: data.featureCodes } : {}),
};
}
diff --git a/packages/layers/src/pointsRenderAttributes.ts b/packages/layers/src/pointsRenderAttributes.ts
index 63525244..a03cab54 100644
--- a/packages/layers/src/pointsRenderAttributes.ts
+++ b/packages/layers/src/pointsRenderAttributes.ts
@@ -21,8 +21,26 @@ export interface PointsRenderAttributes {
featureCodes?: Float32Array;
}
+/**
+ * The `data` prop handed to deck: a length plus binary attribute descriptors.
+ *
+ * Deck compares `props.data` BY IDENTITY. A fresh object here — even one wrapping
+ * the very same buffers — sets `dataChanged` and invalidates every attribute, so
+ * the whole position buffer is re-uploaded. Memoizing the buffers is not enough;
+ * the wrapper has to be stable too.
+ */
+export interface PointsDeckData {
+ length: number;
+ attributes: {
+ getPosition: { value: Float32Array; size: number };
+ getFeatureCode?: { value: Float32Array; size: number };
+ };
+}
+
interface CacheEntry extends PointsRenderAttributes {
use3d: boolean;
+ /** Memoized `data` wrappers, one per colour mode (the attribute set differs). */
+ deckData: { colored?: PointsDeckData; plain?: PointsDeckData };
}
const cache = new WeakMap();
@@ -62,7 +80,44 @@ export function buildPointsAttributes(
featureCodes = codes instanceof Float32Array ? codes : Float32Array.from(codes);
}
- const entry: CacheEntry = { length, positions, featureCodes, use3d };
+ const entry: CacheEntry = { length, positions, featureCodes, use3d, deckData: {} };
cache.set(batch, entry);
return entry;
}
+
+/**
+ * The deck `data` prop for a batch — the SAME object on every call for a given
+ * (batch, use3d, colorByFeature).
+ *
+ * Rebuilding it per render made deck treat the layer's data as new and re-upload
+ * every binary attribute, so an unrelated prop change (point size, opacity, a
+ * hover) cost a full position re-upload: ~80ms of `bufferSubData` for a 3.7M-point
+ * element, ~44MB of positions. Neither `getRadius` vs `radiusScale` nor the
+ * `updateTriggers` entry mattered, because `dataChanged` invalidates everything
+ * regardless of triggers.
+ */
+export function buildPointsDeckData(
+ batch: ColumnarNdarrayPointsBatch,
+ use3d: boolean,
+ colorByFeature: boolean
+): PointsDeckData {
+ const attributes = buildPointsAttributes(batch, use3d);
+ const entry = cache.get(batch);
+ const codes = colorByFeature ? attributes.featureCodes : undefined;
+ const slot = codes ? 'colored' : 'plain';
+ const cached = entry?.deckData[slot];
+ if (cached) {
+ return cached;
+ }
+ const data: PointsDeckData = {
+ length: attributes.length,
+ attributes: {
+ getPosition: { value: attributes.positions, size: 3 },
+ ...(codes ? { getFeatureCode: { value: codes, size: 1 } } : {}),
+ },
+ };
+ if (entry) {
+ entry.deckData[slot] = data;
+ }
+ return data;
+}
diff --git a/packages/layers/src/pointsScatterLayer.ts b/packages/layers/src/pointsScatterLayer.ts
index c1373870..be6ec697 100644
--- a/packages/layers/src/pointsScatterLayer.ts
+++ b/packages/layers/src/pointsScatterLayer.ts
@@ -1,17 +1,46 @@
import type { Matrix4 } from '@math.gl/core';
import { ScatterplotLayer } from 'deck.gl';
+import type { FeatureColorOverrides } from './pointsFeatureColor.js';
import { PointsFeatureColorExtension } from './pointsFeatureColorExtension.js';
import type { ColumnarNdarrayPointsBatch } from './pointsLoader.js';
-import { buildPointsAttributes } from './pointsRenderAttributes.js';
+import { buildPointsAttributes, buildPointsDeckData } from './pointsRenderAttributes.js';
/** Orthographic zoom at which configured pointSize applies at full scale. */
export const POINT_SIZE_ZOOM_REFERENCE = 0;
/** Minimum radius multiplier when zoomed out (reduces fragment overdraw). */
export const MIN_POINT_SIZE_SCALE = 0.15;
export const DEFAULT_POINT_SIZE = 0.1;
-export const DEFAULT_POINT_RADIUS_MIN_PIXELS = 0.1;
+export const DEFAULT_POINT_RADIUS_MIN_PIXELS = 0.01;
export const DEFAULT_POINT_RADIUS_MAX_PIXELS = 3;
+/**
+ * The uniform scale factor a model matrix applies to positions.
+ *
+ * Deck runs `modelMatrix` over POSITIONS but not over a `'common'`-unit radius, so
+ * the two disagree by exactly this factor. A SpatialData element's transform is
+ * whatever the writer chose — a Xenium element may scale by 4.7, while an element
+ * expressed in millimetres scales by 0.00012 — and without correction the same
+ * `pointSize` renders ~40000x differently between those two.
+ *
+ * Returns the geometric mean of the x and y basis lengths, so a mildly anisotropic
+ * transform gets a sensible single radius; 1 for a missing or degenerate matrix,
+ * which leaves sizing exactly as it was.
+ */
+export function modelMatrixUniformScale(matrix: Matrix4 | null | undefined): number {
+ if (!matrix) {
+ return 1;
+ }
+ // Column-major: column 0 is the x basis, column 1 the y basis.
+ const m = matrix as unknown as ArrayLike;
+ if (typeof m[0] !== 'number') {
+ return 1;
+ }
+ const scaleX = Math.hypot(m[0] ?? 0, m[1] ?? 0, m[2] ?? 0);
+ const scaleY = Math.hypot(m[4] ?? 0, m[5] ?? 0, m[6] ?? 0);
+ const scale = Math.sqrt(scaleX * scaleY);
+ return Number.isFinite(scale) && scale > 0 ? scale : 1;
+}
+
export function zoomScaledPointSize(
pointSize: number,
zoom: number | null | undefined,
@@ -39,6 +68,12 @@ export interface PointsScatterStyleProps {
tileSubLayer?: boolean;
/** Colour points by their per-point feature code (requires batch codes). */
colorByFeature?: boolean;
+ /** Number of feature codes the colour LUT must cover (catalog `maxCode + 1`). */
+ featureCodeSpaceSize?: number;
+ /** Per-feature colour overrides (`code → [r,g,b]`); absent codes keep the default. */
+ featureColorOverrides?: FeatureColorOverrides | null;
+ /** Emphasise one feature code (others desaturate + dim); -1 highlights nothing. */
+ highlightFeatureCode?: number;
}
// One shared extension instance: it is stateless, so every scatter layer that
@@ -60,6 +95,13 @@ export function renderColumnarScatterLayer(
const radiusUnits: 'common' | 'pixels' = isTile ? 'pixels' : 'common';
const radiusMinPixels = props.pointRadiusMinPixels ?? DEFAULT_POINT_RADIUS_MIN_PIXELS;
const radiusMaxPixels = props.pointRadiusMaxPixels ?? DEFAULT_POINT_RADIUS_MAX_PIXELS;
+ // `pointSize` means "this many units of the ELEMENT's own coordinate space".
+ // Deck transforms positions by `modelMatrix` but leaves a common-unit radius
+ // alone, so without folding the matrix scale in here the same pointSize renders
+ // wildly differently per element: an element with a 0.00012 mm affine drew points
+ // ~8000x too large for its data, swamping the view and shredding fill rate.
+ // Pixel-unit tiles are already viewport-relative and must not be rescaled.
+ const transformScale = isTile ? 1 : modelMatrixUniformScale(props.modelMatrix);
// Feed deck GPU-ready binary attributes (interleaved positions) instead of a
// per-object `getPosition` closure. The buffer is memoized on the batch, so a
@@ -84,22 +126,23 @@ export function renderColumnarScatterLayer(
return new ScatterplotLayer({
id,
coordinateSystem: 'cartesian',
- data: {
- length: attributes.length,
- attributes: {
- getPosition: { value: attributes.positions, size: 3 },
- ...(colorByFeature
- ? { getFeatureCode: { value: attributes.featureCodes as Float32Array, size: 1 } }
- : {}),
- },
- },
+ // Memoized per (batch, use3d, colorByFeature): deck compares `data` by
+ // IDENTITY, so a fresh wrapper — even around identical buffers — invalidates
+ // every attribute and re-uploads the whole position buffer.
+ data: buildPointsDeckData(batch, props.use3d === true, colorByFeature),
...(props.tileBounds ? { bounds: props.tileBounds } : {}),
extensions: [pointsFeatureColorExtension],
+ // Sizes the colour LUT texture and supplies any per-feature overrides — read by
+ // the extension to (re)build `pfcPalette`.
+ featureCodeSpaceSize: props.featureCodeSpaceSize ?? 0,
+ ...(props.featureColorOverrides ? { featureColorOverrides: props.featureColorOverrides } : {}),
+ highlightFeatureCode: props.highlightFeatureCode ?? -1,
// Constant default: the binary getFeatureCode attribute overrides it when
// colouring; when it is withdrawn (colour off), deck reverts to this -1, so
// the shader's `featureCode >= 0.0` guard falls through to the flat colour.
getFeatureCode: -1,
- getRadius: props.pointSize,
+ // getRadius: props.pointSize,
+ radiusScale: props.pointSize * transformScale,
radiusUnits,
radiusMinPixels,
radiusMaxPixels,
@@ -109,8 +152,8 @@ export function renderColumnarScatterLayer(
pickable: true,
autoHighlight: true,
highlightColor: [255, 255, 0, 200],
- updateTriggers: {
- getRadius: [props.pointSize],
- },
+ // updateTriggers: {
+ // radiusScale: [props.pointSize],
+ // },
});
}
diff --git a/packages/layers/src/preloadedScatterStrategy.ts b/packages/layers/src/preloadedScatterStrategy.ts
index 9486f5dd..b1fe5931 100644
--- a/packages/layers/src/preloadedScatterStrategy.ts
+++ b/packages/layers/src/preloadedScatterStrategy.ts
@@ -1,7 +1,11 @@
import { applyRenderCapToColumnar } from '@spatialdata/core';
import type { Layer, LayersList } from 'deck.gl';
import type { PointsLayer } from './PointsLayer.js';
-import { featureFilterAwaitingRowCodes, filterBatchSignature } from './pointsFeatureCodes.js';
+import {
+ featureCodesSignature,
+ featureFilterAwaitingRowCodes,
+ filterBatchSignature,
+} from './pointsFeatureCodes.js';
import type { ColumnarNdarrayPointsBatch } from './pointsLoader.js';
import type { PointsRenderStrategy } from './pointsRenderStrategies.js';
import { DEFAULT_POINT_SIZE, renderColumnarScatterLayer } from './pointsScatterLayer.js';
@@ -38,20 +42,21 @@ function resolveScatterBatch(layer: PointsLayer): ColumnarNdarrayPointsBatch | u
return state.filteredBatch;
}
- // The selection changed and the new filtered batch is still computing off-thread.
- // Keep showing the PREVIOUS filtered result rather than flashing the full
- // unfiltered batch — but only when that previous result was itself a real
- // selection, not the unfiltered "all features" batch (whose signature matches
- // `featureCodes === undefined`). Reusing the unfiltered batch here is exactly
- // the flash-of-all-points the filter is meant to avoid.
- const unfilteredSignature = filterBatchSignature(undefined, preloadedFeatureCodes, renderCap);
- if (state.filteredBatch && state.filteredBatchSignature !== unfilteredSignature) {
+ // A new filtered batch is still computing off-thread. We may keep the PREVIOUS
+ // filtered result on screen ONLY while it is the SAME SET OF GENES — e.g. the render
+ // cap or the row-code buffer moved but the selection did not. If the SELECTION
+ // itself changed (A → B), reusing the previous batch would draw gene A under a
+ // gene-B selection: the "wrong gene shown" bug. The gene signature is the first
+ // segment of `filterBatchSignature` (`featureCodesSignature | preloaded | renderCap`).
+ const currentGeneSignature = featureCodesSignature(featureCodes);
+ const staleGeneSignature = state.filteredBatchSignature?.split('|')[0];
+ if (state.filteredBatch && staleGeneSignature === currentGeneSignature) {
return state.filteredBatch;
}
- // No reusable filtered batch. Draw the full batch only when nothing is selected;
- // while a selection's first filter is pending, draw nothing (a brief blank beats
- // a misleading flash of every feature).
+ // No reusable filtered batch for these genes. Draw the full batch only when nothing
+ // is selected; while a changed selection's first filter is pending, draw nothing (a
+ // brief blank beats showing either every feature or the previous selection's genes).
if (featureCodes === undefined) {
return cappedPreloaded();
}
@@ -71,6 +76,9 @@ export const preloadedScatterStrategy: PointsRenderStrategy = {
color = [255, 100, 100, 200],
use3d,
colorByFeature,
+ featureCodeSpaceSize,
+ featureColorOverrides,
+ highlightFeatureCode,
} = layer.props;
if (!visible) {
@@ -97,6 +105,9 @@ export const preloadedScatterStrategy: PointsRenderStrategy = {
modelMatrix: layer.props.modelMatrix,
use3d,
colorByFeature,
+ ...(featureCodeSpaceSize !== undefined ? { featureCodeSpaceSize } : {}),
+ ...(featureColorOverrides ? { featureColorOverrides } : {}),
+ ...(highlightFeatureCode !== undefined ? { highlightFeatureCode } : {}),
});
},
};
diff --git a/packages/layers/tests/pointsDataEngine.spec.ts b/packages/layers/tests/pointsDataEngine.spec.ts
index 43c9acd2..bf9850f7 100644
--- a/packages/layers/tests/pointsDataEngine.spec.ts
+++ b/packages/layers/tests/pointsDataEngine.spec.ts
@@ -334,21 +334,26 @@ describe('PointsDataEngine — feature catalog', () => {
expect(engine.getFeatureCatalog('pts:other')).toBeUndefined();
});
- it('records null and stays settled when the scan rejects', async () => {
+ it('leaves the catalog retryable when the scan rejects, and retry() recovers it', async () => {
+ // A4: a failed full-catalog scan no longer settles permanently as null — it is a
+ // retryable `failed`, so it is not "loaded", not "loading", and retry() re-runs it.
const engine = new PointsDataEngine();
+ let attempts = 0;
const element = {
key: 'pts:boom',
listFeaturesWithCounts: vi.fn(async () => {
- throw new Error('scan failed');
+ attempts += 1;
+ if (attempts === 1) throw new Error('scan failed');
+ return sampleCatalog;
}),
} as unknown as PointsElement;
- const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
await engine.ensureFeatureCatalog({ key: 'pts:boom', layerId: 'l', element });
-
- expect(engine.getFeatureCatalog('pts:boom')).toBeNull();
+ expect(engine.getFeatureCatalog('pts:boom')).toBeUndefined();
expect(engine.isFeatureCatalogLoading('pts:boom')).toBe(false);
- errSpy.mockRestore();
+
+ await engine.retry('pts:boom');
+ expect(engine.getFeatureCatalog('pts:boom')).toEqual(sampleCatalog);
});
});
@@ -363,7 +368,14 @@ describe('PointsDataEngine — row feature codes', () => {
expect(engine.hasRowFeatureCodes('pts:rc')).toBe(true);
expect(Array.from(engine.getRowFeatureCodes('pts:rc')!)).toEqual([0, 1, 0]);
- expect(loadRowFeatureCodes).toHaveBeenCalledWith({ featureCatalog: sampleCatalog });
+ // R5 fix (Track A): the codes are read at the resident preload's cap so they
+ // stay row-aligned with the batch. No preload ran here, so the cap is the default.
+ expect(loadRowFeatureCodes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ featureCatalog: sampleCatalog,
+ memoryCap: DEFAULT_POINTS_MEMORY_CAP,
+ })
+ );
});
it('passes undefined catalog when none is built yet (core scans internally)', async () => {
@@ -372,7 +384,13 @@ describe('PointsDataEngine — row feature codes', () => {
await engine.ensureRowFeatureCodes({ key: 'pts:rc2', layerId: 'l', element });
- expect(loadRowFeatureCodes).toHaveBeenCalledWith({ featureCatalog: undefined });
+ // R5 fix (Track A): the cap is threaded through even with no catalog yet.
+ expect(loadRowFeatureCodes).toHaveBeenCalledWith(
+ expect.objectContaining({
+ featureCatalog: undefined,
+ memoryCap: DEFAULT_POINTS_MEMORY_CAP,
+ })
+ );
});
it('is idempotent', async () => {
@@ -853,3 +871,60 @@ describe('PointsDataEngine — shed complete batch on lower', () => {
});
});
});
+
+describe('PointsDataEngine — colour LUT inputs', () => {
+ it('reports the feature code-space size as maxCode + 1, memoised on catalog identity', async () => {
+ const engine = new PointsDataEngine();
+ const { element } = makeFeatureElement('pts:lut');
+ expect(engine.getFeatureCodeSpaceSize('pts:lut')).toBe(0); // no catalog yet
+ await engine.ensureFeatureCatalog({ key: 'pts:lut', layerId: 'l', element });
+ expect(engine.getFeatureCodeSpaceSize('pts:lut')).toBe(2); // codes {0,1} → width 2
+ });
+
+ it('resolves by-name colour overrides to a code→rgb map via the catalog', async () => {
+ const engine = new PointsDataEngine();
+ const { element } = makeFeatureElement('pts:ov');
+ await engine.ensureFeatureCatalog({ key: 'pts:ov', layerId: 'l', element });
+
+ const map = engine.getFeatureColorOverrideMap('pts:ov', {
+ GeneB: [10, 20, 30],
+ });
+ expect(map?.get(1)).toEqual([10, 20, 30]); // GeneB is code 1
+ expect(map?.has(0)).toBe(false);
+ // A name absent from the catalog is dropped, not thrown.
+ expect(engine.getFeatureColorOverrideMap('pts:ov', { Nope: [1, 2, 3] })).toBeNull();
+ });
+
+ it('returns null (all-default palette) with no overrides, and a stable map identity otherwise', async () => {
+ const engine = new PointsDataEngine();
+ const { element } = makeFeatureElement('pts:ov2');
+ await engine.ensureFeatureCatalog({ key: 'pts:ov2', layerId: 'l', element });
+ expect(engine.getFeatureColorOverrideMap('pts:ov2', undefined)).toBeNull();
+
+ const overrides = { GeneA: [1, 2, 3] as [number, number, number] };
+ const first = engine.getFeatureColorOverrideMap('pts:ov2', overrides);
+ // Same (config, catalog) → same map identity, so the palette texture downstream
+ // is not rebuilt on every getLayers frame.
+ expect(engine.getFeatureColorOverrideMap('pts:ov2', overrides)).toBe(first);
+ });
+
+ it('holds a per-element hover highlight and notifies subscribers on change only', () => {
+ const engine = new PointsDataEngine();
+ let notifications = 0;
+ engine.subscribe(() => {
+ notifications += 1;
+ });
+ expect(engine.getHighlightedFeature('pts:h')).toBe(-1); // none by default
+
+ engine.setHighlightedFeature('pts:h', 3);
+ expect(engine.getHighlightedFeature('pts:h')).toBe(3);
+ expect(notifications).toBe(1);
+
+ engine.setHighlightedFeature('pts:h', 3); // unchanged → no repaint
+ expect(notifications).toBe(1);
+
+ engine.setHighlightedFeature('pts:h', null); // clear
+ expect(engine.getHighlightedFeature('pts:h')).toBe(-1);
+ expect(notifications).toBe(2);
+ });
+});
diff --git a/packages/layers/tests/pointsFeatureColor.spec.ts b/packages/layers/tests/pointsFeatureColor.spec.ts
index a8b03335..2776bc77 100644
--- a/packages/layers/tests/pointsFeatureColor.spec.ts
+++ b/packages/layers/tests/pointsFeatureColor.spec.ts
@@ -1,5 +1,11 @@
import { describe, expect, it } from 'vitest';
-import { featureCodeToCssColor, featureCodeToRgb } from '../src/pointsFeatureColor.js';
+import {
+ buildFeaturePalette,
+ DEFAULT_FEATURE_PALETTE_WIDTH,
+ featureCodeToCssColor,
+ featureCodeToRgb,
+ featurePaletteWidth,
+} from '../src/pointsFeatureColor.js';
describe('featureCodeToRgb', () => {
it('returns grey for the negative "no colour" sentinel', () => {
@@ -24,3 +30,52 @@ describe('featureCodeToRgb', () => {
expect(featureCodeToCssColor(42)).toBe(`rgb(${r}, ${g}, ${b})`);
});
});
+
+describe('buildFeaturePalette', () => {
+ it('lays out one RGBA texel per code, defaulting to featureCodeToRgb', () => {
+ const palette = buildFeaturePalette(3);
+ // Width is a floor, not the requested size — see DEFAULT_FEATURE_PALETTE_WIDTH.
+ expect(palette.width).toBe(DEFAULT_FEATURE_PALETTE_WIDTH);
+ expect(palette.data.length).toBe(DEFAULT_FEATURE_PALETTE_WIDTH * 4);
+ for (let code = 0; code < 3; code += 1) {
+ const [r, g, b] = featureCodeToRgb(code);
+ const o = code * 4;
+ // The LUT must match the procedural colour byte-for-byte (look-preserving swap).
+ expect([
+ palette.data[o],
+ palette.data[o + 1],
+ palette.data[o + 2],
+ palette.data[o + 3],
+ ]).toEqual([r, g, b, 255]);
+ }
+ });
+
+ it('patches only the overridden codes, leaving the rest at their default', () => {
+ const palette = buildFeaturePalette(4, new Map([[2, [10, 20, 30] as const]]));
+ expect([palette.data[8], palette.data[9], palette.data[10], palette.data[11]]).toEqual([
+ 10, 20, 30, 255,
+ ]);
+ const [r, g, b] = featureCodeToRgb(1);
+ expect([palette.data[4], palette.data[5], palette.data[6]]).toEqual([r, g, b]);
+ });
+
+ it('covers a default code space even with NO catalog, so colour works before one loads', () => {
+ // The regression this pins: sizing the LUT from the catalog made it 1 texel wide
+ // until the catalog landed, and the shader clamps every code to texel 0 — the
+ // whole layer one flat colour for the entire load of a big element.
+ const palette = buildFeaturePalette(0);
+ expect(palette.width).toBe(DEFAULT_FEATURE_PALETTE_WIDTH);
+
+ // Distinct codes must get distinct texels with no catalog at all.
+ const texel = (code: number) => [...palette.data.slice(code * 4, code * 4 + 3)];
+ expect(texel(7)).toEqual([...featureCodeToRgb(7)]);
+ expect(texel(500)).not.toEqual(texel(7));
+ });
+
+ it('widens past the default when the catalog code space is larger', () => {
+ expect(buildFeaturePalette(10_000).width).toBe(10_000);
+ expect(featurePaletteWidth(10_000)).toBe(10_000);
+ // …and never narrows below the default for a small/unknown space.
+ expect(featurePaletteWidth(12)).toBe(DEFAULT_FEATURE_PALETTE_WIDTH);
+ });
+});
diff --git a/packages/layers/tests/pointsFeatureColorExtension.spec.ts b/packages/layers/tests/pointsFeatureColorExtension.spec.ts
index 24250ea8..460df2a2 100644
--- a/packages/layers/tests/pointsFeatureColorExtension.spec.ts
+++ b/packages/layers/tests/pointsFeatureColorExtension.spec.ts
@@ -32,4 +32,27 @@ describe('PointsFeatureColorExtension', () => {
expect(mainEnd).toContain('featureCode >= 0.0');
expect(mainEnd).toContain('vFillColor');
});
+
+ it('samples the colour from the pfcPalette LUT (not a procedural formula)', () => {
+ const mainEnd = shaders.inject['vs:#main-end'];
+ expect(mainEnd).toContain('texelFetch(pfcPalette');
+ // The palette module must declare the sampler + the width used to clamp the index.
+ const paletteModule = (shaders.modules as Array<{ name: string; vs: string }>).find(
+ (m) => m.name === 'pfcColor'
+ );
+ expect(paletteModule?.vs).toContain('sampler2D pfcPalette');
+ expect(paletteModule?.vs).toContain('paletteWidth');
+ });
+
+ it('declares the LUT-sizing and override props so deck tracks them', () => {
+ // Without these on the extension's defaultProps, deck would not diff them and the
+ // texture would never rebuild when the code space or overrides change.
+ expect(PointsFeatureColorExtension.defaultProps.featureCodeSpaceSize).toEqual({
+ type: 'number',
+ value: 0,
+ });
+ expect(PointsFeatureColorExtension.defaultProps.featureColorOverrides).toMatchObject({
+ type: 'object',
+ });
+ });
});
diff --git a/packages/layers/tests/pointsLayerLoadOrdering.spec.ts b/packages/layers/tests/pointsLayerLoadOrdering.spec.ts
new file mode 100644
index 00000000..b67ad97a
--- /dev/null
+++ b/packages/layers/tests/pointsLayerLoadOrdering.spec.ts
@@ -0,0 +1,138 @@
+import { describe, expect, it } from 'vitest';
+import { PointsLayer } from '../src/PointsLayer.js';
+import type { ColumnarNdarrayPointsBatch, PointsLoader } from '../src/pointsLoader.js';
+
+/**
+ * `preloadedBatch` is written after an `await`, so whichever read resolves LAST
+ * wins regardless of which one is current. Two ways that goes wrong:
+ *
+ * 1. The streaming overlay bumps `resourceRevision` per chunk, so several reads
+ * of the same loader can be in flight. An earlier, slower one landing last
+ * replaces the grown buffer with a smaller one — points disappear mid-stream.
+ * 2. A loader swap (a cap raise) resets the batch state and starts a fresh read,
+ * but a read already in flight against the OLD loader still resolves and
+ * overwrites it. A revision check alone misses this: revisions are per-holder
+ * and can coincide across the swap.
+ *
+ * (1) is currently masked by the adapter's `loadAll` being await-free, so it
+ * resolves in call order — an accident of one implementation, not a guarantee of
+ * the `PointsLoader` contract these methods are written against. These drive the
+ * reads directly with a controllable `loadAll` so both are pinned regardless.
+ */
+
+function batchOf(pointCount: number): ColumnarNdarrayPointsBatch {
+ return {
+ format: 'columnar-ndarray',
+ shape: [2, pointCount],
+ data: [new Float32Array(pointCount), new Float32Array(pointCount)],
+ pointCount,
+ };
+}
+
+type Deferred = { promise: Promise; resolve: () => void };
+
+function deferredLoader(batch: ColumnarNdarrayPointsBatch): {
+ loader: PointsLoader;
+ release: () => void;
+} {
+ let release = (): void => {};
+ const gate = new Promise((resolve) => {
+ release = resolve;
+ });
+ const loader = {
+ capabilities: { kind: 'preloaded-columnar' },
+ loadInBounds: async () => batch,
+ loadAll: async () => {
+ await gate;
+ return batch;
+ },
+ } as unknown as PointsLoader;
+ return { loader, release };
+}
+
+/**
+ * `PointsLayer` extends deck's `CompositeLayer`, whose `props`/`state` are managed
+ * by the layer lifecycle. Drive the private reads against a stand-in with the same
+ * two accessors rather than booting a deck instance — the race lives entirely in
+ * the await/setState ordering.
+ */
+function harness(initialLoader: PointsLoader) {
+ const layer = Object.create(PointsLayer.prototype) as PointsLayer & {
+ ensurePreloadedBatch(): Promise;
+ refreshPreloadedBatch(): Promise;
+ };
+ let state: Record = { filterGeneration: 0 };
+ let props: Record = {
+ resource: { loader: initialLoader },
+ resourceRevision: 0,
+ };
+ Object.defineProperty(layer, 'props', { get: () => props });
+ Object.defineProperty(layer, 'state', {
+ get: () => state,
+ set: (next: Record) => {
+ state = next;
+ },
+ });
+ (layer as unknown as { setState(patch: Record): void }).setState = (patch) => {
+ state = { ...state, ...patch };
+ };
+ return {
+ layer,
+ setProps(next: { loader?: PointsLoader; revision?: number }) {
+ props = {
+ resource: { loader: next.loader ?? (props.resource as { loader: PointsLoader }).loader },
+ resourceRevision: next.revision ?? props.resourceRevision,
+ };
+ },
+ preloadedPointCount: () =>
+ (state.preloadedBatch as ColumnarNdarrayPointsBatch | undefined)?.pointCount,
+ };
+}
+
+describe('PointsLayer — out-of-order loadAll resolutions', () => {
+ it('does not let a slower earlier revision overwrite a newer batch', async () => {
+ const small = deferredLoader(batchOf(10)); // revision 1: 10 points
+ const grown = deferredLoader(batchOf(40)); // revision 2: the same buffer, grown
+
+ // One loader identity whose backing buffer grows — model it as two reads by
+ // swapping which deferred `loadAll` the props expose, keeping the revision
+ // bump as the only signal, exactly as the streaming overlay does.
+ const h = harness(small.loader);
+ h.setProps({ revision: 1 });
+ const first = h.layer.refreshPreloadedBatch();
+
+ h.setProps({ loader: grown.loader, revision: 2 });
+ const second = h.layer.refreshPreloadedBatch();
+
+ // The NEWER read completes first, then the older one resolves late.
+ grown.release();
+ await second;
+ expect(h.preloadedPointCount()).toBe(40);
+
+ small.release();
+ await first;
+ expect(h.preloadedPointCount()).toBe(40); // not clobbered back down to 10
+ });
+
+ it('does not let a read against a replaced loader overwrite the new one', async () => {
+ // A cap raise swaps the loader; `updateState` resets the batch state and starts
+ // a fresh read. The read already in flight against the old loader must not land.
+ const oldLoader = deferredLoader(batchOf(10));
+ const newLoader = deferredLoader(batchOf(99));
+
+ const h = harness(oldLoader.loader);
+ const stale = h.layer.refreshPreloadedBatch();
+
+ // Same revision across the swap — the case a revision-only guard misses.
+ h.setProps({ loader: newLoader.loader });
+ const fresh = h.layer.ensurePreloadedBatch();
+
+ newLoader.release();
+ await fresh;
+ expect(h.preloadedPointCount()).toBe(99);
+
+ oldLoader.release();
+ await stale;
+ expect(h.preloadedPointCount()).toBe(99); // the old loader's batch never lands
+ });
+});
diff --git a/packages/layers/tests/pointsRenderAttributes.spec.ts b/packages/layers/tests/pointsRenderAttributes.spec.ts
index 5e8a429c..a9a6926d 100644
--- a/packages/layers/tests/pointsRenderAttributes.spec.ts
+++ b/packages/layers/tests/pointsRenderAttributes.spec.ts
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest';
import type { ColumnarNdarrayPointsBatch } from '../src/pointsLoader.js';
-import { buildPointsAttributes } from '../src/pointsRenderAttributes.js';
+import { buildPointsAttributes, buildPointsDeckData } from '../src/pointsRenderAttributes.js';
function batch(overrides: Partial): ColumnarNdarrayPointsBatch {
return {
@@ -48,3 +48,42 @@ describe('buildPointsAttributes', () => {
expect(buildPointsAttributes(b, true).positions).not.toBe(first.positions);
});
});
+
+describe('buildPointsDeckData — identity stability', () => {
+ // Deck compares `props.data` by identity: a fresh wrapper marks the data as
+ // changed and re-uploads every binary attribute. On a 3.7M-point element that
+ // was ~80ms of bufferSubData whenever an unrelated prop (point size, opacity)
+ // changed, because the wrapper was rebuilt on every render.
+ it('returns the SAME object for repeated calls on one batch', () => {
+ const b = batch({});
+ const first = buildPointsDeckData(b, false, false);
+ const second = buildPointsDeckData(b, false, false);
+ expect(second).toBe(first);
+ expect(second.attributes.getPosition.value).toBe(first.attributes.getPosition.value);
+ });
+
+ it('keeps separate stable wrappers per colour mode', () => {
+ const b = batch({ featureCodes: new Float32Array([0, 1, 0]) });
+ const colored = buildPointsDeckData(b, false, true);
+ const plain = buildPointsDeckData(b, false, false);
+
+ expect(colored).not.toBe(plain);
+ expect(colored.attributes.getFeatureCode?.value).toBeInstanceOf(Float32Array);
+ expect(plain.attributes.getFeatureCode).toBeUndefined();
+ // …and each stays stable across repeat calls, so toggling colour back and
+ // forth does not churn the wrapper.
+ expect(buildPointsDeckData(b, false, true)).toBe(colored);
+ expect(buildPointsDeckData(b, false, false)).toBe(plain);
+ });
+
+ it('gives a different wrapper for a different batch', () => {
+ const a = buildPointsDeckData(batch({}), false, false);
+ const b = buildPointsDeckData(batch({}), false, false);
+ expect(b).not.toBe(a);
+ });
+
+ it('omits getFeatureCode when the batch carries no codes', () => {
+ const data = buildPointsDeckData(batch({}), false, true);
+ expect(data.attributes.getFeatureCode).toBeUndefined();
+ });
+});
diff --git a/packages/layers/tests/pointsRenderStrategies.spec.ts b/packages/layers/tests/pointsRenderStrategies.spec.ts
index c4c4996f..b68afd25 100644
--- a/packages/layers/tests/pointsRenderStrategies.spec.ts
+++ b/packages/layers/tests/pointsRenderStrategies.spec.ts
@@ -1,5 +1,6 @@
import { describe, expect, it } from 'vitest';
import type { PointsLayer } from '../src/PointsLayer.js';
+import { filterBatchSignature } from '../src/pointsFeatureCodes.js';
import { resolvePointsRenderStrategy } from '../src/pointsRenderStrategies.js';
import { preloadedScatterStrategy } from '../src/preloadedScatterStrategy.js';
@@ -62,3 +63,57 @@ describe('preloadedScatterStrategy sublayer id', () => {
expect(layer?.id).not.toBe(compositeId);
});
});
+
+describe('preloadedScatterStrategy — never shows the previous selection while a new one filters', () => {
+ const codes = new Int32Array([0, 1, 0]);
+ const preloadedBatch = {
+ format: 'columnar-ndarray' as const,
+ data: [new Float32Array([0, 1, 2]), new Float32Array([0, 1, 2])],
+ shape: [2, 3] as [number, number],
+ pointCount: 3,
+ featureCodes: codes,
+ };
+ // A previously-computed filtered batch for gene {0} (2 of the 3 rows).
+ const filteredForZero = {
+ format: 'columnar-ndarray' as const,
+ data: [new Float32Array([0, 2]), new Float32Array([0, 2])],
+ shape: [2, 2] as [number, number],
+ pointCount: 2,
+ featureCodes: new Int32Array([0, 0]),
+ };
+ const drawnCount = (layer: unknown): number | undefined =>
+ (layer as { props?: { data?: { length?: number } } } | null)?.props?.data?.length;
+
+ function layerWith(props: Record): PointsLayer {
+ return {
+ props: { id: 'points:x', visible: true, ...props },
+ state: {
+ preloadedBatch,
+ filteredBatch: filteredForZero,
+ filteredBatchSignature: filterBatchSignature([0], codes, undefined),
+ },
+ } as unknown as PointsLayer;
+ }
+
+ it('draws nothing (not the old gene) when the selection changed and the new filter is pending', () => {
+ // Selection moved {0} → {1}; the stale filteredBatch still holds gene {0}. Reusing
+ // it would draw gene 0 under a gene-1 selection — the "wrong gene shown" bug.
+ const layer = layerWith({ featureCodes: [1], preloadedFeatureCodes: codes });
+ expect(preloadedScatterStrategy.renderLayers(layer)).toBeNull();
+ });
+
+ it('reuses the previous filtered batch when only the render cap moved (same genes)', () => {
+ // Same selection {0}, only renderCap differs → the full signature changed but the
+ // GENE signature did not, so keeping the stale batch on screen is correct (no flash).
+ const layer = layerWith({ featureCodes: [0], preloadedFeatureCodes: codes, renderCap: 100 });
+ const result = preloadedScatterStrategy.renderLayers(layer);
+ expect(drawnCount(Array.isArray(result) ? result[0] : result)).toBe(2);
+ });
+
+ it('falls back to the full batch for the "all features" view, not the stale selection', () => {
+ // No selection: draw everything (3 rows), never the previous {0} filtered batch.
+ const layer = layerWith({ featureCodes: undefined, preloadedFeatureCodes: codes });
+ const result = preloadedScatterStrategy.renderLayers(layer);
+ expect(drawnCount(Array.isArray(result) ? result[0] : result)).toBe(3);
+ });
+});
diff --git a/packages/layers/tests/pointsResourceIdentity.spec.ts b/packages/layers/tests/pointsResourceIdentity.spec.ts
index 1185ee07..17d7e5b0 100644
--- a/packages/layers/tests/pointsResourceIdentity.spec.ts
+++ b/packages/layers/tests/pointsResourceIdentity.spec.ts
@@ -226,7 +226,7 @@ describe('partial render resource — getMatchingPartialResource', () => {
await pending;
});
- it('CHANGES identity when the scan grows the buffer — and only then', async () => {
+ it('HOLDS identity when the scan grows the buffer, bumping a revision instead (D10)', async () => {
const engine = new PointsDataEngine();
const scan = deferred();
const first = batch(2);
@@ -253,17 +253,20 @@ describe('partial render resource — getMatchingPartialResource', () => {
const atFirstChunk = engine.getMatchingPartialResource(element, 'pts');
expect(atFirstChunk).not.toBeNull();
+ const revisionBefore = engine.getMatchingPartialRevision('pts');
- // A new chunk grows the buffer. The memo keys on the partial's IDENTITY, so
- // this must produce a new resource — otherwise the overlay stops filling in.
+ // A new chunk grows the buffer. D10: the resource identity is held STABLE for the
+ // scan (so PointsLayer does not tear the overlay down per chunk) and the revision
+ // bumps instead — the composite re-reads the grown buffer on that prop change.
emit({ matchedRows: 5, scannedRows: 30, partialResult: grown });
const atSecondChunk = engine.getMatchingPartialResource(element, 'pts');
- expect(atSecondChunk).not.toBeNull();
- expect(atSecondChunk).not.toBe(atFirstChunk);
+ expect(atSecondChunk).toBe(atFirstChunk); // SAME resource — no teardown
+ expect(engine.getMatchingPartialRevision('pts')).toBe(revisionBefore + 1);
- // ...and is then stable again until the next chunk.
+ // Stable across reads until the next growth.
expect(engine.getMatchingPartialResource(element, 'pts')).toBe(atSecondChunk);
+ expect(engine.getMatchingPartialRevision('pts')).toBe(revisionBefore + 1);
scan.resolve(grown);
await pending;
@@ -291,3 +294,52 @@ describe('partial render resource — getMatchingPartialResource', () => {
expect(engine.getMatchingResource(element, 'pts')).not.toBeNull();
});
});
+
+describe('base render resource — getBaseResource (P2)', () => {
+ // The base layer's "current best view" evolves resident → resident-filtered →
+ // matched over an element's life. Each is a different batch; the old code drew
+ // them from two different resources under one layer id, so every swap changed the
+ // loader identity and PointsLayer hard-reset — the base flicker. getBaseResource
+ // holds ONE identity per element and swaps the backing batch, bumping a revision.
+ function makeElement(key: string) {
+ return {
+ key,
+ loadPoints: vi.fn(async () => batch(4)),
+ } as unknown as PointsElement;
+ }
+
+ it('holds identity across a resident↔matched batch swap, bumping the revision', () => {
+ const engine = new PointsDataEngine();
+ const element = makeElement('pts');
+ const resident = batch(4);
+ const matched = batch(2);
+
+ const first = engine.getBaseResource(element, 'pts', resident);
+ expect(first).not.toBeNull();
+ expect(engine.getBaseRevision('pts')).toBe(0);
+
+ // Same batch, repeated reads (pan frames) → same resource, no revision bump.
+ expect(engine.getBaseResource(element, 'pts', resident)).toBe(first);
+ expect(engine.getBaseRevision('pts')).toBe(0);
+
+ // Swap to the matched batch (a scan settled and now covers) → SAME resource
+ // identity (no teardown), revision bumped so PointsLayer re-reads.
+ const afterSwap = engine.getBaseResource(element, 'pts', matched);
+ expect(afterSwap).toBe(first);
+ expect(engine.getBaseRevision('pts')).toBe(1);
+
+ // Stable again until the next swap.
+ expect(engine.getBaseResource(element, 'pts', matched)).toBe(first);
+ expect(engine.getBaseRevision('pts')).toBe(1);
+ });
+
+ it('is null (and clears) when there is no batch', () => {
+ const engine = new PointsDataEngine();
+ const element = makeElement('pts');
+ engine.getBaseResource(element, 'pts', batch(4));
+ expect(engine.getBaseResource(element, 'pts', undefined)).toBeNull();
+ // Rebuilt fresh afterwards (revision resets).
+ expect(engine.getBaseResource(element, 'pts', batch(4))).not.toBeNull();
+ expect(engine.getBaseRevision('pts')).toBe(0);
+ });
+});
diff --git a/packages/layers/tests/pointsScatterSizing.spec.ts b/packages/layers/tests/pointsScatterSizing.spec.ts
new file mode 100644
index 00000000..70b14ec5
--- /dev/null
+++ b/packages/layers/tests/pointsScatterSizing.spec.ts
@@ -0,0 +1,48 @@
+import { Matrix4 } from '@math.gl/core';
+import { describe, expect, it } from 'vitest';
+import { modelMatrixUniformScale } from '../src/pointsScatterLayer.js';
+
+/**
+ * Point size is expressed in the ELEMENT's coordinate units. Deck applies
+ * `modelMatrix` to positions but not to a `'common'`-unit radius, so the layer
+ * folds the matrix scale into `radiusScale` itself. These pin that factor against
+ * transforms taken from real SpatialData elements.
+ */
+describe('modelMatrixUniformScale', () => {
+ it('reads the scale of a millimetre affine (points were ~8300x oversized)', () => {
+ const scale = 0.00012028094454887216;
+ expect(modelMatrixUniformScale(new Matrix4().scale([scale, scale, 1]))).toBeCloseTo(scale, 12);
+ });
+
+ it('reads a plain upscale transform', () => {
+ const scale = 4.705882352941177;
+ expect(modelMatrixUniformScale(new Matrix4().scale([scale, scale, 1]))).toBeCloseTo(scale, 9);
+ });
+
+ it('is 1 for identity, so untransformed elements are unaffected', () => {
+ expect(modelMatrixUniformScale(new Matrix4())).toBe(1);
+ });
+
+ it('is 1 for a missing matrix', () => {
+ expect(modelMatrixUniformScale(null)).toBe(1);
+ expect(modelMatrixUniformScale(undefined)).toBe(1);
+ });
+
+ it('takes the geometric mean of anisotropic axes', () => {
+ expect(modelMatrixUniformScale(new Matrix4().scale([4, 9, 1]))).toBeCloseTo(6, 9);
+ });
+
+ it('ignores translation', () => {
+ const m = new Matrix4().translate([1000, -2000, 0]).scale([3, 3, 1]);
+ expect(modelMatrixUniformScale(m)).toBeCloseTo(3, 9);
+ });
+
+ it('is unaffected by rotation (basis length, not raw elements)', () => {
+ const m = new Matrix4().rotateZ(Math.PI / 4).scale([2, 2, 1]);
+ expect(modelMatrixUniformScale(m)).toBeCloseTo(2, 9);
+ });
+
+ it('falls back to 1 for a degenerate (zero) matrix rather than collapsing points', () => {
+ expect(modelMatrixUniformScale(new Matrix4().scale([0, 0, 0]))).toBe(1);
+ });
+});
diff --git a/packages/vis/src/SpatialCanvas/PointsFeatureFilterPanel.tsx b/packages/vis/src/SpatialCanvas/PointsFeatureFilterPanel.tsx
index f2890251..18a27d75 100644
--- a/packages/vis/src/SpatialCanvas/PointsFeatureFilterPanel.tsx
+++ b/packages/vis/src/SpatialCanvas/PointsFeatureFilterPanel.tsx
@@ -1,20 +1,74 @@
-import { featureCodeToCssColor } from '@spatialdata/layers';
+import { featureNamesForCodes, resolveFeatureSelectionCodes } from '@spatialdata/core';
+import { featureCodeToRgb } from '@spatialdata/layers';
import type { CSSProperties } from 'react';
-import { useEffect, useMemo, useState } from 'react';
+import { useEffect, useMemo, useRef, useState } from 'react';
import { useSpatialCanvasActions } from './context';
import { describeFeatureRowState, featureRowOpacity } from './featureRowState';
import { usePointsFeatureState } from './PointsFeatureState';
import type { PointsLayerConfig } from './types';
// we need a pass on how we manage styles
-const swatchStyle: CSSProperties = {
- width: 10,
- height: 10,
- borderRadius: 2,
+
+// The colour swatch IS the picker: this span's background shows the feature's
+// effective colour, and a transparent native colour input overlays it. `inline-block`
+// + `box-sizing: border-box` make the 12×12 size hold regardless of flex context and
+// keep the 1px border inside the box (an inline span would ignore width/height, and a
+// content-box border would overflow — the layout bug this replaces).
+const colorSwatchStyle: CSSProperties = {
+ position: 'relative',
+ display: 'inline-block',
+ boxSizing: 'border-box',
+ width: 12,
+ height: 12,
flexShrink: 0,
+ borderRadius: 2,
border: '1px solid rgba(255, 255, 255, 0.25)',
};
+const colorSwatchOverriddenStyle: CSSProperties = {
+ borderColor: '#6cb6ff',
+ boxShadow: '0 0 0 1px #6cb6ff',
+};
+
+const colorInputStyle: CSSProperties = {
+ position: 'absolute',
+ inset: 0,
+ width: '100%',
+ height: '100%',
+ margin: 0,
+ padding: 0,
+ border: 'none',
+ opacity: 0,
+ cursor: 'pointer',
+ appearance: 'none',
+ WebkitAppearance: 'none',
+};
+
+const resetOverrideStyle: CSSProperties = {
+ color: '#888',
+ fontSize: '11px',
+ padding: '0 3px',
+ border: '1px solid #444',
+ borderRadius: 3,
+ background: '#222',
+ cursor: 'pointer',
+ flexShrink: 0,
+};
+
+const hex2 = (value: number): string =>
+ Math.max(0, Math.min(255, value)).toString(16).padStart(2, '0');
+
+/** `[r,g,b]` (0–255) → `#rrggbb` for a native colour input's value. */
+function rgbToHex([r, g, b]: readonly [number, number, number]): string {
+ return `#${hex2(r)}${hex2(g)}${hex2(b)}`;
+}
+
+/** `#rrggbb` → `[r,g,b]` (0–255). */
+function hexToRgb(hex: string): [number, number, number] {
+ const n = Number.parseInt(hex.slice(1), 16);
+ return [(n >> 16) & 255, (n >> 8) & 255, n & 255];
+}
+
const panelStyle: CSSProperties = {
display: 'flex',
flexDirection: 'column',
@@ -108,8 +162,10 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
loadedMatchingCodes,
supportsOnDemandLoad,
matchingLoadState,
+ residentFeatureCounts,
requestCatalog,
- } = usePointsFeatureState(config.featureCodes);
+ setHighlightedFeature,
+ } = usePointsFeatureState(config);
const [searchQuery, setSearchQuery] = useState('');
// Request the full-dataset catalog whenever this panel is shown for a layer.
@@ -118,19 +174,38 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
useEffect(() => {
requestCatalog();
}, [requestCatalog]);
+ // Clear any lingering hover highlight when the panel unmounts (or its layer
+ // changes), so an emphasis doesn't stick after the pointer is long gone.
+ useEffect(() => () => setHighlightedFeature(null), [setHighlightedFeature]);
const entries = useMemo(() => catalog?.entries ?? [], [catalog?.entries]);
const hasCounts = entries.some((entry) => entry.count !== undefined);
- const allSelected = config.featureCodes === undefined;
- const noneSelected = config.featureCodes !== undefined && config.featureCodes.length === 0;
+ // Authoritative dataset counts only arrive with the catalog's counts scan. Until
+ // then fall back to the running resident-window tally accumulated while the points
+ // streamed in — enough to populate and sort the column immediately. Partial values
+ // are marked with a leading "≥" so they are never mistaken for dataset totals.
+ const partialCounts = residentFeatureCounts;
+ const hasAnyCounts = hasCounts || (partialCounts?.size ?? 0) > 0;
+ const effectiveCount = (entry: { code: number; count?: number }): number | undefined =>
+ entry.count ?? partialCounts?.get(entry.code);
+ const countIsPartial = (entry: { code: number; count?: number }): boolean =>
+ entry.count === undefined && partialCounts?.get(entry.code) !== undefined;
+ // The selection persists as NAMES (see `PointsLayerConfig.featureNames`), but the
+ // rest of this panel — checkboxes, greying, the engine reads — works in codes.
+ // Resolve once here against the catalog we are already rendering.
+ const selection = resolveFeatureSelectionCodes(config, catalog);
+ const allSelected = selection === undefined;
+ const noneSelected = selection !== undefined && selection.length === 0;
const selectedCodes = allSelected
? new Set(entries.map((entry) => entry.code))
- : new Set(config.featureCodes ?? []);
+ : new Set(selection ?? []);
const sortedEntries = useMemo(() => {
const list = [...entries];
- if (hasCounts) {
+ const rank = (entry: { code: number; count?: number }): number =>
+ entry.count ?? partialCounts?.get(entry.code) ?? -1;
+ if (hasAnyCounts) {
list.sort((left, right) => {
- const countDiff = (right.count ?? -1) - (left.count ?? -1);
+ const countDiff = rank(right) - rank(left);
if (countDiff !== 0) {
return countDiff;
}
@@ -140,7 +215,7 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
list.sort((left, right) => left.name.localeCompare(right.name));
}
return list;
- }, [entries, hasCounts]);
+ }, [entries, hasAnyCounts, partialCounts]);
const visibleEntries = useMemo(() => {
const query = searchQuery.trim().toLowerCase();
@@ -150,31 +225,92 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
return sortedEntries.filter((entry) => entry.name.toLowerCase().includes(query));
}, [sortedEntries, searchQuery]);
- const setFeatureCodes = (nextCodes: number[] | undefined) => {
- updateLayer(layerId, { featureCodes: nextCodes });
+ // Write NAMES, and clear any legacy `featureCodes` so the two cannot disagree —
+ // `featureNames` wins when both are set, and a stale code list left behind in a
+ // saved config is exactly the confusion this change exists to remove.
+ const setSelectedCodes = (nextCodes: number[] | undefined) => {
+ updateLayer(layerId, {
+ featureNames: nextCodes ? featureNamesForCodes(nextCodes, catalog) : undefined,
+ featureCodes: undefined,
+ });
+ };
+
+ // Per-feature colour overrides, keyed by feature NAME (survives code remapping).
+ const colorOverrides = config.featureColorOverrides;
+ const effectiveRgb = (name: string, code: number): [number, number, number] =>
+ colorOverrides?.[name] ?? featureCodeToRgb(code);
+ // `` fires change continuously while the picker is dragged,
+ // and each commit is a layer-config write → new palette → deck layer update, on a
+ // layer that can be holding millions of points. Coalesce to one write per frame:
+ // the canvas still previews live (which is the whole point of the control), but
+ // the work is bounded by the display rather than by event rate.
+ // The pending value is the FULL next overrides map, not one entry: successive
+ // edits inside a frame accumulate into it, so two features recoloured before the
+ // frame fires both survive, and the merge base is taken at schedule time — no ref
+ // read during render, and no dependence on which render created the handler.
+ const pendingColorRef = useRef | null>(null);
+ const colorFrameRef = useRef(null);
+ useEffect(
+ () => () => {
+ if (colorFrameRef.current !== null) {
+ cancelAnimationFrame(colorFrameRef.current);
+ }
+ },
+ []
+ );
+ const setColorOverride = (name: string, rgb: [number, number, number]) => {
+ pendingColorRef.current = { ...(pendingColorRef.current ?? colorOverrides ?? {}), [name]: rgb };
+ if (colorFrameRef.current !== null) {
+ return;
+ }
+ colorFrameRef.current = requestAnimationFrame(() => {
+ colorFrameRef.current = null;
+ const pending = pendingColorRef.current;
+ pendingColorRef.current = null;
+ if (pending) {
+ updateLayer(layerId, { featureColorOverrides: pending });
+ }
+ });
+ };
+ const clearColorOverride = (name: string) => {
+ // A coalesced write may still be queued for this feature. It carries the whole
+ // map, so letting it land after the clear would put the override straight back.
+ if (pendingColorRef.current && name in pendingColorRef.current) {
+ delete pendingColorRef.current[name];
+ }
+ if (!colorOverrides || !(name in colorOverrides)) {
+ return;
+ }
+ const next = { ...colorOverrides };
+ delete next[name];
+ updateLayer(layerId, {
+ featureColorOverrides: Object.keys(next).length > 0 ? next : undefined,
+ });
};
const toggleFeature = (code: number, checked: boolean) => {
- const current = new Set(
- allSelected ? entries.map((entry) => entry.code) : (config.featureCodes ?? [])
- );
+ const current = new Set(allSelected ? entries.map((entry) => entry.code) : (selection ?? []));
if (checked) {
current.add(code);
} else {
current.delete(code);
}
if (current.size === 0) {
- setFeatureCodes([]);
+ setSelectedCodes([]);
return;
}
if (current.size === entries.length) {
- setFeatureCodes(undefined);
+ setSelectedCodes(undefined);
return;
}
- setFeatureCodes([...current].sort((left, right) => left - right));
+ setSelectedCodes([...current].sort((left, right) => left - right));
};
- if (catalogLoading) {
+ // Only block on loading when there is NOTHING to show. The catalog scan publishes
+ // the names/codes list before its (slow) per-feature counts pass, so once that
+ // partial arrives the list is usable — features can be seen, coloured and selected
+ // while the counts column is still filling in.
+ if (catalogLoading && !catalog) {
return (
Loading features…
@@ -239,10 +375,14 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
{' '}
· {selectedCount}/{entries.length} selected
- {hasCounts ? ' · sorted by count' : ''}
+ {hasAnyCounts ? (hasCounts ? ' · sorted by count' : ' · sorted by count so far') : ''}
{catalogRefining ?
Loading the full feature list…
: null}
+ {catalogLoading && !catalogRefining ? (
+ // The list is already usable; only the per-feature counts are outstanding.
+
Counting features…
+ ) : null}
{notLoadedCount > 0 ? (
{notLoadedCount} of {entries.length} feature{entries.length === 1 ? '' : 's'}{' '}
@@ -266,7 +406,7 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
checked={allSelected}
onChange={(event) => {
if (event.target.checked) {
- setFeatureCodes(undefined);
+ setSelectedCodes(undefined);
}
}}
/>
@@ -278,7 +418,7 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
checked={noneSelected}
onChange={(event) => {
if (event.target.checked) {
- setFeatureCodes([]);
+ setSelectedCodes([]);
}
}}
/>
@@ -300,6 +440,8 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
entry.count !== undefined ? ` · ${entry.count.toLocaleString()} pts` : '';
// Multi-line diagnostic: the human state + reason, then the raw signals
// that drove the decision (what made this row grey / not grey).
+ const overridden = colorOverrides?.[entry.name] !== undefined;
+ const rgb = effectiveRgb(entry.name, entry.code);
const title =
`${entry.name} · code ${entry.code}${countStr}\n` +
`${state.label}: ${state.reason}\n` +
@@ -310,21 +452,65 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
key={entry.code}
style={{ ...checkboxLabelStyle, opacity: featureRowOpacity(state) }}
title={title}
+ onMouseEnter={() => setHighlightedFeature(entry.code)}
+ onMouseLeave={() => setHighlightedFeature(null)}
>
toggleFeature(entry.code, event.target.checked)}
/>
+ {/* Swatch = colour picker: this span's background is the effective
+ colour and a transparent colour input overlays it. Interactive content
+ inside the label, so operating it does not toggle the checkbox. */}
+ style={{
+ ...colorSwatchStyle,
+ background: `rgb(${rgb[0]}, ${rgb[1]}, ${rgb[2]})`,
+ ...(overridden ? colorSwatchOverriddenStyle : {}),
+ }}
+ title={`${entry.name} colour${overridden ? ' (overridden)' : ''}`}
+ >
+ event.stopPropagation()}
+ onChange={(event) => setColorOverride(entry.name, hexToRgb(event.target.value))}
+ />
+
{entry.name}
{state.greyed ? ' ·' : ''}
- {hasCounts ? {formatFeatureCount(entry.count)} : null}
+ {overridden ? (
+
+ ) : null}
+ {hasAnyCounts ? (
+
+ {countIsPartial(entry) ? '≥' : ''}
+ {formatFeatureCount(effectiveCount(entry))}
+
+ ) : null}
);
})}
diff --git a/packages/vis/src/SpatialCanvas/PointsFeatureState.tsx b/packages/vis/src/SpatialCanvas/PointsFeatureState.tsx
index c343b978..822ea4b5 100644
--- a/packages/vis/src/SpatialCanvas/PointsFeatureState.tsx
+++ b/packages/vis/src/SpatialCanvas/PointsFeatureState.tsx
@@ -22,6 +22,7 @@
* consumers read `pointsEngine` + `resolvePointsTarget` off the renderer-hook
* result, wrap a subtree in this provider, and consume `usePointsFeatureState`.
*/
+import { resolveFeatureSelectionCodes } from '@spatialdata/core';
import type { PointsDataEngine, PointsLoadTarget } from '@spatialdata/layers';
import {
createContext,
@@ -94,6 +95,12 @@ function usePointsFeatureContext(): PointsFeatureStateContextValue {
return value;
}
+/** A layer's feature-filter selection: durable names, or already-resolved codes. */
+export interface PointsFeatureSelection {
+ featureNames?: readonly string[] | undefined;
+ featureCodes?: readonly number[] | undefined;
+}
+
export interface PointsFeatureState {
/** The feature catalog: `undefined` until requested/settled, `null` when the
* element has no `feature_key`, else the catalog. */
@@ -116,11 +123,20 @@ export interface PointsFeatureState {
/** Truncation of what's on screen for the selection passed to the hook (so the
* UI can show when raising the memory cap would load more). */
truncation: ReturnType;
+ /** Running per-feature counts over the resident window (`code → rows`), available
+ * while the whole-dataset counts scan is still running. Partial by construction. */
+ residentFeatureCounts: ReturnType;
/** Stable callback — trigger the full-dataset catalog build (idempotent). */
requestCatalog: () => void;
+ /** Stable callback — set (or clear, with null) the hover-highlighted feature code
+ * for this layer, so its points are emphasised on the canvas. */
+ setHighlightedFeature: (featureCode: number | null) => void;
}
-const EMPTY_POINTS_FEATURE_STATE: Omit = {
+const EMPTY_POINTS_FEATURE_STATE: Omit<
+ PointsFeatureState,
+ 'requestCatalog' | 'setHighlightedFeature'
+> = {
catalog: undefined,
catalogLoading: false,
catalogRefining: false,
@@ -129,6 +145,7 @@ const EMPTY_POINTS_FEATURE_STATE: Omit = {
supportsOnDemandLoad: false,
matchingLoadState: undefined,
truncation: undefined,
+ residentFeatureCounts: undefined,
};
/**
@@ -138,7 +155,9 @@ const EMPTY_POINTS_FEATURE_STATE: Omit = {
* (the active selection — pass `config.featureCodes`), plus a stable
* `requestCatalog`.
*/
-export function usePointsFeatureState(featureCodes?: readonly number[]): PointsFeatureState {
+export function usePointsFeatureState(
+ selection?: readonly number[] | PointsFeatureSelection
+): PointsFeatureState {
'use no memo';
const { engine, target, subscribe, getVersion } = usePointsFeatureContext();
// Reactivity: re-render this component on every engine mutation. The returned
@@ -147,12 +166,27 @@ export function usePointsFeatureState(featureCodes?: readonly number[]): PointsF
const requestCatalog = useCallback(() => {
if (target) void engine.ensureFeatureCatalog(target);
}, [engine, target]);
+ const setHighlightedFeature = useCallback(
+ (featureCode: number | null) => {
+ if (target) engine.setHighlightedFeature(target.key, featureCode);
+ },
+ [engine, target]
+ );
if (!target) {
- return { ...EMPTY_POINTS_FEATURE_STATE, requestCatalog };
+ return { ...EMPTY_POINTS_FEATURE_STATE, requestCatalog, setHighlightedFeature };
}
const key = target.key;
const scannable = engine.supportsFeatureScan(key);
+ // A selection persists as NAMES, so resolve it here against the catalog this
+ // hook is already reading — callers pass their config and keep working in codes.
+ // An array is still accepted as already-resolved codes.
+ const featureCodes = Array.isArray(selection)
+ ? selection
+ : resolveFeatureSelectionCodes(
+ (selection ?? {}) as PointsFeatureSelection,
+ engine.getFeatureCatalog(key)
+ );
const hasSelection = !!featureCodes && featureCodes.length > 0;
return {
catalog: engine.getFeatureCatalog(key),
@@ -166,6 +200,8 @@ export function usePointsFeatureState(featureCodes?: readonly number[]): PointsF
matchingLoadState:
hasSelection && scannable ? engine.getMatchingLoadState(key, featureCodes) : undefined,
truncation: engine.getActiveTruncation(key, featureCodes),
+ residentFeatureCounts: engine.getResidentFeatureCounts(key),
requestCatalog,
+ setHighlightedFeature,
};
}
diff --git a/packages/vis/src/SpatialCanvas/PointsLayerPanel.tsx b/packages/vis/src/SpatialCanvas/PointsLayerPanel.tsx
index fbc54fe9..1032c470 100644
--- a/packages/vis/src/SpatialCanvas/PointsLayerPanel.tsx
+++ b/packages/vis/src/SpatialCanvas/PointsLayerPanel.tsx
@@ -73,7 +73,7 @@ function ShowMatchingPoints({ config }: { config: PointsLayerConfig }) {
// read is engine-backed and updates on notify; the compiler would otherwise
// memoize this line's JSX and never repaint it as the scan progresses.
'use no memo';
- const { truncation: t } = usePointsFeatureState(config.featureCodes);
+ const { truncation: t } = usePointsFeatureState(config);
if (!t) return null;
// Report the batch held in memory (always true), NOT a per-selection matched
// count: t.loaded is the covered-batch size, which overstates the selection
@@ -98,9 +98,39 @@ function ShowMatchingPoints({ config }: { config: PointsLayerConfig }) {
);
}
+function PointSizeControl({ config }: { config: PointsLayerConfig }) {
+ const actions = useSpatialCanvasActions();
+ return (
+
+ );
+}
+
export default function PointsLayerPanel({ config, engine, resolveTarget }: PointsLayerPanelProps) {
return (
+
diff --git a/packages/vis/src/SpatialCanvas/index.tsx b/packages/vis/src/SpatialCanvas/index.tsx
index 103a0d28..8c1c182c 100644
--- a/packages/vis/src/SpatialCanvas/index.tsx
+++ b/packages/vis/src/SpatialCanvas/index.tsx
@@ -768,31 +768,6 @@ function SpatialCanvasInner({
}
/>
- {selectedConfig.type === 'points' && (
-
- )}
{selectedConfig.type === 'points' && (
, and consume the usePoints* hooks.
diff --git a/packages/vis/src/SpatialCanvas/types.ts b/packages/vis/src/SpatialCanvas/types.ts
index 0ce4938a..8ba2d6e4 100644
--- a/packages/vis/src/SpatialCanvas/types.ts
+++ b/packages/vis/src/SpatialCanvas/types.ts
@@ -113,12 +113,32 @@ export interface PointsLayerConfig extends BaseLayerConfig {
*/
colorByFeature?: boolean;
/**
- * Feature-filter selection by Feature Code. `undefined` means "all features
- * shown" (no filter); an array restricts the drawn points to those codes. This
- * is serializable Stack-Entry state (persists in a saved config), distinct from
- * the runtime-only Feature Highlight added in MVP step 3.
+ * Feature-filter selection by feature NAME — the durable, serializable form,
+ * and what the UI writes. `undefined` means "all features shown" (no filter);
+ * an array restricts the drawn points to those features.
+ *
+ * Names rather than codes because for a dictionary-only element (a Xenium
+ * `transcripts` has `feature_name` and no code column) the codes are
+ * APP-ASSIGNED — a first-seen index from whichever catalog scan ran — so a
+ * stored code can come back meaning a different gene. Names are also readable
+ * in a saved config. Takes precedence over {@link featureCodes}; resolved
+ * against the settled catalog by `resolveFeatureSelectionCodes`.
+ */
+ featureNames?: string[];
+ /**
+ * Feature-filter selection by Feature Code. Retained for runtime use and for
+ * configs written before {@link featureNames} existed, which takes precedence.
+ * Prefer names for anything that is persisted — see the note there.
*/
featureCodes?: number[];
+ /**
+ * Per-feature colour overrides, keyed by feature NAME (not code): `{ "EPCAM":
+ * [220, 30, 30] }` draws that gene in that RGB instead of its default categorical
+ * colour. Keyed by name so an override survives the code remapping between the
+ * resident-preview catalog and the full one. Absent features keep their default.
+ * Serializable Stack-Entry state.
+ */
+ featureColorOverrides?: Record;
}
export interface LabelsLayerConfig extends BaseLayerConfig {
diff --git a/packages/vis/src/SpatialCanvas/useLayerData.ts b/packages/vis/src/SpatialCanvas/useLayerData.ts
index 0f51adc3..e8b855e6 100644
--- a/packages/vis/src/SpatialCanvas/useLayerData.ts
+++ b/packages/vis/src/SpatialCanvas/useLayerData.ts
@@ -21,6 +21,7 @@ import {
getTooltipSignature,
type LabelsElement,
type PointsElement,
+ resolveFeatureSelectionCodes,
resolvePointsMemoryCap,
resolveTooltipItems,
type ShapesElement,
@@ -34,6 +35,7 @@ import {
import {
buildShapeFillColorByFeatureId,
buildShapesPrebuiltData,
+ featureFilterAwaitingRowCodes,
PointsDataEngine,
PointsLayer,
type PointsLoadTarget,
@@ -376,7 +378,11 @@ export function useLayerData(
layersRef.current = layers;
const [layerLoadStates, setLayerLoadStates] = useState>({});
- const [, setLoadedDataRevision] = useState(0);
+ // Bumped on every resolver settle. The reconcile effect depends on it so that an
+ // async settle (e.g. the preload landing, which flips `supportsFeatureScan`) re-runs
+ // planning — this is what lets the row-codes and feature-index scan be planned from
+ // the commit phase (Track A) instead of kicked imperatively during render.
+ const [loadedDataRevision, setLoadedDataRevision] = useState(0);
const notifyLoadedDataChanged = useCallback(() => {
setLoadedDataRevision((revision) => revision + 1);
@@ -549,6 +555,11 @@ export function useLayerData(
// system switch that makes a previously unavailable element resolvable. The map is
// memoised on `availableElements`, so this adds no per-render churn.
useEffect(() => {
+ // Bare reference: `loadedDataRevision` is a re-trigger, not a value we read. A
+ // resolver settle (the preload landing flips `supportsFeatureScan`) must replan so
+ // the scan/row-codes tasks get emitted; touching it here declares that dependency
+ // honestly to exhaustive-deps. The plan/load dedup makes the extra runs convergent.
+ void loadedDataRevision;
const contexts: AnyResolveContext[] = [];
for (const layerId of layerOrder) {
const config = layers[layerId];
@@ -586,21 +597,38 @@ export function useLayerData(
transform: elem.transform,
});
} else if (elem.type === 'points' && config.type === 'points') {
- // Only the preload is planned here; row-codes and the feature-index scan
- // stay on the render-phase engine calls in `getLayers` (Track A), so the
- // config deliberately carries just the memory cap.
+ // The full points config drives planning (Track A): `plan()` emits the
+ // preload, and — once the preload makes a scan possible — the row-codes and
+ // feature-index scan tasks from `featureCodes`/`colorByFeature`. These used
+ // to be kicked imperatively from `getLayers` during render.
contexts.push({
entryId: layerId,
elementKey: elem.key,
kind: 'points',
element: elem.element,
- config: { pointsMemoryCap: resolvePointsMemoryCap(config.pointsMemoryCap) },
+ config: {
+ pointsMemoryCap: resolvePointsMemoryCap(config.pointsMemoryCap),
+ // Names are the durable selection; resolve them against the settled
+ // catalog so everything downstream keeps working in codes.
+ ...(() => {
+ const codes = resolveFeatureSelectionCodes(
+ config,
+ pointsEngine.getFeatureCatalog(elem.key)
+ );
+ return codes ? { featureCodes: codes } : {};
+ })(),
+ ...(config.colorByFeature ? { colorByFeature: true } : {}),
+ },
transform: elem.transform,
});
}
}
void store.reconcile(contexts);
- }, [layers, layerOrder, store, elementMapValue]);
+ // `pointsEngine` is the stable useState value, so this adds no churn; it is in
+ // the list because the name→code resolution above reads its catalog. The catalog
+ // ARRIVING is covered by `loadedDataRevision` (bumped on every resolver settle),
+ // which is what re-resolves a name selection that could not be resolved yet.
+ }, [layers, layerOrder, store, elementMapValue, loadedDataRevision, pointsEngine]);
// --- Shapes projection memos (Renderer Adapter side, kept in vis) -------------
@@ -919,17 +947,33 @@ export function useLayerData(
}
} else if (config.type === 'points') {
const element = elem.element as PointsElement;
- const featureCodes = config.featureCodes;
+ const featureCodes = resolveFeatureSelectionCodes(
+ config,
+ pointsEngine.getFeatureCatalog(elem.key)
+ );
const selectionActive = featureCodes !== undefined && featureCodes.length > 0;
-
- // Feature-index render scan: when a selection is active, load the WHOLE
- // dataset's matching points (footer stats skip the row groups a selected
- // feature can't live in), so features outside the resident preload window
- // still render. The scan is idempotent per selection; kicking it here is a
- // no-op once resident/in-flight. On settle it notifies → re-render → the
- // matched resource appears below. `getMatchingResource` returns the LAST
- // completed matched batch, so a selection change keeps showing the prior
- // selection's points until the new scan settles (no blank mid-scan).
+ // Sizes the colour LUT so every point's feature code indexes a real texel.
+ const featureCodeSpaceSize = pointsEngine.getFeatureCodeSpaceSize(elem.key);
+ // Resolve config's by-name colour overrides to a stable code→rgb map for the
+ // LUT (identity-stable so the palette texture is not rebuilt every frame).
+ const featureColorOverrides = pointsEngine.getFeatureColorOverrideMap(
+ elem.key,
+ config.featureColorOverrides
+ );
+ // Hover highlight (runtime): emphasise the hovered feature's points, -1 (no
+ // highlight) otherwise. Held on the engine — the shared external store the
+ // feature panel writes to and this render path reads — so it needs no store /
+ // renderStack plumbing. A uniform, so it costs nothing per frame.
+ const highlightFeatureCode = pointsEngine.getHighlightedFeature(elem.key);
+
+ // Feature-index render scan: when a selection is active, the WHOLE
+ // dataset's matching points are loaded (footer stats skip the row groups a
+ // selected feature can't live in), so features outside the resident preload
+ // window still render. The scan is PLANNED from the reconcile effect (Track
+ // A) — `getLayers` only READS its result here. `getMatchingResource`
+ // returns the LAST completed matched batch, so a selection change keeps
+ // showing the prior selection's points until the new scan settles (no blank
+ // mid-scan).
//
// Gated on scan capability: an authoritative code column (footer stats
// skip row groups) OR a dictionary-only element with a catalog loaded —
@@ -939,81 +983,115 @@ export function useLayerData(
// catalog loads (no shared code space) there is nothing to match names
// against, so it falls through to resident in-memory filtering.
const canFeatureScan = pointsEngine.supportsFeatureScan(elem.key);
- let matchingResource: PointsRenderResource | null = null;
let partialResource: PointsRenderResource | null = null;
- if (selectionActive && canFeatureScan) {
- void pointsEngine.ensureMatchingFeaturesLoaded(
- { key: elem.key, layerId, element },
- featureCodes,
- resolvePointsMemoryCap(config.pointsMemoryCap)
- );
- matchingResource = pointsEngine.getMatchingResource(element, elem.key);
- // The in-flight scan's growing buffer (all matched chunks so far), drawn
- // as an extra overlay sub-layer below so the base (resident preview /
- // prior matched batch) stays visible while points progressively fill in.
+ // The selected genes the LAST-GOOD scan already covers. This is the pivot of
+ // the show/hide policy: the whole-dataset matched batch survives a selection
+ // change as `stale`, so `covered` is the previous scan's genes. Intersecting
+ // it with the CURRENT selection gives exactly the genes we may safely draw
+ // from that batch — never a deselected gene (that would be bug B: a gene shown
+ // when it shouldn't be), and never dropping a still-wanted gene the scan
+ // already has (bug A: a wanted gene vanishing when the selection grows).
+ let coveredSelection: readonly number[] | undefined;
+ if (selectionActive && canFeatureScan && featureCodes !== undefined) {
+ const covered = pointsEngine.getLoadedMatchingFeatureCodes(elem.key);
+ coveredSelection =
+ covered !== undefined ? featureCodes.filter((code) => covered.has(code)) : [];
partialResource = pointsEngine.getMatchingPartialResource(element, elem.key);
}
- if (matchingResource) {
- // The matched batch covers the selection (or a superset of it, when the
- // selection just shrank). Pass the batch's per-row codes + the current
- // selection so the layer filters IN MEMORY down to the selected codes —
- // this is what makes removing a feature a free filter instead of a
- // re-scan. When the selection equals what was scanned, skip the filter
- // (render the batch whole); the batch's own codes still drive colour.
- const matchedRowCodes = pointsEngine.getMatchingRowFeatureCodes(elem.key);
- const coveredSize = pointsEngine.getLoadedMatchingFeatureCodes(elem.key)?.size ?? 0;
- const filterMatched = featureCodes !== undefined && featureCodes.length < coveredSize;
+ // Choose the base batch: the whole-dataset matched batch whenever it covers
+ // ANY still-wanted gene (drawn filtered to that covered subset), else the
+ // resident preload. Growing [A]→[A,B] keeps A on screen from the matched batch
+ // while B's scan streams in via the overlay, instead of blinking A out to the
+ // resident window. Both flow through ONE stable base resource
+ // (`getBaseResource`) whose backing batch swaps under it — so the base never
+ // tears down as the view evolves resident↔matched (the base flicker). An empty
+ // matched batch (a scan that matched nothing) falls back to resident.
+ const matchedCandidate =
+ coveredSelection && coveredSelection.length > 0
+ ? pointsEngine.getMatchedBatch(elem.key)
+ : undefined;
+ // The filter that would have to be applied to draw the matched batch: a
+ // strict subset of what the scan covers means genes in the batch must be
+ // held back.
+ const matchedRowCodes = matchedCandidate
+ ? pointsEngine.getMatchingRowFeatureCodes(elem.key)
+ : undefined;
+ const matchedCoveredSize =
+ pointsEngine.getLoadedMatchingFeatureCodes(elem.key)?.size ?? 0;
+ const matchedFilter =
+ coveredSelection !== undefined && coveredSelection.length < matchedCoveredSize
+ ? coveredSelection
+ : undefined;
+ // ...but `PointsLayer` cannot apply a feature filter without row-aligned
+ // codes, and its strategy resolves that case by drawing the batch WHOLE.
+ // Handing it a filter it will decline is therefore not a no-op — it
+ // surfaces every gene the scan covered, including the one just deselected.
+ // Gate on the layer's OWN predicate (imported, not re-derived, so the two
+ // cannot drift) and fall back to the resident base, which filters in memory.
+ const matchedBatch = featureFilterAwaitingRowCodes(matchedFilter, matchedRowCodes)
+ ? undefined
+ : matchedCandidate;
+ const useMatched = matchedBatch !== undefined && (matchedBatch.shape[1] ?? 0) > 0;
+ // Falling back to the in-flight preload's growing buffer (D3) is what makes a
+ // COLD load paint progressively: until the first full window settles there is
+ // no resident batch, and the base would otherwise draw nothing. It flows
+ // through the same stable base resource, so the growth is a revision bump —
+ // no teardown, no flicker.
+ const baseBatch = useMatched
+ ? matchedBatch
+ : (pointsEngine.getData(elem.key) ?? pointsEngine.getPreloadPartialBatch(elem.key));
+
+ // Colour-by-feature is ON BY DEFAULT (opt-out via `colorByFeature: false`), so
+ // thread the per-row codes whenever colour is not explicitly disabled — the
+ // "all features" view (no selection) needs them too, or it draws flat.
+ const wantsRowCodes = config.colorByFeature !== false || featureCodes !== undefined;
+ let basePreloadedCodes: ArrayLike | undefined;
+ let baseFilter: readonly number[] | undefined;
+ if (useMatched) {
+ basePreloadedCodes = matchedRowCodes;
+ // Filter the whole-dataset batch to the still-wanted covered subset unless
+ // the selection is exactly the scanned set (then render it whole). The batch
+ // only holds covered genes, so this can only ever DROP a deselected gene —
+ // never surface an unselected one. That holds because the gate above
+ // guarantees the filter is applicable; without it the drop silently
+ // becomes a no-op and the deselected gene stays on screen.
+ baseFilter = matchedFilter;
+ } else {
+ // Row codes drive both the in-memory filter (resident → selection) and
+ // colour-by-feature. The row-codes LOAD is planned from the reconcile
+ // effect (Track A), not kicked here.
+ basePreloadedCodes = wantsRowCodes
+ ? pointsEngine.getRowFeatureCodes(elem.key)
+ : undefined;
+ baseFilter = featureCodes;
+ }
+
+ const baseResource = pointsEngine.getBaseResource(element, elem.key, baseBatch);
+ if (baseResource) {
deckLayers.push(
new PointsLayer({
id: layerId,
- resource: matchingResource,
+ resource: baseResource,
+ // Stable resource; the revision bumps when the base batch is swapped
+ // (resident↔matched↔streaming) so the composite re-reads without a
+ // teardown — no base flicker on selection change or scan settle.
+ resourceRevision: pointsEngine.getBaseRevision(elem.key),
modelMatrix: elem.transform,
opacity: config.opacity,
visible: config.visible,
+ // Legacy renderPointsLayer defaulted radius to 1px; preserve that for
+ // parity (the composite's own default is smaller).
pointSize: config.pointSize ?? 1,
- ...(filterMatched ? { featureCodes } : {}),
- ...(matchedRowCodes ? { preloadedFeatureCodes: matchedRowCodes } : {}),
+ ...(baseFilter ? { featureCodes: baseFilter } : {}),
+ ...(basePreloadedCodes ? { preloadedFeatureCodes: basePreloadedCodes } : {}),
...(config.color ? { color: config.color } : {}),
...(config.colorByFeature ? { colorByFeature: true } : {}),
+ featureCodeSpaceSize,
+ ...(featureColorOverrides ? { featureColorOverrides } : {}),
+ highlightFeatureCode,
})
);
- } else {
- // Resident batch: the default view (no selection), and an instant preview
- // of the resident subset while the feature-index scan is still running.
- // The engine returns a STABLE render resource (memoized by signature), so
- // re-running getLayers every pan/zoom frame reuses the same loader
- // identity and the composite does not reset its batch (no flashing).
- const resource = pointsEngine.getResource(element, elem.key);
- if (resource) {
- const filterActive = featureCodes !== undefined;
- // Row codes are needed to filter by feature AND to colour by feature.
- // Colour-by-feature applies even with no filter ("all features"), so
- // load/pass the codes whenever either is on — not just when filtering.
- const needsRowCodes = filterActive || config.colorByFeature === true;
- if (needsRowCodes && !pointsEngine.hasRowFeatureCodes(elem.key)) {
- void pointsEngine.ensureRowFeatureCodes({ key: elem.key, layerId, element });
- }
- const preloadedFeatureCodes = needsRowCodes
- ? pointsEngine.getRowFeatureCodes(elem.key)
- : undefined;
- deckLayers.push(
- new PointsLayer({
- id: layerId,
- resource,
- modelMatrix: elem.transform,
- opacity: config.opacity,
- visible: config.visible,
- // Legacy renderPointsLayer defaulted radius to 1px; preserve that
- // for parity (the composite's own default is smaller).
- pointSize: config.pointSize ?? 1,
- ...(config.color ? { color: config.color } : {}),
- ...(config.colorByFeature ? { colorByFeature: true } : {}),
- ...(featureCodes ? { featureCodes } : {}),
- ...(preloadedFeatureCodes ? { preloadedFeatureCodes } : {}),
- })
- );
- }
}
// Overlay the in-flight scan's growing buffer as a SEPARATE sub-layer on
@@ -1028,6 +1106,9 @@ export function useLayerData(
new PointsLayer({
id: `${layerId}__partial`,
resource: partialResource,
+ // Stable resource across chunks; the revision bumps as the buffer
+ // grows so the overlay re-reads without a per-chunk teardown (D10).
+ resourceRevision: pointsEngine.getMatchingPartialRevision(elem.key),
modelMatrix: elem.transform,
opacity: config.opacity,
visible: config.visible,
@@ -1036,6 +1117,9 @@ export function useLayerData(
...(partialRowCodes ? { preloadedFeatureCodes: partialRowCodes } : {}),
...(config.color ? { color: config.color } : {}),
...(config.colorByFeature ? { colorByFeature: true } : {}),
+ featureCodeSpaceSize,
+ ...(featureColorOverrides ? { featureColorOverrides } : {}),
+ highlightFeatureCode,
})
);
}
diff --git a/packages/vis/tests/useLayerData.spec.tsx b/packages/vis/tests/useLayerData.spec.tsx
index 10d155dd..7e7a200c 100644
--- a/packages/vis/tests/useLayerData.spec.tsx
+++ b/packages/vis/tests/useLayerData.spec.tsx
@@ -283,3 +283,260 @@ describe('useLayerData — resolver lifecycle across a dataset swap', () => {
expect(pointsResource()).toBe(before);
});
});
+
+describe('useLayerData — coverage-gated base (never shows the wrong gene)', () => {
+ // The reported bug: select gene A, deselect, select disjoint gene B → the base
+ // drew ALL of A's points (the matched batch survives a selection change as
+ // `stale`) until B's scan settled. The base must use the matched batch ONLY when
+ // it covers the current selection; otherwise show the resident preload (filtered
+ // to B) while B streams in.
+ function scanPointsElement(key: string): AvailableElement {
+ const resident = {
+ shape: [2, 3],
+ data: [new Float32Array([0, 1, 2]), new Float32Array([3, 4, 5])],
+ featureCodes: new Int32Array([0, 1, 0]),
+ hasFeatureCodeColumn: true, // → supportsFeatureScan true right after preload
+ // Truncated on purpose: a matching scan is only planned when rows exist
+ // beyond the resident window. With a complete batch the resolver filters in
+ // memory and never scans, so there would be no matched batch to gate on.
+ preloadTruncated: true,
+ totalRowCount: 100,
+ };
+ const matchedForZero = {
+ shape: [2, 2],
+ data: [new Float32Array([0, 1]), new Float32Array([0, 1])],
+ featureCodes: new Int32Array([0, 0]),
+ };
+ const element = {
+ key,
+ loadPoints: vi.fn(async () => resident),
+ loadRowFeatureCodes: vi.fn(async () => new Int32Array([0, 1, 0])),
+ listFeaturesWithCounts: vi.fn(async () => null),
+ // The {0} scan settles; the {1} scan is left in flight so `lastGood` stays {0}
+ // — the exact window where the old code drew the wrong gene.
+ loadPointsMatchingFeatureCodes: vi.fn((opts: { featureCodes: readonly number[] }) =>
+ opts.featureCodes[0] === 0 ? Promise.resolve(matchedForZero) : new Promise(() => {})
+ ),
+ } as unknown as PointsElement;
+ return { key, type: 'points', element, transform: new Matrix4() };
+ }
+
+ it('draws the resident batch (not the stale matched batch), via one stable base resource', async () => {
+ const pts = scanPointsElement('transcripts');
+ const elements: ElementsByType = { ...EMPTY_ELEMENTS, points: [pts] };
+
+ const { result, rerender } = renderHook(
+ ({ l }: { l: Record }) =>
+ useLayerData(l, Object.keys(l), elements, null),
+ {
+ initialProps: {
+ l: { 'layer-p': { ...pointsConfig('layer-p', 'transcripts'), featureCodes: [0] } },
+ },
+ }
+ );
+ type LoadAllResource = { loader: { loadAll?: () => Promise<{ shape: number[] }> } };
+ const baseResource = () =>
+ (result.current.getLayers()[0]?.props as { resource?: LoadAllResource } | undefined)
+ ?.resource;
+ const baseRowCount = async () => (await baseResource()?.loader.loadAll?.())?.shape[1];
+
+ // The {0} scan settles → the matched batch covers {0}; the base draws it (2 rows).
+ await waitFor(() => {
+ expect(result.current.pointsEngine.getLoadedMatchingFeatureCodes('transcripts')?.has(0)).toBe(
+ true
+ );
+ });
+ const before = baseResource();
+ expect(await baseRowCount()).toBe(2); // matched-{0}
+
+ // Switch to a DISJOINT gene {1}; its scan is in flight, so `lastGood` is still {0}.
+ rerender({
+ l: { 'layer-p': { ...pointsConfig('layer-p', 'transcripts'), featureCodes: [1] } },
+ });
+ await waitFor(() => {
+ expect(result.current.pointsEngine.isMatchingLoading('transcripts', [1])).toBe(true);
+ });
+
+ // P2: the base resource identity is STABLE across the resident↔matched swap — no
+ // teardown, no flicker. P1: it now draws the RESIDENT batch (3 rows), never the
+ // stale matched-{0} batch (2 rows).
+ expect(baseResource()).toBe(before);
+ expect(await baseRowCount()).toBe(3);
+ });
+});
+
+describe('useLayerData — selection show/hide + colour', () => {
+ // Two more reported bugs beyond the disjoint switch above:
+ // (A) GROWING a selection ([0] → [0,1]) blinked gene 0 out to the resident window
+ // until gene 1's scan settled — a wanted gene vanishing.
+ // (colour) the "all features" view (no selection, no explicit flag) drew flat
+ // because per-row codes were never threaded, though colour-by-feature is on by
+ // default in the renderer.
+ function coverableElement(key: string): AvailableElement {
+ const resident = {
+ shape: [2, 3],
+ data: [new Float32Array([0, 1, 2]), new Float32Array([3, 4, 5])],
+ featureCodes: new Int32Array([0, 1, 0]),
+ hasFeatureCodeColumn: true,
+ // See the note in scanPointsElement: a scan is only planned for a truncated
+ // resident batch, which is the situation these matched-vs-resident cases model.
+ preloadTruncated: true,
+ totalRowCount: 100,
+ };
+ const matchedForZero = {
+ shape: [2, 2],
+ data: [new Float32Array([0, 1]), new Float32Array([0, 1])],
+ featureCodes: new Int32Array([0, 0]),
+ };
+ const element = {
+ key,
+ loadPoints: vi.fn(async () => resident),
+ loadRowFeatureCodes: vi.fn(async () => new Int32Array([0, 1, 0])),
+ listFeaturesWithCounts: vi.fn(async () => null),
+ // ONLY the exact {0} scan settles; any other selection (e.g. the grown {0,1})
+ // stays in flight, so `lastGood` — and thus coverage — remains {0}.
+ loadPointsMatchingFeatureCodes: vi.fn((opts: { featureCodes: readonly number[] }) =>
+ opts.featureCodes.length === 1 && opts.featureCodes[0] === 0
+ ? Promise.resolve(matchedForZero)
+ : new Promise