From b45d156732cc414af694d797fbad0b5fa9455442 Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Fri, 19 Jun 2026 18:22:14 +0100 Subject: [PATCH 01/38] ADR doc with some plans for points/shapes --- .../0002-spatially-aware-vector-loading.md | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 docs/adr/0002-spatially-aware-vector-loading.md diff --git a/docs/adr/0002-spatially-aware-vector-loading.md b/docs/adr/0002-spatially-aware-vector-loading.md new file mode 100644 index 00000000..e8a3bd26 --- /dev/null +++ b/docs/adr/0002-spatially-aware-vector-loading.md @@ -0,0 +1,56 @@ +# Spatially-Aware Vector Loading + +SpatialData points and shapes can be large enough that whole-element Parquet +loads are not a viable browser default. We will treat viewport-bounded vector +loading as a first-class source API and keep persisted optimization artifacts in +Parquet/GeoParquet rather than inventing a deck.gl-specific storage format. + +## Decision + +- Points v1 follows current Vitessce practice: a SpatialData Points Parquet + element may be sorted by 2D Morton order with a `morton_code_2d` column, a + feature-code column, controlled row-group sizes, and 2-4 leading sentinel rows + whose `morton_code_2d` is `0` and whose coordinates encode the full point + extent. +- `@spatialdata/core` exposes bounded point loading through + `PointsElement.loadPointsInBounds()`. When the Parquet module supports + Vitessce's row-group APIs (`readMetadata` and `readParquetRowGroup`) and the + store supports range reads, the loader may fetch selected row groups. Otherwise + it degrades to the existing full-table read followed by bounds filtering. +- `@spatialdata/vis` may render compatible points through a deck.gl `TileLayer`. + The tile layer owns async viewport loads and abort signals; ordinary + `ScatterplotLayer` rendering remains the fallback for preloaded point data. +- `points.experimental/` and `shapes.experimental/` are reserved as + top-level Experimental Optimization Collections. They link back to the source + element by key and metadata rather than modifying canonical SpatialData + element semantics. +- GeoParquet is the durable shape optimization target. GeoArrow is a runtime + columnar layout / deck adapter option, not a duplicate persisted artifact. + +## Prior Art + +- scverse Padua hackathon points work: + and + . +- Padua branch prototype: + . +- Vitessce tiled SpatialData Points: + . +- Vitessce sentinel bbox update: + and + . +- Vitessce shapes format `0.3` compatibility: + and + . + +## Consequences + +- Source loaders must expose typed/columnar batches and remain independent of + deck.gl. Rendering packages decide whether to use TileLayer, ScatterplotLayer, + or a future GeoArrow-aware layer. +- Whole-table point loading is still supported and is the compatibility fallback, + but render paths can opt into experimental optimizations with a single + `experimentalOptimizations` switch. +- Shapes format `0.3` remains on the current modern Parquet-backed path in + `VShapesSource`; large-shape spatial tiling still needs a separate GeoParquet + artifact/writer slice. From 42dbf219dcfa0ceb58a5b2d9a0cad51fbca2394f Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Fri, 19 Jun 2026 18:24:13 +0100 Subject: [PATCH 02/38] Add points tiling functionality and integrate with existing models - Introduced `pointsTiling.ts` to handle Morton code-based tiling for point data. - Updated `index.ts` to export the new points tiling module. - Enhanced `VPointsSource` and `VShapesSource` to support loading points tiling metadata and filtering points within specified bounds. - Modified `PointsColumnarData` type to use `ArrayLike[]` for better compatibility. - Added tests for points tiling functions to ensure correctness. - Updated relevant components in the visualization layer to utilize the new points tiling features. --- packages/core/src/index.ts | 1 + packages/core/src/models/VPointsSource.ts | 244 ++++++++++++- packages/core/src/models/VShapesSource.ts | 46 ++- packages/core/src/models/VTableSource.ts | 321 +++++++++++++++--- packages/core/src/models/index.ts | 12 + packages/core/src/pointsTiling.ts | 265 +++++++++++++++ packages/core/src/spatialViewFit.ts | 2 +- packages/core/tests/pointsTiling.spec.ts | 94 +++++ .../src/SpatialCanvas/SpatialCanvasViewer.tsx | 17 +- packages/vis/src/SpatialCanvas/index.tsx | 8 +- .../SpatialCanvas/renderers/pointsRenderer.ts | 162 +++++++-- packages/vis/src/SpatialCanvas/types.ts | 1 + .../vis/src/SpatialCanvas/useLayerData.ts | 137 ++++++-- 13 files changed, 1193 insertions(+), 117 deletions(-) create mode 100644 packages/core/src/pointsTiling.ts create mode 100644 packages/core/tests/pointsTiling.spec.ts diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index c5dd569e..e4c412ee 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,6 +9,7 @@ export * from './types.js'; export * from './store/index.js'; export * from './models/index.js'; export * from './spatialViewFit.js'; +export * from './pointsTiling.js'; export * from './shapes.js'; export { inferShapesGeometryKindFromParquet, diff --git a/packages/core/src/models/VPointsSource.ts b/packages/core/src/models/VPointsSource.ts index 03d19743..cad783e9 100644 --- a/packages/core/src/models/VPointsSource.ts +++ b/packages/core/src/models/VPointsSource.ts @@ -1,5 +1,14 @@ -import type { Axis } from '../schemas'; import { basename } from '../Vutils'; +import { + MORTON_CODE_2D_COLUMN, + type PointsInBoundsOptions, + type PointsInBoundsResult, + type PointsTilingMetadata, + extractSentinelBoundingBox, + filterPointsToBounds, + mortonIntervalsForBounds, +} from '../pointsTiling.js'; +import type { Axis } from '../schemas'; // import { normalizeAxes } from '@vitessce/spatial-utils'; import SpatialDataTableSource from './VTableSource'; @@ -68,7 +77,42 @@ function getParquetPath(arrPath?: string) { throw new Error(`Cannot determine parquet path for points array path: ${arrPath}`); } +function arrowSchemaFieldNames(table: { schema: { fields?: Array<{ name?: unknown }> } } | null) { + return ( + table?.schema.fields?.flatMap((field) => + typeof field.name === 'string' ? [field.name] : [] + ) ?? [] + ); +} + +function selectFeatureCodeColumn(fields: string[], featureKey: string | undefined) { + const candidates = [ + featureKey ? `${featureKey}_codes` : undefined, + 'feature_name_codes', + 'feature_index', + ].filter((value): value is string => typeof value === 'string'); + return candidates.find((candidate) => fields.includes(candidate)); +} + +function checkAbort(signal?: AbortSignal) { + if (signal?.aborted) { + throw new DOMException('The operation was aborted.', 'AbortError'); + } +} + +function rowGroupCountForIndex(metadata: PointsTilingMetadata, rowGroupIndex: number) { + if (rowGroupIndex < 0 || rowGroupIndex >= metadata.totalRowGroups) { + return 0; + } + return metadata.rowGroupRowCounts?.[rowGroupIndex] ?? metadata.maxRowsPerGroup; +} + export default class SpatialDataPointsSource extends SpatialDataTableSource { + private readonly pointTilingMetadataCache = new Map< + string, + Promise + >(); + /** * * @param path A path to within shapes. @@ -156,4 +200,202 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource { data: axisColumnArrs, }; } + + async getPointsTilingMetadata(elementPath: string): Promise { + if (this.pointTilingMetadataCache.has(elementPath)) { + return this.pointTilingMetadataCache.get(elementPath) ?? null; + } + const promise = this.loadPointsTilingMetadataUncached(elementPath); + this.pointTilingMetadataCache.set(elementPath, promise); + return promise; + } + + private async loadPointsTilingMetadataUncached( + elementPath: string + ): Promise { + const parquetPath = getParquetPath(elementPath); + const zattrs = await this.loadSpatialDataElementAttrs(elementPath); + const { axes, spatialdata_attrs: spatialDataAttrs } = zattrs; + const normAxes = normalizeAxes(axes); + const axisNames = normAxes.map((axis: { name: string }) => axis.name); + const { feature_key: featureKey } = spatialDataAttrs; + + const datasetMetadata = await this.loadParquetDatasetMetadata(parquetPath); + const schemaTable = datasetMetadata ? null : await this.loadParquetSchemaTable(parquetPath); + const fields = datasetMetadata?.schema?.fields + ? datasetMetadata.schema.fields.flatMap((field) => + typeof field.name === 'string' ? [field.name] : [] + ) + : arrowSchemaFieldNames(schemaTable); + + const featureCodeColumnName = selectFeatureCodeColumn(fields, featureKey); + if ( + !fields.includes('x') || + !fields.includes('y') || + !fields.includes(MORTON_CODE_2D_COLUMN) || + !featureCodeColumnName + ) { + return null; + } + + const canLoadRowGroups = await this.canLoadParquetRowGroups(); + const firstRowGroup = + datasetMetadata && canLoadRowGroups + ? await this.loadParquetRowGroupByGroupIndex(parquetPath, 0) + : null; + const bounds = firstRowGroup + ? (extractSentinelBoundingBox(firstRowGroup) ?? undefined) + : undefined; + const rowGroupSizes = datasetMetadata?.rowGroupRows ?? []; + + return { + kind: 'morton-points', + parquetPath, + axisNames, + featureKey, + featureCodeColumnName, + mortonCodeColumnName: MORTON_CODE_2D_COLUMN, + totalRows: datasetMetadata?.totalNumRows ?? 0, + totalRowGroups: datasetMetadata?.totalNumRowGroups ?? 0, + maxRowsPerGroup: rowGroupSizes.length ? Math.max(...rowGroupSizes) : 0, + rowGroupRowCounts: datasetMetadata?.rowGroupRows, + supportsRowGroupRangeReads: Boolean(datasetMetadata && canLoadRowGroups && bounds), + bounds, + }; + } + + async loadPointsInBounds( + elementPath: string, + options: PointsInBoundsOptions + ): Promise { + checkAbort(options.signal); + const metadata = await this.getPointsTilingMetadata(elementPath); + if (metadata?.supportsRowGroupRangeReads && metadata.bounds) { + const rowGroupResult = await this.loadMortonPointsInBounds(elementPath, metadata, options); + if (rowGroupResult) { + return rowGroupResult; + } + } + checkAbort(options.signal); + const full = await this.loadPoints(elementPath); + return filterPointsToBounds(full, options.bounds); + } + + private async bisectRowGroupsRight( + parquetPath: string, + totalRowGroups: number, + targetValue: number + ) { + let lo = 0; + let hi = totalRowGroups; + while (lo < hi) { + const mid = Math.floor((lo + hi) / 2); + const extent = await this.loadParquetRowGroupColumnExtent( + parquetPath, + MORTON_CODE_2D_COLUMN, + mid + ); + const max = extent?.max; + if (max === null || max === undefined || targetValue <= max) { + hi = mid; + } else { + lo = mid + 1; + } + } + return lo; + } + + private async loadMortonPointsInBounds( + elementPath: string, + metadata: PointsTilingMetadata, + options: PointsInBoundsOptions + ): Promise { + if (!metadata.bounds || metadata.totalRowGroups <= 0) { + return null; + } + checkAbort(options.signal); + const intervals = mortonIntervalsForBounds(metadata.bounds, options.bounds); + const rowGroupSet = new Set(); + for (const [start, end] of intervals) { + const first = await this.bisectRowGroupsRight( + metadata.parquetPath, + metadata.totalRowGroups, + start + ); + const last = await this.bisectRowGroupsRight( + metadata.parquetPath, + metadata.totalRowGroups, + end + ); + for (let rowGroup = first; rowGroup <= last; rowGroup++) { + if (rowGroup >= 0 && rowGroup < metadata.totalRowGroups) { + rowGroupSet.add(rowGroup); + } + } + } + const rowGroups = [...rowGroupSet].sort((a, b) => a - b); + const totalRowsUpperBound = rowGroups.reduce( + (sum, rowGroup) => sum + rowGroupCountForIndex(metadata, rowGroup), + 0 + ); + if (totalRowsUpperBound === 0) { + return { + data: [new Float32Array(0), new Float32Array(0)], + shape: [2, 0], + bounds: options.bounds, + loadMode: 'row-groups', + tiling: metadata, + }; + } + + const xs: number[] = []; + const ys: number[] = []; + const zs: number[] = []; + const hasZ = metadata.axisNames.includes('z'); + for (const rowGroup of rowGroups) { + checkAbort(options.signal); + const table = await this.loadParquetRowGroupByGroupIndex(metadata.parquetPath, rowGroup); + const xColumn = table?.getChild('x'); + const yColumn = table?.getChild('y'); + const zColumn = hasZ ? table?.getChild('z') : undefined; + const mortonColumn = table?.getChild(metadata.mortonCodeColumnName); + if (!table || !xColumn || !yColumn) { + continue; + } + for (let i = 0; i < table.numRows; i++) { + if (rowGroup === 0 && i < 4 && mortonColumn?.get(i) === 0) { + continue; + } + const x = xColumn.get(i); + const y = yColumn.get(i); + if (typeof x !== 'number' || typeof y !== 'number') { + continue; + } + if ( + x < options.bounds.minX || + x > options.bounds.maxX || + y < options.bounds.minY || + y > options.bounds.maxY + ) { + continue; + } + xs.push(x); + ys.push(y); + if (hasZ) { + const z = zColumn?.get(i); + zs.push(typeof z === 'number' ? z : 0); + } + } + } + + return { + data: hasZ + ? [new Float32Array(xs), new Float32Array(ys), new Float32Array(zs)] + : [new Float32Array(xs), new Float32Array(ys)], + shape: [hasZ ? 3 : 2, xs.length], + bounds: options.bounds, + loadMode: 'row-groups', + tiling: metadata, + }; + } } diff --git a/packages/core/src/models/VShapesSource.ts b/packages/core/src/models/VShapesSource.ts index 803a4fc1..e5bf115e 100644 --- a/packages/core/src/models/VShapesSource.ts +++ b/packages/core/src/models/VShapesSource.ts @@ -23,15 +23,28 @@ const log = console; // import SpatialDataTableSource from './SpatialDataTableSource.js'; -import type { TypedArray as ZarrTypedArray, Chunk, NumberDataType } from 'zarrita'; import type { Table as ArrowTable } from 'apache-arrow'; import type { Vector } from 'apache-arrow/vector'; -import SpatialDataTableSource from './VTableSource'; +import type { Chunk, NumberDataType, TypedArray as ZarrTypedArray } from 'zarrita'; +import type { SpatialBounds } from '../pointsTiling.js'; import type { ShapesGeometryKind, ShapesRenderData } from '../shapes'; +import SpatialDataTableSource from './VTableSource'; export type PolygonShape = Array>; //nb, not totally happy with this type. export type ZarrNumericArray = ZarrTypedArray | BigInt64Array | Array; +export interface ShapesInBoundsOptions { + bounds: SpatialBounds; + zoom?: number; + signal?: AbortSignal; + columns?: string[]; +} + +export type ShapesInBoundsResult = ShapesRenderData & { + bounds: SpatialBounds; + loadMode: 'full-filter'; +}; + // If the array path starts with table/something/rest // capture table/something. @@ -64,6 +77,12 @@ function getParquetPath(arrPath?: string) { throw new Error(`Cannot determine parquet path for shapes array path: ${arrPath}`); } +function checkAbort(signal?: AbortSignal) { + if (signal?.aborted) { + throw new DOMException('The operation was aborted.', 'AbortError'); + } +} + /** * Converts BigInt64Array or Float64Array to Float32Array if needed. * TODO: remove this and support BigInts/Float64s in downstream code. @@ -363,11 +382,10 @@ export default class SpatialDataShapesSource extends SpatialDataTableSource { // However this may complicate applying transformations, at least in the current way. // Reference: https://deck.gl/docs/api-reference/layers/polygon-layer#data-accessors return arr.map((geom: ArrayBuffer) => { - const coords = - wkb - .readGeometry(geom) - // @ts-expect-error - getCoordinates is not a method of Geometry, check this<<< - .getCoordinates(); + const coords = wkb + .readGeometry(geom) + // @ts-expect-error - getCoordinates is not a method of Geometry, check this<<< + .getCoordinates(); // Take first polygon (if multipolygon) return coords[0]; }); @@ -502,6 +520,20 @@ export default class SpatialDataShapesSource extends SpatialDataTableSource { }; } + async loadShapesInBounds( + elementPath: string, + options: ShapesInBoundsOptions + ): Promise { + checkAbort(options.signal); + const renderData = await this.loadShapesRenderData(elementPath); + checkAbort(options.signal); + return { + ...renderData, + bounds: options.bounds, + loadMode: 'full-filter', + }; + } + /** * * @param path diff --git a/packages/core/src/models/VTableSource.ts b/packages/core/src/models/VTableSource.ts index f303344d..b62d42cb 100644 --- a/packages/core/src/models/VTableSource.ts +++ b/packages/core/src/models/VTableSource.ts @@ -1,16 +1,100 @@ // this is a direct copy of the Vitessce implementation, with changes mostly to make it more normal TypeScript. -import { tableFromIPC, type Table as ArrowTable } from 'apache-arrow'; +import { type Table as ArrowTable, tableFromIPC } from 'apache-arrow'; import type { DataSourceParams } from '../Vutils'; import type { TableColumnData } from '../types'; import AnnDataSource from './VAnnDataSource'; +interface ParquetWasmTableLike { + intoIPCStream(): Uint8Array; +} + +interface ParquetWasmFileMetadata { + numRows(): number; +} + +interface ParquetWasmRowGroupMetadata { + numRows(): number; + fileOffset(): number | bigint; + compressedSize(): number | bigint; +} + +interface ParquetWasmMetadata { + fileMetadata(): ParquetWasmFileMetadata; + numRowGroups(): number; + rowGroup(index: number): ParquetWasmRowGroupMetadata; +} + +interface ParquetModule { + readParquet: (bytes: Uint8Array, options?: { columns?: string[] }) => ParquetWasmTableLike; + readSchema: (bytes: Uint8Array) => ParquetWasmTableLike; + readMetadata?: (bytes: Uint8Array) => ParquetWasmMetadata; + readParquetRowGroup?: ( + schemaBytes: Uint8Array, + rowGroupBytes: Uint8Array, + rowGroupIndex: number + ) => ParquetWasmTableLike; +} + +export interface ParquetPartMetadata { + path: string; + schema: ArrowTable['schema']; + schemaBytes: Uint8Array; + metadata: ParquetWasmMetadata; +} + +export interface ParquetDatasetMetadata { + totalNumRows: number; + totalNumRowGroups: number; + numRowsByPart: number[]; + numRowGroupsByPart: number[]; + numRowsPerGroupByPart: number[]; + rowGroupRows: number[]; + schema: ArrowTable['schema'] | null; + parts: ParquetPartMetadata[]; +} + // Note: This file also serves as the parent for // SpatialDataPointsSource and SpatialDataShapesSource, // because when a table annotates points and shapes, it can be helpful to // have all of the required functionality to load the // table data and the parquet data. +function normalizeParquetModule(module: unknown): ParquetModule { + if (typeof module !== 'object' || module === null) { + throw new Error('parquet-wasm module did not load as an object'); + } + // External WASM builds have drifted API surfaces and incomplete declarations; + // keep the boundary narrow and capability-check every optional method. + const candidate = module as Record; + const { readParquet, readSchema, readMetadata, readParquetRowGroup } = candidate; + if (typeof readParquet !== 'function' || typeof readSchema !== 'function') { + throw new Error('parquet-wasm module is missing required readParquet/readSchema APIs'); + } + return { + readParquet: readParquet as ParquetModule['readParquet'], + readSchema: readSchema as ParquetModule['readSchema'], + readMetadata: + typeof readMetadata === 'function' + ? (readMetadata as ParquetModule['readMetadata']) + : undefined, + readParquetRowGroup: + typeof readParquetRowGroup === 'function' + ? (readParquetRowGroup as ParquetModule['readParquetRowGroup']) + : undefined, + }; +} + +async function initializeParquetModule(module: unknown) { + if (typeof module !== 'object' || module === null) { + return; + } + const maybeInit = (module as Record).default; + if (typeof maybeInit === 'function') { + await maybeInit(); + } +} + async function getParquetModule() { // Dynamic import for code-splitting. parquet-wasm is a WebAssembly module // that needs to be initialized before use in browser environments. @@ -23,10 +107,8 @@ async function getParquetModule() { // Try local import first (works in Node.js, tests, and production builds) try { const module = await import('parquet-wasm'); - if (typeof module.default === 'function') { - await module.default(); - } - return { readParquet: module.readParquet, readSchema: module.readSchema }; + await initializeParquetModule(module); + return normalizeParquetModule(module); } catch (error) { // Local import failed, try CDN fallback (needed in vite dev server) // Reference: https://observablehq.com/@kylebarron/geoparquet-on-the-web @@ -41,8 +123,8 @@ async function getParquetModule() { // @ts-expect-error - CDN import not recognized by TypeScript 'https://cdn.vitessce.io/parquet-wasm@2c23652/esm/parquet_wasm.js' ); - await cdnModule.default(); - return { readParquet: cdnModule.readParquet, readSchema: cdnModule.readSchema }; + await initializeParquetModule(cdnModule); + return normalizeParquetModule(cdnModule); } catch (cdnError) { // Both imports failed, throw an error const localErrorMsg = error instanceof Error ? error.message : String(error); @@ -162,6 +244,14 @@ function hasParquetTailMagic(bytes: Uint8Array) { return bytes.length >= 8 && hasParquetMagic(bytes, bytes.length - 4); } +function toSafeNumber(value: number | bigint, label: string) { + const n = typeof value === 'bigint' ? Number(value) : value; + if (!Number.isSafeInteger(n) || n < 0) { + throw new Error(`Invalid parquet ${label}: ${String(value)}`); + } + return n; +} + /** * This class is a parent class for tables, shapes, and points. * This is because these share functionality, for example: @@ -170,10 +260,7 @@ function hasParquetTailMagic(bytes: Uint8Array) { * - logic for manipulating spatialdata element paths is shared across all elements. */ export default class SpatialDataTableSource extends AnnDataSource { - static parquetModulePromise: Promise<{ - readParquet: (bytes: Uint8Array, options?: { columns?: string[] }) => any; - readSchema: (bytes: Uint8Array) => any; - }>; + static parquetModulePromise: Promise; rootAttrs: { softwareVersion: string; formatVersion: string } | null; // biome-ignore lint/suspicious/noExplicitAny: elementAttrs type should be a tree-ish thing elementAttrs: Record; @@ -322,44 +409,12 @@ export default class SpatialDataTableSource extends AnnDataSource { async loadParquetSchemaBytes(parquetPath: string) { const { store } = this.storeRoot; if (store.getRange) { - // Step 1: Fetch last 8 bytes to get footer length and magic number - const TAIL_LENGTH = 8; let lastError: Error | null = null; for (const candidatePath of getParquetCandidatePaths(parquetPath)) { try { - const tailBytes = await store.getRange(`/${candidatePath}`, { - suffixLength: TAIL_LENGTH, - }); - const normalizedTailBytes = toUint8Array(tailBytes); - if (!normalizedTailBytes || !hasParquetTailMagic(normalizedTailBytes)) { - continue; - } - - // Step 2: Extract footer length and magic number - // little-endian - const footerLength = new DataView( - normalizedTailBytes.buffer, - normalizedTailBytes.byteOffset, - normalizedTailBytes.byteLength - ).getInt32(0, true); - - // Step 3. Fetch the full footer bytes - const footerBytes = await store.getRange(`/${candidatePath}`, { - suffixLength: footerLength + TAIL_LENGTH, - }); - const normalizedFooterBytes = toUint8Array(footerBytes); - if ( - !normalizedFooterBytes || - normalizedFooterBytes.length !== footerLength + TAIL_LENGTH || - !hasParquetTailMagic(normalizedFooterBytes) - ) { - lastError = new Error(`Failed to load parquet footer bytes for ${parquetPath}`); - continue; - } - - // Step 4: Return the footer bytes - return normalizedFooterBytes; + const footerBytes = await this.loadParquetFooterBytesForPath(candidatePath); + if (footerBytes) return footerBytes; } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); } @@ -371,6 +426,177 @@ export default class SpatialDataTableSource extends AnnDataSource { return null; } + private async loadParquetFooterBytesForPath(path: string): Promise { + const { store } = this.storeRoot; + if (!store.getRange) { + return null; + } + const tailLength = 8; + const tailBytes = await store.getRange(`/${path}`, { + suffixLength: tailLength, + }); + const normalizedTailBytes = toUint8Array(tailBytes); + if (!normalizedTailBytes || !hasParquetTailMagic(normalizedTailBytes)) { + return null; + } + + const footerLength = new DataView( + normalizedTailBytes.buffer, + normalizedTailBytes.byteOffset, + normalizedTailBytes.byteLength + ).getInt32(0, true); + + const footerBytes = await store.getRange(`/${path}`, { + suffixLength: footerLength + tailLength, + }); + const normalizedFooterBytes = toUint8Array(footerBytes); + if ( + !normalizedFooterBytes || + normalizedFooterBytes.length !== footerLength + tailLength || + !hasParquetTailMagic(normalizedFooterBytes) + ) { + return null; + } + return normalizedFooterBytes; + } + + async loadParquetSchemaTable(parquetPath: string): Promise { + const schemaBytes = await this.loadParquetSchemaBytes(parquetPath); + if (!schemaBytes) { + return null; + } + const { readSchema } = await SpatialDataTableSource.parquetModulePromise; + const wasmSchema = readSchema(schemaBytes); + return tableFromIPC(wasmSchema.intoIPCStream()); + } + + private async loadParquetPartMetadata(path: string): Promise { + const { readMetadata, readSchema } = await SpatialDataTableSource.parquetModulePromise; + if (!readMetadata) { + return null; + } + const schemaBytes = await this.loadParquetFooterBytesForPath(path); + if (!schemaBytes) { + return null; + } + const schemaTable = await tableFromIPC(readSchema(schemaBytes).intoIPCStream()); + return { + path, + schema: schemaTable.schema, + schemaBytes, + metadata: readMetadata(schemaBytes), + }; + } + + async loadParquetDatasetMetadata(parquetPath: string): Promise { + const { readMetadata } = await SpatialDataTableSource.parquetModulePromise; + const { store } = this.storeRoot; + if (!readMetadata || !store.getRange) { + return null; + } + + const directPart = await this.loadParquetPartMetadata(parquetPath); + const parts: ParquetPartMetadata[] = []; + if (directPart) { + parts.push(directPart); + } else { + for (let partIndex = 0; ; partIndex++) { + const part = await this.loadParquetPartMetadata(`${parquetPath}/part.${partIndex}.parquet`); + if (!part) { + break; + } + parts.push(part); + } + } + + if (parts.length === 0) { + return null; + } + + const numRowsByPart = parts.map((part) => part.metadata.fileMetadata().numRows()); + const numRowGroupsByPart = parts.map((part) => part.metadata.numRowGroups()); + const numRowsPerGroupByPart = parts.map((part) => + part.metadata.numRowGroups() > 0 ? part.metadata.rowGroup(0).numRows() : 0 + ); + const rowGroupRows = parts.flatMap((part) => + Array.from({ length: part.metadata.numRowGroups() }, (_value, rowGroupIndex) => + part.metadata.rowGroup(rowGroupIndex).numRows() + ) + ); + return { + totalNumRows: numRowsByPart.reduce((acc, cur) => acc + cur, 0), + totalNumRowGroups: numRowGroupsByPart.reduce((acc, cur) => acc + cur, 0), + numRowsByPart, + numRowGroupsByPart, + numRowsPerGroupByPart, + rowGroupRows, + schema: parts[0]?.schema ?? null, + parts, + }; + } + + async canLoadParquetRowGroups(): Promise { + const module = await SpatialDataTableSource.parquetModulePromise; + return ( + typeof module.readMetadata === 'function' && typeof module.readParquetRowGroup === 'function' + ); + } + + async loadParquetRowGroupByGroupIndex( + parquetPath: string, + rowGroupIndex: number + ): Promise { + const { readParquetRowGroup } = await SpatialDataTableSource.parquetModulePromise; + const { store } = this.storeRoot; + if (!readParquetRowGroup || !store.getRange) { + return null; + } + const dataset = await this.loadParquetDatasetMetadata(parquetPath); + if (!dataset || rowGroupIndex < 0 || rowGroupIndex >= dataset.totalNumRowGroups) { + return null; + } + + let cumulativeRowGroups = 0; + for (const part of dataset.parts) { + const partRowGroupCount = part.metadata.numRowGroups(); + if (rowGroupIndex >= cumulativeRowGroups + partRowGroupCount) { + cumulativeRowGroups += partRowGroupCount; + continue; + } + const relativeRowGroupIndex = rowGroupIndex - cumulativeRowGroups; + const rowGroup = part.metadata.rowGroup(relativeRowGroupIndex); + const offset = toSafeNumber(rowGroup.fileOffset(), 'row-group file offset'); + const length = toSafeNumber(rowGroup.compressedSize(), 'row-group compressed size'); + const bytes = await store.getRange(`/${part.path}`, { offset, length }); + const rowGroupBytes = toUint8Array(bytes); + if (!rowGroupBytes) { + return null; + } + return tableFromIPC( + readParquetRowGroup(part.schemaBytes, rowGroupBytes, relativeRowGroupIndex).intoIPCStream() + ); + } + return null; + } + + async loadParquetRowGroupColumnExtent( + parquetPath: string, + columnName: string, + rowGroupIndex: number + ): Promise<{ min: number | null; max: number | null } | null> { + const table = await this.loadParquetRowGroupByGroupIndex(parquetPath, rowGroupIndex); + const column = table?.getChild(columnName); + if (!column || column.length === 0) { + return null; + } + const min = column.get(0); + const max = column.get(column.length - 1); + return { + min: typeof min === 'number' ? min : null, + max: typeof max === 'number' ? max : null, + }; + } + /** * Get the index column from a parquet table. * @param parquetPath A path to a parquet file (or directory). @@ -409,7 +635,10 @@ export default class SpatialDataTableSource extends AnnDataSource { return tablePromise; } - private async _loadParquetTableUncached(parquetPath: string, columns?: string[]): Promise { + private async _loadParquetTableUncached( + parquetPath: string, + columns?: string[] + ): Promise { const { readParquet, readSchema } = await SpatialDataTableSource.parquetModulePromise; const options = { diff --git a/packages/core/src/models/index.ts b/packages/core/src/models/index.ts index 253fef4e..a858ab20 100644 --- a/packages/core/src/models/index.ts +++ b/packages/core/src/models/index.ts @@ -494,6 +494,10 @@ export class ShapesElement extends AbstractSpatialElement<'shapes', ShapesAttrs> }); return renderData; } + + async loadShapesInBounds(options: Parameters[1]) { + return this.vShapes.loadShapesInBounds(`shapes/${this.key}`, options); + } } // ============================================ @@ -534,6 +538,14 @@ export class PointsElement extends AbstractSpatialElement<'points', PointsAttrs> //we have points.parquet/part.0.parquet etc. return this.vPoints.loadPoints(`points/${this.key}`); } + + async getPointsTilingMetadata() { + return this.vPoints.getPointsTilingMetadata(`points/${this.key}`); + } + + async loadPointsInBounds(options: Parameters[1]) { + return this.vPoints.loadPointsInBounds(`points/${this.key}`, options); + } } // ============================================ diff --git a/packages/core/src/pointsTiling.ts b/packages/core/src/pointsTiling.ts new file mode 100644 index 00000000..6131f2da --- /dev/null +++ b/packages/core/src/pointsTiling.ts @@ -0,0 +1,265 @@ +import type { Table as ArrowTable } from 'apache-arrow'; +import type { AxisAlignedBounds, PointsColumnarData } from './spatialViewFit.js'; + +export const MORTON_CODE_2D_COLUMN = 'morton_code_2d'; +export const MORTON_CODE_EXTREME_VALUE_INDICATOR = 0; +export const MORTON_CODE_BITS_PER_AXIS = 16; +export const MORTON_CODE_VALUE_MAX = 2 ** MORTON_CODE_BITS_PER_AXIS - 1; + +export type SpatialBounds = AxisAlignedBounds; + +export interface PointsInBoundsOptions { + bounds: SpatialBounds; + zoom?: number; + signal?: AbortSignal; + columns?: string[]; +} + +export interface PointsTilingMetadata { + kind: 'morton-points'; + parquetPath: string; + axisNames: string[]; + featureKey?: string; + featureCodeColumnName: string; + mortonCodeColumnName: typeof MORTON_CODE_2D_COLUMN; + totalRows: number; + totalRowGroups: number; + maxRowsPerGroup: number; + rowGroupRowCounts?: number[]; + supportsRowGroupRangeReads: boolean; + bounds?: SpatialBounds; +} + +export type PointsInBoundsResult = PointsColumnarData & { + bounds: SpatialBounds; + loadMode: 'row-groups' | 'full-filter'; + tiling?: PointsTilingMetadata; + featureIndices?: ArrayLike; +}; + +export function origCoordToNormCoord(x: number, y: number, bbox: SpatialBounds): [number, number] { + const xRange = bbox.maxX - bbox.minX; + const yRange = bbox.maxY - bbox.minY; + if (xRange <= 0 || yRange <= 0) { + return [0, 0]; + } + return [ + Math.max( + 0, + Math.min( + MORTON_CODE_VALUE_MAX, + Math.floor(((x - bbox.minX) / xRange) * MORTON_CODE_VALUE_MAX) + ) + ), + Math.max( + 0, + Math.min( + MORTON_CODE_VALUE_MAX, + Math.floor(((y - bbox.minY) / yRange) * MORTON_CODE_VALUE_MAX) + ) + ), + ]; +} + +function intersects( + ax0: number, + ay0: number, + ax1: number, + ay1: number, + bx0: number, + by0: number, + bx1: number, + by1: number +) { + return !(ax1 < bx0 || bx1 < ax0 || ay1 < by0 || by1 < ay0); +} + +function contained( + ix0: number, + iy0: number, + ix1: number, + iy1: number, + ox0: number, + oy0: number, + ox1: number, + oy1: number +) { + return ox0 <= ix0 && ix0 <= ix1 && ix1 <= ox1 && oy0 <= iy0 && iy0 <= iy1 && iy1 <= oy1; +} + +function cellRange(prefix: number, level: number, bits: number): [number, number] { + const shift = 2 * (bits - level); + const power = 2 ** shift; + return [prefix * power, (prefix + 1) * power - 1]; +} + +export function mergeAdjacentIntervals( + intervals: Array<[number, number]> +): Array<[number, number]> { + if (intervals.length === 0) { + return []; + } + const sorted = [...intervals].sort((a, b) => a[0] - b[0]); + const merged: Array<[number, number]> = [sorted[0]]; + for (const [lo, hi] of sorted.slice(1)) { + const last = merged[merged.length - 1]; + if (lo <= last[1] + 1) { + last[1] = Math.max(last[1], hi); + } else { + merged.push([lo, hi]); + } + } + return merged; +} + +export function zcoverRectangle( + rx0: number, + ry0: number, + rx1: number, + ry1: number, + bits = MORTON_CODE_BITS_PER_AXIS +): Array<[number, number]> { + const maxCoord = 2 ** bits - 1; + const x0 = Math.max(0, Math.min(maxCoord, Math.min(rx0, rx1))); + const x1 = Math.max(0, Math.min(maxCoord, Math.max(rx0, rx1))); + const y0 = Math.max(0, Math.min(maxCoord, Math.min(ry0, ry1))); + const y1 = Math.max(0, Math.min(maxCoord, Math.max(ry0, ry1))); + + const intervals: Array<[number, number]> = []; + const stack: Array<[number, number, number, number, number, number]> = [ + [0, 0, 0, 0, maxCoord, maxCoord], + ]; + + while (stack.length > 0) { + const current = stack.pop(); + if (!current) { + continue; + } + const [prefix, level, xmin, ymin, xmax, ymax] = current; + if (!intersects(xmin, ymin, xmax, ymax, x0, y0, x1, y1)) { + continue; + } + if (contained(xmin, ymin, xmax, ymax, x0, y0, x1, y1) || level === bits) { + intervals.push(cellRange(prefix, level, bits)); + continue; + } + + const midx = Math.floor((xmin + xmax) / 2); + const midy = Math.floor((ymin + ymax) / 2); + stack.push([(prefix << 2) | 0, level + 1, xmin, ymin, midx, midy]); + stack.push([(prefix << 2) | 1, level + 1, midx + 1, ymin, xmax, midy]); + stack.push([(prefix << 2) | 2, level + 1, xmin, midy + 1, midx, ymax]); + stack.push([(prefix << 2) | 3, level + 1, midx + 1, midy + 1, xmax, ymax]); + } + + return mergeAdjacentIntervals(intervals); +} + +export function mortonIntervalsForBounds( + allPointsBounds: SpatialBounds, + queryBounds: SpatialBounds +): Array<[number, number]> { + const [x0, y0] = origCoordToNormCoord(queryBounds.minX, queryBounds.minY, allPointsBounds); + const [x1, y1] = origCoordToNormCoord(queryBounds.maxX, queryBounds.maxY, allPointsBounds); + return zcoverRectangle(x0, y0, x1, y1); +} + +function getNumericValue(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value)) { + return null; + } + return value; +} + +export function extractSentinelBoundingBox( + table: ArrowTable, + xColumnName = 'x', + yColumnName = 'y', + mortonColumnName = MORTON_CODE_2D_COLUMN +): SpatialBounds | null { + const xColumn = table.getChild(xColumnName); + const yColumn = table.getChild(yColumnName); + const mortonColumn = table.getChild(mortonColumnName); + if (!xColumn || !yColumn || !mortonColumn) { + return null; + } + + const maxRows = Math.min(4, table.numRows); + const xs: number[] = []; + const ys: number[] = []; + for (let i = 0; i < maxRows; i++) { + if (mortonColumn.get(i) !== MORTON_CODE_EXTREME_VALUE_INDICATOR) { + break; + } + const x = getNumericValue(xColumn.get(i)); + const y = getNumericValue(yColumn.get(i)); + if (x === null || y === null) { + continue; + } + xs.push(x); + ys.push(y); + } + if (xs.length < 2 || ys.length < 2) { + return null; + } + return { + minX: Math.min(...xs), + minY: Math.min(...ys), + maxX: Math.max(...xs), + maxY: Math.max(...ys), + }; +} + +export function filterPointsToBounds( + data: PointsColumnarData, + bounds: SpatialBounds, + featureIndices?: ArrayLike +): PointsInBoundsResult { + const xs = data.data[0]; + const ys = data.data[1]; + const zs = data.data[2]; + const keep: number[] = []; + const n = Math.min(xs?.length ?? 0, ys?.length ?? 0); + for (let i = 0; i < n; i++) { + const x = xs[i]; + const y = ys[i]; + if ( + Number.isFinite(x) && + Number.isFinite(y) && + x >= bounds.minX && + x <= bounds.maxX && + y >= bounds.minY && + y <= bounds.maxY + ) { + keep.push(i); + } + } + + const outX = new Float32Array(keep.length); + const outY = new Float32Array(keep.length); + const outZ = zs ? new Float32Array(keep.length) : undefined; + const outFeatureIndices = featureIndices ? new Uint32Array(keep.length) : undefined; + for (let i = 0; i < keep.length; i++) { + const sourceIndex = keep[i]; + outX[i] = xs[sourceIndex]; + outY[i] = ys[sourceIndex]; + if (outZ) { + outZ[i] = zs?.[sourceIndex] ?? 0; + } + if (outFeatureIndices) { + outFeatureIndices[i] = featureIndices?.[sourceIndex] ?? 0; + } + } + + return { + data: outZ ? [outX, outY, outZ] : [outX, outY], + shape: [outZ ? 3 : 2, keep.length], + bounds, + loadMode: 'full-filter', + featureIndices: outFeatureIndices, + }; +} + +export function boundsFromStoredPointsBounds(bounds: SpatialBounds): SpatialBounds { + return bounds; +} diff --git a/packages/core/src/spatialViewFit.ts b/packages/core/src/spatialViewFit.ts index 3850713b..b1ebc827 100644 --- a/packages/core/src/spatialViewFit.ts +++ b/packages/core/src/spatialViewFit.ts @@ -22,7 +22,7 @@ export type OrthographicViewState2D = { /** Ndarray-style columnar points: data[0]=x, data[1]=y, optional data[2]=z. */ export type PointsColumnarData = { - data: number[][]; + data: ArrayLike[]; shape?: number[]; }; diff --git a/packages/core/tests/pointsTiling.spec.ts b/packages/core/tests/pointsTiling.spec.ts new file mode 100644 index 00000000..765b150c --- /dev/null +++ b/packages/core/tests/pointsTiling.spec.ts @@ -0,0 +1,94 @@ +import type { Table as ArrowTable } from 'apache-arrow'; +import { describe, expect, it } from 'vitest'; +import { + extractSentinelBoundingBox, + filterPointsToBounds, + mergeAdjacentIntervals, + mortonIntervalsForBounds, + zcoverRectangle, +} from '../src/pointsTiling.js'; + +function vector(values: unknown[]) { + return { + length: values.length, + get: (index: number) => values[index], + }; +} + +function table(columns: Record): ArrowTable { + const first = Object.values(columns)[0] ?? []; + return { + numRows: first.length, + getChild: (name: string) => { + const values = columns[name]; + return values ? vector(values) : null; + }, + } as unknown as ArrowTable; +} + +describe('points tiling helpers', () => { + it('extracts the Vitessce sentinel bounding box from the leading rows', () => { + const arrowTable = table({ + x: [10, 20, 15, 17, 99], + y: [5, 8, 40, 12, 99], + morton_code_2d: [0, 0, 0, 0, 123], + }); + + expect(extractSentinelBoundingBox(arrowTable)).toEqual({ + minX: 10, + minY: 5, + maxX: 20, + maxY: 40, + }); + }); + + it('rejects missing or incomplete sentinel bounds', () => { + expect( + extractSentinelBoundingBox( + table({ + x: [10, 20], + y: [5, 8], + morton_code_2d: [7, 8], + }) + ) + ).toBeNull(); + }); + + it('merges adjacent Morton intervals', () => { + expect( + mergeAdjacentIntervals([ + [10, 12], + [13, 15], + [20, 21], + ]) + ).toEqual([ + [10, 15], + [20, 21], + ]); + }); + + it('covers a full rectangle with the full Morton range', () => { + expect(zcoverRectangle(0, 0, 65535, 65535)).toEqual([[0, 4294967295]]); + }); + + it('produces intervals for a query rectangle inside a stored bbox', () => { + const intervals = mortonIntervalsForBounds( + { minX: 0, minY: 0, maxX: 100, maxY: 100 }, + { minX: 10, minY: 10, maxX: 20, maxY: 20 } + ); + expect(intervals.length).toBeGreaterThan(0); + expect(intervals.every(([lo, hi]) => lo <= hi)).toBe(true); + }); + + it('filters columnar points to bounds without changing source arrays', () => { + const xs = new Float32Array([0, 5, 10]); + const ys = new Float32Array([0, 5, 20]); + const filtered = filterPointsToBounds( + { data: [xs, ys], shape: [2, 3] }, + { minX: 1, minY: 1, maxX: 10, maxY: 10 } + ); + expect(Array.from(filtered.data[0])).toEqual([5]); + expect(Array.from(filtered.data[1])).toEqual([5]); + expect(filtered.shape).toEqual([2, 1]); + }); +}); diff --git a/packages/vis/src/SpatialCanvas/SpatialCanvasViewer.tsx b/packages/vis/src/SpatialCanvas/SpatialCanvasViewer.tsx index 8d46cce1..3e2b29a7 100644 --- a/packages/vis/src/SpatialCanvas/SpatialCanvasViewer.tsx +++ b/packages/vis/src/SpatialCanvas/SpatialCanvasViewer.tsx @@ -21,13 +21,13 @@ import { SpatialViewer } from './SpatialViewer'; import { VivLoaderRegistryProvider } from './VivLoaderRegistry'; import { getDeckFromDeckGlRef, resolveHoverFeatureTooltip } from './featureTooltipHover'; import { + type RenderStackHostLayerResolver, + type RenderStackLayerInputs, + type UnknownRenderStackHostLayerHandler, renderStackOrder, renderStackToLayerInputs, resolveRenderStackHostLayers, sortLayersByRenderStackOrder, - type RenderStackHostLayerResolver, - type RenderStackLayerInputs, - type UnknownRenderStackHostLayerHandler, } from './renderStackAdapters'; import type { ElementsByType, LayerConfig, ShapesLayerPickEvent, ViewState } from './types'; import { @@ -85,6 +85,7 @@ export interface SpatialCanvasViewerProps { * When true (default), hover tooltips aggregate picks from all layers under the cursor. */ aggregateHoverTooltips?: boolean; + experimentalOptimizations?: 'auto' | 'off'; } interface AutoFitInput { @@ -146,6 +147,7 @@ export interface UseSpatialCanvasRendererOptions { hostLayerResolver?: RenderStackHostLayerResolver; onUnknownHostLayer?: UnknownRenderStackHostLayerHandler; autoFit?: boolean; + experimentalOptimizations?: 'auto' | 'off'; } interface UseSpatialCanvasRendererFromLayerInputsOptions { @@ -161,6 +163,7 @@ interface UseSpatialCanvasRendererFromLayerInputsOptions { externalDeckLayers?: Layer[]; sortDeckLayers?: boolean; autoFit?: boolean; + experimentalOptimizations?: 'auto' | 'off'; } export function useSpatialCanvasRendererFromLayerInputs({ @@ -176,6 +179,7 @@ export function useSpatialCanvasRendererFromLayerInputs({ externalDeckLayers, sortDeckLayers, autoFit = true, + experimentalOptimizations = 'auto', }: UseSpatialCanvasRendererFromLayerInputsOptions) { const availableElements = useMemo(() => { if (!spatialData || !coordinateSystem) { @@ -191,7 +195,8 @@ export function useSpatialCanvasRendererFromLayerInputs({ layerInputs.layerOrder, availableElements, coordinateSystem, - spatialData ?? undefined + spatialData ?? undefined, + experimentalOptimizations ); const generatedDeckLayers = layerData.getLayers(); @@ -269,6 +274,7 @@ export function useSpatialCanvasRenderer({ hostLayerResolver, onUnknownHostLayer, autoFit = true, + experimentalOptimizations = 'auto', }: UseSpatialCanvasRendererOptions) { const layerInputs = useMemo(() => renderStackToLayerInputs(renderStack), [renderStack]); const hostDeckLayers = useMemo( @@ -292,6 +298,7 @@ export function useSpatialCanvasRenderer({ hostDeckLayers, sortDeckLayers: true, autoFit, + experimentalOptimizations, }); } @@ -348,6 +355,7 @@ function SpatialCanvasViewerInner({ autoFit = true, style, aggregateHoverTooltips = true, + experimentalOptimizations = 'auto', }: SpatialCanvasViewerProps) { const [measureRef, { width, height }] = useMeasure(); const viewerContainerRef = useRef(null); @@ -385,6 +393,7 @@ function SpatialCanvasViewerInner({ externalDeckLayers, sortDeckLayers: Boolean(renderStack), autoFit, + experimentalOptimizations, }); const hoverPickLayerIds = useMemo( () => Array.from(renderer.enabledLayerIds), diff --git a/packages/vis/src/SpatialCanvas/index.tsx b/packages/vis/src/SpatialCanvas/index.tsx index e15650e8..51bafb15 100644 --- a/packages/vis/src/SpatialCanvas/index.tsx +++ b/packages/vis/src/SpatialCanvas/index.tsx @@ -39,8 +39,8 @@ import { TooltipFieldsPanel } from './TooltipFieldsPanel'; import { VivLoaderRegistryProvider } from './VivLoaderRegistry'; import { SpatialCanvasProvider, useSpatialCanvasActions, useSpatialCanvasStore } from './context'; import { getDeckFromDeckGlRef, resolveHoverFeatureTooltip } from './featureTooltipHover'; -import type { SpatialCanvasStoreApi } from './stores'; import { layerConfig } from './layerConfig'; +import type { SpatialCanvasStoreApi } from './stores'; import type { AvailableElement, ElementsByType, ViewState } from './types'; import type { ImageLayerConfig } from './useLayerData'; import { generateLayerId, getAllCoordinateSystems } from './utils'; @@ -365,12 +365,14 @@ interface SpatialCanvasInnerProps { * When true (default), hover tooltips include picks from all layers under the cursor. */ aggregateHoverTooltips?: boolean; + experimentalOptimizations?: 'auto' | 'off'; } function SpatialCanvasInner({ tooltipContainer, renderTooltip, aggregateHoverTooltips = true, + experimentalOptimizations = 'auto', }: SpatialCanvasInnerProps) { const { spatialData, loading: sdLoading } = useSpatialData(); const [measureRef, { width, height }] = useMeasure(); @@ -426,6 +428,7 @@ function SpatialCanvasInner({ // are managed entirely by ViewerSection so this hook never re-runs on pan. width: vw, height: vh, + experimentalOptimizations, }); const hoverPickLayerIds = useMemo(() => Array.from(enabledLayerIds), [enabledLayerIds]); @@ -869,6 +872,7 @@ export interface SpatialCanvasProps { * When true (default), hover tooltips aggregate picks from all layers under the cursor. */ aggregateHoverTooltips?: boolean; + experimentalOptimizations?: 'auto' | 'off'; } /** @@ -924,6 +928,7 @@ export default function SpatialCanvas({ tooltipContainer, renderTooltip, aggregateHoverTooltips, + experimentalOptimizations, }: SpatialCanvasProps) { return ( @@ -932,6 +937,7 @@ export default function SpatialCanvas({ tooltipContainer={tooltipContainer} renderTooltip={renderTooltip} aggregateHoverTooltips={aggregateHoverTooltips} + experimentalOptimizations={experimentalOptimizations} /> diff --git a/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts b/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts index e8bda519..2a077c97 100644 --- a/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts +++ b/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts @@ -4,9 +4,9 @@ * Renders point cloud data from SpatialData points elements. */ -import { ScatterplotLayer } from 'deck.gl'; import type { Matrix4 } from '@math.gl/core'; -import type { PointsElement } from '@spatialdata/core'; +import type { PointsElement, PointsTilingMetadata, SpatialBounds } from '@spatialdata/core'; +import { ScatterplotLayer, TileLayer } from 'deck.gl'; import type { Layer } from 'deck.gl'; export interface PointDataX { @@ -20,7 +20,7 @@ export interface PointDataX { export interface PointData { shape: number[]; // this should most definitely be TypedArray... - data: number[][]; + data: ArrayLike[]; } export interface PointsLayerRenderConfig { @@ -40,9 +40,82 @@ export interface PointsLayerRenderConfig { color?: [number, number, number, number]; /** ndarray - if we want other data for properties like color/radius etc they will be handled differently */ pointData?: PointData; + pointTilingMetadata?: PointsTilingMetadata; use3d?: boolean; } +type PointTileBbox = { + left: number; + right: number; + top: number; + bottom: number; +}; + +type PointTileLoadProps = { + bbox: unknown; + signal?: AbortSignal; +}; + +function isAbortError(error: unknown) { + return error instanceof DOMException && error.name === 'AbortError'; +} + +function isPointTileBbox(value: unknown): value is PointTileBbox { + if (!value || typeof value !== 'object') { + return false; + } + const candidate = value as Record; + return ( + typeof candidate.left === 'number' && + typeof candidate.right === 'number' && + typeof candidate.top === 'number' && + typeof candidate.bottom === 'number' + ); +} + +function boundsFromTileBbox(bbox: unknown): SpatialBounds | null { + if (!isPointTileBbox(bbox)) { + return null; + } + return { + minX: Math.min(bbox.left, bbox.right), + maxX: Math.max(bbox.left, bbox.right), + minY: Math.min(bbox.top, bbox.bottom), + maxY: Math.max(bbox.top, bbox.bottom), + }; +} + +function renderPointScatterSubLayer( + id: string, + data: PointData, + props: { + color: [number, number, number, number]; + pointSize: number; + opacity: number; + modelMatrix: Matrix4; + use3d?: boolean; + } +) { + const d = data.data; + return new ScatterplotLayer({ + id, + data: d[0], + getPosition: (_d, { index, target }) => [ + d[0][index], + d[1][index], + props.use3d ? d[2]?.[index] || 0 : 0, + ], + getRadius: props.pointSize, + radiusUnits: 'pixels', + getFillColor: props.color, + opacity: props.opacity, + modelMatrix: props.modelMatrix, + pickable: true, + autoHighlight: true, + highlightColor: [255, 255, 0, 200], + }); +} + /** * Create a deck.gl ScatterplotLayer for points data. * @@ -59,41 +132,72 @@ export function renderPointsLayer(config: PointsLayerRenderConfig): Layer | null pointSize = 1, color = [255, 100, 100, 200], pointData, + pointTilingMetadata, use3d, } = config; if (!visible) return null; if (!pointData) { - // Data not loaded yet - console.debug( - `[PointsRenderer] No point data for layer "${id}" from ${element.url ?? element.path}` - ); + if (!pointTilingMetadata?.bounds) { + console.debug( + `[PointsRenderer] No point data for layer "${id}" from ${element.url ?? element.path}` + ); + return null; + } + return new TileLayer({ + id, + data: pointTilingMetadata.parquetPath, + extent: [ + pointTilingMetadata.bounds.minX, + pointTilingMetadata.bounds.minY, + pointTilingMetadata.bounds.maxX, + pointTilingMetadata.bounds.maxY, + ], + tileSize: 512, + minZoom: -12, + maxZoom: 12, + refinementStrategy: 'best-available', + updateTriggers: { + getTileData: [element, pointTilingMetadata.parquetPath], + }, + async getTileData({ bbox, signal }: PointTileLoadProps) { + const bounds = boundsFromTileBbox(bbox); + if (!bounds) { + return null; + } + try { + return await element.loadPointsInBounds({ bounds, signal }); + } catch (error) { + if (signal?.aborted || isAbortError(error)) { + return null; + } + throw error; + } + }, + renderSubLayers: (props: { id: string; data?: PointData | null }) => { + if (!props.data) { + return null; + } + return renderPointScatterSubLayer(`${props.id}-scatter`, props.data, { + color, + pointSize, + opacity, + modelMatrix, + use3d, + }); + }, + }); + } + + if (!pointData) { return null; } - const d = pointData.data; - return new ScatterplotLayer({ - id, - data: d[0], //just for index really - // todo: more robust ndarray handling, be more efficient with target - // see https://deck.gl/docs/developer-guide/performance#supply-attributes-directly - // spatial data-structure (quad/oct-tree) vs pushing raw attributes. - // with ways of querying within view. - // also allow accessors for other props - getPosition: (_d, { index, target }) => [ - d[0][index], - d[1][index], - use3d ? d[2]?.[index] || 0 : 0, - ], - getRadius: pointSize, - radiusUnits: 'pixels', - getFillColor: color, + return renderPointScatterSubLayer(id, pointData, { + color, + pointSize, opacity, - // Apply coordinate transformation modelMatrix, - // Picking - pickable: true, - autoHighlight: true, - highlightColor: [255, 255, 0, 200], + use3d, }); } diff --git a/packages/vis/src/SpatialCanvas/types.ts b/packages/vis/src/SpatialCanvas/types.ts index 78fae21e..229e6bf9 100644 --- a/packages/vis/src/SpatialCanvas/types.ts +++ b/packages/vis/src/SpatialCanvas/types.ts @@ -95,6 +95,7 @@ export interface PointsLayerConfig extends BaseLayerConfig { // should be able to filter etc. Some kind of LOD... pointSize?: number; color?: [number, number, number, number]; + experimentalOptimizations?: 'auto' | 'off'; } export interface LabelsLayerConfig extends BaseLayerConfig { diff --git a/packages/vis/src/SpatialCanvas/useLayerData.ts b/packages/vis/src/SpatialCanvas/useLayerData.ts index 63d2cbf2..d76de6de 100644 --- a/packages/vis/src/SpatialCanvas/useLayerData.ts +++ b/packages/vis/src/SpatialCanvas/useLayerData.ts @@ -23,6 +23,7 @@ import { type LabelsElement, type LabelsTooltipMetadata, type PointsElement, + type PointsTilingMetadata, type ShapesElement, type ShapesRenderData, type ShapesTooltipMetadata, @@ -101,6 +102,7 @@ export interface WorldBoundsCacheEntry { interface LoadedData { shapes: Map; points: Map; + pointTilingMetadata: Map; images: Map; // Viv loaders with computed channel data labels: Map; /** @@ -315,6 +317,33 @@ function getWorldBoundsCacheKey(elem: AvailableElement): string { return `${elem.type}:${elem.key}`; } +function transformAxisAlignedBounds( + bounds: AxisAlignedBounds, + modelMatrix: Matrix4 +): AxisAlignedBounds | null { + const corners: [number, number, number][] = [ + [bounds.minX, bounds.minY, 0], + [bounds.maxX, bounds.minY, 0], + [bounds.maxX, bounds.maxY, 0], + [bounds.minX, bounds.maxY, 0], + ]; + let minX = Number.POSITIVE_INFINITY; + let minY = Number.POSITIVE_INFINITY; + let maxX = Number.NEGATIVE_INFINITY; + let maxY = Number.NEGATIVE_INFINITY; + for (const corner of corners) { + const transformed = modelMatrix.transformAsPoint(corner); + if (!Number.isFinite(transformed[0]) || !Number.isFinite(transformed[1])) { + return null; + } + minX = Math.min(minX, transformed[0]); + minY = Math.min(minY, transformed[1]); + maxX = Math.max(maxX, transformed[0]); + maxY = Math.max(maxY, transformed[1]); + } + return { minX, minY, maxX, maxY }; +} + export function resolveLayerElement( layerId: string, config: LayerConfig | undefined, @@ -448,7 +477,8 @@ export function useLayerData( layerOrder: string[], availableElements: ElementsByType, coordinateSystem: string | null, - spatialData?: SpatialData + spatialData?: SpatialData, + experimentalOptimizations: 'auto' | 'off' = 'auto' ): UseLayerDataResult { const { getOmeZarrMultiscalesData } = useVivLoaderRegistry(); @@ -456,6 +486,7 @@ export function useLayerData( const loadedDataRef = useRef({ shapes: new Map(), points: new Map(), + pointTilingMetadata: new Map(), images: new Map(), labels: new Map(), shapePrebuiltData: new Map(), @@ -546,6 +577,7 @@ export function useLayerData( loadFillColor: boolean; loadImage: boolean; loadPoints: boolean; + loadPointTilingMetadata: boolean; loadLabels: boolean; }> = []; @@ -575,6 +607,7 @@ export function useLayerData( loadFillColor, loadImage: false, loadPoints: false, + loadPointTilingMetadata: false, loadLabels: false, }); } @@ -592,20 +625,32 @@ export function useLayerData( loadFillColor: false, loadImage: false, loadPoints: false, + loadPointTilingMetadata: false, loadLabels, }); } - } else if (config.type === 'points' && !loaded.points.has(elem.key)) { - toLoad.push({ - layerId, - element: elem, - loadGeometry: false, - loadTooltip: false, - loadFillColor: false, - loadImage: false, - loadPoints: true, - loadLabels: false, - }); + } else if (config.type === 'points') { + const wantsOptimized = + experimentalOptimizations !== 'off' && config.experimentalOptimizations !== 'off'; + const metadataKnown = loaded.pointTilingMetadata.has(elem.key); + const tiledMetadata = loaded.pointTilingMetadata.get(elem.key); + const loadPointTilingMetadata = wantsOptimized && !metadataKnown; + const loadPoints = + !loaded.points.has(elem.key) && + (!wantsOptimized || (metadataKnown && tiledMetadata === null)); + if (loadPointTilingMetadata || loadPoints) { + toLoad.push({ + layerId, + element: elem, + loadGeometry: false, + loadTooltip: false, + loadFillColor: false, + loadImage: false, + loadPoints, + loadPointTilingMetadata, + loadLabels: false, + }); + } } else if (config.type === 'image' && !loaded.images.has(elem.key)) { toLoad.push({ layerId, @@ -615,6 +660,7 @@ export function useLayerData( loadFillColor: false, loadImage: true, loadPoints: false, + loadPointTilingMetadata: false, loadLabels: false, }); } @@ -633,6 +679,7 @@ export function useLayerData( loadFillColor, loadImage, loadPoints, + loadPointTilingMetadata, loadLabels, }) => { if (element.type === 'shapes') { @@ -769,17 +816,38 @@ export function useLayerData( } } } - } else if (element.type === 'points' && loadPoints) { - try { - setLayerResourceStatus(layerId, 'geometry', 'loading'); - // todo better type-guards etc here. - const e = element.element as PointsElement; - const data = await e.loadPoints(); - loadedDataRef.current.points.set(element.key, data); - setLayerResourceStatus(layerId, 'geometry', 'ready'); - } catch (error) { - setLayerResourceStatus(layerId, 'geometry', 'error'); - console.error(`Failed to load points for ${layerId}:`, error); + } else if (element.type === 'points') { + const e = element.element as PointsElement; + if (loadPointTilingMetadata) { + try { + setLayerResourceStatus(layerId, 'geometry', 'loading'); + const metadata = await e.getPointsTilingMetadata(); + const renderableMetadata = + metadata?.supportsRowGroupRangeReads && metadata.bounds ? metadata : null; + loadedDataRef.current.pointTilingMetadata.set(element.key, renderableMetadata); + setLayerResourceStatus( + layerId, + 'geometry', + renderableMetadata ? 'ready' : 'idle' + ); + notifyLoadedDataChanged(); + } catch (error) { + loadedDataRef.current.pointTilingMetadata.set(element.key, null); + setLayerResourceStatus(layerId, 'geometry', 'error'); + console.error(`Failed to inspect point tiling metadata for ${layerId}:`, error); + notifyLoadedDataChanged(); + } + } + if (loadPoints) { + try { + setLayerResourceStatus(layerId, 'geometry', 'loading'); + const data = await e.loadPoints(); + loadedDataRef.current.points.set(element.key, data); + setLayerResourceStatus(layerId, 'geometry', 'ready'); + } catch (error) { + setLayerResourceStatus(layerId, 'geometry', 'error'); + console.error(`Failed to load points for ${layerId}:`, error); + } } } else if (element.type === 'image' && loadImage) { try { @@ -1048,6 +1116,7 @@ export function useLayerData( spatialData, setLayerResourceStatus, notifyLoadedDataChanged, + experimentalOptimizations, ]); const reloadElement = useCallback((type: string, key: string) => { @@ -1064,6 +1133,7 @@ export function useLayerData( } } else if (type === 'points') { loaded.points.delete(key); + loaded.pointTilingMetadata.delete(key); loaded.worldBounds.delete(`points:${key}`); } else if (type === 'image') { loaded.images.delete(key); @@ -1093,7 +1163,10 @@ export function useLayerData( return loadedDataRef.current.shapes.has(elem.key); } if (elem.type === 'points') { - return loadedDataRef.current.points.has(elem.key); + return ( + loadedDataRef.current.points.has(elem.key) || + Boolean(loadedDataRef.current.pointTilingMetadata.get(elem.key)?.bounds) + ); } if (elem.type === 'image') { return loadedDataRef.current.images.has(elem.key); @@ -1134,13 +1207,19 @@ export function useLayerData( } if (elem.type === 'points') { const pointData = loaded.points.get(elem.key); - if (!pointData) return null; + const tilingMetadata = loaded.pointTilingMetadata.get(elem.key); + if (!pointData && !tilingMetadata?.bounds) return null; return getCachedWorldBounds( loaded.worldBounds, getWorldBoundsCacheKey(elem), - pointData, + pointData ?? tilingMetadata, elem.transform, - () => boundsFromPoints(pointData, elem.transform, false) + () => + pointData + ? boundsFromPoints(pointData, elem.transform, false) + : tilingMetadata?.bounds + ? transformAxisAlignedBounds(tilingMetadata.bounds, elem.transform) + : null ); } if (elem.type === 'image') { @@ -1238,7 +1317,8 @@ export function useLayerData( } } else if (config.type === 'points') { const pointData = loaded.points.get(elem.key); - if (pointData) { + const pointTilingMetadata = loaded.pointTilingMetadata.get(elem.key) ?? undefined; + if (pointData || pointTilingMetadata) { const layer = renderPointsLayer({ element: elem.element as PointsElement, id: layerId, @@ -1248,6 +1328,7 @@ export function useLayerData( pointSize: config.pointSize, color: config.color, pointData, + pointTilingMetadata, }); if (layer) deckLayers.push(layer); } From ea9cbf1ea696384d4cedbdfb913df846517df525 Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Fri, 19 Jun 2026 18:36:47 +0100 Subject: [PATCH 03/38] Initialize spatialdata-experimental-writer package - Added `pyproject.toml` for package configuration, including dependencies and build system. - Created `README.md` with an overview of the experimental vector optimization writers for SpatialData. - Implemented core functionality in `src/spatialdata_experimental_writer`, including Morton sorting and multiscale Parquet writing. - Developed command-line interface in `cli.py` for various operations on SpatialData. - Introduced Zarr integration for reading and writing points data. - Added tests for key functionalities in `tests/test_points.py` and `tests/test_zarr.py`. - Established package structure with necessary modules and scripts. --- .../spatialdata-experimental-writer/README.md | 16 + .../pyproject.toml | 34 ++ .../__init__.py | 17 + .../spatialdata_experimental_writer/cli.py | 330 ++++++++++++++++++ .../spatialdata_experimental_writer/points.py | 217 ++++++++++++ .../spatialdata_experimental_writer/zarr.py | 62 ++++ .../tests/test_points.py | 69 ++++ .../tests/test_zarr.py | 66 ++++ .../spatialdata-experimental-writer/uv.lock | 283 +++++++++++++++ 9 files changed, 1094 insertions(+) create mode 100644 python/spatialdata-experimental-writer/README.md create mode 100644 python/spatialdata-experimental-writer/pyproject.toml create mode 100644 python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/__init__.py create mode 100644 python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/cli.py create mode 100644 python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/points.py create mode 100644 python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/zarr.py create mode 100644 python/spatialdata-experimental-writer/tests/test_points.py create mode 100644 python/spatialdata-experimental-writer/tests/test_zarr.py create mode 100644 python/spatialdata-experimental-writer/uv.lock diff --git a/python/spatialdata-experimental-writer/README.md b/python/spatialdata-experimental-writer/README.md new file mode 100644 index 00000000..8db60cf2 --- /dev/null +++ b/python/spatialdata-experimental-writer/README.md @@ -0,0 +1,16 @@ +# spatialdata-experimental-writer + +Experimental vector optimization writers for browser-oriented SpatialData +rendering. + +The initial writer targets Vitessce-compatible Morton-sorted Points Parquet: + +- `x`, `y`, optional `z` coordinates are preserved. +- `morton_code_2d` is added using 16 bits per axis. +- the first 2-4 rows are sentinel/extreme rows with `morton_code_2d == 0`; + readers can infer the full point bounding box from these rows. +- string/categorical columns are placed at the right side of the table. +- row-group size is controlled when writing Parquet. + +The package also includes a small multiscale Parquet writer hook that stores +Padua-style `spatialdata_multiscale` JSON metadata in the Parquet schema. diff --git a/python/spatialdata-experimental-writer/pyproject.toml b/python/spatialdata-experimental-writer/pyproject.toml new file mode 100644 index 00000000..f63e32e7 --- /dev/null +++ b/python/spatialdata-experimental-writer/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "spatialdata-experimental-writer" +version = "0.1.0" +description = "Experimental SpatialData vector optimization writers" +requires-python = ">=3.12" +readme = "README.md" +license = { text = "MIT" } +authors = [{ name = "SpatialData.js contributors" }] +dependencies = [ + "numpy>=2.0", + "pandas>=2.2", + "pyarrow>=18", +] + +[project.scripts] +spatialdata-experimental-writer = "spatialdata_experimental_writer.cli:main" + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.uv] +package = true + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[dependency-groups] +dev = [ + "pytest>=8.0", +] diff --git a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/__init__.py b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/__init__.py new file mode 100644 index 00000000..99eaa590 --- /dev/null +++ b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/__init__.py @@ -0,0 +1,17 @@ +from .points import ( + MORTON_CODE_2D_COLUMN, + MORTON_CODE_EXTREME_VALUE_INDICATOR, + build_spatialdata_multiscale_metadata, + morton_sort_points, + write_morton_points_parquet, + write_multiscale_points_parquet, +) + +__all__ = [ + "MORTON_CODE_2D_COLUMN", + "MORTON_CODE_EXTREME_VALUE_INDICATOR", + "build_spatialdata_multiscale_metadata", + "morton_sort_points", + "write_morton_points_parquet", + "write_multiscale_points_parquet", +] diff --git a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/cli.py b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/cli.py new file mode 100644 index 00000000..277d2a24 --- /dev/null +++ b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/cli.py @@ -0,0 +1,330 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import pandas as pd + +from .points import ( + build_spatialdata_multiscale_metadata, + write_morton_points_parquet, + write_multiscale_points_parquet, +) +from .zarr import ( + experimental_points_output_path, + list_points_keys, + points_parquet_path, + read_points_dataframe, + read_points_element_attrs, +) + +_EPILOG = """\ +examples: + # List Points elements in a SpatialData Zarr store + spatialdata-experimental-writer list-points ~/data/xenium.zarr + + # Morton-sort transcripts from a Zarr store into points.experimental/ + spatialdata-experimental-writer morton-points-from-zarr \\ + ~/data/xenium.zarr --points-key transcripts + + # Morton-sort a CSV or single Parquet file + spatialdata-experimental-writer morton-points input.csv output.parquet \\ + --feature-key feature_name + + # Write multiscale Parquet with embedded spatialdata_multiscale metadata + spatialdata-experimental-writer multiscale-points input.parquet output.parquet +""" + + +def _positive_int(value: str) -> int: + parsed = int(value) + if parsed <= 0: + raise argparse.ArgumentTypeError("value must be positive") + return parsed + + +def _read_dataframe(path: str) -> pd.DataFrame: + input_path = Path(path) + if input_path.is_dir(): + return read_points_dataframe(input_path) + suffix = input_path.suffix.lower() + if suffix == ".csv": + return pd.read_csv(input_path) + if suffix in {".parquet", ".pq"}: + return pd.read_parquet(input_path) + raise SystemExit( + f"Unsupported input: {path}\n" + "Expected a .csv file, .parquet file, or a directory of Parquet parts." + ) + + +def _morton_points(args: argparse.Namespace) -> None: + df = _read_dataframe(args.input) + sorted_df = write_morton_points_parquet( + df, + args.output, + feature_key=args.feature_key, + row_group_size=args.row_group_size, + compression=args.compression, + ) + print( + json.dumps( + { + "format": "morton-points", + "rows": int(len(sorted_df)), + "output": str(args.output), + "row_group_size": args.row_group_size, + }, + indent=2, + sort_keys=True, + ) + ) + + +def _multiscale_points(args: argparse.Namespace) -> None: + df = _read_dataframe(args.input) + if args.metadata_json: + metadata = json.loads(Path(args.metadata_json).read_text()) + else: + metadata = build_spatialdata_multiscale_metadata(df) + write_multiscale_points_parquet( + df, + args.output, + metadata=metadata, + row_group_size=args.row_group_size, + compression=args.compression, + ) + print( + json.dumps( + { + "format": "spatialdata_multiscale_points", + "rows": int(len(df)), + "output": str(args.output), + "row_group_size": args.row_group_size, + }, + indent=2, + sort_keys=True, + ) + ) + + +def _list_points(args: argparse.Namespace) -> None: + keys = list_points_keys(args.zarr) + if not keys: + raise SystemExit(f"No Points elements found under {Path(args.zarr) / 'points'}") + print(json.dumps({"zarr": str(args.zarr), "points_keys": keys}, indent=2, sort_keys=True)) + + +def _morton_points_from_zarr(args: argparse.Namespace) -> None: + zarr_path = Path(args.zarr) + keys = list_points_keys(zarr_path) + if not keys: + raise SystemExit(f"No Points elements found under {zarr_path / 'points'}") + + points_key = args.points_key + if points_key is None: + if len(keys) == 1: + points_key = keys[0] + else: + raise SystemExit( + "Multiple Points elements found; pass --points-key.\n" + f"Available keys: {', '.join(keys)}" + ) + if points_key not in keys: + raise SystemExit( + f"Unknown Points element {points_key!r}.\nAvailable keys: {', '.join(keys)}" + ) + + attrs = read_points_element_attrs(zarr_path, points_key) + feature_key = args.feature_key or attrs.get("feature_key") + source_parquet = points_parquet_path(zarr_path, points_key) + output = Path(args.output) if args.output else experimental_points_output_path( + zarr_path, points_key + ) + + df = read_points_dataframe(source_parquet) + sorted_df = write_morton_points_parquet( + df, + output, + feature_key=feature_key, + row_group_size=args.row_group_size, + compression=args.compression, + ) + print( + json.dumps( + { + "format": "morton-points", + "zarr": str(zarr_path), + "points_key": points_key, + "source": str(source_parquet), + "output": str(output), + "feature_key": feature_key, + "rows": int(len(sorted_df)), + "row_group_size": args.row_group_size, + }, + indent=2, + sort_keys=True, + ) + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Write browser-oriented SpatialData vector optimization artifacts " + "(Morton-sorted Points Parquet and multiscale metadata)." + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=_EPILOG, + ) + subparsers = parser.add_subparsers(dest="command", required=True, metavar="command") + + list_points = subparsers.add_parser( + "list-points", + help="list Points element keys in a SpatialData Zarr store", + description="List Points element keys under /points/.", + ) + list_points.add_argument( + "zarr", + metavar="ZARR", + help="Path to a SpatialData Zarr store (directory containing points/)", + ) + list_points.set_defaults(func=_list_points) + + morton_from_zarr = subparsers.add_parser( + "morton-points-from-zarr", + help="Morton-sort a Points element from a SpatialData Zarr store", + description=( + "Read points//points.parquet from a SpatialData Zarr store, " + "add morton_code_2d sentinel rows, and write Vitessce-compatible Parquet. " + "Defaults to points.experimental//points.parquet inside the store." + ), + ) + morton_from_zarr.add_argument( + "zarr", + metavar="ZARR", + help="Path to a SpatialData Zarr store", + ) + morton_from_zarr.add_argument( + "--points-key", + metavar="KEY", + help=( + "Points element name under points/ (for example transcripts). " + "Required when the store has more than one Points element." + ), + ) + morton_from_zarr.add_argument( + "--output", + metavar="PATH", + help=( + "Output Parquet path (default: /points.experimental//points.parquet)" + ), + ) + morton_from_zarr.add_argument( + "--feature-key", + metavar="COLUMN", + help=( + "Column used to derive _codes (default: spatialdata_attrs.feature_key " + "from the element zarr.json)" + ), + ) + morton_from_zarr.add_argument( + "--row-group-size", + type=_positive_int, + default=50_000, + metavar="N", + help="Target row-group size after sentinel rows (default: 50000)", + ) + morton_from_zarr.add_argument( + "--compression", + default="zstd", + help="Parquet compression codec (default: zstd)", + ) + morton_from_zarr.set_defaults(func=_morton_points_from_zarr) + + morton = subparsers.add_parser( + "morton-points", + help="Morton-sort points from CSV or Parquet", + description=( + "Sort x/y points by 2D Morton order, prepend sentinel bbox rows, " + "and write Vitessce-compatible Parquet." + ), + ) + morton.add_argument( + "input", + metavar="INPUT", + help="Input .csv, .parquet file, or directory of Parquet parts", + ) + morton.add_argument( + "output", + metavar="OUTPUT", + help="Output .parquet file", + ) + morton.add_argument( + "--feature-key", + metavar="COLUMN", + help="Column used to derive _codes for categorical features", + ) + morton.add_argument( + "--row-group-size", + type=_positive_int, + default=50_000, + metavar="N", + help="Target row-group size after sentinel rows (default: 50000)", + ) + morton.add_argument( + "--compression", + default="zstd", + help="Parquet compression codec (default: zstd)", + ) + morton.set_defaults(func=_morton_points) + + multiscale = subparsers.add_parser( + "multiscale-points", + help="write multiscale Points Parquet with spatialdata_multiscale metadata", + description=( + "Write Points Parquet with Padua-style spatialdata_multiscale JSON " + "stored in the file schema metadata." + ), + ) + multiscale.add_argument( + "input", + metavar="INPUT", + help="Input .csv, .parquet file, or directory of Parquet parts", + ) + multiscale.add_argument( + "output", + metavar="OUTPUT", + help="Output .parquet file", + ) + multiscale.add_argument( + "--metadata-json", + metavar="PATH", + help="Optional spatialdata_multiscale metadata JSON (default: inferred from input)", + ) + multiscale.add_argument( + "--row-group-size", + type=_positive_int, + default=50_000, + metavar="N", + help="Target row-group size (default: 50000)", + ) + multiscale.add_argument( + "--compression", + default="zstd", + help="Parquet compression codec (default: zstd)", + ) + multiscale.set_defaults(func=_multiscale_points) + + return parser + + +def main(argv: list[str] | None = None) -> None: + parser = build_parser() + args = parser.parse_args(argv) + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/points.py b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/points.py new file mode 100644 index 00000000..ed629c8b --- /dev/null +++ b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/points.py @@ -0,0 +1,217 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +MORTON_CODE_2D_COLUMN = "morton_code_2d" +MORTON_CODE_EXTREME_VALUE_INDICATOR = np.uint32(0) +MORTON_CODE_BITS_PER_AXIS = 16 +MORTON_CODE_VALUE_MAX = np.uint32((2**MORTON_CODE_BITS_PER_AXIS) - 1) + + +def _norm_series_to_uint(series: pd.Series, v_min: float, v_max: float) -> pd.Series: + if v_max == v_min: + return pd.Series(np.zeros(len(series), dtype=np.uint32), index=series.index) + normalized = (series.astype("float64") - v_min) / (v_max - v_min) + clipped = normalized.clip(0.0, 1.0).fillna(0.0) + return (clipped * int(MORTON_CODE_VALUE_MAX)).astype(np.uint32) + + +def _part1by1_16(values: np.ndarray) -> np.ndarray: + x = values.astype(np.uint32) & np.uint32(0x0000FFFF) + x = (x | np.left_shift(x, 8)) & np.uint32(0x00FF00FF) + x = (x | np.left_shift(x, 4)) & np.uint32(0x0F0F0F0F) + x = (x | np.left_shift(x, 2)) & np.uint32(0x33333333) + x = (x | np.left_shift(x, 1)) & np.uint32(0x55555555) + return x + + +def morton_code_2d(x_uint: pd.Series, y_uint: pd.Series) -> np.ndarray: + xs = _part1by1_16(x_uint.to_numpy(np.uint32)) + ys = _part1by1_16(y_uint.to_numpy(np.uint32)) + return (np.left_shift(ys.astype(np.uint64), 1) | xs.astype(np.uint64)).astype(np.uint32) + + +def _extreme_indices(df: pd.DataFrame) -> list[Any]: + extreme_values = [ + ("x", df["x"].min()), + ("x", df["x"].max()), + ("y", df["y"].min()), + ("y", df["y"].max()), + ] + result: list[Any] = [] + for column, value in extreme_values: + matches = df.index[df[column] == value] + if len(matches) == 0: + continue + index = matches[0] + if index not in result: + result.append(index) + return result + + +def _append_feature_codes(df: pd.DataFrame, feature_key: str | None) -> pd.DataFrame: + if not feature_key or feature_key not in df.columns: + return df + code_column = f"{feature_key}_codes" + if code_column in df.columns: + return df + out = df.copy() + values = out[feature_key] + if isinstance(values.dtype, pd.CategoricalDtype): + out[code_column] = values.cat.codes.astype("int32") + else: + categories = pd.Categorical(values) + out[code_column] = categories.codes.astype("int32") + return out + + +def _move_string_like_columns_right(df: pd.DataFrame) -> pd.DataFrame: + string_like: list[str] = [] + other: list[str] = [] + for column in df.columns: + dtype = df[column].dtype + if isinstance(dtype, pd.CategoricalDtype) or pd.api.types.is_string_dtype(dtype): + string_like.append(column) + else: + other.append(column) + return df[[*other, *string_like]] + + +def morton_sort_points(df: pd.DataFrame, *, feature_key: str | None = None) -> pd.DataFrame: + missing = [column for column in ("x", "y") if column not in df.columns] + if missing: + raise ValueError("Points dataframe is missing required columns: " + ", ".join(missing)) + + out = _append_feature_codes(df.copy(), feature_key) + x_min = float(out["x"].min()) + x_max = float(out["x"].max()) + y_min = float(out["y"].min()) + y_max = float(out["y"].max()) + out["x_uint"] = _norm_series_to_uint(out["x"], x_min, x_max) + out["y_uint"] = _norm_series_to_uint(out["y"], y_min, y_max) + out[MORTON_CODE_2D_COLUMN] = morton_code_2d(out["x_uint"], out["y_uint"]) + + sentinel_indices = _extreme_indices(out) + sentinel = out.loc[sentinel_indices].copy().reset_index(drop=True) + sentinel[MORTON_CODE_2D_COLUMN] = MORTON_CODE_EXTREME_VALUE_INDICATOR + + rest = out.drop(index=sentinel_indices) + sort_columns = [MORTON_CODE_2D_COLUMN] + if "z" in rest.columns and rest["z"].nunique(dropna=False) < 100: + sort_columns = ["z", MORTON_CODE_2D_COLUMN] + rest = rest.sort_values(sort_columns, kind="mergesort").reset_index(drop=True) + + combined = pd.concat([sentinel, rest], ignore_index=True) + return _move_string_like_columns_right(combined) + + +def _write_arrow_table_in_row_groups( + table: pa.Table, + output_path: Path, + *, + row_group_size: int, + metadata: dict[str, Any] | None = None, + compression: str = "zstd", +) -> None: + if row_group_size <= 0: + raise ValueError("row_group_size must be positive") + output_path.parent.mkdir(parents=True, exist_ok=True) + schema = table.schema + if metadata: + merged = dict(schema.metadata or {}) + merged[b"spatialdata_multiscale"] = json.dumps(metadata).encode() + schema = schema.with_metadata(merged) + + writer = pq.ParquetWriter(output_path, schema, compression=compression, write_statistics=True) + try: + sentinel_count = 0 + if MORTON_CODE_2D_COLUMN in table.column_names: + morton_column = table.column(MORTON_CODE_2D_COLUMN).combine_chunks() + for i in range(min(4, table.num_rows)): + if morton_column[i].as_py() != 0: + break + sentinel_count += 1 + if sentinel_count: + writer.write_table(table.slice(0, sentinel_count), row_group_size=sentinel_count) + for start in range(sentinel_count, table.num_rows, row_group_size): + chunk = table.slice(start, min(row_group_size, table.num_rows - start)) + writer.write_table(chunk, row_group_size=chunk.num_rows) + finally: + writer.close() + + +def write_morton_points_parquet( + df: pd.DataFrame, + output_path: str | Path, + *, + feature_key: str | None = None, + row_group_size: int = 50_000, + compression: str = "zstd", +) -> pd.DataFrame: + sorted_df = morton_sort_points(df, feature_key=feature_key) + table = pa.Table.from_pandas(sorted_df, preserve_index=False) + _write_arrow_table_in_row_groups( + table, + Path(output_path), + row_group_size=row_group_size, + compression=compression, + ) + return sorted_df + + +def build_spatialdata_multiscale_metadata( + df: pd.DataFrame, + *, + axes: tuple[str, ...] = ("x", "y", "z"), + coordinate_space: str = "raw", + version: str = "1.0", + levels: list[dict[str, Any]] | None = None, + limit: int | None = None, +) -> dict[str, Any]: + available_axes = [axis for axis in axes if axis in df.columns] + if not available_axes: + raise ValueError("No requested coordinate axes are present in the dataframe.") + return { + "version": version, + "format": "spatialdata_multiscale_points", + "axes": available_axes, + "bounding_box": { + "min": [float(df[axis].min()) for axis in available_axes], + "max": [float(df[axis].max()) for axis in available_axes], + }, + "coordinate_space": coordinate_space, + "limit": limit, + "levels": levels or [], + "n_points_total": int(len(df)), + } + + +def write_multiscale_points_parquet( + df: pd.DataFrame, + output_path: str | Path, + *, + metadata: dict[str, Any], + row_group_size: int = 50_000, + compression: str = "zstd", +) -> None: + table = pa.Table.from_pandas(df, preserve_index=False) + if {"__spatial_index__", "__morton__"}.issubset(df.columns): + sort_keys = [("__spatial_index__", "ascending"), ("__morton__", "ascending")] + if "gene" in df.columns: + sort_keys.insert(0, ("gene", "ascending")) + table = table.take(pc.sort_indices(table, sort_keys=sort_keys)) + _write_arrow_table_in_row_groups( + table, + Path(output_path), + row_group_size=row_group_size, + metadata=metadata, + compression=compression, + ) diff --git a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/zarr.py b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/zarr.py new file mode 100644 index 00000000..9f273023 --- /dev/null +++ b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/zarr.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pandas as pd +import pyarrow.dataset as ds + + +def _points_root(zarr_path: Path) -> Path: + return zarr_path / "points" + + +def list_points_keys(zarr_path: str | Path) -> list[str]: + root = _points_root(Path(zarr_path)) + if not root.is_dir(): + return [] + return sorted( + child.name + for child in root.iterdir() + if child.is_dir() and (child / "zarr.json").is_file() + ) + + +def _read_zarr_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text()) + + +def read_points_element_attrs(zarr_path: str | Path, points_key: str) -> dict[str, Any]: + element_json = _points_root(Path(zarr_path)) / points_key / "zarr.json" + if not element_json.is_file(): + raise FileNotFoundError(f"Points element not found: points/{points_key}") + attrs = _read_zarr_json(element_json).get("attributes", {}) + spatialdata_attrs = attrs.get("spatialdata_attrs", {}) + if not isinstance(spatialdata_attrs, dict): + spatialdata_attrs = {} + return { + "axes": attrs.get("axes", []), + "feature_key": spatialdata_attrs.get("feature_key"), + "instance_key": spatialdata_attrs.get("instance_key"), + "version": spatialdata_attrs.get("version"), + } + + +def points_parquet_path(zarr_path: str | Path, points_key: str) -> Path: + return _points_root(Path(zarr_path)) / points_key / "points.parquet" + + +def experimental_points_output_path(zarr_path: str | Path, points_key: str) -> Path: + return Path(zarr_path) / "points.experimental" / points_key / "points.parquet" + + +def read_points_dataframe(parquet_path: str | Path) -> pd.DataFrame: + path = Path(parquet_path) + if not path.exists(): + raise FileNotFoundError(f"Points Parquet not found: {path}") + if path.is_dir(): + table = ds.dataset(path, format="parquet").to_table() + else: + table = ds.dataset(path, format="parquet").to_table() + return table.to_pandas() diff --git a/python/spatialdata-experimental-writer/tests/test_points.py b/python/spatialdata-experimental-writer/tests/test_points.py new file mode 100644 index 00000000..de80608e --- /dev/null +++ b/python/spatialdata-experimental-writer/tests/test_points.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import json + +import pandas as pd +import pyarrow.parquet as pq + +from spatialdata_experimental_writer import ( + MORTON_CODE_2D_COLUMN, + build_spatialdata_multiscale_metadata, + morton_sort_points, + write_morton_points_parquet, + write_multiscale_points_parquet, +) + + +def test_morton_sort_points_adds_sentinel_rows_and_feature_codes() -> None: + df = pd.DataFrame( + { + "x": [0.0, 10.0, 5.0, 2.0], + "y": [3.0, 4.0, 20.0, 0.0], + "feature_name": ["b", "a", "b", "c"], + } + ) + + sorted_df = morton_sort_points(df, feature_key="feature_name") + + assert MORTON_CODE_2D_COLUMN in sorted_df.columns + assert "feature_name_codes" in sorted_df.columns + assert sorted_df[MORTON_CODE_2D_COLUMN].iloc[:4].eq(0).all() + assert sorted_df.columns[-1] == "feature_name" + + +def test_write_morton_points_parquet_uses_small_sentinel_row_group(tmp_path) -> None: + df = pd.DataFrame( + { + "x": [0.0, 10.0, 5.0, 2.0, 8.0], + "y": [3.0, 4.0, 20.0, 0.0, 9.0], + "feature_name": ["b", "a", "b", "c", "a"], + } + ) + output = tmp_path / "points.parquet" + + write_morton_points_parquet(df, output, feature_key="feature_name", row_group_size=2) + + parquet = pq.ParquetFile(output) + assert parquet.num_row_groups >= 2 + assert parquet.metadata.row_group(0).num_rows <= 4 + + +def test_write_multiscale_points_parquet_stores_metadata(tmp_path) -> None: + df = pd.DataFrame( + { + "x": [0.0, 10.0], + "y": [3.0, 4.0], + "__spatial_index__": [0, 1], + "__morton__": [0, 1], + } + ) + output = tmp_path / "points.parquet" + metadata = build_spatialdata_multiscale_metadata(df, axes=("x", "y")) + + write_multiscale_points_parquet(df, output, metadata=metadata, row_group_size=2) + + schema_metadata = pq.ParquetFile(output).schema_arrow.metadata + assert schema_metadata is not None + stored = json.loads(schema_metadata[b"spatialdata_multiscale"]) + assert stored["format"] == "spatialdata_multiscale_points" + assert stored["bounding_box"]["min"] == [0.0, 3.0] diff --git a/python/spatialdata-experimental-writer/tests/test_zarr.py b/python/spatialdata-experimental-writer/tests/test_zarr.py new file mode 100644 index 00000000..55584fec --- /dev/null +++ b/python/spatialdata-experimental-writer/tests/test_zarr.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from spatialdata_experimental_writer.zarr import ( + experimental_points_output_path, + list_points_keys, + points_parquet_path, + read_points_dataframe, + read_points_element_attrs, +) + + +def _write_points_element( + zarr_root: Path, + key: str, + *, + feature_key: str = "feature_name", +) -> None: + element_dir = zarr_root / "points" / key + parquet_dir = element_dir / "points.parquet" + parquet_dir.mkdir(parents=True) + table = pa.Table.from_pandas( + pd.DataFrame( + { + "x": [0.0, 1.0], + "y": [2.0, 3.0], + "feature_name": ["a", "b"], + } + ), + preserve_index=False, + ) + pq.write_table(table, parquet_dir / "part.0.parquet") + element_dir.joinpath("zarr.json").write_text( + json.dumps( + { + "attributes": { + "encoding-type": "ngff:points", + "axes": ["x", "y"], + "spatialdata_attrs": { + "feature_key": feature_key, + "version": "0.2", + }, + }, + "zarr_format": 3, + "node_type": "group", + } + ) + ) + + +def test_list_points_keys_and_read_element(tmp_path: Path) -> None: + _write_points_element(tmp_path, "transcripts") + assert list_points_keys(tmp_path) == ["transcripts"] + attrs = read_points_element_attrs(tmp_path, "transcripts") + assert attrs["feature_key"] == "feature_name" + df = read_points_dataframe(points_parquet_path(tmp_path, "transcripts")) + assert list(df.columns) == ["x", "y", "feature_name"] + assert experimental_points_output_path(tmp_path, "transcripts") == ( + tmp_path / "points.experimental" / "transcripts" / "points.parquet" + ) diff --git a/python/spatialdata-experimental-writer/uv.lock b/python/spatialdata-experimental-writer/uv.lock new file mode 100644 index 00000000..f0daa17a --- /dev/null +++ b/python/spatialdata-experimental-writer/uv.lock @@ -0,0 +1,283 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/ad/fed0499ce6a338d2a03ebae59cd15093910c8875328855781952abf6c2fe/numpy-2.4.6.tar.gz", hash = "sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda", size = 20735807, upload-time = "2026-05-18T23:37:14.07Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/2a/3d7b5ac8aac24feaf9ad7ed58f45b0bbc06d37e4338ae84c9f2298b570f9/numpy-2.4.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1", size = 16689119, upload-time = "2026-05-18T23:33:54.065Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/92c4c131527599e8288d6918e888d88726f84d805d784b771f32408aeaef/numpy-2.4.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb", size = 14699246, upload-time = "2026-05-18T23:33:57.621Z" }, + { url = "https://files.pythonhosted.org/packages/ad/fe/c0a6b7b2ca128a8fb228575147073b660656734b8ebe4d76c8fd748dcc79/numpy-2.4.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41", size = 5204410, upload-time = "2026-05-18T23:34:00.302Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d4/9770d14ba719432bb90a421bfd443872ed0f70f7264b64bec12ea363d5fd/numpy-2.4.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698", size = 6551240, upload-time = "2026-05-18T23:34:02.852Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c6/50a46a6205feba2343f1d6d17438107c5dc491ed1c736e6ea68689fd906b/numpy-2.4.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f", size = 15671012, upload-time = "2026-05-18T23:34:05.485Z" }, + { url = "https://files.pythonhosted.org/packages/99/60/14115e6364fa676c5397c2ad3004e527e9aa487abf5d0706ec81bbd08529/numpy-2.4.6-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853", size = 16645538, upload-time = "2026-05-18T23:34:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/ae/c5/693cbe59e57db94d2231fa519ca3978dc9e19da5a8f088588f5c6e947ff2/numpy-2.4.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a", size = 17020706, upload-time = "2026-05-18T23:34:13.053Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fc/85b7c4eff9b4966ade25c2273cf7e7012e92366c032058653934b37de044/numpy-2.4.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2", size = 18368541, upload-time = "2026-05-18T23:34:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/e1b27545deedce7f4a0b348618c6b62d74e36a4dc9ccd42f3eb2f85eee32/numpy-2.4.6-cp312-cp312-win32.whl", hash = "sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45", size = 5962825, upload-time = "2026-05-18T23:34:20.3Z" }, + { url = "https://files.pythonhosted.org/packages/ab/ca/feab00bd44aa5fe1ad2c18f08b4d3bb92e26484b0b1d1443897809ed528c/numpy-2.4.6-cp312-cp312-win_amd64.whl", hash = "sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751", size = 12321687, upload-time = "2026-05-18T23:34:23.095Z" }, + { url = "https://files.pythonhosted.org/packages/63/cf/5a6d34850a39d1093558564f77ee8e8e0bee5061151b8f05a55711001ec7/numpy-2.4.6-cp312-cp312-win_arm64.whl", hash = "sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8", size = 10221482, upload-time = "2026-05-18T23:34:25.876Z" }, + { url = "https://files.pythonhosted.org/packages/fb/82/bdab26d7438c6791ca31b7c024ca37c1eab8b726ba236129005cd4a06e45/numpy-2.4.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0", size = 16684648, upload-time = "2026-05-18T23:34:29.41Z" }, + { url = "https://files.pythonhosted.org/packages/1b/30/a80189bcc7f5e4258b3fbc3968d909d1756f54d023299ecc39ad6fdb9ef8/numpy-2.4.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb", size = 14693902, upload-time = "2026-05-18T23:34:33.013Z" }, + { url = "https://files.pythonhosted.org/packages/97/12/70b5d0d7c15e1ebb8a6a84a8caa1d19e181d84fb58bb6d70aca29099dec1/numpy-2.4.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f", size = 5198992, upload-time = "2026-05-18T23:34:36.132Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/ebd2a8f8a83541f8d38cc5667e8c2b69cecfd30da6e45693e8158857d44b/numpy-2.4.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3", size = 6546944, upload-time = "2026-05-18T23:34:38.484Z" }, + { url = "https://files.pythonhosted.org/packages/bb/c5/7b863a97a91671a0338f4253bd3b5a3d3852f0692dae91711c9f4a10e787/numpy-2.4.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b", size = 15669392, upload-time = "2026-05-18T23:34:41.257Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9d/3584b9984ca4c047aea75214ce1a4c4c73d849bd71b604264b7f5653f8a8/numpy-2.4.6-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089", size = 16633220, upload-time = "2026-05-18T23:34:45.075Z" }, + { url = "https://files.pythonhosted.org/packages/05/ae/7c67fba23bd98caec7c99261f3a16072ade14813486b0282cb29846de832/numpy-2.4.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a", size = 17020800, upload-time = "2026-05-18T23:34:49.065Z" }, + { url = "https://files.pythonhosted.org/packages/d9/5d/3b6725cb31d983c5e66916f5d36f6d7e5521129e4c4404d64f918292a5b6/numpy-2.4.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605", size = 18357600, upload-time = "2026-05-18T23:34:52.709Z" }, + { url = "https://files.pythonhosted.org/packages/f7/da/2ccc6c2fe8898dee01d90c75c5f5f914a23daf99e3e0f59516a08760c8b5/numpy-2.4.6-cp313-cp313-win32.whl", hash = "sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91", size = 5961134, upload-time = "2026-05-18T23:34:55.618Z" }, + { url = "https://files.pythonhosted.org/packages/b5/cd/9cc4dc876fb065d5c220aae4d5e14826b2715331bb7618ce1fb07a679d99/numpy-2.4.6-cp313-cp313-win_amd64.whl", hash = "sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359", size = 12318598, upload-time = "2026-05-18T23:34:58.928Z" }, + { url = "https://files.pythonhosted.org/packages/39/1e/c0bcba1f8694116485fe28fd1be698c278fcda4141c5b0e53a2aed8b12a8/numpy-2.4.6-cp313-cp313-win_arm64.whl", hash = "sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778", size = 10222272, upload-time = "2026-05-18T23:35:02.167Z" }, + { url = "https://files.pythonhosted.org/packages/63/6d/cc5619247c8f4204e507f5883528372e4ac4bb189e579fb859a12e480b1f/numpy-2.4.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1", size = 14821197, upload-time = "2026-05-18T23:35:05.468Z" }, + { url = "https://files.pythonhosted.org/packages/00/58/f1c39161c87d9e9bed660f1ed4bafc0e403d5ec9650b6dd77aead07d489b/numpy-2.4.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe", size = 5326287, upload-time = "2026-05-18T23:35:08.693Z" }, + { url = "https://files.pythonhosted.org/packages/af/57/3917ab0fd97f271a8694513581b8a36c655f111c446852c302f04ccdb6fc/numpy-2.4.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997", size = 6646763, upload-time = "2026-05-18T23:35:11.459Z" }, + { url = "https://files.pythonhosted.org/packages/eb/0f/037e64c494b67581ae18193d770adef354c41f3f2c8ebf865602d949bf8f/numpy-2.4.6-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20", size = 15728070, upload-time = "2026-05-18T23:35:14.79Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/5d2bae9c9542eb4df16dc9c46dc79c186e9bad53805dfa5399a6023c6db0/numpy-2.4.6-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d", size = 16681752, upload-time = "2026-05-18T23:35:18.836Z" }, + { url = "https://files.pythonhosted.org/packages/92/14/23d1dfb410ae362cd59ce53e936b1513d545eb40db3949ced632e19a459e/numpy-2.4.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67", size = 17086024, upload-time = "2026-05-18T23:35:22.52Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/23595a2c642cdf3bc567877064bdd7f91c8b0038a4453cf2daf7248eafe9/numpy-2.4.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd", size = 18403398, upload-time = "2026-05-18T23:35:26.398Z" }, + { url = "https://files.pythonhosted.org/packages/8a/90/0ac3bc947217e66dec77e7cbc6a1979d1af70b6461b82f620d3bccd5e4c8/numpy-2.4.6-cp313-cp313t-win32.whl", hash = "sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab", size = 6084971, upload-time = "2026-05-18T23:35:29.387Z" }, + { url = "https://files.pythonhosted.org/packages/77/71/5673e351671a1d2bd6063b91b44f70c0affea7d1516fa7a6572941ba4aa1/numpy-2.4.6-cp313-cp313t-win_amd64.whl", hash = "sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75", size = 12458532, upload-time = "2026-05-18T23:35:32.175Z" }, + { url = "https://files.pythonhosted.org/packages/3f/88/19d3503c5046e688f049274b27a3ef3d771152fa80d3ba3d01a3dff61abe/numpy-2.4.6-cp313-cp313t-win_arm64.whl", hash = "sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd", size = 10291881, upload-time = "2026-05-18T23:35:35.465Z" }, + { url = "https://files.pythonhosted.org/packages/f8/91/3ab2044d05fd16d343c5ac2e69b127f1b2854040dd20b193257c78028bd3/numpy-2.4.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079", size = 16683458, upload-time = "2026-05-18T23:35:38.353Z" }, + { url = "https://files.pythonhosted.org/packages/8e/62/764ce66fa4147ae6d73071a3abf804ffe606f174618697c571acdf26a7c9/numpy-2.4.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7", size = 14704559, upload-time = "2026-05-18T23:35:42.14Z" }, + { url = "https://files.pythonhosted.org/packages/60/61/23f27c172f022e04025b7dc2367f4d63c1a398120607ec896228649a6f48/numpy-2.4.6-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5", size = 5209716, upload-time = "2026-05-18T23:35:45.377Z" }, + { url = "https://files.pythonhosted.org/packages/03/71/21cf70dc6ea3e3acb95fc53a265b2fc248b981f0194ceb5b475271b8809d/numpy-2.4.6-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096", size = 6543947, upload-time = "2026-05-18T23:35:47.926Z" }, + { url = "https://files.pythonhosted.org/packages/d5/91/64288395ee1799bd2e0b04a305dce9666da90c961e1f3fe982a05ee1c036/numpy-2.4.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b", size = 15685197, upload-time = "2026-05-18T23:35:50.863Z" }, + { url = "https://files.pythonhosted.org/packages/f3/eb/ebffaa97dc55502df69584a8f0dcf07f69a3e0b3e2323670a2722db9aa39/numpy-2.4.6-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8", size = 16638245, upload-time = "2026-05-18T23:35:54.752Z" }, + { url = "https://files.pythonhosted.org/packages/b8/0b/54f9da33128d7e350fab89c7455902eeae70349ee52bddb448dc4a576f45/numpy-2.4.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402", size = 17036587, upload-time = "2026-05-18T23:35:58.355Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f0/fdebc1052db1cc37c64beb22072d67cd6d1c71adca1299f53dec2b5e20d3/numpy-2.4.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb", size = 18363226, upload-time = "2026-05-18T23:36:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b4/298628d98c72b57e57f7165ae6a481a1deaf6f3c28262a6e4c739c275930/numpy-2.4.6-cp314-cp314-win32.whl", hash = "sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1", size = 6010196, upload-time = "2026-05-18T23:36:05.92Z" }, + { url = "https://files.pythonhosted.org/packages/df/ac/46de6dda46478f7942f839e094970be2d4a861e005c4b3bf07c92e291a09/numpy-2.4.6-cp314-cp314-win_amd64.whl", hash = "sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261", size = 12450334, upload-time = "2026-05-18T23:36:09.107Z" }, + { url = "https://files.pythonhosted.org/packages/78/92/b8b798ac784102c0da830d2257d59358e3d3d90d1e2b3f2575dad976c5cf/numpy-2.4.6-cp314-cp314-win_arm64.whl", hash = "sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6", size = 10495678, upload-time = "2026-05-18T23:36:12.766Z" }, + { url = "https://files.pythonhosted.org/packages/30/34/ec28d1aa8115971537c01469ab2011ee96827930f0a124de1000cc2a7ed7/numpy-2.4.6-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a", size = 14823672, upload-time = "2026-05-18T23:36:16.473Z" }, + { url = "https://files.pythonhosted.org/packages/16/bd/f6d1fede4e54e8042a7ff97bb495510f3c220f94bcd9e8b228e87c92cc0d/numpy-2.4.6-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e", size = 5328731, upload-time = "2026-05-18T23:36:19.767Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f0/e105b9e2fd728a9910103884decd6951d9dd73896b914a98d9a231de02ee/numpy-2.4.6-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e", size = 6649805, upload-time = "2026-05-18T23:36:22.266Z" }, + { url = "https://files.pythonhosted.org/packages/82/dd/1206a7ca6ab15e3f02069707ca96222e202af681bb73756da7527f3cb837/numpy-2.4.6-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43", size = 15730496, upload-time = "2026-05-18T23:36:25.713Z" }, + { url = "https://files.pythonhosted.org/packages/51/e7/38d3ea825dcab85a591734decb2f6c67caa7c8367d374df1a1c3842f9b07/numpy-2.4.6-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e", size = 16679616, upload-time = "2026-05-18T23:36:29.652Z" }, + { url = "https://files.pythonhosted.org/packages/93/b7/caabfdf53edf663e0b4eb74d7d405d83baef09eb5e83bcd32d601d72b93e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895", size = 17085145, upload-time = "2026-05-18T23:36:33.449Z" }, + { url = "https://files.pythonhosted.org/packages/f9/45/68d7c33a6bcf3e5aa3bdbd57a367e6f615286dfd6482f97e8ffeb734306e/numpy-2.4.6-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4", size = 18403813, upload-time = "2026-05-18T23:36:37.369Z" }, + { url = "https://files.pythonhosted.org/packages/9c/50/0753655aa844c99cd9e018aacf76f130f1bd81d881bb74bc0aef5d73a8ba/numpy-2.4.6-cp314-cp314t-win32.whl", hash = "sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063", size = 6156982, upload-time = "2026-05-18T23:36:40.817Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d4/7c67becf668f973cb490cec3e98dfd799d866f9c989a54d355672cfa0db6/numpy-2.4.6-cp314-cp314t-win_amd64.whl", hash = "sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627", size = 12638908, upload-time = "2026-05-18T23:36:43.996Z" }, + { url = "https://files.pythonhosted.org/packages/43/bb/e1c71a4295b1b1d1393d50dbb4f2a36283c6859d9d3892e84f00ec5a91d5/numpy-2.4.6-cp314-cp314t-win_arm64.whl", hash = "sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66", size = 10565867, upload-time = "2026-05-18T23:36:47.114Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pandas" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/f1/392f8c5bfc16f66a0d2d41561c01627c228fe7ed2a0d056ef11315042570/pandas-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fed2ff7fd9779120e388e285fc029bd5cf9490cdd2e4166a9ee22c0e49a9ab09", size = 10357846, upload-time = "2026-05-11T18:52:36.143Z" }, + { url = "https://files.pythonhosted.org/packages/cf/3d/b16412745651e855f357e5e66930248688378853a6e2698a214e331fba1f/pandas-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b168fc218fd80a6cbdbdbc1a97ddc7889ed057d7eb45f50d866ceab5f39904c4", size = 9899550, upload-time = "2026-05-11T18:52:38.976Z" }, + { url = "https://files.pythonhosted.org/packages/31/a8/fa2535168fffcedf67f4f6de28d2dd903a747ca7c8ea6989451aaeb3a92f/pandas-3.0.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0383c72c75cdcca61a9e116e611143902dbfd08bff356829c2f6d1cf40a9ca8c", size = 10412965, upload-time = "2026-05-11T18:52:41.915Z" }, + { url = "https://files.pythonhosted.org/packages/65/b6/09b01cdbc15224e2850365192d17b7bdebb8bdbd8780ed221fcdf0d9a515/pandas-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6dc0b3fd2169c9157deed50b4d519553a3655c8c6a96027136d654592be973a9", size = 10894600, upload-time = "2026-05-11T18:52:45.02Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a4/2eb28f2fccb4ced4a2c79ab2a5dee9ade1ebf44922ebad6fea158c9f95d4/pandas-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7e65d5407dc0b394f509699650e4a2ec01c0514f21850f453fa60f3be79a5dbf", size = 11422824, upload-time = "2026-05-11T18:52:48.058Z" }, + { url = "https://files.pythonhosted.org/packages/f8/45/830bb57f533a4604b355e07edcb8ea18cf88b5f94e5fca92f27052d7c597/pandas-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f8894dc474d648fe7b6ff0ca9b0bd73950d19952bc1a6534540762c5d79d305c", size = 11950889, upload-time = "2026-05-11T18:52:50.905Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c5/fc1b368f303087d20e8c9bf3d6ceb186263cfac0ade735cd938538bea839/pandas-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:c7be265b62cef88e253a941e4698604973736dcfe242fdb5198f0f7bc473cdcc", size = 9755463, upload-time = "2026-05-11T18:52:53.386Z" }, + { url = "https://files.pythonhosted.org/packages/86/bd/fda8f9705b1b09c6ebe14bfc0fa0e4ec8584d54ea673628f157ff55131af/pandas-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:557409bc4178e70ee8d9ddb494798e51ebf6ea59330f6be22c51bab2a7db6c49", size = 9066158, upload-time = "2026-05-11T18:52:56.038Z" }, + { url = "https://files.pythonhosted.org/packages/c5/90/62d8302883c44308c477e222c3daf7c813a34c8e96985882fbd53d964352/pandas-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:67b3b64c11910cfa29f4e94a14d3bff9ee693b6fc76055e7cad549cee0aec5fa", size = 10331071, upload-time = "2026-05-11T18:52:58.838Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ae/6a6493c783a101f165e4356953ba3c74d6f77f0042fa7d753da9dfbb640c/pandas-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:39436b377d56d2a2e52d0395bdbee171f01068e99af5250509aceeb929f765c7", size = 9875690, upload-time = "2026-05-11T18:53:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/62/7c/5df8e9f56c69a2769fbe9382a5ef8f2658c007e376434e1e2cbb57ad895f/pandas-3.0.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4be06d68f9ddcfc645b87534911da79a8fbffc7573c80e0edcf42a5020624d8", size = 10381634, upload-time = "2026-05-11T18:53:04.393Z" }, + { url = "https://files.pythonhosted.org/packages/99/68/1237369725aa617bb358263d535803e3053fdbc593513ec5ed9c9896b5b6/pandas-3.0.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a4eeb6830daf35a71cc09649bd823e2b542dac246cdee9614c6e4bd65028cd6a", size = 10891243, upload-time = "2026-05-11T18:53:07.643Z" }, + { url = "https://files.pythonhosted.org/packages/25/93/77d108e8af7222b4a503ebde0e30215b1c2e4f8e53a526431890f22d5586/pandas-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1928e07221f82db493cd4af1e23c1bfca524a19a4699887975bff68f49a72bfb", size = 11388659, upload-time = "2026-05-11T18:53:10.634Z" }, + { url = "https://files.pythonhosted.org/packages/d0/bd/eff5b4399f332ac386c853f6cd2bd3fa2ca0061b9f36ecd9c4d7c4265649/pandas-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51b1fe551acb77dac643c6fda86084d8d446c10fe64b06a9cc29c4cc8540e7f2", size = 11942880, upload-time = "2026-05-11T18:53:13.536Z" }, + { url = "https://files.pythonhosted.org/packages/2c/20/559ace4200982c3887d0b86bfd0d856a2143ef8ddab63cc07934951a964c/pandas-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:a82d532a3351d435432cd913edbccaf8b8e01d4dd0e5ced5a8d2e8ecd94c7e44", size = 9757091, upload-time = "2026-05-11T18:53:16.306Z" }, + { url = "https://files.pythonhosted.org/packages/3a/66/69055a09fe200f29f922a3eeec4804611900b95f52d932ece3393c3c0c19/pandas-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:275c14e0fce14a2ec20eee474aecd305478ea3c1e6f6a9d8fe219a165542717e", size = 9057282, upload-time = "2026-05-11T18:53:18.768Z" }, + { url = "https://files.pythonhosted.org/packages/57/0e/efe801b0e6811e8e650cd21b7f2608e30f08a7067e2bf6e8752b0d56ee3c/pandas-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:46997386d528eb40376ecd6b033cf4a8a1e5282580f68f43de875b78cba2199d", size = 10767016, upload-time = "2026-05-11T18:53:21.227Z" }, + { url = "https://files.pythonhosted.org/packages/ea/dc/eb55135a1d5f0f0519f28da1f609a206d2cad1f9c35c32d51e38dd7261ae/pandas-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:261e308dfb22448384b7580cf719d2f998fe2966c92893c3e77d14008af1f066", size = 10420210, upload-time = "2026-05-11T18:53:23.982Z" }, + { url = "https://files.pythonhosted.org/packages/c6/3e/b1d5d955ce33ffecb407465a60bc32769d74fcf68224b7ae67ae11d4dea4/pandas-3.0.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dd1a5d1def6a46002e964510bdc67c368aa0951df5d1d9f8365336f5a1f490cd", size = 10336126, upload-time = "2026-05-11T18:53:26.731Z" }, + { url = "https://files.pythonhosted.org/packages/f5/76/a01261711ab60a22d71b862f0de20e4c504bf80457270ad8cb42110f6abc/pandas-3.0.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d72828c20c6d6e83e1e22a6a3b47b326b71664112fa9705dcbccfd7a39b62085", size = 10728051, upload-time = "2026-05-11T18:53:29.125Z" }, + { url = "https://files.pythonhosted.org/packages/e9/21/ea191195e587b18cf682e97f433f81b2d0fbe341380e80a3e0d6e4403c8e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d26cbe1fcfc12e8fd900e2454163e466b2d3af84f7c75481df7683ffc073d870", size = 11350796, upload-time = "2026-05-11T18:53:32.056Z" }, + { url = "https://files.pythonhosted.org/packages/64/69/f0eaaf54939f0e8c6768fd06be9af2cef9b36048b96dfb9e1b2c685a807e/pandas-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:3e91cec1879ada0624fc3dc9953c5cbd60208e59c0db28f540c5d6d47502422f", size = 11799741, upload-time = "2026-05-11T18:53:34.985Z" }, + { url = "https://files.pythonhosted.org/packages/45/a4/865e0e510cae5fc2194de4db28be638952de942571ba9125934fd9c01d47/pandas-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:08d789b41f87e0905880e293cedf6197ce71fe67cc081358b1e148a491b9bd13", size = 10499958, upload-time = "2026-05-11T18:53:37.857Z" }, + { url = "https://files.pythonhosted.org/packages/86/54/effdcc3c0ff7a08037889200e148ebe94c16c4f653be078c7b3675955df1/pandas-3.0.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3650109c0f22879df8bd6179ab9ee3d7f1d1d4e7e0094a3f0032d9f51e2e64ac", size = 10336065, upload-time = "2026-05-11T18:53:41.099Z" }, + { url = "https://files.pythonhosted.org/packages/68/10/bf2d6738d72748b961a3751ab89522d58c54efc36a8e1a12161216cd45cf/pandas-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:bab900348131a7db1f69a7309ef141fd5680f1487094193bcbbb61791573bf8f", size = 9926101, upload-time = "2026-05-11T18:53:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e9/e35cf11c8a136e757b956f5f0efdcaa50aecde85ea055f1898dfc68262f3/pandas-3.0.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba7e08b9ac1d54569cd1e256e3668975ed624d6826f7b68df0342b012007bddb", size = 10457553, upload-time = "2026-05-11T18:53:46.394Z" }, + { url = "https://files.pythonhosted.org/packages/58/3b/1cdec6772bdbaf7b25dab360c59f03cadf05492dd724c6540af905389b07/pandas-3.0.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d71c63ae4ebdbf70209742096f1fc46a83a0613c99d4b23766cced9ff8cd62a", size = 10914065, upload-time = "2026-05-11T18:53:49.134Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/1ef644445fcd72e3627bceec77e3560636f87ddce4ed841afe76b83b5bf9/pandas-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e3a2ec42c98ffa2565a67e08e218d06d72576d758d90facb7c00805194d8f360", size = 11459188, upload-time = "2026-05-11T18:53:52.527Z" }, + { url = "https://files.pythonhosted.org/packages/7e/49/4d8d4f42cbc9c4adc7a1870f269c02cbd6cd40d059622c06fb298addcbad/pandas-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:335f62418ed562cfc3c49e9e196375c28b729dcef8543abf4f9438e381bf3c76", size = 11982966, upload-time = "2026-05-11T18:53:55.043Z" }, + { url = "https://files.pythonhosted.org/packages/38/55/792619469bab9882d8bbd5865d45a72f6478762d04a9af4bf0d08c503e95/pandas-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:3c20a521bbb85902f79f7270c80a59e1b5452d96d170c034f207181870f97ac5", size = 9876755, upload-time = "2026-05-11T18:53:58.067Z" }, + { url = "https://files.pythonhosted.org/packages/2a/af/33c469653b0ba03b50c3a98192d4c07f0c75c66b263ceb097fce0ee97d31/pandas-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:a2d2dff8a04f3917b55ab3910c32990f8ddf7eceba114947838cefa976a68977", size = 9198658, upload-time = "2026-05-11T18:54:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fa/b8c257bd76b8bd060c3a9151c1fca05e9b9c5e3af5d0f549c0356f6d143d/pandas-3.0.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:0d589105b3c14645af1738ff279b2995102d8f7a03b0a66dc8d95550eb513e04", size = 10787242, upload-time = "2026-05-11T18:54:03.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/eb/f19206ffb0bf1919002969aa448b4702c6594845156a6f8050674855aac3/pandas-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:13fc1e853d9e04743d11ba75a985ccbc2a317fe07d8af61e445a6fd24dacd6a6", size = 10436369, upload-time = "2026-05-11T18:54:06.311Z" }, + { url = "https://files.pythonhosted.org/packages/fd/24/c7c39fb4fe22b71a0c2d78bf0c585c600092d85f94f086d2b3b2f6ca27e2/pandas-3.0.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:819959dab7bbd0049c15623fbac4e29a191b9528160a61fb1032242d8ced2d9c", size = 10358306, upload-time = "2026-05-11T18:54:09.085Z" }, + { url = "https://files.pythonhosted.org/packages/16/ec/dd2a9eb7fa1204df88c0864164e35b228ac581062ac612ba0a67fd812e4c/pandas-3.0.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:60ae316d3fd75d1858d450d0db0103ea2be3e7d4a95ec2f064f7e2ae63f7b028", size = 10758394, upload-time = "2026-05-11T18:54:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/95/6e/00c61ea8e85b4f6d8d35e11852a1a4998fc7fafc91c6a602d1cc9c972d64/pandas-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd3a518890b400d32f9023722dc9a9a5c969f00b415419a3c06c043f09bb5d7d", size = 11375717, upload-time = "2026-05-11T18:54:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/31/89/8fc1c268969fac43688d65fd92e67df24bd128d53cb4d2eee534cd307399/pandas-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9c39be2d709d01fa972a0cabc522389fceca4f3969332ba25a7d6c5802cf976a", size = 11828897, upload-time = "2026-05-11T18:54:17.146Z" }, + { url = "https://files.pythonhosted.org/packages/56/3b/e7d20dea247a3e6dc0bd8a6953854afbedc03951def4e7371e05e7263e25/pandas-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4db8c527972a821cf5286b40ccc57642a39bc62e62022b42f99f8a67fca8c3a1", size = 10900855, upload-time = "2026-05-11T18:54:19.72Z" }, + { url = "https://files.pythonhosted.org/packages/0f/54/68a0978d1ef8502b8492099beaa6e7a0c1b32e3b5d4f677f5810cb08711c/pandas-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b2c95f8bfc1ee412bf482605d7bfd30c12d1d26bd59fdd91efeef1d4718decb1", size = 9466464, upload-time = "2026-05-11T18:54:22.754Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pyarrow" +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, + { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, + { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, + { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, + { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, + { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, + { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "spatialdata-experimental-writer" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "numpy" }, + { name = "pandas" }, + { name = "pyarrow" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "numpy", specifier = ">=2.0" }, + { name = "pandas", specifier = ">=2.2" }, + { name = "pyarrow", specifier = ">=18" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.0" }] + +[[package]] +name = "tzdata" +version = "2026.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, +] From d99f9fb06d6240793353ae6b5716b33498053cfb Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Fri, 19 Jun 2026 20:36:55 +0100 Subject: [PATCH 04/38] adr notes --- .../0002-spatially-aware-vector-loading.md | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/docs/adr/0002-spatially-aware-vector-loading.md b/docs/adr/0002-spatially-aware-vector-loading.md index e8a3bd26..4c9f5258 100644 --- a/docs/adr/0002-spatially-aware-vector-loading.md +++ b/docs/adr/0002-spatially-aware-vector-loading.md @@ -27,6 +27,92 @@ Parquet/GeoParquet rather than inventing a deck.gl-specific storage format. - GeoParquet is the durable shape optimization target. GeoArrow is a runtime columnar layout / deck adapter option, not a duplicate persisted artifact. +## Experimental Optimization Collections + +Use `points.experimental//` and `shapes.experimental//` only for +persisted layouts that **standard SpatialData / Vitessce readers cannot correctly +consume** — not for every browser optimization. + +| Layout | Where it lives | Why | +|--------|----------------|-----| +| **Morton v1** (`morton_code_2d`, sentinels, `{feature_key}_codes`, row groups) | **Canonical** `points//points.parquet` | Follows Vitessce practice. Extra columns are additive; Python `spatialdata` full-table reads still work. | +| **Feature-primary sort** (Morton not primary key) | `points.experimental//` | Breaks Morton row-group bisect; needs a new tiling `kind` | +| **Padua multiscale** (`__spatial_index__`, levels in schema metadata) | `points.experimental//` | Non-standard vs morton-points v1 | +| **GeoParquet shapes tiling** | `shapes.experimental//` | Future | + +`experimentalOptimizations` in `@spatialdata/vis` means use TileLayer / row-group +reads when **canonical** parquet schema supports morton tiling — not “look in +`points.experimental/`”. + +The experimental writer defaults to **in-place** Morton sorting on +`points//points.parquet`. Use `--experimental` only when writing a layout +that must not replace the canonical element. + +## Multi-part Parquet (reader) + +Wild-type SpatialData points may store `points//points.parquet` as a +**directory** with `part.0.parquet`, `part.1.parquet`, … The logical path remains +`points//points.parquet`. `@spatialdata/core` supports both single-file and +multipart layouts for metadata, schema, and row-group range reads. The +experimental writer outputs a **single-file** Morton artifact by design; row-group +range reads fetch only the byte ranges needed per viewport. + +## Feature / gene filtering + +Transcript and other feature-bearing points declare `feature_key` in element +`spatialdata_attrs` (for example `"feature_name"` on xenium transcripts). This is +distinct from `instance_key` (for example `"cell_id"`), which identifies the +object a point belongs to. + +The Morton writer adds `{feature_key}_codes` (for example `feature_name_codes`) +as `int32` categorical codes alongside the string feature column. Sorting is +**spatial** (Morton on x/y) by default; row groups are spatial chunks. + +**Core API** — extend bounded loading with optional feature codes: + +```typescript +interface PointsInBoundsOptions { + bounds: SpatialBounds; + /** Integer codes matching `{feature_key}_codes` in the parquet artifact */ + featureCodes?: readonly number[]; + signal?: AbortSignal; +} +``` + +**Vis API** — extend `PointsLayerConfig` with `featureCodes?: number[]` and +wire through TileLayer `updateTriggers.getTileData`. + +v1 applies feature filtering as a **read-time row predicate** after spatial +bounds filtering (and after row-group fetch on the Morton path). It does not +skip row groups by gene. String-based `features?: string[]` and a codebook +artifact are deferred. + +Feature filtering is separate from **feature-primary sort** experiments +(`[feature_codes, morton_code_2d]`), which may require a new tiling `kind` if +promoted. Use `write-index-permutations` on a derivative Zarr store to benchmark +sort strategies; see the writer README. + +A hypothetical **per-gene density map** (2D histogram / KDE for one feature) is +out of scope for the Morton tile path and may be an offline aggregation or +dedicated viz mode later. + +## Sort strategy experiments + +Default Morton v1 sort is spatial on `morton_code_2d` (optionally `z` when +low-cardinality). Multi-key sorts under evaluation include +`[morton_code_2d, feature_name_codes]` and `[feature_name_codes, morton_code_2d]`. +The reader's row-group bisect assumes Morton is the **primary** sort key; do not +silently swap sort order under the existing `morton-points` format id. + +Generate comparable permutations with: + +```bash +spatialdata-experimental-writer write-index-permutations SOURCE_ZARR DEST_ZARR +``` + +The derivative store includes sibling `points//` elements and +`index-manifest.json` for benchmark tooling. + ## Prior Art - scverse Padua hackathon points work: From 42c3ecee5b60a164e7007ee9136453e91bccc4e0 Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Sat, 20 Jun 2026 08:31:19 +0100 Subject: [PATCH 05/38] WIP improvements to points tiling functionality and visualization integration - Added `featureCodes` option to `PointsInBoundsOptions` for filtering points based on feature codes. - Introduced utility functions `featureCodeAllowSet` and `rowMatchesFeatureCode` for feature code management. - Updated `filterPointsToBounds` to incorporate feature code filtering logic. - Enhanced `VPointsSource` to support loading points with optional feature codes and improved row group reading. - Implemented tests for new functionality in `mortonPointsTiling.spec.ts` and `pointsTiling.spec.ts`. - Added `PointsStylePanel` and loading message handling in the visualization layer for better user feedback during point loading. --- packages/core/src/models/VPointsSource.ts | 101 ++++++++- packages/core/src/models/VTableSource.ts | 112 ++++++++-- packages/core/src/pointsTiling.ts | 62 +++++- .../core/tests/mortonPointsTiling.spec.ts | 205 ++++++++++++++++++ packages/core/tests/pointsTiling.spec.ts | 30 +++ .../src/SpatialCanvas/PointsStylePanel.tsx | 87 ++++++++ .../src/SpatialCanvas/SpatialCanvasViewer.tsx | 15 +- packages/vis/src/SpatialCanvas/index.tsx | 42 +++- .../src/SpatialCanvas/pointsTileProgress.ts | 72 ++++++ .../SpatialCanvas/renderers/pointsRenderer.ts | 177 ++++++++++++--- packages/vis/src/SpatialCanvas/types.ts | 9 + .../vis/src/SpatialCanvas/useLayerData.ts | 114 +++++++++- packages/vis/tests/pointsRenderer.spec.ts | 30 +++ packages/vis/tests/pointsTileProgress.spec.ts | 58 +++++ 14 files changed, 1033 insertions(+), 81 deletions(-) create mode 100644 packages/core/tests/mortonPointsTiling.spec.ts create mode 100644 packages/vis/src/SpatialCanvas/PointsStylePanel.tsx create mode 100644 packages/vis/src/SpatialCanvas/pointsTileProgress.ts create mode 100644 packages/vis/tests/pointsRenderer.spec.ts create mode 100644 packages/vis/tests/pointsTileProgress.spec.ts diff --git a/packages/core/src/models/VPointsSource.ts b/packages/core/src/models/VPointsSource.ts index cad783e9..f451b6ff 100644 --- a/packages/core/src/models/VPointsSource.ts +++ b/packages/core/src/models/VPointsSource.ts @@ -5,8 +5,11 @@ import { type PointsInBoundsResult, type PointsTilingMetadata, extractSentinelBoundingBox, + featureCodeAllowSet, filterPointsToBounds, + isMortonSentinelValue, mortonIntervalsForBounds, + rowMatchesFeatureCode, } from '../pointsTiling.js'; import type { Axis } from '../schemas'; // import { normalizeAxes } from '@vitessce/spatial-utils'; @@ -241,14 +244,17 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource { const canLoadRowGroups = await this.canLoadParquetRowGroups(); const firstRowGroup = datasetMetadata && canLoadRowGroups - ? await this.loadParquetRowGroupByGroupIndex(parquetPath, 0) + ? await this.loadParquetRowGroupByGroupIndex(parquetPath, 0, { + columns: ['x', 'y', MORTON_CODE_2D_COLUMN], + limit: 4, + }) : null; const bounds = firstRowGroup ? (extractSentinelBoundingBox(firstRowGroup) ?? undefined) : undefined; const rowGroupSizes = datasetMetadata?.rowGroupRows ?? []; - return { + const metadata: PointsTilingMetadata = { kind: 'morton-points', parquetPath, axisNames, @@ -262,6 +268,8 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource { supportsRowGroupRangeReads: Boolean(datasetMetadata && canLoadRowGroups && bounds), bounds, }; + + return metadata; } async loadPointsInBounds( @@ -277,8 +285,66 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource { } } checkAbort(options.signal); - const full = await this.loadPoints(elementPath); - return filterPointsToBounds(full, options.bounds); + const full = await this.loadPointsWithOptionalFeatureCodes(elementPath, metadata, options); + return filterPointsToBounds( + full.data, + options.bounds, + undefined, + options.featureCodes, + full.featureCodes + ); + } + + private async loadPointsWithOptionalFeatureCodes( + elementPath: string, + metadata: PointsTilingMetadata | null, + options: PointsInBoundsOptions + ) { + const parquetPath = getParquetPath(elementPath); + const zattrs = await this.loadSpatialDataElementAttrs(elementPath); + const { axes, spatialdata_attrs: spatialDataAttrs } = zattrs; + const normAxes = normalizeAxes(axes); + const axisNames = normAxes.map((axis: { name: string }) => axis.name); + const { feature_key: featureKey } = spatialDataAttrs; + let featureCodeColumnName = metadata?.featureCodeColumnName; + if (!featureCodeColumnName && options.featureCodes?.length) { + const datasetMetadata = await this.loadParquetDatasetMetadata(parquetPath); + const schemaTable = datasetMetadata ? null : await this.loadParquetSchemaTable(parquetPath); + const fields = datasetMetadata?.schema?.fields + ? datasetMetadata.schema.fields.flatMap((field) => + typeof field.name === 'string' ? [field.name] : [] + ) + : arrowSchemaFieldNames(schemaTable); + featureCodeColumnName = selectFeatureCodeColumn(fields, featureKey); + } + const resolvedFeatureCodeColumn = + typeof featureCodeColumnName === 'string' ? featureCodeColumnName : undefined; + const columnNames = [...axisNames]; + const needsFeatureCodes = Boolean( + options.featureCodes?.length && resolvedFeatureCodeColumn + ); + if (needsFeatureCodes && resolvedFeatureCodeColumn) { + columnNames.push(resolvedFeatureCodeColumn); + } + const arrowTable = await this.loadParquetTable(parquetPath, columnNames); + const axisColumnArrs = axisNames.map((name: string) => { + const column = arrowTable.getChild(name); + if (!column) { + throw new Error(`Column "${name}" not found in the arrow table.`); + } + return column.toArray(); + }); + let featureCodes: ArrayLike | undefined; + if (needsFeatureCodes && resolvedFeatureCodeColumn) { + featureCodes = arrowTable.getChild(resolvedFeatureCodeColumn)?.toArray(); + } + return { + data: { + shape: [axisColumnArrs.length, arrowTable.numRows], + data: axisColumnArrs, + }, + featureCodes, + }; } private async bisectRowGroupsRight( @@ -314,6 +380,7 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource { return null; } checkAbort(options.signal); + const allowedFeatureCodes = featureCodeAllowSet(options.featureCodes); const intervals = mortonIntervalsForBounds(metadata.bounds, options.bounds); const rowGroupSet = new Set(); for (const [start, end] of intervals) { @@ -352,18 +419,40 @@ export default class SpatialDataPointsSource extends SpatialDataTableSource { const ys: number[] = []; const zs: number[] = []; const hasZ = metadata.axisNames.includes('z'); + const featureCodeColumnName = + allowedFeatureCodes && metadata.featureCodeColumnName + ? metadata.featureCodeColumnName + : undefined; + const rowGroupColumns = [ + 'x', + 'y', + ...(hasZ ? ['z'] : []), + metadata.mortonCodeColumnName, + ...(featureCodeColumnName ? [featureCodeColumnName] : []), + ]; for (const rowGroup of rowGroups) { checkAbort(options.signal); - const table = await this.loadParquetRowGroupByGroupIndex(metadata.parquetPath, rowGroup); + const table = await this.loadParquetRowGroupByGroupIndex(metadata.parquetPath, rowGroup, { + columns: rowGroupColumns, + }); const xColumn = table?.getChild('x'); const yColumn = table?.getChild('y'); const zColumn = hasZ ? table?.getChild('z') : undefined; const mortonColumn = table?.getChild(metadata.mortonCodeColumnName); + const featureCodeColumn = featureCodeColumnName + ? table?.getChild(featureCodeColumnName) + : undefined; if (!table || !xColumn || !yColumn) { continue; } for (let i = 0; i < table.numRows; i++) { - if (rowGroup === 0 && i < 4 && mortonColumn?.get(i) === 0) { + if (rowGroup === 0 && i < 4 && isMortonSentinelValue(mortonColumn?.get(i))) { + continue; + } + if ( + featureCodeColumn && + !rowMatchesFeatureCode(featureCodeColumn.get(i), allowedFeatureCodes) + ) { continue; } const x = xColumn.get(i); diff --git a/packages/core/src/models/VTableSource.ts b/packages/core/src/models/VTableSource.ts index b62d42cb..5612f241 100644 --- a/packages/core/src/models/VTableSource.ts +++ b/packages/core/src/models/VTableSource.ts @@ -25,17 +25,34 @@ interface ParquetWasmMetadata { rowGroup(index: number): ParquetWasmRowGroupMetadata; } +export interface ParquetRowGroupReadOptions { + columns?: string[]; + limit?: number; + offset?: number; +} + interface ParquetModule { - readParquet: (bytes: Uint8Array, options?: { columns?: string[] }) => ParquetWasmTableLike; + readParquet: (bytes: Uint8Array, options?: ParquetRowGroupReadOptions) => ParquetWasmTableLike; readSchema: (bytes: Uint8Array) => ParquetWasmTableLike; readMetadata?: (bytes: Uint8Array) => ParquetWasmMetadata; readParquetRowGroup?: ( schemaBytes: Uint8Array, rowGroupBytes: Uint8Array, - rowGroupIndex: number + rowGroupIndex: number, + options?: ParquetRowGroupReadOptions ) => ParquetWasmTableLike; } +function parquetColumnValueToNumber(value: unknown): number | null { + if (typeof value === 'number' && Number.isFinite(value)) { + return value; + } + if (typeof value === 'bigint') { + return Number(value); + } + return null; +} + export interface ParquetPartMetadata { path: string; schema: ArrowTable['schema']; @@ -95,6 +112,21 @@ async function initializeParquetModule(module: unknown) { } } +function parquetModuleSupportsRowGroupReads(module: ParquetModule): boolean { + return ( + typeof module.readMetadata === 'function' && typeof module.readParquetRowGroup === 'function' + ); +} + +async function loadParquetModuleFromCdn(): Promise { + const cdnModule = await import( + // @ts-expect-error - CDN import not recognized by TypeScript + 'https://cdn.vitessce.io/parquet-wasm@2c23652/esm/parquet_wasm.js' + ); + await initializeParquetModule(cdnModule); + return normalizeParquetModule(cdnModule); +} + async function getParquetModule() { // Dynamic import for code-splitting. parquet-wasm is a WebAssembly module // that needs to be initialized before use in browser environments. @@ -104,11 +136,21 @@ async function getParquetModule() { // - probably ultimately may be using geoarrow-wasm / investigate deck.gl arrow layer // think about how that fits our 'core' (no deck deps) vs 'vis' structure etc. + const useCdnForMissingRowGroupApis = typeof window !== 'undefined'; + // Try local import first (works in Node.js, tests, and production builds) try { const module = await import('parquet-wasm'); await initializeParquetModule(module); - return normalizeParquetModule(module); + const normalized = normalizeParquetModule(module); + if (!parquetModuleSupportsRowGroupReads(normalized) && useCdnForMissingRowGroupApis) { + console.warn( + '[VTableSource] Local parquet-wasm lacks row-group APIs; falling back to CDN build.' + ); + const cdnNormalized = await loadParquetModuleFromCdn(); + return cdnNormalized; + } + return normalized; } catch (error) { // Local import failed, try CDN fallback (needed in vite dev server) // Reference: https://observablehq.com/@kylebarron/geoparquet-on-the-web @@ -119,12 +161,8 @@ async function getParquetModule() { ); try { - const cdnModule = await import( - // @ts-expect-error - CDN import not recognized by TypeScript - 'https://cdn.vitessce.io/parquet-wasm@2c23652/esm/parquet_wasm.js' - ); - await initializeParquetModule(cdnModule); - return normalizeParquetModule(cdnModule); + const cdnNormalized = await loadParquetModuleFromCdn(); + return cdnNormalized; } catch (cdnError) { // Both imports failed, throw an error const localErrorMsg = error instanceof Error ? error.message : String(error); @@ -273,6 +311,8 @@ export default class SpatialDataTableSource extends AnnDataSource { * `loadPolygonShapes` all target the same file). */ parquetTableCache: Record>; + /** Morton min/max per row group — avoids re-decoding row groups during bisect. */ + rowGroupColumnExtentCache: Map; obsIndices: Record>; varIndices: Record>; varAliases: Record; @@ -293,6 +333,7 @@ export default class SpatialDataTableSource extends AnnDataSource { // TODO: change to column-specific storage. this.parquetTableBytes = {}; this.parquetTableCache = {}; + this.rowGroupColumnExtentCache = new Map(); // Table-specific properties this.obsIndices = {}; @@ -544,7 +585,8 @@ export default class SpatialDataTableSource extends AnnDataSource { async loadParquetRowGroupByGroupIndex( parquetPath: string, - rowGroupIndex: number + rowGroupIndex: number, + readOptions?: ParquetRowGroupReadOptions ): Promise { const { readParquetRowGroup } = await SpatialDataTableSource.parquetModulePromise; const { store } = this.storeRoot; @@ -573,7 +615,12 @@ export default class SpatialDataTableSource extends AnnDataSource { return null; } return tableFromIPC( - readParquetRowGroup(part.schemaBytes, rowGroupBytes, relativeRowGroupIndex).intoIPCStream() + readParquetRowGroup( + part.schemaBytes, + rowGroupBytes, + relativeRowGroupIndex, + readOptions + ).intoIPCStream() ); } return null; @@ -584,17 +631,44 @@ export default class SpatialDataTableSource extends AnnDataSource { columnName: string, rowGroupIndex: number ): Promise<{ min: number | null; max: number | null } | null> { - const table = await this.loadParquetRowGroupByGroupIndex(parquetPath, rowGroupIndex); - const column = table?.getChild(columnName); - if (!column || column.length === 0) { + const cacheKey = `${parquetPath}::${rowGroupIndex}::${columnName}`; + const cached = this.rowGroupColumnExtentCache.get(cacheKey); + if (cached) { + return cached; + } + const dataset = await this.loadParquetDatasetMetadata(parquetPath); + const rowCount = dataset?.rowGroupRows?.[rowGroupIndex]; + if (!rowCount) { return null; } - const min = column.get(0); - const max = column.get(column.length - 1); - return { - min: typeof min === 'number' ? min : null, - max: typeof max === 'number' ? max : null, + const columnOptions: ParquetRowGroupReadOptions = { columns: [columnName] }; + const minTable = await this.loadParquetRowGroupByGroupIndex( + parquetPath, + rowGroupIndex, + { ...columnOptions, limit: 1 } + ); + const minColumn = minTable?.getChild(columnName); + if (!minColumn || minColumn.length === 0) { + return null; + } + let maxValue: number | null = parquetColumnValueToNumber(minColumn.get(0)); + if (rowCount > 1) { + const maxTable = await this.loadParquetRowGroupByGroupIndex(parquetPath, rowGroupIndex, { + ...columnOptions, + offset: rowCount - 1, + limit: 1, + }); + const maxColumn = maxTable?.getChild(columnName); + if (maxColumn && maxColumn.length > 0) { + maxValue = parquetColumnValueToNumber(maxColumn.get(0)); + } + } + const extent = { + min: parquetColumnValueToNumber(minColumn.get(0)), + max: maxValue, }; + this.rowGroupColumnExtentCache.set(cacheKey, extent); + return extent; } /** diff --git a/packages/core/src/pointsTiling.ts b/packages/core/src/pointsTiling.ts index 6131f2da..781ad46c 100644 --- a/packages/core/src/pointsTiling.ts +++ b/packages/core/src/pointsTiling.ts @@ -10,6 +10,8 @@ export type SpatialBounds = AxisAlignedBounds; export interface PointsInBoundsOptions { bounds: SpatialBounds; + /** Integer codes matching `{feature_key}_codes` in the Morton Parquet artifact. */ + featureCodes?: readonly number[]; zoom?: number; signal?: AbortSignal; columns?: string[]; @@ -165,10 +167,17 @@ export function mortonIntervalsForBounds( } function getNumericValue(value: unknown): number | null { - if (typeof value !== 'number' || !Number.isFinite(value)) { - return null; + if (typeof value === 'number' && Number.isFinite(value)) { + return value; + } + if (typeof value === 'bigint') { + return Number(value); } - return value; + return null; +} + +export function isMortonSentinelValue(value: unknown): boolean { + return getNumericValue(value) === MORTON_CODE_EXTREME_VALUE_INDICATOR; } export function extractSentinelBoundingBox( @@ -188,7 +197,7 @@ export function extractSentinelBoundingBox( const xs: number[] = []; const ys: number[] = []; for (let i = 0; i < maxRows; i++) { - if (mortonColumn.get(i) !== MORTON_CODE_EXTREME_VALUE_INDICATOR) { + if (!isMortonSentinelValue(mortonColumn.get(i))) { break; } const x = getNumericValue(xColumn.get(i)); @@ -210,11 +219,33 @@ export function extractSentinelBoundingBox( }; } +export function featureCodeAllowSet( + featureCodes: readonly number[] | undefined +): Set | null { + if (!featureCodes?.length) { + return null; + } + return new Set(featureCodes); +} + +export function rowMatchesFeatureCode( + code: unknown, + allowed: Set | null +): boolean { + if (!allowed) { + return true; + } + return typeof code === 'number' && Number.isFinite(code) && allowed.has(code); +} + export function filterPointsToBounds( data: PointsColumnarData, bounds: SpatialBounds, - featureIndices?: ArrayLike + featureIndices?: ArrayLike, + featureCodes?: readonly number[], + sourceFeatureCodes?: ArrayLike ): PointsInBoundsResult { + const allowedFeatureCodes = featureCodeAllowSet(featureCodes); const xs = data.data[0]; const ys = data.data[1]; const zs = data.data[2]; @@ -224,15 +255,22 @@ export function filterPointsToBounds( const x = xs[i]; const y = ys[i]; if ( - Number.isFinite(x) && - Number.isFinite(y) && - x >= bounds.minX && - x <= bounds.maxX && - y >= bounds.minY && - y <= bounds.maxY + !Number.isFinite(x) || + !Number.isFinite(y) || + x < bounds.minX || + x > bounds.maxX || + y < bounds.minY || + y > bounds.maxY + ) { + continue; + } + if ( + allowedFeatureCodes && + !rowMatchesFeatureCode(sourceFeatureCodes?.[i], allowedFeatureCodes) ) { - keep.push(i); + continue; } + keep.push(i); } const outX = new Float32Array(keep.length); diff --git a/packages/core/tests/mortonPointsTiling.spec.ts b/packages/core/tests/mortonPointsTiling.spec.ts new file mode 100644 index 00000000..2727ea5f --- /dev/null +++ b/packages/core/tests/mortonPointsTiling.spec.ts @@ -0,0 +1,205 @@ +import { execSync } from 'node:child_process'; +import { mkdtemp, readFile, writeFile, mkdir } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import SpatialDataPointsSource from '../src/models/VPointsSource.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const projectRoot = join(__dirname, '../../..'); +const writerRoot = join(projectRoot, 'python/spatialdata-experimental-writer'); + +async function writeSyntheticPointsZarr(root: string) { + const elementDir = join(root, 'points', 'transcripts'); + await mkdir(elementDir, { recursive: true }); + await writeFile( + join(root, 'zarr.json'), + JSON.stringify({ zarr_format: 3, node_type: 'group' }) + ); + await writeFile( + join(elementDir, 'zarr.json'), + JSON.stringify({ + attributes: { + 'encoding-type': 'ngff:points', + axes: ['x', 'y'], + spatialdata_attrs: { + feature_key: 'feature_name', + version: '0.2', + }, + }, + zarr_format: 3, + node_type: 'group', + }) + ); + + execSync( + `uv run python - <<'PY' +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +from pathlib import Path + +root = Path(${JSON.stringify(elementDir)}) +rows = 500 +df = pd.DataFrame( + { + "x": [float(i % 100) for i in range(rows)], + "y": [float((i * 3) % 100) for i in range(rows)], + "feature_name": (["gene_a", "gene_b", "gene_c"] * rows)[:rows], + } +) +pq.write_table(pa.Table.from_pandas(df, preserve_index=False), root / "points.parquet") +PY`, + { cwd: writerRoot, stdio: 'pipe' } + ); + + execSync( + `uv run spatialdata-experimental-writer morton-points-from-zarr ${JSON.stringify(root)} --points-key transcripts --row-group-size 100`, + { cwd: writerRoot, stdio: 'pipe' } + ); +} + +function createStore(files: Record) { + let getRangeCalls = 0; + let getCalls = 0; + const store = { + getRangeCalls: () => getRangeCalls, + getCalls: () => getCalls, + resetCalls: () => { + getRangeCalls = 0; + getCalls = 0; + }, + store: { + async get(path: string) { + getCalls += 1; + return files[path.slice(1)] ?? null; + }, + async getRange( + path: string, + range: { offset?: number; length?: number; suffixLength?: number } + ) { + getRangeCalls += 1; + const bytes = files[path.slice(1)]; + if (!bytes) { + return null; + } + if (range.suffixLength !== undefined) { + const start = Math.max(0, bytes.length - range.suffixLength); + return bytes.slice(start); + } + const offset = range.offset ?? 0; + const length = range.length ?? bytes.length - offset; + return bytes.slice(offset, offset + length); + }, + }, + }; + return store; +} + +describe('Morton points tiling (canonical parquet)', () => { + let fixtureRoot: string; + let source: SpatialDataPointsSource; + let mockStore: ReturnType; + + beforeAll(async () => { + fixtureRoot = await mkdtemp(join(tmpdir(), 'morton-points-')); + await writeSyntheticPointsZarr(fixtureRoot); + + const parquetPath = join(fixtureRoot, 'points/transcripts/points.parquet'); + const elementJsonPath = join(fixtureRoot, 'points/transcripts/zarr.json'); + mockStore = createStore({ + 'points/transcripts/points.parquet': new Uint8Array(await readFile(parquetPath)), + 'points/transcripts/zarr.json': new Uint8Array(await readFile(elementJsonPath)), + }); + + source = new SpatialDataPointsSource({ + store: mockStore.store, + fileType: '.zarr', + }); + }, 120_000); + + afterAll(async () => { + execSync(`rm -rf ${JSON.stringify(fixtureRoot)}`, { stdio: 'pipe' }); + }); + + it('detects morton tiling metadata on canonical points.parquet', async () => { + mockStore.resetCalls(); + const metadata = await source.getPointsTilingMetadata('points/transcripts'); + expect(metadata).toMatchObject({ + kind: 'morton-points', + featureCodeColumnName: 'feature_name_codes', + }); + expect(mockStore.getRangeCalls()).toBeGreaterThan(0); + }); + + it('loads a bounded viewport without returning the full table', async () => { + const full = await source.loadPoints('points/transcripts'); + const xs = full.data[0]; + const ys = full.data[1]; + const minX = Math.min(...xs); + const maxX = Math.max(...xs); + const minY = Math.min(...ys); + const maxY = Math.max(...ys); + const bounds = { + minX: minX + 5, + maxX: minX + 15, + minY: minY + 5, + maxY: minY + 15, + }; + + mockStore.resetCalls(); + const loadTable = vi.spyOn(source, 'loadParquetTable'); + const result = await source.loadPointsInBounds('points/transcripts', { bounds }); + expect(result.shape[1]).toBeGreaterThan(0); + expect(result.shape[1]).toBeLessThan(full.shape[1]); + expect(['row-groups', 'full-filter']).toContain(result.loadMode); + if (result.loadMode === 'full-filter') { + expect(loadTable).toHaveBeenCalled(); + } + loadTable.mockRestore(); + }); + + it('filters loaded points by feature codes', async () => { + const full = await source.loadPoints('points/transcripts'); + const xs = full.data[0]; + const ys = full.data[1]; + const bounds = { + minX: Math.min(...xs), + maxX: Math.max(...xs), + minY: Math.min(...ys), + maxY: Math.max(...ys), + }; + + const unfiltered = await source.loadPointsInBounds('points/transcripts', { bounds }); + const filtered = await source.loadPointsInBounds('points/transcripts', { + bounds, + featureCodes: [0], + }); + expect(filtered.shape[1]).toBeGreaterThan(0); + expect(filtered.shape[1]).toBeLessThan(unfiltered.shape[1] ?? Number.MAX_SAFE_INTEGER); + }); + + it('uses row-group reads when parquet-wasm exposes row-group APIs', async () => { + const canRowGroups = await source.canLoadParquetRowGroups(); + if (!canRowGroups) { + return; + } + + const metadata = await source.getPointsTilingMetadata('points/transcripts'); + expect(metadata?.supportsRowGroupRangeReads).toBe(true); + expect(metadata?.bounds).toBeDefined(); + + mockStore.resetCalls(); + const bounds = { + minX: metadata!.bounds!.minX + 10, + maxX: metadata!.bounds!.minX + 30, + minY: metadata!.bounds!.minY + 10, + maxY: metadata!.bounds!.minY + 30, + }; + const result = await source.loadPointsInBounds('points/transcripts', { bounds }); + expect(result.loadMode).toBe('row-groups'); + expect(mockStore.getRangeCalls()).toBeGreaterThan(0); + expect(mockStore.getCalls()).toBe(0); + }); +}); diff --git a/packages/core/tests/pointsTiling.spec.ts b/packages/core/tests/pointsTiling.spec.ts index 765b150c..b5e3e1d8 100644 --- a/packages/core/tests/pointsTiling.spec.ts +++ b/packages/core/tests/pointsTiling.spec.ts @@ -42,6 +42,21 @@ describe('points tiling helpers', () => { }); }); + it('accepts bigint morton sentinel values', () => { + const arrowTable = table({ + x: [10, 20, 15, 17], + y: [5, 8, 40, 12], + morton_code_2d: [0n, 0n, 0n, 0n], + }); + + expect(extractSentinelBoundingBox(arrowTable)).toEqual({ + minX: 10, + minY: 5, + maxX: 20, + maxY: 40, + }); + }); + it('rejects missing or incomplete sentinel bounds', () => { expect( extractSentinelBoundingBox( @@ -91,4 +106,19 @@ describe('points tiling helpers', () => { expect(Array.from(filtered.data[1])).toEqual([5]); expect(filtered.shape).toEqual([2, 1]); }); + + it('filters columnar points by feature codes after spatial bounds', () => { + const xs = new Float32Array([5, 5, 5]); + const ys = new Float32Array([5, 5, 5]); + const featureCodes = new Int32Array([0, 1, 2]); + const filtered = filterPointsToBounds( + { data: [xs, ys], shape: [2, 3] }, + { minX: 0, minY: 0, maxX: 10, maxY: 10 }, + undefined, + [1], + featureCodes + ); + expect(Array.from(filtered.data[0])).toEqual([5]); + expect(filtered.shape).toEqual([2, 1]); + }); }); diff --git a/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx b/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx new file mode 100644 index 00000000..ae5179ae --- /dev/null +++ b/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx @@ -0,0 +1,87 @@ +import type { CSSProperties } from 'react'; +import { + DEFAULT_POINT_RADIUS_MAX_PIXELS, + DEFAULT_POINT_RADIUS_MIN_PIXELS, + DEFAULT_POINT_SIZE, +} from './renderers/pointsRenderer'; +import type { PointsLayerConfig } from './types'; + +const rangeLabelStyle: CSSProperties = { + color: '#ccc', + fontSize: '12px', + display: 'flex', + flexDirection: 'column', + gap: 4, +}; + +const tileProgressStyle: CSSProperties = { + color: '#aaa', + fontSize: '11px', +}; + +export interface PointsStylePanelProps { + layerId: string; + config: PointsLayerConfig; + tileLoadingMessage?: string | null; + updateLayer: (id: string, updates: Partial) => void; +} + +export function PointsStylePanel({ + layerId, + config, + tileLoadingMessage, + updateLayer, +}: PointsStylePanelProps) { + return ( + <> + + + + {tileLoadingMessage ? ( +
{tileLoadingMessage}
+ ) : null} + + ); +} diff --git a/packages/vis/src/SpatialCanvas/SpatialCanvasViewer.tsx b/packages/vis/src/SpatialCanvas/SpatialCanvasViewer.tsx index 3e2b29a7..11bf5f0f 100644 --- a/packages/vis/src/SpatialCanvas/SpatialCanvasViewer.tsx +++ b/packages/vis/src/SpatialCanvas/SpatialCanvasViewer.tsx @@ -156,6 +156,8 @@ interface UseSpatialCanvasRendererFromLayerInputsOptions { layerInputs: RenderStackLayerInputs; renderOrder?: string[]; viewState?: ViewState | null; + /** Orthographic zoom for points layer radius scaling (without subscribing to pan target). */ + viewZoom?: number | null; onViewStateChange?: (viewState: ViewState) => void; width: number; height: number; @@ -172,6 +174,7 @@ export function useSpatialCanvasRendererFromLayerInputs({ layerInputs, renderOrder, viewState, + viewZoom: viewZoomProp, onViewStateChange, width, height, @@ -196,7 +199,8 @@ export function useSpatialCanvasRendererFromLayerInputs({ availableElements, coordinateSystem, spatialData ?? undefined, - experimentalOptimizations + experimentalOptimizations, + viewZoomProp ?? viewState?.zoom ?? null ); const generatedDeckLayers = layerData.getLayers(); @@ -395,6 +399,7 @@ function SpatialCanvasViewerInner({ autoFit, experimentalOptimizations, }); + const pointsTileLoadingMessage = renderer.getPointsTileLoadingMessage(); const hoverPickLayerIds = useMemo( () => Array.from(renderer.enabledLayerIds), [renderer.enabledLayerIds] @@ -565,7 +570,13 @@ function SpatialCanvasViewerInner({ {showLoadingOverlay && renderer.isBlocking && (
Loading layer data...
)} - {showLoadingOverlay && renderer.isLoading && !renderer.isBlocking && ( + {showLoadingOverlay && !renderer.isBlocking && pointsTileLoadingMessage && ( +
{pointsTileLoadingMessage}
+ )} + {showLoadingOverlay && + renderer.isLoading && + !renderer.isBlocking && + !pointsTileLoadingMessage && (
Refreshing layer metadata...
diff --git a/packages/vis/src/SpatialCanvas/index.tsx b/packages/vis/src/SpatialCanvas/index.tsx index 51bafb15..dfb6ff69 100644 --- a/packages/vis/src/SpatialCanvas/index.tsx +++ b/packages/vis/src/SpatialCanvas/index.tsx @@ -24,6 +24,7 @@ import { createPortal } from 'react-dom'; import { ImageChannelPanel } from './ImageChannelPanel'; import { LabelsChannelPanel } from './LabelsChannelPanel'; import { LayerOrderList } from './LayerOrderList'; +import { PointsStylePanel } from './PointsStylePanel'; import { ShapeFillColorPanel } from './ShapeFillColorPanel'; import { shouldAutoFitSpatialView, @@ -40,6 +41,7 @@ import { VivLoaderRegistryProvider } from './VivLoaderRegistry'; import { SpatialCanvasProvider, useSpatialCanvasActions, useSpatialCanvasStore } from './context'; import { getDeckFromDeckGlRef, resolveHoverFeatureTooltip } from './featureTooltipHover'; import { layerConfig } from './layerConfig'; +import { pointsTileLoadingMessage as formatPointsTileLoadingMessage } from './pointsTileProgress'; import type { SpatialCanvasStoreApi } from './stores'; import type { AvailableElement, ElementsByType, ViewState } from './types'; import type { ImageLayerConfig } from './useLayerData'; @@ -213,6 +215,7 @@ interface ViewerSectionProps { hasEnabledLayers: boolean; isBlocking: boolean; isLoading: boolean; + pointsTileLoadingMessage: string | null; hasLayersDrawn: boolean; getWorldBoundsForVisibleLayers: () => import('@spatialdata/core').AxisAlignedBounds | null; vw: number; @@ -229,6 +232,7 @@ function ViewerSection({ hasEnabledLayers, isBlocking, isLoading, + pointsTileLoadingMessage, hasLayersDrawn, getWorldBoundsForVisibleLayers, vw, @@ -318,7 +322,23 @@ function ViewerSection({ Loading layer data... )} - {isLoading && !isBlocking && ( + {!isBlocking && pointsTileLoadingMessage && ( +
+ {pointsTileLoadingMessage} +
+ )} + {isLoading && !isBlocking && !pointsTileLoadingMessage && (
s.selectedLayerId); + const viewZoom = useSpatialCanvasStore((s) => s.viewState?.zoom ?? null); const actions = useSpatialCanvasActions(); @@ -412,6 +433,8 @@ function SpatialCanvasInner({ getImageLayerLoadedData, getLabelsLayerLoadedData, getLayerLoadState, + getPointsTileLoadProgress, + getPointsTileLoadingMessage, getWorldBoundsForLayer, getWorldBoundsForVisibleLayers, hasEnabledLayers, @@ -424,12 +447,14 @@ function SpatialCanvasInner({ spatialData, coordinateSystem, layerInputs: { layers, layerOrder }, - // viewState and onViewStateChange are omitted: auto-fit and pan handling - // are managed entirely by ViewerSection so this hook never re-runs on pan. + // viewState target is not subscribed here; zoom alone drives point-size scaling. + viewZoom, width: vw, height: vh, experimentalOptimizations, }); + const pointsTileLoadingMessage = getPointsTileLoadingMessage(); + const hoverPickLayerIds = useMemo(() => Array.from(enabledLayerIds), [enabledLayerIds]); useEffect(() => { @@ -681,6 +706,7 @@ function SpatialCanvasInner({ hasEnabledLayers={hasEnabledLayers} isBlocking={isBlocking} isLoading={isLoading} + pointsTileLoadingMessage={getPointsTileLoadingMessage()} hasLayersDrawn={hasLayersDrawn} getWorldBoundsForVisibleLayers={getWorldBoundsForVisibleLayers} vw={vw} @@ -806,6 +832,16 @@ function SpatialCanvasInner({ updateLayer={actions.updateLayer} /> )} + {selectedConfig.type === 'points' && ( + + )} {selectedConfig.type === 'shapes' && ( void; + onTileLoadStart?: () => void; + onTileLoadEnd?: (success: boolean) => void; +} + +export function emptyPointsTileLoadProgress(): PointsTileLoadProgress { + return { inFlight: 0, loaded: 0, viewportTotal: 0 }; +} + +export function aggregatePointsTileLoadProgress( + progressByLayer: ReadonlyMap +): PointsTileLoadProgress { + let inFlight = 0; + let loaded = 0; + let viewportTotal = 0; + for (const progress of progressByLayer.values()) { + inFlight += progress.inFlight; + loaded += progress.loaded; + viewportTotal += progress.viewportTotal; + } + return { inFlight, loaded, viewportTotal }; +} + +export function pointsTileLoadingMessage(progress: PointsTileLoadProgress): string | null { + const { inFlight, loaded, viewportTotal } = progress; + const awaitingViewport = + viewportTotal > 0 && loaded < viewportTotal && inFlight === 0; + if (inFlight <= 0 && !awaitingViewport) { + return null; + } + if (viewportTotal > 0) { + return `Loading points… (${loaded}/${viewportTotal} tiles)`; + } + return inFlight > 0 ? 'Loading points…' : null; +} + +export function isPointsTileLoading(progress: PointsTileLoadProgress): boolean { + return pointsTileLoadingMessage(progress) !== null; +} + +export function createPointsTileLoadCallbacks( + getProgress: () => PointsTileLoadProgress, + setProgress: (progress: PointsTileLoadProgress) => void +): PointsTileLoadCallbacks { + return { + onViewportTilesRequested: (count) => { + setProgress({ inFlight: 0, loaded: 0, viewportTotal: count }); + }, + onTileLoadStart: () => { + const current = getProgress(); + setProgress({ ...current, inFlight: current.inFlight + 1 }); + }, + onTileLoadEnd: (success) => { + const current = getProgress(); + setProgress({ + ...current, + inFlight: Math.max(0, current.inFlight - 1), + loaded: success ? current.loaded + 1 : current.loaded, + }); + }, + }; +} diff --git a/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts b/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts index 2a077c97..0e517662 100644 --- a/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts +++ b/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts @@ -6,8 +6,10 @@ import type { Matrix4 } from '@math.gl/core'; import type { PointsElement, PointsTilingMetadata, SpatialBounds } from '@spatialdata/core'; +import { COORDINATE_SYSTEM } from '@deck.gl/core'; import { ScatterplotLayer, TileLayer } from 'deck.gl'; import type { Layer } from 'deck.gl'; +import type { PointsTileLoadCallbacks } from '../pointsTileProgress'; export interface PointDataX { position: [number, number] | [number, number, number]; @@ -23,6 +25,27 @@ export interface PointData { data: ArrayLike[]; } +/** Orthographic zoom at which configured pointSize applies at full scale. */ +export const POINT_SIZE_ZOOM_REFERENCE = 0; +/** Minimum radius multiplier when zoomed out (reduces fragment overdraw). */ +export const MIN_POINT_SIZE_SCALE = 0.15; +export const DEFAULT_POINT_SIZE = 1; +export const DEFAULT_POINT_RADIUS_MIN_PIXELS = 1; +export const DEFAULT_POINT_RADIUS_MAX_PIXELS = 3; + +export function zoomScaledPointSize( + pointSize: number, + zoom: number | null | undefined, + zoomReference = POINT_SIZE_ZOOM_REFERENCE, + minScale = MIN_POINT_SIZE_SCALE +): number { + if (zoom === null || zoom === undefined || !Number.isFinite(zoom)) { + return pointSize; + } + const scale = 2 ** (zoom - zoomReference); + return pointSize * Math.min(1, Math.max(minScale, scale)); +} + export interface PointsLayerRenderConfig { /** The points element to render */ element: PointsElement; @@ -36,11 +59,19 @@ export interface PointsLayerRenderConfig { visible: boolean; /** Point radius in pixels */ pointSize?: number; + pointRadiusMinPixels?: number; + pointRadiusMaxPixels?: number; + pointMinSizeScale?: number; + /** Orthographic view zoom used to scale pointSize when zoomed out */ + viewZoom?: number | null; /** Point color [r, g, b, a] (0-255) */ color?: [number, number, number, number]; + /** Integer codes matching `{feature_key}_codes` in the Morton Parquet artifact. */ + featureCodes?: readonly number[]; /** ndarray - if we want other data for properties like color/radius etc they will be handled differently */ pointData?: PointData; pointTilingMetadata?: PointsTilingMetadata; + tileLoadCallbacks?: PointsTileLoadCallbacks; use3d?: boolean; } @@ -73,10 +104,25 @@ function isPointTileBbox(value: unknown): value is PointTileBbox { ); } -function boundsFromTileBbox(bbox: unknown): SpatialBounds | null { - if (!isPointTileBbox(bbox)) { +export function intersectBounds( + query: SpatialBounds, + clip: SpatialBounds +): SpatialBounds | null { + const minX = Math.max(query.minX, clip.minX); + const maxX = Math.min(query.maxX, clip.maxX); + const minY = Math.max(query.minY, clip.minY); + const maxY = Math.min(query.maxY, clip.maxY); + if (minX > maxX || minY > maxY) { return null; } + return { minX, minY, maxX, maxY }; +} + +function scatterBoundsFromTileBbox(bbox: PointTileBbox): [number, number, number, number] { + return [bbox.left, bbox.top, bbox.right, bbox.bottom]; +} + +function boundsFromTileBbox(bbox: PointTileBbox): SpatialBounds { return { minX: Math.min(bbox.left, bbox.right), maxX: Math.max(bbox.left, bbox.right), @@ -91,21 +137,43 @@ function renderPointScatterSubLayer( props: { color: [number, number, number, number]; pointSize: number; + pointRadiusMinPixels?: number; + pointRadiusMaxPixels?: number; + pointMinSizeScale?: number; + viewZoom?: number | null; opacity: number; modelMatrix: Matrix4; use3d?: boolean; + tileBounds?: [number, number, number, number]; + /** Tile sublayers use fixed pixel radius (Vitessce pattern). */ + tileSubLayer?: boolean; } ) { const d = data.data; + const effectivePointSize = props.tileSubLayer + ? props.pointSize + : zoomScaledPointSize( + props.pointSize, + props.viewZoom, + POINT_SIZE_ZOOM_REFERENCE, + props.pointMinSizeScale ?? MIN_POINT_SIZE_SCALE + ); return new ScatterplotLayer({ id, data: d[0], + ...(props.tileBounds ? { bounds: props.tileBounds } : {}), getPosition: (_d, { index, target }) => [ d[0][index], d[1][index], props.use3d ? d[2]?.[index] || 0 : 0, ], - getRadius: props.pointSize, + getRadius: effectivePointSize, + ...(props.tileSubLayer + ? { + radiusMinPixels: props.pointRadiusMinPixels ?? DEFAULT_POINT_RADIUS_MIN_PIXELS, + radiusMaxPixels: props.pointRadiusMaxPixels ?? DEFAULT_POINT_RADIUS_MAX_PIXELS, + } + : {}), radiusUnits: 'pixels', getFillColor: props.color, opacity: props.opacity, @@ -113,6 +181,15 @@ function renderPointScatterSubLayer( pickable: true, autoHighlight: true, highlightColor: [255, 255, 0, 200], + updateTriggers: { + getRadius: [ + props.pointSize, + props.viewZoom, + props.pointRadiusMinPixels, + props.pointRadiusMaxPixels, + props.pointMinSizeScale, + ], + }, }); } @@ -129,15 +206,33 @@ export function renderPointsLayer(config: PointsLayerRenderConfig): Layer | null modelMatrix, opacity, visible, - pointSize = 1, + pointSize = DEFAULT_POINT_SIZE, + pointRadiusMinPixels, + pointRadiusMaxPixels, + pointMinSizeScale, + viewZoom, color = [255, 100, 100, 200], pointData, pointTilingMetadata, + featureCodes, + tileLoadCallbacks, use3d, } = config; if (!visible) return null; + const scatterStyleProps = { + color, + pointSize, + pointRadiusMinPixels, + pointRadiusMaxPixels, + pointMinSizeScale, + viewZoom, + opacity, + modelMatrix, + use3d, + }; + if (!pointData) { if (!pointTilingMetadata?.bounds) { console.debug( @@ -145,59 +240,81 @@ export function renderPointsLayer(config: PointsLayerRenderConfig): Layer | null ); return null; } + const localBounds = pointTilingMetadata.bounds; return new TileLayer({ id, - data: pointTilingMetadata.parquetPath, + coordinateSystem: COORDINATE_SYSTEM.CARTESIAN, + modelMatrix, extent: [ - pointTilingMetadata.bounds.minX, - pointTilingMetadata.bounds.minY, - pointTilingMetadata.bounds.maxX, - pointTilingMetadata.bounds.maxY, + localBounds.minX, + localBounds.minY, + localBounds.maxX, + localBounds.maxY, ], + opacity, + visible, tileSize: 512, - minZoom: -12, - maxZoom: 12, + // Vitessce: single tile resolution. extent enables z=-1 clamp when viewZoom < -1. + minZoom: -1, + maxZoom: -1, refinementStrategy: 'best-available', updateTriggers: { - getTileData: [element, pointTilingMetadata.parquetPath], + getTileData: [pointTilingMetadata.parquetPath, featureCodes], + renderSubLayers: [ + pointSize, + pointRadiusMinPixels, + pointRadiusMaxPixels, + pointMinSizeScale, + viewZoom, + color, + opacity, + modelMatrix, + use3d, + ], + }, + onViewportLoad(tiles) { + tileLoadCallbacks?.onViewportTilesRequested?.(tiles?.length ?? 0); }, async getTileData({ bbox, signal }: PointTileLoadProps) { - const bounds = boundsFromTileBbox(bbox); + if (!isPointTileBbox(bbox)) { + return null; + } + tileLoadCallbacks?.onTileLoadStart?.(); + const rawBounds = boundsFromTileBbox(bbox); + const bounds = intersectBounds(rawBounds, localBounds); if (!bounds) { + tileLoadCallbacks?.onTileLoadEnd?.(true); return null; } try { - return await element.loadPointsInBounds({ bounds, signal }); + const result = await element.loadPointsInBounds({ bounds, featureCodes, signal }); + tileLoadCallbacks?.onTileLoadEnd?.(true); + return result; } catch (error) { + tileLoadCallbacks?.onTileLoadEnd?.(false); if (signal?.aborted || isAbortError(error)) { return null; } throw error; } }, - renderSubLayers: (props: { id: string; data?: PointData | null }) => { + renderSubLayers: (props: { + id: string; + data?: PointData | null; + tile?: { bbox?: unknown }; + }) => { if (!props.data) { return null; } + const tileBbox = isPointTileBbox(props.tile?.bbox) ? props.tile.bbox : null; return renderPointScatterSubLayer(`${props.id}-scatter`, props.data, { - color, - pointSize, - opacity, - modelMatrix, - use3d, + ...scatterStyleProps, + tileBounds: tileBbox ? scatterBoundsFromTileBbox(tileBbox) : undefined, + tileSubLayer: true, }); }, }); } - if (!pointData) { - return null; - } - return renderPointScatterSubLayer(id, pointData, { - color, - pointSize, - opacity, - modelMatrix, - use3d, - }); + return renderPointScatterSubLayer(id, pointData, scatterStyleProps); } diff --git a/packages/vis/src/SpatialCanvas/types.ts b/packages/vis/src/SpatialCanvas/types.ts index 229e6bf9..4376cacd 100644 --- a/packages/vis/src/SpatialCanvas/types.ts +++ b/packages/vis/src/SpatialCanvas/types.ts @@ -93,8 +93,17 @@ export interface PointsLayerConfig extends BaseLayerConfig { // Points-specific settings // TODO: these should be accessors for getColor etc based on e.g. transcript type // should be able to filter etc. Some kind of LOD... + /** Base point radius in pixels (scatter path; tile sublayer getRadius). */ pointSize?: number; + /** Minimum radius in pixels for tiled points (deck.gl radiusMinPixels). */ + pointRadiusMinPixels?: number; + /** Maximum radius in pixels for tiled points (deck.gl radiusMaxPixels). */ + pointRadiusMaxPixels?: number; + /** Minimum pointSize multiplier when zoomed out on the non-tiled scatter path. */ + pointMinSizeScale?: number; color?: [number, number, number, number]; + /** Filter to these feature code(s). Future: string[] resolved via codebook. */ + featureCodes?: number[]; experimentalOptimizations?: 'auto' | 'off'; } diff --git a/packages/vis/src/SpatialCanvas/useLayerData.ts b/packages/vis/src/SpatialCanvas/useLayerData.ts index d76de6de..f0c61c3e 100644 --- a/packages/vis/src/SpatialCanvas/useLayerData.ts +++ b/packages/vis/src/SpatialCanvas/useLayerData.ts @@ -65,6 +65,15 @@ import { import { createImageLoader } from './renderers/imageRenderer'; import { renderLabelsLayer } from './renderers/labelsRenderer'; import { type PointData, renderPointsLayer } from './renderers/pointsRenderer'; +import { + aggregatePointsTileLoadProgress, + createPointsTileLoadCallbacks, + emptyPointsTileLoadProgress, + isPointsTileLoading, + pointsTileLoadingMessage, + type PointsTileLoadCallbacks, + type PointsTileLoadProgress, +} from './pointsTileProgress'; import { loadShapesData, renderShapesLayer } from './renderers/shapesRenderer'; import type { AvailableElement, ElementsByType, LayerConfig, ShapesLayerConfig } from './types'; @@ -224,6 +233,10 @@ interface UseLayerDataResult { isLoading: boolean; /** Whether any visible layer is still waiting on its first renderable resource. */ isBlocking: boolean; + /** Tile fetch progress for Morton-tiled points layers. */ + getPointsTileLoadProgress: (layerId?: string) => PointsTileLoadProgress; + /** User-facing message while tiled points are loading, if any. */ + getPointsTileLoadingMessage: () => string | null; /** Trigger a reload of data for a specific element */ reloadElement: (type: string, key: string) => void; /** World-space axis-aligned bounds for one visible layer with loaded data, or null. */ @@ -478,7 +491,8 @@ export function useLayerData( availableElements: ElementsByType, coordinateSystem: string | null, spatialData?: SpatialData, - experimentalOptimizations: 'auto' | 'off' = 'auto' + experimentalOptimizations: 'auto' | 'off' = 'auto', + viewZoom: number | null = null ): UseLayerDataResult { const { getOmeZarrMultiscalesData } = useVivLoaderRegistry(); @@ -505,11 +519,71 @@ export function useLayerData( const [layerLoadStates, setLayerLoadStates] = useState>({}); const [, setLoadedDataRevision] = useState(0); + const pointsTileProgressRef = useRef(new Map()); + const pointsTileCallbacksRef = useRef(new Map()); + const [pointsTileProgressRevision, setPointsTileProgressRevision] = useState(0); const notifyLoadedDataChanged = useCallback(() => { setLoadedDataRevision((revision) => revision + 1); }, []); + const getPointsTileCallbacks = useCallback((layerId: string): PointsTileLoadCallbacks => { + let callbacks = pointsTileCallbacksRef.current.get(layerId); + if (!callbacks) { + callbacks = createPointsTileLoadCallbacks( + () => pointsTileProgressRef.current.get(layerId) ?? emptyPointsTileLoadProgress(), + (progress) => { + pointsTileProgressRef.current.set(layerId, progress); + setPointsTileProgressRevision((revision) => revision + 1); + } + ); + pointsTileCallbacksRef.current.set(layerId, callbacks); + } + return callbacks; + }, []); + + const getPointsTileLoadProgress = useCallback( + (layerId?: string): PointsTileLoadProgress => { + void pointsTileProgressRevision; + if (layerId) { + return pointsTileProgressRef.current.get(layerId) ?? emptyPointsTileLoadProgress(); + } + const visibleProgress = new Map(); + for (const id of layerOrder) { + const config = layers[id]; + if (!config?.visible || config.type !== 'points') continue; + const progress = pointsTileProgressRef.current.get(id); + if (progress) { + visibleProgress.set(id, progress); + } + } + return aggregatePointsTileLoadProgress(visibleProgress); + }, + [layerOrder, layers, pointsTileProgressRevision] + ); + + const getPointsTileLoadingMessage = useCallback((): string | null => { + return pointsTileLoadingMessage(getPointsTileLoadProgress()); + }, [getPointsTileLoadProgress]); + + const prevExperimentalOptimizationsRef = useRef(experimentalOptimizations); + useEffect(() => { + const prev = prevExperimentalOptimizationsRef.current; + prevExperimentalOptimizationsRef.current = experimentalOptimizations; + if (prev === 'off' && experimentalOptimizations !== 'off') { + const loaded = loadedDataRef.current; + for (const layerId of layerOrder) { + const config = layers[layerId]; + if (config?.type !== 'points' || !config.visible) continue; + const elem = resolveLayerElement(layerId, config, elementMap.current); + if (elem) { + loaded.points.delete(elem.key); + } + } + notifyLoadedDataChanged(); + } + }, [experimentalOptimizations, layerOrder, layers, notifyLoadedDataChanged]); + // Build a map of element key -> AvailableElement for quick lookup const elementMap = useRef>(new Map()); @@ -825,6 +899,9 @@ export function useLayerData( const renderableMetadata = metadata?.supportsRowGroupRangeReads && metadata.bounds ? metadata : null; loadedDataRef.current.pointTilingMetadata.set(element.key, renderableMetadata); + if (renderableMetadata) { + loadedDataRef.current.points.delete(element.key); + } setLayerResourceStatus( layerId, 'geometry', @@ -1326,9 +1403,15 @@ export function useLayerData( opacity: config.opacity, visible: config.visible, pointSize: config.pointSize, + pointRadiusMinPixels: config.pointRadiusMinPixels, + pointRadiusMaxPixels: config.pointRadiusMaxPixels, + pointMinSizeScale: config.pointMinSizeScale, + viewZoom, color: config.color, + featureCodes: config.featureCodes, pointData, pointTilingMetadata, + tileLoadCallbacks: pointTilingMetadata ? getPointsTileCallbacks(layerId) : undefined, }); if (layer) deckLayers.push(layer); } @@ -1380,7 +1463,7 @@ export function useLayerData( } return deckLayers; - }, [layers, layerOrder, getStableSelections]); + }, [layers, layerOrder, getStableSelections, viewZoom, getPointsTileCallbacks]); const getImageLayerLoadedData = useCallback((layerId: string): ImageLoaderData | undefined => { const elem = resolveLayerElement(layerId, layersRef.current[layerId], elementMap.current); @@ -1639,13 +1722,24 @@ export function useLayerData( return vivProps; }, [layers, layerOrder, getStableSelections]); - const isLoading = useMemo( - () => - Object.values(layerLoadStates).some((state) => - Object.values(state).some((status) => status === 'loading') - ), - [layerLoadStates] - ); + const isLoading = useMemo(() => { + const resourceLoading = Object.values(layerLoadStates).some((state) => + Object.values(state).some((status) => status === 'loading') + ); + if (resourceLoading) { + return true; + } + void pointsTileProgressRevision; + for (const layerId of layerOrder) { + const config = layers[layerId]; + if (!config?.visible || config.type !== 'points') continue; + const progress = pointsTileProgressRef.current.get(layerId); + if (progress && isPointsTileLoading(progress)) { + return true; + } + } + return false; + }, [layerLoadStates, layerOrder, layers, pointsTileProgressRevision]); const isBlocking = useMemo( () => @@ -1680,6 +1774,8 @@ export function useLayerData( getShapePickEvent, isLoading, isBlocking, + getPointsTileLoadProgress, + getPointsTileLoadingMessage, reloadElement, getWorldBoundsForLayer, getWorldBoundsForVisibleLayers, diff --git a/packages/vis/tests/pointsRenderer.spec.ts b/packages/vis/tests/pointsRenderer.spec.ts new file mode 100644 index 00000000..f575cf62 --- /dev/null +++ b/packages/vis/tests/pointsRenderer.spec.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; + +import { + MIN_POINT_SIZE_SCALE, + POINT_SIZE_ZOOM_REFERENCE, + zoomScaledPointSize, +} from '../src/SpatialCanvas/renderers/pointsRenderer.js'; + +describe('zoomScaledPointSize', () => { + it('returns base size at the reference zoom', () => { + expect(zoomScaledPointSize(4, POINT_SIZE_ZOOM_REFERENCE)).toBe(4); + }); + + it('shrinks points when zoomed out', () => { + expect(zoomScaledPointSize(4, -2)).toBe(1); + }); + + it('does not grow beyond the configured size when zoomed in', () => { + expect(zoomScaledPointSize(4, 4)).toBe(4); + }); + + it('clamps to the minimum scale when zoomed far out', () => { + expect(zoomScaledPointSize(4, -10)).toBe(4 * MIN_POINT_SIZE_SCALE); + }); + + it('returns base size when zoom is unavailable', () => { + expect(zoomScaledPointSize(3, null)).toBe(3); + expect(zoomScaledPointSize(3, undefined)).toBe(3); + }); +}); diff --git a/packages/vis/tests/pointsTileProgress.spec.ts b/packages/vis/tests/pointsTileProgress.spec.ts new file mode 100644 index 00000000..6121abd5 --- /dev/null +++ b/packages/vis/tests/pointsTileProgress.spec.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; + +import { + aggregatePointsTileLoadProgress, + createPointsTileLoadCallbacks, + emptyPointsTileLoadProgress, + isPointsTileLoading, + pointsTileLoadingMessage, +} from '../src/SpatialCanvas/pointsTileProgress.js'; + +describe('pointsTileProgress', () => { + it('aggregates progress across layers', () => { + const aggregate = aggregatePointsTileLoadProgress( + new Map([ + ['a', { inFlight: 2, loaded: 1, viewportTotal: 4 }], + ['b', { inFlight: 1, loaded: 3, viewportTotal: 6 }], + ]) + ); + expect(aggregate).toEqual({ inFlight: 3, loaded: 4, viewportTotal: 10 }); + }); + + it('reports loading while tiles are in flight', () => { + expect( + pointsTileLoadingMessage({ inFlight: 2, loaded: 1, viewportTotal: 6 }) + ).toBe('Loading points… (1/6 tiles)'); + expect(isPointsTileLoading({ inFlight: 2, loaded: 1, viewportTotal: 6 })).toBe(true); + }); + + it('clears the message when the viewport batch completes', () => { + expect( + pointsTileLoadingMessage({ inFlight: 0, loaded: 6, viewportTotal: 6 }) + ).toBeNull(); + }); + + it('tracks tile lifecycle through callbacks', () => { + let progress = emptyPointsTileLoadProgress(); + const callbacks = createPointsTileLoadCallbacks( + () => progress, + (next) => { + progress = next; + } + ); + + callbacks.onViewportTilesRequested?.(2); + expect(progress).toEqual({ inFlight: 0, loaded: 0, viewportTotal: 2 }); + + callbacks.onTileLoadStart?.(); + callbacks.onTileLoadStart?.(); + expect(progress.inFlight).toBe(2); + + callbacks.onTileLoadEnd?.(true); + expect(progress).toEqual({ inFlight: 1, loaded: 1, viewportTotal: 2 }); + + callbacks.onTileLoadEnd?.(true); + expect(progress).toEqual({ inFlight: 0, loaded: 2, viewportTotal: 2 }); + expect(pointsTileLoadingMessage(progress)).toBeNull(); + }); +}); From 8c6d7abfda3862221d21dca09dcc19abe848c716 Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Sat, 20 Jun 2026 08:32:23 +0100 Subject: [PATCH 06/38] spatialdata-experimental-writer point index options - Updated README.md to include detailed installation instructions and command usage for the experimental writer. - Added a new script for benchmarking points index permutations, allowing users to evaluate performance based on index-manifest.json. - Implemented a new CLI command for writing derivative Zarr stores with transcript index sort permutations. - Introduced consolidated metadata registration for points elements in Zarr stores. - Enhanced morton sorting functionality to support custom sort orders and ensure uint columns are not persisted in output Parquet files. - Added integration tests to validate new features and ensure correct functionality across the package. --- .../spatialdata-experimental-writer/README.md | 88 +++++++- .../scripts/benchmark_points_index.py | 155 +++++++++++++ .../spatialdata_experimental_writer/cli.py | 84 ++++++- .../index_permutations.py | 210 ++++++++++++++++++ .../spatialdata_experimental_writer/points.py | 31 ++- .../spatialdata_experimental_writer/zarr.py | 64 +++++- .../tests/test_integration.py | 136 ++++++++++++ .../tests/test_zarr.py | 39 ++++ 8 files changed, 786 insertions(+), 21 deletions(-) create mode 100644 python/spatialdata-experimental-writer/scripts/benchmark_points_index.py create mode 100644 python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/index_permutations.py create mode 100644 python/spatialdata-experimental-writer/tests/test_integration.py diff --git a/python/spatialdata-experimental-writer/README.md b/python/spatialdata-experimental-writer/README.md index 8db60cf2..e5cba490 100644 --- a/python/spatialdata-experimental-writer/README.md +++ b/python/spatialdata-experimental-writer/README.md @@ -7,10 +7,92 @@ The initial writer targets Vitessce-compatible Morton-sorted Points Parquet: - `x`, `y`, optional `z` coordinates are preserved. - `morton_code_2d` is added using 16 bits per axis. -- the first 2-4 rows are sentinel/extreme rows with `morton_code_2d == 0`; +- the first 2–4 rows are sentinel/extreme rows with `morton_code_2d == 0`; readers can infer the full point bounding box from these rows. +- `{feature_key}_codes` (for example `feature_name_codes`) are added when + `feature_key` is set in element attrs. - string/categorical columns are placed at the right side of the table. - row-group size is controlled when writing Parquet. +- intermediate Morton uint columns are not persisted in the output Parquet. -The package also includes a small multiscale Parquet writer hook that stores -Padua-style `spatialdata_multiscale` JSON metadata in the Parquet schema. +Morton v1 belongs on the **canonical** element path +`points//points.parquet`. Use `--experimental` only for layouts that +standard readers cannot consume (see +[ADR 0002](../../docs/adr/0002-spatially-aware-vector-loading.md)). + +Feature / gene filtering in the browser is documented in ADR 0002; pass integer +`featureCodes` through `@spatialdata/core` `loadPointsInBounds()` and +`PointsLayerConfig.featureCodes` in `@spatialdata/vis`. + +## Install + +```bash +cd python/spatialdata-experimental-writer +uv sync +``` + +## Commands + +```bash +# List Points elements in a store +uv run spatialdata-experimental-writer list-points ~/data/xenium.zarr + +# Morton-sort transcripts in-place on canonical points//points.parquet +uv run spatialdata-experimental-writer morton-points-from-zarr \ + ~/data/xenium.zarr --points-key transcripts + +# Optional: write to points.experimental/ instead of canonical path +uv run spatialdata-experimental-writer morton-points-from-zarr \ + ~/data/xenium.zarr --points-key transcripts --experimental + +# Build a derivative store with transcript index sort permutations +uv run spatialdata-experimental-writer write-index-permutations \ + ~/data/xenium_rep1_io.zarr \ + ~/data/xenium_rep1_index-permutations.zarr + +# Morton-sort a CSV or Parquet file +uv run spatialdata-experimental-writer morton-points input.csv output.parquet \ + --feature-key feature_name +``` + +## Xenium workflow + +Standard sandbox datasets are listed in the +[spatialdata datasets docs](https://spatialdata.scverse.org/en/stable/tutorials/notebooks/datasets/README.html): + +| Dataset | URL | +|---------|-----| +| `xenium_rep1_io.zarr` | `https://s3.embl.de/spatialdata/spatialdata-sandbox/xenium_rep1_io.zarr/` | +| `xenium_rep2_io.zarr` | `https://s3.embl.de/spatialdata/spatialdata-sandbox/xenium_rep2_io.zarr/` | +| `visium_associated_xenium_io.zarr` | `https://s3.embl.de/spatialdata/spatialdata-sandbox/visium_associated_xenium_io.zarr/` | + +After downloading a store locally: + +```bash +uv run spatialdata-experimental-writer morton-points-from-zarr \ + ~/data/spatialdata/sdata_inputs/xenium_rep1_io.zarr \ + --points-key transcripts +``` + +This replaces `points/transcripts/points.parquet` in place (single-file output; +multipart source directories are replaced). Open the store in `@spatialdata/vis` +with `experimentalOptimizations="auto"` to use TileLayer row-group reads. + +For sort-strategy benchmarks on a **copy** of the store: + +```bash +uv run spatialdata-experimental-writer write-index-permutations \ + ~/data/spatialdata/sdata_inputs/xenium_rep1_io.zarr \ + ~/data/spatialdata/sdata_inputs/xenium_rep1_index-permutations.zarr \ + --max-rows 500000 + +uv run python scripts/benchmark_points_index.py \ + ~/data/spatialdata/sdata_inputs/xenium_rep1_index-permutations.zarr +``` + +## Multiscale hook + +The package also includes a Padua-style multiscale Parquet writer that stores +`spatialdata_multiscale` JSON metadata in the Parquet schema. That layout is +non-standard for morton-points v1 and belongs under `points.experimental/` if +persisted. diff --git a/python/spatialdata-experimental-writer/scripts/benchmark_points_index.py b/python/spatialdata-experimental-writer/scripts/benchmark_points_index.py new file mode 100644 index 00000000..57b08aaa --- /dev/null +++ b/python/spatialdata-experimental-writer/scripts/benchmark_points_index.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Benchmark points index permutations using index-manifest.json.""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import pandas as pd +import pyarrow.parquet as pq + + +def _load_bounds(manifest: dict, scenario_id: str | None) -> dict[str, float]: + scenarios = manifest.get("benchmark_scenarios") or [] + if scenario_id: + for scenario in scenarios: + if scenario.get("id") == scenario_id: + return scenario["bounds"] + raise SystemExit(f"Unknown scenario id: {scenario_id}") + if scenarios: + return scenarios[0]["bounds"] + raise SystemExit("Manifest has no benchmark_scenarios") + + +def _feature_codes(manifest: dict, scenario_id: str | None) -> list[int] | None: + scenarios = manifest.get("benchmark_scenarios") or [] + if not scenario_id: + return None + for scenario in scenarios: + if scenario.get("id") == scenario_id: + codes = scenario.get("feature_codes") + return list(codes) if codes is not None else None + return None + + +def _parquet_path(store: Path, element_path: str) -> Path: + return store / element_path / "points.parquet" + + +def _read_rows_in_bounds( + parquet_path: Path, + bounds: dict[str, float], + feature_codes: list[int] | None, + feature_key: str | None, +) -> tuple[int, int]: + if parquet_path.is_dir(): + parts = sorted(parquet_path.glob("part.*.parquet")) + if not parts: + raise FileNotFoundError(f"No parquet parts under {parquet_path}") + frames = [pd.read_parquet(part) for part in parts] + df = pd.concat(frames, ignore_index=True) + bytes_read = sum(part.stat().st_size for part in parts) + else: + bytes_read = parquet_path.stat().st_size + df = pd.read_parquet(parquet_path) + + mask = ( + (df["x"] >= bounds["minX"]) + & (df["x"] <= bounds["maxX"]) + & (df["y"] >= bounds["minY"]) + & (df["y"] <= bounds["maxY"]) + ) + if feature_codes is not None: + code_column = f"{feature_key}_codes" if feature_key else "feature_name_codes" + if code_column not in df.columns: + raise KeyError(f"Missing feature code column {code_column!r}") + mask &= df[code_column].isin(feature_codes) + return int(mask.sum()), int(bytes_read) + + +def _estimate_row_group_bytes(parquet_path: Path, bounds: dict[str, float]) -> int | None: + if not parquet_path.is_file(): + return None + if "morton_code_2d" not in pq.ParquetFile(parquet_path).schema_arrow.names: + return None + # Upper bound only: full file size when row-group APIs are unavailable in this script. + return parquet_path.stat().st_size + + +def benchmark_store( + store: Path, + *, + scenario_id: str | None, + conditions: list[str] | None, +) -> list[dict]: + manifest_path = store / "index-manifest.json" + if not manifest_path.exists(): + raise FileNotFoundError(f"Missing index-manifest.json under {store}") + manifest = json.loads(manifest_path.read_text()) + bounds = _load_bounds(manifest, scenario_id) + feature_codes = _feature_codes(manifest, scenario_id) + feature_key = manifest.get("feature_key") + selected = conditions or [entry["id"] for entry in manifest.get("conditions", [])] + + results: list[dict] = [] + for condition in manifest.get("conditions", []): + condition_id = condition["id"] + if condition_id not in selected: + continue + element_path = condition["element_path"] + parquet_path = _parquet_path(store, element_path) + started = time.perf_counter() + try: + rows, bytes_read = _read_rows_in_bounds( + parquet_path, bounds, feature_codes, feature_key + ) + row_group_hint = _estimate_row_group_bytes(parquet_path, bounds) + except Exception as error: # noqa: BLE001 - report per condition + results.append( + { + "condition": condition_id, + "element_path": element_path, + "error": str(error), + } + ) + continue + elapsed_ms = (time.perf_counter() - started) * 1000 + results.append( + { + "condition": condition_id, + "element_path": element_path, + "sort_order": condition.get("sort_order"), + "tiling_kind": condition.get("tiling_kind"), + "rows_in_bounds": rows, + "bytes_read_estimate": bytes_read, + "morton_row_group_bytes_upper_bound": row_group_hint, + "latency_ms": round(elapsed_ms, 2), + "bounds": bounds, + "feature_codes": feature_codes, + } + ) + return results + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Benchmark points index permutations from index-manifest.json" + ) + parser.add_argument("store", type=Path, help="Derivative Zarr store path") + parser.add_argument("--scenario", metavar="ID", help="benchmark_scenarios id") + parser.add_argument( + "--conditions", + metavar="IDS", + help="Comma-separated condition ids (default: all in manifest)", + ) + args = parser.parse_args() + condition_ids = args.conditions.split(",") if args.conditions else None + results = benchmark_store(args.store, scenario_id=args.scenario, conditions=condition_ids) + print(json.dumps({"store": str(args.store), "results": results}, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/cli.py b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/cli.py index 277d2a24..c631dab8 100644 --- a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/cli.py +++ b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/cli.py @@ -2,17 +2,18 @@ import argparse import json +import shutil from pathlib import Path import pandas as pd +from .index_permutations import DEFAULT_CONDITIONS, write_index_permutations from .points import ( build_spatialdata_multiscale_metadata, write_morton_points_parquet, write_multiscale_points_parquet, ) from .zarr import ( - experimental_points_output_path, list_points_keys, points_parquet_path, read_points_dataframe, @@ -24,10 +25,14 @@ # List Points elements in a SpatialData Zarr store spatialdata-experimental-writer list-points ~/data/xenium.zarr - # Morton-sort transcripts from a Zarr store into points.experimental/ + # Morton-sort transcripts in-place on the canonical points element spatialdata-experimental-writer morton-points-from-zarr \\ ~/data/xenium.zarr --points-key transcripts + # Build a derivative store with transcript index permutations + spatialdata-experimental-writer write-index-permutations \\ + ~/data/xenium_rep1_io.zarr ~/data/xenium_rep1_index-permutations.zarr + # Morton-sort a CSV or single Parquet file spatialdata-experimental-writer morton-points input.csv output.parquet \\ --feature-key feature_name @@ -139,11 +144,19 @@ def _morton_points_from_zarr(args: argparse.Namespace) -> None: attrs = read_points_element_attrs(zarr_path, points_key) feature_key = args.feature_key or attrs.get("feature_key") source_parquet = points_parquet_path(zarr_path, points_key) - output = Path(args.output) if args.output else experimental_points_output_path( - zarr_path, points_key - ) + if args.output: + output = Path(args.output) + elif args.experimental: + output = zarr_path / "points.experimental" / points_key / "points.parquet" + else: + output = source_parquet df = read_points_dataframe(source_parquet) + if output.exists(): + if output.is_dir(): + shutil.rmtree(output) + else: + output.unlink() sorted_df = write_morton_points_parquet( df, output, @@ -159,6 +172,7 @@ def _morton_points_from_zarr(args: argparse.Namespace) -> None: "points_key": points_key, "source": str(source_parquet), "output": str(output), + "in_place": not args.experimental and args.output is None, "feature_key": feature_key, "rows": int(len(sorted_df)), "row_group_size": args.row_group_size, @@ -169,6 +183,29 @@ def _morton_points_from_zarr(args: argparse.Namespace) -> None: ) +def _write_index_permutations(args: argparse.Namespace) -> None: + condition_ids = args.conditions.split(",") if args.conditions else None + selected = None + if condition_ids: + by_id = {condition.id: condition for condition in DEFAULT_CONDITIONS} + missing = [value for value in condition_ids if value not in by_id] + if missing: + raise SystemExit(f"Unknown conditions: {', '.join(missing)}") + selected = tuple(by_id[value] for value in condition_ids) + + manifest = write_index_permutations( + args.source_zarr, + args.dest_zarr, + points_key=args.points_key, + max_rows=args.max_rows, + conditions=selected, + overwrite=args.overwrite, + row_group_size=args.row_group_size, + compression=args.compression, + ) + print(json.dumps(manifest, indent=2, sort_keys=True)) + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( @@ -198,7 +235,7 @@ def build_parser() -> argparse.ArgumentParser: description=( "Read points//points.parquet from a SpatialData Zarr store, " "add morton_code_2d sentinel rows, and write Vitessce-compatible Parquet. " - "Defaults to points.experimental//points.parquet inside the store." + "Defaults to in-place replacement of points//points.parquet." ), ) morton_from_zarr.add_argument( @@ -206,6 +243,11 @@ def build_parser() -> argparse.ArgumentParser: metavar="ZARR", help="Path to a SpatialData Zarr store", ) + morton_from_zarr.add_argument( + "--experimental", + action="store_true", + help="Write to points.experimental//points.parquet instead of canonical path", + ) morton_from_zarr.add_argument( "--points-key", metavar="KEY", @@ -218,7 +260,8 @@ def build_parser() -> argparse.ArgumentParser: "--output", metavar="PATH", help=( - "Output Parquet path (default: /points.experimental//points.parquet)" + "Output Parquet path (default: in-place on points//points.parquet, " + "or points.experimental//points.parquet with --experimental)" ), ) morton_from_zarr.add_argument( @@ -317,6 +360,33 @@ def build_parser() -> argparse.ArgumentParser: ) multiscale.set_defaults(func=_multiscale_points) + index_permutations = subparsers.add_parser( + "write-index-permutations", + help="write derivative Zarr with transcript index sort permutations", + description=( + "Copy a SpatialData Zarr store and add sibling points elements with " + "different transcript sort/index layouts plus index-manifest.json." + ), + ) + index_permutations.add_argument("source_zarr", metavar="SOURCE_ZARR") + index_permutations.add_argument("dest_zarr", metavar="DEST_ZARR") + index_permutations.add_argument("--points-key", metavar="KEY") + index_permutations.add_argument("--max-rows", type=_positive_int, metavar="N") + index_permutations.add_argument( + "--conditions", + metavar="IDS", + help="Comma-separated condition ids (default: all)", + ) + index_permutations.add_argument("--overwrite", action="store_true") + index_permutations.add_argument( + "--row-group-size", + type=_positive_int, + default=50_000, + metavar="N", + ) + index_permutations.add_argument("--compression", default="zstd") + index_permutations.set_defaults(func=_write_index_permutations) + return parser diff --git a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/index_permutations.py b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/index_permutations.py new file mode 100644 index 00000000..9463044b --- /dev/null +++ b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/index_permutations.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import json +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Sequence + +import pandas as pd + +from .points import MORTON_CODE_2D_COLUMN, write_morton_points_parquet +from .zarr import ( + list_points_keys, + points_parquet_path, + read_points_dataframe, + read_points_element_attrs, + register_points_elements_in_consolidated_metadata, +) + + +@dataclass(frozen=True) +class IndexCondition: + id: str + element_suffix: str + sort_order: tuple[str, ...] | None + tiling_kind: str | None + + +DEFAULT_CONDITIONS: tuple[IndexCondition, ...] = ( + IndexCondition("canonical", "", None, None), + IndexCondition("morton", "_morton", (MORTON_CODE_2D_COLUMN,), "morton-points"), + IndexCondition( + "morton-then-feature", + "_morton_then_feature", + (MORTON_CODE_2D_COLUMN, "feature_name_codes"), + "morton-points", + ), + IndexCondition( + "feature-then-morton", + "_feature_then_morton", + ("feature_name_codes", MORTON_CODE_2D_COLUMN), + "experimental", + ), +) + + +def _resolve_feature_code_column(feature_key: str | None) -> str: + if feature_key: + return f"{feature_key}_codes" + return "feature_name_codes" + + +def _condition_sort_order( + condition: IndexCondition, feature_key: str | None +) -> list[str] | None: + if condition.sort_order is None: + return None + feature_code_column = _resolve_feature_code_column(feature_key) + return [ + feature_code_column if column == "feature_name_codes" else column + for column in condition.sort_order + ] + + +def _copy_store_shell(source: Path, dest: Path, *, overwrite: bool) -> None: + if dest.exists(): + if not overwrite: + raise FileExistsError(f"Destination already exists: {dest}") + shutil.rmtree(dest) + + def ignore_points(directory: str, names: list[str]) -> set[str]: + if Path(directory) == source: + return {"points"} if "points" in names else set() + return set() + + shutil.copytree(source, dest, ignore=ignore_points) + + +def _write_element_zarr_json(source_element_dir: Path, dest_element_dir: Path) -> None: + source_json = source_element_dir / "zarr.json" + dest_element_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(source_json, dest_element_dir / "zarr.json") + + +def _copy_canonical_parquet(source_parquet: Path, dest_parquet: Path) -> None: + dest_parquet.parent.mkdir(parents=True, exist_ok=True) + if source_parquet.is_dir(): + if dest_parquet.exists(): + shutil.rmtree(dest_parquet) + shutil.copytree(source_parquet, dest_parquet) + else: + shutil.copy2(source_parquet, dest_parquet) + + +def write_index_permutations( + source_zarr: str | Path, + dest_zarr: str | Path, + *, + points_key: str | None = None, + max_rows: int | None = None, + conditions: Sequence[IndexCondition] | None = None, + overwrite: bool = False, + row_group_size: int = 50_000, + compression: str = "zstd", +) -> dict[str, Any]: + source_path = Path(source_zarr) + dest_path = Path(dest_zarr) + keys = list_points_keys(source_path) + if not keys: + raise FileNotFoundError(f"No Points elements found under {source_path / 'points'}") + + resolved_key = points_key or (keys[0] if len(keys) == 1 else None) + if resolved_key is None: + raise ValueError( + "Multiple Points elements found; pass points_key. " + f"Available keys: {', '.join(keys)}" + ) + if resolved_key not in keys: + raise ValueError(f"Unknown points key {resolved_key!r}. Available: {', '.join(keys)}") + + attrs = read_points_element_attrs(source_path, resolved_key) + feature_key = attrs.get("feature_key") + source_element_dir = source_path / "points" / resolved_key + source_parquet = points_parquet_path(source_path, resolved_key) + + _copy_store_shell(source_path, dest_path, overwrite=overwrite) + + df = read_points_dataframe(source_parquet) + if max_rows is not None and len(df) > max_rows: + df = df.sample(n=max_rows, random_state=0).reset_index(drop=True) + + selected = tuple(conditions or DEFAULT_CONDITIONS) + manifest_conditions: list[dict[str, Any]] = [] + + for condition in selected: + element_key = ( + resolved_key if condition.id == "canonical" else f"{resolved_key}{condition.element_suffix}" + ) + element_dir = dest_path / "points" / element_key + output_parquet = element_dir / "points.parquet" + _write_element_zarr_json(source_element_dir, element_dir) + + if condition.sort_order is None: + if max_rows is not None: + output_parquet.parent.mkdir(parents=True, exist_ok=True) + if output_parquet.exists(): + if output_parquet.is_dir(): + shutil.rmtree(output_parquet) + else: + output_parquet.unlink() + df.to_parquet(output_parquet, index=False) + else: + _copy_canonical_parquet(source_parquet, output_parquet) + else: + sort_order = _condition_sort_order(condition, feature_key) + write_morton_points_parquet( + df, + output_parquet, + feature_key=feature_key, + sort_order=sort_order, + row_group_size=row_group_size, + compression=compression, + ) + + manifest_conditions.append( + { + "id": condition.id, + "element_path": f"points/{element_key}", + "sort_order": list(condition.sort_order) if condition.sort_order else None, + "tiling_kind": condition.tiling_kind, + } + ) + + manifest = { + "version": "0.1", + "store_path": str(dest_path), + "source_store": str(source_path), + "source_element": f"points/{resolved_key}", + "feature_key": feature_key, + "n_points": int(len(df)), + "conditions": manifest_conditions, + "benchmark_scenarios": [ + { + "id": "center-tile", + "bounds": { + "minX": float(df["x"].quantile(0.25)), + "maxX": float(df["x"].quantile(0.75)), + "minY": float(df["y"].quantile(0.25)), + "maxY": float(df["y"].quantile(0.75)), + }, + } + ], + } + element_keys = [ + ( + resolved_key + if condition.id == "canonical" + else f"{resolved_key}{condition.element_suffix}" + ) + for condition in selected + ] + register_points_elements_in_consolidated_metadata( + dest_path, + element_keys, + template_key=resolved_key, + ) + + manifest_path = dest_path / "index-manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + return manifest diff --git a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/points.py b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/points.py index ed629c8b..f8d2895d 100644 --- a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/points.py +++ b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/points.py @@ -2,7 +2,7 @@ import json from pathlib import Path -from typing import Any +from typing import Any, Sequence import numpy as np import pandas as pd @@ -85,7 +85,12 @@ def _move_string_like_columns_right(df: pd.DataFrame) -> pd.DataFrame: return df[[*other, *string_like]] -def morton_sort_points(df: pd.DataFrame, *, feature_key: str | None = None) -> pd.DataFrame: +def morton_sort_points( + df: pd.DataFrame, + *, + feature_key: str | None = None, + sort_order: Sequence[str] | None = None, +) -> pd.DataFrame: missing = [column for column in ("x", "y") if column not in df.columns] if missing: raise ValueError("Points dataframe is missing required columns: " + ", ".join(missing)) @@ -95,18 +100,21 @@ def morton_sort_points(df: pd.DataFrame, *, feature_key: str | None = None) -> p x_max = float(out["x"].max()) y_min = float(out["y"].min()) y_max = float(out["y"].max()) - out["x_uint"] = _norm_series_to_uint(out["x"], x_min, x_max) - out["y_uint"] = _norm_series_to_uint(out["y"], y_min, y_max) - out[MORTON_CODE_2D_COLUMN] = morton_code_2d(out["x_uint"], out["y_uint"]) + x_uint = _norm_series_to_uint(out["x"], x_min, x_max) + y_uint = _norm_series_to_uint(out["y"], y_min, y_max) + out[MORTON_CODE_2D_COLUMN] = morton_code_2d(x_uint, y_uint) sentinel_indices = _extreme_indices(out) sentinel = out.loc[sentinel_indices].copy().reset_index(drop=True) sentinel[MORTON_CODE_2D_COLUMN] = MORTON_CODE_EXTREME_VALUE_INDICATOR rest = out.drop(index=sentinel_indices) - sort_columns = [MORTON_CODE_2D_COLUMN] - if "z" in rest.columns and rest["z"].nunique(dropna=False) < 100: - sort_columns = ["z", MORTON_CODE_2D_COLUMN] + if sort_order is None: + sort_columns: list[str] = [MORTON_CODE_2D_COLUMN] + if "z" in rest.columns and rest["z"].nunique(dropna=False) < 100: + sort_columns = ["z", MORTON_CODE_2D_COLUMN] + else: + sort_columns = list(sort_order) rest = rest.sort_values(sort_columns, kind="mergesort").reset_index(drop=True) combined = pd.concat([sentinel, rest], ignore_index=True) @@ -153,11 +161,14 @@ def write_morton_points_parquet( output_path: str | Path, *, feature_key: str | None = None, + sort_order: Sequence[str] | None = None, row_group_size: int = 50_000, compression: str = "zstd", ) -> pd.DataFrame: - sorted_df = morton_sort_points(df, feature_key=feature_key) - table = pa.Table.from_pandas(sorted_df, preserve_index=False) + sorted_df = morton_sort_points(df, feature_key=feature_key, sort_order=sort_order) + indexed = sorted_df.copy() + indexed.index.name = "__index_level_0__" + table = pa.Table.from_pandas(indexed, preserve_index=True) _write_arrow_table_in_row_groups( table, Path(output_path), diff --git a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/zarr.py b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/zarr.py index 9f273023..3a0ae59f 100644 --- a/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/zarr.py +++ b/python/spatialdata-experimental-writer/src/spatialdata_experimental_writer/zarr.py @@ -2,7 +2,7 @@ import json from pathlib import Path -from typing import Any +from typing import Any, Sequence import pandas as pd import pyarrow.dataset as ds @@ -51,6 +51,68 @@ def experimental_points_output_path(zarr_path: str | Path, points_key: str) -> P return Path(zarr_path) / "points.experimental" / points_key / "points.parquet" +def _points_element_consolidated_entry( + zarr_path: Path, + points_key: str, + *, + metadata: dict[str, Any] | None = None, +) -> dict[str, Any]: + if metadata is not None: + entry = metadata.get(f"points/{points_key}") + if isinstance(entry, dict): + return json.loads(json.dumps(entry)) + element_json = _points_root(zarr_path) / points_key / "zarr.json" + element_doc = _read_zarr_json(element_json) + return { + "attributes": element_doc.get("attributes", {}), + "node_type": element_doc.get("node_type", "group"), + "zarr_format": element_doc.get("zarr_format", 3), + } + + +def read_store_consolidated_metadata(zarr_path: str | Path) -> dict[str, Any]: + root_json = Path(zarr_path) / "zarr.json" + if not root_json.is_file(): + raise FileNotFoundError(f"Missing store metadata: {root_json}") + doc = _read_zarr_json(root_json) + consolidated = doc.get("consolidated_metadata") + if not isinstance(consolidated, dict): + raise ValueError(f"Store has no consolidated metadata: {root_json}") + metadata = consolidated.get("metadata") + if not isinstance(metadata, dict): + raise ValueError(f"Store consolidated metadata has no metadata map: {root_json}") + return metadata + + +def register_points_elements_in_consolidated_metadata( + zarr_path: str | Path, + element_keys: Sequence[str], + *, + template_key: str, +) -> None: + """Register sibling points elements in the store root consolidated metadata.""" + store_path = Path(zarr_path) + root_json = store_path / "zarr.json" + doc = _read_zarr_json(root_json) + consolidated = doc.get("consolidated_metadata") + if not isinstance(consolidated, dict): + consolidated = {"kind": "inline", "metadata": {}} + doc["consolidated_metadata"] = consolidated + metadata = consolidated.get("metadata") + if not isinstance(metadata, dict): + metadata = {} + consolidated["metadata"] = metadata + + template_entry = _points_element_consolidated_entry( + store_path, + template_key, + metadata=metadata, + ) + for key in element_keys: + metadata[f"points/{key}"] = json.loads(json.dumps(template_entry)) + root_json.write_text(json.dumps(doc, indent=2) + "\n") + + def read_points_dataframe(parquet_path: str | Path) -> pd.DataFrame: path = Path(parquet_path) if not path.exists(): diff --git a/python/spatialdata-experimental-writer/tests/test_integration.py b/python/spatialdata-experimental-writer/tests/test_integration.py new file mode 100644 index 00000000..5b491c6e --- /dev/null +++ b/python/spatialdata-experimental-writer/tests/test_integration.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +from spatialdata_experimental_writer.index_permutations import write_index_permutations +from spatialdata_experimental_writer.points import MORTON_CODE_2D_COLUMN, write_morton_points_parquet +from spatialdata_experimental_writer.zarr import list_points_keys + + +def _write_points_element( + zarr_root: Path, + key: str, + *, + feature_key: str = "feature_name", + rows: int = 200, +) -> None: + element_dir = zarr_root / "points" / key + parquet_path = element_dir / "points.parquet" + element_dir.mkdir(parents=True) + rng = pd.Series(range(rows)) + table = pa.Table.from_pandas( + pd.DataFrame( + { + "x": (rng.astype("float64") % 100).tolist(), + "y": ((rng * 3).astype("float64") % 100).tolist(), + "feature_name": (["gene_a", "gene_b", "gene_c"] * rows)[:rows], + } + ), + preserve_index=False, + ) + pq.write_table(table, parquet_path) + element_dir.joinpath("zarr.json").write_text( + json.dumps( + { + "attributes": { + "encoding-type": "ngff:points", + "axes": ["x", "y"], + "spatialdata_attrs": { + "feature_key": feature_key, + "version": "0.2", + }, + }, + "zarr_format": 3, + "node_type": "group", + } + ) + ) + zarr_root.joinpath("zarr.json").write_text( + json.dumps({"zarr_format": 3, "node_type": "group"}) + ) + + +def test_morton_parquet_does_not_persist_uint_columns(tmp_path: Path) -> None: + df = pd.DataFrame( + { + "x": [0.0, 10.0, 5.0, 2.0, 8.0], + "y": [3.0, 4.0, 20.0, 0.0, 9.0], + "feature_name": ["b", "a", "b", "c", "a"], + } + ) + output = tmp_path / "points.parquet" + write_morton_points_parquet(df, output, feature_key="feature_name", row_group_size=2) + columns = pq.ParquetFile(output).schema_arrow.names + assert "x_uint" not in columns + assert "y_uint" not in columns + assert MORTON_CODE_2D_COLUMN in columns + assert "feature_name_codes" in columns + + +def test_morton_points_from_zarr_defaults_to_canonical_path(tmp_path: Path) -> None: + zarr_root = tmp_path / "store.zarr" + _write_points_element(zarr_root, "transcripts") + canonical = zarr_root / "points" / "transcripts" / "points.parquet" + + subprocess.run( + [ + sys.executable, + "-m", + "spatialdata_experimental_writer.cli", + "morton-points-from-zarr", + str(zarr_root), + "--points-key", + "transcripts", + "--row-group-size", + "50", + ], + check=True, + cwd=Path(__file__).resolve().parents[1], + ) + + assert canonical.is_file() + columns = pq.ParquetFile(canonical).schema_arrow.names + assert MORTON_CODE_2D_COLUMN in columns + assert "feature_name_codes" in columns + + +def test_write_index_permutations_writes_manifest(tmp_path: Path) -> None: + source = tmp_path / "source.zarr" + dest = tmp_path / "dest.zarr" + _write_points_element(source, "transcripts", rows=120) + + manifest = write_index_permutations( + source, + dest, + points_key="transcripts", + row_group_size=40, + conditions=tuple( + condition + for condition in __import__( + "spatialdata_experimental_writer.index_permutations", + fromlist=["DEFAULT_CONDITIONS"], + ).DEFAULT_CONDITIONS + if condition.id in {"canonical", "morton"} + ), + ) + + assert (dest / "index-manifest.json").exists() + assert manifest["source_element"] == "points/transcripts" + assert (dest / "points" / "transcripts" / "points.parquet").exists() + assert (dest / "points" / "transcripts_morton" / "points.parquet").exists() + assert list_points_keys(dest) == ["transcripts", "transcripts_morton"] + consolidated = json.loads((dest / "zarr.json").read_text())["consolidated_metadata"][ + "metadata" + ] + assert "points/transcripts_morton" in consolidated + morton_columns = pq.ParquetFile( + dest / "points" / "transcripts_morton" / "points.parquet" + ).schema_arrow.names + assert MORTON_CODE_2D_COLUMN in morton_columns diff --git a/python/spatialdata-experimental-writer/tests/test_zarr.py b/python/spatialdata-experimental-writer/tests/test_zarr.py index 55584fec..829a9481 100644 --- a/python/spatialdata-experimental-writer/tests/test_zarr.py +++ b/python/spatialdata-experimental-writer/tests/test_zarr.py @@ -13,6 +13,7 @@ points_parquet_path, read_points_dataframe, read_points_element_attrs, + register_points_elements_in_consolidated_metadata, ) @@ -54,6 +55,44 @@ def _write_points_element( ) +def test_register_points_elements_in_consolidated_metadata(tmp_path: Path) -> None: + _write_points_element(tmp_path, "transcripts") + root_json = tmp_path / "zarr.json" + root_json.write_text( + json.dumps( + { + "zarr_format": 3, + "node_type": "group", + "consolidated_metadata": { + "kind": "inline", + "metadata": { + "points/transcripts": { + "attributes": { + "encoding-type": "ngff:points", + "axes": ["x", "y"], + "spatialdata_attrs": { + "feature_key": "feature_name", + "version": "0.2", + }, + }, + "node_type": "group", + "zarr_format": 3, + } + }, + }, + } + ) + ) + register_points_elements_in_consolidated_metadata( + tmp_path, + ["transcripts", "transcripts_morton"], + template_key="transcripts", + ) + metadata = json.loads(root_json.read_text())["consolidated_metadata"]["metadata"] + assert "points/transcripts_morton" in metadata + assert metadata["points/transcripts_morton"]["attributes"]["encoding-type"] == "ngff:points" + + def test_list_points_keys_and_read_element(tmp_path: Path) -> None: _write_points_element(tmp_path, "transcripts") assert list_points_keys(tmp_path) == ["transcripts"] From 35176469b008fb43e6ad5f1deb705489f54df96b Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Sat, 20 Jun 2026 09:18:33 +0100 Subject: [PATCH 07/38] Refactor, Implement Points Render Resource and Loader Functionality and add loading debug info - Added detailed documentation for the Points Render Resource, including its structure and usage. - Introduced `PointsLoader` capabilities and encoding strategies for handling point data. - Implemented `createMortonTiledPointsLoader` and `createPreloadedColumnarPointsLoader` functions for efficient data loading. - Developed a new `PointsLayer` class to manage rendering strategies and integrate with deck.gl. - Added support for tile debugging and enhanced point visualization features. - Created tests for the new points loader functionality to ensure reliability and performance. - Updated package exports to include new points-related modules and types. --- CONTEXT.md | 12 + docs/adr/0003-points-render-resource.md | 72 +++++ packages/core/src/index.ts | 14 + packages/core/src/pointsLoader.ts | 178 ++++++++++ packages/core/tests/pointsLoader.spec.ts | 28 ++ packages/layers/package.json | 1 + packages/layers/src/PointsLayer.ts | 94 ++++++ packages/layers/src/geoArrowStrategies.ts | 24 ++ packages/layers/src/index.ts | 42 +++ packages/layers/src/mortonTiledStrategy.ts | 226 +++++++++++++ packages/layers/src/pointsBbox.ts | 67 ++++ packages/layers/src/pointsLoader.ts | 81 +++++ packages/layers/src/pointsLoaderAdapter.ts | 46 +++ packages/layers/src/pointsRenderStrategies.ts | 21 ++ packages/layers/src/pointsScatterLayer.ts | 92 ++++++ packages/layers/src/pointsTileDebug.ts | 269 +++++++++++++++ .../layers/src/pointsTileLoadCallbacks.ts | 23 ++ packages/layers/src/pointsTiledDebugHooks.ts | 62 ++++ .../layers/src/preloadedScatterStrategy.ts | 47 +++ .../tests/pointsRenderStrategies.spec.ts | 35 ++ packages/layers/tests/pointsTileDebug.spec.ts | 58 ++++ packages/layers/vite.config.ts | 9 +- .../src/SpatialCanvas/PointsStylePanel.tsx | 24 ++ packages/vis/src/SpatialCanvas/index.tsx | 2 + .../src/SpatialCanvas/pointsTileProgress.ts | 18 +- .../SpatialCanvas/renderers/pointsRenderer.ts | 306 +++--------------- .../resolvePointsRenderResource.ts | 59 ++++ packages/vis/src/SpatialCanvas/types.ts | 2 + .../vis/src/SpatialCanvas/useLayerData.ts | 79 ++++- packages/vis/tests/pointsRenderer.spec.ts | 2 +- packages/vis/tests/pointsTileProgress.spec.ts | 16 +- .../tests/resolvePointsRenderResource.spec.ts | 24 ++ pnpm-lock.yaml | 3 + 33 files changed, 1752 insertions(+), 284 deletions(-) create mode 100644 docs/adr/0003-points-render-resource.md create mode 100644 packages/core/src/pointsLoader.ts create mode 100644 packages/core/tests/pointsLoader.spec.ts create mode 100644 packages/layers/src/PointsLayer.ts create mode 100644 packages/layers/src/geoArrowStrategies.ts create mode 100644 packages/layers/src/mortonTiledStrategy.ts create mode 100644 packages/layers/src/pointsBbox.ts create mode 100644 packages/layers/src/pointsLoader.ts create mode 100644 packages/layers/src/pointsLoaderAdapter.ts create mode 100644 packages/layers/src/pointsRenderStrategies.ts create mode 100644 packages/layers/src/pointsScatterLayer.ts create mode 100644 packages/layers/src/pointsTileDebug.ts create mode 100644 packages/layers/src/pointsTileLoadCallbacks.ts create mode 100644 packages/layers/src/pointsTiledDebugHooks.ts create mode 100644 packages/layers/src/preloadedScatterStrategy.ts create mode 100644 packages/layers/tests/pointsRenderStrategies.spec.ts create mode 100644 packages/layers/tests/pointsTileDebug.spec.ts create mode 100644 packages/vis/src/SpatialCanvas/resolvePointsRenderResource.ts create mode 100644 packages/vis/tests/resolvePointsRenderResource.spec.ts diff --git a/CONTEXT.md b/CONTEXT.md index 9f9f37ee..2b1570b2 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -40,6 +40,18 @@ _Avoid_: serializable prop, stack entry prop A small MDV/control-layer UI area that edits observable state directly while passing plain values through renderer and third-party boundaries. _Avoid_: MobX renderer contract +**Points Render Resource**: +The **Resource Resolver** output for a points **Spatial Entry**: a bundle `{ element, loader }` pairing the canonical `PointsElement` with a frozen **`PointsLoader`** facet. +_Avoid_: treating `PointsLoader` alone as the full render resource, or storing the loader on/mutating the element + +**PointsLoader**: +The loader facet of a **Points Render Resource**: encoding capabilities plus a fetch API (`loadInBounds`, optional `loadAll`). Built by `@spatialdata/core` store-I/O factories; consumed by `@spatialdata/layers` render strategies — not by calling `PointsElement` methods directly from deck code. +_Avoid_: conflating with Viv/image `loader` when discussing SpatialData element identity + +**Points Encoding**: +The render-time points layout selected after resolver probing, e.g. `preloaded-columnar`, `morton-tiled`, or future `geoarrow-*` kinds. Distinct from persisted Parquet layout described in ADR 0002. +_Avoid_: `experimentalOptimizations` as a synonym for encoding kind + ## Relationships - A **Render Stack** contains zero or more ordered **Stack Entries**. diff --git a/docs/adr/0003-points-render-resource.md b/docs/adr/0003-points-render-resource.md new file mode 100644 index 00000000..d0d161a9 --- /dev/null +++ b/docs/adr/0003-points-render-resource.md @@ -0,0 +1,72 @@ +# Points Render Resource + +ADR 0002 describes persisted Morton Parquet artifacts and bounded loading APIs on +`PointsElement`. This ADR describes the **render-time** boundary between store +I/O, the Resource Resolver, and the deck.gl `PointsLayer` composite. + +## Decision + +- A points **Spatial Entry** (`PointsElement`) remains the canonical spatial + identity handle. Deck layers stay associated with that element for picks, + tooltips, and Render Stack `elementKey`. +- The **Resource Resolver** (today `resolvePointsRenderResource()` in + `@spatialdata/vis`) probes once and returns a **Points Render Resource** + bundle `{ element, loader }` with **frozen** encoding capabilities. +- **`PointsLoader`** is the loader facet only: encoding kind, batch format, + bounds, and fetch methods. Render strategies call `loader.loadInBounds()` — + not `element.loadPointsInBounds()` directly from `@spatialdata/layers`. +- **`PointsLayer`** (`@spatialdata/layers` `CompositeLayer`) takes + `resource: PointsRenderResource` plus cosmetic props. It delegates to + encoding-specific render strategies selected by `loader.capabilities.kind`. +- **Store I/O loader factories** live in `@spatialdata/core` and close over + `PointsElement`. **Render strategies** and tile-debug overlay logic live in + `@spatialdata/layers`. The vis resolver associates element + loader. + +## Encoding selection (v1) + +| Condition | Encoding kind | Strategy | +|-----------|---------------|----------| +| Full table preloaded in resolver cache | `preloaded-columnar` | `ScatterplotLayer` | +| Morton metadata with row-group range reads + bounds | `morton-tiled` | `TileLayer` + per-tile scatter | +| Future GeoArrow batch from core | `geoarrow-binary` | stub → `GeoArrowScatterplotLayer` | +| Future tiled Arrow/Parquet deck path | `geoarrow-tiled` | stub | + +Resolver probing is **eager**: capabilities do not change mid-session unless +the element or resolver cache inputs change. + +## GeoArrow boundary + +- **Core** may later expose deck-free Apache Arrow `RecordBatch` batches from + Parquet row groups (x/y/z columns or geometry). +- **Layers** owns [deck.gl-geoarrow](https://github.com/geoarrow/deck.gl-geoarrow) + integration: GeoArrow geometry shaping and `GeoArrowScatterplotLayer` / + future tiled deck paths. +- Core must not import deck.gl or `@geoarrow/deck.gl-geoarrow`. + +## Batch contract + +`PointsBatch` is a tagged union: + +- `columnar-ndarray` — v1 Morton and preloaded paths +- `arrow-record-batch` — reserved for GeoArrow strategies + +## Tile debug overlay + +When `showTileDebugOverlay` is enabled on a tiled encoding, the morton strategy +emits a pickable `PolygonLayer` sublayer with per-tile status (pending, loading, +loaded, empty, error, aborted). This is cosmetic for tile fetching and must not +appear in `TileLayer.updateTriggers.getTileData`. + +## Relationship to ADR 0002 + +- ADR 0002: persisted artifacts and `PointsElement.loadPointsInBounds()` API. +- ADR 0003: render-time bundle, strategy registry, and deck composite ownership. + +## Consequences + +- Swapping encodings or deck.gl parquet layers requires new loader factories + and/or strategies — not changes to `PointsLayer` public props. +- `PointsElement` does not grow a mutable `renderResource` attachment; the + resolver cache holds stable bundle references per element key. +- Image precedent: `ImageElement` + Viv loader built in vis; points precedent: + `PointsElement` + `PointsLoader` built in vis, rendered by `PointsLayer`. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index e4c412ee..a8fae35d 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -10,6 +10,20 @@ export * from './store/index.js'; export * from './models/index.js'; export * from './spatialViewFit.js'; export * from './pointsTiling.js'; +export { + createMortonTiledPointsLoader, + createPointsLoaderForElement, + createPreloadedColumnarPointsLoader, + resolvePointsEncoding, + type CorePointsLoader, + type ColumnarNdarrayPointsBatch, + type PreloadedColumnarInput, + type PointsBatch, + type PointsBatchFormat, + type PointsEncodingKind, + type PointsLoadInBoundsOptions, + type PointsLoaderCapabilities, +} from './pointsLoader.js'; export * from './shapes.js'; export { inferShapesGeometryKindFromParquet, diff --git a/packages/core/src/pointsLoader.ts b/packages/core/src/pointsLoader.ts new file mode 100644 index 00000000..bcf9a998 --- /dev/null +++ b/packages/core/src/pointsLoader.ts @@ -0,0 +1,178 @@ +import type { PointsElement } from './models/index.js'; +import type { + PointsInBoundsResult, + PointsTilingMetadata, + SpatialBounds, +} from './pointsTiling.js'; + +export type PointsEncodingKind = + | 'preloaded-columnar' + | 'morton-tiled' + | 'geoarrow-binary' + | 'geoarrow-tiled'; + +export type PointsBatchFormat = 'columnar-ndarray' | 'arrow-record-batch'; + +export interface PointsLoaderCapabilities { + kind: PointsEncodingKind; + batchFormat: PointsBatchFormat; + bounds?: SpatialBounds; + supportsViewportTiles: boolean; + supportsFeatureCodes?: boolean; +} + +export interface ColumnarNdarrayPointsBatch { + format: 'columnar-ndarray'; + data: ArrayLike[]; + shape: number[]; + bounds?: SpatialBounds; + loadMode?: string; + pointCount?: number; +} + +export type PointsBatch = ColumnarNdarrayPointsBatch; + +export interface PointsLoadInBoundsOptions { + bounds: SpatialBounds; + featureCodes?: readonly number[]; + signal?: AbortSignal; +} + +export interface CorePointsLoader { + readonly capabilities: PointsLoaderCapabilities; + loadInBounds(options: PointsLoadInBoundsOptions): Promise; + loadAll?(options?: { signal?: AbortSignal }): Promise; +} + +export interface PreloadedColumnarInput { + shape: number[]; + data: ArrayLike[]; +} + +export function resolvePointsEncoding( + preloaded: PreloadedColumnarInput | null | undefined, + metadata: PointsTilingMetadata | null | undefined, + wantsOptimized: boolean +): PointsEncodingKind { + if (preloaded) { + return 'preloaded-columnar'; + } + if (wantsOptimized && metadata?.supportsRowGroupRangeReads && metadata.bounds) { + return 'morton-tiled'; + } + return 'preloaded-columnar'; +} + +function toColumnarBatch( + result: PointsInBoundsResult | PreloadedColumnarInput, + overrides?: Partial +): ColumnarNdarrayPointsBatch { + const shape = result.shape ?? []; + const pointCount = shape[0] ?? 0; + return { + format: 'columnar-ndarray', + data: result.data, + shape, + bounds: 'bounds' in result ? result.bounds : overrides?.bounds, + loadMode: 'loadMode' in result ? result.loadMode : overrides?.loadMode, + pointCount, + ...overrides, + }; +} + +export function createMortonTiledPointsLoader( + element: PointsElement, + metadata: PointsTilingMetadata +): CorePointsLoader { + const capabilities: PointsLoaderCapabilities = { + kind: 'morton-tiled', + batchFormat: 'columnar-ndarray', + bounds: metadata.bounds, + supportsViewportTiles: true, + supportsFeatureCodes: Boolean(metadata.featureKey), + }; + + return { + capabilities, + async loadInBounds(options: PointsLoadInBoundsOptions): Promise { + const result = await element.loadPointsInBounds(options); + return toColumnarBatch(result); + }, + }; +} + +export function createPreloadedColumnarPointsLoader( + element: PointsElement, + preloaded: PreloadedColumnarInput +): CorePointsLoader { + const batch = toColumnarBatch(preloaded, { loadMode: 'full-filter' }); + const capabilities: PointsLoaderCapabilities = { + kind: 'preloaded-columnar', + batchFormat: 'columnar-ndarray', + bounds: inferBoundsFromColumnar(preloaded), + supportsViewportTiles: false, + supportsFeatureCodes: true, + }; + + return { + capabilities, + async loadInBounds(options: PointsLoadInBoundsOptions): Promise { + void element; + void options; + return batch; + }, + async loadAll() { + return batch; + }, + }; +} + +function inferBoundsFromColumnar(preloaded: PreloadedColumnarInput) { + const xs = preloaded.data[0]; + const ys = preloaded.data[1]; + if (!xs || !ys || preloaded.shape[0] === 0) { + return undefined; + } + let minX = Number.POSITIVE_INFINITY; + let maxX = Number.NEGATIVE_INFINITY; + let minY = Number.POSITIVE_INFINITY; + let maxY = Number.NEGATIVE_INFINITY; + const count = preloaded.shape[0]; + for (let index = 0; index < count; index += 1) { + const x = xs[index]; + const y = ys[index]; + if (x < minX) minX = x; + if (x > maxX) maxX = x; + if (y < minY) minY = y; + if (y > maxY) maxY = y; + } + if (!Number.isFinite(minX) || !Number.isFinite(minY)) { + return undefined; + } + return { minX, minY, maxX, maxY }; +} + +export function createPointsLoaderForElement( + element: PointsElement, + options: { + preloaded?: PreloadedColumnarInput | null; + tilingMetadata?: PointsTilingMetadata | null; + wantsOptimized: boolean; + } +): CorePointsLoader | null { + const encoding = resolvePointsEncoding( + options.preloaded, + options.tilingMetadata, + options.wantsOptimized + ); + + if (encoding === 'morton-tiled' && options.tilingMetadata?.bounds) { + return createMortonTiledPointsLoader(element, options.tilingMetadata); + } + + if (options.preloaded) { + return createPreloadedColumnarPointsLoader(element, options.preloaded); + } + + return null; +} diff --git a/packages/core/tests/pointsLoader.spec.ts b/packages/core/tests/pointsLoader.spec.ts new file mode 100644 index 00000000..9c7bc4cc --- /dev/null +++ b/packages/core/tests/pointsLoader.spec.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest'; + +import { resolvePointsEncoding } from '../src/pointsLoader.js'; + +describe('resolvePointsEncoding', () => { + it('prefers preloaded data when present', () => { + expect( + resolvePointsEncoding({ shape: [1], data: [[0], [0]] }, null, true) + ).toBe('preloaded-columnar'); + }); + + it('selects morton tiling when metadata supports row-group reads', () => { + expect( + resolvePointsEncoding(null, { + kind: 'morton-points', + parquetPath: 'points/a/points.parquet', + axisNames: ['x', 'y'], + featureCodeColumnName: 'feature_name_codes', + mortonCodeColumnName: 'morton_code_2d', + totalRows: 10, + totalRowGroups: 1, + maxRowsPerGroup: 10, + supportsRowGroupRangeReads: true, + bounds: { minX: 0, minY: 0, maxX: 10, maxY: 10 }, + }, true) + ).toBe('morton-tiled'); + }); +}); diff --git a/packages/layers/package.json b/packages/layers/package.json index de0f939f..505835ac 100644 --- a/packages/layers/package.json +++ b/packages/layers/package.json @@ -29,6 +29,7 @@ "@deck.gl/core": "catalog:", "@hms-dbmi/viv": "catalog:", "@math.gl/core": "catalog:", + "@spatialdata/core": "workspace:*", "zod": "catalog:" }, "devDependencies": { diff --git a/packages/layers/src/PointsLayer.ts b/packages/layers/src/PointsLayer.ts new file mode 100644 index 00000000..75b604e2 --- /dev/null +++ b/packages/layers/src/PointsLayer.ts @@ -0,0 +1,94 @@ +import type { Matrix4 } from '@math.gl/core'; +import type { UpdateParameters } from '@deck.gl/core'; +import { CompositeLayer } from 'deck.gl'; +import type { Layer, LayersList } from 'deck.gl'; +import type { PointsRenderResource } from './pointsLoader.js'; +import type { PointsTileLoadCallbacks } from './pointsTileLoadCallbacks.js'; +import type { ColumnarNdarrayPointsBatch } from './pointsLoader.js'; +import type { PointsTileDebugEntry } from './pointsTileDebug.js'; +import { resolvePointsRenderStrategy } from './pointsRenderStrategies.js'; +import { + DEFAULT_POINT_RADIUS_MAX_PIXELS, + DEFAULT_POINT_RADIUS_MIN_PIXELS, + DEFAULT_POINT_SIZE, +} from './pointsScatterLayer.js'; + +export interface PointsLayerProps { + id: string; + resource: PointsRenderResource; + visible?: boolean; + opacity?: number; + modelMatrix: Matrix4; + pointSize?: number; + pointRadiusMinPixels?: number; + pointRadiusMaxPixels?: number; + pointMinSizeScale?: number; + viewZoom?: number | null; + color?: [number, number, number, number]; + featureCodes?: readonly number[]; + showTileDebugOverlay?: boolean; + tileLoadCallbacks?: PointsTileLoadCallbacks; + use3d?: boolean; +} + +interface PointsLayerState { + preloadedBatch?: ColumnarNdarrayPointsBatch; + tileDebugEntries?: PointsTileDebugEntry[]; +} + +export class PointsLayer extends CompositeLayer { + static layerName = 'PointsLayer'; + + static defaultProps = { + visible: true, + opacity: 1, + pointSize: DEFAULT_POINT_SIZE, + pointRadiusMinPixels: DEFAULT_POINT_RADIUS_MIN_PIXELS, + pointRadiusMaxPixels: DEFAULT_POINT_RADIUS_MAX_PIXELS, + showTileDebugOverlay: false, + } satisfies Partial; + + initializeState(): void { + this.state = {}; + void this.ensurePreloadedBatch(); + } + + updateState(params: UpdateParameters): void { + const { props, oldProps } = params; + if ( + props.resource.loader !== oldProps.resource.loader || + props.resource.element !== oldProps.resource.element + ) { + this.setState({ preloadedBatch: undefined, tileDebugEntries: [] }); + void this.ensurePreloadedBatch(); + } + } + + private async ensurePreloadedBatch(): Promise { + const { resource } = this.props; + if (resource.loader.capabilities.kind !== 'preloaded-columnar') { + return; + } + const existing = (this.state as PointsLayerState).preloadedBatch; + if (existing) { + return; + } + const batch = await resource.loader.loadAll?.(); + if (batch?.format === 'columnar-ndarray') { + this.setState({ preloadedBatch: batch }); + } + } + + /** Public wrapper for strategy modules outside this class. */ + subLayerProps

>(props: P & { id: string }): P { + return this.getSubLayerProps(props); + } + + renderLayers(): Layer | null | LayersList { + const { visible = true, resource } = this.props; + if (!visible || !resource?.loader) { + return null; + } + return resolvePointsRenderStrategy(resource.loader).renderLayers(this); + } +} diff --git a/packages/layers/src/geoArrowStrategies.ts b/packages/layers/src/geoArrowStrategies.ts new file mode 100644 index 00000000..a2b68f97 --- /dev/null +++ b/packages/layers/src/geoArrowStrategies.ts @@ -0,0 +1,24 @@ +import type { Layer, LayersList } from 'deck.gl'; +import type { PointsLayer } from './PointsLayer.js'; +import type { PointsRenderStrategy } from './pointsRenderStrategies.js'; + +export const geoArrowBinaryStrategy: PointsRenderStrategy = { + renderLayers(): Layer | null | LayersList { + return null; + }, +}; + +export const geoArrowTiledStrategy: PointsRenderStrategy = { + renderLayers(): Layer | null | LayersList { + return null; + }, +}; + +export const unsupportedPointsStrategy: PointsRenderStrategy = { + renderLayers(layer): Layer | null | LayersList { + console.debug( + `[PointsLayer] Unsupported points encoding for element "${layer.props.resource.element.key}"` + ); + return null; + }, +}; diff --git a/packages/layers/src/index.ts b/packages/layers/src/index.ts index e7ed66ff..8a65a137 100644 --- a/packages/layers/src/index.ts +++ b/packages/layers/src/index.ts @@ -68,3 +68,45 @@ export type { RenderStackSpatialElementType, RenderStackSpatialEntry, } from './renderStack'; +export { PointsLayer } from './PointsLayer'; +export type { PointsLayerProps } from './PointsLayer'; +export { + columnarBatchFromPointData, + pointDataFromColumnarBatch, + type ArrowRecordBatchPointsBatch, + type ColumnarNdarrayPointsBatch, + type PointData, + type PointsBatch, + type PointsBatchFormat, + type PointsEncodingKind, + type PointsLoadInBoundsOptions, + type PointsLoader, + type PointsLoaderCapabilities, + type PointsRenderResource, +} from './pointsLoader.js'; +export { + createPointsRenderResource, + coreLoaderToPointsLoader, +} from './pointsLoaderAdapter.js'; +export { + DEFAULT_POINT_RADIUS_MAX_PIXELS, + DEFAULT_POINT_RADIUS_MIN_PIXELS, + DEFAULT_POINT_SIZE, + MIN_POINT_SIZE_SCALE, + POINT_SIZE_ZOOM_REFERENCE, + zoomScaledPointSize, +} from './pointsScatterLayer.js'; +export type { PointsTileHandle, PointsTileLoadResult, PointsTileLoadCallbacks } from './pointsTileLoadCallbacks.js'; +export { + POINTS_TILE_DEBUG_PICK_KIND, + formatPointsTileDebugTooltip, + isPointsTileDebugPickObject, + reduceTileDebugEntries, + tileDebugEntriesSignature, + tileDebugStatusFillColor, + tileDebugStatusLineColor, + type PointsTileDebugEntry, + type PointsTileDebugPickObject, + type PointsTileLoadProgress, + type PointsTileStatus, +} from './pointsTileDebug.js'; diff --git a/packages/layers/src/mortonTiledStrategy.ts b/packages/layers/src/mortonTiledStrategy.ts new file mode 100644 index 00000000..a48fec89 --- /dev/null +++ b/packages/layers/src/mortonTiledStrategy.ts @@ -0,0 +1,226 @@ +import { COORDINATE_SYSTEM } from '@deck.gl/core'; +import { PolygonLayer, TileLayer } from 'deck.gl'; +import type { Layer, LayersList } from 'deck.gl'; +import type { PointsLayer } from './PointsLayer.js'; +import { + boundsFromTileBbox, + intersectBounds, + isPointTileBbox, + scatterBoundsFromTileBbox, + tileHandleFromDeckTile, +} from './pointsBbox.js'; +import type { ColumnarNdarrayPointsBatch } from './pointsLoader.js'; +import { + DEFAULT_POINT_RADIUS_MAX_PIXELS, + DEFAULT_POINT_RADIUS_MIN_PIXELS, + DEFAULT_POINT_SIZE, + renderColumnarScatterLayer, +} from './pointsScatterLayer.js'; +import type { PointsRenderStrategy } from './pointsRenderStrategies.js'; +import { createTiledPointsDebugHooks } from './pointsTiledDebugHooks.js'; +import { + POINTS_TILE_DEBUG_PICK_KIND, + pointsTileDebugPolygonData, + tileDebugStatusFillColor, + tileDebugStatusLineColor, +} from './pointsTileDebug.js'; + +function isAbortError(error: unknown) { + return error instanceof DOMException && error.name === 'AbortError'; +} + +function isColumnarBatch(value: unknown): value is ColumnarNdarrayPointsBatch { + return ( + !!value && + typeof value === 'object' && + (value as ColumnarNdarrayPointsBatch).format === 'columnar-ndarray' + ); +} + +export const mortonTiledStrategy: PointsRenderStrategy = { + renderLayers(layer): Layer | null | LayersList { + const { + resource, + featureCodes, + showTileDebugOverlay, + tileLoadCallbacks, + opacity = 1, + visible = true, + pointSize = DEFAULT_POINT_SIZE, + pointRadiusMinPixels, + pointRadiusMaxPixels, + color = [255, 100, 100, 200], + use3d, + } = layer.props; + + const localBounds = resource.loader.capabilities.bounds; + if (!localBounds) { + return null; + } + + const debugHooks = createTiledPointsDebugHooks(layer, tileLoadCallbacks); + const scatterStyleProps = { + color, + pointSize, + pointRadiusMinPixels, + pointRadiusMaxPixels, + opacity, + modelMatrix: layer.props.modelMatrix, + use3d, + }; + + const layers: LayersList = [ + new TileLayer( + layer.subLayerProps({ + id: 'tiles', + coordinateSystem: COORDINATE_SYSTEM.CARTESIAN, + modelMatrix: layer.props.modelMatrix, + extent: [localBounds.minX, localBounds.minY, localBounds.maxX, localBounds.maxY], + opacity, + visible, + tileSize: 512, + minZoom: -1, + maxZoom: -1, + refinementStrategy: 'best-available', + updateTriggers: { + getTileData: [resource.element.key, featureCodes], + renderSubLayers: [ + pointSize, + pointRadiusMinPixels, + pointRadiusMaxPixels, + color, + opacity, + layer.props.modelMatrix, + use3d, + ], + }, + onViewportLoad(tiles: Array<{ index?: { x: number; y: number; z: number }; id?: string; bbox?: unknown }> | null) { + const handles = (tiles ?? []) + .map((tile: { index?: { x: number; y: number; z: number }; id?: string; bbox?: unknown }) => + tileHandleFromDeckTile(tile) + ) + .filter( + (handle): handle is NonNullable> => + handle != null + ); + debugHooks.onViewportTilesRequested(handles); + }, + async getTileData(tileProps: { index?: { x: number; y: number; z: number }; id?: string; bbox?: unknown; signal?: AbortSignal }) { + const tile = tileHandleFromDeckTile(tileProps); + if (!tile || !isPointTileBbox(tileProps.bbox)) { + return null; + } + debugHooks.onTileLoadStart(tile); + const rawBounds = boundsFromTileBbox(tile.bbox); + const bounds = intersectBounds(rawBounds, localBounds); + if (!bounds) { + debugHooks.onTileLoadEnd( + tile, + { success: true, clippedBounds: null, pointCount: 0, loadMode: 'clipped' }, + rawBounds + ); + return null; + } + try { + const batch = await resource.loader.loadInBounds({ + bounds, + featureCodes, + signal: tileProps.signal, + }); + if (!batch || !isColumnarBatch(batch)) { + debugHooks.onTileLoadEnd( + tile, + { success: true, clippedBounds: bounds, pointCount: 0 }, + rawBounds + ); + return null; + } + debugHooks.onTileLoadEnd( + tile, + { + success: true, + clippedBounds: bounds, + pointCount: batch.pointCount ?? batch.shape[0] ?? 0, + loadMode: batch.loadMode, + }, + rawBounds + ); + return batch; + } catch (error) { + const aborted = Boolean(tileProps.signal?.aborted) || isAbortError(error); + debugHooks.onTileLoadEnd( + tile, + { + success: false, + aborted, + clippedBounds: bounds, + errorMessage: aborted ? 'aborted' : String(error), + }, + rawBounds + ); + if (aborted) { + return null; + } + throw error; + } + }, + renderSubLayers: (props: { + id: string; + data?: ColumnarNdarrayPointsBatch | null; + tile?: { bbox?: unknown }; + }) => { + if (!props.data || !isColumnarBatch(props.data)) { + return null; + } + const tileBbox = isPointTileBbox(props.tile?.bbox) ? props.tile.bbox : null; + return renderColumnarScatterLayer(`${props.id}-scatter`, props.data, { + ...scatterStyleProps, + tileBounds: tileBbox ? scatterBoundsFromTileBbox(tileBbox) : undefined, + tileSubLayer: true, + }); + }, + }) + ), + ]; + + if (showTileDebugOverlay) { + const entries = debugHooks.getTileDebugEntries(); + const polygonData = pointsTileDebugPolygonData(entries).map(({ polygon, entry }) => ({ + polygon, + entry, + kind: POINTS_TILE_DEBUG_PICK_KIND as typeof POINTS_TILE_DEBUG_PICK_KIND, + })); + layers.push( + new PolygonLayer( + layer.subLayerProps({ + id: 'tile-debug', + coordinateSystem: COORDINATE_SYSTEM.CARTESIAN, + modelMatrix: layer.props.modelMatrix, + data: polygonData, + pickable: true, + autoHighlight: true, + highlightColor: [255, 255, 255, 120], + getPolygon: (d: { polygon: [number, number][] }) => d.polygon, + getFillColor: (d: { entry: { status: import('./pointsTileDebug.js').PointsTileStatus } }) => + tileDebugStatusFillColor(d.entry.status), + getLineColor: (d: { entry: { status: import('./pointsTileDebug.js').PointsTileStatus } }) => + tileDebugStatusLineColor(d.entry.status), + getLineWidth: 2, + lineWidthUnits: 'pixels', + filled: true, + stroked: true, + opacity: Math.min(1, opacity + 0.15), + visible, + updateTriggers: { + getFillColor: [debugHooks.getTileDebugSignature()], + getLineColor: [debugHooks.getTileDebugSignature()], + getPolygon: [debugHooks.getTileDebugSignature()], + }, + }) + ) + ); + } + + return layers; + }, +}; diff --git a/packages/layers/src/pointsBbox.ts b/packages/layers/src/pointsBbox.ts new file mode 100644 index 00000000..3cad54ff --- /dev/null +++ b/packages/layers/src/pointsBbox.ts @@ -0,0 +1,67 @@ +import type { SpatialBounds } from '@spatialdata/core'; +import type { PointsTileHandle } from './pointsTileLoadCallbacks.js'; + +export type PointTileBbox = { + left: number; + right: number; + top: number; + bottom: number; +}; + +export function isPointTileBbox(value: unknown): value is PointTileBbox { + if (!value || typeof value !== 'object') { + return false; + } + const candidate = value as Record; + return ( + typeof candidate.left === 'number' && + typeof candidate.right === 'number' && + typeof candidate.top === 'number' && + typeof candidate.bottom === 'number' + ); +} + +export function intersectBounds( + query: SpatialBounds, + clip: SpatialBounds +): SpatialBounds | null { + const minX = Math.max(query.minX, clip.minX); + const maxX = Math.min(query.maxX, clip.maxX); + const minY = Math.max(query.minY, clip.minY); + const maxY = Math.min(query.maxY, clip.maxY); + if (minX > maxX || minY > maxY) { + return null; + } + return { minX, minY, maxX, maxY }; +} + +export function boundsFromTileBbox(bbox: PointTileBbox): SpatialBounds { + return { + minX: Math.min(bbox.left, bbox.right), + maxX: Math.max(bbox.left, bbox.right), + minY: Math.min(bbox.top, bbox.bottom), + maxY: Math.max(bbox.top, bbox.bottom), + }; +} + +export function scatterBoundsFromTileBbox( + bbox: PointTileBbox +): [number, number, number, number] { + return [bbox.left, bbox.top, bbox.right, bbox.bottom]; +} + +export function tileHandleFromDeckTile(tile: { + index?: { x: number; y: number; z: number }; + id?: string; + bbox?: unknown; +}): PointsTileHandle | null { + if (!tile.index || !isPointTileBbox(tile.bbox)) { + return null; + } + const { x, y, z } = tile.index; + return { + tileId: tile.id ?? `${x}-${y}-${z}`, + index: { x, y, z }, + bbox: tile.bbox, + }; +} diff --git a/packages/layers/src/pointsLoader.ts b/packages/layers/src/pointsLoader.ts new file mode 100644 index 00000000..be6d8b65 --- /dev/null +++ b/packages/layers/src/pointsLoader.ts @@ -0,0 +1,81 @@ +import type { SpatialBounds } from '@spatialdata/core'; +import type { PointsElement } from '@spatialdata/core'; + +export type PointsEncodingKind = + | 'preloaded-columnar' + | 'morton-tiled' + | 'geoarrow-binary' + | 'geoarrow-tiled'; + +export type PointsBatchFormat = 'columnar-ndarray' | 'arrow-record-batch'; + +export interface PointsLoaderCapabilities { + kind: PointsEncodingKind; + batchFormat: PointsBatchFormat; + bounds?: SpatialBounds; + supportsViewportTiles: boolean; + supportsFeatureCodes?: boolean; +} + +export interface ColumnarNdarrayPointsBatch { + format: 'columnar-ndarray'; + data: ArrayLike[]; + shape: number[]; + bounds?: SpatialBounds; + loadMode?: string; + pointCount?: number; +} + +/** Placeholder for future GeoArrow strategies. */ +export interface ArrowRecordBatchPointsBatch { + format: 'arrow-record-batch'; + batch: unknown; + bounds?: SpatialBounds; + loadMode?: string; + pointCount?: number; +} + +export type PointsBatch = ColumnarNdarrayPointsBatch | ArrowRecordBatchPointsBatch; + +export interface PointsLoadInBoundsOptions { + bounds: SpatialBounds; + featureCodes?: readonly number[]; + signal?: AbortSignal; +} + +export interface PointsLoader { + readonly capabilities: PointsLoaderCapabilities; + loadInBounds(options: PointsLoadInBoundsOptions): Promise; + loadAll?(options?: { signal?: AbortSignal }): Promise; +} + +export interface PointsRenderResource { + element: PointsElement; + loader: PointsLoader; +} + +export interface PointData { + shape: number[]; + data: ArrayLike[]; +} + +export function columnarBatchFromPointData( + data: PointData, + options?: { loadMode?: string; bounds?: SpatialBounds } +): ColumnarNdarrayPointsBatch { + return { + format: 'columnar-ndarray', + data: data.data, + shape: data.shape, + bounds: options?.bounds, + loadMode: options?.loadMode, + pointCount: data.shape[0] ?? 0, + }; +} + +export function pointDataFromColumnarBatch(batch: ColumnarNdarrayPointsBatch): PointData { + return { + data: batch.data, + shape: batch.shape, + }; +} diff --git a/packages/layers/src/pointsLoaderAdapter.ts b/packages/layers/src/pointsLoaderAdapter.ts new file mode 100644 index 00000000..d4b9a238 --- /dev/null +++ b/packages/layers/src/pointsLoaderAdapter.ts @@ -0,0 +1,46 @@ +import type { PointsElement } from '@spatialdata/core'; +import type { + PointsBatch, + PointsLoader, + PointsLoaderCapabilities, + PointsLoadInBoundsOptions, + PointsRenderResource, +} from './pointsLoader.js'; + +type CorePointsLoader = { + readonly capabilities: PointsLoaderCapabilities; + loadInBounds(options: PointsLoadInBoundsOptions): Promise; + loadAll?(options?: { signal?: AbortSignal }): Promise; +}; + +export type { + ArrowRecordBatchPointsBatch, + ColumnarNdarrayPointsBatch, + PointData, + PointsBatch, + PointsBatchFormat, + PointsEncodingKind, + PointsLoadInBoundsOptions, + PointsLoader, + PointsLoaderCapabilities, + PointsRenderResource, +} from './pointsLoader.js'; + +export { + columnarBatchFromPointData, + pointDataFromColumnarBatch, +} from './pointsLoader.js'; + +export function coreLoaderToPointsLoader(loader: CorePointsLoader): PointsLoader { + return loader; +} + +export function createPointsRenderResource( + element: PointsElement, + loader: CorePointsLoader +): PointsRenderResource { + return { + element, + loader: coreLoaderToPointsLoader(loader), + }; +} diff --git a/packages/layers/src/pointsRenderStrategies.ts b/packages/layers/src/pointsRenderStrategies.ts new file mode 100644 index 00000000..0dba7a61 --- /dev/null +++ b/packages/layers/src/pointsRenderStrategies.ts @@ -0,0 +1,21 @@ +import type { Layer, LayersList } from 'deck.gl'; +import type { PointsEncodingKind, PointsLoader } from './pointsLoader.js'; +import type { PointsLayer } from './PointsLayer.js'; +import { geoArrowBinaryStrategy, geoArrowTiledStrategy, unsupportedPointsStrategy } from './geoArrowStrategies.js'; +import { mortonTiledStrategy } from './mortonTiledStrategy.js'; +import { preloadedScatterStrategy } from './preloadedScatterStrategy.js'; + +export interface PointsRenderStrategy { + renderLayers(layer: PointsLayer): Layer | null | LayersList; +} + +const STRATEGIES: Record = { + 'preloaded-columnar': preloadedScatterStrategy, + 'morton-tiled': mortonTiledStrategy, + 'geoarrow-binary': geoArrowBinaryStrategy, + 'geoarrow-tiled': geoArrowTiledStrategy, +}; + +export function resolvePointsRenderStrategy(loader: PointsLoader): PointsRenderStrategy { + return STRATEGIES[loader.capabilities.kind] ?? unsupportedPointsStrategy; +} diff --git a/packages/layers/src/pointsScatterLayer.ts b/packages/layers/src/pointsScatterLayer.ts new file mode 100644 index 00000000..b4058c92 --- /dev/null +++ b/packages/layers/src/pointsScatterLayer.ts @@ -0,0 +1,92 @@ +import type { Matrix4 } from '@math.gl/core'; +import { COORDINATE_SYSTEM } from '@deck.gl/core'; +import { ScatterplotLayer } from 'deck.gl'; +import type { ColumnarNdarrayPointsBatch } from './pointsLoader.js'; +import { pointDataFromColumnarBatch } from './pointsLoader.js'; + +/** Orthographic zoom at which configured pointSize applies at full scale. */ +export const POINT_SIZE_ZOOM_REFERENCE = 0; +/** Minimum radius multiplier when zoomed out (reduces fragment overdraw). */ +export const MIN_POINT_SIZE_SCALE = 0.15; +export const DEFAULT_POINT_SIZE = 1; +export const DEFAULT_POINT_RADIUS_MIN_PIXELS = 0.1; +export const DEFAULT_POINT_RADIUS_MAX_PIXELS = 3; + +export function zoomScaledPointSize( + pointSize: number, + zoom: number | null | undefined, + zoomReference = POINT_SIZE_ZOOM_REFERENCE, + minScale = MIN_POINT_SIZE_SCALE +): number { + if (zoom === null || zoom === undefined || !Number.isFinite(zoom)) { + return pointSize; + } + const scale = 2 ** (zoom - zoomReference); + return pointSize * Math.min(1, Math.max(minScale, scale)); +} + +export interface PointsScatterStyleProps { + color: [number, number, number, number]; + pointSize: number; + pointRadiusMinPixels?: number; + pointRadiusMaxPixels?: number; + pointMinSizeScale?: number; + viewZoom?: number | null; + opacity: number; + modelMatrix: Matrix4; + use3d?: boolean; + tileBounds?: [number, number, number, number]; + tileSubLayer?: boolean; +} + +export function renderColumnarScatterLayer( + id: string, + batch: ColumnarNdarrayPointsBatch, + props: PointsScatterStyleProps +) { + const pointData = pointDataFromColumnarBatch(batch); + const d = pointData.data; + const effectivePointSize = props.tileSubLayer + ? props.pointSize + : zoomScaledPointSize( + props.pointSize, + props.viewZoom, + POINT_SIZE_ZOOM_REFERENCE, + props.pointMinSizeScale ?? MIN_POINT_SIZE_SCALE + ); + + return new ScatterplotLayer({ + id, + coordinateSystem: COORDINATE_SYSTEM.CARTESIAN, + data: d[0], + ...(props.tileBounds ? { bounds: props.tileBounds } : {}), + getPosition: (_d, { index, target }) => [ + d[0][index], + d[1][index], + props.use3d ? d[2]?.[index] || 0 : 0, + ], + getRadius: effectivePointSize, + ...(props.tileSubLayer + ? { + radiusMinPixels: props.pointRadiusMinPixels ?? DEFAULT_POINT_RADIUS_MIN_PIXELS, + radiusMaxPixels: props.pointRadiusMaxPixels ?? DEFAULT_POINT_RADIUS_MAX_PIXELS, + } + : {}), + radiusUnits: 'pixels', + getFillColor: props.color, + opacity: props.opacity, + modelMatrix: props.modelMatrix, + pickable: true, + autoHighlight: true, + highlightColor: [255, 255, 0, 200], + updateTriggers: { + getRadius: [ + props.pointSize, + props.viewZoom, + props.pointRadiusMinPixels, + props.pointRadiusMaxPixels, + props.pointMinSizeScale, + ], + }, + }); +} diff --git a/packages/layers/src/pointsTileDebug.ts b/packages/layers/src/pointsTileDebug.ts new file mode 100644 index 00000000..68a86ce7 --- /dev/null +++ b/packages/layers/src/pointsTileDebug.ts @@ -0,0 +1,269 @@ +import type { SpatialBounds } from '@spatialdata/core'; +import type { PointsTileHandle, PointsTileLoadResult } from './pointsTileLoadCallbacks.js'; + +export type PointsTileStatus = + | 'pending' + | 'loading' + | 'loaded' + | 'empty' + | 'error' + | 'aborted'; + +export interface PointsTileLoadProgress { + inFlight: number; + loaded: number; + viewportTotal: number; +} + +export interface PointsTileDebugEntry { + tileId: string; + index: { x: number; y: number; z: number }; + bbox: SpatialBounds; + clippedBounds: SpatialBounds | null; + status: PointsTileStatus; + requestedAt: number; + startedAt?: number; + completedAt?: number; + pointCount?: number; + loadMode?: string; + errorMessage?: string; +} + +export const POINTS_TILE_DEBUG_PICK_KIND = 'spatialdata-points-tile-debug' as const; + +export interface PointsTileDebugPickObject { + kind: typeof POINTS_TILE_DEBUG_PICK_KIND; + entry: PointsTileDebugEntry; +} + +export function isPointsTileDebugPickObject( + value: unknown +): value is PointsTileDebugPickObject { + if (!value || typeof value !== 'object') { + return false; + } + const candidate = value as Partial; + return candidate.kind === POINTS_TILE_DEBUG_PICK_KIND && candidate.entry != null; +} + +export type PointsTileDebugEvent = + | { type: 'viewport'; tiles: readonly PointsTileHandle[]; at: number } + | { type: 'start'; tile: PointsTileHandle; at: number } + | { + type: 'end'; + tile: PointsTileHandle; + result: PointsTileLoadResult; + at: number; + clipBounds: SpatialBounds; + }; + +export function reduceTileDebugEntries( + previous: readonly PointsTileDebugEntry[], + event: PointsTileDebugEvent +): PointsTileDebugEntry[] { + const byId = new Map(previous.map((entry) => [entry.tileId, entry])); + + if (event.type === 'viewport') { + const next = new Map(); + for (const tile of event.tiles) { + const rawBounds = boundsFromHandle(tile); + const existing = byId.get(tile.tileId); + next.set(tile.tileId, { + tileId: tile.tileId, + index: tile.index, + bbox: rawBounds, + clippedBounds: existing?.clippedBounds ?? null, + status: + existing?.status === 'loaded' || existing?.status === 'empty' + ? existing.status + : 'pending', + requestedAt: event.at, + startedAt: existing?.startedAt, + completedAt: existing?.completedAt, + pointCount: existing?.pointCount, + loadMode: existing?.loadMode, + errorMessage: existing?.errorMessage, + }); + } + return [...next.values()]; + } + + if (event.type === 'start') { + const rawBounds = boundsFromHandle(event.tile); + const existing = byId.get(event.tile.tileId); + byId.set(event.tile.tileId, { + tileId: event.tile.tileId, + index: event.tile.index, + bbox: rawBounds, + clippedBounds: existing?.clippedBounds ?? null, + status: 'loading', + requestedAt: existing?.requestedAt ?? event.at, + startedAt: event.at, + completedAt: undefined, + pointCount: undefined, + loadMode: undefined, + errorMessage: undefined, + }); + return [...byId.values()]; + } + + const rawBounds = boundsFromHandle(event.tile); + const { result } = event; + let status: PointsTileStatus = 'error'; + if (result.aborted) { + status = 'aborted'; + } else if (result.success) { + status = (result.pointCount ?? 0) > 0 ? 'loaded' : 'empty'; + } + + byId.set(event.tile.tileId, { + tileId: event.tile.tileId, + index: event.tile.index, + bbox: rawBounds, + clippedBounds: result.clippedBounds ?? event.clipBounds, + status, + requestedAt: byId.get(event.tile.tileId)?.requestedAt ?? event.at, + startedAt: byId.get(event.tile.tileId)?.startedAt ?? event.at, + completedAt: event.at, + pointCount: result.pointCount, + loadMode: result.loadMode, + errorMessage: result.errorMessage, + }); + return [...byId.values()]; +} + +function boundsFromHandle(tile: PointsTileHandle): SpatialBounds { + const { bbox } = tile; + return { + minX: Math.min(bbox.left, bbox.right), + maxX: Math.max(bbox.left, bbox.right), + minY: Math.min(bbox.top, bbox.bottom), + maxY: Math.max(bbox.top, bbox.bottom), + }; +} + +export interface PointsTileDebugPolygonDatum { + polygon: [number, number][]; + entry: PointsTileDebugEntry; +} + +export function pointsTileDebugPolygonData( + entries: readonly PointsTileDebugEntry[] +): PointsTileDebugPolygonDatum[] { + return entries.map((entry) => { + const bounds = entry.clippedBounds ?? entry.bbox; + const { minX, minY, maxX, maxY } = bounds; + return { + entry, + polygon: [ + [minX, minY], + [maxX, minY], + [maxX, maxY], + [minX, maxY], + ], + }; + }); +} + +function formatBounds(bounds: SpatialBounds): string { + return `[${bounds.minX.toFixed(1)}, ${bounds.minY.toFixed(1)}]–[${bounds.maxX.toFixed(1)}, ${bounds.maxY.toFixed(1)}]`; +} + +function formatDuration(ms: number): string { + if (ms < 1000) { + return `${Math.round(ms)}ms`; + } + return `${(ms / 1000).toFixed(2)}s`; +} + +export function formatPointsTileDebugTooltip( + entry: PointsTileDebugEntry, + batchProgress: PointsTileLoadProgress, + now = Date.now() +): { title: string; items: Array<{ label: string; value: string }> } { + const items: Array<{ label: string; value: string }> = [ + { label: 'tile', value: entry.tileId }, + { label: 'status', value: entry.status }, + { + label: 'batch', + value: `${batchProgress.loaded}/${batchProgress.viewportTotal} (${batchProgress.inFlight} in flight)`, + }, + { label: 'index', value: `x=${entry.index.x} y=${entry.index.y} z=${entry.index.z}` }, + { label: 'bbox', value: formatBounds(entry.bbox) }, + ]; + + if (entry.clippedBounds) { + items.push({ label: 'clipped', value: formatBounds(entry.clippedBounds) }); + } + if (entry.startedAt !== undefined) { + if (entry.completedAt !== undefined) { + items.push({ + label: 'duration', + value: formatDuration(entry.completedAt - entry.startedAt), + }); + } else { + items.push({ + label: 'elapsed', + value: formatDuration(now - entry.startedAt), + }); + } + } + if (entry.pointCount !== undefined) { + items.push({ label: 'points', value: String(entry.pointCount) }); + } + if (entry.loadMode) { + items.push({ label: 'load mode', value: entry.loadMode }); + } + if (entry.errorMessage) { + items.push({ label: 'error', value: entry.errorMessage }); + } + + return { + title: `Tile ${entry.tileId}`, + items, + }; +} + +export function tileDebugStatusFillColor( + status: PointsTileStatus +): [number, number, number, number] { + switch (status) { + case 'pending': + return [120, 120, 120, 30]; + case 'loading': + return [255, 180, 0, 80]; + case 'loaded': + return [80, 200, 80, 25]; + case 'empty': + return [120, 160, 200, 35]; + case 'error': + return [220, 60, 60, 70]; + case 'aborted': + return [180, 80, 80, 45]; + } +} + +export function tileDebugStatusLineColor( + status: PointsTileStatus +): [number, number, number, number] { + switch (status) { + case 'pending': + return [180, 180, 180, 180]; + case 'loading': + return [255, 200, 0, 255]; + case 'loaded': + return [80, 220, 80, 220]; + case 'empty': + return [140, 180, 220, 220]; + case 'error': + return [255, 80, 80, 255]; + case 'aborted': + return [220, 120, 120, 220]; + } +} + +export function tileDebugEntriesSignature(entries: readonly PointsTileDebugEntry[]): string { + return entries + .map((entry) => `${entry.tileId}:${entry.status}:${entry.pointCount ?? ''}`) + .join('|'); +} diff --git a/packages/layers/src/pointsTileLoadCallbacks.ts b/packages/layers/src/pointsTileLoadCallbacks.ts new file mode 100644 index 00000000..7deef609 --- /dev/null +++ b/packages/layers/src/pointsTileLoadCallbacks.ts @@ -0,0 +1,23 @@ +import type { SpatialBounds } from '@spatialdata/core'; +import type { PointTileBbox } from './pointsBbox.js'; + +export interface PointsTileHandle { + tileId: string; + index: { x: number; y: number; z: number }; + bbox: PointTileBbox; +} + +export interface PointsTileLoadResult { + success: boolean; + aborted?: boolean; + clippedBounds?: SpatialBounds | null; + pointCount?: number; + loadMode?: string; + errorMessage?: string; +} + +export interface PointsTileLoadCallbacks { + onViewportTilesRequested?: (tiles: readonly PointsTileHandle[]) => void; + onTileLoadStart?: (tile: PointsTileHandle) => void; + onTileLoadEnd?: (tile: PointsTileHandle, result: PointsTileLoadResult) => void; +} diff --git a/packages/layers/src/pointsTiledDebugHooks.ts b/packages/layers/src/pointsTiledDebugHooks.ts new file mode 100644 index 00000000..5d9b5d67 --- /dev/null +++ b/packages/layers/src/pointsTiledDebugHooks.ts @@ -0,0 +1,62 @@ +import type { PointsLayer } from './PointsLayer.js'; +import type { PointsTileLoadCallbacks } from './pointsTileLoadCallbacks.js'; +import { + reduceTileDebugEntries, + tileDebugEntriesSignature, + type PointsTileDebugEntry, +} from './pointsTileDebug.js'; + +export interface TiledPointsDebugState { + tileDebugEntries: PointsTileDebugEntry[]; +} + +export function createTiledPointsDebugHooks( + layer: PointsLayer, + tileLoadCallbacks?: PointsTileLoadCallbacks +) { + const updateDebugEntries = (updater: (entries: readonly PointsTileDebugEntry[]) => PointsTileDebugEntry[]) => { + const current = + ((layer.state as unknown as TiledPointsDebugState | undefined)?.tileDebugEntries) ?? []; + const next = updater(current); + layer.setState({ tileDebugEntries: next }); + }; + + return { + onViewportTilesRequested(tiles: Parameters>[0]) { + tileLoadCallbacks?.onViewportTilesRequested?.(tiles); + updateDebugEntries((entries) => + reduceTileDebugEntries(entries, { type: 'viewport', tiles, at: Date.now() }) + ); + }, + onTileLoadStart(tile: Parameters>[0]) { + tileLoadCallbacks?.onTileLoadStart?.(tile); + updateDebugEntries((entries) => + reduceTileDebugEntries(entries, { type: 'start', tile, at: Date.now() }) + ); + }, + onTileLoadEnd( + tile: Parameters>[0], + result: Parameters>[1], + clipBounds: { minX: number; minY: number; maxX: number; maxY: number } + ) { + tileLoadCallbacks?.onTileLoadEnd?.(tile, result); + updateDebugEntries((entries) => + reduceTileDebugEntries(entries, { + type: 'end', + tile, + result, + at: Date.now(), + clipBounds, + }) + ); + }, + getTileDebugEntries(): PointsTileDebugEntry[] { + return ((layer.state as unknown as TiledPointsDebugState | undefined)?.tileDebugEntries) ?? []; + }, + getTileDebugSignature(): string { + return tileDebugEntriesSignature( + ((layer.state as unknown as TiledPointsDebugState | undefined)?.tileDebugEntries) ?? [] + ); + }, + }; +} diff --git a/packages/layers/src/preloadedScatterStrategy.ts b/packages/layers/src/preloadedScatterStrategy.ts new file mode 100644 index 00000000..8d7a01f7 --- /dev/null +++ b/packages/layers/src/preloadedScatterStrategy.ts @@ -0,0 +1,47 @@ +import type { Layer, LayersList } from 'deck.gl'; +import type { PointsLayer } from './PointsLayer.js'; +import type { PointsRenderStrategy } from './pointsRenderStrategies.js'; +import { + DEFAULT_POINT_SIZE, + renderColumnarScatterLayer, +} from './pointsScatterLayer.js'; +import type { ColumnarNdarrayPointsBatch } from './pointsLoader.js'; + +export const preloadedScatterStrategy: PointsRenderStrategy = { + renderLayers(layer): Layer | null | LayersList { + const { + resource, + opacity = 1, + visible = true, + pointSize = DEFAULT_POINT_SIZE, + pointRadiusMinPixels, + pointRadiusMaxPixels, + pointMinSizeScale, + viewZoom, + color = [255, 100, 100, 200], + use3d, + } = layer.props; + + if (!visible) { + return null; + } + + const cached = (layer.state as { preloadedBatch?: ColumnarNdarrayPointsBatch }) + .preloadedBatch; + if (!cached) { + return null; + } + + return renderColumnarScatterLayer(layer.props.id, cached, { + color, + pointSize, + pointRadiusMinPixels, + pointRadiusMaxPixels, + pointMinSizeScale, + viewZoom, + opacity, + modelMatrix: layer.props.modelMatrix, + use3d, + }); + }, +}; diff --git a/packages/layers/tests/pointsRenderStrategies.spec.ts b/packages/layers/tests/pointsRenderStrategies.spec.ts new file mode 100644 index 00000000..16a7b1bb --- /dev/null +++ b/packages/layers/tests/pointsRenderStrategies.spec.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; + +import { resolvePointsRenderStrategy } from '../src/pointsRenderStrategies.js'; + +describe('resolvePointsRenderStrategy', () => { + it('selects morton and preloaded strategies by encoding kind', () => { + expect( + resolvePointsRenderStrategy({ + capabilities: { + kind: 'morton-tiled', + batchFormat: 'columnar-ndarray', + supportsViewportTiles: true, + }, + loadInBounds: async () => null, + }).renderLayers + ).toBeTypeOf('function'); + + expect( + resolvePointsRenderStrategy({ + capabilities: { + kind: 'preloaded-columnar', + batchFormat: 'columnar-ndarray', + supportsViewportTiles: false, + }, + loadAll: async () => ({ + format: 'columnar-ndarray', + data: [[0], [0]], + shape: [1], + pointCount: 1, + }), + loadInBounds: async () => null, + }).renderLayers + ).toBeTypeOf('function'); + }); +}); diff --git a/packages/layers/tests/pointsTileDebug.spec.ts b/packages/layers/tests/pointsTileDebug.spec.ts new file mode 100644 index 00000000..93ce2584 --- /dev/null +++ b/packages/layers/tests/pointsTileDebug.spec.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; + +import { + formatPointsTileDebugTooltip, + reduceTileDebugEntries, +} from '../src/pointsTileDebug.js'; + +const sampleTile = { + tileId: '1-2--1', + index: { x: 1, y: 2, z: -1 }, + bbox: { left: 512, top: 1024, right: 1024, bottom: 512 }, +}; + +describe('pointsTileDebug', () => { + it('transitions tile status through viewport, start, and end events', () => { + const at = 1_000; + let entries = reduceTileDebugEntries([], { + type: 'viewport', + tiles: [sampleTile], + at, + }); + expect(entries[0]?.status).toBe('pending'); + + entries = reduceTileDebugEntries(entries, { type: 'start', tile: sampleTile, at: at + 10 }); + expect(entries[0]?.status).toBe('loading'); + expect(entries[0]?.startedAt).toBe(at + 10); + + entries = reduceTileDebugEntries(entries, { + type: 'end', + tile: sampleTile, + at: at + 100, + clipBounds: { minX: 512, minY: 512, maxX: 1024, maxY: 1024 }, + result: { success: true, pointCount: 42, loadMode: 'row-groups' }, + }); + expect(entries[0]?.status).toBe('loaded'); + expect(entries[0]?.pointCount).toBe(42); + expect(entries[0]?.completedAt).toBe(at + 100); + }); + + it('formats tooltip with elapsed time for in-flight tiles', () => { + const tooltip = formatPointsTileDebugTooltip( + { + tileId: sampleTile.tileId, + index: sampleTile.index, + bbox: { minX: 512, minY: 512, maxX: 1024, maxY: 1024 }, + clippedBounds: null, + status: 'loading', + requestedAt: 1_000, + startedAt: 1_500, + }, + { inFlight: 1, loaded: 0, viewportTotal: 3 }, + 2_000 + ); + expect(tooltip.items.some((item) => item.label === 'elapsed' && item.value === '500ms')).toBe( + true + ); + }); +}); diff --git a/packages/layers/vite.config.ts b/packages/layers/vite.config.ts index 2cdfa994..5ebfce03 100644 --- a/packages/layers/vite.config.ts +++ b/packages/layers/vite.config.ts @@ -26,7 +26,14 @@ export default defineConfig({ formats: ['es'], }, rollupOptions: { - external: ['@deck.gl/core', '@hms-dbmi/viv', '@math.gl/core', 'deck.gl', 'zod'], + external: [ + '@deck.gl/core', + '@hms-dbmi/viv', + '@math.gl/core', + '@spatialdata/core', + 'deck.gl', + 'zod', + ], }, }, test: { diff --git a/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx b/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx index ae5179ae..0ce76ddd 100644 --- a/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx +++ b/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx @@ -19,10 +19,19 @@ const tileProgressStyle: CSSProperties = { fontSize: '11px', }; +const checkboxLabelStyle: CSSProperties = { + color: '#ccc', + fontSize: '12px', + display: 'flex', + alignItems: 'center', + gap: 6, +}; + export interface PointsStylePanelProps { layerId: string; config: PointsLayerConfig; tileLoadingMessage?: string | null; + supportsTileDebugOverlay?: boolean; updateLayer: (id: string, updates: Partial) => void; } @@ -30,6 +39,7 @@ export function PointsStylePanel({ layerId, config, tileLoadingMessage, + supportsTileDebugOverlay = false, updateLayer, }: PointsStylePanelProps) { return ( @@ -79,6 +89,20 @@ export function PointsStylePanel({ } /> + {supportsTileDebugOverlay ? ( + + ) : null} {tileLoadingMessage ? (

{tileLoadingMessage}
) : null} diff --git a/packages/vis/src/SpatialCanvas/index.tsx b/packages/vis/src/SpatialCanvas/index.tsx index dfb6ff69..7c68a9e3 100644 --- a/packages/vis/src/SpatialCanvas/index.tsx +++ b/packages/vis/src/SpatialCanvas/index.tsx @@ -435,6 +435,7 @@ function SpatialCanvasInner({ getLayerLoadState, getPointsTileLoadProgress, getPointsTileLoadingMessage, + getPointsLayerSupportsTileDebug, getWorldBoundsForLayer, getWorldBoundsForVisibleLayers, hasEnabledLayers, @@ -839,6 +840,7 @@ function SpatialCanvasInner({ tileLoadingMessage={formatPointsTileLoadingMessage( getPointsTileLoadProgress(selectedConfig.id) )} + supportsTileDebugOverlay={getPointsLayerSupportsTileDebug(selectedConfig.id)} updateLayer={actions.updateLayer} /> )} diff --git a/packages/vis/src/SpatialCanvas/pointsTileProgress.ts b/packages/vis/src/SpatialCanvas/pointsTileProgress.ts index ec466566..81d4eba3 100644 --- a/packages/vis/src/SpatialCanvas/pointsTileProgress.ts +++ b/packages/vis/src/SpatialCanvas/pointsTileProgress.ts @@ -1,3 +1,5 @@ +import type { PointsTileHandle, PointsTileLoadResult } from '@spatialdata/layers'; + export interface PointsTileLoadProgress { /** Tiles currently fetching data. */ inFlight: number; @@ -7,10 +9,12 @@ export interface PointsTileLoadProgress { viewportTotal: number; } +export type { PointsTileHandle, PointsTileLoadResult }; + export interface PointsTileLoadCallbacks { - onViewportTilesRequested?: (count: number) => void; - onTileLoadStart?: () => void; - onTileLoadEnd?: (success: boolean) => void; + onViewportTilesRequested?: (tiles: readonly PointsTileHandle[]) => void; + onTileLoadStart?: (tile: PointsTileHandle) => void; + onTileLoadEnd?: (tile: PointsTileHandle, result: PointsTileLoadResult) => void; } export function emptyPointsTileLoadProgress(): PointsTileLoadProgress { @@ -53,15 +57,17 @@ export function createPointsTileLoadCallbacks( setProgress: (progress: PointsTileLoadProgress) => void ): PointsTileLoadCallbacks { return { - onViewportTilesRequested: (count) => { - setProgress({ inFlight: 0, loaded: 0, viewportTotal: count }); + onViewportTilesRequested: (tiles) => { + setProgress({ inFlight: 0, loaded: 0, viewportTotal: tiles.length }); }, onTileLoadStart: () => { const current = getProgress(); setProgress({ ...current, inFlight: current.inFlight + 1 }); }, - onTileLoadEnd: (success) => { + onTileLoadEnd: (tile, result) => { + void tile; const current = getProgress(); + const success = result.success && !result.aborted; setProgress({ ...current, inFlight: Math.max(0, current.inFlight - 1), diff --git a/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts b/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts index 0e517662..caaef4c0 100644 --- a/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts +++ b/packages/vis/src/SpatialCanvas/renderers/pointsRenderer.ts @@ -1,54 +1,26 @@ /** - * Points layer renderer using deck.gl ScatterplotLayer - * - * Renders point cloud data from SpatialData points elements. + * Points layer renderer adapter for SpatialCanvas. */ import type { Matrix4 } from '@math.gl/core'; -import type { PointsElement, PointsTilingMetadata, SpatialBounds } from '@spatialdata/core'; -import { COORDINATE_SYSTEM } from '@deck.gl/core'; -import { ScatterplotLayer, TileLayer } from 'deck.gl'; +import { PointsLayer, type PointsRenderResource } from '@spatialdata/layers'; import type { Layer } from 'deck.gl'; import type { PointsTileLoadCallbacks } from '../pointsTileProgress'; -export interface PointDataX { - position: [number, number] | [number, number, number]; - // Additional properties can be added for coloring, sizing, etc. - [key: string]: unknown; -} - -// this is ndarray and should be defined elsewhere -// not that we wouldn't also want to be able to have other data & accessors -export interface PointData { - shape: number[]; - // this should most definitely be TypedArray... - data: ArrayLike[]; -} +export { + DEFAULT_POINT_RADIUS_MAX_PIXELS, + DEFAULT_POINT_RADIUS_MIN_PIXELS, + DEFAULT_POINT_SIZE, + MIN_POINT_SIZE_SCALE, + POINT_SIZE_ZOOM_REFERENCE, + zoomScaledPointSize, +} from '@spatialdata/layers'; -/** Orthographic zoom at which configured pointSize applies at full scale. */ -export const POINT_SIZE_ZOOM_REFERENCE = 0; -/** Minimum radius multiplier when zoomed out (reduces fragment overdraw). */ -export const MIN_POINT_SIZE_SCALE = 0.15; -export const DEFAULT_POINT_SIZE = 1; -export const DEFAULT_POINT_RADIUS_MIN_PIXELS = 1; -export const DEFAULT_POINT_RADIUS_MAX_PIXELS = 3; - -export function zoomScaledPointSize( - pointSize: number, - zoom: number | null | undefined, - zoomReference = POINT_SIZE_ZOOM_REFERENCE, - minScale = MIN_POINT_SIZE_SCALE -): number { - if (zoom === null || zoom === undefined || !Number.isFinite(zoom)) { - return pointSize; - } - const scale = 2 ** (zoom - zoomReference); - return pointSize * Math.min(1, Math.max(minScale, scale)); -} +export type { PointData } from '@spatialdata/layers'; export interface PointsLayerRenderConfig { - /** The points element to render */ - element: PointsElement; + /** Resolved points render resource from the Resource Resolver. */ + resource: PointsRenderResource; /** Unique layer ID */ id: string; /** Transformation matrix to target coordinate system */ @@ -68,253 +40,59 @@ export interface PointsLayerRenderConfig { color?: [number, number, number, number]; /** Integer codes matching `{feature_key}_codes` in the Morton Parquet artifact. */ featureCodes?: readonly number[]; - /** ndarray - if we want other data for properties like color/radius etc they will be handled differently */ - pointData?: PointData; - pointTilingMetadata?: PointsTilingMetadata; + showTileDebugOverlay?: boolean; tileLoadCallbacks?: PointsTileLoadCallbacks; use3d?: boolean; } -type PointTileBbox = { - left: number; - right: number; - top: number; - bottom: number; -}; - -type PointTileLoadProps = { - bbox: unknown; - signal?: AbortSignal; -}; - -function isAbortError(error: unknown) { - return error instanceof DOMException && error.name === 'AbortError'; -} - -function isPointTileBbox(value: unknown): value is PointTileBbox { - if (!value || typeof value !== 'object') { - return false; - } - const candidate = value as Record; - return ( - typeof candidate.left === 'number' && - typeof candidate.right === 'number' && - typeof candidate.top === 'number' && - typeof candidate.bottom === 'number' - ); -} - -export function intersectBounds( - query: SpatialBounds, - clip: SpatialBounds -): SpatialBounds | null { - const minX = Math.max(query.minX, clip.minX); - const maxX = Math.min(query.maxX, clip.maxX); - const minY = Math.max(query.minY, clip.minY); - const maxY = Math.min(query.maxY, clip.maxY); - if (minX > maxX || minY > maxY) { - return null; - } - return { minX, minY, maxX, maxY }; -} - -function scatterBoundsFromTileBbox(bbox: PointTileBbox): [number, number, number, number] { - return [bbox.left, bbox.top, bbox.right, bbox.bottom]; -} - -function boundsFromTileBbox(bbox: PointTileBbox): SpatialBounds { - return { - minX: Math.min(bbox.left, bbox.right), - maxX: Math.max(bbox.left, bbox.right), - minY: Math.min(bbox.top, bbox.bottom), - maxY: Math.max(bbox.top, bbox.bottom), - }; -} - -function renderPointScatterSubLayer( - id: string, - data: PointData, - props: { - color: [number, number, number, number]; - pointSize: number; - pointRadiusMinPixels?: number; - pointRadiusMaxPixels?: number; - pointMinSizeScale?: number; - viewZoom?: number | null; - opacity: number; - modelMatrix: Matrix4; - use3d?: boolean; - tileBounds?: [number, number, number, number]; - /** Tile sublayers use fixed pixel radius (Vitessce pattern). */ - tileSubLayer?: boolean; - } -) { - const d = data.data; - const effectivePointSize = props.tileSubLayer - ? props.pointSize - : zoomScaledPointSize( - props.pointSize, - props.viewZoom, - POINT_SIZE_ZOOM_REFERENCE, - props.pointMinSizeScale ?? MIN_POINT_SIZE_SCALE - ); - return new ScatterplotLayer({ - id, - data: d[0], - ...(props.tileBounds ? { bounds: props.tileBounds } : {}), - getPosition: (_d, { index, target }) => [ - d[0][index], - d[1][index], - props.use3d ? d[2]?.[index] || 0 : 0, - ], - getRadius: effectivePointSize, - ...(props.tileSubLayer - ? { - radiusMinPixels: props.pointRadiusMinPixels ?? DEFAULT_POINT_RADIUS_MIN_PIXELS, - radiusMaxPixels: props.pointRadiusMaxPixels ?? DEFAULT_POINT_RADIUS_MAX_PIXELS, - } - : {}), - radiusUnits: 'pixels', - getFillColor: props.color, - opacity: props.opacity, - modelMatrix: props.modelMatrix, - pickable: true, - autoHighlight: true, - highlightColor: [255, 255, 0, 200], - updateTriggers: { - getRadius: [ - props.pointSize, - props.viewZoom, - props.pointRadiusMinPixels, - props.pointRadiusMaxPixels, - props.pointMinSizeScale, - ], - }, - }); -} - -/** - * Create a deck.gl ScatterplotLayer for points data. - * - * Note: This requires the point data to be pre-loaded since deck.gl layers - * are synchronous. The data loading should happen at a higher level. - */ export function renderPointsLayer(config: PointsLayerRenderConfig): Layer | null { const { - element, + resource, id, modelMatrix, opacity, visible, - pointSize = DEFAULT_POINT_SIZE, + pointSize, pointRadiusMinPixels, pointRadiusMaxPixels, pointMinSizeScale, viewZoom, - color = [255, 100, 100, 200], - pointData, - pointTilingMetadata, + color, featureCodes, + showTileDebugOverlay, tileLoadCallbacks, use3d, } = config; - if (!visible) return null; + if (!visible) { + return null; + } + + if ( + !resource.loader.capabilities.bounds && + resource.loader.capabilities.kind === 'morton-tiled' + ) { + console.debug( + `[PointsRenderer] No tiling bounds for layer "${id}" from ${resource.element.path}` + ); + return null; + } - const scatterStyleProps = { - color, + return new PointsLayer({ + id, + resource, + modelMatrix, + opacity, + visible, pointSize, pointRadiusMinPixels, pointRadiusMaxPixels, pointMinSizeScale, viewZoom, - opacity, - modelMatrix, + color, + featureCodes, + showTileDebugOverlay, + tileLoadCallbacks, use3d, - }; - - if (!pointData) { - if (!pointTilingMetadata?.bounds) { - console.debug( - `[PointsRenderer] No point data for layer "${id}" from ${element.url ?? element.path}` - ); - return null; - } - const localBounds = pointTilingMetadata.bounds; - return new TileLayer({ - id, - coordinateSystem: COORDINATE_SYSTEM.CARTESIAN, - modelMatrix, - extent: [ - localBounds.minX, - localBounds.minY, - localBounds.maxX, - localBounds.maxY, - ], - opacity, - visible, - tileSize: 512, - // Vitessce: single tile resolution. extent enables z=-1 clamp when viewZoom < -1. - minZoom: -1, - maxZoom: -1, - refinementStrategy: 'best-available', - updateTriggers: { - getTileData: [pointTilingMetadata.parquetPath, featureCodes], - renderSubLayers: [ - pointSize, - pointRadiusMinPixels, - pointRadiusMaxPixels, - pointMinSizeScale, - viewZoom, - color, - opacity, - modelMatrix, - use3d, - ], - }, - onViewportLoad(tiles) { - tileLoadCallbacks?.onViewportTilesRequested?.(tiles?.length ?? 0); - }, - async getTileData({ bbox, signal }: PointTileLoadProps) { - if (!isPointTileBbox(bbox)) { - return null; - } - tileLoadCallbacks?.onTileLoadStart?.(); - const rawBounds = boundsFromTileBbox(bbox); - const bounds = intersectBounds(rawBounds, localBounds); - if (!bounds) { - tileLoadCallbacks?.onTileLoadEnd?.(true); - return null; - } - try { - const result = await element.loadPointsInBounds({ bounds, featureCodes, signal }); - tileLoadCallbacks?.onTileLoadEnd?.(true); - return result; - } catch (error) { - tileLoadCallbacks?.onTileLoadEnd?.(false); - if (signal?.aborted || isAbortError(error)) { - return null; - } - throw error; - } - }, - renderSubLayers: (props: { - id: string; - data?: PointData | null; - tile?: { bbox?: unknown }; - }) => { - if (!props.data) { - return null; - } - const tileBbox = isPointTileBbox(props.tile?.bbox) ? props.tile.bbox : null; - return renderPointScatterSubLayer(`${props.id}-scatter`, props.data, { - ...scatterStyleProps, - tileBounds: tileBbox ? scatterBoundsFromTileBbox(tileBbox) : undefined, - tileSubLayer: true, - }); - }, - }); - } - - return renderPointScatterSubLayer(id, pointData, scatterStyleProps); + }) as Layer; } diff --git a/packages/vis/src/SpatialCanvas/resolvePointsRenderResource.ts b/packages/vis/src/SpatialCanvas/resolvePointsRenderResource.ts new file mode 100644 index 00000000..d0a16db4 --- /dev/null +++ b/packages/vis/src/SpatialCanvas/resolvePointsRenderResource.ts @@ -0,0 +1,59 @@ +import { + createPointsLoaderForElement, + type PointsElement, + type PointsTilingMetadata, +} from '@spatialdata/core'; +import { + createPointsRenderResource, + type PointsRenderResource, +} from '@spatialdata/layers'; + +export interface ResolvePointsRenderResourceCache { + preloaded?: { shape: number[]; data: ArrayLike[] } | null; + tilingMetadata?: PointsTilingMetadata | null; + metadataKnown?: boolean; +} + +export interface ResolvePointsRenderResourceOptions { + experimentalOptimizations: 'auto' | 'off'; +} + +export function resolvePointsRenderResource( + element: PointsElement, + cache: ResolvePointsRenderResourceCache, + options: ResolvePointsRenderResourceOptions +): PointsRenderResource | null { + const wantsOptimized = options.experimentalOptimizations !== 'off'; + const canTile = + wantsOptimized && + cache.metadataKnown && + cache.tilingMetadata?.supportsRowGroupRangeReads && + cache.tilingMetadata.bounds; + + const loader = createPointsLoaderForElement(element, { + preloaded: cache.preloaded ?? null, + tilingMetadata: canTile ? cache.tilingMetadata : null, + wantsOptimized, + }); + + if (!loader) { + return null; + } + + return createPointsRenderResource(element, loader); +} + +export function pointsRenderResourceSignature( + element: PointsElement, + cache: ResolvePointsRenderResourceCache, + options: ResolvePointsRenderResourceOptions +): string { + return [ + element.key, + options.experimentalOptimizations, + cache.metadataKnown ? 'meta' : 'nometa', + cache.tilingMetadata?.parquetPath ?? '', + cache.tilingMetadata?.supportsRowGroupRangeReads ? 'rg' : '', + cache.preloaded ? `pre:${cache.preloaded.shape[0] ?? 0}` : 'nopre', + ].join('|'); +} diff --git a/packages/vis/src/SpatialCanvas/types.ts b/packages/vis/src/SpatialCanvas/types.ts index 4376cacd..07f5c1de 100644 --- a/packages/vis/src/SpatialCanvas/types.ts +++ b/packages/vis/src/SpatialCanvas/types.ts @@ -105,6 +105,8 @@ export interface PointsLayerConfig extends BaseLayerConfig { /** Filter to these feature code(s). Future: string[] resolved via codebook. */ featureCodes?: number[]; experimentalOptimizations?: 'auto' | 'off'; + /** Show viewport tile polygons and loading stats for tiled points layers. */ + showTileDebugOverlay?: boolean; } export interface LabelsLayerConfig extends BaseLayerConfig { diff --git a/packages/vis/src/SpatialCanvas/useLayerData.ts b/packages/vis/src/SpatialCanvas/useLayerData.ts index f0c61c3e..9c86d7c9 100644 --- a/packages/vis/src/SpatialCanvas/useLayerData.ts +++ b/packages/vis/src/SpatialCanvas/useLayerData.ts @@ -51,6 +51,8 @@ import { buildShapeFeatureStateRuntime, buildShapeFillColorByFeatureId, buildShapesPrebuiltData, + formatPointsTileDebugTooltip, + isPointsTileDebugPickObject, resolveShapeFeatureFromPick, resolveShapeTooltipFromPickInfo, resolveShapeTooltipRowIndex, @@ -74,6 +76,10 @@ import { type PointsTileLoadCallbacks, type PointsTileLoadProgress, } from './pointsTileProgress'; +import { + pointsRenderResourceSignature, + resolvePointsRenderResource, +} from './resolvePointsRenderResource'; import { loadShapesData, renderShapesLayer } from './renderers/shapesRenderer'; import type { AvailableElement, ElementsByType, LayerConfig, ShapesLayerConfig } from './types'; @@ -237,6 +243,8 @@ interface UseLayerDataResult { getPointsTileLoadProgress: (layerId?: string) => PointsTileLoadProgress; /** User-facing message while tiled points are loading, if any. */ getPointsTileLoadingMessage: () => string | null; + /** Whether a points layer uses viewport tile loading (tile debug overlay eligible). */ + getPointsLayerSupportsTileDebug: (layerId: string) => boolean; /** Trigger a reload of data for a specific element */ reloadElement: (type: string, key: string) => void; /** World-space axis-aligned bounds for one visible layer with loaded data, or null. */ @@ -521,6 +529,9 @@ export function useLayerData( const [, setLoadedDataRevision] = useState(0); const pointsTileProgressRef = useRef(new Map()); const pointsTileCallbacksRef = useRef(new Map()); + const pointsRenderResourceCacheRef = useRef( + new Map }>() + ); const [pointsTileProgressRevision, setPointsTileProgressRevision] = useState(0); const notifyLoadedDataChanged = useCallback(() => { @@ -1212,6 +1223,7 @@ export function useLayerData( loaded.points.delete(key); loaded.pointTilingMetadata.delete(key); loaded.worldBounds.delete(`points:${key}`); + pointsRenderResourceCacheRef.current.delete(key); } else if (type === 'image') { loaded.images.delete(key); loaded.worldBounds.delete(`image:${key}`); @@ -1394,10 +1406,42 @@ export function useLayerData( } } else if (config.type === 'points') { const pointData = loaded.points.get(elem.key); - const pointTilingMetadata = loaded.pointTilingMetadata.get(elem.key) ?? undefined; - if (pointData || pointTilingMetadata) { + const pointTilingMetadata = loaded.pointTilingMetadata.get(elem.key); + const metadataKnown = loaded.pointTilingMetadata.has(elem.key); + const wantsOptimized = + experimentalOptimizations !== 'off' && config.experimentalOptimizations !== 'off'; + const signature = pointsRenderResourceSignature( + elem.element as PointsElement, + { + preloaded: pointData ?? null, + tilingMetadata: pointTilingMetadata, + metadataKnown, + }, + { experimentalOptimizations: wantsOptimized ? 'auto' : 'off' } + ); + let cachedResource = pointsRenderResourceCacheRef.current.get(elem.key); + if (!cachedResource || cachedResource.signature !== signature) { + const resource = resolvePointsRenderResource( + elem.element as PointsElement, + { + preloaded: pointData ?? null, + tilingMetadata: pointTilingMetadata, + metadataKnown, + }, + { experimentalOptimizations: wantsOptimized ? 'auto' : 'off' } + ); + if (resource) { + cachedResource = { signature, resource }; + pointsRenderResourceCacheRef.current.set(elem.key, cachedResource); + } else { + pointsRenderResourceCacheRef.current.delete(elem.key); + } + } + if (cachedResource?.resource) { + const supportsViewportTiles = + cachedResource.resource.loader.capabilities.supportsViewportTiles; const layer = renderPointsLayer({ - element: elem.element as PointsElement, + resource: cachedResource.resource, id: layerId, modelMatrix: elem.transform, opacity: config.opacity, @@ -1409,9 +1453,10 @@ export function useLayerData( viewZoom, color: config.color, featureCodes: config.featureCodes, - pointData, - pointTilingMetadata, - tileLoadCallbacks: pointTilingMetadata ? getPointsTileCallbacks(layerId) : undefined, + showTileDebugOverlay: config.showTileDebugOverlay, + tileLoadCallbacks: supportsViewportTiles + ? getPointsTileCallbacks(layerId) + : undefined, }); if (layer) deckLayers.push(layer); } @@ -1463,7 +1508,7 @@ export function useLayerData( } return deckLayers; - }, [layers, layerOrder, getStableSelections, viewZoom, getPointsTileCallbacks]); + }, [layers, layerOrder, getStableSelections, viewZoom, getPointsTileCallbacks, experimentalOptimizations]); const getImageLayerLoadedData = useCallback((layerId: string): ImageLoaderData | undefined => { const elem = resolveLayerElement(layerId, layersRef.current[layerId], elementMap.current); @@ -1540,6 +1585,16 @@ export function useLayerData( ); } + if (elem.type === 'points') { + if (isPointsTileDebugPickObject(pickInfo.object)) { + const progress = + pointsTileProgressRef.current.get(layerId) ?? emptyPointsTileLoadProgress(); + const tooltip = formatPointsTileDebugTooltip(pickInfo.object.entry, progress); + return attachTooltipElementContext(tooltip, elementContext); + } + return undefined; + } + if (!isShapesAvailableElement(elem)) { return undefined; } @@ -1762,6 +1817,15 @@ export function useLayerData( [layerLoadStates, layerOrder, layers, hasRenderableLayerData] ); + const getPointsLayerSupportsTileDebug = useCallback((layerId: string): boolean => { + const elem = resolveLayerElement(layerId, layersRef.current[layerId], elementMap.current); + if (!elem || elem.type !== 'points') { + return false; + } + const cached = pointsRenderResourceCacheRef.current.get(elem.key); + return cached?.resource?.loader.capabilities.supportsViewportTiles ?? false; + }, []); + return { getLayers, getVivLayerProps, @@ -1776,6 +1840,7 @@ export function useLayerData( isBlocking, getPointsTileLoadProgress, getPointsTileLoadingMessage, + getPointsLayerSupportsTileDebug, reloadElement, getWorldBoundsForLayer, getWorldBoundsForVisibleLayers, diff --git a/packages/vis/tests/pointsRenderer.spec.ts b/packages/vis/tests/pointsRenderer.spec.ts index f575cf62..f0eae5b5 100644 --- a/packages/vis/tests/pointsRenderer.spec.ts +++ b/packages/vis/tests/pointsRenderer.spec.ts @@ -4,7 +4,7 @@ import { MIN_POINT_SIZE_SCALE, POINT_SIZE_ZOOM_REFERENCE, zoomScaledPointSize, -} from '../src/SpatialCanvas/renderers/pointsRenderer.js'; +} from '@spatialdata/layers'; describe('zoomScaledPointSize', () => { it('returns base size at the reference zoom', () => { diff --git a/packages/vis/tests/pointsTileProgress.spec.ts b/packages/vis/tests/pointsTileProgress.spec.ts index 6121abd5..4f81780b 100644 --- a/packages/vis/tests/pointsTileProgress.spec.ts +++ b/packages/vis/tests/pointsTileProgress.spec.ts @@ -8,6 +8,12 @@ import { pointsTileLoadingMessage, } from '../src/SpatialCanvas/pointsTileProgress.js'; +const sampleTile = { + tileId: '0-0--1', + index: { x: 0, y: 0, z: -1 }, + bbox: { left: 0, top: 512, right: 512, bottom: 0 }, +}; + describe('pointsTileProgress', () => { it('aggregates progress across layers', () => { const aggregate = aggregatePointsTileLoadProgress( @@ -41,17 +47,17 @@ describe('pointsTileProgress', () => { } ); - callbacks.onViewportTilesRequested?.(2); + callbacks.onViewportTilesRequested?.([sampleTile, sampleTile]); expect(progress).toEqual({ inFlight: 0, loaded: 0, viewportTotal: 2 }); - callbacks.onTileLoadStart?.(); - callbacks.onTileLoadStart?.(); + callbacks.onTileLoadStart?.(sampleTile); + callbacks.onTileLoadStart?.(sampleTile); expect(progress.inFlight).toBe(2); - callbacks.onTileLoadEnd?.(true); + callbacks.onTileLoadEnd?.(sampleTile, { success: true, pointCount: 10 }); expect(progress).toEqual({ inFlight: 1, loaded: 1, viewportTotal: 2 }); - callbacks.onTileLoadEnd?.(true); + callbacks.onTileLoadEnd?.(sampleTile, { success: true, pointCount: 0 }); expect(progress).toEqual({ inFlight: 0, loaded: 2, viewportTotal: 2 }); expect(pointsTileLoadingMessage(progress)).toBeNull(); }); diff --git a/packages/vis/tests/resolvePointsRenderResource.spec.ts b/packages/vis/tests/resolvePointsRenderResource.spec.ts new file mode 100644 index 00000000..24e98dbd --- /dev/null +++ b/packages/vis/tests/resolvePointsRenderResource.spec.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; + +import { pointsRenderResourceSignature } from '../src/SpatialCanvas/resolvePointsRenderResource.js'; + +describe('pointsRenderResourceSignature', () => { + it('changes when preload or metadata inputs change', () => { + const element = { key: 'transcripts' } as { key: string }; + const base = pointsRenderResourceSignature( + element as never, + { metadataKnown: true, tilingMetadata: null, preloaded: null }, + { experimentalOptimizations: 'auto' } + ); + const withPreload = pointsRenderResourceSignature( + element as never, + { + metadataKnown: true, + tilingMetadata: null, + preloaded: { shape: [2], data: [[0, 1], [0, 1]] }, + }, + { experimentalOptimizations: 'auto' } + ); + expect(base).not.toEqual(withPreload); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 784796f6..a7c66a78 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -269,6 +269,9 @@ importers: '@math.gl/core': specifier: 'catalog:' version: 4.1.0 + '@spatialdata/core': + specifier: workspace:* + version: link:../core zod: specifier: 'catalog:' version: 4.1.13 From a0667df7dcb54e8787224cf84ecf8e65587767e8 Mon Sep 17 00:00:00 2001 From: Peter Todd Date: Sat, 20 Jun 2026 10:13:27 +0100 Subject: [PATCH 08/38] More consistent state in Points Layer and Debugging visualisation - Introduced `columnarPointCount` function to accurately determine point counts from columnar data. - Updated `toColumnarBatch` to utilize the new point count logic. - Added `createTileDebugStore` and `createTiledPointsDebugHooks` for improved tile debugging capabilities. - Enhanced `PointsLayer` to support tile debugging and updated state management. - Modified `pointsTileProgress` to track loaded points and improve loading messages. - Updated tests to cover new debugging features and ensure correct functionality across layers. - Adjusted visualization components to integrate tile debugging options seamlessly. --- packages/core/src/pointsLoader.ts | 16 +- packages/core/tests/pointsLoader.spec.ts | 36 +++- packages/layers/src/PointsLayer.ts | 10 +- packages/layers/src/index.ts | 6 + packages/layers/src/mortonTiledStrategy.ts | 24 ++- packages/layers/src/pointsLoader.ts | 6 +- packages/layers/src/pointsScatterLayer.ts | 2 +- packages/layers/src/pointsTileDebug.ts | 157 +++++++++++++--- packages/layers/src/pointsTiledDebugHooks.ts | 175 +++++++++++++++--- packages/layers/tests/pointsTileDebug.spec.ts | 83 ++++++++- .../src/SpatialCanvas/PointsStylePanel.tsx | 2 +- packages/vis/src/SpatialCanvas/index.tsx | 1 + .../src/SpatialCanvas/pointsTileProgress.ts | 95 ++++++++-- .../SpatialCanvas/renderers/pointsRenderer.ts | 10 +- .../vis/src/SpatialCanvas/useLayerData.ts | 55 +++++- packages/vis/tests/pointsTileProgress.spec.ts | 49 +++-- 16 files changed, 614 insertions(+), 113 deletions(-) diff --git a/packages/core/src/pointsLoader.ts b/packages/core/src/pointsLoader.ts index bcf9a998..f66cf381 100644 --- a/packages/core/src/pointsLoader.ts +++ b/packages/core/src/pointsLoader.ts @@ -63,15 +63,27 @@ export function resolvePointsEncoding( return 'preloaded-columnar'; } +function columnarPointCount(shape: number[], data: ArrayLike[]): number { + if (shape.length >= 2 && Number.isFinite(shape[1])) { + return shape[1]; + } + const fromData = data[0]?.length; + if (typeof fromData === 'number') { + return fromData; + } + return shape[0] ?? 0; +} + function toColumnarBatch( result: PointsInBoundsResult | PreloadedColumnarInput, overrides?: Partial ): ColumnarNdarrayPointsBatch { const shape = result.shape ?? []; - const pointCount = shape[0] ?? 0; + const data = result.data; + const pointCount = columnarPointCount(shape, data); return { format: 'columnar-ndarray', - data: result.data, + data, shape, bounds: 'bounds' in result ? result.bounds : overrides?.bounds, loadMode: 'loadMode' in result ? result.loadMode : overrides?.loadMode, diff --git a/packages/core/tests/pointsLoader.spec.ts b/packages/core/tests/pointsLoader.spec.ts index 9c7bc4cc..7c7a04c3 100644 --- a/packages/core/tests/pointsLoader.spec.ts +++ b/packages/core/tests/pointsLoader.spec.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; -import { resolvePointsEncoding } from '../src/pointsLoader.js'; +import { createMortonTiledPointsLoader, resolvePointsEncoding } from '../src/pointsLoader.js'; +import type { PointsElement } from '../src/models/index.js'; describe('resolvePointsEncoding', () => { it('prefers preloaded data when present', () => { @@ -26,3 +27,36 @@ describe('resolvePointsEncoding', () => { ).toBe('morton-tiled'); }); }); + +describe('createMortonTiledPointsLoader', () => { + it('uses columnar shape[1] as point count, not shape[0] axis count', async () => { + const element = { + async loadPointsInBounds() { + return { + shape: [2, 1_000], + data: [new Float64Array(1_000), new Float64Array(1_000)], + bounds: { minX: 0, minY: 0, maxX: 10, maxY: 10 }, + loadMode: 'row-groups', + }; + }, + } as unknown as PointsElement; + + const loader = createMortonTiledPointsLoader(element, { + kind: 'morton-points', + parquetPath: 'points/a/points.parquet', + axisNames: ['x', 'y'], + featureCodeColumnName: 'feature_name_codes', + mortonCodeColumnName: 'morton_code_2d', + totalRows: 1_000, + totalRowGroups: 1, + maxRowsPerGroup: 1_000, + supportsRowGroupRangeReads: true, + bounds: { minX: 0, minY: 0, maxX: 10, maxY: 10 }, + }); + + const batch = await loader.loadInBounds({ + bounds: { minX: 0, minY: 0, maxX: 10, maxY: 10 }, + }); + expect(batch?.pointCount).toBe(1_000); + }); +}); diff --git a/packages/layers/src/PointsLayer.ts b/packages/layers/src/PointsLayer.ts index 75b604e2..b629d907 100644 --- a/packages/layers/src/PointsLayer.ts +++ b/packages/layers/src/PointsLayer.ts @@ -4,8 +4,8 @@ import { CompositeLayer } from 'deck.gl'; import type { Layer, LayersList } from 'deck.gl'; import type { PointsRenderResource } from './pointsLoader.js'; import type { PointsTileLoadCallbacks } from './pointsTileLoadCallbacks.js'; +import type { TileDebugStore } from './pointsTiledDebugHooks.js'; import type { ColumnarNdarrayPointsBatch } from './pointsLoader.js'; -import type { PointsTileDebugEntry } from './pointsTileDebug.js'; import { resolvePointsRenderStrategy } from './pointsRenderStrategies.js'; import { DEFAULT_POINT_RADIUS_MAX_PIXELS, @@ -28,12 +28,14 @@ export interface PointsLayerProps { featureCodes?: readonly number[]; showTileDebugOverlay?: boolean; tileLoadCallbacks?: PointsTileLoadCallbacks; + tileDebugStore?: TileDebugStore; + /** Bumps when {@link tileDebugStore} contents change; forces debug overlay refresh. */ + tileDebugSignature?: string; use3d?: boolean; } interface PointsLayerState { preloadedBatch?: ColumnarNdarrayPointsBatch; - tileDebugEntries?: PointsTileDebugEntry[]; } export class PointsLayer extends CompositeLayer { @@ -45,7 +47,7 @@ export class PointsLayer extends CompositeLayer { pointSize: DEFAULT_POINT_SIZE, pointRadiusMinPixels: DEFAULT_POINT_RADIUS_MIN_PIXELS, pointRadiusMaxPixels: DEFAULT_POINT_RADIUS_MAX_PIXELS, - showTileDebugOverlay: false, + showTileDebugOverlay: true, } satisfies Partial; initializeState(): void { @@ -59,7 +61,7 @@ export class PointsLayer extends CompositeLayer { props.resource.loader !== oldProps.resource.loader || props.resource.element !== oldProps.resource.element ) { - this.setState({ preloadedBatch: undefined, tileDebugEntries: [] }); + this.setState({ preloadedBatch: undefined }); void this.ensurePreloadedBatch(); } } diff --git a/packages/layers/src/index.ts b/packages/layers/src/index.ts index 8a65a137..1951fa39 100644 --- a/packages/layers/src/index.ts +++ b/packages/layers/src/index.ts @@ -97,6 +97,12 @@ export { zoomScaledPointSize, } from './pointsScatterLayer.js'; export type { PointsTileHandle, PointsTileLoadResult, PointsTileLoadCallbacks } from './pointsTileLoadCallbacks.js'; +export { + createTileDebugStore, + createTiledPointsDebugHooks, + type TileDebugStore, + type TiledPointsDebugState, +} from './pointsTiledDebugHooks.js'; export { POINTS_TILE_DEBUG_PICK_KIND, formatPointsTileDebugTooltip, diff --git a/packages/layers/src/mortonTiledStrategy.ts b/packages/layers/src/mortonTiledStrategy.ts index a48fec89..bda65337 100644 --- a/packages/layers/src/mortonTiledStrategy.ts +++ b/packages/layers/src/mortonTiledStrategy.ts @@ -37,8 +37,18 @@ function isColumnarBatch(value: unknown): value is ColumnarNdarrayPointsBatch { ); } +function renderedPointCount(batch: ColumnarNdarrayPointsBatch): number { + if (batch.pointCount !== undefined) { + return batch.pointCount; + } + if (batch.shape.length >= 2 && Number.isFinite(batch.shape[1])) { + return batch.shape[1]; + } + return batch.data[0]?.length ?? 0; +} + export const mortonTiledStrategy: PointsRenderStrategy = { - renderLayers(layer): Layer | null | LayersList { + renderLayers(layer: PointsLayer): Layer | null | LayersList { const { resource, featureCodes, @@ -58,7 +68,7 @@ export const mortonTiledStrategy: PointsRenderStrategy = { return null; } - const debugHooks = createTiledPointsDebugHooks(layer, tileLoadCallbacks); + const debugHooks = createTiledPointsDebugHooks(layer.props.tileDebugStore, tileLoadCallbacks); const scatterStyleProps = { color, pointSize, @@ -140,7 +150,7 @@ export const mortonTiledStrategy: PointsRenderStrategy = { { success: true, clippedBounds: bounds, - pointCount: batch.pointCount ?? batch.shape[0] ?? 0, + pointCount: renderedPointCount(batch), loadMode: batch.loadMode, }, rawBounds @@ -185,6 +195,7 @@ export const mortonTiledStrategy: PointsRenderStrategy = { if (showTileDebugOverlay) { const entries = debugHooks.getTileDebugEntries(); + const debugSignature = layer.props.tileDebugSignature ?? debugHooks.getTileDebugSignature(); const polygonData = pointsTileDebugPolygonData(entries).map(({ polygon, entry }) => ({ polygon, entry, @@ -212,9 +223,10 @@ export const mortonTiledStrategy: PointsRenderStrategy = { opacity: Math.min(1, opacity + 0.15), visible, updateTriggers: { - getFillColor: [debugHooks.getTileDebugSignature()], - getLineColor: [debugHooks.getTileDebugSignature()], - getPolygon: [debugHooks.getTileDebugSignature()], + data: [debugSignature], + getFillColor: [debugSignature], + getLineColor: [debugSignature], + getPolygon: [debugSignature], }, }) ) diff --git a/packages/layers/src/pointsLoader.ts b/packages/layers/src/pointsLoader.ts index be6d8b65..f3940591 100644 --- a/packages/layers/src/pointsLoader.ts +++ b/packages/layers/src/pointsLoader.ts @@ -63,13 +63,17 @@ export function columnarBatchFromPointData( data: PointData, options?: { loadMode?: string; bounds?: SpatialBounds } ): ColumnarNdarrayPointsBatch { + const pointCount = + data.shape.length >= 2 && Number.isFinite(data.shape[1]) + ? data.shape[1] + : (data.data[0]?.length ?? data.shape[0] ?? 0); return { format: 'columnar-ndarray', data: data.data, shape: data.shape, bounds: options?.bounds, loadMode: options?.loadMode, - pointCount: data.shape[0] ?? 0, + pointCount, }; } diff --git a/packages/layers/src/pointsScatterLayer.ts b/packages/layers/src/pointsScatterLayer.ts index b4058c92..006db602 100644 --- a/packages/layers/src/pointsScatterLayer.ts +++ b/packages/layers/src/pointsScatterLayer.ts @@ -8,7 +8,7 @@ import { pointDataFromColumnarBatch } from './pointsLoader.js'; export const POINT_SIZE_ZOOM_REFERENCE = 0; /** Minimum radius multiplier when zoomed out (reduces fragment overdraw). */ export const MIN_POINT_SIZE_SCALE = 0.15; -export const DEFAULT_POINT_SIZE = 1; +export const DEFAULT_POINT_SIZE = 0.1; export const DEFAULT_POINT_RADIUS_MIN_PIXELS = 0.1; export const DEFAULT_POINT_RADIUS_MAX_PIXELS = 3; diff --git a/packages/layers/src/pointsTileDebug.ts b/packages/layers/src/pointsTileDebug.ts index 68a86ce7..62a45bb5 100644 --- a/packages/layers/src/pointsTileDebug.ts +++ b/packages/layers/src/pointsTileDebug.ts @@ -12,6 +12,7 @@ export type PointsTileStatus = export interface PointsTileLoadProgress { inFlight: number; loaded: number; + loadedPoints: number; viewportTotal: number; } @@ -46,8 +47,29 @@ export function isPointsTileDebugPickObject( return candidate.kind === POINTS_TILE_DEBUG_PICK_KIND && candidate.entry != null; } +export interface PointsTileCompletedSnapshot { + status: PointsTileStatus; + pointCount?: number; + loadMode?: string; + clippedBounds: SpatialBounds | null; + errorMessage?: string; + startedAt?: number; + completedAt: number; +} + +export interface PointsTileDebugViewportContext { + loadingTileIds: ReadonlySet; + completedTilesById: ReadonlyMap; + tileHandlesById: ReadonlyMap; +} + export type PointsTileDebugEvent = - | { type: 'viewport'; tiles: readonly PointsTileHandle[]; at: number } + | { + type: 'viewport'; + tiles: readonly PointsTileHandle[]; + at: number; + context: PointsTileDebugViewportContext; + } | { type: 'start'; tile: PointsTileHandle; at: number } | { type: 'end'; @@ -57,6 +79,64 @@ export type PointsTileDebugEvent = clipBounds: SpatialBounds; }; +export function completedSnapshotFromLoadResult( + result: PointsTileLoadResult, + clipBounds: SpatialBounds, + completedAt: number, + startedAt?: number +): PointsTileCompletedSnapshot { + let status: PointsTileStatus = 'error'; + if (result.aborted) { + status = 'aborted'; + } else if (result.success) { + status = (result.pointCount ?? 0) > 0 ? 'loaded' : 'empty'; + } + + return { + status, + pointCount: result.pointCount, + loadMode: result.loadMode, + clippedBounds: result.clippedBounds ?? clipBounds, + errorMessage: result.errorMessage, + startedAt, + completedAt, + }; +} + +function resolveViewportTileStatus( + tileId: string, + existing: PointsTileDebugEntry | undefined, + context: PointsTileDebugViewportContext +): PointsTileStatus { + if (context.loadingTileIds.has(tileId)) { + return 'loading'; + } + const completed = context.completedTilesById.get(tileId); + if (completed) { + return completed.status; + } + if (existing?.status === 'loaded' || existing?.status === 'empty') { + return existing.status; + } + return 'pending'; +} + +function applyCompletedSnapshot( + entry: PointsTileDebugEntry, + completed: PointsTileCompletedSnapshot +): PointsTileDebugEntry { + return { + ...entry, + status: completed.status, + clippedBounds: completed.clippedBounds, + pointCount: completed.pointCount, + loadMode: completed.loadMode, + errorMessage: completed.errorMessage, + startedAt: completed.startedAt ?? entry.startedAt, + completedAt: completed.completedAt, + }; +} + export function reduceTileDebugEntries( previous: readonly PointsTileDebugEntry[], event: PointsTileDebugEvent @@ -64,26 +144,42 @@ export function reduceTileDebugEntries( const byId = new Map(previous.map((entry) => [entry.tileId, entry])); if (event.type === 'viewport') { - const next = new Map(); + const tileHandlesById = new Map(event.context.tileHandlesById); for (const tile of event.tiles) { + tileHandlesById.set(tile.tileId, tile); + } + const activeTileIds = new Set([ + ...event.tiles.map((tile) => tile.tileId), + ...event.context.loadingTileIds, + ...event.context.completedTilesById.keys(), + ]); + const next = new Map(); + for (const tileId of activeTileIds) { + const tile = tileHandlesById.get(tileId); + if (!tile) { + continue; + } const rawBounds = boundsFromHandle(tile); const existing = byId.get(tile.tileId); - next.set(tile.tileId, { + const completed = event.context.completedTilesById.get(tile.tileId); + const status = resolveViewportTileStatus(tile.tileId, existing, event.context); + let entry: PointsTileDebugEntry = { tileId: tile.tileId, index: tile.index, bbox: rawBounds, - clippedBounds: existing?.clippedBounds ?? null, - status: - existing?.status === 'loaded' || existing?.status === 'empty' - ? existing.status - : 'pending', - requestedAt: event.at, - startedAt: existing?.startedAt, - completedAt: existing?.completedAt, - pointCount: existing?.pointCount, - loadMode: existing?.loadMode, - errorMessage: existing?.errorMessage, - }); + clippedBounds: existing?.clippedBounds ?? completed?.clippedBounds ?? null, + status, + requestedAt: existing?.requestedAt ?? event.at, + startedAt: existing?.startedAt ?? completed?.startedAt, + completedAt: existing?.completedAt ?? completed?.completedAt, + pointCount: existing?.pointCount ?? completed?.pointCount, + loadMode: existing?.loadMode ?? completed?.loadMode, + errorMessage: existing?.errorMessage ?? completed?.errorMessage, + }; + if (completed && (status === 'loaded' || status === 'empty' || status === 'error' || status === 'aborted')) { + entry = applyCompletedSnapshot(entry, completed); + } + next.set(tile.tileId, entry); } return [...next.values()]; } @@ -109,25 +205,25 @@ export function reduceTileDebugEntries( const rawBounds = boundsFromHandle(event.tile); const { result } = event; - let status: PointsTileStatus = 'error'; - if (result.aborted) { - status = 'aborted'; - } else if (result.success) { - status = (result.pointCount ?? 0) > 0 ? 'loaded' : 'empty'; - } + const snapshot = completedSnapshotFromLoadResult( + result, + event.clipBounds, + event.at, + byId.get(event.tile.tileId)?.startedAt ?? event.at + ); byId.set(event.tile.tileId, { tileId: event.tile.tileId, index: event.tile.index, bbox: rawBounds, - clippedBounds: result.clippedBounds ?? event.clipBounds, - status, + clippedBounds: snapshot.clippedBounds, + status: snapshot.status, requestedAt: byId.get(event.tile.tileId)?.requestedAt ?? event.at, - startedAt: byId.get(event.tile.tileId)?.startedAt ?? event.at, - completedAt: event.at, - pointCount: result.pointCount, - loadMode: result.loadMode, - errorMessage: result.errorMessage, + startedAt: snapshot.startedAt, + completedAt: snapshot.completedAt, + pointCount: snapshot.pointCount, + loadMode: snapshot.loadMode, + errorMessage: snapshot.errorMessage, }); return [...byId.values()]; } @@ -186,7 +282,10 @@ export function formatPointsTileDebugTooltip( { label: 'status', value: entry.status }, { label: 'batch', - value: `${batchProgress.loaded}/${batchProgress.viewportTotal} (${batchProgress.inFlight} in flight)`, + value: + batchProgress.loadedPoints > 0 + ? `${batchProgress.loaded}/${batchProgress.viewportTotal} (${batchProgress.inFlight} in flight, ${batchProgress.loadedPoints.toLocaleString()} points)` + : `${batchProgress.loaded}/${batchProgress.viewportTotal} (${batchProgress.inFlight} in flight)`, }, { label: 'index', value: `x=${entry.index.x} y=${entry.index.y} z=${entry.index.z}` }, { label: 'bbox', value: formatBounds(entry.bbox) }, diff --git a/packages/layers/src/pointsTiledDebugHooks.ts b/packages/layers/src/pointsTiledDebugHooks.ts index 5d9b5d67..28fe4df0 100644 --- a/packages/layers/src/pointsTiledDebugHooks.ts +++ b/packages/layers/src/pointsTiledDebugHooks.ts @@ -1,38 +1,149 @@ -import type { PointsLayer } from './PointsLayer.js'; -import type { PointsTileLoadCallbacks } from './pointsTileLoadCallbacks.js'; +import type { PointsTileHandle, PointsTileLoadCallbacks } from './pointsTileLoadCallbacks.js'; import { + completedSnapshotFromLoadResult, reduceTileDebugEntries, tileDebugEntriesSignature, + type PointsTileCompletedSnapshot, type PointsTileDebugEntry, } from './pointsTileDebug.js'; export interface TiledPointsDebugState { tileDebugEntries: PointsTileDebugEntry[]; + completedTilesById?: Record; + loadingTileIds?: string[]; + lastViewportTiles?: readonly PointsTileHandle[]; + tileHandlesById?: Record; +} + +function rememberTileHandle( + state: TiledPointsDebugState, + tile: PointsTileHandle +): Record { + return { ...(state.tileHandlesById ?? {}), [tile.tileId]: tile }; +} + +function rebuildActiveDebugEntries( + entries: readonly PointsTileDebugEntry[], + state: TiledPointsDebugState, + at: number +): PointsTileDebugEntry[] { + return reduceTileDebugEntries(entries, { + type: 'viewport', + tiles: state.lastViewportTiles ?? [], + at, + context: { + loadingTileIds: new Set(state.loadingTileIds ?? []), + completedTilesById: new Map(Object.entries(state.completedTilesById ?? {})), + tileHandlesById: new Map(Object.entries(state.tileHandlesById ?? {})), + }, + }); +} + +export interface TileDebugStore { + getState(): TiledPointsDebugState; + update(updater: (state: TiledPointsDebugState) => TiledPointsDebugState): void; +} + +function emptyDebugState(): TiledPointsDebugState { + return { tileDebugEntries: [], completedTilesById: {}, loadingTileIds: [], tileHandlesById: {} }; +} + +function debugStateSignature(state: TiledPointsDebugState): string { + const completedKeys = Object.keys(state.completedTilesById ?? {}).sort().join(','); + const loadingKeys = [...(state.loadingTileIds ?? [])].sort().join(','); + const handleKeys = Object.keys(state.tileHandlesById ?? {}).sort().join(','); + return `${tileDebugEntriesSignature(state.tileDebugEntries)}|${loadingKeys}|${completedKeys}|${handleKeys}`; +} + +export function createTileDebugStore(onChange?: () => void): TileDebugStore { + let state = emptyDebugState(); + return { + getState() { + return state; + }, + update(updater) { + const next = updater(state); + if (debugStateSignature(state) === debugStateSignature(next)) { + return; + } + state = next; + onChange?.(); + }, + }; } export function createTiledPointsDebugHooks( - layer: PointsLayer, + store: TileDebugStore | undefined, tileLoadCallbacks?: PointsTileLoadCallbacks ) { - const updateDebugEntries = (updater: (entries: readonly PointsTileDebugEntry[]) => PointsTileDebugEntry[]) => { - const current = - ((layer.state as unknown as TiledPointsDebugState | undefined)?.tileDebugEntries) ?? []; - const next = updater(current); - layer.setState({ tileDebugEntries: next }); - }; + if (!store) { + return { + onViewportTilesRequested( + tiles: Parameters>[0] + ) { + tileLoadCallbacks?.onViewportTilesRequested?.(tiles); + }, + onTileLoadStart(tile: Parameters>[0]) { + tileLoadCallbacks?.onTileLoadStart?.(tile); + }, + onTileLoadEnd( + tile: Parameters>[0], + result: Parameters>[1], + _clipBounds: { minX: number; minY: number; maxX: number; maxY: number } + ) { + tileLoadCallbacks?.onTileLoadEnd?.(tile, result); + }, + getTileDebugEntries(): PointsTileDebugEntry[] { + return []; + }, + getTileDebugSignature(): string { + return ''; + }, + }; + } return { onViewportTilesRequested(tiles: Parameters>[0]) { tileLoadCallbacks?.onViewportTilesRequested?.(tiles); - updateDebugEntries((entries) => - reduceTileDebugEntries(entries, { type: 'viewport', tiles, at: Date.now() }) - ); + store.update((state) => { + const at = Date.now(); + const tileHandlesById = { ...(state.tileHandlesById ?? {}) }; + for (const tile of tiles) { + tileHandlesById[tile.tileId] = tile; + } + const nextState: TiledPointsDebugState = { + ...state, + lastViewportTiles: tiles, + tileHandlesById, + }; + return { + ...nextState, + tileDebugEntries: rebuildActiveDebugEntries(state.tileDebugEntries, nextState, at), + }; + }); }, onTileLoadStart(tile: Parameters>[0]) { tileLoadCallbacks?.onTileLoadStart?.(tile); - updateDebugEntries((entries) => - reduceTileDebugEntries(entries, { type: 'start', tile, at: Date.now() }) - ); + store.update((state) => { + const at = Date.now(); + const nextState: TiledPointsDebugState = { + ...state, + tileHandlesById: rememberTileHandle(state, tile), + loadingTileIds: [...new Set([...(state.loadingTileIds ?? []), tile.tileId])], + completedTilesById: Object.fromEntries( + Object.entries(state.completedTilesById ?? {}).filter(([tileId]) => tileId !== tile.tileId) + ), + }; + const afterStart = reduceTileDebugEntries(state.tileDebugEntries, { + type: 'start', + tile, + at, + }); + return { + ...nextState, + tileDebugEntries: rebuildActiveDebugEntries(afterStart, nextState, at), + }; + }); }, onTileLoadEnd( tile: Parameters>[0], @@ -40,23 +151,35 @@ export function createTiledPointsDebugHooks( clipBounds: { minX: number; minY: number; maxX: number; maxY: number } ) { tileLoadCallbacks?.onTileLoadEnd?.(tile, result); - updateDebugEntries((entries) => - reduceTileDebugEntries(entries, { - type: 'end', - tile, + const at = Date.now(); + store.update((state) => { + const loadingTileIds = (state.loadingTileIds ?? []).filter((tileId) => tileId !== tile.tileId); + const completedTilesById = { ...(state.completedTilesById ?? {}) }; + const startedAt = + state.tileDebugEntries.find((entry) => entry.tileId === tile.tileId)?.startedAt ?? at; + completedTilesById[tile.tileId] = completedSnapshotFromLoadResult( result, - at: Date.now(), clipBounds, - }) - ); + at, + startedAt + ); + const nextState: TiledPointsDebugState = { + ...state, + tileHandlesById: rememberTileHandle(state, tile), + loadingTileIds, + completedTilesById, + }; + return { + ...nextState, + tileDebugEntries: rebuildActiveDebugEntries(state.tileDebugEntries, nextState, at), + }; + }); }, getTileDebugEntries(): PointsTileDebugEntry[] { - return ((layer.state as unknown as TiledPointsDebugState | undefined)?.tileDebugEntries) ?? []; + return store.getState().tileDebugEntries; }, getTileDebugSignature(): string { - return tileDebugEntriesSignature( - ((layer.state as unknown as TiledPointsDebugState | undefined)?.tileDebugEntries) ?? [] - ); + return tileDebugEntriesSignature(store.getState().tileDebugEntries); }, }; } diff --git a/packages/layers/tests/pointsTileDebug.spec.ts b/packages/layers/tests/pointsTileDebug.spec.ts index 93ce2584..874cd90d 100644 --- a/packages/layers/tests/pointsTileDebug.spec.ts +++ b/packages/layers/tests/pointsTileDebug.spec.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { + completedSnapshotFromLoadResult, formatPointsTileDebugTooltip, reduceTileDebugEntries, } from '../src/pointsTileDebug.js'; @@ -11,6 +12,18 @@ const sampleTile = { bbox: { left: 512, top: 1024, right: 1024, bottom: 512 }, }; +const emptyViewportContext = { + loadingTileIds: new Set(), + completedTilesById: new Map(), + tileHandlesById: new Map(), +}; + +const sampleTile2 = { + tileId: '3-4--1', + index: { x: 3, y: 4, z: -1 }, + bbox: { left: 1536, top: 2048, right: 2048, bottom: 1536 }, +}; + describe('pointsTileDebug', () => { it('transitions tile status through viewport, start, and end events', () => { const at = 1_000; @@ -18,6 +31,10 @@ describe('pointsTileDebug', () => { type: 'viewport', tiles: [sampleTile], at, + context: { + ...emptyViewportContext, + tileHandlesById: new Map([[sampleTile.tileId, sampleTile]]), + }, }); expect(entries[0]?.status).toBe('pending'); @@ -37,6 +54,70 @@ describe('pointsTileDebug', () => { expect(entries[0]?.completedAt).toBe(at + 100); }); + it('restores completed tiles after they re-enter the viewport', () => { + const at = 1_000; + const completedTilesById = new Map([ + [ + sampleTile.tileId, + completedSnapshotFromLoadResult( + { success: true, pointCount: 42, loadMode: 'row-groups' }, + { minX: 512, minY: 512, maxX: 1024, maxY: 1024 }, + at + 100, + at + 10 + ), + ], + ]); + + const entries = reduceTileDebugEntries([], { + type: 'viewport', + tiles: [sampleTile], + at: at + 200, + context: { + loadingTileIds: new Set(), + completedTilesById, + tileHandlesById: new Map([[sampleTile.tileId, sampleTile]]), + }, + }); + + expect(entries[0]?.status).toBe('loaded'); + expect(entries[0]?.pointCount).toBe(42); + }); + + it('includes loading and completed tiles not reported in the latest viewport event', () => { + const at = 1_000; + const completedTilesById = new Map([ + [ + sampleTile2.tileId, + completedSnapshotFromLoadResult( + { success: true, pointCount: 99, loadMode: 'row-groups' }, + { minX: 1536, minY: 1536, maxX: 2048, maxY: 2048 }, + at + 50, + at + 10 + ), + ], + ]); + + const entries = reduceTileDebugEntries([], { + type: 'viewport', + tiles: [sampleTile], + at: at + 100, + context: { + loadingTileIds: new Set([sampleTile.tileId]), + completedTilesById, + tileHandlesById: new Map([ + [sampleTile.tileId, sampleTile], + [sampleTile2.tileId, sampleTile2], + ]), + }, + }); + + expect(entries.map((entry) => entry.tileId).sort()).toEqual( + [sampleTile.tileId, sampleTile2.tileId].sort() + ); + expect(entries.find((entry) => entry.tileId === sampleTile.tileId)?.status).toBe('loading'); + expect(entries.find((entry) => entry.tileId === sampleTile2.tileId)?.pointCount).toBe(99); + }); + it('formats tooltip with elapsed time for in-flight tiles', () => { const tooltip = formatPointsTileDebugTooltip( { @@ -48,7 +129,7 @@ describe('pointsTileDebug', () => { requestedAt: 1_000, startedAt: 1_500, }, - { inFlight: 1, loaded: 0, viewportTotal: 3 }, + { inFlight: 1, loaded: 0, loadedPoints: 0, viewportTotal: 3 }, 2_000 ); expect(tooltip.items.some((item) => item.label === 'elapsed' && item.value === '500ms')).toBe( diff --git a/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx b/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx index 0ce76ddd..6045e579 100644 --- a/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx +++ b/packages/vis/src/SpatialCanvas/PointsStylePanel.tsx @@ -93,7 +93,7 @@ export function PointsStylePanel({