Tell a fully-loaded feature apart from one the memory cap only sampled

.changeset/resident-feature-coverage.md

@@ -0,0 +1,25 @@
+---
+'@spatialdata/core': patch
+'@spatialdata/vis': minor
+---
+
+Distinguish a feature that is fully loaded from one the memory cap only sampled.
+
+`resident` means a feature has **at least one** point inside the memory cap. On a
+truncated element that is true of nearly every feature, so the panel greyed nothing,
+showed each feature's full dataset count beside it, and presented a sample as the
+whole answer. On a Xenium transcripts element (8.07M points, 4M cap) all 541
+features read as resident while half the data was absent.
+
+`describeFeatureRowState` takes optional `residentPointCount` / `datasetPointCount`
+and returns a new `partial` tone — drawn, so not greyed, but labelled and explained
+with both counts and the share. A completed feature-index scan vetoes it: that
+supplies the feature whole, so its resident shortfall is no longer what is on screen.
+The built-in panel shows `resident / dataset` on those rows and a summary line, and
+falls back to the previous behaviour whenever counts are unknown.
+
+Fixes a latent bug this exposed: `getResidentFeatureCounts` answered from the preload
+result's own tally, which is frozen in the resident preview's code space and is not
+remapped when the full catalog supersedes it. For a dictionary-only element that
+attributed one gene's count to another. Counts now derive from the reconciled row
+codes, memoised on the same identity as the resident-codes set.

packages/core/src/engine/PointsResolver.ts

@@ -140,6 +140,9 @@ interface PointsEntry {
    * identity. A DATA memo (a Set), not a render resource — it stays in core. */
   residentCodes?: ReadonlySet<number>;
   residentCodesSource?: ArrayLike<number>;
+  /** Memoized per-code tally of the resident {@link rowCodes}, invalidated by the
+   * same identity. Shares `residentCodesSource` — both are derived from one array. */
+  residentCounts?: ReadonlyMap<number, number>;
   /**
    * Whole-dataset points for the active selection — the feature-index scan — as a
    * {@link RequestSlot}. Keyed by `` `${signature}#${cap}` ``: the selected-codes
@@ -514,16 +517,42 @@ export class PointsResolver implements ResourceResolver<PointsResolveConfig, Poi
   }
 
   /**
-   * Running per-feature point counts for the resident batch (`code → rows`),
-   * accumulated while the preload streamed. These are counts over the RESIDENT
-   * WINDOW, not the dataset — the authoritative dataset totals come from the catalog
-   * scan — but they are available almost immediately, so the panel can show stats
-   * instead of blanks while that scan runs. Reads the in-flight partial first so the
-   * numbers climb as points arrive.
+   * Per-feature point counts for the resident batch (`code → rows`).
+   *
+   * Counts over the RESIDENT WINDOW, not the dataset — the authoritative totals come
+   * from the catalog scan. Comparing the two is what tells a panel that a feature it
+   * is drawing is only partly there: `resident` means "has at least one point inside
+   * the memory cap", so on a truncated element every feature can be resident while
+   * half the dataset is absent.
+   *
+   * Derived from the settled {@link rowCodes} in preference to the preload result's
+   * own tally, because for a dictionary-only element the codes get re-expressed when
+   * the full catalog supersedes the resident preview ({@link reconcileRowCodes}) and
+   * the preload's frozen map is NOT remapped with them — it would keep answering in
+   * the old code space, attributing one gene's count to another. The preload tally
+   * remains the fallback: it streams, so the numbers climb while points arrive and
+   * before row codes settle, and no remap can have happened yet at that point.
    */
   getResidentFeatureCounts(key: string): ReadonlyMap<number, number> | undefined {
     const entry = this.entries.get(key);
-    return entry?.preload.partial?.featureCodeCounts ?? entry?.preload.lastGood?.featureCodeCounts;
+    if (!entry) {
+      return undefined;
+    }
+    const rowCodes = entry.rowCodes.value;
+    if (rowCodes !== undefined) {
+      if (entry.residentCounts && entry.residentCodesSource === rowCodes) {
+        return entry.residentCounts;
+      }
+      const counts = new Map<number, number>();
+      for (let i = 0; i < rowCodes.length; i += 1) {
+        const code = rowCodes[i] as number;
+        counts.set(code, (counts.get(code) ?? 0) + 1);
+      }
+      entry.residentCounts = counts;
+      entry.residentCodesSource = rowCodes;
+      return counts;
+    }
+    return entry.preload.partial?.featureCodeCounts ?? entry.preload.lastGood?.featureCodeCounts;
   }
 
   /**
@@ -706,6 +735,7 @@ export class PointsResolver implements ResourceResolver<PointsResolveConfig, Poi
         // in-flight reload for a different cap.
         slot.settle(memoryCap, PointsResolver.sliceResidentBatch(resident, memoryCap));
         entry.residentCodes = undefined;
+        entry.residentCounts = undefined;
         entry.residentCodesSource = undefined;
         const codes = entry.rowCodes.value;
         // Deliberately conditional, and NOT re-keyed unconditionally on a shed.
@@ -765,6 +795,7 @@ export class PointsResolver implements ResourceResolver<PointsResolveConfig, Poi
       // load is exactly the corruption R1/R5 were.
       if (signal.aborted) return data;
       entry.residentCodes = undefined;
+      entry.residentCounts = undefined;
       entry.residentCodesSource = undefined;
       entry.featureCodeColumn = data.hasFeatureCodeColumn === true;
       if (data.featureCatalog !== undefined) {
@@ -1071,6 +1102,7 @@ export class PointsResolver implements ResourceResolver<PointsResolveConfig, Poi
     entry.rowCodes.settle(cap, remapRowFeatureCodes(codes, source, target));
     entry.rowCodesCatalog = target;
     entry.residentCodes = undefined;
+    entry.residentCounts = undefined;
     entry.residentCodesSource = undefined;
   }
 

packages/layers/tests/pointsDataEngine.spec.ts

@@ -530,6 +530,14 @@ describe('PointsDataEngine — codes with the geometry preload', () => {
     expect(Array.from(engine.getRowFeatureCodes('pts:remap')!)).toEqual([1, 0, 1]);
     // The resident-codes memo reflects the remapped values.
     expect([...engine.getResidentFeatureCodes('pts:remap')!].sort()).toEqual([0, 1]);
+    // ...and so do the per-feature COUNTS. The preload result carries its own
+    // `featureCodeCounts`, frozen in the preview space and never remapped, so
+    // answering from it here would report GeneB's two points as GeneA's — a panel
+    // showing "resident 2 of N" against the wrong gene. Counts are derived from the
+    // reconciled row codes for exactly that reason.
+    const counts = engine.getResidentFeatureCounts('pts:remap')!;
+    expect(counts.get(1)).toBe(2); // GeneB, full space
+    expect(counts.get(0)).toBe(1); // GeneA, full space
   });
 
   it('does not remap when codes are authoritative (a real feature-code column)', async () => {

packages/vis/src/SpatialCanvas/PointsFeatureFilterPanel.tsx

@@ -111,6 +111,16 @@ const errorStatStyle: CSSProperties = {
   fontSize: '11px',
 };
 
+/** The resident figure in a `resident / dataset` pair — the number that is actually
+ * on screen, so it carries the emphasis. */
+const shortfallStyle: CSSProperties = {
+  color: '#d0a24c',
+};
+
+const ofTotalStyle: CSSProperties = {
+  color: '#777',
+};
+
 const countStyle: CSSProperties = {
   color: '#888',
   fontSize: '11px',
@@ -199,6 +209,19 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
     entry.count ?? partialCounts?.get(entry.code);
   const countIsPartial = (entry: { code: number; count?: number }): boolean =>
     entry.count === undefined && partialCounts?.get(entry.code) !== undefined;
+  // Dataset totals by code, so a row can be compared against what is resident without
+  // rescanning `entries` per row.
+  const datasetCountByCode = new Map<number, number>(
+    entries.flatMap((entry) => (entry.count !== undefined ? [[entry.code, entry.count]] : []))
+  );
+  /** Resident points for a feature, when that is meaningfully LESS than the dataset —
+   * i.e. there is a shortfall worth showing. `undefined` otherwise. */
+  const residentShortfall = (entry: { code: number; count?: number }): number | undefined => {
+    if (entry.count === undefined) return undefined;
+    const resident = residentFeatureCounts?.get(entry.code);
+    if (resident === undefined || resident >= entry.count) return undefined;
+    return resident;
+  };
   // The selection persists as NAMES (see `PointsLayerConfig.featureNames`), but the
   // rest of this panel — checkboxes, greying, the engine reads — works in codes.
   // Resolve once here against the catalog we are already rendering.
@@ -371,12 +394,24 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
       scanning,
       supportsOnDemandLoad,
       residentKnown,
+      residentPointCount: residentFeatureCounts?.get(code),
+      datasetPointCount: datasetCountByCode.get(code),
     });
     return { resident, rendered, selected, state };
   };
   const notLoadedCount = residentKnown
     ? entries.reduce((total, entry) => total + (rowInfo(entry.code).state.greyed ? 1 : 0), 0)
     : 0;
+  // Features that ARE drawn but only in part. Counted separately from `notLoadedCount`
+  // because they are the opposite failure of understanding: those rows look completely
+  // healthy — un-greyed, with a full dataset count beside them — while most of their
+  // points are outside the cap.
+  const partialCount = residentKnown
+    ? entries.reduce(
+        (total, entry) => total + (rowInfo(entry.code).state.tone === 'partial' ? 1 : 0),
+        0
+      )
+    : 0;
 
   return (
     <div style={panelStyle}>
@@ -401,6 +436,13 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
             : "not in the loaded sample (greyed below) — this dataset has no feature index, so they can't be shown until the row cap is raised or it's rewritten with one."}
         </div>
       ) : null}
+      {partialCount > 0 ? (
+        <div style={helperStyle}>
+          {partialCount} of {entries.length} feature{entries.length === 1 ? '' : 's'} only partly
+          loaded — the resident window is capped, so the canvas is drawing a sample of each. Select
+          one to fetch it in full, or raise the memory cap.
+        </div>
+      ) : null}
       {matchingLoadState?.failed ? (
         // A failed scan still DRAWS: the render path falls back to filtering the
         // resident batch, so the canvas shows whichever part of the selection was
@@ -526,19 +568,43 @@ export function PointsFeatureFilterPanel({ config }: PointsFeatureFilterPanelPro
                   ⟲
                 </button>
               ) : null}
-              {hasAnyCounts ? (
-                <span
-                  style={countStyle}
-                  title={
-                    countIsPartial(entry)
-                      ? 'Points loaded so far (resident window) — dataset total still counting'
-                      : 'Points in the dataset'
-                  }
-                >
-                  {countIsPartial(entry) ? '≥' : ''}
-                  {formatFeatureCount(effectiveCount(entry))}
-                </span>
-              ) : null}
+              {hasAnyCounts
+                ? (() => {
+                    // Once dataset totals land, keep showing the resident tally too when
+                    // it falls short — the panel used to drop it, which is what let a
+                    // capped element present "1,182,402" for a feature it was drawing a
+                    // fraction of. `rendered` means a scan supplied it whole, so the
+                    // shortfall is no longer what's on screen.
+                    const shortfall =
+                      state.tone === 'partial' ? residentShortfall(entry) : undefined;
+                    return (
+                      <span
+                        style={countStyle}
+                        title={
+                          shortfall !== undefined
+                            ? `${shortfall.toLocaleString()} of ${formatFeatureCount(
+                                entry.count
+                              )} points are inside the memory cap`
+                            : countIsPartial(entry)
+                              ? 'Points loaded so far (resident window) — dataset total still counting'
+                              : 'Points in the dataset'
+                        }
+                      >
+                        {shortfall !== undefined ? (
+                          <>
+                            <span style={shortfallStyle}>{shortfall.toLocaleString()}</span>
+                            <span style={ofTotalStyle}> / {formatFeatureCount(entry.count)}</span>
+                          </>
+                        ) : (
+                          <>
+                            {countIsPartial(entry) ? '≥' : ''}
+                            {formatFeatureCount(effectiveCount(entry))}
+                          </>
+                        )}
+                      </span>
+                    );
+                  })()
+                : null}
             </label>
           );
         })}

packages/vis/src/SpatialCanvas/featureRowState.ts

@@ -8,7 +8,14 @@
 
 /** Why a feature row is (or isn't) greyed — drives both the dimming and the
  * diagnostic tooltip so they can never disagree. */
-export type FeatureRowTone = 'resident' | 'loaded' | 'cached' | 'loading' | 'noIndex' | 'notLoaded';
+export type FeatureRowTone =
+  | 'resident'
+  | 'partial'
+  | 'loaded'
+  | 'cached'
+  | 'loading'
+  | 'noIndex'
+  | 'notLoaded';
 
 export interface FeatureRowState {
   tone: FeatureRowTone;
@@ -33,6 +40,18 @@ export interface FeatureRowStateInput {
   supportsOnDemandLoad: boolean;
   /** The resident set is known (false → we can't distinguish, treat as shown). */
   residentKnown: boolean;
+  /**
+   * Points of this feature inside the resident window, and in the whole dataset.
+   *
+   * Together these separate "resident" from "all here", which `resident` alone
+   * cannot: it means *at least one* point made the memory cap. On a truncated
+   * element every feature is typically resident — one point each is enough — while
+   * most of the data is absent, and a row that says nothing about that reads as a
+   * complete answer. Both `undefined` until the counts are known, which is the only
+   * time this classification falls back to the older, blunter one.
+   */
+  residentPointCount?: number;
+  datasetPointCount?: number;
 }
 
 /**
@@ -52,6 +71,8 @@ export function describeFeatureRowState({
   scanning,
   supportsOnDemandLoad,
   residentKnown,
+  residentPointCount,
+  datasetPointCount,
 }: FeatureRowStateInput): FeatureRowState {
   if (!residentKnown) {
     return {
@@ -62,6 +83,30 @@ export function describeFeatureRowState({
     };
   }
   if (resident) {
+    // Resident but incomplete: drawn, so not greyed, but the row must not imply the
+    // whole feature is on screen. Ranked above plain `resident` because the shortfall
+    // is the more useful fact when it exists.
+    //
+    // `rendered` vetoes it: a completed scan supplies the feature whole, so its
+    // resident shortfall says nothing about what is drawn. Without this veto a
+    // feature would be loaded in full and still be labelled partial.
+    if (
+      !rendered &&
+      residentPointCount !== undefined &&
+      datasetPointCount !== undefined &&
+      residentPointCount < datasetPointCount
+    ) {
+      const percent = datasetPointCount > 0 ? (residentPointCount / datasetPointCount) * 100 : 0;
+      return {
+        tone: 'partial',
+        greyed: false,
+        label: 'partial',
+        reason:
+          `Only ${residentPointCount.toLocaleString()} of ${datasetPointCount.toLocaleString()} points ` +
+          `(${formatCoveragePercent(percent)}) are inside the memory cap, and that is what is drawn. ` +
+          'Select it to fetch the rest, or raise the cap.',
+      };
+    }
     return {
       tone: 'resident',
       greyed: false,
@@ -111,6 +156,18 @@ export function describeFeatureRowState({
   };
 }
 
+/**
+ * A coverage share, rounded so it never reads as more certain than it is: `<1%`
+ * rather than `0%` for a feature with a handful of points in a huge window, and
+ * `>99%` rather than `100%` for one that is all-but-complete — the two cases where a
+ * naive round would claim the opposite of the truth.
+ */
+function formatCoveragePercent(percent: number): string {
+  if (percent > 0 && percent < 1) return '<1%';
+  if (percent < 100 && percent > 99) return '>99%';
+  return `${Math.round(percent)}%`;
+}
+
 /** Opacity for a row given its state: crisp when its points are on screen,
  * mid-dim while loading, fully dim when not loaded. */
 export function featureRowOpacity(state: FeatureRowState): number {