DEGA-279-numerical-attributes

docs/overview/usage.md

@@ -25,4 +25,4 @@ cgm = dega.viz.Clustergram(parquet_data=pq_data, width=500, height=500)
 cgm
 ```
 
-`Clustergram` also accepts `network` or `matrix` arguments, but `parquet_data` is recommended for large datasets.
+`Clustergram` can also be initialized directly from a `Matrix` instance which internally uses `export_viz_parquet`. Passing a legacy JSON `network` is now deprecated.

docs/python/viz/api.md

@@ -4,7 +4,8 @@
 
 The `Clustergram` widget accepts a `parquet_data` argument for efficient
 initialization. Use [`Matrix.export_viz_parquet`](../clust/api.md#celldega.clust.matrix.Matrix.export_viz_parquet)
-to generate this data from a clustered matrix.
+to generate this data from a clustered matrix. Passing a JSON ``network``
+object is deprecated; pass ``matrix`` or ``parquet_data`` instead.
 
 ::: celldega.viz
 

js/deck-gl/matrix/matrix_tooltip.js

@@ -10,37 +10,33 @@ export const get_tooltip = (viz_state, params) => {
   if (object) {
     // Check which layer the tooltip is currently over
     if (layer.id === 'row-label-layer') {
-      // Display the row label when hovering over the row_label_layer
       return {
         html: `Row Label: ${object.name}`,
         style: { color: 'white' },
       };
     } else if (layer.id === 'col-label-layer') {
-      // Display the row label when hovering over the row_label_layer
       return {
         html: `Col Label: ${object.name}`,
         style: { color: 'white' },
       };
     } else if (layer.id === 'row-layer') {
-      // Display the row label when hovering over the row_label_layer
+      const row_attr_name = viz_state.attr.names.row[object.level];
       return {
-        html: `Row Label: ${object.name}`,
+        html: `${row_attr_name}: ${object.name}`,
         style: { color: 'white' },
       };
     } else if (layer.id === 'col-layer') {
-      // Display the row label when hovering over the row_label_layer
+      const col_attr_name = viz_state.attr.names.col[object.level];
       return {
-        html: `Col Label: ${object.name}`,
+        html: `${col_attr_name}: ${object.name}`,
         style: { color: 'white' },
       };
     } else if (layer.id === 'row-dendro-layer') {
-      // Display the row label when hovering over the row_label_layer
       return {
         html: `row-dendro-${object.properties.name}<br>${object.properties.all_names}`,
         style: { color: 'white' },
       };
     } else if (layer.id === 'col-dendro-layer') {
-      // Display the row label when hovering over the row_label_layer
       return {
         html: `row-dendro-${object.properties.name}<br>${object.properties.all_names}`,
         style: { color: 'white' },

js/matrix/cat_data.js

@@ -8,12 +8,16 @@ const colorToRgba = (colorStr, alpha = 255) => {
   return [d3col.r, d3col.g, d3col.b, alpha];
 };
 
-const set_cat_data = (network, viz_state, axis) => {
+export const set_cat_data = (network, viz_state, axis) => {
+
   const isRow = axis === 'row';
   const nodes = isRow ? network.row_nodes : network.col_nodes;
-  const num_cats = isRow
-    ? viz_state.cats.num_cats.row
-    : viz_state.cats.num_cats.col;
+  const num_attr = isRow
+    ? viz_state.attr.num.row
+    : viz_state.attr.num.col;
+  const max_abs = isRow
+    ? viz_state.attr.maxabs.row
+    : viz_state.attr.maxabs.col;
   const cat_offset = isRow
     ? viz_state.viz.row_cat_offset
     : viz_state.viz.col_cat_offset;
@@ -26,43 +30,48 @@ const set_cat_data = (network, viz_state, axis) => {
 
   const cat_data = nodes
     .flatMap((node, node_index) => {
-      return Array.from({ length: num_cats }).map((_, cat_index) => {
-        const cat_name = `cat-${cat_index}`;
-        const inst_cat = node[cat_name];
+      return Array.from({ length: num_attr }).map((_, attr_index) => {
+        let value;
+        let color_rgba;
+        const maxVal = max_abs[attr_index];
+        const isNumeric = maxVal !== null && maxVal !== undefined;
 
-        if (!inst_cat) {
-          return null;
+        if (isNumeric) {
+          const attr_name = `num-${attr_index}`;
+          value = node[attr_name];
+          if (value === undefined || value === null || isNaN(value)) {
+            return null;
+          }
+          // orange rgba for positive, dark gray rgba for negative
+          const neg = [255, 165, 0];
+          const pos = [169, 169, 169];
+          const color = value >= 0 ? pos : neg;
+          const scale = maxVal === 0 ? 1 : maxVal;
+          const alpha = Math.min(1, Math.abs(value) / scale);
+          color_rgba = [...color, Math.round(alpha * 255)];
+        } else {
+          const cat_name = `cat-${attr_index}`;
+          value = node[cat_name];
+          if (!value) {
+            return null;
+          }
+          const ini_color = network.global_cat_colors[value];
+          color_rgba = colorToRgba(ini_color, 255);
         }
 
-        const ini_color = network.global_cat_colors[inst_cat];
-        const color_rgba = colorToRgba(ini_color, 255);
-
         return {
           position: isRow
-            ? [
-                cat_offset * (cat_index + 0.5) + 20,
-                node_offset * (node_index + 0.5),
-              ]
-            : [
-                node_offset * (node_index + 0.5),
-                cat_offset * (cat_index + 1.5) - 30,
-              ],
+            ? [cat_offset * (attr_index + 0.5) + 20, node_offset * (node_index + 0.5)]
+            : [node_offset * (node_index + 0.5), cat_offset * (attr_index + 1.5) - 30],
           color: color_rgba,
-          name: inst_cat,
-          level: cat_index,
+          name: value,
+          level: attr_index,
           original_index: node_index,
         };
       });
     })
     .filter(Boolean);
 
   return cat_data;
-};
 
-export const set_row_cat_data = (network, viz_state) => {
-  return set_cat_data(network, viz_state, 'row');
-};
-
-export const set_col_cat_data = (network, viz_state) => {
-  return set_cat_data(network, viz_state, 'col');
-};
+};

js/matrix/mat_data.js

@@ -1,30 +1,37 @@
 export const set_mat_data = (network, viz_state) => {
-  // Iterate over each row and column in network.mat
-  let inst_color;
-  network.mat.forEach((rowArray, index_row) => {
-    rowArray.forEach((tile_value, index_col) => {
-      if (tile_value >= 0) {
-        inst_color = [255, 0, 0];
-      } else {
-        inst_color = [0, 0, 255];
-      }
+  const { mat } = network;
+  const { col_offset, row_offset } = viz_state.viz;
+  const max_abs_value = viz_state.mat.max_abs_value;
+  const mat_data = viz_state.mat.mat_data;
+
+  for (let index_row = 0; index_row < mat.length; index_row++) {
+    const rowArray = mat[index_row];
+
+    for (let index_col = 0; index_col < rowArray.length; index_col++) {
+      const tile_value = rowArray[index_col];
+
+      // Optional: skip small/zero values to reduce memory
+      if (tile_value == null || Math.abs(tile_value) < 1e-6) continue;
+
+      const inst_color = tile_value >= 0 ? [255, 0, 0] : [0, 0, 255];
 
       const p = {
         position: [
-          viz_state.viz.col_offset * (index_col + 0.5),
-          viz_state.viz.row_offset * (index_row + 1.5),
+          col_offset * (index_col + 0.5),
+          row_offset * (index_row + 1.5),
         ],
         color: [
           inst_color[0],
           inst_color[1],
           inst_color[2],
-          (255 * Math.abs(tile_value)) / viz_state.mat.max_abs_value,
+          (255 * Math.abs(tile_value)) / max_abs_value,
         ],
         value: tile_value,
         row: index_row,
         col: index_col,
       };
-      viz_state.mat.mat_data.push(p);
-    });
-  });
+
+      mat_data.push(p);
+    }
+  }
 };

js/matrix/set_constants.js

@@ -26,12 +26,19 @@ export const set_mat_constants = (
   viz_state.viz = {};
   viz_state.viz.height_margin = 100;
 
-  viz_state.cats = {};
-  viz_state.cats.num_cats = {};
-
-  // use network cat_colors to figure out how many categories there are
-  viz_state.cats.num_cats.row = Object.keys(network.row_cats).length;
-  viz_state.cats.num_cats.col = Object.keys(network.col_cats).length;
+  viz_state.attr = {};
+  viz_state.attr.names = {
+    row: network.row_attr || [],
+    col: network.col_attr || [],
+  };
+  viz_state.attr.maxabs = {
+    row: network.row_attr_maxabs || [],
+    col: network.col_attr_maxabs || [],
+  };
+  viz_state.attr.num = {
+    row: viz_state.attr.names.row.length,
+    col: viz_state.attr.names.col.length,
+  };
 
   viz_state.root.style.height = `${height + viz_state.viz.height_margin}px`;
 
@@ -40,9 +47,9 @@ export const set_mat_constants = (
   viz_state.viz.col_cat_offset = 9;
 
   viz_state.viz.mat_width =
-    width - viz_state.viz.row_cat_offset * viz_state.cats.num_cats.row;
+    width - viz_state.viz.row_cat_offset * viz_state.attr.num.row;
   viz_state.viz.mat_height =
-    height - viz_state.viz.col_cat_offset * viz_state.cats.num_cats.col;
+    height - viz_state.viz.col_cat_offset * viz_state.attr.num.col;
 
   viz_state.mat = {};
   viz_state.mat.num_rows = network.mat.length;
@@ -99,12 +106,12 @@ export const set_mat_constants = (
 
   viz_state.viz.col_region =
     (viz_state.viz.col_cat_height + viz_state.viz.extra_space.col) *
-      viz_state.cats.num_cats.col +
+      viz_state.attr.num.col +
     viz_state.viz.col_label;
 
   viz_state.viz.row_region =
     (viz_state.viz.row_cat_width + viz_state.viz.extra_space.row) *
-      viz_state.cats.num_cats.row +
+      viz_state.attr.num.row +
     viz_state.viz.row_label;
 
   viz_state.viz.col_width = viz_state.viz.mat_width / viz_state.mat.num_cols;

js/viz/matrix_viz.js

@@ -52,7 +52,7 @@ import { get_tooltip } from '../deck-gl/matrix/matrix_tooltip';
 import { on_view_state_change } from '../deck-gl/matrix/on_view_state_change';
 import { ini_views, ini_view_state } from '../deck-gl/matrix/views';
 import { ini_zoom_data } from '../deck-gl/matrix/zoom';
-import { set_row_cat_data, set_col_cat_data } from '../matrix/cat_data';
+import { set_cat_data } from '../matrix/cat_data';
 import { calc_dendro_polygons, ini_dendro } from '../matrix/dendro';
 import { set_row_label_data, set_col_label_data } from '../matrix/label_data';
 import { set_mat_data } from '../matrix/mat_data';
@@ -101,8 +101,9 @@ export const matrix_viz = async (
   set_row_label_data(network, viz_state);
   set_col_label_data(network, viz_state);
 
-  viz_state.cats.row_cat_data = set_row_cat_data(network, viz_state);
-  viz_state.cats.col_cat_data = set_col_cat_data(network, viz_state);
+  viz_state.cats = {};
+  viz_state.cats.row_cat_data = set_cat_data(network, viz_state, 'row');
+  viz_state.cats.col_cat_data = set_cat_data(network, viz_state, 'col');
 
   ini_dendro(viz_state);
 
@@ -147,4 +148,6 @@ export const matrix_viz = async (
 
   el.appendChild(ui_container);
   el.appendChild(viz_state.root);
+
+  return () => deck_mat.finalize();
 };

js/widget.js

@@ -118,12 +118,15 @@ const render_landscape = async ({ model, el }) => {
 };
 
 const render_matrix_new = async ({ model, el }) => {
-  let network = model.get('network');
+  // let network = model.get('network');
+  let network;
   const width = model.get('width');
   const height = model.get('height');
 
   const matBytes = model.get('mat_parquet');
   if (matBytes && matBytes.byteLength > 0) {
+
+    console.log('Loading network from Parquet files');
     network = await networkFromParquet(
       model.get('network_meta'),
       matBytes,
@@ -134,11 +137,12 @@ const render_matrix_new = async ({ model, el }) => {
     );
   }
 
-  matrix_viz(model, el, network, width, height);
+  return matrix_viz(model, el, network, width, height);
 };
 
 // Main render function - no export keyword
-function render({ model, el }) {
+async function render({ model, el }) {
+  let cleanup = null;
   try {
     const componentType = model.get('component');
 
@@ -152,15 +156,32 @@ function render({ model, el }) {
 
     switch (componentType) {
       case 'Landscape':
-        return render_landscape({ model, el });
+        cleanup = await render_landscape({ model, el });
+        break;
       case 'Matrix':
-        return render_matrix_new({ model, el });
+        cleanup = await render_matrix_new({ model, el });
+        break;
       default:
         handleValidationWarning(`Unknown component type: ${componentType}`, {
           data: { componentType, model: model?.id || 'unknown' },
         });
         return;
     }
+
+    model.on('msg:custom', (msg) => {
+      if (msg.event === 'finalize' && cleanup) {
+        try {
+          if (typeof cleanup === 'function') {
+            cleanup();
+          } else if (cleanup.finalize) {
+            cleanup.finalize();
+          }
+        } catch (e) {
+          console.error('Error finalizing deck:', e);
+        }
+        cleanup = null;
+      }
+    });
   } catch (error) {
     const errorResult = handleAsyncError(error, {
       context: 'render function',

src/celldega/clust/constants.py

@@ -149,7 +149,7 @@ def normalize_axis(axis: AxisInput) -> str:
     "memory_threshold": 2e9,  # 2GB for memory mapping
     "cache_size_limit": 5,
     "large_matrix_threshold": 10000,
-    "memory_warning_threshold": 50_000_000,
+    "matrix_cell_threshold": 5_000_000,
     "sample_hash_size": 100,
 }
 
@@ -174,6 +174,10 @@ def normalize_axis(axis: AxisInput) -> str:
     "col_nodes": [],
     "mat": [],
     "linkage": {Axis.ROW.value: [], Axis.COL.value: []},
+    "row_attr": [],
+    "col_attr": [],
+    "row_attr_maxabs": [],
+    "col_attr_maxabs": [],
     "cat_colors": {Axis.ROW.value: {}, Axis.COL.value: {}},
     "matrix_colors": {"pos": "red", "neg": "blue"},
     "views": [],
@@ -187,7 +191,7 @@ def normalize_axis(axis: AxisInput) -> str:
     "invalid_norm": "Normalization '{}' not supported. Use: total, zscore, qn",
     "missing_category": "Category '{}' not found in {}",
     "no_valid_features": "No valid {} features found",
-    "clustering_size": "Matrix has {} columns. Use force=True to override.",
+    "clustering_size": "Matrix has {} rows and {} columns exceeding the total recommended matrix cell count of {}. Use force=True to override.",
     "missing_scanpy": "scanpy required: pip install scanpy",
     "missing_metadata": "{} metadata missing for: {}...",
 }