use ipydatawidgets to continuously sync classData as array

.vscode/settings.json

@@ -2,5 +2,5 @@
     "python.linting.pylintEnabled": false,
     "python.linting.pylamaEnabled": true,
     "python.linting.enabled": true,
-    "python.pythonPath": "/home/ian/anaconda3/envs/colonies/bin/python"
+    "python.pythonPath": "/home/ian/anaconda3/envs/ipysegment/bin/python"
 }

examples/Untitled.ipynb

@@ -0,0 +1,87 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "from ipydatawidgets import NDArrayWidget\n",
+    "\n",
+    "raw_data = np.ones((100, 100, 3), dtype=np.float32)\n",
+    "datawidget = NDArrayWidget(raw_data)\n",
+    "\n",
+    "# Below, my_other_widget has a trait `data` of the type `Instance(NDArrayWidget)`\n",
+    "# my_other_widget.data = datawidget"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "NDArrayWidget(array=array([[[1., 1., 1.],\n",
+       "        [1., 1., 1.],\n",
+       "        [1., 1., 1.],\n",
+       "        ...,\n",
+       "        [1.…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "datawidget"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ipydatawidgets import DataUnion"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.3"
+  },
+  "widgets": {
+   "application/vnd.jupyter.widget-state+json": {
+    "state": {},
+    "version_major": 2,
+    "version_minor": 0
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

ipysegment/segment.py

@@ -12,13 +12,17 @@
 from ._frontend import module_name, module_version
 from .utils import binary_image
 from numpy import frombuffer, uint8, copy
+import zlib
+from ipydatawidgets import shape_constraints, DataUnion, data_union_serialization
+import numpy as np
+# from ipydatawidgets.traits import shape_constraints
 
 from traitlets import Bytes, Bool, CInt, Unicode
 import logging
 logger = logging.getLogger()
-import zlib
 # import sys
 
+
 def deserializeImage(json, obj):
     logger.debug('yikes')
     logger.debug('deserializing:', json)
@@ -40,7 +44,6 @@ def deserializeImage(json, obj):
     'from_json': deserializeImage,
 }
 
-
 class segmenter(DOMWidget):
     """TODO: Add docstring here
     """
@@ -55,6 +58,12 @@ class segmenter(DOMWidget):
     erasing = Bool(True).tag(sync=True)
     # underlying info for labels - this handles the syncing to ts
     _labels = Bytes(default_value=None, allow_none=True, read_only=True).tag(sync=True, **labels_serialization)
+    # yikes = 
+
+    data = DataUnion(
+        np.zeros([10, 10, 4], dtype=np.uint8),
+        dtype='uint8', shape_constraint=shape_constraints(None, None, 4),  # 2D RGBA
+    ).tag(sync=True, **data_union_serialization)
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)

ipysegment/utils.py

@@ -6,18 +6,18 @@
 """Binary module."""
 from io import BytesIO
 
-from PIL import Image as PILImage
+# from PIL import Image as PILImage
 
 import numpy as np
 
 
-def image_bytes_to_array(im_bytes):
-    """Turn raw image bytes into a NumPy array."""
-    im_file = BytesIO(im_bytes)
+# def image_bytes_to_array(im_bytes):
+#     """Turn raw image bytes into a NumPy array."""
+#     im_file = BytesIO(im_bytes)
 
-    im = PILImage.open(im_file)
+#     im = PILImage.open(im_file)
 
-    return np.array(im)
+#     return np.array(im)
 
 
 def binary_image(ar):

package.json

@@ -52,13 +52,16 @@
   },
   "dependencies": {
     "@jupyter-widgets/base": "^1.1.10 || ^2.0.1 || ^3",
+    "jupyter-dataserializers": "^2.2.0",
+    "ndarray": "^1.0.19",
     "pako": "^1.0.11"
   },
   "devDependencies": {
     "@phosphor/application": "^1.6.0",
     "@phosphor/widgets": "^1.6.0",
     "@types/expect.js": "^0.3.29",
     "@types/mocha": "^5.2.5",
+    "@types/ndarray": "^1.0.8",
     "@types/node": "^10.11.6",
     "@types/pako": "^1.0.1",
     "@types/webpack-env": "^1.13.6",

setup.py

@@ -87,6 +87,7 @@
     include_package_data = True,
     install_requires = [
         'ipywidgets>=7.0.0',
+        'ipydatawidgets>=4.0.1'
     ],
     extras_require = {
         'test': [

src/widget.ts

@@ -6,10 +6,14 @@ import { DOMWidgetModel, DOMWidgetView, ISerializers, Dict } from '@jupyter-widg
 import { MODULE_NAME, MODULE_VERSION } from './version';
 
 import * as pako from 'pako';
+import { data_union_serialization, getArray, listenToUnion } from 'jupyter-dataserializers';
+
+// import ndarray = require('ndarray');
 // import { toBytes } from './utils';
 
 // Import the CSS
 import '../css/widget.css';
+import ndarray from 'ndarray';
 
 function serializeImageData(image: ImageData) {
   // seems like it would make more sense to just pass the whole object
@@ -52,11 +56,15 @@ export class segmentModel extends DOMWidgetModel {
       serialize: serializeImageData,
       deserialize: deserializeImageData,
     },
+    data: data_union_serialization,
   };
 
   initialize(attributes: any, options: any) {
     super.initialize(attributes, options);
 
+    console.log('yikes');
+    listenToUnion(this, 'data', this._updateClassData.bind(this), true);
+
     this.imgCanvas = document.createElement('canvas');
     this.classCanvas = document.createElement('canvas');
     this.imgContext = getContext(this.imgCanvas);
@@ -102,6 +110,25 @@ export class segmentModel extends DOMWidgetModel {
     this.save_changes();
   }
 
+  private _updateClassData() {
+    console.log('here?');
+    const data = getArray(this.get('data'));
+    if (data) {
+      const imageData = new ImageData(
+        new Uint8ClampedArray(data.data),
+        data.shape[0],
+        data.shape[1]
+      );
+      this.classContext.putImageData(imageData, 0, 0);
+      console.log(this.classCanvas);
+    }
+    this._forEachView((view) => {
+      view.resize();
+      view.redraw();
+    });
+    console.log(data);
+  }
+
   putImageData(bufferMetadata: any, buffers: any) {
     // const [bufferMetadata, dx, dy] = args;
 
@@ -180,7 +207,7 @@ export class segmentView extends DOMWidgetView {
     container.appendChild(this.displayCanvas);
     container.appendChild(this.previewCanvas);
 
-    console.log(this.el);
+    // console.log(this.el);
     // this.el
     this.previewContext = getContext(this.previewCanvas);
     this.displayContext = getContext(this.displayCanvas);
@@ -276,6 +303,13 @@ export class segmentView extends DOMWidgetView {
       this.drawImageScaled();
       this.lassoing = false;
       this.previewContext.clearRect(0, 0, this.previewCanvas.width, this.previewCanvas.height);
+      // sync the data after all the drawing to keep drawing as snappy as possible
+      const h = this.model.classCanvas.height;
+      const w = this.model.classCanvas.width;
+      const imgData = this.model.classContext.getImageData(0, 0, w, h);
+      const nd = ndarray(imgData.data, [w, h, 4]);
+      this.model.set('data', nd);
+      this.model.save_changes();
     }
     this.panning = false;
   };