Incremental Hypergrid loading; viewer transform updates no longer stack.

.travis.yml

@@ -9,9 +9,6 @@ node_js:
 env:
   global:
     - EM_USE_GLOBAL_CACHE=1
-    - CACHE_DIR=$HOME/.cache/docker
-    - CACHE_FILE_EMSCRIPTEN=$CACHE_DIR/emscripten.tar.gz
-    - CACHE_FILE_PUPPETEER=$CACHE_DIR/puppeteer.tar.gz
 
 addons:
   apt:
@@ -23,15 +20,9 @@ sudo: required
 services:
   - docker
 
-cache:
-  directories:
-    - $CACHE_DIR
-
 before_install:
   - mkdir boost_includes
   - cp -r /usr/include/boost boost_includes/
-  - if [ -f ${CACHE_FILE_EMSCRIPTEN} ]; then gunzip -c ${CACHE_FILE_EMSCRIPTEN} | docker load; fi
-  - if [ -f ${CACHE_FILE_PUPPETEER} ]; then gunzip -c ${CACHE_FILE_PUPPETEER} | docker load; fi
   - docker run -dit --name emscripten -v $(pwd):/src trzeci/emscripten:sdk-incoming-64bit bash
 
 install:
@@ -40,7 +31,3 @@ install:
 script:
   - docker exec -it emscripten npm run start
   - npm run travis_test
-
-before_cache:
-  - if [ ! -f ${CACHE_FILE_EMSCRIPTEN} ]; then docker save trzeci/emscripten:sdk-incoming-64bit | gzip > ${CACHE_FILE_EMSCRIPTEN}; fi
-  - if [ ! -f ${CACHE_FILE_PUPPETEER} ]; then docker save zenato/puppeteer | gzip > ${CACHE_FILE_PUPPETEER}; fi

packages/perspective-common/common.config.js

@@ -53,7 +53,7 @@ module.exports = function() {
                 loader: "babel-loader",
                 options: {
                     presets: ['env'],
-                    plugins: ['transform-runtime', ["transform-es2015-for-of", {
+                    plugins: ['transform-promise-to-bluebird', 'transform-async-to-bluebird', 'transform-runtime', ["transform-es2015-for-of", {
                         "loose": true
                     }]]
                 }

packages/perspective-common/index.js

@@ -7,6 +7,11 @@
  *
  */
 
+require('bluebird').config({
+    cancellation: true,
+    longStackTraces: false
+});
+
 /**
  * Detect Internet Explorer.
  *

packages/perspective-common/package.json

@@ -16,12 +16,15 @@
   "author": "",
   "license": "Apache",
   "dependencies": {
+    "bluebird": "^3.5.1",
     "babel-runtime": "^6.26.0"
   },
   "devDependencies": {
     "babel-core": "^6.26.0",
     "babel-loader": "^7.1.2",
+    "babel-plugin-transform-async-to-bluebird": "^1.1.1",
     "babel-plugin-transform-es2015-for-of": "^6.23.0",
+    "babel-plugin-transform-promise-to-bluebird": "^1.1.1",
     "babel-plugin-transform-runtime": "^6.23.0",
     "babel-polyfill": "^6.26.0",
     "babel-preset-env": "^1.6.0",

packages/perspective-viewer-highcharts/src/js/highcharts.js

@@ -7,6 +7,8 @@
  *
  */
 
+import '@jpmorganchase/perspective-common';
+
 import highcharts from 'highcharts';
 
 const Highcharts = highcharts;
@@ -133,7 +135,7 @@ function _make_series(js, pivots, col_pivots, mode, hidden) {
                     for (let prop of columns) {
                         var sname = prop.split(',');
                         var gname = sname[sname.length - 1];
-                        sname = sname.slice(0, sname.length - 1).join(", ") || gname;
+                        sname = sname.slice(0, sname.length - 1).join(", ") || " ";
                         var s;
                         if (prev === undefined) prev = sname;
                         for (var sidx = 0; sidx < series.length; sidx++) {
@@ -170,7 +172,7 @@ function _make_series(js, pivots, col_pivots, mode, hidden) {
                     if (is_stacked) {
                         sname = sname.join(", ") || gname;
                     } else {
-                        sname = sname.slice(0, sname.length - 1).join(", ") || gname;
+                        sname = sname.slice(0, sname.length - 1).join(", ") || " ";
                     }
                     var s;
                     for (var sidx=0; sidx<series.length; sidx++) {

packages/perspective-viewer-hypergrid/src/js/hypergrid.js

@@ -568,7 +568,7 @@ function PerspectiveDataModel(grid) {
     grid.mixIn.call(grid.behavior.dataModel, {
 
         // Override setData
-        setData: function (dataPayload, schema) {
+        setData: function (dataPayload, schema, cache_update) {
             this.viewData = dataPayload;
             this.source.setData(dataPayload, schema);
         },
@@ -738,47 +738,50 @@ function psp2hypergrid(data, schema) {
         }
     }
     var is_tree = data[0].hasOwnProperty('__ROW_PATH__');
-    var row_paths = data.map(function(row) {
-        if (is_tree) {
-            return ["ROOT"].concat(row.__ROW_PATH__) || ["ROOT"];
-        } else {
-            return [];
-        }
-    });
-    var columnPaths = Object.keys(data[0]).filter(function(row) {
-        return row !== "__ROW_PATH__";
-    }).map(function(row) {
-        return row.split(',');
-    });
+
+    var columnPaths = Object.keys(data[0])
+        .filter(row => row !== "__ROW_PATH__")
+        .map(row => row.split(','));
+
     let flat_columns = columnPaths.map(col => col.join(","));
-    let rows = data.map(function(row, idx) {
+
+    let row_paths = [];
+    let rows = [];
+    let row_leaves = [];
+    for (let idx = 0; idx < data.length; idx++) {
+        const row = data[idx] || {};
+        let new_row = [];
         if (is_tree) {
+            if (row.__ROW_PATH__ === undefined) {
+                row.__ROW_PATH__ = [];
+            }
+            row_paths.push(["ROOT"].concat(row.__ROW_PATH__));
             let name = row['__ROW_PATH__'][row['__ROW_PATH__'].length - 1];
             if (name === undefined && idx === 0) name = "TOTAL"
-            var new_row = [name];
+            new_row = [name];
+            row_leaves.push(row.__ROW_PATH__.length >= (data[idx + 1] ? data[idx + 1].__ROW_PATH__.length : 0));
         } else {
-            new_row = [];
+            row_paths.push([]);
         }
-
         for (var col of flat_columns) {
             new_row.push(row[col]);
         }
-        return new_row;
-    })
+        rows.push(new_row);
+    }
+
     var hg_data = {
         rowPaths: row_paths,
         data: rows,
         isTree: is_tree,
         configuration: {},
         columnPaths: (is_tree ? [[" "]] : []).concat(columnPaths),
-        columnTypes: (is_tree ? ["str"] : []).concat(columnPaths.map(function(col) { return conv[schema[col[col.length - 1]]] }))
+        columnTypes: (is_tree ? ["str"] : []).concat(columnPaths.map(col => conv[schema[col[col.length - 1]]]))
     };
 
     if (is_tree) {
-        hg_data['rowLeaf'] = data.map(function(row, idx) {
-            return row.__ROW_PATH__.length >= (data[idx + 1] ? data[idx + 1].__ROW_PATH__.length : 0);
-        })
+        hg_data['rowLeaf'] = row_leaves;
     }
+
     return hg_data
 }
 
@@ -865,40 +868,69 @@ registerElement(TEMPLATE, {
 
 });
 
+const PAGE_SIZE = 1000;
 
-async function grid(div, view, hidden) {
-    let [json, schema] = await Promise.all([view.to_json(), view.schema()]);
+async function fill_page(view, json, hidden, start_row, end_row) {
+    let next_page = await view.to_json({start_row: start_row, end_row: end_row});
     if (hidden.length > 0) {
-        let first = json[0];
+        let first = next_page[0];
         let to_delete = [];
         for (let key in first) {
             let split_key = key.split(',');
             if (hidden.indexOf(split_key[split_key.length - 1].trim()) >= 0) {
                 to_delete.push(key);
             }
         }
-        for (let row of json) {
+        for (let row of next_page) {
             for (let h of to_delete) {
                 delete row[h];
             }
         }
     }
+    for (let idx = 0; idx < next_page.length; idx++) {
+        json[start_row + idx] = next_page[idx];
+    }
+    return json;
+}
+
+async function load_incrementally(view, schema, hidden, json, total, page) {
+    json = await fill_page(view, json, hidden, page * PAGE_SIZE, (page + 1) * PAGE_SIZE);
+    this.grid.set_data(json, schema);
+    this.grid.grid.canvas.resize();
+    this.grid.grid.canvas.resize();
+    if ((page + 1) * PAGE_SIZE < total) {
+        await load_incrementally.call(this, view, schema, hidden, json, total, page + 1);
+    }
+}
 
+async function grid(div, view, hidden) {
+    let [nrows, json, schema] = await Promise.all([
+        view.num_rows(), 
+        view.to_json({end_row: 1}), 
+        view.schema()
+    ]);
+    let visible_rows = [];
     if (!this.grid) {
         this.grid = document.createElement('perspective-hypergrid');
+        visible_rows = [0, 0, 100];
     } else if (this.grid.grid) {
         this.grid.grid.canvas.stopResizing();
+        visible_rows = this.grid.grid.getVisibleRows();
+    }
+    json.length = nrows;
+    if (visible_rows.length > 0) {
+        json = await fill_page(view, json, hidden, visible_rows[1], visible_rows[visible_rows.length - 1] + 1);        
     }
     if (!(document.contains ? document.contains(this.grid) : false)) {
         div.innerHTML = "";
         div.appendChild(this.grid);
     }
-    this.grid.set_data(json, schema);
-
-    // TODO this resolves a bug in the TreeRenderer, the calculated tree column
-    // width is 0 initially.
-    this.grid.grid.canvas.resize();
-    this.grid.grid.canvas.resize();
+    if (visible_rows.length > 0) {
+        this.grid.set_data(json, schema);
+        this.grid.grid.canvas.resize();
+        this.grid.grid.canvas.resize();
+    }
+    await load_incrementally.call(this, view, schema, hidden, json, nrows, 0);
 }
 
 global.registerPlugin("hypergrid", {

packages/perspective-viewer/package.json

@@ -34,6 +34,7 @@
     "@jpmorganchase/perspective-common": "^0.0.1",
     "babel-polyfill": "^6.26.0",
     "babel-runtime": "^6.26.0",
+    "bluebird": "^3.5.1",
     "d3-array": "^1.2.1",
     "mobile-drag-drop": "^2.2.0",
     "underscore": "^1.8.3"
@@ -42,7 +43,9 @@
     "@jpmorganchase/perspective-common": "^0.0.1",
     "babel-core": "^6.26.0",
     "babel-loader": "^7.1.2",
+    "babel-plugin-transform-async-to-bluebird": "^1.1.1",
     "babel-plugin-transform-es2015-for-of": "^6.23.0",
+    "babel-plugin-transform-promise-to-bluebird": "^1.1.1",
     "babel-plugin-transform-runtime": "^6.23.0",
     "babel-preset-env": "^1.6.0",
     "css-loader": "^0.28.7",

packages/perspective-viewer/src/js/view.js

@@ -175,9 +175,9 @@ async function loadTable(table) {
     }
 
     this._column_names.innerHTML = "";
+    this._table = table;
 
     let [cols, schema] = await Promise.all([table.columns(), table.schema()]);
-    this._table = table;
 
     if (!this.hasAttribute('columns')) {
         this.setAttribute('columns', JSON.stringify(cols));
@@ -238,6 +238,7 @@ function update() {
     let column_pivots = this._view_columns('#column_pivots perspective-row:not(.off)');
     let filters = JSON.parse(this.getAttribute('filters'));
     let aggregates = this._get_view_aggregates();
+    if (aggregates.length === 0) return;
     if (row_pivots.length === 0 && column_pivots.length > 0) {
         row_pivots = column_pivots;
         column_pivots = [];
@@ -269,15 +270,21 @@ function update() {
             timeout = Math.min(10000, Math.max(0, timeout));
             this._debounced = setTimeout(() => {
                 this._debounced = undefined;
-                this._plugin.create.call(this, this._datavis, this._view, hidden, false);
+                const t = performance.now();
+                this._plugin.create.call(this, this._datavis, this._view, hidden, false).then(() => {
+                    this.setAttribute('render_time', performance.now() - t);
+                });
             }, timeout || 0);
         }
     });
     this._drop_target.style.display = 'none';
 
     const t = performance.now();
     this._render_count = (this._render_count || 0) + 1;
-    this._plugin.create.call(this, this._datavis, this._view, hidden, true).then(() => {
+    if (this._task) {
+        this._task.cancel();
+    }
+    this._task = this._plugin.create.call(this, this._datavis, this._view, hidden, true).then(() => {
         if (!this.hasAttribute('render_time')) {
             this.dispatchEvent(new Event('loaded', {bubbles: true}));
         }

packages/perspective/package.json

@@ -40,6 +40,7 @@
   "dependencies": {
     "@jpmorganchase/perspective-common": "^0.0.1",
     "babel-runtime": "^6.26.0",
+    "bluebird": "^3.5.1",
     "d3-array": "^1.2.1",
     "moment": "^2.19.1",
     "papaparse": "^4.3.6",
@@ -49,7 +50,9 @@
     "@jpmorganchase/perspective-common": "^0.0.1",
     "babel-core": "^6.26.0",
     "babel-loader": "^7.1.2",
+    "babel-plugin-transform-async-to-bluebird": "^1.1.1",
     "babel-plugin-transform-es2015-for-of": "^6.23.0",
+    "babel-plugin-transform-promise-to-bluebird": "^1.1.1",
     "babel-plugin-transform-runtime": "^6.23.0",
     "babel-polyfill": "^6.26.0",
     "babel-preset-env": "^1.6.0",

packages/perspective/src/cpp/main.cpp

@@ -490,6 +490,30 @@ scalar_to_val(const t_tscalvec& scalars, t_uint32 idx)
     }
 }
 
+/**
+ *
+ *
+ * Params
+ * ------
+ *
+ *
+ * Returns
+ * -------
+ *
+ */
+template<typename T>
+val
+get_data(T ctx, t_uint32 start_row, t_uint32 end_row, t_uint32 start_col, t_uint32 end_col)
+{
+    auto slice = ctx->get_data(start_row, end_row, start_col, end_col);
+    val arr = val::array();
+    for (auto idx = 0; idx < slice.size(); ++idx)
+    {
+        arr.set(idx, scalar_to_val(slice, idx));
+    }
+    return arr;
+}
+
 /**
  * Main
  */
@@ -761,4 +785,7 @@ EMSCRIPTEN_BINDINGS(perspective)
     function("make_context_one", &make_context_one);
     function("make_context_two", &make_context_two);
     function("scalar_to_val", &scalar_to_val);
+    function("get_data_zero", &get_data<t_ctx0_sptr>);
+    function("get_data_one", &get_data<t_ctx1_sptr>);
+    function("get_data_two", &get_data<t_ctx2_sptr>);
 }