Merge pull request #138 from jpmorganchase/master

Update develop from remote master

cpp/perspective/src/cpp/context_two.cpp

@@ -138,15 +138,15 @@ t_ctx2::close(t_header header, t_index idx) {
             m_row_depth = 0;
             retval = m_rtraversal->collapse_node(idx);
             m_rows_changed = (retval > 0);
-        }
+        } break;
         case HEADER_COLUMN: {
             if (!m_ctraversal->is_valid_idx(idx))
                 return 0;
             m_column_depth_set = false;
             m_column_depth = 0;
             retval = m_ctraversal->collapse_node(idx);
             m_columns_changed = (retval > 0);
-        }
+        } break;
         default: {
             PSP_COMPLAIN_AND_ABORT("Invalid header type detected.");
             return INVALID_INDEX;

cpp/perspective/src/cpp/emscripten.cpp

@@ -685,7 +685,7 @@ namespace binding {
 
     void
     _fill_col_int64(val accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow) {
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update) {
         t_uindex nrows = col->size();
 
         if (is_arrow) {
@@ -700,7 +700,7 @@ namespace binding {
 
     void
     _fill_col_time(val accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow) {
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update) {
         t_uindex nrows = col->size();
 
         if (is_arrow) {
@@ -729,7 +729,11 @@ namespace binding {
                     continue;
 
                 if (item.isNull()) {
-                    col->unset(i);
+                    if (is_update) {
+                        col->unset(i);
+                    } else {
+                        col->clear(i);
+                    }
                     continue;
                 }
 
@@ -742,7 +746,7 @@ namespace binding {
 
     void
     _fill_col_date(val accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow) {
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update) {
         t_uindex nrows = col->size();
 
         if (is_arrow) {
@@ -771,7 +775,11 @@ namespace binding {
                     continue;
 
                 if (item.isNull()) {
-                    col->unset(i);
+                    if (is_update) {
+                        col->unset(i);
+                    } else {
+                        col->clear(i);
+                    }
                     continue;
                 }
 
@@ -782,7 +790,7 @@ namespace binding {
 
     void
     _fill_col_bool(val accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow) {
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update) {
         t_uindex nrows = col->size();
 
         if (is_arrow) {
@@ -801,7 +809,11 @@ namespace binding {
                     continue;
 
                 if (item.isNull()) {
-                    col->unset(i);
+                    if (is_update) {
+                        col->unset(i);
+                    } else {
+                        col->clear(i);
+                    }
                     continue;
                 }
 
@@ -813,7 +825,7 @@ namespace binding {
 
     void
     _fill_col_string(val accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow) {
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update) {
 
         t_uindex nrows = col->size();
 
@@ -866,7 +878,11 @@ namespace binding {
                     continue;
 
                 if (item.isNull()) {
-                    col->unset(i);
+                    if (is_update) {
+                        col->unset(i);
+                    } else {
+                        col->clear(i);
+                    }
                     continue;
                 }
 
@@ -880,7 +896,7 @@ namespace binding {
 
     void
     _fill_col_numeric(val accessor, t_table& tbl, std::shared_ptr<t_column> col,
-        std::string name, std::int32_t cidx, t_dtype type, bool is_arrow) {
+        std::string name, std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update) {
         t_uindex nrows = col->size();
 
         if (is_arrow) {
@@ -913,7 +929,11 @@ namespace binding {
                     continue;
 
                 if (item.isNull()) {
-                    col->unset(i);
+                    if (is_update) {
+                        col->unset(i);
+                    } else {
+                        col->clear(i);
+                    }
                     continue;
                 }
 
@@ -939,7 +959,8 @@ namespace binding {
                             std::cout << "Promoting to string" << std::endl;
                             tbl.promote_column(name, DTYPE_STR, i, false);
                             col = tbl.get_column(name);
-                            _fill_col_string(accessor, col, name, cidx, DTYPE_STR, is_arrow);
+                            _fill_col_string(
+                                accessor, col, name, cidx, DTYPE_STR, is_arrow, is_update);
                             return;
                         } else {
                             col->set_nth(i, static_cast<std::int32_t>(fval));
@@ -976,7 +997,7 @@ namespace binding {
      */
     void
     _fill_data(t_table& tbl, std::vector<std::string> ocolnames, val accessor,
-        std::vector<t_dtype> odt, std::uint32_t offset, bool is_arrow) {
+        std::vector<t_dtype> odt, std::uint32_t offset, bool is_arrow, bool is_update) {
 
         for (auto cidx = 0; cidx < ocolnames.size(); ++cidx) {
             auto name = ocolnames[cidx];
@@ -993,25 +1014,26 @@ namespace binding {
 
             switch (col_type) {
                 case DTYPE_INT64: {
-                    _fill_col_int64(dcol, col, name, cidx, col_type, is_arrow);
+                    _fill_col_int64(dcol, col, name, cidx, col_type, is_arrow, is_update);
                 } break;
                 case DTYPE_BOOL: {
-                    _fill_col_bool(dcol, col, name, cidx, col_type, is_arrow);
+                    _fill_col_bool(dcol, col, name, cidx, col_type, is_arrow, is_update);
                 } break;
                 case DTYPE_DATE: {
-                    _fill_col_date(dcol, col, name, cidx, col_type, is_arrow);
+                    _fill_col_date(dcol, col, name, cidx, col_type, is_arrow, is_update);
                 } break;
                 case DTYPE_TIME: {
-                    _fill_col_time(dcol, col, name, cidx, col_type, is_arrow);
+                    _fill_col_time(dcol, col, name, cidx, col_type, is_arrow, is_update);
                 } break;
                 case DTYPE_STR: {
-                    _fill_col_string(dcol, col, name, cidx, col_type, is_arrow);
+                    _fill_col_string(dcol, col, name, cidx, col_type, is_arrow, is_update);
                 } break;
                 case DTYPE_NONE: {
                     break;
                 }
                 default:
-                    _fill_col_numeric(dcol, tbl, col, name, cidx, col_type, is_arrow);
+                    _fill_col_numeric(
+                        dcol, tbl, col, name, cidx, col_type, is_arrow, is_update);
             }
 
             if (is_arrow) {
@@ -1418,9 +1440,7 @@ namespace binding {
      * A gnode.
      */
     std::shared_ptr<t_gnode>
-    make_gnode(const t_table& table) {
-        auto iscm = table.get_schema();
-
+    make_gnode(const t_schema& iscm) {
         std::vector<std::string> ocolnames(iscm.columns());
         std::vector<t_dtype> odt(iscm.types());
 
@@ -1496,7 +1516,14 @@ namespace binding {
         tbl.init();
         tbl.extend(size);
 
-        _fill_data(tbl, colnames, accessor, dtypes, offset, is_arrow);
+        bool is_new_gnode = gnode.isUndefined();
+        std::shared_ptr<t_gnode> new_gnode;
+        if (!is_new_gnode) {
+            new_gnode = gnode.as<std::shared_ptr<t_gnode>>();
+        }
+
+        _fill_data(tbl, colnames, accessor, dtypes, offset, is_arrow,
+            !(is_new_gnode || new_gnode->mapping_size() == 0));
 
         // Set up pkey and op columns
         if (is_delete) {
@@ -1522,19 +1549,15 @@ namespace binding {
             tbl.clone_column(index, "psp_okey");
         }
 
-        std::shared_ptr<t_gnode> new_gnode;
-
-        if (gnode.isUndefined()) {
-            new_gnode = make_gnode(tbl);
-            pool->register_gnode(new_gnode.get());
-        } else {
-            new_gnode = gnode.as<std::shared_ptr<t_gnode>>();
-        }
-
         if (!computed.isUndefined()) {
             table_add_computed_column(tbl, computed);
         }
 
+        if (is_new_gnode) {
+            new_gnode = make_gnode(tbl.get_schema());
+            pool->register_gnode(new_gnode.get());
+        }
+
         pool->send(new_gnode->get_id(), 0, tbl);
         pool->_process();
 
@@ -1556,7 +1579,7 @@ namespace binding {
     clone_gnode_table(t_pool* pool, std::shared_ptr<t_gnode> gnode, val computed) {
         t_table* tbl = gnode->_get_pkeyed_table();
         table_add_computed_column(*tbl, computed);
-        std::shared_ptr<t_gnode> new_gnode = make_gnode(*tbl);
+        std::shared_ptr<t_gnode> new_gnode = make_gnode(tbl->get_schema());
         pool->register_gnode(new_gnode.get());
         pool->send(new_gnode->get_id(), 0, *tbl);
         pool->_process();

cpp/perspective/src/cpp/gnode.cpp

@@ -692,6 +692,11 @@ t_gnode::_process() {
     psp_log_time(repr() + " _process.noinit_path.exit");
 }
 
+t_uindex
+t_gnode::mapping_size() const {
+    return m_state->mapping_size();
+}
+
 t_table*
 t_gnode::_get_otable(t_uindex portidx) {
     PSP_TRACE_SENTINEL();

cpp/perspective/src/include/perspective/binding.h

@@ -125,27 +125,27 @@ namespace binding {
 
     template <typename T>
     void _fill_col_numeric(T accessor, t_table& tbl, std::shared_ptr<t_column> col,
-        std::string name, std::int32_t cidx, t_dtype type, bool is_arrow);
+        std::string name, std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update);
 
     template <typename T>
     void _fill_col_int64(T accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow);
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update);
 
     template <typename T>
     void _fill_col_time(T accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow);
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update);
 
     template <typename T>
     void _fill_col_date(T accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow);
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update);
 
     template <typename T>
     void _fill_col_bool(T accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow);
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update);
 
     template <typename T>
     void _fill_col_string(T accessor, std::shared_ptr<t_column> col, std::string name,
-        std::int32_t cidx, t_dtype type, bool is_arrow);
+        std::int32_t cidx, t_dtype type, bool is_arrow, bool is_update);
 
     /**
      * Fills the table with data from language.
@@ -165,7 +165,7 @@ namespace binding {
      */
     template <typename T>
     void _fill_data(t_table& tbl, std::vector<std::string> ocolnames, T accessor,
-        std::vector<t_dtype> odt, std::uint32_t offset, bool is_arrow);
+        std::vector<t_dtype> odt, std::uint32_t offset, bool is_arrow, bool is_update);
 
     /******************************************************************************
      *
@@ -223,7 +223,7 @@ namespace binding {
      * -------
      * A gnode.
      */
-    std::shared_ptr<t_gnode> make_gnode(const t_table& table);
+    std::shared_ptr<t_gnode> make_gnode(const t_schema& iscm);
 
     /**
      * Create a populated table.

cpp/perspective/src/include/perspective/gnode.h

@@ -130,6 +130,8 @@ class PERSPECTIVE_EXPORT t_gnode {
     bool was_updated() const;
     void clear_updated();
 
+    t_uindex mapping_size() const;
+
     // helper function for tests
     std::shared_ptr<t_table> tstep(std::shared_ptr<const t_table> input_table);
 

package.json

@@ -81,6 +81,7 @@
             "packages/perspective-viewer-hypergrid",
             "packages/perspective-viewer-highcharts",
             "packages/perspective-viewer-d3fc"
-        ]
+        ],
+        "verbose": true
     }
 }

packages/perspective-viewer-d3fc/src/js/plugin/plugin.js

@@ -33,16 +33,16 @@ charts.forEach(chart => {
 function drawChart(chart) {
     return async function(el, view, task) {
         // FIXME: super tight coupling to private viewer methods
-        const row_pivots = this._get_view_row_pivots();
-        const col_pivots = this._get_view_column_pivots();
         const aggregates = this._get_view_aggregates();
         const hidden = this._get_view_hidden(aggregates);
         const filter = this._view._config.filter;
 
-        const [tschema, json] = await Promise.all([this._table.schema(), view.to_json()]);
+        const [tschema, json, config] = await Promise.all([this._table.schema(), view.to_json(), view.get_config()]);
         if (task.cancelled) {
             return;
         }
+        const row_pivots = config.row_pivot;
+        const col_pivots = config.column_pivot;
 
         const filtered = row_pivots.length > 0 ? json.filter(col => col.__ROW_PATH__ && col.__ROW_PATH__.length == row_pivots.length) : json;
         const dataMap = !row_pivots.length ? (col, i) => ({...removeHiddenData(col, hidden), __ROW_PATH__: [i]}) : col => removeHiddenData(col, hidden);

packages/perspective-viewer-highcharts/src/js/config.js

@@ -135,8 +135,8 @@ export function default_config(aggregates, mode) {
     //
 
     // read this + define chart schema using _view()
-    const that = this,
-        config = that._view._config;
+    const that = this;
+    const config = this._config;
 
     const axis_titles = get_axis_titles(config.aggregate);
     const pivot_titles = get_pivot_titles(config.row_pivot, config.column_pivot);

packages/perspective-viewer-highcharts/src/js/draw.js

@@ -37,11 +37,19 @@ function get_or_create_element(div) {
     return perspective_highcharts_element;
 }
 
-export const draw = mode =>
+export const draw = (mode, set_config) =>
     async function(el, view, task) {
+        if (set_config) {
+            this._config = await view.get_config();
+            if (task.cancelled) {
+                return;
+            }
+        }
+
+        const row_pivots = this._config.row_pivot;
+        const col_pivots = this._config.column_pivot;
+
         // FIXME: super tight coupling to private viewer methods
-        const row_pivots = this._get_view_row_pivots();
-        const col_pivots = this._get_view_column_pivots();
         const aggregates = this._get_view_aggregates();
         const hidden = this._get_view_hidden(aggregates);