From 0c49eb1d6f3a5e35285c62840b6bfd3d6b4bfb57 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 17 Apr 2018 13:22:59 -0400 Subject: [PATCH 1/3] Added date filter support to C++ --- packages/perspective/src/cpp/main.cpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages/perspective/src/cpp/main.cpp b/packages/perspective/src/cpp/main.cpp index 148890f1ad..fe358ca9d4 100644 --- a/packages/perspective/src/cpp/main.cpp +++ b/packages/perspective/src/cpp/main.cpp @@ -108,13 +108,10 @@ _get_fterms(t_schema schema, val j_filters) term = mktscalar(filter[2].as()); break; case DTYPE_TIME: - { - std::cout << "Date filters not handled yet" << std::endl; - } - break; + term = mktscalar(t_time(static_cast(filter[2].as()))); + break; default: { - //std::cout << filter[2].as().c_str() << std::endl; term = mktscalar(get_interned_cstr(filter[2].as().c_str())); } } From 60e73ff8723c9534f7037acdf00129321ec5235d Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 17 Apr 2018 13:23:54 -0400 Subject: [PATCH 2/3] Refactored date parsing logic --- packages/perspective/src/js/date_parser.js | 72 ++++++++++++++++++++++ packages/perspective/src/js/perspective.js | 46 ++------------ 2 files changed, 76 insertions(+), 42 deletions(-) create mode 100644 packages/perspective/src/js/date_parser.js diff --git a/packages/perspective/src/js/date_parser.js b/packages/perspective/src/js/date_parser.js new file mode 100644 index 0000000000..45caea7b1c --- /dev/null +++ b/packages/perspective/src/js/date_parser.js @@ -0,0 +1,72 @@ +/****************************************************************************** + * + * Copyright (c) 2017, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +import moment from "moment"; + +const DATE_PARSE_CANDIDATES = [ + moment.ISO_8601, + moment.RFC_2822, + 'YYYY-MM-DD\\DHH:mm:ss.SSSS', + 'MM-DD-YYYY', + 'MM/DD/YYYY', + 'M/D/YYYY', + 'M/D/YY', + 'DD MMM YYYY', + 'HH:mm:ss.SSS', +]; + +/** + * + * + * @export + * @param {string} x + * @returns + */ +export function is_valid_date(x) { + return moment(x, DATE_PARSE_CANDIDATES, true).isValid(); +} + +/** + * + * + * @export + * @class DateParser + */ +export class DateParser { + + constructor() { + this.date_types = []; + this.date_candidates = DATE_PARSE_CANDIDATES.slice(); + this.date_exclusions = []; + } + + parse(input) { + if (this.date_exclusions.indexOf(input) > -1) { + return -1;; + } else { + let val = input; + if (typeof val === "string") { + val = moment(input, this.date_types, true); + if (!val.isValid() || this.date_types.length === 0) { + for (let candidate of this.date_candidates) { + val = moment(input, candidate, true); + if (val.isValid()) { + this.date_types.push(candidate); + this.date_candidates.splice(this.date_candidates.indexOf(candidate), 1); + return +val; + } + } + this.date_exclusions.push(input); + return -1; + } + } + return +val; + } + } +} \ No newline at end of file diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 8844cfdf5e..19e9f25d2f 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -8,8 +8,8 @@ */ import papaparse from "papaparse"; -import moment from "moment"; import * as Arrow from "@apache-arrow/es5-esm"; +import {is_valid_date, DateParser} from "./date_parser.js"; import {TYPE_AGGREGATES, AGGREGATE_DEFAULTS, TYPE_FILTERS, FILTER_DEFAULTS} from "./defaults.js"; @@ -50,7 +50,7 @@ function infer_type(x) { t = __MODULE__.t_dtype.DTYPE_TIME; } else if (!isNaN(Number(x)) && x !== '') { t = __MODULE__.t_dtype.DTYPE_FLOAT64; - } else if (typeof x === "string" && moment(x, DATE_PARSE_CANDIDATES, true).isValid()) { + } else if (typeof x === "string" && is_valid_date(x)) { t = __MODULE__.t_dtype.DTYPE_TIME; } else if (typeof x === "string") { let lower = x.toLowerCase(); @@ -63,18 +63,6 @@ function infer_type(x) { return t; } -const DATE_PARSE_CANDIDATES = [ - moment.ISO_8601, - moment.RFC_2822, - 'YYYY-MM-DD\\DHH:mm:ss.SSSS', - 'MM-DD-YYYY', - 'MM/DD/YYYY', - 'M/D/YYYY', - 'M/D/YY', - 'DD MMM YYYY', - 'HH:mm:ss.SSS', -]; - /** * Do any necessary data transforms on columns. Currently it does the following * transforms @@ -169,9 +157,7 @@ function parse_data(data, names, types) { inferredType = __MODULE__.t_dtype.DTYPE_STR; } col = []; - const date_types = []; - const date_candidates = DATE_PARSE_CANDIDATES.slice(); - const date_exclusions = []; + const parser = new DateParser(); for (let x = 0; x < data.length; x ++) { if (!(name in data[x]) || data[x][name] === undefined) continue; if (inferredType.value === __MODULE__.t_dtype.DTYPE_FLOAT64.value) { @@ -194,32 +180,8 @@ function parse_data(data, names, types) { col.push(cell); } } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_TIME.value) { - if (date_exclusions.indexOf(data[x][name]) > -1) { - col.push(-1); - } else { let val = data[x][name]; - if (typeof val === "string") { - val = moment(data[x][name], date_types, true); - if (!val.isValid() || date_types.length === 0) { - let found = false; - for (let candidate of date_candidates) { - val = moment(data[x][name], candidate, true); - if (val.isValid()) { - date_types.push(candidate); - date_candidates.splice(date_candidates.indexOf(candidate), 1); - found = true; - break; - } - } - if (!found) { - date_exclusions.push(data[x][name]); - col.push(-1); - continue; - } - } - } - col.push(+val); - } + col.push(parser.parse(val)); } else { col.push(data[x][name] === null ? (types[types.length - 1].value === 19 ? "" : 0) : "" + data[x][name]); // TODO this is not right - might not be a string. Need a data cleaner } From 8668688e41165859c7484398f579a109b5afa503 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 17 Apr 2018 13:24:16 -0400 Subject: [PATCH 3/3] Added date filter to JS API and tests --- packages/perspective/src/js/perspective.js | 34 ++++++++----- packages/perspective/test/js/filters.js | 56 ++++++++++++++++------ 2 files changed, 65 insertions(+), 25 deletions(-) diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 19e9f25d2f..1c642dd5f9 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -180,7 +180,7 @@ function parse_data(data, names, types) { col.push(cell); } } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_TIME.value) { - let val = data[x][name]; + let val = data[x][name]; col.push(parser.parse(val)); } else { col.push(data[x][name] === null ? (types[types.length - 1].value === 19 ? "" : 0) : "" + data[x][name]); // TODO this is not right - might not be a string. Need a data cleaner @@ -766,15 +766,7 @@ table.prototype.size = async function() { return this.gnode.get_table().size(); } -/** - * The schema of this {@link table}. A schema is an Object, the keys of which - * are the columns of this {@link table}, and the values are their string type names. - * - * @async - * - * @returns {Promise} A Promise of this {@link table}'s schema. - */ -table.prototype.schema = async function() { +table.prototype._schema = function () { let schema = this.gnode.get_tblschema(); let columns = schema.columns(); let types = schema.types(); @@ -795,9 +787,24 @@ table.prototype.schema = async function() { new_schema[columns.get(key)] = "date"; } } + schema.delete(); + columns.delete(); + types.delete(); return new_schema; } +/** + * The schema of this {@link table}. A schema is an Object, the keys of which + * are the columns of this {@link table}, and the values are their string type names. + * + * @async + * + * @returns {Promise} A Promise of this {@link table}'s schema. + */ +table.prototype.schema = async function() { + return this._schema(); +} + /** * Create a new {@link view} from this table with a specified * configuration. @@ -898,8 +905,13 @@ table.prototype.view = function(config) { let filter_op = __MODULE__.t_filter_op.FILTER_OP_AND; if (config.filter) { + let schema = this._schema(); filters = config.filter.map(function(filter) { - return [filter[0], _string_to_filter_op[filter[1]], filter[2]]; + if (schema[filter[0]] === "date") { + return [filter[0], _string_to_filter_op[filter[1]], +new DateParser().parse(filter[2])]; + } else { + return [filter[0], _string_to_filter_op[filter[1]], filter[2]]; + } }); if (config.filter_op) { filter_op = _string_to_filter_op[config.filter_op]; diff --git a/packages/perspective/test/js/filters.js b/packages/perspective/test/js/filters.js index f5558d5fff..26c9094511 100644 --- a/packages/perspective/test/js/filters.js +++ b/packages/perspective/test/js/filters.js @@ -7,11 +7,22 @@ * */ +var yesterday = new Date(); +yesterday.setDate(yesterday.getDate() - 1); +var now = new Date(); + var data = [ - {'x': 1, 'y':'a', 'z': true}, - {'x': 2, 'y':'b', 'z': false}, - {'x': 3, 'y':'c', 'z': true}, - {'x': 4, 'y':'d', 'z': false} + {'w': now, 'x': 1, 'y':'a', 'z': true}, + {'w': now, 'x': 2, 'y':'b', 'z': false}, + {'w': now, 'x': 3, 'y':'c', 'z': true}, + {'w': yesterday, 'x': 4, 'y':'d', 'z': false} +]; + +var rdata = [ + {'w': +now, 'x': 1, 'y':'a', 'z': true}, + {'w': +now, 'x': 2, 'y':'b', 'z': false}, + {'w': +now, 'x': 3, 'y':'c', 'z': true}, + {'w': +yesterday, 'x': 4, 'y':'d', 'z': false} ]; module.exports = (perspective) => { @@ -41,7 +52,7 @@ module.exports = (perspective) => { filter: [['x', '>', 2.0]] }); let json = await view.to_json(); - expect(data.slice(2)).toEqual(json); + expect(rdata.slice(2)).toEqual(json); }); it("x < 3", async function () { @@ -50,7 +61,7 @@ module.exports = (perspective) => { filter: [['x', '<', 3.0]] }); let json = await view.to_json(); - expect(data.slice(0, 2)).toEqual(json); + expect(rdata.slice(0, 2)).toEqual(json); }); it("x > 4", async function () { @@ -81,7 +92,7 @@ module.exports = (perspective) => { filter: [['x', '==', 1]] }); let json = await view.to_json(); - expect(data.slice(0, 1)).toEqual(json); + expect(rdata.slice(0, 1)).toEqual(json); }); it("x == 5", async function () { @@ -99,7 +110,7 @@ module.exports = (perspective) => { filter: [['y', '==', 'a']] }); let json = await view.to_json(); - expect(data.slice(0, 1)).toEqual(json); + expect(rdata.slice(0, 1)).toEqual(json); }); it("y == 'e'", async function () { @@ -117,7 +128,7 @@ module.exports = (perspective) => { filter: [['z', '==', true]] }); let json = await view.to_json(); - expect([data[0], data[2]]).toEqual(json); + expect([rdata[0], rdata[2]]).toEqual(json); }); it("z == false", async function () { @@ -126,9 +137,26 @@ module.exports = (perspective) => { filter: [['z', '==', false]] }); let json = await view.to_json(); - expect([data[1], data[3]]).toEqual(json); + expect([rdata[1], rdata[3]]).toEqual(json); + }); + + it("w == yesterday", async function () { + var table = perspective.table(data); + var view = table.view({ + filter: [['w', '==', yesterday]] + }); + let json = await view.to_json(); + expect([rdata[3]]).toEqual(json); }); + it("w != yesterday", async function () { + var table = perspective.table(data); + var view = table.view({ + filter: [['w', '!=', yesterday]] + }); + let json = await view.to_json(); + expect(rdata.slice(0, 3)).toEqual(json); + }); }); describe("in", function() { @@ -138,7 +166,7 @@ module.exports = (perspective) => { filter: [['y', 'in', ['a', 'b']]] }); let json = await view.to_json(); - expect(data.slice(0, 2)).toEqual(json); + expect(rdata.slice(0, 2)).toEqual(json); }); }); @@ -151,7 +179,7 @@ module.exports = (perspective) => { filter: [['y', 'contains', 'a']] }); let json = await view.to_json(); - expect(data.slice(0, 1)).toEqual(json); + expect(rdata.slice(0, 1)).toEqual(json); }); }); @@ -167,7 +195,7 @@ module.exports = (perspective) => { ] }); let json = await view.to_json(); - expect(data.slice(1, 3)).toEqual(json); + expect(rdata.slice(1, 3)).toEqual(json); }); it("y contains 'a' | y contains 'b'", async function () { @@ -180,7 +208,7 @@ module.exports = (perspective) => { ] }); let json = await view.to_json(); - expect(data.slice(0, 2)).toEqual(json); + expect(rdata.slice(0, 2)).toEqual(json); }); });