Skip to content

Commit

Permalink
Merge pull request #93 from jpmorganchase/date-filters
Browse files Browse the repository at this point in the history 
Date filters
  • Loading branch information
@texodus
texodus authored Apr 17, 2018
2 parents 335c4e3 + 8668688 commit 0beb7ba
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 72 deletions.
7 changes: 2 additions & 5 deletions packages/perspective/src/cpp/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,13 +108,10 @@ _get_fterms(t_schema schema, val j_filters)
term = mktscalar(filter[2].as<bool>());
break;
case DTYPE_TIME:
{
std::cout << "Date filters not handled yet" << std::endl;
}
break;
term = mktscalar(t_time(static_cast<t_int64>(filter[2].as<t_float64>())));
break;
default:
{
//std::cout << filter[2].as<std::string>().c_str() << std::endl;
term = mktscalar(get_interned_cstr(filter[2].as<std::string>().c_str()));
}
}
Expand Down
72 changes: 72 additions & 0 deletions packages/perspective/src/js/date_parser.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
/******************************************************************************
*
* Copyright (c) 2017, the Perspective Authors.
*
* This file is part of the Perspective library, distributed under the terms of
* the Apache License 2.0. The full license can be found in the LICENSE file.
*
*/

import moment from "moment";

const DATE_PARSE_CANDIDATES = [
moment.ISO_8601,
moment.RFC_2822,
'YYYY-MM-DD\\DHH:mm:ss.SSSS',
'MM-DD-YYYY',
'MM/DD/YYYY',
'M/D/YYYY',
'M/D/YY',
'DD MMM YYYY',
'HH:mm:ss.SSS',
];

/**
*
*
* @export
* @param {string} x
* @returns
*/
export function is_valid_date(x) {
return moment(x, DATE_PARSE_CANDIDATES, true).isValid();
}

/**
*
*
* @export
* @class DateParser
*/
export class DateParser {

constructor() {
this.date_types = [];
this.date_candidates = DATE_PARSE_CANDIDATES.slice();
this.date_exclusions = [];
}

parse(input) {
if (this.date_exclusions.indexOf(input) > -1) {
return -1;;
} else {
let val = input;
if (typeof val === "string") {
val = moment(input, this.date_types, true);
if (!val.isValid() || this.date_types.length === 0) {
for (let candidate of this.date_candidates) {
val = moment(input, candidate, true);
if (val.isValid()) {
this.date_types.push(candidate);
this.date_candidates.splice(this.date_candidates.indexOf(candidate), 1);
return +val;
}
}
this.date_exclusions.push(input);
return -1;
}
}
return +val;
}
}
}
80 changes: 27 additions & 53 deletions packages/perspective/src/js/perspective.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
*/

import papaparse from "papaparse";
import moment from "moment";
import * as Arrow from "@apache-arrow/es5-esm";
import {is_valid_date, DateParser} from "./date_parser.js";

import {TYPE_AGGREGATES, AGGREGATE_DEFAULTS, TYPE_FILTERS, FILTER_DEFAULTS} from "./defaults.js";

Expand Down Expand Up @@ -50,7 +50,7 @@ function infer_type(x) {
t = __MODULE__.t_dtype.DTYPE_TIME;
} else if (!isNaN(Number(x)) && x !== '') {
t = __MODULE__.t_dtype.DTYPE_FLOAT64;
} else if (typeof x === "string" && moment(x, DATE_PARSE_CANDIDATES, true).isValid()) {
} else if (typeof x === "string" && is_valid_date(x)) {
t = __MODULE__.t_dtype.DTYPE_TIME;
} else if (typeof x === "string") {
let lower = x.toLowerCase();
Expand All @@ -63,18 +63,6 @@ function infer_type(x) {
return t;
}

const DATE_PARSE_CANDIDATES = [
moment.ISO_8601,
moment.RFC_2822,
'YYYY-MM-DD\\DHH:mm:ss.SSSS',
'MM-DD-YYYY',
'MM/DD/YYYY',
'M/D/YYYY',
'M/D/YY',
'DD MMM YYYY',
'HH:mm:ss.SSS',
];

/**
* Do any necessary data transforms on columns. Currently it does the following
* transforms
Expand Down Expand Up @@ -169,9 +157,7 @@ function parse_data(data, names, types) {
inferredType = __MODULE__.t_dtype.DTYPE_STR;
}
col = [];
const date_types = [];
const date_candidates = DATE_PARSE_CANDIDATES.slice();
const date_exclusions = [];
const parser = new DateParser();
for (let x = 0; x < data.length; x ++) {
if (!(name in data[x]) || data[x][name] === undefined) continue;
if (inferredType.value === __MODULE__.t_dtype.DTYPE_FLOAT64.value) {
Expand All @@ -194,32 +180,8 @@ function parse_data(data, names, types) {
col.push(cell);
}
} else if (inferredType.value === __MODULE__.t_dtype.DTYPE_TIME.value) {
if (date_exclusions.indexOf(data[x][name]) > -1) {
col.push(-1);
} else {
let val = data[x][name];
if (typeof val === "string") {
val = moment(data[x][name], date_types, true);
if (!val.isValid() || date_types.length === 0) {
let found = false;
for (let candidate of date_candidates) {
val = moment(data[x][name], candidate, true);
if (val.isValid()) {
date_types.push(candidate);
date_candidates.splice(date_candidates.indexOf(candidate), 1);
found = true;
break;
}
}
if (!found) {
date_exclusions.push(data[x][name]);
col.push(-1);
continue;
}
}
}
col.push(+val);
}
let val = data[x][name];
col.push(parser.parse(val));
} else {
col.push(data[x][name] === null ? (types[types.length - 1].value === 19 ? "" : 0) : "" + data[x][name]); // TODO this is not right - might not be a string. Need a data cleaner
}
Expand Down Expand Up @@ -804,15 +766,7 @@ table.prototype.size = async function() {
return this.gnode.get_table().size();
}

/**
* The schema of this {@link table}. A schema is an Object, the keys of which
* are the columns of this {@link table}, and the values are their string type names.
*
* @async
*
* @returns {Promise<Object>} A Promise of this {@link table}'s schema.
*/
table.prototype.schema = async function() {
table.prototype._schema = function () {
let schema = this.gnode.get_tblschema();
let columns = schema.columns();
let types = schema.types();
Expand All @@ -833,9 +787,24 @@ table.prototype.schema = async function() {
new_schema[columns.get(key)] = "date";
}
}
schema.delete();
columns.delete();
types.delete();
return new_schema;
}

/**
* The schema of this {@link table}. A schema is an Object, the keys of which
* are the columns of this {@link table}, and the values are their string type names.
*
* @async
*
* @returns {Promise<Object>} A Promise of this {@link table}'s schema.
*/
table.prototype.schema = async function() {
return this._schema();
}

/**
* Create a new {@link view} from this table with a specified
* configuration.
Expand Down Expand Up @@ -936,8 +905,13 @@ table.prototype.view = function(config) {
let filter_op = __MODULE__.t_filter_op.FILTER_OP_AND;

if (config.filter) {
let schema = this._schema();
filters = config.filter.map(function(filter) {
return [filter[0], _string_to_filter_op[filter[1]], filter[2]];
if (schema[filter[0]] === "date") {
return [filter[0], _string_to_filter_op[filter[1]], +new DateParser().parse(filter[2])];
} else {
return [filter[0], _string_to_filter_op[filter[1]], filter[2]];
}
});
if (config.filter_op) {
filter_op = _string_to_filter_op[config.filter_op];
Expand Down
56 changes: 42 additions & 14 deletions packages/perspective/test/js/filters.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,22 @@
*
*/

var yesterday = new Date();
yesterday.setDate(yesterday.getDate() - 1);
var now = new Date();

var data = [
{'x': 1, 'y':'a', 'z': true},
{'x': 2, 'y':'b', 'z': false},
{'x': 3, 'y':'c', 'z': true},
{'x': 4, 'y':'d', 'z': false}
{'w': now, 'x': 1, 'y':'a', 'z': true},
{'w': now, 'x': 2, 'y':'b', 'z': false},
{'w': now, 'x': 3, 'y':'c', 'z': true},
{'w': yesterday, 'x': 4, 'y':'d', 'z': false}
];

var rdata = [
{'w': +now, 'x': 1, 'y':'a', 'z': true},
{'w': +now, 'x': 2, 'y':'b', 'z': false},
{'w': +now, 'x': 3, 'y':'c', 'z': true},
{'w': +yesterday, 'x': 4, 'y':'d', 'z': false}
];

module.exports = (perspective) => {
Expand Down Expand Up @@ -41,7 +52,7 @@ module.exports = (perspective) => {
filter: [['x', '>', 2.0]]
});
let json = await view.to_json();
expect(data.slice(2)).toEqual(json);
expect(rdata.slice(2)).toEqual(json);
});

it("x < 3", async function () {
Expand All @@ -50,7 +61,7 @@ module.exports = (perspective) => {
filter: [['x', '<', 3.0]]
});
let json = await view.to_json();
expect(data.slice(0, 2)).toEqual(json);
expect(rdata.slice(0, 2)).toEqual(json);
});

it("x > 4", async function () {
Expand Down Expand Up @@ -81,7 +92,7 @@ module.exports = (perspective) => {
filter: [['x', '==', 1]]
});
let json = await view.to_json();
expect(data.slice(0, 1)).toEqual(json);
expect(rdata.slice(0, 1)).toEqual(json);
});

it("x == 5", async function () {
Expand All @@ -99,7 +110,7 @@ module.exports = (perspective) => {
filter: [['y', '==', 'a']]
});
let json = await view.to_json();
expect(data.slice(0, 1)).toEqual(json);
expect(rdata.slice(0, 1)).toEqual(json);
});

it("y == 'e'", async function () {
Expand All @@ -117,7 +128,7 @@ module.exports = (perspective) => {
filter: [['z', '==', true]]
});
let json = await view.to_json();
expect([data[0], data[2]]).toEqual(json);
expect([rdata[0], rdata[2]]).toEqual(json);
});

it("z == false", async function () {
Expand All @@ -126,9 +137,26 @@ module.exports = (perspective) => {
filter: [['z', '==', false]]
});
let json = await view.to_json();
expect([data[1], data[3]]).toEqual(json);
expect([rdata[1], rdata[3]]).toEqual(json);
});

it("w == yesterday", async function () {
var table = perspective.table(data);
var view = table.view({
filter: [['w', '==', yesterday]]
});
let json = await view.to_json();
expect([rdata[3]]).toEqual(json);
});

it("w != yesterday", async function () {
var table = perspective.table(data);
var view = table.view({
filter: [['w', '!=', yesterday]]
});
let json = await view.to_json();
expect(rdata.slice(0, 3)).toEqual(json);
});
});

describe("in", function() {
Expand All @@ -138,7 +166,7 @@ module.exports = (perspective) => {
filter: [['y', 'in', ['a', 'b']]]
});
let json = await view.to_json();
expect(data.slice(0, 2)).toEqual(json);
expect(rdata.slice(0, 2)).toEqual(json);
});

});
Expand All @@ -151,7 +179,7 @@ module.exports = (perspective) => {
filter: [['y', 'contains', 'a']]
});
let json = await view.to_json();
expect(data.slice(0, 1)).toEqual(json);
expect(rdata.slice(0, 1)).toEqual(json);
});

});
Expand All @@ -167,7 +195,7 @@ module.exports = (perspective) => {
]
});
let json = await view.to_json();
expect(data.slice(1, 3)).toEqual(json);
expect(rdata.slice(1, 3)).toEqual(json);
});

it("y contains 'a' | y contains 'b'", async function () {
Expand All @@ -180,7 +208,7 @@ module.exports = (perspective) => {
]
});
let json = await view.to_json();
expect(data.slice(0, 2)).toEqual(json);
expect(rdata.slice(0, 2)).toEqual(json);
});

});
Expand Down

0 comments on commit 0beb7ba

Please sign in to comment.