diff --git a/caravel/assets/javascripts/explore/explore.jsx b/caravel/assets/javascripts/explore/explore.jsx
index 76c548aa750e..e3c1ea38c79e 100644
--- a/caravel/assets/javascripts/explore/explore.jsx
+++ b/caravel/assets/javascripts/explore/explore.jsx
@@ -23,39 +23,12 @@ require('../../vendor/pygments.css');
require('../../stylesheets/explore.css');
let slice;
+let filterCount = 1;
const getPanelClass = function (fieldPrefix) {
return (fieldPrefix === 'flt' ? 'filter' : 'having') + '_panel';
};
-function prepForm() {
- // Assigning the right id to form elements in filters
- const fixId = function ($filter, fieldPrefix, i) {
- $filter.attr('id', function () {
- return fieldPrefix + '_' + i;
- });
-
- ['col', 'op', 'eq'].forEach(function (fieldMiddle) {
- const fieldName = fieldPrefix + '_' + fieldMiddle;
- $filter.find('[id^=' + fieldName + '_]')
- .attr('id', function () {
- return fieldName + '_' + i;
- })
- .attr('name', function () {
- return fieldName + '_' + i;
- });
- });
- };
-
- ['flt', 'having'].forEach(function (fieldPrefix) {
- let i = 1;
- $('#' + getPanelClass(fieldPrefix) + ' #filters > div').each(function () {
- fixId($(this), fieldPrefix, i);
- i++;
- });
- });
-}
-
function query(forceUpdate, pushState) {
let force = forceUpdate;
if (force === undefined) {
@@ -67,10 +40,8 @@ function query(forceUpdate, pushState) {
$('div.alert').remove();
}
$('#is_cached').hide();
- prepForm();
if (pushState !== false) {
- // update the url after prepForm() fix the field ids
history.pushState({}, document.title, slice.querystring());
}
slice.render(force);
@@ -104,7 +75,6 @@ function saveSlice() {
return;
}
$('#action').val(action);
- prepForm();
$('#query').submit();
}
@@ -341,15 +311,120 @@ function initExploreView() {
$('[data-toggle="tooltip"]').tooltip({ container: 'body' });
$('.ui-helper-hidden-accessible').remove(); // jQuery-ui 1.11+ creates a div for every tooltip
+ function createChoices(term, data) {
+ const filtered = $(data).filter(function () {
+ return this.text.localeCompare(term) === 0;
+ });
+ if (filtered.length === 0) {
+ return {
+ id: term,
+ text: term,
+ };
+ }
+ return {};
+ }
+
+ function initSelectionToValue(element, callback) {
+ callback({
+ id: element.val(),
+ text: element.val(),
+ });
+ }
+
+ function convertSelect(selectId) {
+ const parent = $(selectId).parent();
+ const name = $(selectId).attr('name');
+ const l = [];
+ let selected = '';
+ const options = $(selectId + ' option');
+ const classes = $(selectId).attr('class');
+ for (let i = 0; i < options.length; i++) {
+ l.push({
+ id: options[i].value,
+ text: options[i].text,
+ });
+ if (options[i].selected) {
+ selected = options[i].value;
+ }
+ }
+ $(selectId).remove();
+ parent.append(
+ ''
+ );
+ $(`input[name='${name}']`).select2({
+ createSearchChoice: createChoices,
+ initSelection: initSelectionToValue,
+ dropdownAutoWidth: true,
+ multiple: false,
+ data: l,
+ sortResults(data) {
+ return data.sort(function (a, b) {
+ if (a.text > b.text) {
+ return 1;
+ } else if (a.text < b.text) {
+ return -1;
+ }
+ return 0;
+ });
+ },
+ });
+ }
+
+ function insertFilterChoices(i, fieldPrefix) {
+ const column = $('#' + fieldPrefix + '_col_' + i).val();
+ const eq = '#' + fieldPrefix + '_eq_' + i;
+ $(eq).empty();
+
+ $.getJSON(slice.filterEndpoint(column), function (data) {
+ $.each(data, function (key, value) {
+ $(eq).append($('').attr('value', value).text(value));
+ });
+ $(eq).select2('destroy');
+ convertSelect(eq);
+ });
+ }
+
function addFilter(i, fieldPrefix) {
+ const isHaving = fieldPrefix === 'having';
const cp = $('#' + fieldPrefix + '0').clone();
$(cp).appendTo('#' + getPanelClass(fieldPrefix) + ' #filters');
+ $(cp).attr('id', fieldPrefix + filterCount);
$(cp).show();
+
+ const eqId = fieldPrefix + '_eq_' + filterCount;
+ const $eq = $(cp).find('#' + fieldPrefix + '_eq_0');
+ $eq.attr('id', eqId).attr('name', eqId);
+
+ const opId = fieldPrefix + '_op_' + filterCount;
+ const $op = $(cp).find('#' + fieldPrefix + '_op_0');
+ $op.attr('id', opId).attr('name', opId);
+
+ const colId = fieldPrefix + '_col_' + filterCount;
+ const $col = $(cp).find('#' + fieldPrefix + '_col_0');
+ $col.attr('id', colId).attr('name', colId);
+
+ // Set existing values
if (i !== undefined) {
- $(cp).find('#' + fieldPrefix + '_eq_0').val(px.getParam(fieldPrefix + '_eq_' + i));
- $(cp).find('#' + fieldPrefix + '_op_0').val(px.getParam(fieldPrefix + '_op_' + i));
- $(cp).find('#' + fieldPrefix + '_col_0').val(px.getParam(fieldPrefix + '_col_' + i));
+ $op.val(px.getParam(fieldPrefix + '_op_' + i));
+ $col.val(px.getParam(fieldPrefix + '_col_' + i));
+
+ if (isHaving || !slice.filterSelectEnabled) {
+ $eq.val(px.getParam(fieldPrefix + '_eq_' + i));
+ } else {
+ insertFilterChoices(filterCount, fieldPrefix);
+ const eqVal = px.getParam(fieldPrefix + '_eq_' + i);
+ $eq.append($('').attr('value', eqVal).text(eqVal));
+ }
+ }
+
+ if (slice.filterSelectEnabled && !isHaving) {
+ const currentFilter = filterCount;
+ $col.change(function () {
+ insertFilterChoices(currentFilter, fieldPrefix);
+ });
}
+
$(cp).find('select').select2();
$(cp).find('.remove').click(function () {
$(this)
@@ -357,6 +432,7 @@ function initExploreView() {
.parent()
.remove();
});
+ filterCount++;
}
function setFilters() {
@@ -395,52 +471,16 @@ function initExploreView() {
queryAndSaveBtnsEl
);
- function createChoices(term, data) {
- const filtered = $(data).filter(function () {
- return this.text.localeCompare(term) === 0;
- });
- if (filtered.length === 0) {
- return {
- id: term,
- text: term,
- };
- }
- return {};
- }
-
- function initSelectionToValue(element, callback) {
- callback({
- id: element.val(),
- text: element.val(),
- });
- }
+ $(window).bind('popstate', function () {
+ // Browser back button
+ const returnLocation = history.location || document.location;
+ // Could do something more lightweight here, but we're not optimizing
+ // for the use of the back button anyways
+ returnLocation.reload();
+ });
$('.select2_freeform').each(function () {
- const parent = $(this).parent();
- const name = $(this).attr('name');
- const l = [];
- let selected = '';
- for (let i = 0; i < this.options.length; i++) {
- l.push({
- id: this.options[i].value,
- text: this.options[i].text,
- });
- if (this.options[i].selected) {
- selected = this.options[i].value;
- }
- }
- parent.append(
- ``
- );
- $(`input[name='${name}']`).select2({
- createSearchChoice: createChoices,
- initSelection: initSelectionToValue,
- dropdownAutoWidth: true,
- multiple: false,
- data: l,
- });
- $(this).remove();
+ convertSelect('#' + $(this).attr('id'));
});
function prepSaveDialog() {
@@ -489,13 +529,11 @@ function initExploreView() {
$(document).ready(function () {
const data = $('.slice').data('slice');
-
- initExploreView();
-
slice = px.Slice(data);
-
$('.slice').data('slice', slice);
+ initExploreView();
+
// call vis render method, which issues ajax
query(false, false);
diff --git a/caravel/assets/javascripts/modules/caravel.js b/caravel/assets/javascripts/modules/caravel.js
index 4b013fa87a3a..484bb67f90e8 100644
--- a/caravel/assets/javascripts/modules/caravel.js
+++ b/caravel/assets/javascripts/modules/caravel.js
@@ -59,6 +59,7 @@ const px = function () {
const selector = '#' + containerId;
const container = $(selector);
const sliceId = data.sliceId;
+ const filterSelectEnabled = data.filter_select_enabled;
let dttm = 0;
const stopwatch = function () {
dttm += 10;
@@ -75,6 +76,7 @@ const px = function () {
data,
container,
containerId,
+ filterSelectEnabled,
selector,
querystring(params) {
const newParams = params || {};
@@ -111,6 +113,11 @@ const px = function () {
endpoint += '&force=' + this.force;
return endpoint;
},
+ filterEndpoint(column) {
+ const parser = document.createElement('a');
+ parser.href = data.filter_endpoint;
+ return parser.pathname + column + '/' + this.querystring();
+ },
d3format(col, number) {
// uses the utils memoized d3format function and formats based on
// column level defined preferences
diff --git a/caravel/forms.py b/caravel/forms.py
index 3cd689230d40..20e166dc6adb 100755
--- a/caravel/forms.py
+++ b/caravel/forms.py
@@ -1071,13 +1071,17 @@ def add_to_form(attrs):
_('Filter 1'),
default=col_choices[0][0],
choices=col_choices))
- setattr(QueryForm, field_prefix + '_op_' + str(i), SelectField(
- _('Filter 1'),
- default=op_choices[0][0],
- choices=op_choices))
- setattr(
- QueryForm, field_prefix + '_eq_' + str(i),
- TextField(_("Super"), default=''))
+ setattr(QueryForm, field_prefix + '_op_' + str(i),
+ SelectField(
+ _('Filter 1'),
+ default=op_choices[0][0],
+ choices=op_choices))
+ if viz.datasource.filter_select_enabled and not is_having_filter:
+ setattr(QueryForm, field_prefix + '_eq_' + str(i),
+ FreeFormSelectField(_("Super"), choices=[]))
+ else:
+ setattr(QueryForm, field_prefix + '_eq_' + str(i),
+ TextField(_("Super"), default=''))
if time_fields:
QueryForm.fieldsets = ({
diff --git a/caravel/migrations/versions/17fc73813b42_datasource_table_filter_select.py b/caravel/migrations/versions/17fc73813b42_datasource_table_filter_select.py
new file mode 100644
index 000000000000..b48912c71f0f
--- /dev/null
+++ b/caravel/migrations/versions/17fc73813b42_datasource_table_filter_select.py
@@ -0,0 +1,24 @@
+"""datasource_table_filter_select
+
+Revision ID: 17fc73813b42
+Revises: 3c3ffe173e4f
+Create Date: 2016-08-12 16:41:25.629004
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '17fc73813b42'
+down_revision = '3c3ffe173e4f'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ op.add_column('datasources', sa.Column('filter_select_enabled', sa.Boolean(), default=False))
+ op.add_column('tables', sa.Column('filter_select_enabled', sa.Boolean(), default=False))
+
+
+def downgrade():
+ op.drop_column('tables', 'filter_select_enabled')
+ op.drop_column('datasources', 'filter_select_enabled')
diff --git a/caravel/models.py b/caravel/models.py
index 34834e9d8eff..eff7b906b8c4 100644
--- a/caravel/models.py
+++ b/caravel/models.py
@@ -27,6 +27,7 @@
from flask_babel import lazy_gettext as _
from pydruid.client import PyDruid
+from pydruid.utils.aggregators import count
from pydruid.utils.filters import Dimension, Filter
from pydruid.utils.postaggregator import Postaggregator
from pydruid.utils.having import Aggregation
@@ -606,6 +607,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
default_endpoint = Column(Text)
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
is_featured = Column(Boolean, default=False)
+ filter_select_enabled = Column(Boolean, default=False)
user_id = Column(Integer, ForeignKey('ab_user.id'))
owner = relationship('User', backref='tables', foreign_keys=[user_id])
database = relationship(
@@ -698,6 +700,42 @@ def get_col(self, col_name):
if col_name == col.column_name:
return col
+ def values_for_column(self,
+ column_name,
+ from_dttm,
+ to_dttm,
+ limit=500):
+ """Runs query against sqla to retrieve some sample values for the given column."""
+ granularity = self.main_dttm_col
+
+ cols = {col.column_name: col for col in self.columns}
+ target_col = cols[column_name]
+
+ dttm_col = cols[granularity]
+ timestamp = dttm_col.sqla_col.label('timestamp')
+ time_filter = [
+ timestamp >= text(dttm_col.dttm_sql_literal(from_dttm)),
+ timestamp <= text(dttm_col.dttm_sql_literal(to_dttm)),
+ ]
+
+ tbl = table(self.table_name)
+ qry = select([target_col.sqla_col])
+ qry = qry.select_from(tbl)
+ qry = qry.where(and_(*time_filter))
+ qry = qry.distinct(column_name)
+ qry = qry.limit(limit)
+
+ engine = self.database.get_sqla_engine()
+ sql = "{}".format(
+ qry.compile(
+ engine, compile_kwargs={"literal_binds": True}, ),
+ )
+
+ return pd.read_sql_query(
+ sql=sql,
+ con=engine
+ )
+
def query( # sqla
self, groupby, metrics,
granularity,
@@ -1135,6 +1173,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable):
datasource_name = Column(String(255), unique=True)
is_featured = Column(Boolean, default=False)
is_hidden = Column(Boolean, default=False)
+ filter_select_enabled = Column(Boolean, default=False)
description = Column(Text)
default_endpoint = Column(Text)
user_id = Column(Integer, ForeignKey('ab_user.id'))
@@ -1289,6 +1328,35 @@ def sync_to_db(cls, name, cluster):
col_obj.generate_metrics()
session.flush()
+ def values_for_column(self,
+ column_name,
+ from_dttm,
+ to_dttm,
+ limit=500):
+ """Runs query against Druid to retrieve some values for the given column"""
+ # TODO: Use Lexicographic TopNMeticSpec onces supported by PyDruid
+ from_dttm = from_dttm.replace(tzinfo=config.get("DRUID_TZ"))
+ to_dttm = to_dttm.replace(tzinfo=config.get("DRUID_TZ"))
+
+ qry = dict(
+ datasource=self.datasource_name,
+ granularity="all",
+ intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(),
+ aggregations=dict(count=count("count")),
+ dimension=column_name,
+ metric="count",
+ threshold=limit,
+ )
+
+ client = self.cluster.get_pydruid_client()
+ client.topn(**qry)
+ df = client.export_pandas()
+
+ if df is None or df.size == 0:
+ raise Exception(_("No data was returned."))
+
+ return df
+
def query( # druid
self, groupby, metrics,
granularity,
diff --git a/caravel/views.py b/caravel/views.py
index e2b24a57e6b9..54d3277dd9db 100755
--- a/caravel/views.py
+++ b/caravel/views.py
@@ -476,8 +476,8 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa
'table_name', 'database', 'schema',
'default_endpoint', 'offset', 'cache_timeout']
edit_columns = [
- 'table_name', 'sql', 'is_featured', 'database', 'schema',
- 'description', 'owner',
+ 'table_name', 'sql', 'is_featured', 'filter_select_enabled',
+ 'database', 'schema', 'description', 'owner',
'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
related_views = [TableColumnInlineView, SqlMetricInlineView]
base_order = ('changed_on', 'desc')
@@ -501,6 +501,7 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa
'database': _("Database"),
'changed_on_': _("Last Changed"),
'is_featured': _("Is Featured"),
+ 'filter_select_enabled': _("Enable Filter Select"),
'schema': _("Schema"),
'default_endpoint': _("Default Endpoint"),
'offset': _("Offset"),
@@ -783,8 +784,8 @@ class DruidDatasourceModelView(CaravelModelView, DeleteMixin): # noqa
related_views = [DruidColumnInlineView, DruidMetricInlineView]
edit_columns = [
'datasource_name', 'cluster', 'description', 'owner',
- 'is_featured', 'is_hidden', 'default_endpoint', 'offset',
- 'cache_timeout']
+ 'is_featured', 'is_hidden', 'filter_select_enabled', 'default_endpoint',
+ 'offset', 'cache_timeout']
add_columns = edit_columns
page_size = 500
base_order = ('datasource_name', 'asc')
@@ -801,6 +802,7 @@ class DruidDatasourceModelView(CaravelModelView, DeleteMixin): # noqa
'owner': _("Owner"),
'is_featured': _("Is Featured"),
'is_hidden': _("Is Hidden"),
+ 'filter_select_enabled': _("Enable Filter Select"),
'default_endpoint': _("Default Endpoint"),
'offset': _("Time Offset"),
'cache_timeout': _("Cache Timeout"),
@@ -1013,6 +1015,58 @@ def explore(self, datasource_type, datasource_id, slice_id=None):
return resp
+ @api
+ @has_access_api
+ @expose("/filter////")
+ def filter(self, datasource_type, datasource_id, column):
+ """
+ Endpoint to retrieve values for specified column.
+
+ :param datasource_type: Type of datasource e.g. table
+ :param datasource_id: Datasource id
+ :param column: Column name to retrieve values for
+ :return:
+ """
+ # TODO: Cache endpoint by user, datasource and column
+ error_redirect = '/slicemodelview/list/'
+ datasource_class = models.SqlaTable \
+ if datasource_type == "table" else models.DruidDatasource
+
+ datasource = db.session.query(
+ datasource_class).filter_by(id=datasource_id).first()
+
+ if not datasource:
+ flash(__("The datasource seems to have been deleted"), "alert")
+ return redirect(error_redirect)
+
+ all_datasource_access = self.can_access(
+ 'all_datasource_access', 'all_datasource_access')
+ datasource_access = self.can_access(
+ 'datasource_access', datasource.perm)
+ if not (all_datasource_access or datasource_access):
+ flash(__("You don't seem to have access to this datasource"), "danger")
+ return redirect(error_redirect)
+
+ viz_type = request.args.get("viz_type")
+ if not viz_type and datasource.default_endpoint:
+ return redirect(datasource.default_endpoint)
+ if not viz_type:
+ viz_type = "table"
+ try:
+ obj = viz.viz_types[viz_type](
+ datasource,
+ form_data=request.args,
+ slice_=None)
+ except Exception as e:
+ flash(str(e), "danger")
+ return redirect(error_redirect)
+ status = 200
+ payload = obj.get_values_for_column(column)
+ return Response(
+ payload,
+ status=status,
+ mimetype="application/json")
+
def save_or_overwrite_slice(
self, args, slc, slice_add_perm, slice_edit_perm):
"""Save or overwrite a slice"""
diff --git a/caravel/viz.py b/caravel/viz.py
index 13f99f8854a3..a3fab7da7cd6 100755
--- a/caravel/viz.py
+++ b/caravel/viz.py
@@ -145,6 +145,34 @@ def get_url(self, for_cache_key=False, **kwargs):
del od['force']
return href(od)
+ def get_filter_url(self):
+ """Returns the URL to retrieve column values used in the filter dropdown"""
+ d = self.orig_form_data.copy()
+ # Remove unchecked checkboxes because HTML is weird like that
+ od = MultiDict()
+ for key in sorted(d.keys()):
+ # if MultiDict is initialized with MD({key:[emptyarray]}),
+ # key is included in d.keys() but accessing it throws
+ try:
+ if d[key] is False:
+ del d[key]
+ continue
+ except IndexError:
+ pass
+
+ if isinstance(d, (MultiDict, ImmutableMultiDict)):
+ v = d.getlist(key)
+ else:
+ v = d.get(key)
+ if not isinstance(v, list):
+ v = [v]
+ for item in v:
+ od.add(key, item)
+ href = Href(
+ '/caravel/filter/{self.datasource.type}/'
+ '{self.datasource.id}/'.format(**locals()))
+ return href(od)
+
def get_df(self, query_obj=None):
"""Returns a pandas dataframe based on the query object"""
if not query_obj:
@@ -308,6 +336,7 @@ def get_json(self):
'form_data': self.form_data,
'json_endpoint': self.json_endpoint,
'query': self.query,
+ 'filter_endpoint': self.filter_endpoint,
'standalone_endpoint': self.standalone_endpoint,
}
payload['cached_dttm'] = datetime.now().isoformat().split('.')[0]
@@ -341,9 +370,11 @@ def data(self):
'csv_endpoint': self.csv_endpoint,
'form_data': self.form_data,
'json_endpoint': self.json_endpoint,
+ 'filter_endpoint': self.filter_endpoint,
'standalone_endpoint': self.standalone_endpoint,
'token': self.token,
'viz_name': self.viz_type,
+ 'filter_select_enabled': self.datasource.filter_select_enabled,
'column_formats': {
m.metric_name: m.d3format
for m in self.datasource.metrics
@@ -357,6 +388,34 @@ def get_csv(self):
include_index = not isinstance(df.index, pd.RangeIndex)
return df.to_csv(index=include_index, encoding="utf-8")
+ def get_values_for_column(self, column):
+ """
+ Retrieves values for a column to be used by the filter dropdown.
+
+ :param column: column name
+ :return: JSON containing the some values for a column
+ """
+ form_data = self.form_data
+
+ since = form_data.get("since", "1 year ago")
+ from_dttm = utils.parse_human_datetime(since)
+ now = datetime.now()
+ if from_dttm > now:
+ from_dttm = now - (from_dttm - now)
+ until = form_data.get("until", "now")
+ to_dttm = utils.parse_human_datetime(until)
+ if from_dttm > to_dttm:
+ flasher("The date range doesn't seem right.", "danger")
+ from_dttm = to_dttm # Making them identical to not raise
+
+ kwargs = dict(
+ column_name=column,
+ from_dttm=from_dttm,
+ to_dttm=to_dttm,
+ )
+ df = self.datasource.values_for_column(**kwargs)
+ return df[column].to_json()
+
def get_data(self):
return []
@@ -364,6 +423,10 @@ def get_data(self):
def json_endpoint(self):
return self.get_url(json="true")
+ @property
+ def filter_endpoint(self):
+ return self.get_filter_url()
+
@property
def cache_key(self):
url = self.get_url(for_cache_key=True, json="true", force="false")