diff --git a/caravel/assets/javascripts/explore/explore.jsx b/caravel/assets/javascripts/explore/explore.jsx index 76c548aa750e..e3c1ea38c79e 100644 --- a/caravel/assets/javascripts/explore/explore.jsx +++ b/caravel/assets/javascripts/explore/explore.jsx @@ -23,39 +23,12 @@ require('../../vendor/pygments.css'); require('../../stylesheets/explore.css'); let slice; +let filterCount = 1; const getPanelClass = function (fieldPrefix) { return (fieldPrefix === 'flt' ? 'filter' : 'having') + '_panel'; }; -function prepForm() { - // Assigning the right id to form elements in filters - const fixId = function ($filter, fieldPrefix, i) { - $filter.attr('id', function () { - return fieldPrefix + '_' + i; - }); - - ['col', 'op', 'eq'].forEach(function (fieldMiddle) { - const fieldName = fieldPrefix + '_' + fieldMiddle; - $filter.find('[id^=' + fieldName + '_]') - .attr('id', function () { - return fieldName + '_' + i; - }) - .attr('name', function () { - return fieldName + '_' + i; - }); - }); - }; - - ['flt', 'having'].forEach(function (fieldPrefix) { - let i = 1; - $('#' + getPanelClass(fieldPrefix) + ' #filters > div').each(function () { - fixId($(this), fieldPrefix, i); - i++; - }); - }); -} - function query(forceUpdate, pushState) { let force = forceUpdate; if (force === undefined) { @@ -67,10 +40,8 @@ function query(forceUpdate, pushState) { $('div.alert').remove(); } $('#is_cached').hide(); - prepForm(); if (pushState !== false) { - // update the url after prepForm() fix the field ids history.pushState({}, document.title, slice.querystring()); } slice.render(force); @@ -104,7 +75,6 @@ function saveSlice() { return; } $('#action').val(action); - prepForm(); $('#query').submit(); } @@ -341,15 +311,120 @@ function initExploreView() { $('[data-toggle="tooltip"]').tooltip({ container: 'body' }); $('.ui-helper-hidden-accessible').remove(); // jQuery-ui 1.11+ creates a div for every tooltip + function createChoices(term, data) { + const filtered = $(data).filter(function () { + return this.text.localeCompare(term) === 0; + }); + if (filtered.length === 0) { + return { + id: term, + text: term, + }; + } + return {}; + } + + function initSelectionToValue(element, callback) { + callback({ + id: element.val(), + text: element.val(), + }); + } + + function convertSelect(selectId) { + const parent = $(selectId).parent(); + const name = $(selectId).attr('name'); + const l = []; + let selected = ''; + const options = $(selectId + ' option'); + const classes = $(selectId).attr('class'); + for (let i = 0; i < options.length; i++) { + l.push({ + id: options[i].value, + text: options[i].text, + }); + if (options[i].selected) { + selected = options[i].value; + } + } + $(selectId).remove(); + parent.append( + '' + ); + $(`input[name='${name}']`).select2({ + createSearchChoice: createChoices, + initSelection: initSelectionToValue, + dropdownAutoWidth: true, + multiple: false, + data: l, + sortResults(data) { + return data.sort(function (a, b) { + if (a.text > b.text) { + return 1; + } else if (a.text < b.text) { + return -1; + } + return 0; + }); + }, + }); + } + + function insertFilterChoices(i, fieldPrefix) { + const column = $('#' + fieldPrefix + '_col_' + i).val(); + const eq = '#' + fieldPrefix + '_eq_' + i; + $(eq).empty(); + + $.getJSON(slice.filterEndpoint(column), function (data) { + $.each(data, function (key, value) { + $(eq).append($('').attr('value', value).text(value)); + }); + $(eq).select2('destroy'); + convertSelect(eq); + }); + } + function addFilter(i, fieldPrefix) { + const isHaving = fieldPrefix === 'having'; const cp = $('#' + fieldPrefix + '0').clone(); $(cp).appendTo('#' + getPanelClass(fieldPrefix) + ' #filters'); + $(cp).attr('id', fieldPrefix + filterCount); $(cp).show(); + + const eqId = fieldPrefix + '_eq_' + filterCount; + const $eq = $(cp).find('#' + fieldPrefix + '_eq_0'); + $eq.attr('id', eqId).attr('name', eqId); + + const opId = fieldPrefix + '_op_' + filterCount; + const $op = $(cp).find('#' + fieldPrefix + '_op_0'); + $op.attr('id', opId).attr('name', opId); + + const colId = fieldPrefix + '_col_' + filterCount; + const $col = $(cp).find('#' + fieldPrefix + '_col_0'); + $col.attr('id', colId).attr('name', colId); + + // Set existing values if (i !== undefined) { - $(cp).find('#' + fieldPrefix + '_eq_0').val(px.getParam(fieldPrefix + '_eq_' + i)); - $(cp).find('#' + fieldPrefix + '_op_0').val(px.getParam(fieldPrefix + '_op_' + i)); - $(cp).find('#' + fieldPrefix + '_col_0').val(px.getParam(fieldPrefix + '_col_' + i)); + $op.val(px.getParam(fieldPrefix + '_op_' + i)); + $col.val(px.getParam(fieldPrefix + '_col_' + i)); + + if (isHaving || !slice.filterSelectEnabled) { + $eq.val(px.getParam(fieldPrefix + '_eq_' + i)); + } else { + insertFilterChoices(filterCount, fieldPrefix); + const eqVal = px.getParam(fieldPrefix + '_eq_' + i); + $eq.append($('').attr('value', eqVal).text(eqVal)); + } + } + + if (slice.filterSelectEnabled && !isHaving) { + const currentFilter = filterCount; + $col.change(function () { + insertFilterChoices(currentFilter, fieldPrefix); + }); } + $(cp).find('select').select2(); $(cp).find('.remove').click(function () { $(this) @@ -357,6 +432,7 @@ function initExploreView() { .parent() .remove(); }); + filterCount++; } function setFilters() { @@ -395,52 +471,16 @@ function initExploreView() { queryAndSaveBtnsEl ); - function createChoices(term, data) { - const filtered = $(data).filter(function () { - return this.text.localeCompare(term) === 0; - }); - if (filtered.length === 0) { - return { - id: term, - text: term, - }; - } - return {}; - } - - function initSelectionToValue(element, callback) { - callback({ - id: element.val(), - text: element.val(), - }); - } + $(window).bind('popstate', function () { + // Browser back button + const returnLocation = history.location || document.location; + // Could do something more lightweight here, but we're not optimizing + // for the use of the back button anyways + returnLocation.reload(); + }); $('.select2_freeform').each(function () { - const parent = $(this).parent(); - const name = $(this).attr('name'); - const l = []; - let selected = ''; - for (let i = 0; i < this.options.length; i++) { - l.push({ - id: this.options[i].value, - text: this.options[i].text, - }); - if (this.options[i].selected) { - selected = this.options[i].value; - } - } - parent.append( - `` - ); - $(`input[name='${name}']`).select2({ - createSearchChoice: createChoices, - initSelection: initSelectionToValue, - dropdownAutoWidth: true, - multiple: false, - data: l, - }); - $(this).remove(); + convertSelect('#' + $(this).attr('id')); }); function prepSaveDialog() { @@ -489,13 +529,11 @@ function initExploreView() { $(document).ready(function () { const data = $('.slice').data('slice'); - - initExploreView(); - slice = px.Slice(data); - $('.slice').data('slice', slice); + initExploreView(); + // call vis render method, which issues ajax query(false, false); diff --git a/caravel/assets/javascripts/modules/caravel.js b/caravel/assets/javascripts/modules/caravel.js index 4b013fa87a3a..484bb67f90e8 100644 --- a/caravel/assets/javascripts/modules/caravel.js +++ b/caravel/assets/javascripts/modules/caravel.js @@ -59,6 +59,7 @@ const px = function () { const selector = '#' + containerId; const container = $(selector); const sliceId = data.sliceId; + const filterSelectEnabled = data.filter_select_enabled; let dttm = 0; const stopwatch = function () { dttm += 10; @@ -75,6 +76,7 @@ const px = function () { data, container, containerId, + filterSelectEnabled, selector, querystring(params) { const newParams = params || {}; @@ -111,6 +113,11 @@ const px = function () { endpoint += '&force=' + this.force; return endpoint; }, + filterEndpoint(column) { + const parser = document.createElement('a'); + parser.href = data.filter_endpoint; + return parser.pathname + column + '/' + this.querystring(); + }, d3format(col, number) { // uses the utils memoized d3format function and formats based on // column level defined preferences diff --git a/caravel/forms.py b/caravel/forms.py index 3cd689230d40..20e166dc6adb 100755 --- a/caravel/forms.py +++ b/caravel/forms.py @@ -1071,13 +1071,17 @@ def add_to_form(attrs): _('Filter 1'), default=col_choices[0][0], choices=col_choices)) - setattr(QueryForm, field_prefix + '_op_' + str(i), SelectField( - _('Filter 1'), - default=op_choices[0][0], - choices=op_choices)) - setattr( - QueryForm, field_prefix + '_eq_' + str(i), - TextField(_("Super"), default='')) + setattr(QueryForm, field_prefix + '_op_' + str(i), + SelectField( + _('Filter 1'), + default=op_choices[0][0], + choices=op_choices)) + if viz.datasource.filter_select_enabled and not is_having_filter: + setattr(QueryForm, field_prefix + '_eq_' + str(i), + FreeFormSelectField(_("Super"), choices=[])) + else: + setattr(QueryForm, field_prefix + '_eq_' + str(i), + TextField(_("Super"), default='')) if time_fields: QueryForm.fieldsets = ({ diff --git a/caravel/migrations/versions/17fc73813b42_datasource_table_filter_select.py b/caravel/migrations/versions/17fc73813b42_datasource_table_filter_select.py new file mode 100644 index 000000000000..b48912c71f0f --- /dev/null +++ b/caravel/migrations/versions/17fc73813b42_datasource_table_filter_select.py @@ -0,0 +1,24 @@ +"""datasource_table_filter_select + +Revision ID: 17fc73813b42 +Revises: 3c3ffe173e4f +Create Date: 2016-08-12 16:41:25.629004 + +""" + +# revision identifiers, used by Alembic. +revision = '17fc73813b42' +down_revision = '3c3ffe173e4f' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('datasources', sa.Column('filter_select_enabled', sa.Boolean(), default=False)) + op.add_column('tables', sa.Column('filter_select_enabled', sa.Boolean(), default=False)) + + +def downgrade(): + op.drop_column('tables', 'filter_select_enabled') + op.drop_column('datasources', 'filter_select_enabled') diff --git a/caravel/models.py b/caravel/models.py index 34834e9d8eff..eff7b906b8c4 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -27,6 +27,7 @@ from flask_babel import lazy_gettext as _ from pydruid.client import PyDruid +from pydruid.utils.aggregators import count from pydruid.utils.filters import Dimension, Filter from pydruid.utils.postaggregator import Postaggregator from pydruid.utils.having import Aggregation @@ -606,6 +607,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable): default_endpoint = Column(Text) database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) is_featured = Column(Boolean, default=False) + filter_select_enabled = Column(Boolean, default=False) user_id = Column(Integer, ForeignKey('ab_user.id')) owner = relationship('User', backref='tables', foreign_keys=[user_id]) database = relationship( @@ -698,6 +700,42 @@ def get_col(self, col_name): if col_name == col.column_name: return col + def values_for_column(self, + column_name, + from_dttm, + to_dttm, + limit=500): + """Runs query against sqla to retrieve some sample values for the given column.""" + granularity = self.main_dttm_col + + cols = {col.column_name: col for col in self.columns} + target_col = cols[column_name] + + dttm_col = cols[granularity] + timestamp = dttm_col.sqla_col.label('timestamp') + time_filter = [ + timestamp >= text(dttm_col.dttm_sql_literal(from_dttm)), + timestamp <= text(dttm_col.dttm_sql_literal(to_dttm)), + ] + + tbl = table(self.table_name) + qry = select([target_col.sqla_col]) + qry = qry.select_from(tbl) + qry = qry.where(and_(*time_filter)) + qry = qry.distinct(column_name) + qry = qry.limit(limit) + + engine = self.database.get_sqla_engine() + sql = "{}".format( + qry.compile( + engine, compile_kwargs={"literal_binds": True}, ), + ) + + return pd.read_sql_query( + sql=sql, + con=engine + ) + def query( # sqla self, groupby, metrics, granularity, @@ -1135,6 +1173,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable): datasource_name = Column(String(255), unique=True) is_featured = Column(Boolean, default=False) is_hidden = Column(Boolean, default=False) + filter_select_enabled = Column(Boolean, default=False) description = Column(Text) default_endpoint = Column(Text) user_id = Column(Integer, ForeignKey('ab_user.id')) @@ -1289,6 +1328,35 @@ def sync_to_db(cls, name, cluster): col_obj.generate_metrics() session.flush() + def values_for_column(self, + column_name, + from_dttm, + to_dttm, + limit=500): + """Runs query against Druid to retrieve some values for the given column""" + # TODO: Use Lexicographic TopNMeticSpec onces supported by PyDruid + from_dttm = from_dttm.replace(tzinfo=config.get("DRUID_TZ")) + to_dttm = to_dttm.replace(tzinfo=config.get("DRUID_TZ")) + + qry = dict( + datasource=self.datasource_name, + granularity="all", + intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(), + aggregations=dict(count=count("count")), + dimension=column_name, + metric="count", + threshold=limit, + ) + + client = self.cluster.get_pydruid_client() + client.topn(**qry) + df = client.export_pandas() + + if df is None or df.size == 0: + raise Exception(_("No data was returned.")) + + return df + def query( # druid self, groupby, metrics, granularity, diff --git a/caravel/views.py b/caravel/views.py index e2b24a57e6b9..54d3277dd9db 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -476,8 +476,8 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa 'table_name', 'database', 'schema', 'default_endpoint', 'offset', 'cache_timeout'] edit_columns = [ - 'table_name', 'sql', 'is_featured', 'database', 'schema', - 'description', 'owner', + 'table_name', 'sql', 'is_featured', 'filter_select_enabled', + 'database', 'schema', 'description', 'owner', 'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout'] related_views = [TableColumnInlineView, SqlMetricInlineView] base_order = ('changed_on', 'desc') @@ -501,6 +501,7 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa 'database': _("Database"), 'changed_on_': _("Last Changed"), 'is_featured': _("Is Featured"), + 'filter_select_enabled': _("Enable Filter Select"), 'schema': _("Schema"), 'default_endpoint': _("Default Endpoint"), 'offset': _("Offset"), @@ -783,8 +784,8 @@ class DruidDatasourceModelView(CaravelModelView, DeleteMixin): # noqa related_views = [DruidColumnInlineView, DruidMetricInlineView] edit_columns = [ 'datasource_name', 'cluster', 'description', 'owner', - 'is_featured', 'is_hidden', 'default_endpoint', 'offset', - 'cache_timeout'] + 'is_featured', 'is_hidden', 'filter_select_enabled', 'default_endpoint', + 'offset', 'cache_timeout'] add_columns = edit_columns page_size = 500 base_order = ('datasource_name', 'asc') @@ -801,6 +802,7 @@ class DruidDatasourceModelView(CaravelModelView, DeleteMixin): # noqa 'owner': _("Owner"), 'is_featured': _("Is Featured"), 'is_hidden': _("Is Hidden"), + 'filter_select_enabled': _("Enable Filter Select"), 'default_endpoint': _("Default Endpoint"), 'offset': _("Time Offset"), 'cache_timeout': _("Cache Timeout"), @@ -1013,6 +1015,58 @@ def explore(self, datasource_type, datasource_id, slice_id=None): return resp + @api + @has_access_api + @expose("/filter////") + def filter(self, datasource_type, datasource_id, column): + """ + Endpoint to retrieve values for specified column. + + :param datasource_type: Type of datasource e.g. table + :param datasource_id: Datasource id + :param column: Column name to retrieve values for + :return: + """ + # TODO: Cache endpoint by user, datasource and column + error_redirect = '/slicemodelview/list/' + datasource_class = models.SqlaTable \ + if datasource_type == "table" else models.DruidDatasource + + datasource = db.session.query( + datasource_class).filter_by(id=datasource_id).first() + + if not datasource: + flash(__("The datasource seems to have been deleted"), "alert") + return redirect(error_redirect) + + all_datasource_access = self.can_access( + 'all_datasource_access', 'all_datasource_access') + datasource_access = self.can_access( + 'datasource_access', datasource.perm) + if not (all_datasource_access or datasource_access): + flash(__("You don't seem to have access to this datasource"), "danger") + return redirect(error_redirect) + + viz_type = request.args.get("viz_type") + if not viz_type and datasource.default_endpoint: + return redirect(datasource.default_endpoint) + if not viz_type: + viz_type = "table" + try: + obj = viz.viz_types[viz_type]( + datasource, + form_data=request.args, + slice_=None) + except Exception as e: + flash(str(e), "danger") + return redirect(error_redirect) + status = 200 + payload = obj.get_values_for_column(column) + return Response( + payload, + status=status, + mimetype="application/json") + def save_or_overwrite_slice( self, args, slc, slice_add_perm, slice_edit_perm): """Save or overwrite a slice""" diff --git a/caravel/viz.py b/caravel/viz.py index 13f99f8854a3..a3fab7da7cd6 100755 --- a/caravel/viz.py +++ b/caravel/viz.py @@ -145,6 +145,34 @@ def get_url(self, for_cache_key=False, **kwargs): del od['force'] return href(od) + def get_filter_url(self): + """Returns the URL to retrieve column values used in the filter dropdown""" + d = self.orig_form_data.copy() + # Remove unchecked checkboxes because HTML is weird like that + od = MultiDict() + for key in sorted(d.keys()): + # if MultiDict is initialized with MD({key:[emptyarray]}), + # key is included in d.keys() but accessing it throws + try: + if d[key] is False: + del d[key] + continue + except IndexError: + pass + + if isinstance(d, (MultiDict, ImmutableMultiDict)): + v = d.getlist(key) + else: + v = d.get(key) + if not isinstance(v, list): + v = [v] + for item in v: + od.add(key, item) + href = Href( + '/caravel/filter/{self.datasource.type}/' + '{self.datasource.id}/'.format(**locals())) + return href(od) + def get_df(self, query_obj=None): """Returns a pandas dataframe based on the query object""" if not query_obj: @@ -308,6 +336,7 @@ def get_json(self): 'form_data': self.form_data, 'json_endpoint': self.json_endpoint, 'query': self.query, + 'filter_endpoint': self.filter_endpoint, 'standalone_endpoint': self.standalone_endpoint, } payload['cached_dttm'] = datetime.now().isoformat().split('.')[0] @@ -341,9 +370,11 @@ def data(self): 'csv_endpoint': self.csv_endpoint, 'form_data': self.form_data, 'json_endpoint': self.json_endpoint, + 'filter_endpoint': self.filter_endpoint, 'standalone_endpoint': self.standalone_endpoint, 'token': self.token, 'viz_name': self.viz_type, + 'filter_select_enabled': self.datasource.filter_select_enabled, 'column_formats': { m.metric_name: m.d3format for m in self.datasource.metrics @@ -357,6 +388,34 @@ def get_csv(self): include_index = not isinstance(df.index, pd.RangeIndex) return df.to_csv(index=include_index, encoding="utf-8") + def get_values_for_column(self, column): + """ + Retrieves values for a column to be used by the filter dropdown. + + :param column: column name + :return: JSON containing the some values for a column + """ + form_data = self.form_data + + since = form_data.get("since", "1 year ago") + from_dttm = utils.parse_human_datetime(since) + now = datetime.now() + if from_dttm > now: + from_dttm = now - (from_dttm - now) + until = form_data.get("until", "now") + to_dttm = utils.parse_human_datetime(until) + if from_dttm > to_dttm: + flasher("The date range doesn't seem right.", "danger") + from_dttm = to_dttm # Making them identical to not raise + + kwargs = dict( + column_name=column, + from_dttm=from_dttm, + to_dttm=to_dttm, + ) + df = self.datasource.values_for_column(**kwargs) + return df[column].to_json() + def get_data(self): return [] @@ -364,6 +423,10 @@ def get_data(self): def json_endpoint(self): return self.get_url(json="true") + @property + def filter_endpoint(self): + return self.get_filter_url() + @property def cache_key(self): url = self.get_url(for_cache_key=True, json="true", force="false")