diff --git a/caravel/assets/javascripts/explore/explore.jsx b/caravel/assets/javascripts/explore/explore.jsx index 43850e5c5bdf..81d06e5ea084 100644 --- a/caravel/assets/javascripts/explore/explore.jsx +++ b/caravel/assets/javascripts/explore/explore.jsx @@ -24,39 +24,12 @@ require('../../vendor/pygments.css'); require('../../stylesheets/explore.css'); let slice; +let filterCount = 1; const getPanelClass = function (fieldPrefix) { return (fieldPrefix === 'flt' ? 'filter' : 'having') + '_panel'; }; -function prepForm() { - // Assigning the right id to form elements in filters - const fixId = function ($filter, fieldPrefix, i) { - $filter.attr('id', function () { - return fieldPrefix + '_' + i; - }); - - ['col', 'op', 'eq'].forEach(function (fieldMiddle) { - const fieldName = fieldPrefix + '_' + fieldMiddle; - $filter.find('[id^=' + fieldName + '_]') - .attr('id', function () { - return fieldName + '_' + i; - }) - .attr('name', function () { - return fieldName + '_' + i; - }); - }); - }; - - ['flt', 'having'].forEach(function (fieldPrefix) { - let i = 1; - $('#' + getPanelClass(fieldPrefix) + ' #filters > div').each(function () { - fixId($(this), fieldPrefix, i); - i++; - }); - }); -} - function query(forceUpdate, pushState) { let force = forceUpdate; if (force === undefined) { @@ -67,10 +40,8 @@ function query(forceUpdate, pushState) { $('div.alert').remove(); } $('#is_cached').hide(); - prepForm(); if (pushState !== false) { - // update the url after prepForm() fix the field ids history.pushState({}, document.title, slice.querystring()); } slice.render(force); @@ -104,7 +75,6 @@ function saveSlice() { return; } $('#action').val(action); - prepForm(); $('#query').submit(); } @@ -199,15 +169,120 @@ function initExploreView() { $('[data-toggle="tooltip"]').tooltip({ container: 'body' }); $('.ui-helper-hidden-accessible').remove(); // jQuery-ui 1.11+ creates a div for every tooltip + function createChoices(term, data) { + const filtered = $(data).filter(function () { + return this.text.localeCompare(term) === 0; + }); + if (filtered.length === 0) { + return { + id: term, + text: term, + }; + } + return {}; + } + + function convertSelect(selectId, multiple) { + const parent = $(selectId).parent(); + const name = $(selectId).attr('name'); + const l = []; + let selected = ''; + const options = $(selectId + ' option'); + const classes = $(selectId).attr('class'); + for (let i = 0; i < options.length; i++) { + l.push({ + id: options[i].value, + text: options[i].text, + }); + if (options[i].selected) { + selected = options[i].value; + } + } + $(selectId).remove(); + parent.append( + '' + ); + $(`input[name='${name}']`).select2({ + createSearchChoice: createChoices, + dropdownAutoWidth: true, + multiple, + data: l, + sortResults(data) { + return data.sort(function (a, b) { + if (a.text > b.text) { + return 1; + } else if (a.text < b.text) { + return -1; + } + return 0; + }); + }, + }); + } + + function insertFilterChoices(i, fieldPrefix) { + const column = $('#' + fieldPrefix + '_col_' + i).val(); + const eq = '#' + fieldPrefix + '_eq_' + i; + $(eq).empty(); + + $.getJSON(slice.filterEndpoint(column), function (data) { + $(eq).append($('').attr('value', null).text('')); + $.each(data, function (key, value) { + let wrappedValue = value; + if (value.indexOf(',') !== -1) { + wrappedValue = '\'' + value + '\''; + } + $(eq).append($('') + .attr('value', wrappedValue) + .text(wrappedValue)); + }); + $(eq).select2('destroy'); + convertSelect(eq, true); + }); + } + function addFilter(i, fieldPrefix) { + const isHaving = fieldPrefix === 'having'; const cp = $('#' + fieldPrefix + '0').clone(); $(cp).appendTo('#' + getPanelClass(fieldPrefix) + ' #filters'); + $(cp).attr('id', fieldPrefix + filterCount); $(cp).show(); + + const eqId = fieldPrefix + '_eq_' + filterCount; + const $eq = $(cp).find('#' + fieldPrefix + '_eq_0'); + $eq.attr('id', eqId).attr('name', eqId); + + const opId = fieldPrefix + '_op_' + filterCount; + const $op = $(cp).find('#' + fieldPrefix + '_op_0'); + $op.attr('id', opId).attr('name', opId); + + const colId = fieldPrefix + '_col_' + filterCount; + const $col = $(cp).find('#' + fieldPrefix + '_col_0'); + $col.attr('id', colId).attr('name', colId); + + // Set existing values if (i !== undefined) { - $(cp).find('#' + fieldPrefix + '_eq_0').val(px.getParam(fieldPrefix + '_eq_' + i)); - $(cp).find('#' + fieldPrefix + '_op_0').val(px.getParam(fieldPrefix + '_op_' + i)); - $(cp).find('#' + fieldPrefix + '_col_0').val(px.getParam(fieldPrefix + '_col_' + i)); + $op.val(px.getParam(fieldPrefix + '_op_' + i)); + $col.val(px.getParam(fieldPrefix + '_col_' + i)); + + if (isHaving || !slice.filterSelectEnabled) { + $eq.val(px.getParam(fieldPrefix + '_eq_' + i)); + } else { + insertFilterChoices(filterCount, fieldPrefix); + const eqVal = px.getParam(fieldPrefix + '_eq_' + i); + $eq.append($('') + .attr('value', eqVal).text(eqVal)); + } } + + if (slice.filterSelectEnabled && !isHaving) { + const currentFilter = filterCount; + $col.change(function () { + insertFilterChoices(currentFilter, fieldPrefix); + }); + } + $(cp).find('select').select2(); $(cp).find('.remove').click(function () { $(this) @@ -215,6 +290,7 @@ function initExploreView() { .parent() .remove(); }); + filterCount++; } function setFilters() { @@ -244,52 +320,8 @@ function initExploreView() { addFilter(undefined, 'having'); }); - function createChoices(term, data) { - const filtered = $(data).filter(function () { - return this.text.localeCompare(term) === 0; - }); - if (filtered.length === 0) { - return { - id: term, - text: term, - }; - } - return {}; - } - - function initSelectionToValue(element, callback) { - callback({ - id: element.val(), - text: element.val(), - }); - } - $('.select2_freeform').each(function () { - const parent = $(this).parent(); - const name = $(this).attr('name'); - const l = []; - let selected = ''; - for (let i = 0; i < this.options.length; i++) { - l.push({ - id: this.options[i].value, - text: this.options[i].text, - }); - if (this.options[i].selected) { - selected = this.options[i].value; - } - } - parent.append( - `` - ); - $(`input[name='${name}']`).select2({ - createSearchChoice: createChoices, - initSelection: initSelectionToValue, - dropdownAutoWidth: true, - multiple: false, - data: l, - }); - $(this).remove(); + convertSelect('#' + $(this).attr('id'), false); }); function prepSaveDialog() { @@ -387,9 +419,10 @@ exploreController = Object.assign({}, utils.controllerInterface, exploreControll $(document).ready(function () { const data = $('.slice').data('slice'); - initExploreView(); - slice = px.Slice(data, exploreController); + $('.slice').data('slice', slice); + + initExploreView(); // call vis render method, which issues ajax // calls render on the slice for the first time diff --git a/caravel/assets/javascripts/modules/caravel.js b/caravel/assets/javascripts/modules/caravel.js index 73e9a93680c3..154131ea6a24 100644 --- a/caravel/assets/javascripts/modules/caravel.js +++ b/caravel/assets/javascripts/modules/caravel.js @@ -61,6 +61,7 @@ const px = function () { const container = $(selector); const sliceId = data.slice_id; const origJsonEndpoint = data.json_endpoint; + const filterSelectEnabled = data.filter_select_enabled; let dttm = 0; const stopwatch = function () { dttm += 10; @@ -77,6 +78,7 @@ const px = function () { data, container, containerId, + filterSelectEnabled, selector, querystring() { const parser = document.createElement('a'); @@ -111,6 +113,11 @@ const px = function () { endpoint += '&force=' + this.force; return endpoint; }, + filterEndpoint(column) { + const parser = document.createElement('a'); + parser.href = data.filter_endpoint; + return parser.pathname + column + '/' + this.querystring(); + }, d3format(col, number) { // uses the utils memoized d3format function and formats based on // column level defined preferences diff --git a/caravel/forms.py b/caravel/forms.py index 01b9da5641f9..6e42b9d5f931 100755 --- a/caravel/forms.py +++ b/caravel/forms.py @@ -1088,13 +1088,17 @@ def add_to_form(attrs): _('Filter 1'), default=col_choices[0][0], choices=col_choices)) - setattr(QueryForm, field_prefix + '_op_' + str(i), SelectField( - _('Filter 1'), - default=op_choices[0][0], - choices=op_choices)) - setattr( - QueryForm, field_prefix + '_eq_' + str(i), - TextField(_("Super"), default='')) + setattr(QueryForm, field_prefix + '_op_' + str(i), + SelectField( + _('Filter 1'), + default=op_choices[0][0], + choices=op_choices)) + if viz.datasource.filter_select_enabled and not is_having_filter: + setattr(QueryForm, field_prefix + '_eq_' + str(i), + FreeFormSelectField(_("Super"), choices=[])) + else: + setattr(QueryForm, field_prefix + '_eq_' + str(i), + TextField(_("Super"), default='')) if time_fields: QueryForm.fieldsets = ({ diff --git a/caravel/migrations/versions/6584a92c17a3_enable_filter_select.py b/caravel/migrations/versions/6584a92c17a3_enable_filter_select.py new file mode 100644 index 000000000000..ebd4b700e59c --- /dev/null +++ b/caravel/migrations/versions/6584a92c17a3_enable_filter_select.py @@ -0,0 +1,26 @@ +"""Enable Filter Select + +Revision ID: 6584a92c17a3 +Revises: c611f2b591b8 +Create Date: 2016-11-07 17:25:33.081565 + +""" + +# revision identifiers, used by Alembic. +revision = '6584a92c17a3' +down_revision = 'c611f2b591b8' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.add_column('datasources', sa.Column('filter_select_enabled', + sa.Boolean(), default=False)) + op.add_column('tables', sa.Column('filter_select_enabled', + sa.Boolean(), default=False)) + + +def downgrade(): + op.drop_column('tables', 'filter_select_enabled') + op.drop_column('datasources', 'filter_select_enabled') diff --git a/caravel/models.py b/caravel/models.py index 8494d4716a81..a4fdf22661ed 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -32,6 +32,7 @@ from flask_babel import lazy_gettext as _ from pydruid.client import PyDruid +from pydruid.utils.aggregators import count from pydruid.utils.filters import Dimension, Filter from pydruid.utils.postaggregator import Postaggregator from pydruid.utils.having import Aggregation @@ -841,6 +842,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable, ImportMixin): default_endpoint = Column(Text) database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False) is_featured = Column(Boolean, default=False) + filter_select_enabled = Column(Boolean, default=False) user_id = Column(Integer, ForeignKey('ab_user.id')) owner = relationship('User', backref='tables', foreign_keys=[user_id]) database = relationship( @@ -945,6 +947,45 @@ def get_col(self, col_name): if col_name == col.column_name: return col + def values_for_column(self, + column_name, + from_dttm, + to_dttm, + limit=500): + """Runs query against sqla to retrieve some + sample values for the given column. + """ + granularity = self.main_dttm_col + + cols = {col.column_name: col for col in self.columns} + target_col = cols[column_name] + + tbl = table(self.table_name) + qry = select([target_col.sqla_col]) + qry = qry.select_from(tbl) + qry = qry.distinct(column_name) + qry = qry.limit(limit) + + if granularity: + dttm_col = cols[granularity] + timestamp = dttm_col.sqla_col.label('timestamp') + time_filter = [ + timestamp >= text(dttm_col.dttm_sql_literal(from_dttm)), + timestamp <= text(dttm_col.dttm_sql_literal(to_dttm)), + ] + qry = qry.where(and_(*time_filter)) + + engine = self.database.get_sqla_engine() + sql = "{}".format( + qry.compile( + engine, compile_kwargs={"literal_binds": True}, ), + ) + + return pd.read_sql_query( + sql=sql, + con=engine + ) + def query( # sqla self, groupby, metrics, granularity, @@ -1547,6 +1588,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable): datasource_name = Column(String(255), unique=True) is_featured = Column(Boolean, default=False) is_hidden = Column(Boolean, default=False) + filter_select_enabled = Column(Boolean, default=False) description = Column(Text) default_endpoint = Column(Text) user_id = Column(Integer, ForeignKey('ab_user.id')) @@ -1849,6 +1891,37 @@ def granularity(period_name, timezone=None, origin=None): period_name).total_seconds() * 1000 return granularity + def values_for_column(self, + column_name, + from_dttm, + to_dttm, + limit=500): + """ + Runs query against Druid to retrieve some values for the given column + """ + # TODO: Use Lexicographic TopNMeticSpec onces supported by PyDruid + from_dttm = from_dttm.replace(tzinfo=config.get("DRUID_TZ")) + to_dttm = to_dttm.replace(tzinfo=config.get("DRUID_TZ")) + + qry = dict( + datasource=self.datasource_name, + granularity="all", + intervals=from_dttm.isoformat() + '/' + to_dttm.isoformat(), + aggregations=dict(count=count("count")), + dimension=column_name, + metric="count", + threshold=limit, + ) + + client = self.cluster.get_pydruid_client() + client.topn(**qry) + df = client.export_pandas() + + if df is None or df.size == 0: + raise Exception(_("No data was returned.")) + + return df + def query( # druid self, groupby, metrics, granularity, diff --git a/caravel/views.py b/caravel/views.py index 28e81041e6e6..7c4516605ff2 100755 --- a/caravel/views.py +++ b/caravel/views.py @@ -600,8 +600,8 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa 'link', 'database', 'is_featured', 'changed_on_'] add_columns = ['table_name', 'database', 'schema'] edit_columns = [ - 'table_name', 'sql', 'is_featured', 'database', 'schema', - 'description', 'owner', + 'table_name', 'sql', 'is_featured', 'filter_select_enabled', + 'database', 'schema', 'description', 'owner', 'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout'] related_views = [TableColumnInlineView, SqlMetricInlineView] base_order = ('changed_on', 'desc') @@ -627,6 +627,7 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa 'database': _("Database"), 'changed_on_': _("Last Changed"), 'is_featured': _("Is Featured"), + 'filter_select_enabled': _("Enable Filter Select"), 'schema': _("Schema"), 'default_endpoint': _("Default Endpoint"), 'offset': _("Offset"), @@ -977,8 +978,8 @@ class DruidDatasourceModelView(CaravelModelView, DeleteMixin): # noqa related_views = [DruidColumnInlineView, DruidMetricInlineView] edit_columns = [ 'datasource_name', 'cluster', 'description', 'owner', - 'is_featured', 'is_hidden', 'default_endpoint', 'offset', - 'cache_timeout'] + 'is_featured', 'is_hidden', 'filter_select_enabled', + 'default_endpoint', 'offset', 'cache_timeout'] add_columns = edit_columns page_size = 500 base_order = ('datasource_name', 'asc') @@ -996,6 +997,7 @@ class DruidDatasourceModelView(CaravelModelView, DeleteMixin): # noqa 'owner': _("Owner"), 'is_featured': _("Is Featured"), 'is_hidden': _("Is Hidden"), + 'filter_select_enabled': _("Enable Filter Select"), 'default_endpoint': _("Default Endpoint"), 'offset': _("Time Offset"), 'cache_timeout': _("Cache Timeout"), @@ -1389,6 +1391,59 @@ def explore(self, datasource_type, datasource_id): userid=g.user.get_id() if g.user else '' ) + @api + @has_access_api + @expose("/filter////") + def filter(self, datasource_type, datasource_id, column): + """ + Endpoint to retrieve values for specified column. + + :param datasource_type: Type of datasource e.g. table + :param datasource_id: Datasource id + :param column: Column name to retrieve values for + :return: + """ + # TODO: Cache endpoint by user, datasource and column + error_redirect = '/slicemodelview/list/' + datasource_class = models.SqlaTable \ + if datasource_type == "table" else models.DruidDatasource + + datasource = db.session.query( + datasource_class).filter_by(id=datasource_id).first() + + if not datasource: + flash(__("The datasource seems to have been deleted"), "alert") + return redirect(error_redirect) + + all_datasource_access = self.can_access( + 'all_datasource_access', 'all_datasource_access') + datasource_access = self.can_access( + 'datasource_access', datasource.perm) + if not (all_datasource_access or datasource_access): + flash(__("You don't seem to have access to this datasource"), + "danger") + return redirect(error_redirect) + + viz_type = request.args.get("viz_type") + if not viz_type and datasource.default_endpoint: + return redirect(datasource.default_endpoint) + if not viz_type: + viz_type = "table" + try: + obj = viz.viz_types[viz_type]( + datasource, + form_data=request.args, + slice_=None) + except Exception as e: + flash(str(e), "danger") + return redirect(error_redirect) + status = 200 + payload = obj.get_values_for_column(column) + return Response( + payload, + status=status, + mimetype="application/json") + def save_or_overwrite_slice( self, args, slc, slice_add_perm, slice_edit_perm): """Save or overwrite a slice""" diff --git a/caravel/viz.py b/caravel/viz.py index 7b34daa5cacf..e09b7eb11504 100755 --- a/caravel/viz.py +++ b/caravel/viz.py @@ -151,6 +151,36 @@ def get_url(self, for_cache_key=False, json_endpoint=False, **kwargs): del od['force'] return href(od) + def get_filter_url(self): + """ + Returns the URL to retrieve column values used in the filter dropdown + """ + d = self.orig_form_data.copy() + # Remove unchecked checkboxes because HTML is weird like that + od = MultiDict() + for key in sorted(d.keys()): + # if MultiDict is initialized with MD({key:[emptyarray]}), + # key is included in d.keys() but accessing it throws + try: + if d[key] is False: + del d[key] + continue + except IndexError: + pass + + if isinstance(d, (MultiDict, ImmutableMultiDict)): + v = d.getlist(key) + else: + v = d.get(key) + if not isinstance(v, list): + v = [v] + for item in v: + od.add(key, item) + href = Href( + '/caravel/filter/{self.datasource.type}/' + '{self.datasource.id}/'.format(**locals())) + return href(od) + def get_df(self, query_obj=None): """Returns a pandas dataframe based on the query object""" if not query_obj: @@ -325,6 +355,7 @@ def get_json(self, force=False): 'form_data': self.form_data, 'json_endpoint': self.json_endpoint, 'query': self.query, + 'filter_endpoint': self.filter_endpoint, 'standalone_endpoint': self.standalone_endpoint, 'column_formats': self.data['column_formats'], } @@ -359,9 +390,11 @@ def data(self): 'csv_endpoint': self.csv_endpoint, 'form_data': self.form_data, 'json_endpoint': self.json_endpoint, + 'filter_endpoint': self.filter_endpoint, 'standalone_endpoint': self.standalone_endpoint, 'token': self.token, 'viz_name': self.viz_type, + 'filter_select_enabled': self.datasource.filter_select_enabled, 'column_formats': { m.metric_name: m.d3format for m in self.datasource.metrics @@ -375,6 +408,34 @@ def get_csv(self): include_index = not isinstance(df.index, pd.RangeIndex) return df.to_csv(index=include_index, encoding="utf-8") + def get_values_for_column(self, column): + """ + Retrieves values for a column to be used by the filter dropdown. + + :param column: column name + :return: JSON containing the some values for a column + """ + form_data = self.form_data + + since = form_data.get("since", "1 year ago") + from_dttm = utils.parse_human_datetime(since) + now = datetime.now() + if from_dttm > now: + from_dttm = now - (from_dttm - now) + until = form_data.get("until", "now") + to_dttm = utils.parse_human_datetime(until) + if from_dttm > to_dttm: + flasher("The date range doesn't seem right.", "danger") + from_dttm = to_dttm # Making them identical to not raise + + kwargs = dict( + column_name=column, + from_dttm=from_dttm, + to_dttm=to_dttm, + ) + df = self.datasource.values_for_column(**kwargs) + return df[column].to_json() + def get_data(self): return [] @@ -382,6 +443,10 @@ def get_data(self): def json_endpoint(self): return self.get_url(json_endpoint=True) + @property + def filter_endpoint(self): + return self.get_filter_url() + @property def cache_key(self): url = self.get_url(for_cache_key=True, json="true", force="false")