Custom WHERE clause for tables (not druid) + error handling refactor

panoramix/forms.py

@@ -1,4 +1,5 @@
 from wtforms import Field, Form, SelectMultipleField, SelectField, TextField
+from copy import copy
 
 
 class OmgWtForm(Form):
@@ -24,7 +25,8 @@ def field_css_classes(self, fieldname):
         return ""
 
 
-def form_factory(datasource, viz, form_args=None):
+def form_factory(viz):
+    datasource = viz.datasource
     from panoramix.viz import viz_types
     row_limits = [10, 50, 100, 500, 1000, 5000, 10000]
     series_limits = [0, 5, 10, 25, 50, 100, 500]
@@ -57,6 +59,7 @@ def form_factory(datasource, viz, form_args=None):
         'x': SelectField('X Axis', choices=datasource.metrics_combo),
         'y': SelectField('Y Axis', choices=datasource.metrics_combo),
         'size': SelectField('Bubble Size', choices=datasource.metrics_combo),
+        'where': TextField('Custom WHERE clause'),
     }
     field_css_classes = {k: ['form-control'] for k in px_form_fields.keys()}
     select2 = [
@@ -70,7 +73,7 @@ def form_factory(datasource, viz, form_args=None):
         field_css_classes[field] += ['select2']
 
     class QueryForm(OmgWtForm):
-        field_order = viz.form_fields
+        field_order = copy(viz.form_fields)
         css_classes = field_css_classes
 
     for i in range(10):
@@ -85,4 +88,9 @@ class QueryForm(OmgWtForm):
             ff = [ff]
         for s in ff:
             setattr(QueryForm, s, px_form_fields[s])
+
+    # datasource type specific form elements
+    if datasource.__class__.__name__ == 'Table':
+        QueryForm.field_order += ['where']
+        setattr(QueryForm, 'where', px_form_fields['where'])
     return QueryForm

panoramix/models.py

@@ -11,7 +11,7 @@
 from sqlalchemy import Table as sqlaTable
 from sqlalchemy import create_engine, MetaData, desc, select, and_
 from sqlalchemy.orm import relationship
-from sqlalchemy.sql import table, literal_column
+from sqlalchemy.sql import table, literal_column, text
 
 from copy import deepcopy, copy
 from collections import namedtuple
@@ -99,7 +99,9 @@ def query_bkp(
             limit_spec=None,
             filter=None,
             is_timeseries=True,
-            timeseries_limit=15, row_limit=None):
+            timeseries_limit=15,
+            row_limit=None,
+            extras=None):
         """
         Unused, legacy way of querying by building a SQL string without
         using the sqlalchemy expression API (new approach which supports
@@ -192,7 +194,8 @@ def query(
             limit_spec=None,
             filter=None,
             is_timeseries=True,
-            timeseries_limit=15, row_limit=None):
+            timeseries_limit=15, row_limit=None,
+            extras=None):
 
         qry_start_dttm = datetime.now()
         timestamp = literal_column(
@@ -236,6 +239,8 @@ def query(
                 if op == 'not in':
                     cond = ~cond
                 where_clause_and.append(cond)
+        if extras and 'where' in extras:
+            where_clause_and += [text(extras['where'])]
         qry = qry.where(and_(*where_clause_and))
         qry = qry.order_by(desc(main_metric_expr))
         qry = qry.limit(row_limit)
@@ -530,7 +535,8 @@ def query(
             filter=None,
             is_timeseries=True,
             timeseries_limit=None,
-            row_limit=None):
+            row_limit=None,
+            extras=None):
         qry_start_dttm = datetime.now()
 
         # add tzinfo to native datetime with config

panoramix/templates/panoramix/datasource.html

@@ -1,6 +1,7 @@
 {% extends "panoramix/base.html" %}
 {% block head_css %}
 {{super()}}
+{% set datasource = viz.datasource %}
 <style>
 .select2-container-multi .select2-choices {
   height: 70px; 
@@ -97,7 +98,7 @@ <h3>{{ viz.verbose_name }}
     <hr/>
     {% block viz %}
         {% if error_msg %}
-            <span class="alert alert-danger">{{ error_msg }}</span>
+            <div class="alert alert-danger">{{ error_msg }}</div>
         {% endif %}
     {% endblock %}
 

panoramix/views.py

@@ -196,7 +196,7 @@ def table(self, table_id):
                 json.dumps(obj.get_query(), indent=4),
                 status=200,
                 mimetype="application/json")
-        return obj.render()
+        return obj.check_and_render()
 
     @has_access
     @expose("/datasource/<datasource_name>/")
@@ -224,7 +224,7 @@ def datasource(self, datasource_name):
         if not hasattr(obj, 'df') or obj.df is None or obj.df.empty:
             return obj.render_no_data()
 
-        return obj.render()
+        return obj.check_and_render()
 
     @has_access
     @expose("/refresh_datasources/")

panoramix/viz.py

@@ -46,7 +46,7 @@ def __init__(self, datasource, form_data, view):
             self.error_msg = str(e)
 
     def form_class(self):
-        return form_factory(self.datasource, self, request.args)
+        return form_factory(self)
 
     def query_filters(self):
         args = self.form_data
@@ -86,6 +86,12 @@ def query_obj(self):
         if from_dttm >= to_dttm:
             flash("The date range doesn't seem right.", "danger")
             from_dttm = to_dttm  # Making them identicial to not raise
+
+        # extras are used to query elements specific to a datasource type
+        # for instance the extra where clause that applies only to Tables
+        extras = {
+            'where': args.get("where")
+        }
         d = {
             'granularity': granularity,
             'from_dttm': from_dttm,
@@ -96,13 +102,20 @@ def query_obj(self):
             'row_limit': row_limit,
             'filter': self.query_filters(),
             'timeseries_limit': limit,
+            'extras': extras,
         }
         return d
 
     def render_no_data(self):
         self.template = "panoramix/no_data.html"
         return BaseViz.render(self)
 
+    def check_and_render(self, *args, **kwards):
+        if self.error_msg:
+            return BaseViz.render(self, error_msg=self.error_msg)
+        else:
+            return self.render(*args, **kwards)
+
     def render(self, *args, **kwargs):
         form = self.form_class(self.form_data)
         return self.view.render_template(
@@ -123,9 +136,6 @@ def query_obj(self):
         return d
 
     def render(self):
-        if self.error_msg:
-            return super(TableViz, self).render(error_msg=self.error_msg)
-
         df = self.df
         if df is None or df.empty:
             return super(TableViz, self).render(error_msg="No data.")
@@ -138,9 +148,6 @@ def render(self):
                 df[m + '__perc'] = np.rint((df[m] / np.max(df[m])) * 100)
         return super(TableViz, self).render(df=df)
 
-    def form_class(self):
-        return form_factory(self.datasource, self, request.args)
-
 
 class HighchartsViz(BaseViz):
     verbose_name = "Base Highcharts Viz"
@@ -159,9 +166,6 @@ class BubbleViz(HighchartsViz):
         'viz_type', 'since', 'until',
         'series', 'entity', 'x', 'y', 'size', 'limit']
 
-    def form_class(self):
-        return form_factory(self.datasource, self, request.args)
-
     def query_obj(self):
         d = super(BubbleViz, self).query_obj()
         d['granularity'] = 'all'
@@ -184,17 +188,14 @@ def query_obj(self):
         return d
 
     def render(self):
-        if not self.error_msg:
-            df = self.df.fillna(0)
-            df['x'] = df[[self.x_metric]]
-            df['y'] = df[[self.y_metric]]
-            df['z'] = df[[self.z_metric]]
-            df['name'] = df[[self.entity]]
-            df['group'] = df[[self.series]]
-            chart = HighchartBubble(df)
-            return super(BubbleViz, self).render(chart_js=chart.javascript_cmd)
-        else:
-            return super(BubbleViz, self).render(error_msg=self.error_msg)
+        df = self.df.fillna(0)
+        df['x'] = df[[self.x_metric]]
+        df['y'] = df[[self.y_metric]]
+        df['z'] = df[[self.z_metric]]
+        df['name'] = df[[self.entity]]
+        df['group'] = df[[self.series]]
+        chart = HighchartBubble(df)
+        return super(BubbleViz, self).render(chart_js=chart.javascript_cmd)
 
 
 class TimeSeriesViz(HighchartsViz):
@@ -243,9 +244,6 @@ def render(self):
             **CHART_ARGS)
         return super(TimeSeriesViz, self).render(chart_js=chart.javascript_cmd)
 
-    def form_class(self):
-        return form_factory(self.datasource, self, request.args)
-
     def bake_query(self):
         """
         Doing a 2 phase query where we limit the number of series.
@@ -283,6 +281,7 @@ class TimeSeriesStackedBarViz(TimeSeriesViz):
 class DistributionBarViz(HighchartsViz):
     verbose_name = "Distribution - Bar Chart"
     chart_type = "column"
+    form_fields = BaseViz.form_fields + ['limit']
 
     def query_obj(self):
         d = super(DistributionBarViz, self).query_obj()
@@ -305,6 +304,7 @@ def render(self):
 class DistributionPieViz(HighchartsViz):
     verbose_name = "Distribution - Pie Chart"
     chart_type = "pie"
+    form_fields = BaseViz.form_fields + ['limit']
 
     def query_obj(self):
         d = super(DistributionPieViz, self).query_obj()