@@ -191,11 +193,12 @@ class VisualizeModal extends React.Component {
}
VisualizeModal.propTypes = {
query: React.PropTypes.object,
- show: React.PropTypes.boolean,
+ show: React.PropTypes.bool,
onHide: React.PropTypes.function,
};
VisualizeModal.defaultProps = {
show: false,
+ onHide: () => {},
};
function mapStateToProps() {
diff --git a/caravel/assets/javascripts/SqlLab/main.css b/caravel/assets/javascripts/SqlLab/main.css
index f130a162b43b..7c46f18811d1 100644
--- a/caravel/assets/javascripts/SqlLab/main.css
+++ b/caravel/assets/javascripts/SqlLab/main.css
@@ -239,3 +239,6 @@ div.tablePopover:hover {
.SouthPane .tab-content {
padding-top: 10px;
}
+.SqlEditor textarea {
+ display: none;
+}
diff --git a/caravel/assets/javascripts/dashboard/Dashboard.jsx b/caravel/assets/javascripts/dashboard/Dashboard.jsx
index 1f2137c70592..13677ae4b1f3 100644
--- a/caravel/assets/javascripts/dashboard/Dashboard.jsx
+++ b/caravel/assets/javascripts/dashboard/Dashboard.jsx
@@ -154,7 +154,7 @@ function dashboardContainer(dashboardData) {
refreshExcept(sliceId) {
const immune = this.metadata.filter_immune_slices || [];
this.slices.forEach(function (slice) {
- if (slice.data.slice_id !== sliceId && immune.indexOf(slice.data.sliceId) === -1) {
+ if (slice.data.slice_id !== sliceId && immune.indexOf(slice.data.slice_id) === -1) {
slice.render();
}
});
diff --git a/caravel/assets/javascripts/modules/caravel.js b/caravel/assets/javascripts/modules/caravel.js
index 4b013fa87a3a..2e673aef9398 100644
--- a/caravel/assets/javascripts/modules/caravel.js
+++ b/caravel/assets/javascripts/modules/caravel.js
@@ -127,8 +127,9 @@ const px = function () {
cachedSelector = $('#is_cached');
if (data !== undefined && data.is_cached) {
cachedSelector
- .attr('title',
- 'Served from data cached at ' + data.cached_dttm + '. Click to force-refresh')
+ .attr(
+ 'title',
+ `Served from data cached at ${data.cached_dttm}. Click [Query] to force-refresh`)
.show()
.tooltip('fixTitle');
} else {
diff --git a/caravel/assets/javascripts/welcome.js b/caravel/assets/javascripts/welcome.js
index 4cc6a3c77e72..28e48a3934ef 100644
--- a/caravel/assets/javascripts/welcome.js
+++ b/caravel/assets/javascripts/welcome.js
@@ -5,9 +5,7 @@ require('../stylesheets/welcome.css');
require('bootstrap');
require('datatables.net-bs');
require('../node_modules/datatables-bootstrap3-plugin/media/css/datatables-bootstrap3.css');
-require('../node_modules/cal-heatmap/cal-heatmap.css');
const d3 = require('d3');
-const CalHeatMap = require('cal-heatmap');
function modelViewTable(selector, modelView, orderCol, order) {
// Builds a dataTable from a flask appbuilder api endpoint
let url = '/' + modelView.toLowerCase() + '/api/read';
@@ -51,32 +49,5 @@ function modelViewTable(selector, modelView, orderCol, order) {
});
}
$(document).ready(function () {
- d3.json('/caravel/activity_per_day', function (json) {
- const ext = d3.extent(d3.values(json));
- const cal = new CalHeatMap();
- const range = 10;
- const legendBounds = [];
- const step = (ext[1] - ext[0]) / (range - 1);
- for (let i = 0; i < range; i++) {
- legendBounds.push(i * step + ext[0]);
- }
- cal.init({
- start: new Date().setFullYear(new Date().getFullYear() - 1),
- range: 13,
- data: json,
- legend: legendBounds,
- legendColors: [
- '#D6E685',
- '#1E6823',
- ],
- domain: 'month',
- subDomain: 'day',
- itemName: 'action',
- tooltip: true,
- cellSize: 10,
- cellPadding: 2,
- domainGutter: 22,
- });
- });
modelViewTable('#dash_table', 'DashboardModelViewAsync', 'changed_on', 'desc');
});
diff --git a/caravel/assets/package.json b/caravel/assets/package.json
index 321def76482a..0ccb17f1f71c 100644
--- a/caravel/assets/package.json
+++ b/caravel/assets/package.json
@@ -1,6 +1,6 @@
{
"name": "caravel",
- "version": "0.1.0",
+ "version": "0.10.0",
"description": "Caravel is a data exploration platform designed to be visual, intuitive, and interactive.",
"directories": {
"doc": "docs",
diff --git a/caravel/assets/visualizations/mapbox.css b/caravel/assets/visualizations/mapbox.css
index a64a4ac20977..babb33be0eac 100644
--- a/caravel/assets/visualizations/mapbox.css
+++ b/caravel/assets/visualizations/mapbox.css
@@ -1,16 +1,16 @@
-div.widget .slice_container {
+.mapbox div.widget .slice_container {
cursor: grab;
cursor: -moz-grab;
cursor: -webkit-grab;
overflow: hidden;
}
-div.widget .slice_container:active {
+.mapbox div.widget .slice_container:active {
cursor: grabbing;
cursor: -moz-grabbing;
cursor: -webkit-grabbing;
}
-.slice_container div {
+.mapbox .slice_container div {
padding-top: 0px;
}
diff --git a/caravel/assets/visualizations/nvd3_vis.css b/caravel/assets/visualizations/nvd3_vis.css
index bfa08daa6a3f..d92440e7ef7c 100644
--- a/caravel/assets/visualizations/nvd3_vis.css
+++ b/caravel/assets/visualizations/nvd3_vis.css
@@ -18,6 +18,11 @@ text.nv-axislabel {
.dist_bar svg.nvd3-svg {
width: auto;
+ font-size: 14px;
+}
+
+.nv-x text{
+ font-size: 12px;
}
.bar .slice_container {
diff --git a/caravel/assets/visualizations/nvd3_vis.js b/caravel/assets/visualizations/nvd3_vis.js
index 8ed92235cb42..7d8dbfa25a54 100644
--- a/caravel/assets/visualizations/nvd3_vis.js
+++ b/caravel/assets/visualizations/nvd3_vis.js
@@ -69,14 +69,16 @@ function nvd3Vis(slice) {
// Calculates the longest label size for stretching bottom margin
function calculateStretchMargins(payloadData) {
- const axisLabels = payloadData.data[0].values;
let stretchMargin = 0;
const pixelsPerCharX = 4.5; // approx, depends on font size
- let maxLabelSize = 0;
- for (let i = 0; i < axisLabels.length; i++) {
- maxLabelSize = Math.max(axisLabels[i].x.length, maxLabelSize);
- }
- stretchMargin = Math.ceil(Math.max(stretchMargin, pixelsPerCharX * maxLabelSize));
+ let maxLabelSize = 10; // accomodate for shorter labels
+ payloadData.data.forEach((d) => {
+ const axisLabels = d.values;
+ for (let i = 0; i < axisLabels.length; i++) {
+ maxLabelSize = Math.max(axisLabels[i].x.length, maxLabelSize);
+ }
+ });
+ stretchMargin = Math.ceil(pixelsPerCharX * maxLabelSize);
return stretchMargin;
}
@@ -310,8 +312,12 @@ function nvd3Vis(slice) {
}
if (fd.bottom_margin === 'auto') {
- const stretchMargin = calculateStretchMargins(payload);
- chart.margin({ bottom: stretchMargin });
+ if (vizType === 'dist_bar') {
+ const stretchMargin = calculateStretchMargins(payload);
+ chart.margin({ bottom: stretchMargin });
+ } else {
+ chart.margin({ bottom: 50 });
+ }
} else {
chart.margin({ bottom: fd.bottom_margin });
}
diff --git a/caravel/assets/webpack.config.js b/caravel/assets/webpack.config.js
index 79e5d780aa58..7b851e2e0e8c 100644
--- a/caravel/assets/webpack.config.js
+++ b/caravel/assets/webpack.config.js
@@ -1,5 +1,6 @@
const webpack = require('webpack');
const path = require('path');
+const fs = require('fs');
// input dir
const APP_DIR = path.resolve(__dirname, './');
@@ -7,6 +8,8 @@ const APP_DIR = path.resolve(__dirname, './');
// output dir
const BUILD_DIR = path.resolve(__dirname, './dist');
+const VERSION_STRING = JSON.parse(fs.readFileSync('package.json')).version;
+
const config = {
entry: {
'css-theme': APP_DIR + '/javascripts/css-theme.js',
@@ -19,7 +22,7 @@ const config = {
},
output: {
path: BUILD_DIR,
- filename: '[name].entry.js',
+ filename: `[name].${VERSION_STRING}.entry.js`,
},
resolve: {
extensions: [
diff --git a/caravel/bin/caravel b/caravel/bin/caravel
index 0c9428905328..a582c3e5ecb6 100755
--- a/caravel/bin/caravel
+++ b/caravel/bin/caravel
@@ -73,7 +73,7 @@ def version(verbose):
"-----------------------\n"
"Caravel {version}\n"
"-----------------------").format(
- boat=ascii_art.boat, version=caravel.VERSION)
+ boat=ascii_art.boat, version=config.get('VERSION_STRING'))
print(s)
if verbose:
print("[DB] : " + "{}".format(db.engine))
diff --git a/caravel/config.py b/caravel/config.py
index 233f99a6c782..4856c07fa72c 100644
--- a/caravel/config.py
+++ b/caravel/config.py
@@ -8,7 +8,9 @@
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
+from caravel import app
+import json
import os
from dateutil import tz
@@ -22,6 +24,11 @@
# ---------------------------------------------------------
# Caravel specific config
# ---------------------------------------------------------
+PACKAGE_DIR = os.path.join(BASE_DIR, 'static', 'assets')
+PACKAGE_FILE = os.path.join(PACKAGE_DIR, 'package.json')
+with open(PACKAGE_FILE) as package_file:
+ VERSION_STRING = json.load(package_file)['version']
+
ROW_LIMIT = 50000
CARAVEL_WORKERS = 16
diff --git a/caravel/data/energy.json.gz b/caravel/data/energy.json.gz
index b2f47a148b4f..624d71db683a 100644
Binary files a/caravel/data/energy.json.gz and b/caravel/data/energy.json.gz differ
diff --git a/caravel/models.py b/caravel/models.py
index c0752ff7442f..956c36e26b0e 100644
--- a/caravel/models.py
+++ b/caravel/models.py
@@ -3,6 +3,7 @@
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
+import re
import functools
import json
@@ -21,7 +22,7 @@
import sqlparse
from dateutil.parser import parse
-from flask import request, g
+from flask import escape, g, Markup, request
from flask_appbuilder import Model
from flask_appbuilder.models.mixins import AuditMixin
from flask_appbuilder.models.decorators import renders
@@ -38,10 +39,11 @@
DateTime, Date, Table, Numeric,
create_engine, MetaData, desc, asc, select, and_, func
)
+from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import relationship
from sqlalchemy.sql import table, literal_column, text, column
-from sqlalchemy.sql.expression import TextAsFrom
+from sqlalchemy.sql.expression import ColumnClause, TextAsFrom
from sqlalchemy_utils import EncryptedType
from werkzeug.datastructures import ImmutableMultiDict
@@ -54,6 +56,7 @@
config = app.config
QueryResult = namedtuple('namedtuple', ['df', 'query', 'duration'])
+FillterPattern = re.compile(r'''((?:[^,"']|"[^"]*"|'[^']*')+)''')
class JavascriptPostAggregator(Postaggregator):
@@ -100,12 +103,13 @@ def changed_by_(self):
@renders('changed_on')
def changed_on_(self):
- return '{}'.format(self.changed_on)
+ return Markup(
+ '{}'.format(self.changed_on))
@renders('changed_on')
def modified(self):
s = humanize.naturaltime(datetime.now() - self.changed_on)
- return '{}'.format(s)
+ return Markup('{}'.format(s))
@property
def icons(self):
@@ -250,8 +254,8 @@ def edit_url(self):
@property
def slice_link(self):
url = self.slice_url
- return '{obj.slice_name}'.format(
- url=url, obj=self)
+ name = escape(self.slice_name)
+ return Markup('{name}'.format(**locals()))
def get_viz(self, url_params_multidict=None):
"""Creates :py:class:viz.BaseViz object from the url_params_multidict.
@@ -347,7 +351,9 @@ def sqla_metadata(self):
return metadata.reflect()
def dashboard_link(self):
- return '{obj.dashboard_title}'.format(obj=self)
+ title = escape(self.dashboard_title)
+ return Markup(
+ '{title}'.format(**locals()))
@property
def json_data(self):
@@ -427,6 +433,12 @@ def backend(self):
url = make_url(self.sqlalchemy_uri_decrypted)
return url.get_backend_name()
+ def set_sqlalchemy_uri(self, uri):
+ conn = sqla.engine.url.make_url(uri)
+ self.password = conn.password
+ conn.password = "X" * 10 if conn.password else None
+ self.sqlalchemy_uri = str(conn) # hides the password
+
def get_sqla_engine(self, schema=None):
extra = self.get_extra()
url = make_url(self.sqlalchemy_uri_decrypted)
@@ -589,6 +601,22 @@ def grains(self):
def grains_dict(self):
return {grain.name: grain for grain in self.grains()}
+ def epoch_to_dttm(self, ms=False):
+ """Database-specific SQL to convert unix timestamp to datetime
+ """
+ ts2date_exprs = {
+ 'sqlite': "datetime({col}, 'unixepoch')",
+ 'postgresql': "(timestamp 'epoch' + {col} * interval '1 second')",
+ 'mysql': "from_unixtime({col})",
+ 'mssql': "dateadd(S, {col}, '1970-01-01')"
+ }
+ ts2date_exprs['redshift'] = ts2date_exprs['postgresql']
+ ts2date_exprs['vertica'] = ts2date_exprs['postgresql']
+ for db_type, expr in ts2date_exprs.items():
+ if self.sqlalchemy_uri.startswith(db_type):
+ return expr.replace('{col}', '({col}/1000.0)') if ms else expr
+ raise Exception(_("Unable to convert unix epoch to datetime"))
+
def get_extra(self):
extra = {}
if self.extra:
@@ -666,7 +694,9 @@ def description_markeddown(self):
@property
def link(self):
- return '{self.table_name}'.format(**locals())
+ table_name = escape(self.table_name)
+ return Markup(
+ '{table_name}'.format(**locals()))
@property
def perm(self):
@@ -710,10 +740,6 @@ def html(self):
def name(self):
return self.table_name
- @renders('table_name')
- def table_link(self):
- return '{obj.table_name}'.format(obj=self)
-
@property
def metrics_combo(self):
return sorted(
@@ -786,6 +812,22 @@ def query( # sqla
metrics_exprs = []
if granularity:
+
+ # TODO: sqlalchemy 1.2 release should be doing this on its own.
+ # Patch only if the column clause is specific for DateTime set and
+ # granularity is selected.
+ @compiles(ColumnClause)
+ def _(element, compiler, **kw):
+ text = compiler.visit_column(element, **kw)
+ try:
+ if element.is_literal and hasattr(element.type, 'python_type') and \
+ type(element.type) is DateTime:
+
+ text = text.replace('%%', '%')
+ except NotImplementedError:
+ pass # Some elements raise NotImplementedError for python_type
+ return text
+
dttm_col = cols[granularity]
dttm_expr = dttm_col.sqla_col.label('timestamp')
timestamp = dttm_expr
@@ -793,9 +835,15 @@ def query( # sqla
# Transforming time grain into an expression based on configuration
time_grain_sqla = extras.get('time_grain_sqla')
if time_grain_sqla:
+ if dttm_col.python_date_format == 'epoch_s':
+ dttm_expr = self.database.epoch_to_dttm().format(
+ col=dttm_expr)
+ elif dttm_col.python_date_format == 'epoch_ms':
+ dttm_expr = self.database.epoch_to_dttm(ms=True).format(
+ col=dttm_expr)
udf = self.database.grains_dict().get(time_grain_sqla, '{col}')
timestamp_grain = literal_column(
- udf.function.format(col=dttm_expr)).label('timestamp')
+ udf.function.format(col=dttm_expr), type_=DateTime).label('timestamp')
else:
timestamp_grain = timestamp
@@ -839,7 +887,8 @@ def query( # sqla
for col, op, eq in filter:
col_obj = cols[col]
if op in ('in', 'not in'):
- values = eq.split(",")
+ splitted = FillterPattern.split(eq)[1::2]
+ values = [types.replace("'", '').strip() for types in splitted]
cond = col_obj.sqla_col.in_(values)
if op == 'not in':
cond = ~cond
@@ -1155,6 +1204,9 @@ def refresh_datasources(self):
for datasource in self.get_datasources():
if datasource not in config.get('DRUID_DATA_SOURCE_BLACKLIST'):
DruidDatasource.sync_to_db(datasource, self)
+ @property
+ def perm(self):
+ return "[{obj.cluster_name}].(id:{obj.id})".format(obj=self)
class DruidDatasource(Model, AuditMixinNullable, Queryable):
@@ -1203,9 +1255,8 @@ def perm(self):
@property
def link(self):
- return (
- ''
- '{self.datasource_name}').format(**locals())
+ name = escape(self.datasource_name)
+ return Markup('{name}').format(**locals())
@property
def full_name(self):
@@ -1219,8 +1270,8 @@ def __repr__(self):
@renders('datasource_name')
def datasource_link(self):
url = "/caravel/explore/{obj.type}/{obj.id}/".format(obj=self)
- return '{obj.datasource_name}'.format(
- url=url, obj=self)
+ name = escape(self.datasource_name)
+ return Markup('{name}'.format(**locals()))
def get_metric_obj(self, metric_name):
return [
@@ -1597,9 +1648,11 @@ def get_filters(raw_filters):
cond = ~(Dimension(col) == eq)
elif op in ('in', 'not in'):
fields = []
- splitted = eq.split(',')
- if len(splitted) > 1:
- for s in eq.split(','):
+ # Distinguish quoted values with regular value types
+ splitted = FillterPattern.split(eq)[1::2]
+ values = [types.replace("'", '') for types in splitted]
+ if len(values) > 1:
+ for s in values:
s = s.strip()
fields.append(Dimension(col) == s)
cond = Filter(type="or", fields=fields)
diff --git a/caravel/templates/appbuilder/general/widgets/list.html b/caravel/templates/appbuilder/general/widgets/list.html
index 294aa5b4c755..797787cdca63 100644
--- a/caravel/templates/appbuilder/general/widgets/list.html
+++ b/caravel/templates/appbuilder/general/widgets/list.html
@@ -66,7 +66,7 @@
id="{{ '{}__{}'.format(pk, value) }}"
data-checkbox-api-prefix="/caravel/checkbox/{{ modelview_name }}/{{ pk }}/{{ value }}/">
{% else %}
- {{ item[value]|safe }}
+ {{ item[value] }}
{% endif %}
{% endfor %}
diff --git a/caravel/templates/caravel/base.html b/caravel/templates/caravel/base.html
index 69a6d0a9bdad..49502f29504d 100644
--- a/caravel/templates/caravel/base.html
+++ b/caravel/templates/caravel/base.html
@@ -8,10 +8,14 @@
{% block head_js %}
{{super()}}
-
+ {% with filename="css-theme" %}
+ {% include "caravel/partials/_script_tag.html" %}
+ {% endwith %}
{% endblock %}
{% block tail_js %}
- {{super()}}
-
+ {{super()}}
+ {% with filename="common" %}
+ {% include "caravel/partials/_script_tag.html" %}
+ {% endwith %}
{% endblock %}
diff --git a/caravel/templates/caravel/basic.html b/caravel/templates/caravel/basic.html
index f2545ec73a8c..2ca5f5c29a8b 100644
--- a/caravel/templates/caravel/basic.html
+++ b/caravel/templates/caravel/basic.html
@@ -13,7 +13,9 @@
{% endblock %}
{% block head_js %}
-
+ {% with filename="css-theme" %}
+ {% include "caravel/partials/_script_tag.html" %}
+ {% endwith %}
{% endblock %}
diff --git a/caravel/templates/caravel/dashboard.html b/caravel/templates/caravel/dashboard.html
index de59a5637314..f559e672e6af 100644
--- a/caravel/templates/caravel/dashboard.html
+++ b/caravel/templates/caravel/dashboard.html
@@ -2,7 +2,9 @@
{% block head_js %}
{{ super() }}
-
+ {% with filename="dashboard" %}
+ {% include "caravel/partials/_script_tag.html" %}
+ {% endwith %}
{% endblock %}
{% block title %}[dashboard] {{ dashboard.dashboard_title }}{% endblock %}
{% block body %}
diff --git a/caravel/templates/caravel/explore.html b/caravel/templates/caravel/explore.html
index d12b97c15366..93f2c194fb83 100644
--- a/caravel/templates/caravel/explore.html
+++ b/caravel/templates/caravel/explore.html
@@ -106,8 +106,11 @@
{{ _("Filters") }}
+ data-placement="right"
+ title="{{_("Filters are defined using comma delimited strings as in ")}}.
+ {{_("Leave the value field empty to filter empty strings or nulls")}}.
+ {{_("For filters with comma in values, wrap them in single quotes,
+ as in ")}}">
@@ -186,11 +189,11 @@
-
-
+
+
{% include 'caravel/partials/_explore_title.html' %}
-
+
cached
@@ -332,5 +335,7 @@
{{ _("Save a Slice") }}
{% block tail_js %}
{{ super() }}
-
+ {% with filename="explore" %}
+ {% include "caravel/partials/_script_tag.html" %}
+ {% endwith %}
{% endblock %}
diff --git a/caravel/templates/caravel/models/database/macros.html b/caravel/templates/caravel/models/database/macros.html
index c667c31b0b33..a547a6adcb71 100644
--- a/caravel/templates/caravel/models/database/macros.html
+++ b/caravel/templates/caravel/models/database/macros.html
@@ -8,7 +8,11 @@
var data = {};
try{
- data = JSON.stringify({ uri: $("#sqlalchemy_uri").val(), extras: JSON.parse($("#extra").val()) })
+ data = JSON.stringify({
+ uri: $("#sqlalchemy_uri").val(),
+ name: $('#database_name').val(),
+ extras: JSON.parse($("#extra").val())
+ })
} catch(parse_error){
alert("Malformed JSON in the extras field: " + parse_error);
return false
diff --git a/caravel/templates/caravel/partials/_explore_title.html b/caravel/templates/caravel/partials/_explore_title.html
index 9f9767ce47e4..e098f879a828 100644
--- a/caravel/templates/caravel/partials/_explore_title.html
+++ b/caravel/templates/caravel/partials/_explore_title.html
@@ -26,5 +26,5 @@