diff --git a/.gitignore b/.gitignore
index 4c93a874bad0..eb93aa3c20e7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,8 @@ dist
caravel.egg-info/
app.db
*.bak
+.idea
+*.sqllite
# Node.js, webpack artifacts
*.entry.js
diff --git a/caravel/assets/.eslintrc b/caravel/assets/.eslintrc
index 1346231fbe4f..7a517699feb5 100644
--- a/caravel/assets/.eslintrc
+++ b/caravel/assets/.eslintrc
@@ -8,5 +8,6 @@
"prefer-arrow-callback": 0,
"func-names": 0,
"react/jsx-no-bind": 0,
+ "no-confusing-arrow": 0,
}
}
diff --git a/caravel/assets/javascripts/SqlLab/TODO.md b/caravel/assets/javascripts/SqlLab/TODO.md
new file mode 100644
index 000000000000..db8f247abff0
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/TODO.md
@@ -0,0 +1,19 @@
+
+# TODO
+* Figure out how to organize the left panel, integrate Search
+* collapse sql beyond 10 lines
+* Security per-database (dropdown)
+* Get a to work
+
+## Cosmetic
+* Result set font is too big
+* lmiit/timer/buttons wrap
+* table label is transparent
+* SqlEditor buttons
+* use react-bootstrap-prompt for query title input
+* Make tabs look great
+
+# PROJECT
+* Write Runbook
+* Confirm backups
+* merge chef branch
diff --git a/caravel/assets/javascripts/SqlLab/actions.js b/caravel/assets/javascripts/SqlLab/actions.js
new file mode 100644
index 000000000000..69473b37b6bc
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/actions.js
@@ -0,0 +1,116 @@
+export const RESET_STATE = 'RESET_STATE';
+export const ADD_QUERY_EDITOR = 'ADD_QUERY_EDITOR';
+export const REMOVE_QUERY_EDITOR = 'REMOVE_QUERY_EDITOR';
+export const ADD_TABLE = 'ADD_TABLE';
+export const REMOVE_TABLE = 'REMOVE_TABLE';
+export const START_QUERY = 'START_QUERY';
+export const STOP_QUERY = 'STOP_QUERY';
+export const END_QUERY = 'END_QUERY';
+export const REMOVE_QUERY = 'REMOVE_QUERY';
+export const EXPAND_TABLE = 'EXPAND_TABLE';
+export const COLLAPSE_TABLE = 'COLLAPSE_TABLE';
+export const QUERY_SUCCESS = 'QUERY_SUCCESS';
+export const QUERY_FAILED = 'QUERY_FAILED';
+export const QUERY_EDITOR_SETDB = 'QUERY_EDITOR_SETDB';
+export const QUERY_EDITOR_SET_SCHEMA = 'QUERY_EDITOR_SET_SCHEMA';
+export const QUERY_EDITOR_SET_TITLE = 'QUERY_EDITOR_SET_TITLE';
+export const QUERY_EDITOR_SET_AUTORUN = 'QUERY_EDITOR_SET_AUTORUN';
+export const QUERY_EDITOR_SET_SQL = 'QUERY_EDITOR_SET_SQL';
+export const SET_WORKSPACE_DB = 'SET_WORKSPACE_DB';
+export const ADD_WORKSPACE_QUERY = 'ADD_WORKSPACE_QUERY';
+export const REMOVE_WORKSPACE_QUERY = 'REMOVE_WORKSPACE_QUERY';
+export const SET_ACTIVE_QUERY_EDITOR = 'SET_ACTIVE_QUERY_EDITOR';
+export const ADD_ALERT = 'ADD_ALERT';
+export const REMOVE_ALERT = 'REMOVE_ALERT';
+export const REFRESH_QUERIES = 'REFRESH_QUERIES';
+
+export function resetState() {
+ return { type: RESET_STATE };
+}
+
+export function addQueryEditor(queryEditor) {
+ return { type: ADD_QUERY_EDITOR, queryEditor };
+}
+
+export function addAlert(alert) {
+ return { type: ADD_ALERT, alert };
+}
+
+export function removeAlert(alert) {
+ return { type: REMOVE_ALERT, alert };
+}
+
+export function setActiveQueryEditor(queryEditor) {
+ return { type: SET_ACTIVE_QUERY_EDITOR, queryEditor };
+}
+
+export function removeQueryEditor(queryEditor) {
+ return { type: REMOVE_QUERY_EDITOR, queryEditor };
+}
+
+export function removeQuery(query) {
+ return { type: REMOVE_QUERY, query };
+}
+
+export function queryEditorSetDb(queryEditor, dbId) {
+ return { type: QUERY_EDITOR_SETDB, queryEditor, dbId };
+}
+
+export function queryEditorSetSchema(queryEditor, schema) {
+ return { type: QUERY_EDITOR_SET_SCHEMA, queryEditor, schema };
+}
+
+export function queryEditorSetAutorun(queryEditor, autorun) {
+ return { type: QUERY_EDITOR_SET_AUTORUN, queryEditor, autorun };
+}
+
+export function queryEditorSetTitle(queryEditor, title) {
+ return { type: QUERY_EDITOR_SET_TITLE, queryEditor, title };
+}
+
+export function queryEditorSetSql(queryEditor, sql) {
+ return { type: QUERY_EDITOR_SET_SQL, queryEditor, sql };
+}
+
+export function addTable(table) {
+ return { type: ADD_TABLE, table };
+}
+
+export function expandTable(table) {
+ return { type: EXPAND_TABLE, table };
+}
+
+export function collapseTable(table) {
+ return { type: COLLAPSE_TABLE, table };
+}
+
+export function removeTable(table) {
+ return { type: REMOVE_TABLE, table };
+}
+
+export function startQuery(query) {
+ return { type: START_QUERY, query };
+}
+
+export function stopQuery(query) {
+ return { type: STOP_QUERY, query };
+}
+
+export function querySuccess(query, results) {
+ return { type: QUERY_SUCCESS, query, results };
+}
+
+export function queryFailed(query, msg) {
+ return { type: QUERY_FAILED, query, msg };
+}
+
+export function addWorkspaceQuery(query) {
+ return { type: ADD_WORKSPACE_QUERY, query };
+}
+
+export function removeWorkspaceQuery(query) {
+ return { type: REMOVE_WORKSPACE_QUERY, query };
+}
+export function refreshQueries(alteredQueries) {
+ return { type: REFRESH_QUERIES, alteredQueries };
+}
diff --git a/caravel/assets/javascripts/SqlLab/common.js b/caravel/assets/javascripts/SqlLab/common.js
new file mode 100644
index 000000000000..e1c9c96f9de9
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/common.js
@@ -0,0 +1,5 @@
+export const STATE_BSSTYLE_MAP = {
+ failed: 'danger',
+ running: 'warning',
+ success: 'success',
+};
diff --git a/caravel/assets/javascripts/SqlLab/components/Alerts.jsx b/caravel/assets/javascripts/SqlLab/components/Alerts.jsx
new file mode 100644
index 000000000000..0ad6d845ecda
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/Alerts.jsx
@@ -0,0 +1,41 @@
+import React from 'react';
+import { Alert } from 'react-bootstrap';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+class Alerts extends React.Component {
+ removeAlert(alert) {
+ this.props.actions.removeAlert(alert);
+ }
+ render() {
+ const alerts = this.props.alerts.map((alert) =>
+
+ {alert.msg}
+
+
+ );
+ return (
+
{alerts}
+ );
+ }
+}
+
+Alerts.propTypes = {
+ alerts: React.PropTypes.array,
+ actions: React.PropTypes.object,
+};
+
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+export default connect(null, mapDispatchToProps)(Alerts);
diff --git a/caravel/assets/javascripts/SqlLab/components/ButtonWithTooltip.jsx b/caravel/assets/javascripts/SqlLab/components/ButtonWithTooltip.jsx
new file mode 100644
index 000000000000..622c721f6d3b
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/ButtonWithTooltip.jsx
@@ -0,0 +1,46 @@
+import React from 'react';
+import { Button, OverlayTrigger, Tooltip } from 'react-bootstrap';
+
+const ButtonWithTooltip = (props) => {
+ let tooltip = (
+
+ {props.tooltip}
+
+ );
+ return (
+
+
+ {props.children}
+
+
+ );
+};
+
+ButtonWithTooltip.defaultProps = {
+ onClick: () => {},
+ disabled: false,
+ placement: 'top',
+ bsStyle: 'default',
+};
+
+ButtonWithTooltip.propTypes = {
+ bsStyle: React.PropTypes.string,
+ children: React.PropTypes.element,
+ className: React.PropTypes.string,
+ disabled: React.PropTypes.bool,
+ onClick: React.PropTypes.func,
+ placement: React.PropTypes.string,
+ tooltip: React.PropTypes.string,
+};
+
+export default ButtonWithTooltip;
diff --git a/caravel/assets/javascripts/SqlLab/components/LeftPane.jsx b/caravel/assets/javascripts/SqlLab/components/LeftPane.jsx
new file mode 100644
index 000000000000..b65faddcbc59
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/LeftPane.jsx
@@ -0,0 +1,65 @@
+import React from 'react';
+import { Alert, Button, Label } from 'react-bootstrap';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import QueryLink from './QueryLink';
+
+import 'react-select/dist/react-select.css';
+
+const LeftPane = (props) => {
+ let queryElements;
+ if (props.workspaceQueries.length > 0) {
+ queryElements = props.workspaceQueries.map((q) => );
+ } else {
+ queryElements = (
+
+ Use the save button on the SQL editor to save a query
+ into this section for future reference.
+
+ );
+ }
+ return (
+
+
+
+
+ Saved Queries
+
+ ALPHA
+
+
+
+
+ {queryElements}
+
+
+
+
+ Reset State
+
+
+ );
+};
+
+LeftPane.propTypes = {
+ workspaceQueries: React.PropTypes.array,
+ actions: React.PropTypes.object,
+};
+
+LeftPane.defaultProps = {
+ workspaceQueries: [],
+};
+
+function mapStateToProps(state) {
+ return {
+ workspaceQueries: state.workspaceQueries,
+ };
+}
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(LeftPane);
diff --git a/caravel/assets/javascripts/SqlLab/components/Link.jsx b/caravel/assets/javascripts/SqlLab/components/Link.jsx
new file mode 100644
index 000000000000..4fff9db7e014
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/Link.jsx
@@ -0,0 +1,54 @@
+import React from 'react';
+import { OverlayTrigger, Tooltip } from 'react-bootstrap';
+
+
+class Link extends React.Component {
+ render() {
+ let tooltip = (
+
+ {this.props.tooltip}
+
+ );
+ const link = (
+
+ {this.props.children}
+
+ );
+ if (this.props.tooltip) {
+ return (
+
+ {link}
+
+ );
+ }
+ return link;
+ }
+}
+Link.propTypes = {
+ children: React.PropTypes.object,
+ className: React.PropTypes.string,
+ href: React.PropTypes.string,
+ onClick: React.PropTypes.func,
+ placement: React.PropTypes.string,
+ style: React.PropTypes.object,
+ tooltip: React.PropTypes.string,
+};
+Link.defaultProps = {
+ disabled: false,
+ href: '#',
+ tooltip: null,
+ placement: 'top',
+ onClick: () => {},
+};
+
+export default Link;
diff --git a/caravel/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx b/caravel/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx
new file mode 100644
index 000000000000..f992769d9592
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx
@@ -0,0 +1,58 @@
+import React from 'react';
+import moment from 'moment';
+import { bindActionCreators } from 'redux';
+import { connect } from 'react-redux';
+import * as Actions from '../actions';
+
+$ = require('jquery');
+
+
+class QueryAutoRefresh extends React.Component {
+ componentWillMount() {
+ this.startTimer();
+ }
+ componentWillUnmount() {
+ this.stopTimer();
+ }
+ startTimer() {
+ if (!(this.timer)) {
+ this.timer = setInterval(this.stopwatch.bind(this), 5000);
+ }
+ }
+ stopTimer() {
+ clearInterval(this.timer);
+ this.timer = null;
+ }
+ stopwatch() {
+ const url = '/caravel/queries/0';
+ // No updates in case of failure.
+ $.getJSON(
+ url,
+ (data, status, xhr) => {
+ if (status === "success") {
+ this.props.actions.refreshQueries(data);
+ }
+ });
+ }
+ render() {
+ return null;
+ }
+}
+QueryAutoRefresh.propTypes = {
+ // queries: React.PropTypes.object,
+};
+QueryAutoRefresh.defaultProps = {
+ // queries: null,
+};
+
+function mapStateToProps() {
+ return {};
+}
+
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryAutoRefresh);
diff --git a/caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx b/caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx
new file mode 100644
index 000000000000..af6cb0da0863
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx
@@ -0,0 +1,55 @@
+import React from 'react';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import QueryTable from './QueryTable';
+import { Alert } from 'react-bootstrap';
+
+const QueryHistory = (props) => {
+ const activeQeId = props.tabHistory[props.tabHistory.length - 1];
+ const queriesArray = []
+ for (var query_id in props.queries) {
+ if (props.queries[query_id].sqlEditorId === activeQeId) {
+ queriesArray.push(props.queries[query_id])
+ }
+ }
+ if (queriesArray.length > 0) {
+ return (
+
+ );
+ }
+ return (
+
+ No query history yet...
+
+ );
+};
+
+QueryHistory.defaultProps = {
+ queries: {},
+};
+
+QueryHistory.propTypes = {
+ queries: React.PropTypes.object,
+ tabHistory: React.PropTypes.array,
+ actions: React.PropTypes.object,
+};
+
+function mapStateToProps(state) {
+ return {
+ queries: state.queries,
+ tabHistory: state.tabHistory,
+ };
+}
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryHistory);
diff --git a/caravel/assets/javascripts/SqlLab/components/QueryLink.jsx b/caravel/assets/javascripts/SqlLab/components/QueryLink.jsx
new file mode 100644
index 000000000000..61123ca87578
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/QueryLink.jsx
@@ -0,0 +1,66 @@
+import React from 'react';
+import { ButtonGroup } from 'react-bootstrap';
+import Link from './Link';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class QueryLink extends React.Component {
+ popTab() {
+ const qe = {
+ id: shortid.generate(),
+ title: this.props.query.title,
+ dbId: this.props.query.dbId,
+ autorun: false,
+ sql: this.props.query.sql,
+ };
+ this.props.actions.addQueryEditor(qe);
+ }
+ render() {
+ return (
+
+ );
+ }
+}
+
+QueryLink.propTypes = {
+ query: React.PropTypes.object,
+ actions: React.PropTypes.object,
+};
+
+QueryLink.defaultProps = {
+};
+
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+export default connect(null, mapDispatchToProps)(QueryLink);
+
diff --git a/caravel/assets/javascripts/SqlLab/components/QueryLog.jsx b/caravel/assets/javascripts/SqlLab/components/QueryLog.jsx
new file mode 100644
index 000000000000..bc83a4627c6b
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/QueryLog.jsx
@@ -0,0 +1,51 @@
+import React from 'react';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import QueryTable from './QueryTable';
+import { Alert } from 'react-bootstrap';
+
+class QueryLog extends React.Component {
+ render() {
+ const activeQeId = this.props.tabHistory[this.props.tabHistory.length - 1];
+ const queries = this.props.queries.filter((q) => (q.sqlEditorId === activeQeId));
+ if (queries.length > 0) {
+ return (
+
+ );
+ }
+ return (
+
+ No query history yet...
+
+ );
+ }
+}
+QueryLog.defaultProps = {
+ queries: [],
+};
+
+QueryLog.propTypes = {
+ queries: React.PropTypes.array,
+ tabHistory: React.PropTypes.array,
+ actions: React.PropTypes.object,
+};
+
+function mapStateToProps(state) {
+ return {
+ queries: state.queries,
+ tabHistory: state.tabHistory,
+ };
+}
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryLog);
diff --git a/caravel/assets/javascripts/SqlLab/components/QuerySearch.jsx b/caravel/assets/javascripts/SqlLab/components/QuerySearch.jsx
new file mode 100644
index 000000000000..e0352f9363cb
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/QuerySearch.jsx
@@ -0,0 +1,74 @@
+import React from 'react';
+import SplitPane from 'react-split-pane';
+import Select from 'react-select';
+import { Button } from 'react-bootstrap';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import QueryTable from './QueryTable';
+
+class QuerySearch extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ queryText: '',
+ };
+ }
+ changeQueryText(value) {
+ this.setState({ queryText: value });
+ }
+ render() {
+ const queries = this.props.queries;
+ return (
+
+
+
+
+
+ Search Queries
+
+
+
+
+
+
+
+
+
+
+
+ Search!
+
+ );
+ }
+}
+QuerySearch.propTypes = {
+ queries: React.PropTypes.array,
+};
+QuerySearch.defaultProps = {
+ queries: [],
+};
+
+function mapStateToProps(state) {
+ return {
+ queries: state.queries,
+ };
+}
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QuerySearch);
diff --git a/caravel/assets/javascripts/SqlLab/components/QueryTable.jsx b/caravel/assets/javascripts/SqlLab/components/QueryTable.jsx
new file mode 100644
index 000000000000..841e50d285f1
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/QueryTable.jsx
@@ -0,0 +1,121 @@
+import React from 'react';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import moment from 'moment';
+import { Table } from 'reactable';
+
+import SyntaxHighlighter from 'react-syntax-highlighter';
+import { github } from 'react-syntax-highlighter/dist/styles';
+
+import Link from './Link';
+import VisualizeModal from './VisualizeModal';
+import { STATE_BSSTYLE_MAP } from '../common.js';
+
+
+class QueryTable extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ showVisualizeModal: false,
+ activeQuery: null,
+ };
+ }
+ hideVisualizeModal() {
+ this.setState({ showVisualizeModal: false });
+ }
+ showVisualizeModal(query) {
+ this.setState({ showVisualizeModal: true });
+ this.setState({ activeQuery: query });
+ }
+ notImplemented() {
+ alert('Not implemented yet!');
+ }
+ render() {
+ const data = this.props.queries.map((query) => {
+ const q = Object.assign({}, query);
+ // TODO(bkyryliuk): rename ...Dttm into the ...Timestamp.
+ const since = (q.endDttm) ? q.endDttm : new Date().getTime();
+ const duration = moment.utc(since - q.startDttm);
+ if (q.endDttm) {
+ q.duration = duration.format('HH:mm:ss.SS');
+ }
+ q.started = moment.utc(q.startDttm).format('HH:mm:ss');
+ q.sql = {q.sql} ;
+ q.state = (
+
+ {q.state}
+
+ );
+ q.actions = (
+
+
+
+
+
+
+ );
+
+ return q;
+ }).reverse();
+ return (
+
+ );
+ }
+}
+QueryTable.propTypes = {
+ columns: React.PropTypes.array,
+ actions: React.PropTypes.object,
+ queries: React.PropTypes.array,
+};
+QueryTable.defaultProps = {
+ columns: ['state', 'started', 'duration', 'progress', 'rows', 'sql', 'actions'],
+ queries: [],
+};
+
+function mapStateToProps() {
+ return {};
+}
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+export default connect(mapStateToProps, mapDispatchToProps)(QueryTable);
diff --git a/caravel/assets/javascripts/SqlLab/components/ResultSet.jsx b/caravel/assets/javascripts/SqlLab/components/ResultSet.jsx
new file mode 100644
index 000000000000..25ed558e6ce2
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/ResultSet.jsx
@@ -0,0 +1,94 @@
+import React from 'react';
+import { Alert, Button } from 'react-bootstrap';
+import { Table } from 'reactable';
+
+import VisualizeModal from './VisualizeModal';
+
+
+class ResultSet extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ searchText: '',
+ showModal: false,
+ };
+ }
+ shouldComponentUpdate(nextProps, nextState) {
+ return (
+ this.state.searchText !== nextState.searchText ||
+ this.state.showModal !== nextState.showModal
+ );
+ }
+ changeSearch(event) {
+ this.setState({ searchText: event.target.value });
+ }
+ showModal() {
+ this.setState({ showModal: true });
+ }
+ hideModal() {
+ this.setState({ showModal: false });
+ }
+ render() {
+ const results = this.props.query.results;
+ let controls =
;
+ if (this.props.showControls) {
+ controls = (
+
+
+
+
+ Visualize
+
+ .CSV
+
+
+
+
+
+
+ );
+ }
+ if (results && results.data.length > 0) {
+ return (
+
+ );
+ }
+ return (The query returned no data );
+ }
+}
+ResultSet.propTypes = {
+ query: React.PropTypes.object,
+ showControls: React.PropTypes.boolean,
+ search: React.PropTypes.boolean,
+ searchText: React.PropTypes.string,
+};
+ResultSet.defaultProps = {
+ showControls: true,
+ search: true,
+ searchText: '',
+};
+
+export default ResultSet;
diff --git a/caravel/assets/javascripts/SqlLab/components/SouthPane.jsx b/caravel/assets/javascripts/SqlLab/components/SouthPane.jsx
new file mode 100644
index 000000000000..0da7913b167b
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/SouthPane.jsx
@@ -0,0 +1,42 @@
+import { Tab, Tabs } from 'react-bootstrap';
+import QueryHistory from './QueryHistory';
+import ResultSet from './ResultSet';
+import React from 'react';
+
+const SouthPane = (props) => {
+ let results;
+ if (props.latestQuery) {
+ if (props.latestQuery.state === 'running') {
+ results = (
+
+ );
+ } else if (props.latestQuery.state === 'failed') {
+ results = {props.latestQuery.msg}
;
+ } else if (props.latestQuery.state === 'success') {
+ results = ;
+ }
+ } else {
+ results = Run a query to display results here
;
+ }
+ return (
+
+
+
+ {results}
+
+
+
+
+
+
+ );
+};
+
+SouthPane.propTypes = {
+ latestQuery: React.PropTypes.object,
+};
+
+SouthPane.defaultProps = {
+};
+
+export default SouthPane;
diff --git a/caravel/assets/javascripts/SqlLab/components/SqlEditor.jsx b/caravel/assets/javascripts/SqlLab/components/SqlEditor.jsx
new file mode 100644
index 000000000000..054f1b0e79e0
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/SqlEditor.jsx
@@ -0,0 +1,267 @@
+const $ = window.$ = require('jquery');
+import React from 'react';
+import {
+ Button,
+ ButtonGroup,
+ FormGroup,
+ InputGroup,
+ Form,
+ FormControl,
+ DropdownButton,
+ Label,
+ MenuItem,
+ OverlayTrigger,
+ Tooltip,
+} from 'react-bootstrap';
+
+import AceEditor from 'react-ace';
+import 'brace/mode/sql';
+import 'brace/theme/github';
+import 'brace/ext/language_tools';
+
+import { bindActionCreators } from 'redux';
+import { connect } from 'react-redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+import ButtonWithTooltip from './ButtonWithTooltip';
+import SouthPane from './SouthPane';
+import Timer from './Timer';
+
+import SqlEditorTopToolbar from './SqlEditorTopToolbar';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class SqlEditor extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ autorun: props.queryEditor.autorun,
+ sql: props.queryEditor.sql,
+ ctas: '',
+ };
+ }
+ componentDidMount() {
+ this.onMount();
+ }
+ onMount() {
+ if (this.state.autorun) {
+ this.setState({ autorun: false });
+ this.props.actions.queryEditorSetAutorun(this.props.queryEditor, false);
+ this.startQuery();
+ }
+ }
+ runQuery() {
+ this.startQuery();
+ }
+ startQuery(runAsync = false, ctas = false) {
+ const that = this;
+
+ const query = {
+ id: shortid.generate(),
+ sqlEditorId: this.props.queryEditor.id,
+ sql: this.props.queryEditor.sql,
+ state: 'running',
+ tab: this.props.queryEditor.title,
+ dbId: this.props.queryEditor.dbId,
+ startDttm: new Date().getTime(),
+ };
+
+ // Execute the Query
+ that.props.actions.startQuery(query);
+
+ const sqlJsonUrl = '/caravel/sql_json/';
+ const sqlJsonRequest = {
+ json: true,
+ client_id: query.id,
+ async: runAsync,
+ sql: this.props.queryEditor.sql,
+ database_id: this.props.queryEditor.dbId,
+ sql_editor_id: this.props.queryEditor.id,
+ schema: this.props.queryEditor.schema,
+ tab: this.props.queryEditor.title,
+ json: true,
+ select_as_cta: ctas,
+ tmp_table_name: this.state.ctas,
+ };
+ $.ajax({
+ type: 'POST',
+ dataType: 'json',
+ url: sqlJsonUrl,
+ data: sqlJsonRequest,
+ success(results) {
+ if (runAsync) {
+ // TODO nothing?
+ } else {
+ that.props.actions.querySuccess(query, results);
+ }
+ },
+ error(err) {
+ let msg = '';
+ try {
+ msg = err.responseJSON.error;
+ } catch (e) {
+ msg = (err.responseText) ? err.responseText : e;
+ }
+ that.props.actions.queryFailed(query, msg);
+ },
+ });
+ }
+ stopQuery() {
+ this.props.actions.stopQuery(this.props.latestQuery);
+ }
+ createTableAs() {
+ this.startQuery(true, true);
+ }
+ textChange(text) {
+ this.setState({ sql: text });
+ this.props.actions.queryEditorSetSql(this.props.queryEditor, text);
+ }
+ addWorkspaceQuery() {
+ this.props.actions.addWorkspaceQuery({
+ id: shortid.generate(),
+ sql: this.state.sql,
+ dbId: this.props.queryEditor.dbId,
+ schema: this.props.queryEditor.schema,
+ title: this.props.queryEditor.title,
+ });
+ }
+ ctasChange() {}
+ visualize() {}
+ ctasChanged(event) {
+ this.setState({ ctas: event.target.value });
+ }
+ render() {
+ let runButtons = (
+
+
+ Run Query
+
+
+ );
+ if (this.props.latestQuery && this.props.latestQuery.state === 'running') {
+ runButtons = (
+
+
+ Stop
+
+
+ );
+ }
+ const rightButtons = (
+
+
+
+
+ }>
+
+ export to .csv
+
+
+ export to .json
+
+
+
+ );
+ let limitWarning = null;
+ const rowLimit = 1000;
+ if (this.props.latestQuery && this.props.latestQuery.rows === rowLimit) {
+ const tooltip = (
+
+ It appears that the number of rows in the query results displayed
+ was limited on the server side to the {rowLimit} limit.
+
+ );
+ limitWarning = (
+
+ LIMIT
+
+ );
+ }
+ const editorBottomBar = (
+
+
+
+
+
+ {limitWarning}
+
+ {rightButtons}
+
+
+ );
+ return (
+
+
+
+
+
+ {editorBottomBar}
+
+
+
+
+
+ );
+ }
+}
+
+SqlEditor.propTypes = {
+ queryEditor: React.PropTypes.object,
+ actions: React.PropTypes.object,
+ latestQuery: React.PropTypes.object,
+};
+
+SqlEditor.defaultProps = {
+};
+
+function mapStateToProps() {
+ return {};
+}
+
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(SqlEditor);
diff --git a/caravel/assets/javascripts/SqlLab/components/SqlEditorTopToolbar.jsx b/caravel/assets/javascripts/SqlLab/components/SqlEditorTopToolbar.jsx
new file mode 100644
index 000000000000..11dd7a17667c
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/SqlEditorTopToolbar.jsx
@@ -0,0 +1,271 @@
+const $ = window.$ = require('jquery');
+import React from 'react';
+import { Label, OverlayTrigger, Popover } from 'react-bootstrap';
+
+import { bindActionCreators } from 'redux';
+import { connect } from 'react-redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+import Select from 'react-select';
+import Link from './Link';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class SqlEditorTopToolbar extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ databaseLoading: false,
+ databaseOptions: [],
+ schemaLoading: false,
+ schemaOptions: [],
+ tableLoading: false,
+ tableOptions: [],
+ };
+ }
+ componentWillMount() {
+ this.fetchDatabaseOptions();
+ this.fetchSchemas();
+ this.fetchTables();
+ }
+ getSql(table) {
+ let cols = '';
+ table.columns.forEach(function (col, i) {
+ cols += col.name;
+ if (i < table.columns.length - 1) {
+ cols += ', ';
+ }
+ });
+ return `SELECT ${cols}\nFROM ${table.name}`;
+ }
+ selectStar(table) {
+ this.props.actions.queryEditorSetSql(this.props.queryEditor, this.getSql(table));
+ }
+ popTab(table) {
+ const qe = {
+ id: shortid.generate(),
+ title: table.name,
+ dbId: table.dbId,
+ schema: table.schema,
+ autorun: true,
+ sql: this.getSql(table),
+ };
+ this.props.actions.addQueryEditor(qe);
+ }
+ fetchTables(dbId, schema) {
+ const actualDbId = dbId || this.props.queryEditor.dbId;
+ if (actualDbId) {
+ const actualSchema = schema || this.props.queryEditor.schema;
+ this.setState({ tableLoading: true });
+ this.setState({ tableOptions: [] });
+ const url = `/caravel/tables/${actualDbId}/${actualSchema}`;
+ $.get(url, (data) => {
+ let tableOptions = data.tables.map((s) => ({ value: s, label: s }));
+ const views = data.views.map((s) => ({ value: s, label: '[view] ' + s }));
+ tableOptions = [...tableOptions, ...views];
+ this.setState({ tableOptions });
+ this.setState({ tableLoading: false });
+ });
+ }
+ }
+ changeSchema(schemaOpt) {
+ const schema = (schemaOpt) ? schemaOpt.value : null;
+ this.props.actions.queryEditorSetSchema(this.props.queryEditor, schema);
+ this.fetchTables(this.props.queryEditor.dbId, schema);
+ }
+ fetchSchemas(dbId) {
+ const actualDbId = dbId || this.props.queryEditor.dbId;
+ if (actualDbId) {
+ this.setState({ schemaLoading: true });
+ const url = `/databasetablesasync/api/read?_flt_0_id=${actualDbId}`;
+ $.get(url, (data) => {
+ const schemas = data.result[0].all_schema_names;
+ const schemaOptions = schemas.map((s) => ({ value: s, label: s }));
+ this.setState({ schemaOptions });
+ this.setState({ schemaLoading: false });
+ });
+ }
+ }
+ changeDb(db) {
+ const val = (db) ? db.value : null;
+ this.setState({ schemaOptions: [] });
+ this.props.actions.queryEditorSetDb(this.props.queryEditor, val);
+ if (!(db)) {
+ this.setState({ tableOptions: [] });
+ return;
+ }
+ this.fetchTables(val, this.props.queryEditor.schema);
+ this.fetchSchemas(val);
+ }
+ fetchDatabaseOptions() {
+ this.setState({ databaseLoading: true });
+ const url = '/databaseasync/api/read';
+ $.get(url, (data) => {
+ const options = data.result.map((db) => ({ value: db.id, label: db.database_name }));
+ this.setState({ databaseOptions: options });
+ this.setState({ databaseLoading: false });
+ });
+ }
+ closePopover(ref) {
+ this.refs[ref].hide();
+ }
+ changeTable(tableOpt) {
+ const tableName = tableOpt.value;
+ const qe = this.props.queryEditor;
+ const url = `/caravel/table/${qe.dbId}/${tableName}/${qe.schema}/`;
+ $.get(url, (data) => {
+ this.props.actions.addTable({
+ id: shortid.generate(),
+ dbId: this.props.queryEditor.dbId,
+ queryEditorId: this.props.queryEditor.id,
+ name: data.name,
+ schema: qe.schema,
+ columns: data.columns,
+ expanded: true,
+ showPopup: true,
+ });
+ })
+ .fail(() => {
+ this.props.actions.addAlert({
+ msg: 'Error occurred while fetching metadata',
+ bsStyle: 'danger',
+ });
+ });
+ }
+ render() {
+ const tables = this.props.tables.filter((t) => (t.queryEditorId === this.props.queryEditor.id));
+ const tablesEls = tables.map((table) => {
+ let cols = [];
+ if (table.columns) {
+ cols = table.columns.map((col) => (
+
+
{col.name}
+
{col.type}
+
+ ));
+ }
+ const popoverId = 'tblPopover_' + table.name;
+ const popoverTop = (
+
+ );
+ const popover = (
+
+ {cols}
+
+ );
+ return (
+
+
+
+ {table.name}
+
+
+
+
+ );
+ });
+ return (
+
+
+
+
+
+
+
+
+
+
+
+ {tablesEls}
+
+
+ );
+ }
+}
+
+SqlEditorTopToolbar.propTypes = {
+ queryEditor: React.PropTypes.object,
+ tables: React.PropTypes.array,
+ actions: React.PropTypes.object,
+};
+
+SqlEditorTopToolbar.defaultProps = {
+ tables: [],
+};
+
+function mapStateToProps(state) {
+ return {
+ tables: state.tables,
+ };
+}
+
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(SqlEditorTopToolbar);
diff --git a/caravel/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx b/caravel/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx
new file mode 100644
index 000000000000..74d9ff8dde52
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx
@@ -0,0 +1,119 @@
+import React from 'react';
+import { DropdownButton, MenuItem, Panel, Tab, Tabs } from 'react-bootstrap';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import SqlEditor from './SqlEditor';
+import shortid from 'shortid';
+
+let queryCount = 1;
+
+class QueryEditors extends React.Component {
+ renameTab(qe) {
+ const newTitle = prompt('Enter a new title for the tab');
+ if (newTitle) {
+ this.props.actions.queryEditorSetTitle(qe, newTitle);
+ }
+ }
+ activeQueryEditor() {
+ const qeid = this.props.tabHistory[this.props.tabHistory.length - 1];
+ for (let i = 0; i < this.props.queryEditors.length; i++) {
+ const qe = this.props.queryEditors[i];
+ if (qe.id === qeid) {
+ return qe;
+ }
+ }
+ }
+ newQueryEditor() {
+ queryCount++;
+ const activeQueryEditor = this.activeQueryEditor();
+ const qe = {
+ id: shortid.generate(),
+ title: `Untitled Query ${queryCount}`,
+ dbId: (activeQueryEditor) ? activeQueryEditor.dbId : null,
+ schema: (activeQueryEditor) ? activeQueryEditor.schema : null,
+ autorun: false,
+ sql: 'SELECT ...',
+ };
+ this.props.actions.addQueryEditor(qe);
+ }
+ handleSelect(key) {
+ if (key === 'add_tab') {
+ this.newQueryEditor();
+ } else {
+ this.props.actions.setActiveQueryEditor({ id: key });
+ }
+ }
+ render() {
+ const editors = this.props.queryEditors.map((qe, i) => {
+ let latestQuery = this.props.queries[qe.latestQueryId]
+ const state = (latestQuery) ? latestQuery.state : '';
+ const tabTitle = (
+
+
{qe.title} {' '}
+
+
+ close tab
+
+
+ rename tab
+
+
+
+ );
+ return (
+
+
+ );
+ });
+ return (
+
+
+
+ {editors}
+
} eventKey="add_tab" />
+
+
+
+ );
+ }
+}
+QueryEditors.propTypes = {
+ actions: React.PropTypes.object,
+ queries: React.PropTypes.object,
+ queryEditors: React.PropTypes.array,
+ tabHistory: React.PropTypes.array,
+};
+QueryEditors.defaultProps = {
+ tabHistory: [],
+ queryEditors: [],
+};
+
+function mapStateToProps(state) {
+ return {
+ queryEditors: state.queryEditors,
+ queries: state.queries,
+ tabHistory: state.tabHistory,
+ };
+}
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryEditors);
diff --git a/caravel/assets/javascripts/SqlLab/components/TableMetadata.jsx b/caravel/assets/javascripts/SqlLab/components/TableMetadata.jsx
new file mode 100644
index 000000000000..75bbc6459fc4
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/TableMetadata.jsx
@@ -0,0 +1,23 @@
+import React from 'react';
+import { BootstrapTable, TableHeaderColumn } from 'react-bootstrap-table';
+
+const TableMetadata = function (props) {
+ return (
+
+
+ id
+
+ Name
+ Type
+
+ );
+};
+
+TableMetadata.propTypes = {
+ table: React.PropTypes.object,
+};
+
+export default TableMetadata;
diff --git a/caravel/assets/javascripts/SqlLab/components/TableWorkspaceElement.jsx b/caravel/assets/javascripts/SqlLab/components/TableWorkspaceElement.jsx
new file mode 100644
index 000000000000..d43229ad8ccf
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/TableWorkspaceElement.jsx
@@ -0,0 +1,102 @@
+import React from 'react';
+import { ButtonGroup } from 'react-bootstrap';
+import Link from './Link';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class TableWorkspaceElement extends React.Component {
+ selectStar() {
+ let cols = '';
+ this.props.table.columns.forEach((col, i) => {
+ cols += col.name;
+ if (i < this.props.table.columns.length - 1) {
+ cols += ', ';
+ }
+ });
+ const sql = `SELECT ${cols}\nFROM ${this.props.table.name}`;
+ const qe = {
+ id: shortid.generate(),
+ title: this.props.table.name,
+ dbId: this.props.table.dbId,
+ autorun: true,
+ sql,
+ };
+ this.props.actions.addQueryEditor(qe);
+ }
+ render() {
+ let metadata = null;
+ let buttonToggle;
+ if (!this.props.table.expanded) {
+ buttonToggle = (
+
+ {this.props.table.name}
+
+ );
+ metadata = this.props.table.columns.map((col) =>
+
+ {col.name}
+ {col.type}
+
+ );
+ metadata = (
+ {metadata}
+ );
+ } else {
+ buttonToggle = (
+
+ {this.props.table.name}
+
+ );
+ }
+ return (
+
+ {buttonToggle}
+
+
+
+
+ {metadata}
+
+ );
+ }
+}
+TableWorkspaceElement.propTypes = {
+ table: React.PropTypes.object,
+ actions: React.PropTypes.object,
+};
+TableWorkspaceElement.defaultProps = {
+ table: null,
+};
+
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+export default connect(null, mapDispatchToProps)(TableWorkspaceElement);
+
diff --git a/caravel/assets/javascripts/SqlLab/components/Timer.jsx b/caravel/assets/javascripts/SqlLab/components/Timer.jsx
new file mode 100644
index 000000000000..900195e47365
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/Timer.jsx
@@ -0,0 +1,62 @@
+import React from 'react';
+import moment from 'moment';
+
+import { STATE_BSSTYLE_MAP } from '../common.js';
+
+class Timer extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ clockStr: '',
+ };
+ }
+ componentWillMount() {
+ this.startTimer();
+ }
+ componentWillUnmount() {
+ this.stopTimer();
+ }
+ startTimer() {
+ if (!(this.timer)) {
+ this.timer = setInterval(this.stopwatch.bind(this), 30);
+ }
+ }
+ stopTimer() {
+ clearInterval(this.timer);
+ this.timer = null;
+ }
+ stopwatch() {
+ if (this.props && this.props.query) {
+ let fromDttm = (this.props.query.endDttm) ? this.props.query.endDttm : new Date().getTime();
+ const since = (this.props.query.endDttm) ? this.props.query.endDttm : new Date().getTime();
+ const duration = moment.utc(since - this.props.query.startDttm);
+ const clockStr = duration.format('HH:mm:ss.SS');
+ this.setState({ clockStr });
+ if (this.props.query.state !== 'running') {
+ this.stopTimer();
+ }
+ }
+ }
+ render() {
+ if (this.props.query && this.props.query.state === 'running') {
+ this.startTimer();
+ }
+ let timerSpan = null;
+ if (this.props && this.props.query) {
+ timerSpan = (
+
+ {this.state.clockStr}
+
+ );
+ }
+ return timerSpan;
+ }
+}
+Timer.propTypes = {
+ query: React.PropTypes.object,
+};
+Timer.defaultProps = {
+ query: null,
+};
+
+export default Timer;
diff --git a/caravel/assets/javascripts/SqlLab/components/VisualizeModal.jsx b/caravel/assets/javascripts/SqlLab/components/VisualizeModal.jsx
new file mode 100644
index 000000000000..f0e1aa9f6a15
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/components/VisualizeModal.jsx
@@ -0,0 +1,172 @@
+import React from 'react';
+import { Button, Col, Modal } from 'react-bootstrap';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import Select from 'react-select';
+import { Table } from 'reactable';
+import shortid from 'shortid';
+
+const $ = require('jquery');
+
+class VisualizeModal extends React.Component {
+ constructor(props) {
+ super(props);
+ this.state = {
+ chartType: 'line',
+ datasourceName: shortid.generate(),
+ columns: {},
+ };
+ }
+ changeChartType(option) {
+ this.setState({ chartType: (option) ? option.value : null });
+ }
+ mergedColumns() {
+ const columns = Object.assign({}, this.state.columns);
+ if (this.props.query && this.props.query.results.columns) {
+ this.props.query.results.columns.forEach((col) => {
+ if (columns[col] === undefined) {
+ columns[col] = {};
+ }
+ });
+ }
+ return columns;
+ }
+ visualize() {
+ const vizOptions = {
+ chartType: this.state.chartType,
+ datasourceName: this.state.datasourceName,
+ columns: this.state.columns,
+ sql: this.props.query.sql,
+ dbId: this.props.query.dbId,
+ };
+ window.open('/caravel/sqllab_viz/?data=' + JSON.stringify(vizOptions));
+ }
+ changeDatasourceName(event) {
+ this.setState({ datasourceName: event.target.value });
+ }
+ changeCheckbox(attr, col, event) {
+ let columns = this.mergedColumns();
+ const column = Object.assign({}, columns[col], { [attr]: event.target.checked });
+ columns = Object.assign({}, columns, { [col]: column });
+ this.setState({ columns });
+ }
+ changeAggFunction(col, option) {
+ let columns = this.mergedColumns();
+ const val = (option) ? option.value : null;
+ const column = Object.assign({}, columns[col], { agg: val });
+ columns = Object.assign({}, columns, { [col]: column });
+ this.setState({ columns });
+ }
+ render() {
+ if (!(this.props.query)) {
+ return
;
+ }
+ const tableData = this.props.query.results.columns.map((col) => ({
+ column: col,
+ is_dimension: (
+
+ ),
+ is_date: (
+
+ ),
+ agg_func: (
+
+ ),
+ }));
+ const modal = (
+
+
+
+
+ Visualize under construction
+
+
+
+
+
+ Chart Type
+
+
+
+ Datasource Name
+
+
+
+
+
+
+ Visualize
+
+
+
+
+ );
+ return modal;
+ }
+}
+VisualizeModal.propTypes = {
+ query: React.PropTypes.object,
+ show: React.PropTypes.boolean,
+ onHide: React.PropTypes.function,
+};
+VisualizeModal.defaultProps = {
+ show: false,
+};
+
+function mapStateToProps() {
+ return {};
+}
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+export default connect(mapStateToProps, mapDispatchToProps)(VisualizeModal);
diff --git a/caravel/assets/javascripts/SqlLab/index.jsx b/caravel/assets/javascripts/SqlLab/index.jsx
new file mode 100644
index 000000000000..58c025177bbc
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/index.jsx
@@ -0,0 +1,74 @@
+const $ = window.$ = require('jquery');
+const jQuery = window.jQuery = $;
+require('bootstrap');
+
+import React from 'react';
+import { render } from 'react-dom';
+import * as Actions from './actions';
+
+import { Label, Tab, Tabs } from 'react-bootstrap';
+import LeftPane from './components/LeftPane';
+import TabbedSqlEditors from './components/TabbedSqlEditors';
+import QueryAutoRefresh from './components/QueryAutoRefresh';
+import Alerts from './components/Alerts';
+
+import { bindActionCreators, compose, createStore } from 'redux';
+import { connect, Provider } from 'react-redux';
+
+import { initialState, sqlLabReducer } from './reducers';
+import persistState from 'redux-localstorage';
+
+
+require('./main.css');
+
+let store = createStore(sqlLabReducer, initialState, compose(persistState(), window.devToolsExtension && window.devToolsExtension()));
+
+// jquery hack to highlight the navbar menu
+$('a[href="/caravel/sqllab"]').parent().addClass('active');
+
+class App extends React.Component {
+ render() {
+ return (
+
+ );
+ }
+}
+
+App.propTypes = {
+ alerts: React.PropTypes.array,
+};
+
+
+function mapStateToProps(state) {
+ return {
+ alerts: state.alerts,
+ };
+}
+function mapDispatchToProps(dispatch) {
+ return {
+ actions: bindActionCreators(Actions, dispatch),
+ };
+}
+
+const ReduxedApp = connect(mapStateToProps, mapDispatchToProps)(App);
+
+render(
+
+
+ ,
+ document.getElementById('app')
+);
+
diff --git a/caravel/assets/javascripts/SqlLab/main.css b/caravel/assets/javascripts/SqlLab/main.css
new file mode 100644
index 000000000000..dd5f9b646fb6
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/main.css
@@ -0,0 +1,223 @@
+#app {
+ position: absolute;
+ top: 65;
+ left: 0;
+ right: 0;
+ bottom: 0;
+}
+.inlineBlock {
+ display: inline-block;
+}
+.valignTop {
+ vertical-align: top;
+}
+.inline {
+ display: inline;
+}
+.nopadding {
+ padding: 0px;
+}
+.panel.nopadding .panel-body {
+ padding: 0px;
+}
+
+.loading {
+ width: 50px;
+ margin-top: 15px;
+}
+.pane-cell {
+ padding: 10px;
+ overflow: auto;
+ width: 100%;
+ height: 100%;
+}
+.SqlEditor .header {
+ padding-top: 5px;
+ padding-bottom: 5px;
+}
+.Workspace .btn-sm {
+ box-shadow: 1px 1px 2px rgba(0, 0, 0, 0.1);
+ margin-top: 2px;
+ padding: 4px;
+}
+.Workspace hr {
+ margin-top: 10px;
+ margin-bottom: 10px;
+}
+div.Workspace {
+ height: 100%;
+ margin: 0px;
+}
+.SqlEditor .clock {
+ background-color: orange;
+ padding: 5px;
+}
+
+.padded {
+ padding: 10px;
+}
+
+.p-t-10 {
+ padding-top: 10px;
+}
+.p-t-5 {
+ padding-top: 5px;
+}
+.m-r-5 {
+ margin-right: 5px;
+}
+.m-l-1 {
+ margin-left: 1px;
+}
+.m-r-10 {
+ margin-right: 10px;
+}
+.m-b-10 {
+ margin-bottom: 10px;
+}
+.m-t-5 {
+ margin-top: 5px;
+}
+.m-t-10 {
+ margin-top: 10px;
+}
+.p-t-10 {
+ padding-top: 10px;
+}
+.sqllab-toolbar {
+ padding-top: 5px;
+ border-bottom: 1px solid #DDD;
+}
+.no-shadow {
+ box-shadow: none;
+ background-color: transparent;
+}
+.pane-west {
+ height: 100%;
+ overflow: auto;
+}
+.circle {
+ border-radius: 50%;
+ width: 10px;
+ height: 10px;
+ display: inline-block;
+ background-color: #ccc;
+}
+.Pane2 {
+ width: 0;
+}
+.running {
+ background-color: lime;
+ color: black;
+}
+.success {
+ background-color: green;
+}
+.failed {
+ background-color: red;
+}
+
+.handle {
+ cursor: move;
+}
+.window {
+ z-index: 1000;
+ position: absolute;
+ width: 300px;
+ opacity: 0.85;
+ border: 1px solid #AAA;
+ max-height: 600px;
+ box-shadow: rgba(0, 0, 0, 0.8) 5px 5px 25px
+}
+
+.SqlLab pre {
+ padding: 0px !important;
+ margin: 0px;
+ border: none;
+ font-size: 11px;
+ line-height: 125%;
+ background-color: transparent !important;
+}
+
+.Resizer {
+ background: #000;
+ opacity: .2;
+ z-index: 1;
+ -moz-box-sizing: border-box;
+ -webkit-box-sizing: border-box;
+ box-sizing: border-box;
+ -moz-background-clip: padding;
+ -webkit-background-clip: padding;
+ background-clip: padding-box;
+}
+
+.Resizer:hover {
+ -webkit-transition: all 2s ease;
+ transition: all 2s ease;
+}
+
+.Resizer.horizontal {
+ height: 10px;
+ margin: -5px 0;
+ border-top: 5px solid rgba(255, 255, 255, 0);
+ border-bottom: 5px solid rgba(255, 255, 255, 0);
+ cursor: row-resize;
+ width: 100%;
+ padding: 1px;
+}
+
+.Resizer.horizontal:hover {
+ border-top: 5px solid rgba(0, 0, 0, 0.5);
+ border-bottom: 5px solid rgba(0, 0, 0, 0.5);
+}
+
+.Resizer.vertical {
+ width: 9px;
+ margin: 0 -5px;
+ border-left: 5px solid rgba(255, 255, 255, 0);
+ border-right: 5px solid rgba(255, 255, 255, 0);
+ cursor: col-resize;
+}
+
+.Resizer.vertical:hover {
+ border-left: 5px solid rgba(0, 0, 0, 0.5);
+ border-right: 5px solid rgba(0, 0, 0, 0.5);
+}
+.Resizer.disabled {
+ cursor: not-allowed;
+}
+.Resizer.disabled:hover {
+ border-color: transparent;
+}
+
+.popover{
+ max-width:400px;
+}
+.Select-menu-outer {
+ z-index: 1000;
+}
+.table-label {
+ margin-top: 5px;
+ margin-right: 10px;
+ float: left;
+}
+div.tablePopover {
+ opacity: 0.7 !important;
+}
+div.tablePopover:hover {
+ opacity: 1 !important;
+}
+.ResultSetControls {
+ padding-bottom: 3px;
+ padding-top: 3px;
+}
+
+.ace_editor {
+ border: 1px solid #ccc;
+ margin: 20px 0;
+}
+
+.Select-menu-outer {
+ min-width: 100%;
+ width: inherit;
+}
diff --git a/caravel/assets/javascripts/SqlLab/reducers.js b/caravel/assets/javascripts/SqlLab/reducers.js
new file mode 100644
index 000000000000..96615e914923
--- /dev/null
+++ b/caravel/assets/javascripts/SqlLab/reducers.js
@@ -0,0 +1,199 @@
+import moment from 'moment';
+import shortid from 'shortid';
+import * as actions from './actions';
+
+const defaultQueryEditor = {
+ id: shortid.generate(),
+ title: 'Query 1',
+ sql: 'SELECT *\nFROM\nWHERE',
+ latestQueryId: null,
+ autorun: false,
+ dbId: null,
+};
+
+// TODO(bkyryliuk): document the object schemas
+export const initialState = {
+ alerts: [],
+ queries: {},
+ queryEditors: [defaultQueryEditor],
+ tabHistory: [defaultQueryEditor.id],
+ tables: [],
+ workspaceQueries: [],
+};
+
+function addToObject(state, arrKey, obj) {
+ const newObject = Object.assign({}, state[arrKey]);
+ const copiedObject = Object.assign({}, obj);
+
+ if (!copiedObject.id) {
+ copiedObject.id = shortid.generate();
+ }
+ newObject[copiedObject.id] = copiedObject;
+ return Object.assign({}, state, { [arrKey]: newObject });
+}
+
+function alterInObject(state, arrKey, obj, alterations) {
+ const newObject = Object.assign({}, state[arrKey]);
+ newObject[obj.id] = (Object.assign({}, newObject[obj.id], alterations));
+ return Object.assign({}, state, { [arrKey]: newObject });
+}
+
+function alterInArr(state, arrKey, obj, alterations) {
+ // Finds an item in an array in the state and replaces it with a
+ // new object with an altered property
+ const idKey = 'id';
+ const newArr = [];
+ state[arrKey].forEach((arrItem) => {
+ if (obj[idKey] === arrItem[idKey]) {
+ newArr.push(Object.assign({}, arrItem, alterations));
+ } else {
+ newArr.push(arrItem);
+ }
+ });
+ return Object.assign({}, state, { [arrKey]: newArr });
+}
+
+function replaceInArr(state, arrKey, obj) {
+ // Finds an item in an array in the state and replaces it with a
+ // new object with an altered property
+ const idKey = 'id';
+ const newArr = [];
+ state[arrKey].forEach((arrItem) => {
+ if (obj[idKey] === arrItem[idKey]) {
+ newArr.push(Object.assign({}, obj[idKey]));
+ } else {
+ newArr.push(arrItem);
+ }
+ });
+ return Object.assign({}, state, { [arrKey]: newArr });
+}
+
+function removeFromArr(state, arrKey, obj, idKey = 'id') {
+ const newArr = [];
+ state[arrKey].forEach((arrItem) => {
+ if (!(obj[idKey] === arrItem[idKey])) {
+ newArr.push(arrItem);
+ }
+ });
+ return Object.assign({}, state, { [arrKey]: newArr });
+}
+
+function addToArr(state, arrKey, obj) {
+ const newObj = Object.assign({}, obj);
+ if (!newObj.id) {
+ newObj.id = shortid.generate();
+ }
+ const newState = {};
+ newState[arrKey] = [...state[arrKey], newObj];
+ return Object.assign({}, state, newState);
+}
+
+export const sqlLabReducer = function (state, action) {
+ const actionHandlers = {
+ [actions.ADD_QUERY_EDITOR]() {
+ const tabHistory = state.tabHistory.slice();
+ tabHistory.push(action.queryEditor.id);
+ const newState = Object.assign({}, state, { tabHistory });
+ return addToArr(newState, 'queryEditors', action.queryEditor);
+ },
+ [actions.REMOVE_QUERY_EDITOR]() {
+ let newState = removeFromArr(state, 'queryEditors', action.queryEditor);
+ // List of remaining queryEditor ids
+ const qeIds = newState.queryEditors.map((qe) => qe.id);
+ let th = state.tabHistory.slice();
+ th = th.filter((id) => qeIds.includes(id));
+ newState = Object.assign({}, newState, { tabHistory: th });
+ return newState;
+ },
+ [actions.REMOVE_QUERY]() {
+ const newQueries = Object.assign({}, state['queries']);
+ delete newQueries[action.query.id]
+ return Object.assign({}, state, { queries: newQueries });
+ },
+ [actions.RESET_STATE]() {
+ return Object.assign({}, initialState);
+ },
+ [actions.ADD_TABLE]() {
+ return addToArr(state, 'tables', action.table);
+ },
+ [actions.EXPAND_TABLE]() {
+ return alterInArr(state, 'tables', action.table, { expanded: true });
+ },
+ [actions.COLLAPSE_TABLE]() {
+ return alterInArr(state, 'tables', action.table, { expanded: false });
+ },
+ [actions.REMOVE_TABLE]() {
+ return removeFromArr(state, 'tables', action.table);
+ },
+ [actions.START_QUERY]() {
+ const newState = addToObject(state, 'queries', action.query);
+ const sqlEditor = { id: action.query.sqlEditorId };
+ return alterInArr(newState, 'queryEditors', sqlEditor, { latestQueryId: action.query.id });
+ },
+ [actions.STOP_QUERY]() {
+ return alterInObject(state, 'queries', action.query, { state: 'stopped' });
+ },
+ [actions.QUERY_SUCCESS]() {
+ const alts = {
+ state: 'success',
+ results: action.results,
+ rows: action.results.data.length,
+ endDttm: moment(),
+ };
+ return alterInObject(state, 'queries', action.query, alts);
+ },
+ [actions.QUERY_FAILED]() {
+ const alts = { state: 'failed', msg: action.msg, endDttm: moment() };
+ return alterInObject(state, 'queries', action.query, alts);
+ },
+ [actions.SET_ACTIVE_QUERY_EDITOR]() {
+ const qeIds = state.queryEditors.map((qe) => qe.id);
+ if (qeIds.includes(action.queryEditor.id)) {
+ const tabHistory = state.tabHistory.slice();
+ tabHistory.push(action.queryEditor.id);
+ return Object.assign({}, state, { tabHistory });
+ }
+ return state;
+ },
+ [actions.QUERY_EDITOR_SETDB]() {
+ return alterInArr(state, 'queryEditors', action.queryEditor, { dbId: action.dbId });
+ },
+ [actions.QUERY_EDITOR_SET_SCHEMA]() {
+ return alterInArr(state, 'queryEditors', action.queryEditor, { schema: action.schema });
+ },
+ [actions.QUERY_EDITOR_SET_TITLE]() {
+ return alterInArr(state, 'queryEditors', action.queryEditor, { title: action.title });
+ },
+ [actions.QUERY_EDITOR_SET_SQL]() {
+ return alterInArr(state, 'queryEditors', action.queryEditor, { sql: action.sql });
+ },
+ [actions.QUERY_EDITOR_SET_AUTORUN]() {
+ return alterInArr(state, 'queryEditors', action.queryEditor, { autorun: action.autorun });
+ },
+ [actions.ADD_WORKSPACE_QUERY]() {
+ return addToArr(state, 'workspaceQueries', action.query);
+ },
+ [actions.REMOVE_WORKSPACE_QUERY]() {
+ return removeFromArr(state, 'workspaceQueries', action.query);
+ },
+ [actions.ADD_ALERT]() {
+ return addToArr(state, 'alerts', action.alert);
+ },
+ [actions.REMOVE_ALERT]() {
+ return removeFromArr(state, 'alerts', action.alert);
+ },
+ [actions.REFRESH_QUERIES]() {
+ const newQueries = Object.assign({}, state['queries']);
+ // Fetch the updates to the queries present in the store.
+ for (var queryId in state['queries']) {
+ newQueries[queryId] = Object.assign(newQueries[queryId],
+ action.alteredQueries[queryId])
+ }
+ return Object.assign({}, state, { queries: newQueries });
+ },
+ };
+ if (action.type in actionHandlers) {
+ return actionHandlers[action.type]();
+ }
+ return state;
+};
diff --git a/caravel/assets/javascripts/dashboard/Dashboard.jsx b/caravel/assets/javascripts/dashboard/Dashboard.jsx
index 99c2a1bd3825..4b7fc97429cd 100644
--- a/caravel/assets/javascripts/dashboard/Dashboard.jsx
+++ b/caravel/assets/javascripts/dashboard/Dashboard.jsx
@@ -50,6 +50,7 @@ function dashboardContainer(dashboardData) {
const sliceObjects = [];
const dash = this;
dashboard.slices.forEach((data) => {
+ console.log(data);
if (data.error) {
const html = '' + data.error + '
';
$('#slice_' + data.slice_id).find('.token').html(html);
diff --git a/caravel/assets/javascripts/welcome.js b/caravel/assets/javascripts/welcome.js
index d83cd3e7ee92..4cc6a3c77e72 100644
--- a/caravel/assets/javascripts/welcome.js
+++ b/caravel/assets/javascripts/welcome.js
@@ -14,13 +14,14 @@ function modelViewTable(selector, modelView, orderCol, order) {
url += '?_oc_' + modelView + '=' + orderCol;
url += '&_od_' + modelView + '=' + order;
$.getJSON(url, function (data) {
+ const columns = ['dashboard_link', 'creator', 'modified'];
const tableData = $.map(data.result, function (el) {
- const row = $.map(data.list_columns, function (col) {
+ const row = $.map(columns, function (col) {
return el[col];
});
return [row];
});
- const cols = $.map(data.list_columns, function (col) {
+ const cols = $.map(columns, function (col) {
return { sTitle: data.label_columns[col] };
});
const panel = $(selector).parents('.panel');
diff --git a/caravel/assets/package.json b/caravel/assets/package.json
index d9d9f8b1d396..9192d7ef0b05 100644
--- a/caravel/assets/package.json
+++ b/caravel/assets/package.json
@@ -8,7 +8,7 @@
},
"scripts": {
"test": "npm run lint && mocha --compilers js:babel-core/register --required spec/helpers/browser.js spec/**/*_spec.*",
- "dev": "NODE_ENV=dev webpack -d --watch --colors",
+ "dev": "NODE_ENV=dev webpack -d --watch --colors --progress",
"prod": "NODE_ENV=production webpack -p --colors --progress",
"lint": "npm run --silent lint:js",
"lint:js": "eslint --ignore-path=.eslintignore --ext .js ."
@@ -55,19 +55,29 @@
"jquery": "^2.2.1",
"jquery-ui": "1.10.5",
"mapbox-gl": "^0.20.0",
+ "moment": "^2.14.1",
+ "moments": "0.0.2",
"mustache": "^2.2.1",
"nvd3": "1.8.4",
"react": "^15.2.1",
- "react-bootstrap": "^0.28.3",
- "react-bootstrap-datetimepicker": "0.0.22",
- "react-bootstrap-table": "^2.3.7",
+ "react-ace": "^3.4.1",
+ "react-bootstrap": "^0.30.3",
+ "react-bootstrap-table": "^2.3.8",
"react-dom": "^0.14.8",
- "react-grid-layout": "^0.12.3",
+ "react-draggable": "^2.1.2",
+ "react-grid-layout": "^0.12.4",
"react-map-gl": "^1.0.0-beta-10",
+ "react-redux": "^4.4.5",
"react-resizable": "^1.3.3",
"react-select": "^1.0.0-beta14",
+ "react-syntax-highlighter": "^2.1.1",
+ "reactable": "^0.13.2",
+ "redux": "^3.5.2",
+ "redux-localstorage": "^0.4.1",
"select2": "3.5",
"select2-bootstrap-css": "^1.4.6",
+ "shortid": "^2.2.6",
+ "style-loader": "^0.13.0",
"supercluster": "https://github.com/georgeke/supercluster/tarball/ac3492737e7ce98e07af679623aad452373bbc40",
"topojson": "^1.6.22",
"viewport-mercator-project": "^2.1.0"
diff --git a/caravel/assets/stylesheets/caravel.css b/caravel/assets/stylesheets/caravel.css
index 03dc68129a3d..5a326b4d028b 100644
--- a/caravel/assets/stylesheets/caravel.css
+++ b/caravel/assets/stylesheets/caravel.css
@@ -193,5 +193,10 @@ div.widget .slice_container {
.navbar .alert {
padding: 5px 10px;
- margin-top: 15px;
+ margin-top: 8px;
+ margin-bottom: 0px
+}
+
+.table-condensed {
+ font-size: 12px;
}
diff --git a/caravel/assets/stylesheets/less/cosmo/bootswatch.less b/caravel/assets/stylesheets/less/cosmo/bootswatch.less
index f5cc0d1276ea..ea5d456496e9 100644
--- a/caravel/assets/stylesheets/less/cosmo/bootswatch.less
+++ b/caravel/assets/stylesheets/less/cosmo/bootswatch.less
@@ -24,14 +24,63 @@
// Buttons ====================================================================
-.btn {
- padding: 8px 12px 6px 12px;
-}
.btn-default:hover {
background-color: #efefef;
border-color: #bbb;
}
+.nav-tabs {
+ .dropdown-toggle.btn,
+ .dropdown-toggle.btn:hover,
+ .dropdown-toggle.btn:active,
+ .dropdown-toggle.btn:focus,
+ .btn-group.open .dropdown-toggle.btn,
+ .btn-group.open .dropdown-toggle.btn:hover,
+ .btn-group.open .dropdown-toggle.btn:active,
+ .btn-group.open .dropdown-toggle.btn:focus {
+ border-color: transparent;
+ background-color: transparent;
+ box-shadow: none;
+ }
+}
+
+.caret {
+ border: none;
+ color: @gray;
+}
+
+.caret:before {
+ font-family: "FontAwesome";
+ font-size: 10px;
+ content: "\f078";
+}
+
+.caret:hover {
+ color: @gray-darker;
+}
+
+.dropdown-toggle.btn .caret {
+ margin-left: 6px;
+ margin-top: -21px;
+ margin-right: 6px;
+}
+
+.nav-tabs .dropdown-toggle.btn .caret, {
+ margin-left: -12px;
+ margin-top: -10px;
+}
+
+.navbar-nav .caret,
+.panel-title .caret {
+ margin-left: 6px;
+ margin-top: -24px;
+ margin-right: 6px;
+}
+
+
+
+
+
// Typography =================================================================
body {
@@ -332,3 +381,8 @@ a.list-group-item {
color: @state-success-text;
}
}
+
+// Utils ==============================================================
+hr {
+ margin: 10px 0;
+}
diff --git a/caravel/assets/stylesheets/less/cosmo/variables.less b/caravel/assets/stylesheets/less/cosmo/variables.less
index be76aaea7d03..4804346c710a 100644
--- a/caravel/assets/stylesheets/less/cosmo/variables.less
+++ b/caravel/assets/stylesheets/less/cosmo/variables.less
@@ -11,7 +11,7 @@
@gray-darker: lighten(@gray-base, 13.5%); // #222
@gray-dark: lighten(@gray-base, 20%); // #333
@gray: lighten(@gray-base, 33.5%); // #555
-@gray-light: lighten(@gray-base, 70%); //
+@gray-light: lighten(@gray-base, 70%);
@gray-lighter: lighten(@gray-base, 90%); // #eee
@brand-primary: #515152;
diff --git a/caravel/assets/visualizations/histogram.js b/caravel/assets/visualizations/histogram.js
index 10b6aea7a24e..a7e10e0ddac0 100644
--- a/caravel/assets/visualizations/histogram.js
+++ b/caravel/assets/visualizations/histogram.js
@@ -1,151 +1,151 @@
-// JS
-const d3 = require('d3')
-const px = window.px || require('../javascripts/modules/caravel.js')
+import { category21 } from '../javascripts/modules/colors';
+import d3 from 'd3';
-// CSS
-require('./histogram.css')
+require('./histogram.css');
function histogram(slice) {
-
- const div = d3.select(slice.selector)
-
- const _draw = function(data, numBins) {
-
- // Set Margins
- const margin = {
- top: 50,
- right: 10,
- bottom: 20,
- left: 50,
- };
- const navBarHeight = 36;
- const navBarTitleSize = 12;
- const navBarBuffer = 10;
- const width = slice.width() - margin.left - margin.right;
- const height = slice.height() - margin.top - margin.bottom - navBarHeight - navBarBuffer;
-
- // Set Histogram objects
- const formatNumber = d3.format(',.0f');
- const formatTicks = d3.format(',.00f');
- const x = d3.scale.ordinal();
- const y = d3.scale.linear();
- const xAxis = d3.svg.axis().scale(x).orient('bottom').ticks(numBins).tickFormat(formatTicks);
- const yAxis = d3.svg.axis().scale(y).orient('left').ticks(numBins*3);
- // Calculate bins for the data
- const bins = d3.layout.histogram().bins(numBins)(data);
-
- // Set the x-values
- x.domain(bins.map(function(d) { return d.x;}))
- .rangeRoundBands([0, width], .1);
- // Set the y-values
- y.domain([0, d3.max(bins, function(d) { return d.y;})])
- .range([height, 0]);
-
- // Create the svg value with the bins
- const svg = div.selectAll('svg').data([bins]).enter().append('svg');
-
- // Make a rectangular background fill
- svg.append('rect')
- .attr('width', '100%')
- .attr('height', '100%')
- .attr('fill', '#f6f6f6');
-
- // Transform the svg to make space for the margins
- const gEnter = svg
- .append('g')
- .attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
- // Add the bars and the x axis
- gEnter.append('g').attr('class', 'bars');
- gEnter.append('g').attr('class', 'x axis');
-
- // Add width and height to the svg
- svg.attr('width', slice.width())
- .attr('height', slice.height());
-
- // Create the bars in the svg
- const bar = svg.select('.bars').selectAll('.bar').data(bins);
- bar.enter().append('rect');
- bar.exit().remove();
- // Set the Height and Width for each bar
- bar .attr('width', x.rangeBand())
- .attr('x', function(d) { return x(d.x); })
- .attr('y', function(d) { return y(d.y); })
- .attr('height', function(d) {
- return y.range()[0] - y(d.y);
- })
- .attr('fill', function(d) { return px.color.category21(d.length); })
- .order();
-
- // Find maximum length to position the ticks on top of the bar correctly
- const maxLength = d3.max(bins, function(d) { return d.length;});
- function textAboveBar(d) {
- return d.length/maxLength < 0.1;
- }
-
- // Add a bar text to each bar in the histogram
- svg.selectAll('.bartext')
- .data(bins)
- .enter()
- .append('text')
- .attr('dy', '.75em')
- .attr('y', function(d) {
- let padding = 0.0
- if (textAboveBar(d)) {
- padding = 12.0
- } else {
- padding = -8.0
- }
- return y(d.y) - padding;
- })
- .attr('x', function(d) { return x(d.x) + (x.rangeBand()/2);})
- .attr('text-anchor', 'middle')
- .attr('font-weight', 'bold')
- .attr('font-size', '15px')
- .text(function(d) { return formatNumber(d.y); })
- .attr('fill', function(d) {
- if(textAboveBar(d)) { return 'black'; } else { return 'white'; }
- })
- .attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
-
- // Update the x-axis
- svg.append('g')
- .attr('class', 'axis')
- .attr('transform', 'translate(' + margin.left + ',' + (height + margin.top) + ')')
- .text('values')
- .call(xAxis);
-
- // Update the Y Axis and add minor lines
- svg.append('g')
- .attr('class', 'axis')
- .attr('transform', 'translate(' + margin.left + ',' + margin.top + ')')
- .text('count')
- .call(yAxis)
- .selectAll('g')
- .filter(function(d) { return d; })
- .classed('minor', true);
- };
+ const div = d3.select(slice.selector);
- const render = function() {
-
- d3.json(slice.jsonEndpoint(), function(error, json) {
- if(error !== null) {
- slice.error(error.responseText, error);
- return '';
- }
-
- const numBins = Number(json.form_data.link_length) || 10;
-
- div.selectAll('*').remove();
- _draw(json.data, numBins);
- slice.done(json);
- });
+ const draw = function (data, numBins) {
+ // Set Margins
+ const margin = {
+ top: 50,
+ right: 10,
+ bottom: 20,
+ left: 50,
};
+ const navBarHeight = 36;
+ const navBarBuffer = 10;
+ const width = slice.width() - margin.left - margin.right;
+ const height = slice.height() - margin.top - margin.bottom - navBarHeight - navBarBuffer;
+
+ // Set Histogram objects
+ const formatNumber = d3.format(',.0f');
+ const formatTicks = d3.format(',.00f');
+ const x = d3.scale.ordinal();
+ const y = d3.scale.linear();
+ const xAxis = d3.svg.axis()
+ .scale(x)
+ .orient('bottom')
+ .ticks(numBins)
+ .tickFormat(formatTicks);
+ const yAxis = d3.svg.axis()
+ .scale(y)
+ .orient('left')
+ .ticks(numBins * 3);
+ // Calculate bins for the data
+ const bins = d3.layout.histogram().bins(numBins)(data);
+
+ // Set the x-values
+ x.domain(bins.map((d) => d.x))
+ .rangeRoundBands([0, width], 0.1);
+ // Set the y-values
+ y.domain([0, d3.max(bins, (d) => d.y)])
+ .range([height, 0]);
+
+ // Create the svg value with the bins
+ const svg = div.selectAll('svg')
+ .data([bins])
+ .enter()
+ .append('svg');
+
+ // Make a rectangular background fill
+ svg.append('rect')
+ .attr('width', '100%')
+ .attr('height', '100%')
+ .attr('fill', '#f6f6f6');
+
+ // Transform the svg to make space for the margins
+ const gEnter = svg
+ .append('g')
+ .attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
+
+ // Add the bars and the x axis
+ gEnter.append('g').attr('class', 'bars');
+ gEnter.append('g').attr('class', 'x axis');
+
+ // Add width and height to the svg
+ svg.attr('width', slice.width())
+ .attr('height', slice.height());
+
+ // Create the bars in the svg
+ const bar = svg.select('.bars').selectAll('.bar').data(bins);
+ bar.enter().append('rect');
+ bar.exit().remove();
+ // Set the Height and Width for each bar
+ bar.attr('width', x.rangeBand())
+ .attr('x', (d) => x(d.x))
+ .attr('y', (d) => y(d.y))
+ .attr('height', (d) => y.range()[0] - y(d.y))
+ .style('fill', (d) => category21(d.length))
+ .order();
+
+ // Find maximum length to position the ticks on top of the bar correctly
+ const maxLength = d3.max(bins, (d) => d.length);
+ function textAboveBar(d) {
+ return d.length / maxLength < 0.1;
+ }
+
+ // Add a bar text to each bar in the histogram
+ svg.selectAll('.bartext')
+ .data(bins)
+ .enter()
+ .append('text')
+ .attr('dy', '.75em')
+ .attr('y', function (d) {
+ let padding = 0.0;
+ if (textAboveBar(d)) {
+ padding = 12.0;
+ } else {
+ padding = -8.0;
+ }
+ return y(d.y) - padding;
+ })
+ .attr('x', (d) => x(d.x) + (x.rangeBand() / 2))
+ .attr('text-anchor', 'middle')
+ .attr('font-weight', 'bold')
+ .attr('font-size', '15px')
+ .text((d) => formatNumber(d.y))
+ .attr('fill', (d) => textAboveBar(d) ? 'black' : 'white')
+ .attr('transform', 'translate(' + margin.left + ',' + margin.top + ')');
+
+ // Update the x-axis
+ svg.append('g')
+ .attr('class', 'axis')
+ .attr('transform', 'translate(' + margin.left + ',' + (height + margin.top) + ')')
+ .text('values')
+ .call(xAxis);
+
+ // Update the Y Axis and add minor lines
+ svg.append('g')
+ .attr('class', 'axis')
+ .attr('transform', 'translate(' + margin.left + ',' + margin.top + ')')
+ .text('count')
+ .call(yAxis)
+ .selectAll('g')
+ .filter(function (d) { return d; })
+ .classed('minor', true);
+ };
+
+ const render = function () {
+ d3.json(slice.jsonEndpoint(), function (error, json) {
+ if (error !== null) {
+ slice.error(error.responseText, error);
+ return;
+ }
+
+ const numBins = Number(json.form_data.link_length) || 10;
+
+ div.selectAll('*').remove();
+ draw(json.data, numBins);
+ slice.done(json);
+ });
+ };
- return {
- render: render,
- resize: render,
- };
+ return {
+ render,
+ resize: render,
+ };
}
module.exports = histogram;
diff --git a/caravel/assets/webpack.config.js b/caravel/assets/webpack.config.js
index b877d24df51a..76aa7e4c1da7 100644
--- a/caravel/assets/webpack.config.js
+++ b/caravel/assets/webpack.config.js
@@ -16,6 +16,7 @@ const config = {
sql: APP_DIR + '/javascripts/sql.js',
standalone: APP_DIR + '/javascripts/standalone.js',
common: APP_DIR + '/javascripts/common.js',
+ sqllab: APP_DIR + '/javascripts/SqlLab/index.jsx',
},
output: {
path: BUILD_DIR,
diff --git a/caravel/bin/caravel b/caravel/bin/caravel
index c53cdc2b338f..51d0f63c4e1f 100755
--- a/caravel/bin/caravel
+++ b/caravel/bin/caravel
@@ -5,6 +5,8 @@ from __future__ import print_function
from __future__ import unicode_literals
import logging
+import celery
+from celery.bin import worker as celery_worker
from datetime import datetime
from subprocess import Popen
@@ -126,5 +128,24 @@ def refresh_druid():
session.commit()
+@manager.command
+def worker():
+ """Starts a Caravel worker for async SQL query execution."""
+ # celery -A tasks worker --loglevel=info
+ print("Starting SQL Celery worker.")
+ if config.get('CELERY_CONFIG'):
+ print("Celery broker url: ")
+ print(config.get('CELERY_CONFIG').BROKER_URL)
+
+ application = celery.current_app._get_current_object()
+ c_worker = celery_worker.worker(app=application)
+ options = {
+ 'broker': config.get('CELERY_CONFIG').BROKER_URL,
+ 'loglevel': 'INFO',
+ 'traceback': True,
+ }
+ c_worker.run(**options)
+
+
if __name__ == "__main__":
manager.run()
diff --git a/caravel/config.py b/caravel/config.py
index f4285ebddd35..fd2062e68eca 100644
--- a/caravel/config.py
+++ b/caravel/config.py
@@ -179,11 +179,30 @@
# Set this API key to enable Mapbox visualizations
MAPBOX_API_KEY = ""
+# Maximum number of rows returned in the SQL editor
+SQL_MAX_ROW = 1000
+
# If defined, shows this text in an alert-warning box in the navbar
# one example use case may be "STAGING" to make it clear that this is
# not the production version of the site.
WARNING_MSG = None
+# Default celery config is to use SQLA as a broker, in a production setting
+# you'll want to use a proper broker as specified here:
+# http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html
+"""
+# Example:
+class CeleryConfig(object):
+ BROKER_URL = 'sqla+sqlite:///celerydb.sqlite'
+ CELERY_IMPORTS = ('caravel.tasks', )
+ CELERY_RESULT_BACKEND = 'db+sqlite:///celery_results.sqlite'
+ CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
+CELERY_CONFIG = CeleryConfig
+"""
+CELERY_CONFIG = None
+
+# The db id here results in selecting this one as a default in SQL Lab
+DEFAULT_DB_ID = None
try:
from caravel_config import * # noqa
diff --git a/caravel/extract_table_names.py b/caravel/extract_table_names.py
new file mode 100644
index 000000000000..4bc57074290a
--- /dev/null
+++ b/caravel/extract_table_names.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2016 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This example is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php
+#
+# This example illustrates how to extract table names from nested
+# SELECT statements.
+#
+# See:
+# http://groups.google.com/group/sqlparse/browse_thread/thread/b0bd9a022e9d4895
+
+import sqlparse
+from sqlparse.sql import IdentifierList, Identifier
+from sqlparse.tokens import Keyword, DML
+
+
+def is_subselect(parsed):
+ if not parsed.is_group():
+ return False
+ for item in parsed.tokens:
+ if item.ttype is DML and item.value.upper() == 'SELECT':
+ return True
+ return False
+
+
+def extract_from_part(parsed):
+ from_seen = False
+ for item in parsed.tokens:
+ if from_seen:
+ if is_subselect(item):
+ for x in extract_from_part(item):
+ yield x
+ elif item.ttype is Keyword:
+ raise StopIteration
+ else:
+ yield item
+ elif item.ttype is Keyword and item.value.upper() == 'FROM':
+ from_seen = True
+
+
+def extract_table_identifiers(token_stream):
+ for item in token_stream:
+ if isinstance(item, IdentifierList):
+ for identifier in item.get_identifiers():
+ yield identifier.get_name()
+ elif isinstance(item, Identifier):
+ yield item.get_name()
+ # It's a bug to check for Keyword here, but in the example
+ # above some tables names are identified as keywords...
+ elif item.ttype is Keyword:
+ yield item.value
+
+
+# TODO(bkyryliuk): add logic to support joins and unions.
+def extract_tables(sql):
+ stream = extract_from_part(sqlparse.parse(sql)[0])
+ return list(extract_table_identifiers(stream))
diff --git a/caravel/migrations/versions/3c3ffe173e4f_add_sql_string_to_table.py b/caravel/migrations/versions/3c3ffe173e4f_add_sql_string_to_table.py
new file mode 100644
index 000000000000..5b64bc40df2f
--- /dev/null
+++ b/caravel/migrations/versions/3c3ffe173e4f_add_sql_string_to_table.py
@@ -0,0 +1,22 @@
+"""add_sql_string_to_table
+
+Revision ID: 3c3ffe173e4f
+Revises: ad82a75afd82
+Create Date: 2016-08-18 14:06:28.784699
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '3c3ffe173e4f'
+down_revision = 'ad82a75afd82'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ op.add_column('tables', sa.Column('sql', sa.Text(), nullable=True))
+
+
+def downgrade():
+ op.drop_column('tables', 'sql')
diff --git a/caravel/migrations/versions/ad82a75afd82_add_query_model.py b/caravel/migrations/versions/ad82a75afd82_add_query_model.py
new file mode 100644
index 000000000000..a166bba63746
--- /dev/null
+++ b/caravel/migrations/versions/ad82a75afd82_add_query_model.py
@@ -0,0 +1,58 @@
+"""Update models to support storing the queries.
+
+Revision ID: ad82a75afd82
+Revises: f162a1dea4c4
+Create Date: 2016-07-25 17:48:12.771103
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ad82a75afd82'
+down_revision = 'f162a1dea4c4'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ op.create_table('query',
+ sa.Column('id', sa.Integer(), nullable=False),
+ sa.Column('client_id', sa.String(length=11), nullable=False),
+ sa.Column('database_id', sa.Integer(), nullable=False),
+ sa.Column('tmp_table_name', sa.String(length=256), nullable=True),
+ sa.Column('tab_name', sa.String(length=256),nullable=True),
+ sa.Column('sql_editor_id', sa.String(length=256), nullable=True),
+ sa.Column('user_id', sa.Integer(), nullable=True),
+ sa.Column('status', sa.String(length=16), nullable=True),
+ sa.Column('name', sa.String(length=256), nullable=True),
+ sa.Column('schema', sa.String(length=256), nullable=True),
+ sa.Column('sql', sa.Text(), nullable=True),
+ sa.Column('select_sql', sa.Text(), nullable=True),
+ sa.Column('executed_sql', sa.Text(), nullable=True),
+ sa.Column('limit', sa.Integer(), nullable=True),
+ sa.Column('limit_used', sa.Boolean(), nullable=True),
+ sa.Column('select_as_cta', sa.Boolean(), nullable=True),
+ sa.Column('select_as_cta_used', sa.Boolean(), nullable=True),
+ sa.Column('progress', sa.Integer(), nullable=True),
+ sa.Column('rows', sa.Integer(), nullable=True),
+ sa.Column('error_message', sa.Text(), nullable=True),
+ sa.Column('start_time', sa.DateTime(), nullable=True),
+ sa.Column('changed_on', sa.DateTime(), nullable=True),
+ sa.Column('end_time', sa.DateTime(), nullable=True),
+ sa.ForeignKeyConstraint(['database_id'], [u'dbs.id'], ),
+ sa.ForeignKeyConstraint(['user_id'], [u'ab_user.id'], ),
+ sa.PrimaryKeyConstraint('id')
+ )
+ op.add_column('dbs', sa.Column('select_as_create_table_as', sa.Boolean(),
+ nullable=True))
+ op.create_index(
+ op.f('ti_user_id_changed_on'),
+ 'query', ['user_id', 'changed_on'], unique=False)
+
+
+def downgrade():
+ op.drop_index(op.f('ix_query_user_id'), table_name='query')
+ op.drop_index(op.f('ix_query_changed_on'), table_name='query')
+ op.drop_table('query')
+ op.drop_column('dbs', 'select_as_create_table_as')
+
diff --git a/caravel/models.py b/caravel/models.py
index 5fe42729d627..cb27576286e2 100644
--- a/caravel/models.py
+++ b/caravel/models.py
@@ -16,6 +16,7 @@
import pandas as pd
import requests
import sqlalchemy as sqla
+from sqlalchemy.engine.url import make_url
import sqlparse
from dateutil.parser import parse
@@ -30,13 +31,15 @@
from pydruid.utils.postaggregator import Postaggregator
from pydruid.utils.having import Aggregation
from six import string_types
+
from sqlalchemy import (
- Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date,
- Table, create_engine, MetaData, desc, asc, select, and_, func)
-from sqlalchemy.engine import reflection
+ Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date, Table,
+ create_engine, MetaData, desc, asc, select, and_, func
+)
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import relationship
from sqlalchemy.sql import table, literal_column, text, column
+from sqlalchemy.sql.expression import TextAsFrom
from sqlalchemy_utils import EncryptedType
import caravel
@@ -310,6 +313,11 @@ def metadata_dejson(self):
else:
return {}
+ @property
+ def sqla_metadata(self):
+ metadata = MetaData(bind=self.get_sqla_engine())
+ return metadata.reflect()
+
def dashboard_link(self):
return '{obj.dashboard_title} '.format(obj=self)
@@ -372,6 +380,7 @@ class Database(Model, AuditMixinNullable):
sqlalchemy_uri = Column(String(1024))
password = Column(EncryptedType(String(1024), config.get('SECRET_KEY')))
cache_timeout = Column(Integer)
+ select_as_create_table_as = Column(Boolean, default=False)
extra = Column(Text, default=textwrap.dedent("""\
{
"metadata_params": {},
@@ -382,14 +391,69 @@ class Database(Model, AuditMixinNullable):
def __repr__(self):
return self.database_name
- def get_sqla_engine(self):
+ def get_sqla_engine(self, schema=None):
extra = self.get_extra()
params = extra.get('engine_params', {})
- return create_engine(self.sqlalchemy_uri_decrypted, **params)
+ url = make_url(self.sqlalchemy_uri_decrypted)
+ backend = url.get_backend_name()
+ if backend == 'presto' and schema:
+ if '/' in url.database:
+ url.database = url.database.split('/')[0] + '/' + schema
+ else:
+ url.database += '/' + schema
+ elif schema:
+ url.database = schema
+ return create_engine(url, **params)
+
+ def get_df(self, sql, schema):
+ eng = self.get_sqla_engine(schema=schema)
+ cur = eng.execute(sql, schema=schema)
+ cols = [col[0] for col in cur.cursor.description]
+ df = pd.DataFrame(cur.fetchall(), columns=cols)
+ return df
+
+ def compile_sqla_query(self, qry, schema=None):
+ eng = self.get_sqla_engine(schema=schema)
+ compiled = qry.compile(eng, compile_kwargs={"literal_binds": True})
+ return '{}'.format(compiled)
+
+ def select_star(self, table_name, schema=None, limit=1000):
+ """Generates a ``select *`` statement in the proper dialect"""
+ qry = select('*').select_from(table_name)
+ if limit:
+ qry = qry.limit(limit)
+ return self.compile_sqla_query(qry)
+
+ def wrap_sql_limit(self, sql, limit=1000):
+ qry = (
+ select('*')
+ .select_from(TextAsFrom(text(sql), ['*'])
+ .alias('inner_qry')).limit(limit)
+ )
+ return self.compile_sqla_query(qry)
def safe_sqlalchemy_uri(self):
return self.sqlalchemy_uri
+ @property
+ def inspector(self):
+ engine = self.get_sqla_engine()
+ return sqla.inspect(engine)
+
+ def all_table_names(self, schema=None):
+ return sorted(self.inspector.get_table_names(schema))
+
+ def all_view_names(self, schema=None):
+ views = []
+ try:
+ views = self.inspector.get_view_names(schema)
+ except Exception as e:
+ pass
+ return views
+
+ def all_schema_names(self):
+ return sorted(self.inspector.get_schema_names())
+
def grains(self):
"""Defines time granularity database-specific expressions.
@@ -508,10 +572,8 @@ def get_table(self, table_name, schema=None):
autoload=True,
autoload_with=self.get_sqla_engine())
- def get_columns(self, table_name):
- engine = self.get_sqla_engine()
- insp = reflection.Inspector.from_engine(engine)
- return insp.get_columns(table_name)
+ def get_columns(self, table_name, schema=None):
+ return self.inspector.get_columns(table_name, schema)
@property
def sqlalchemy_uri_decrypted(self):
@@ -554,6 +616,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
offset = Column(Integer, default=0)
cache_timeout = Column(Integer)
schema = Column(String(255))
+ sql = Column(Text)
table_columns = relationship("TableColumn", back_populates="table")
baselink = "tablemodelview"
@@ -737,6 +800,10 @@ def query( # sqla
if self.schema:
tbl.schema = self.schema
+ # Supporting arbitrary SQL statements in place of tables
+ if self.sql:
+ tbl = text('(' + self.sql + ') as expr_qry ')
+
if not columns:
qry = qry.group_by(*groupby_exprs)
@@ -1690,3 +1757,82 @@ class FavStar(Model):
class_name = Column(String(50))
obj_id = Column(Integer)
dttm = Column(DateTime, default=func.now())
+
+
+class QueryStatus:
+ SCHEDULED = 'SCHEDULED'
+ CANCELLED = 'CANCELLED'
+ IN_PROGRESS = 'RUNNING'
+ FINISHED = 'SUCCESS'
+ TIMED_OUT = 'TIMED_OUT'
+ FAILED = 'FAILED'
+
+
+class Query(Model):
+
+ """ORM model for SQL query"""
+
+ __tablename__ = 'query'
+ id = Column(Integer, primary_key=True)
+ client_id = Column(String(11))
+
+ database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
+
+ # Store the tmp table into the DB only if the user asks for it.
+ tmp_table_name = Column(String(256))
+ user_id = Column(
+ Integer, ForeignKey('ab_user.id'), nullable=True)
+
+ status = Column(String(16)) # models.QueryStatus
+
+ name = Column(String(256))
+ tab_name = Column(String(256))
+ sql_editor_id = Column(String(256))
+ schema = Column(String(256))
+ sql = Column(Text)
+ # Query to retrieve the results,
+ # used only in case of select_as_cta_used is true.
+ select_sql = Column(Text)
+ executed_sql = Column(Text)
+ # Could be configured in the caravel config.
+ limit = Column(Integer)
+ limit_used = Column(Boolean)
+ select_as_cta = Column(Boolean)
+ select_as_cta_used = Column(Boolean)
+
+ # 1..100
+ progress = Column(Integer) # TODO should be float
+ # # of rows in the result set or rows modified.
+ rows = Column(Integer)
+ error_message = Column(Text)
+ start_time = Column(DateTime)
+ end_time = Column(DateTime)
+ changed_on = Column(
+ DateTime, default=datetime.now, onupdate=datetime.now, nullable=True)
+
+ database = relationship(
+ 'Database', foreign_keys=[database_id], backref='queries')
+
+ __table_args__ = (
+ sqla.Index('ti_user_id_changed_on', user_id, changed_on),
+ )
+
+ def to_dict(self):
+ return {
+ 'serverId': self.id,
+ 'id': self.client_id,
+ 'dbId': self.database_id,
+ 'tab': self.tab_name,
+ 'sqlEditorId': self.sql_editor_id,
+ 'userId': self.user_id,
+ 'state': self.status.lower(),
+ 'schema': self.schema,
+ 'sql': self.sql,
+ 'limit': self.limit,
+ 'progress': self.progress,
+ 'errorMessage': self.error_message,
+ 'startDttm': self.start_time,
+ 'endDttm': self.end_time,
+ 'changedOn': self.changed_on,
+ 'rows': self.rows,
+ }
diff --git a/caravel/sql_lab.py b/caravel/sql_lab.py
new file mode 100644
index 000000000000..822a5e4e4955
--- /dev/null
+++ b/caravel/sql_lab.py
@@ -0,0 +1,121 @@
+import celery
+from datetime import datetime
+import pandas as pd
+import logging
+from caravel import app, db, models, utils
+
+
+celery_app = celery.Celery(config_source=app.config.get('CELERY_CONFIG'))
+
+
+def is_query_select(sql):
+ return sql.upper().startswith('SELECT')
+
+def create_table_as(sql, table_name, override=False):
+ """Reformats the query into the create table as query.
+
+ Works only for the single select SQL statements, in all other cases
+ the sql query is not modified.
+ :param sql: string, sql query that will be executed
+ :param table_name: string, will contain the results of the query execution
+ :param override, boolean, table table_name will be dropped if true
+ :return: string, create table as query
+ """
+ # TODO(bkyryliuk): enforce that all the columns have names. Presto requires it
+ # for the CTA operation.
+ # TODO(bkyryliuk): drop table if allowed, check the namespace and
+ # the permissions.
+ # TODO raise if multi-statement
+ exec_sql = ''
+ if is_query_select(sql):
+ if override:
+ exec_sql = 'DROP TABLE IF EXISTS {};\n'.format(table_name)
+ exec_sql += "CREATE TABLE {table_name} AS {sql}"
+ else:
+ raise Exception("Could not generate CREATE TABLE statement")
+ return exec_sql.format(**locals())
+
+
+@celery_app.task
+def get_sql_results(query_id):
+ """Executes the sql query returns the results."""
+ db.session.commit() # HACK
+ q = db.session.query(models.Query).all()
+ query = db.session.query(models.Query).filter_by(id=query_id).one()
+ database = query.database
+ executed_sql = query.sql.strip().strip(';')
+
+ # Limit enforced only for retrieving the data, not for the CTA queries.
+ if is_query_select(executed_sql):
+ if query.select_as_cta:
+ if not query.tmp_table_name:
+ query.tmp_table_name = 'tmp_{}_table_{}'.format(
+ query.user_id,
+ query.start_time.strftime('%Y_%m_%d_%H_%M_%S'))
+ executed_sql = create_table_as(executed_sql, query.tmp_table_name)
+ elif query.limit:
+ executed_sql = database.wrap_sql_limit(executed_sql, query.limit)
+ engine = database.get_sqla_engine(schema=query.schema)
+ try:
+ query.executed_sql = executed_sql
+ logging.info("Running query: \n{}".format(executed_sql))
+ result_proxy = engine.execute(query.executed_sql, schema=query.schema)
+ except Exception as e:
+ query.error_message = utils.error_msg_from_exception(e)
+ query.status = models.QueryStatus.FAILED
+ query.tmp_table_name = None
+ raise Exception(query.error_message)
+
+ cursor = result_proxy.cursor
+ if hasattr(cursor, "poll"):
+ query_stats = cursor.poll()
+ # poll returns dict -- JSON status information or ``None``
+ # if the query is done
+ # https://github.com/dropbox/PyHive/blob/
+ # b34bdbf51378b3979eaf5eca9e956f06ddc36ca0/pyhive/presto.py#L178
+ while query_stats:
+ # Update the object and wait for the kill signal.
+ completed_splits = float(query_stats['stats']['completedSplits'])
+ total_splits = float(query_stats['stats']['totalSplits'])
+ progress = 100 * completed_splits / total_splits
+ if progress > self._query.progress:
+ query.progress = progress
+
+ db.session.commit()
+ query_stats = cursor.poll()
+ # TODO(b.kyryliuk): check for the kill signal.
+
+ columns = None
+ data = None
+ if result_proxy.cursor:
+ cols = [col[0] for col in result_proxy.cursor.description]
+ data = result_proxy.fetchall()
+ df = pd.DataFrame(data, columns=cols)
+ df = df.fillna(0)
+ columns = [c for c in df.columns]
+ data = df.to_dict(orient='records')
+
+ query.rows = result_proxy.rowcount
+ query.progress = 100
+ query.status = models.QueryStatus.FINISHED
+ if query.rows == -1 and data:
+ # Presto doesn't provide result_proxy.row_count
+ query.rows = len(data)
+
+ # CTAs queries result in 1 cell having the # of the added rows.
+ if query.select_as_cta:
+ query.select_sql = database.select_star(query.tmp_table_name, query.limit)
+
+ query.end_time = datetime.now()
+ db.session.commit()
+
+ payload = {
+ 'query_id': query.id,
+ 'status': query.status,
+ }
+ if query.status == models.QueryStatus.FINISHED:
+ payload['data'] = data
+ payload['columns'] = columns
+ else:
+ payload['error'] = query.error_message
+ return payload
diff --git a/caravel/templates/caravel/basic.html b/caravel/templates/caravel/basic.html
index 07782418edcb..c82a40e1c114 100644
--- a/caravel/templates/caravel/basic.html
+++ b/caravel/templates/caravel/basic.html
@@ -29,7 +29,7 @@
{% block body %}
{% include 'caravel/flash_wrapper.html' %}
- Oops! React.js is not working properly.
+
{% endblock %}
diff --git a/caravel/templates/caravel/paper-theme.html b/caravel/templates/caravel/paper-theme.html
new file mode 100644
index 000000000000..7f00e2469a64
--- /dev/null
+++ b/caravel/templates/caravel/paper-theme.html
@@ -0,0 +1,577 @@
+{% extends "caravel/basic.html" %}
+
+{% block body %}
+
+
+
+
+
+
+
+
+
+
+
+
Theme example
+
This is a template showcasing the optional theme stylesheet included in Bootstrap. Use it as a starting point to create something more unique by building on or modifying it.
+
+
+
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+ Link
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+ Link
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+ Link
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+ Link
+
+
+
+
+
+
+
+
+
+ #
+ First Name
+ Last Name
+ Username
+
+
+
+
+ 1
+ Mark
+ Otto
+ @mdo
+
+
+ 2
+ Jacob
+ Thornton
+ @fat
+
+
+ 3
+ Larry
+ the Bird
+ @twitter
+
+
+
+
+
+
+
+
+ #
+ First Name
+ Last Name
+ Username
+
+
+
+
+ 1
+ Mark
+ Otto
+ @mdo
+
+
+ 2
+ Jacob
+ Thornton
+ @fat
+
+
+ 3
+ Larry
+ the Bird
+ @twitter
+
+
+
+
+
+
+
+
+
+
+
+ #
+ First Name
+ Last Name
+ Username
+
+
+
+
+ 1
+ Mark
+ Otto
+ @mdo
+
+
+ Mark
+ Otto
+ @TwBootstrap
+
+
+ 2
+ Jacob
+ Thornton
+ @fat
+
+
+ 3
+ Larry the Bird
+ @twitter
+
+
+
+
+
+
+
+
+ #
+ First Name
+ Last Name
+ Username
+
+
+
+
+ 1
+ Mark
+ Otto
+ @mdo
+
+
+ 2
+ Jacob
+ Thornton
+ @fat
+
+
+ 3
+ Larry the Bird
+ @twitter
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+
+
+ Default
+ Primary
+ Success
+ Info
+ Warning
+ Danger
+
+
+
+
+
+ 42
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Well done! You successfully read this important alert message. With a
link .
+
+
+ Heads up! This alert needs your attention, but it's not super important.
+
+
+ Warning! Best check yo self, you're not looking too good.
+
+
+ Oh snap! Change a few things up and try submitting again.
+
+
+
+
+
+
+
40% Complete (success)
+
+
+
+
60% Complete (warning)
+
+
+
80% Complete (danger)
+
+
+
+
35% Complete (success)
+
20% Complete (warning)
+
10% Complete (danger)
+
+
+
+
+
+
+
+ Cras justo odio
+ Dapibus ac facilisis in
+ Morbi leo risus
+ Porta ac consectetur ac
+ Vestibulum at eros
+
+
+
+
+
+
+
+
+
+
+
+
+
Panel title
+
+
+ Panel content
+
+
+
+
+
Panel title
+
+
+ Panel content
+
+
+
+
+
+
+
Panel title
+
+
+ Panel content
+
+
+
+
+
Panel title
+
+
+ Panel content
+
+
+
+
+
+
+
Panel title
+
+
+ Panel content
+
+
+
+
+
Panel title
+
+
+ Panel content
+
+
+
+
+
+
+
+
+
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas sed diam eget risus varius blandit sit amet non magna. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent commodo cursus magna, vel scelerisque nisl consectetur et. Cras mattis consectetur purus sit amet fermentum. Duis mollis, est non commodo luctus, nisi erat porttitor ligula, eget lacinia odio sem nec elit. Aenean lacinia bibendum nulla sed consectetur.
+
+
+ This is tabs above here
+
+
+
+
+{% endblock %}
diff --git a/caravel/templates/caravel/sqllab.html b/caravel/templates/caravel/sqllab.html
new file mode 100644
index 000000000000..a3897b0f8e60
--- /dev/null
+++ b/caravel/templates/caravel/sqllab.html
@@ -0,0 +1,6 @@
+{% extends "caravel/basic.html" %}
+
+{% block tail_js %}
+ {{ super() }}
+
+{% endblock %}
diff --git a/caravel/templates/caravel/theme.html b/caravel/templates/caravel/theme.html
index 433e6d502cf8..4a7ea1a3dccf 100644
--- a/caravel/templates/caravel/theme.html
+++ b/caravel/templates/caravel/theme.html
@@ -337,7 +337,6 @@
Button
-
@@ -1213,37 +1212,35 @@ Panel success
-
+
-
Panel danger
+ Panel info
Panel content
-
+
-
Panel info
+ Panel warning
Panel content
-
+
-
Panel warning
+ Panel danger
Panel content
-
-
diff --git a/caravel/utils.py b/caravel/utils.py
index 5921f3f232ec..bd02afa97b2c 100644
--- a/caravel/utils.py
+++ b/caravel/utils.py
@@ -4,7 +4,7 @@
from __future__ import print_function
from __future__ import unicode_literals
-from datetime import datetime, date
+from datetime import date, datetime
import decimal
import functools
import json
@@ -200,11 +200,16 @@ def init(caravel):
perms = db.session.query(ab_models.PermissionView).all()
for perm in perms:
- if perm.permission.name in ('datasource_access', 'database_access'):
+ if (
+ perm.permission and
+ perm.permission.name in ('datasource_access', 'database_access')):
continue
if perm.view_menu and perm.view_menu.name not in (
- 'UserDBModelView', 'RoleModelView', 'ResetPasswordView',
- 'Security'):
+ 'ResetPasswordView',
+ 'RoleModelView',
+ 'Security',
+ 'UserDBModelView',
+ 'SQL Lab'):
sm.add_permission_role(alpha, perm)
sm.add_permission_role(admin, perm)
gamma = sm.add_role("Gamma")
@@ -217,7 +222,9 @@ def init(caravel):
'ResetPasswordView',
'RoleModelView',
'UserDBModelView',
+ 'SQL Lab',
'Security') and
+ perm.permission and
perm.permission.name not in (
'all_datasource_access',
'can_add',
@@ -304,6 +311,8 @@ def json_iso_dttm_ser(obj):
return val
if isinstance(obj, datetime):
obj = obj.isoformat()
+ elif isinstance(obj, date):
+ obj = obj.isoformat()
else:
raise TypeError(
"Unserializable object {} of type {}".format(obj, type(obj))
@@ -329,13 +338,21 @@ def json_int_dttm_ser(obj):
def error_msg_from_exception(e):
"""Translate exception into error message
+
Database have different ways to handle exception. This function attempts
to make sense of the exception object and construct a human readable
sentence.
+
+ TODO(bkyryliuk): parse the Presto error message from the connection
+ created via create_engine.
+ engine = create_engine('presto://localhost:3506/silver') -
+ gives an e.message as the str(dict)
+ presto.connect("localhost", port=3506, catalog='silver') - as a dict.
+ The latter version is parsed correctly by this function.
"""
msg = ''
if hasattr(e, 'message'):
- if (type(e.message) is dict):
+ if type(e.message) is dict:
msg = e.message.get('message')
elif e.message:
msg = "{}".format(e.message)
diff --git a/caravel/views.py b/caravel/views.py
index 629136a890c5..aa1c0e3211fd 100755
--- a/caravel/views.py
+++ b/caravel/views.py
@@ -10,7 +10,7 @@
import sys
import time
import traceback
-from datetime import datetime
+from datetime import datetime, timedelta
import functools
import pandas as pd
@@ -32,7 +32,9 @@
from wtforms.validators import ValidationError
import caravel
-from caravel import appbuilder, db, models, viz, utils, app, sm, ascii_art
+from caravel import (
+ appbuilder, db, models, viz, utils, app, sm, ascii_art, sql_lab
+)
config = app.config
log_this = models.Log.log_this
@@ -452,6 +454,18 @@ def pre_update(self, db):
category_icon='fa-database',)
+class DatabaseAsync(DatabaseView):
+ list_columns = ['id', 'database_name']
+
+appbuilder.add_view_no_menu(DatabaseAsync)
+
+
+class DatabaseTablesAsync(DatabaseView):
+ list_columns = ['id', 'all_table_names', 'all_schema_names']
+
+appbuilder.add_view_no_menu(DatabaseTablesAsync)
+
+
class TableModelView(CaravelModelView, DeleteMixin): # noqa
datamodel = SQLAInterface(models.SqlaTable)
list_columns = [
@@ -463,7 +477,7 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa
'table_name', 'database', 'schema',
'default_endpoint', 'offset', 'cache_timeout']
edit_columns = [
- 'table_name', 'is_featured', 'database', 'schema',
+ 'table_name', 'sql', 'is_featured', 'database', 'schema',
'description', 'owner',
'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
related_views = [TableColumnInlineView, SqlMetricInlineView]
@@ -476,6 +490,10 @@ class TableModelView(CaravelModelView, DeleteMixin): # noqa
'description': Markup(
"Supports
"
"markdown "),
+ 'sql': (
+ "This fields acts a Caravel view, meaning that Caravel will "
+ "run a query against this string as a subquery."
+ ),
}
base_filters = [['id', TableSlice, lambda: []]]
label_columns = {
@@ -610,7 +628,8 @@ def add(self):
url = "/druiddatasourcemodelview/list/"
msg = _(
"Click on a datasource link to create a Slice, "
- "or click on a table link
here "
+ "or click on a table link "
+ "
here "
"to create a Slice for a table"
)
else:
@@ -728,7 +747,7 @@ def pre_delete(self, obj):
class DashboardModelViewAsync(DashboardModelView): # noqa
- list_columns = ['dashboard_link', 'creator', 'modified']
+ list_columns = ['dashboard_link', 'creator', 'modified', 'dashboard_title']
label_columns = {
'dashboard_link': 'Dashboard',
}
@@ -892,7 +911,8 @@ def explore(self, datasource_type, datasource_id):
datasource_access = self.can_access(
'datasource_access', datasource.perm)
if not (all_datasource_access or datasource_access):
- flash(__("You don't seem to have access to this datasource"), "danger")
+ flash(__("You don't seem to have access to this datasource"),
+ "danger")
return redirect(error_redirect)
action = request.args.get('action')
@@ -911,7 +931,7 @@ def explore(self, datasource_type, datasource_id):
form_data=request.args,
slice_=slc)
except Exception as e:
- flash(str(e), "danger")
+ flash(utils.error_msg_from_exception(e), "danger")
return redirect(error_redirect)
if request.args.get("json") == "true":
status = 200
@@ -923,7 +943,7 @@ def explore(self, datasource_type, datasource_id):
payload = obj.get_json()
except Exception as e:
logging.exception(e)
- payload = str(e)
+ payload = utils.error_msg_from_exception(e)
status = 500
resp = Response(
payload,
@@ -953,7 +973,7 @@ def explore(self, datasource_type, datasource_id):
if config.get("DEBUG"):
raise(e)
return Response(
- str(e),
+ utils.error_msg_from_exception(e),
status=500,
mimetype="application/json")
return resp
@@ -969,7 +989,8 @@ def save_or_overwrite_slice(
del d['action']
del d['previous_viz_type']
- as_list = ('metrics', 'groupby', 'columns', 'all_columns', 'mapbox_label', 'order_by_cols')
+ as_list = ('metrics', 'groupby', 'columns', 'all_columns',
+ 'mapbox_label', 'order_by_cols')
for k in d:
v = d.get(k)
if k in as_list and not isinstance(v, list):
@@ -1080,9 +1101,29 @@ def activity_per_day(self):
.group_by(Log.dt)
.all()
)
- payload = {str(time.mktime(dt.timetuple())): ccount for dt, ccount in qry if dt}
+ payload = {str(time.mktime(dt.timetuple())):
+ ccount for dt, ccount in qry if dt}
return Response(json.dumps(payload), mimetype="application/json")
+ @api
+ @has_access_api
+ @expose("/tables/
/")
+ def tables(self, db_id, schema):
+ """endpoint to power the calendar heatmap on the welcome page"""
+ schema = None if schema in ('null', 'undefined') else schema
+ database = (
+ db.session
+ .query(models.Database)
+ .filter_by(id=db_id)
+ .one()
+ )
+ payload = {
+ 'tables': database.all_table_names(schema),
+ 'views': database.all_view_names(schema),
+ }
+ return Response(
+ json.dumps(payload), mimetype="application/json")
+
@api
@has_access_api
@expose("/save_dash//", methods=['GET', 'POST'])
@@ -1117,9 +1158,11 @@ def add_slices(self, dashboard_id):
data = json.loads(request.form.get('data'))
session = db.session()
Slice = models.Slice # noqa
- dash = session.query(models.Dashboard).filter_by(id=dashboard_id).first()
+ dash = (
+ session.query(models.Dashboard).filter_by(id=dashboard_id).first())
check_ownership(dash, raise_if_false=True)
- new_slices = session.query(Slice).filter(Slice.id.in_(data['slice_ids']))
+ new_slices = session.query(Slice).filter(
+ Slice.id.in_(data['slice_ids']))
dash.slices += new_slices
session.merge(dash)
session.commit()
@@ -1153,13 +1196,18 @@ def favstar(self, class_name, obj_id, action):
FavStar = models.FavStar # noqa
count = 0
favs = session.query(FavStar).filter_by(
- class_name=class_name, obj_id=obj_id, user_id=g.user.get_id()).all()
+ class_name=class_name, obj_id=obj_id,
+ user_id=g.user.get_id()).all()
if action == 'select':
if not favs:
session.add(
FavStar(
- class_name=class_name, obj_id=obj_id, user_id=g.user.get_id(),
- dttm=datetime.now()))
+ class_name=class_name,
+ obj_id=obj_id,
+ user_id=g.user.get_id(),
+ dttm=datetime.now()
+ )
+ )
count = 1
elif action == 'unselect':
for fav in favs:
@@ -1214,10 +1262,55 @@ def dashboard(**kwargs): # noqa
dash_save_perm=dash_save_perm,
dash_edit_perm=dash_edit_perm)
+ @has_access
+ @expose("/sqllab_viz/")
+ @log_this
+ def sqllab_viz(self):
+ data = json.loads(request.args.get('data'))
+ table_name = data.get('datasourceName')
+ table = db.session.query(models.SqlaTable).filter_by(table_name=table_name).first()
+ if not table:
+ table = models.SqlaTable(
+ table_name=table_name,
+ )
+ table.database_id = data.get('databaseId')
+ table.sql = data.get('sql')
+ db.session.add(table)
+ cols = []
+ metrics = []
+ for column_name, config in data.get('columns').items():
+ is_dim = config.get('is_dim', False)
+ cols.append(models.TableColumn(
+ column_name=column_name,
+ filterable=is_dim,
+ groupby=is_dim,
+ ))
+ agg = config.get('agg')
+ if agg:
+ metrics.append(models.SqlMetric(
+ metric_name="{agg}__{column_name}".format(**locals()),
+ expression="{agg}({column_name})".format(**locals()),
+ ))
+ metrics.append(models.SqlMetric(
+ metric_name="count".format(**locals()),
+ expression="count(*)".format(**locals()),
+ ))
+ table.columns = cols
+ table.metrics = metrics
+ db.session.commit()
+ return redirect('/caravel/explore/table/{table.id}/'.format(**locals()))
+
@has_access
@expose("/sql//")
@log_this
def sql(self, database_id):
+ if (
+ not self.can_access(
+ 'all_datasource_access', 'all_datasource_access')):
+ flash(
+ "SQL Lab requires the `all_datasource_access` "
+ "permission", "danger")
+ return redirect("/tablemodelview/list/")
mydb = db.session.query(
models.Database).filter_by(id=database_id).first()
@@ -1240,23 +1333,35 @@ def sql(self, database_id):
db=mydb)
@has_access
- @expose("/table///")
+ @expose("/table////")
@log_this
- def table(self, database_id, table_name):
- mydb = db.session.query(
- models.Database).filter_by(id=database_id).first()
- cols = mydb.get_columns(table_name)
- df = pd.DataFrame([(c['name'], c['type']) for c in cols])
- df.columns = ['col', 'type']
- tbl_cls = (
- "dataframe table table-striped table-bordered "
- "table-condensed sql_results").split(' ')
- return self.render_template(
- "caravel/ajah.html",
- content=df.to_html(
- index=False,
- na_rep='',
- classes=tbl_cls))
+ def table(self, database_id, table_name, schema):
+ schema = None if schema in ('null', 'undefined') else schema
+ mydb = db.session.query(models.Database).filter_by(id=database_id).one()
+ cols = []
+ t = mydb.get_columns(table_name, schema)
+ try:
+ t = mydb.get_columns(table_name, schema)
+ except Exception as e:
+ return Response(
+ json.dumps({'error': utils.error_msg_from_exception(e)}),
+ mimetype="application/json")
+ for col in t:
+ dtype = ""
+ try:
+ dtype = '{}'.format(col['type'])
+ except:
+ pass
+ cols.append({
+ 'name': col['name'],
+ 'type': dtype.split('(')[0] if '(' in dtype else dtype,
+ 'longType': dtype,
+ })
+ tbl = {
+ 'name': table_name,
+ 'columns': cols,
+ }
+ return Response(json.dumps(tbl), mimetype="application/json")
@has_access
@expose("/select_star///")
@@ -1285,55 +1390,6 @@ def select_star(self, database_id, table_name):
content=s
)
- @has_access
- @expose("/runsql/", methods=['POST', 'GET'])
- @log_this
- def runsql(self):
- """Runs arbitrary sql and returns and html table"""
- # TODO deprecate in favor on `sql_json`
- session = db.session()
- limit = 1000
- data = json.loads(request.form.get('data'))
- sql = data.get('sql')
- database_id = data.get('database_id')
- mydb = session.query(models.Database).filter_by(id=database_id).first()
-
- if not (self.can_access(
- 'all_datasource_access', 'all_datasource_access') or
- self.can_access('database_access', mydb.perm)):
- raise utils.CaravelSecurityException(_(
- "SQL Lab requires the `all_datasource_access` or "
- "specific db permission"))
-
- content = ""
- if mydb:
- eng = mydb.get_sqla_engine()
- if limit:
- sql = sql.strip().strip(';')
- qry = (
- select('*')
- .select_from(TextAsFrom(text(sql), ['*'])
- .alias('inner_qry'))
- .limit(limit)
- )
- sql = '{}'.format(qry.compile(
- eng, compile_kwargs={"literal_binds": True}))
- try:
- df = pd.read_sql_query(sql=sql, con=eng)
- content = df.to_html(
- index=False,
- na_rep='',
- classes=(
- "dataframe table table-striped table-bordered "
- "table-condensed sql_results").split(' '))
- except Exception as e:
- content = (
- ''
- "{}
"
- ).format(e.message)
- session.commit()
- return content
-
@expose("/theme/")
def theme(self):
return self.render_template('caravel/theme.html')
@@ -1343,56 +1399,137 @@ def theme(self):
@log_this
def sql_json(self):
"""Runs arbitrary sql and returns and json"""
- session = db.session()
- limit = 1000
+ async = request.form.get('async') == 'true'
sql = request.form.get('sql')
database_id = request.form.get('database_id')
- mydb = session.query(models.Database).filter_by(id=database_id).first()
- if not (self.can_access(
- 'all_datasource_access', 'all_datasource_access') or
- self.can_access('database_access', mydb.perm)):
- raise utils.CaravelSecurityException(_(
- "SQL Lab requires the `all_datasource_access` or "
- "specific DB permission"))
+ def json_error_response(msg, status=None):
+ return Response(json.dumps({
+ 'error': msg,
+ 'status': status,
+ }),
+ status=500,
+ mimetype="application/json"
+ )
+
+ session = db.session()
+ mydb = session.query(models.Database).filter_by(id=database_id).first()
- error_msg = ""
if not mydb:
- error_msg = "The database selected doesn't seem to exist"
- else:
- eng = mydb.get_sqla_engine()
- if limit:
- sql = sql.strip().strip(';')
- qry = (
- select('*')
- .select_from(TextAsFrom(text(sql), ['*'])
- .alias('inner_qry'))
- .limit(limit)
- )
- sql = '{}'.format(qry.compile(
- eng, compile_kwargs={"literal_binds": True}))
- try:
- df = pd.read_sql_query(sql=sql, con=eng)
- df = df.fillna(0) # TODO make sure NULL
- except Exception as e:
- logging.exception(e)
- error_msg = utils.error_msg_from_exception(e)
+ json_error_response(
+ 'Database with id {} is missing.'.format(database_id),
+ models.QueryStatus.FAILED)
+ if not (self.can_access('all_datasource_access', 'all_datasource_access') or
+ self.can_access('database_access', mydb.perm)):
+ json_error_response(__(
+ "SQL Lab requires the `all_datasource_access` or specific DB permission"))
+ session.commit()
+
+ query = models.Query(
+ database_id=int(database_id),
+ limit=int(app.config.get('SQL_MAX_ROW', None)),
+ sql=sql,
+ schema=request.form.get('schema'),
+ select_as_cta=request.form.get('select_as_cta') == 'true',
+ start_time=datetime.now(),
+ status=models.QueryStatus.IN_PROGRESS,
+ tab_name=request.form.get('tab'),
+ sql_editor_id=request.form.get('sql_editor_id'),
+ tmp_table_name=request.form.get('tmp_table_name'),
+ user_id=int(g.user.get_id()),
+ client_id=request.form.get('client_id'),
+ )
+ session.add(query)
session.commit()
- if error_msg:
+ session.flush()
+ query_id = query.id
+
+ # Async request.
+ if async:
+ # Ignore the celery future object and the request may time out.
+ sql_lab.get_sql_results.delay(query_id)
return Response(
- json.dumps({
- 'error': error_msg,
- }),
+ json.dumps({'query': query.to_dict()},
+ default=utils.json_int_dttm_ser,
+ allow_nan=False),
+ status=202, # Accepted
+ mimetype="application/json")
+
+ # Sync request.
+ try:
+ data = sql_lab.get_sql_results(query_id)
+ except Exception as e:
+ return Response(
+ json.dumps({'error': "{}".format(e)}),
status=500,
+ mimetype="application/json"
+ )
+
+ data['query'] = query.to_dict()
+
+ return Response(
+ json.dumps(data, default=utils.json_int_dttm_ser, allow_nan=False),
+ status=200,
+ mimetype="application/json")
+
+ @has_access
+ @expose("/csv/")
+ @log_this
+ def csv(self, query_id):
+ """Download the query results as csv."""
+ s = db.session()
+ query = s.query(models.Query).filter_by(id=int(query_id)).first()
+
+ if not (self.can_access('all_datasource_access', 'all_datasource_access') or
+ self.can_access('database_access', query.database.perm)):
+ flash(_(
+ "SQL Lab requires the `all_datasource_access` or specific DB permission"))
+ redirect('/')
+
+ sql = query.select_sql or query.sql
+ df = query.database.get_df(sql, query.schema)
+ # TODO(bkyryliuk): add compression=gzip for big files.
+ csv = df.to_csv(index=False)
+ response = Response(csv, mimetype='text/csv')
+ response.headers['Content-Disposition'] = (
+ 'attachment; filename={}.csv'.format(query.name))
+ return response
+
+ @has_access
+ @expose("/queries/")
+ @log_this
+ def queries(self, last_updated_ms):
+ """Get the updated queries."""
+ if not g.user.get_id():
+ return Response(
+ json.dumps({'error': "Please login to access the queries."}),
+ status=403,
mimetype="application/json")
- else:
- data = {
- 'columns': [c for c in df.columns],
- 'data': df.to_dict(orient='records'),
- }
- return json.dumps(
- data, default=utils.json_int_dttm_ser, allow_nan=False)
+
+ # Unix time, milliseconds.
+ last_updated_ms_int = int(last_updated_ms) if last_updated_ms else 0
+
+ # Local date time, DO NOT USE IT.
+ # last_updated_dt = datetime.fromtimestamp(int(last_updated_ms) / 1000)
+
+ # UTC date time, same that is stored in the DB.
+ last_updated_dt = utils.EPOCH + timedelta(
+ seconds=last_updated_ms_int / 1000)
+
+ sql_queries = (
+ db.session.query(models.Query)
+ .filter(
+ models.Query.user_id == g.user.get_id() or
+ models.Query.changed_on >= last_updated_dt
+ )
+ .all()
+ )
+ dict_queries = {q.client_id: q.to_dict() for q in sql_queries}
+ return Response(
+ json.dumps(dict_queries, default=utils.json_int_dttm_ser),
+ status=200,
+ mimetype="application/json")
@has_access
@expose("/refresh_datasources/")
@@ -1433,6 +1570,11 @@ def welcome(self):
"""Personalized welcome page"""
return self.render_template('caravel/welcome.html', utils=utils)
+ @has_access
+ @expose("/sqllab")
+ def sqlanvil(self):
+ """SQL Editor"""
+ return self.render_template('caravel/sqllab.html')
appbuilder.add_view_no_menu(Caravel)
@@ -1462,6 +1604,11 @@ class CssTemplateModelView(CaravelModelView, DeleteMixin):
category_label=__("Sources"),
category_icon='')
+appbuilder.add_link(
+ "SQL Lab",
+ href='/caravel/sqllab',
+ icon="fa-flask")
+
# ---------------------------------------------------------------------
# Redirecting URL from previous names
diff --git a/caravel/viz.py b/caravel/viz.py
index 5413abb859bf..1161da553d22 100755
--- a/caravel/viz.py
+++ b/caravel/viz.py
@@ -285,7 +285,8 @@ def get_json(self):
cached_data = cached_data.decode('utf-8')
payload = json.loads(cached_data)
except Exception as e:
- logging.error("Error reading cache")
+ logging.error("Error reading cache: " +
+ utils.error_msg_from_exception(e))
payload = None
logging.info("Serving from cache")
diff --git a/docs/installation.rst b/docs/installation.rst
index a2bffb4d1ca8..c70cc0082ead 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -293,3 +293,19 @@ Upgrading should be as straightforward as running::
pip install caravel --upgrade
caravel db upgrade
+
+
+Making your own build
+---------------------
+
+For more advanced users, you may want to build Caravel from sources. That
+would be the case if you fork the project to add features specific to
+your environment.::
+
+ # assuming $CARAVEL_HOME as the root of the repo
+ cd $CARAVEL_HOME/caravel/assets
+ npm install
+ npm run prod
+ cd $CARAVEL_HOME
+ python setup.py install
+
diff --git a/run_tests.sh b/run_tests.sh
index 189791857651..5f2c0de02739 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,6 +1,8 @@
#!/usr/bin/env bash
echo $DB
rm ~/.caravel/caravel_unittests.db
+rm ~/.caravel/celerydb.sqlite
+rm ~/.caravel/celery_results.sqlite
rm -f .coverage
export CARAVEL_CONFIG=tests.caravel_test_config
set -e
diff --git a/setup.py b/setup.py
index afbed2dfbe48..fd3f6a987ebd 100644
--- a/setup.py
+++ b/setup.py
@@ -16,6 +16,7 @@
zip_safe=False,
scripts=['caravel/bin/caravel'],
install_requires=[
+ 'celery==3.1.23',
'cryptography==1.4',
'flask-appbuilder==1.8.1',
'flask-cache==0.13.1',
@@ -29,6 +30,7 @@
'pandas==0.18.1',
'parsedatetime==2.0.0',
'pydruid==0.3.0',
+ 'PyHive>=0.2.1',
'python-dateutil==2.5.3',
'requests==2.10.0',
'simplejson==3.8.2',
@@ -36,6 +38,8 @@
'sqlalchemy==1.0.13',
'sqlalchemy-utils==0.32.7',
'sqlparse==0.1.19',
+ 'thrift>=0.9.3',
+ 'thrift-sasl>=0.2.1',
'werkzeug==0.11.10',
],
extras_require={
diff --git a/tests/caravel_test_config.py b/tests/caravel_test_config.py
index 1720ec837ba0..1c24ee640bb8 100644
--- a/tests/caravel_test_config.py
+++ b/tests/caravel_test_config.py
@@ -9,3 +9,17 @@
# continuous integration
if 'CARAVEL__SQLALCHEMY_DATABASE_URI' in os.environ:
SQLALCHEMY_DATABASE_URI = os.environ.get('CARAVEL__SQLALCHEMY_DATABASE_URI')
+
+SQL_CELERY_DB_FILE_PATH = os.path.join(DATA_DIR, 'celerydb.sqlite')
+SQL_CELERY_RESULTS_DB_FILE_PATH = os.path.join(DATA_DIR, 'celery_results.sqlite')
+SQL_SELECT_AS_CTA = True
+SQL_MAX_ROW = 666
+
+
+class CeleryConfig(object):
+ BROKER_URL = 'sqla+sqlite:///' + SQL_CELERY_DB_FILE_PATH
+ CELERY_IMPORTS = ('caravel.sql_lab', )
+ CELERY_RESULT_BACKEND = 'db+sqlite:///' + SQL_CELERY_RESULTS_DB_FILE_PATH
+ CELERY_ANNOTATIONS = {'sql_lab.add': {'rate_limit': '10/s'}}
+ CONCURRENCY = 1
+CELERY_CONFIG = CeleryConfig
diff --git a/tests/celery_tests.py b/tests/celery_tests.py
new file mode 100644
index 000000000000..c36f7e749cab
--- /dev/null
+++ b/tests/celery_tests.py
@@ -0,0 +1,429 @@
+"""Unit tests for Caravel Celery worker"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import imp
+import json
+import subprocess
+
+import time
+
+import os
+
+import pandas as pd
+import unittest
+
+import caravel
+from caravel import app, appbuilder, db, models, sql_lab, utils
+
+
+BASE_DIR = app.config.get('BASE_DIR')
+cli = imp.load_source('cli', BASE_DIR + '/bin/caravel')
+
+
+SQL_CELERY_DB_FILE_PATH = '/tmp/celerydb.sqlite'
+SQL_CELERY_RESULTS_DB_FILE_PATH = '/tmp/celery_results.sqlite'
+
+
+class CeleryConfig(object):
+ BROKER_URL = 'sqla+sqlite:///' + SQL_CELERY_DB_FILE_PATH
+ CELERY_IMPORTS = ('caravel.sql_lab', )
+ CELERY_RESULT_BACKEND = 'db+sqlite:///' + SQL_CELERY_RESULTS_DB_FILE_PATH
+ CELERY_ANNOTATIONS = {'sql_lab.add': {'rate_limit': '10/s'}}
+ CONCURRENCY = 1
+app.config['CELERY_CONFIG'] = CeleryConfig
+
+# TODO(bkyryliuk): add ability to run this test separately.
+
+
+class UtilityFunctionTests(unittest.TestCase):
+ # TODO(bkyryliuk): support more cases in CTA function.
+ def test_create_table_as(self):
+ select_query = "SELECT * FROM outer_space;"
+ updated_select_query = sql_lab.create_table_as(
+ select_query, "tmp")
+ self.assertEqual(
+ "CREATE TABLE tmp AS SELECT * FROM outer_space;",
+ updated_select_query)
+
+ updated_select_query_with_drop = sql_lab.create_table_as(
+ select_query, "tmp", override=True)
+ self.assertEqual(
+ "DROP TABLE IF EXISTS tmp;\n"
+ "CREATE TABLE tmp AS SELECT * FROM outer_space;",
+ updated_select_query_with_drop)
+
+ select_query_no_semicolon = "SELECT * FROM outer_space"
+ updated_select_query_no_semicolon = sql_lab.create_table_as(
+ select_query_no_semicolon, "tmp")
+ self.assertEqual(
+ "CREATE TABLE tmp AS SELECT * FROM outer_space",
+ updated_select_query_no_semicolon)
+
+ # incorrect_query = "SMTH WRONG SELECT * FROM outer_space"
+ # updated_incorrect_query = sql_lab.create_table_as(
+ # incorrect_query, "tmp")
+ # self.assertEqual(incorrect_query, updated_incorrect_query)
+ #
+ # insert_query = "INSERT INTO stomach VALUES (beer, chips);"
+ # updated_insert_query = sql_lab.create_table_as(
+ # insert_query, "tmp")
+ # self.assertEqual(insert_query, updated_insert_query)
+
+ multi_line_query = (
+ "SELECT * FROM planets WHERE\n"
+ "Luke_Father = 'Darth Vader';")
+ updated_multi_line_query = sql_lab.create_table_as(
+ multi_line_query, "tmp")
+ expected_updated_multi_line_query = (
+ "CREATE TABLE tmp AS SELECT * FROM planets WHERE\n"
+ "Luke_Father = 'Darth Vader';")
+ self.assertEqual(
+ expected_updated_multi_line_query,
+ updated_multi_line_query)
+
+ # updated_multi_line_query_with_drop = sql_lab.create_table_as(
+ # multi_line_query, "tmp", override=True)
+ # expected_updated_multi_line_query_with_drop = (
+ # "DROP TABLE IF EXISTS tmp;\n"
+ # "CREATE TABLE tmp AS SELECT * FROM planets WHERE\n"
+ # "Luke_Father = 'Darth Vader';")
+ # self.assertEqual(
+ # expected_updated_multi_line_query_with_drop,
+ # updated_multi_line_query_with_drop)
+ #
+ # delete_query = "DELETE FROM planet WHERE name = 'Earth'"
+ # updated_delete_query = sql_lab.create_table_as(delete_query, "tmp")
+ # self.assertEqual(delete_query, updated_delete_query)
+ #
+ # create_table_as = (
+ # "CREATE TABLE pleasure AS SELECT chocolate FROM lindt_store;\n")
+ # updated_create_table_as = sql_lab.create_table_as(
+ # create_table_as, "tmp")
+ # self.assertEqual(create_table_as, updated_create_table_as)
+ #
+ # sql_procedure = (
+ # "CREATE PROCEDURE MyMarriage\n "
+ # "BrideGroom Male (25) ,\n "
+ # "Bride Female(20) AS\n "
+ # "BEGIN\n "
+ # "SELECT Bride FROM ukraine_ Brides\n "
+ # "WHERE\n "
+ # "FatherInLaw = 'Millionaire' AND Count(Car) > 20\n"
+ # " AND HouseStatus ='ThreeStoreyed'\n"
+ # " AND BrideEduStatus IN "
+ # "(B.TECH ,BE ,Degree ,MCA ,MiBA)\n "
+ # "AND Having Brothers= Null AND Sisters =Null"
+ # )
+ # updated_sql_procedure = sql_lab.create_table_as(sql_procedure, "tmp")
+ # self.assertEqual(sql_procedure, updated_sql_procedure)
+ #
+ # multiple_statements = """
+ # DROP HUSBAND;
+ # SELECT * FROM politicians WHERE clue > 0;
+ # INSERT INTO MyCarShed VALUES('BUGATTI');
+ # SELECT standard_disclaimer, witty_remark FROM company_requirements;
+ # select count(*) from developer_brain;
+ # """
+ # updated_multiple_statements = sql_lab.create_table_as(
+ # multiple_statements, "tmp")
+ # self.assertEqual(multiple_statements, updated_multiple_statements)
+
+
+class CeleryTestCase(unittest.TestCase):
+ def __init__(self, *args, **kwargs):
+ super(CeleryTestCase, self).__init__(*args, **kwargs)
+ self.client = app.test_client()
+
+ def get_query_by_name(self, sql):
+ session = db.create_scoped_session()
+ query = session.query(models.Query).filter_by(sql=sql).first()
+ session.close()
+ return query
+
+ def get_query_by_id(self, id):
+ session = db.create_scoped_session()
+ query = session.query(models.Query).filter_by(id=id).first()
+ session.close()
+ return query
+
+ @classmethod
+ def setUpClass(cls):
+ try:
+ os.remove(app.config.get('SQL_CELERY_DB_FILE_PATH'))
+ except OSError as e:
+ app.logger.warn(str(e))
+ try:
+ os.remove(app.config.get('SQL_CELERY_RESULTS_DB_FILE_PATH'))
+ except OSError as e:
+ app.logger.warn(str(e))
+
+ utils.init(caravel)
+
+ worker_command = BASE_DIR + '/bin/caravel worker'
+ subprocess.Popen(
+ worker_command, shell=True, stdout=subprocess.PIPE)
+
+ admin = appbuilder.sm.find_user('admin')
+ if not admin:
+ appbuilder.sm.add_user(
+ 'admin', 'admin', ' user', 'admin@fab.org',
+ appbuilder.sm.find_role('Admin'),
+ password='general')
+ cli.load_examples(load_test_data=True)
+
+
+ @classmethod
+ def tearDownClass(cls):
+ subprocess.call(
+ "ps auxww | grep 'celeryd' | awk '{print $2}' | xargs kill -9",
+ shell=True
+ )
+ subprocess.call(
+ "ps auxww | grep 'caravel worker' | awk '{print $2}' | "
+ "xargs kill -9",
+ shell=True
+ )
+
+ def setUp(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+ def login(self, username='admin', password='general'):
+ resp = self.client.post(
+ '/login/',
+ data=dict(username=username, password=password),
+ follow_redirects=True)
+ assert 'Welcome' in resp.data.decode('utf-8')
+
+ def logout(self):
+ self.client.get('/logout/', follow_redirects=True)
+
+ def run_sql(self, dbid, sql, cta='false', tmp_table='tmp',
+ async='false'):
+ self.login()
+ resp = self.client.post(
+ '/caravel/sql_json/',
+ data=dict(
+ database_id=dbid,
+ sql=sql,
+ async=async,
+ select_as_cta=cta,
+ tmp_table_name=tmp_table,
+ client_id="not_used",
+ ),
+ )
+ self.logout()
+ return json.loads(resp.data.decode('utf-8'))
+
+ def test_add_limit_to_the_query(self):
+ query_session = models.Database.create_scoped_session()
+ db_to_query = query_session.query(models.Database).filter_by(
+ id=1).first()
+ eng = db_to_query.get_sqla_engine()
+
+ select_query = "SELECT * FROM outer_space;"
+ updated_select_query = db_to_query.wrap_sql_limit(select_query, 100, eng)
+ # Different DB engines have their own spacing while compiling
+ # the queries, that's why ' '.join(query.split()) is used.
+ # In addition some of the engines do not include OFFSET 0.
+ self.assertTrue(
+ "SELECT * FROM (SELECT * FROM outer_space;) AS inner_qry "
+ "LIMIT 100" in ' '.join(updated_select_query.split())
+ )
+
+ select_query_no_semicolon = "SELECT * FROM outer_space"
+ updated_select_query_no_semicolon = db_to_query.wrap_sql_limit(
+ select_query_no_semicolon, 100, eng)
+ self.assertTrue(
+ "SELECT * FROM (SELECT * FROM outer_space) AS inner_qry "
+ "LIMIT 100" in
+ ' '.join(updated_select_query_no_semicolon.split())
+ )
+
+ incorrect_query = "SMTH WRONG SELECT * FROM outer_space"
+ updated_incorrect_query = db_to_query.wrap_sql_limit(incorrect_query, 100, eng)
+ self.assertEqual(incorrect_query, updated_incorrect_query)
+
+ insert_query = "INSERT INTO stomach VALUES (beer, chips);"
+ updated_insert_query = db_to_query.wrap_sql_limit(insert_query, 100, eng)
+ self.assertEqual(insert_query, updated_insert_query)
+
+ multi_line_query = (
+ "SELECT * FROM planets WHERE\n Luke_Father = 'Darth Vader';"
+ )
+ updated_multi_line_query = db_to_query.wrap_sql_limit(multi_line_query, 100, eng)
+ self.assertTrue(
+ "SELECT * FROM (SELECT * FROM planets WHERE "
+ "Luke_Father = 'Darth Vader';) AS inner_qry LIMIT 100" in
+ ' '.join(updated_multi_line_query.split())
+ )
+
+ delete_query = "DELETE FROM planet WHERE name = 'Earth'"
+ updated_delete_query = db_to_query.wrap_sql_limit(delete_query, 100, eng)
+ self.assertEqual(delete_query, updated_delete_query)
+
+ create_table_as = (
+ "CREATE TABLE pleasure AS SELECT chocolate FROM lindt_store;\n")
+ updated_create_table_as = db_to_query.wrap_sql_limit(create_table_as, 100, eng)
+ self.assertEqual(create_table_as, updated_create_table_as)
+
+ sql_procedure = (
+ "CREATE PROCEDURE MyMarriage\n "
+ "BrideGroom Male (25) ,\n "
+ "Bride Female(20) AS\n "
+ "BEGIN\n "
+ "SELECT Bride FROM ukraine_ Brides\n "
+ "WHERE\n "
+ "FatherInLaw = 'Millionaire' AND Count(Car) > 20\n"
+ " AND HouseStatus ='ThreeStoreyed'\n"
+ " AND BrideEduStatus IN "
+ "(B.TECH ,BE ,Degree ,MCA ,MiBA)\n "
+ "AND Having Brothers= Null AND Sisters = Null"
+ )
+ updated_sql_procedure = db_to_query.wrap_sql_limit(sql_procedure, 100, eng)
+ self.assertEqual(sql_procedure, updated_sql_procedure)
+
+ def test_run_sync_query(self):
+ main_db = db.session.query(models.Database).filter_by(
+ database_name="main").first()
+ eng = main_db.get_sqla_engine()
+
+ # Case 1.
+ # DB #0 doesn't exist.
+ sql_dont_exist = 'SELECT * FROM dontexist'
+ result1 = self.run_sql(0, sql_dont_exist, cta='true')
+ self.assertFalse('query' in result1)
+ self.assertEqual("'NoneType' object has no attribute 'get_sqla_engine'",
+ result1['error'])
+ self.assertIsNone(self.get_query_by_name(sql_dont_exist))
+
+ # Case 2.
+ # Table doesn't exist.
+ result2 = self.run_sql(1, sql_dont_exist, cta='true', )
+ self.assertTrue('error' in result2)
+ self.assertEqual(models.QueryStatus.FAILED.lower(), result2['query']['status'])
+ query2 = self.get_query_by_id(result2['query']['serverId'])
+ self.assertEqual(models.QueryStatus.FAILED, query2.status)
+
+ # Case 3.
+ # Table and DB exists, CTA call to the backend.
+ sql_where = "SELECT name FROM ab_permission WHERE name='can_sql'"
+ result3 = self.run_sql(
+ 1, sql_where, tmp_table='tmp_table_3', cta='true')
+ self.assertEqual(models.QueryStatus.FINISHED.lower(), result3['query']['status'])
+ self.assertIsNone(result3['data'])
+ self.assertIsNone(result3['columns'])
+ query3 = self.get_query_by_id(result3['query']['serverId'])
+
+ # Check the data in the tmp table.
+ df3 = pd.read_sql_query(sql=query3.select_sql, con=eng)
+ data3 = df3.to_dict(orient='records')
+ self.assertEqual([{'name': 'can_sql'}], data3)
+
+ # Case 4.
+ # Table and DB exists, CTA call to the backend, no data.
+ sql_empty_result = 'SELECT * FROM ab_user WHERE id=666'
+ result4 = self.run_sql(
+ 1, sql_empty_result, tmp_table='tmp_table_4', cta='true',)
+ self.assertEqual(models.QueryStatus.FINISHED.lower(), result4['query']['status'])
+ self.assertIsNone(result4['data'])
+ self.assertIsNone(result4['columns'])
+
+ query4 = self.get_query_by_id(result4['query']['serverId'])
+ self.assertEqual(models.QueryStatus.FINISHED, query4.status)
+ self.assertTrue("SELECT * \nFROM tmp_table_4" in query4.select_sql)
+ self.assertTrue("LIMIT 666" in query4.select_sql)
+ self.assertEqual(
+ "CREATE TABLE tmp_table_4 AS SELECT * FROM ab_user WHERE id=666",
+ query4.executed_sql)
+ self.assertEqual("SELECT * FROM ab_user WHERE id=666", query4.sql)
+ if eng.name != 'sqlite':
+ self.assertEqual(0, query4.rows)
+ self.assertEqual(666, query4.limit)
+ self.assertEqual(False, query4.limit_used)
+ self.assertEqual(True, query4.select_as_cta)
+ self.assertEqual(True, query4.select_as_cta_used)
+
+ # Check the data in the tmp table.
+ df4 = pd.read_sql_query(sql=query4.select_sql, con=eng)
+ data4 = df4.to_dict(orient='records')
+ self.assertEqual([], data4)
+
+ # Case 5.
+ # Table and DB exists, select without CTA.
+ result5 = self.run_sql(1, sql_where, tmp_table='tmp_table_5')
+ self.assertEqual(models.QueryStatus.FINISHED.lower(), result5['query']['status'])
+ self.assertEqual(['name'], result5['columns'])
+ self.assertEqual([{'name': 'can_sql'}], result5['data'])
+
+ query5 = self.get_query_by_id(result5['query_id'])
+ self.assertEqual(sql_where, query5.sql)
+ if eng.name != 'sqlite':
+ self.assertEqual(1, query5.rows)
+ self.assertEqual(666, query5.limit)
+ self.assertEqual(True, query5.limit_used)
+ self.assertEqual(False, query5.select_as_cta)
+ self.assertEqual(False, query5.select_as_cta_used)
+
+ def test_run_async_query(self):
+ main_db = db.session.query(models.Database).filter_by(
+ database_name="main").first()
+ eng = main_db.get_sqla_engine()
+
+ # Schedule queries
+
+ # Case 1.
+ # Table and DB exists, async CTA call to the backend.
+ sql_where = "SELECT name FROM ab_role WHERE name='Admin'"
+ result1 = self.run_sql(
+ 1, sql_where, async='true', tmp_table='tmp_async_1', cta='true')
+ self.assertEqual(models.QueryStatus.IN_PROGRESS.lower(), result1['query']['state'])
+
+ # Case 2.
+ # Table and DB exists, async insert query, no CTAs.
+ insert_query = "INSERT INTO ab_role VALUES (9, 'fake_role')"
+ result2 = self.run_sql(1, insert_query, async='true')
+ self.assertEqual(models.QueryStatus.IN_PROGRESS.lower(), result2['query']['state'])
+
+ time.sleep(2)
+
+ # Case 1.
+ query1 = self.get_query_by_id(result1['query']['serverId'])
+ df1 = pd.read_sql_query(query1.select_sql, con=eng)
+ self.assertEqual(models.QueryStatus.FINISHED, query1.status)
+ self.assertEqual([{'name': 'Admin'}], df1.to_dict(orient='records'))
+ self.assertEqual(models.QueryStatus.FINISHED, query1.status)
+ self.assertTrue("SELECT * \nFROM tmp_async_1" in query1.select_sql)
+ self.assertTrue("LIMIT 666" in query1.select_sql)
+ self.assertEqual(
+ "CREATE TABLE tmp_async_1 AS SELECT name FROM ab_role "
+ "WHERE name='Admin'", query1.executed_sql)
+ self.assertEqual(sql_where, query1.sql)
+ if eng.name != 'sqlite':
+ self.assertEqual(1, query1.rows)
+ self.assertEqual(666, query1.limit)
+ self.assertEqual(False, query1.limit_used)
+ self.assertEqual(True, query1.select_as_cta)
+ self.assertEqual(True, query1.select_as_cta_used)
+
+ # Case 2.
+ query2 = self.get_query_by_id(result2['query']['serverId'])
+ self.assertEqual(models.QueryStatus.FINISHED, query2.status)
+ self.assertIsNone(query2.select_sql)
+ self.assertEqual(insert_query, query2.executed_sql)
+ self.assertEqual(insert_query, query2.sql)
+ if eng.name != 'sqlite':
+ self.assertEqual(1, query2.rows)
+ self.assertEqual(666, query2.limit)
+ self.assertEqual(False, query2.limit_used)
+ self.assertEqual(False, query2.select_as_cta)
+ self.assertEqual(False, query2.select_as_cta_used)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/core_tests.py b/tests/core_tests.py
index ec03422853ec..a391000ed4ef 100644
--- a/tests/core_tests.py
+++ b/tests/core_tests.py
@@ -5,11 +5,14 @@
from __future__ import unicode_literals
from datetime import datetime
+import csv
import doctest
import json
import imp
import os
+import io
import unittest
+
from mock import Mock, patch
from flask import escape
@@ -21,14 +24,19 @@
os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config'
-app.config['TESTING'] = True
+# Disable celery.
+app.config['CELERY_CONFIG'] = None
app.config['CSRF_ENABLED'] = False
+app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True
app.config['SECRET_KEY'] = 'thisismyscretkey'
+app.config['SQL_SELECT_AS_CTA'] = False
+app.config['TESTING'] = True
app.config['WTF_CSRF_ENABLED'] = False
-app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True
+
BASE_DIR = app.config.get("BASE_DIR")
cli = imp.load_source('cli', BASE_DIR + "/bin/caravel")
+
class CaravelTestCase(unittest.TestCase):
def __init__(self, *args, **kwargs):
@@ -40,7 +48,7 @@ def __init__(self, *args, **kwargs):
admin = appbuilder.sm.find_user('admin')
if not admin:
appbuilder.sm.add_user(
- 'admin', 'admin',' user', 'admin@fab.org',
+ 'admin', 'admin', ' user', 'admin@fab.org',
appbuilder.sm.find_role('Admin'),
password='general')
@@ -67,6 +75,12 @@ def login(self, username='admin', password='general'):
follow_redirects=True)
assert 'Welcome' in resp.data.decode('utf-8')
+ def get_query_by_sql(self, sql):
+ session = db.create_scoped_session()
+ query = session.query(models.Query).filter_by(sql=sql).first()
+ session.close()
+ return query
+
def logout(self):
self.client.get('/logout/', follow_redirects=True)
@@ -79,7 +93,7 @@ def setup_public_access_for_dashboard(self, table_name):
public_role = appbuilder.sm.find_role('Public')
perms = db.session.query(ab_models.PermissionView).all()
for perm in perms:
- if ( perm.permission.name == 'datasource_access' and
+ if (perm.permission.name == 'datasource_access' and
perm.view_menu and table_name in perm.view_menu.name):
appbuilder.sm.add_permission_role(public_role, perm)
@@ -87,7 +101,7 @@ def revoke_public_access(self, table_name):
public_role = appbuilder.sm.find_role('Public')
perms = db.session.query(ab_models.PermissionView).all()
for perm in perms:
- if ( perm.permission.name == 'datasource_access' and
+ if (perm.permission.name == 'datasource_access' and
perm.view_menu and table_name in perm.view_menu.name):
appbuilder.sm.del_permission_role(public_role, perm)
@@ -95,22 +109,23 @@ def revoke_public_access(self, table_name):
class CoreTests(CaravelTestCase):
def __init__(self, *args, **kwargs):
- # Load examples first, so that we setup proper permission-view relations
- # for all example data sources.
+ # Load examples first, so that we setup proper permission-view
+ # relations for all example data sources.
super(CoreTests, self).__init__(*args, **kwargs)
@classmethod
def setUpClass(cls):
cli.load_examples(load_test_data=True)
utils.init(caravel)
- cls.table_ids = {tbl.table_name: tbl.id for tbl in (
+ cls.table_ids = {tbl.table_name: tbl.id for tbl in (
db.session
.query(models.SqlaTable)
.all()
)}
def setUp(self):
- pass
+ db.session.query(models.Query).delete()
+
def tearDown(self):
pass
@@ -126,7 +141,12 @@ def test_save_slice(self):
copy_name = "Test Sankey Save"
tbl_id = self.table_ids.get('energy_usage')
- url = "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&collapsed_fieldsets=&action={}&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey"
+ url = (
+ "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&"
+ "groupby=target&metric=sum__value&row_limit=5000&where=&having=&"
+ "flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&"
+ "collapsed_fieldsets=&action={}&datasource_name=energy_usage&"
+ "datasource_id=1&datasource_type=table&previous_viz_type=sankey")
db.session.commit()
resp = self.client.get(
@@ -146,6 +166,8 @@ def test_slices(self):
for slc in db.session.query(Slc).all():
urls += [
(slc.slice_name, 'slice_url', slc.slice_url),
+ (slc.slice_name, 'slice_id_endpoint', '/caravel/slices/{}'.
+ format(slc.id)),
(slc.slice_name, 'json_endpoint', slc.viz.json_endpoint),
(slc.slice_name, 'csv_endpoint', slc.viz.csv_endpoint),
]
@@ -210,13 +232,20 @@ def test_misc(self):
def test_shortner(self):
self.login(username='admin')
- data = "//caravel/explore/table/1/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name=Energy+Sankey&collapsed_fieldsets=&action=&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey"
+ data = (
+ "//caravel/explore/table/1/?viz_type=sankey&groupby=source&"
+ "groupby=target&metric=sum__value&row_limit=5000&where=&having=&"
+ "flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name="
+ "Energy+Sankey&collapsed_fieldsets=&action=&datasource_name="
+ "energy_usage&datasource_id=1&datasource_type=table&"
+ "previous_viz_type=sankey")
resp = self.client.post('/r/shortner/', data=data)
assert '/r/' in resp.data.decode('utf-8')
def test_save_dash(self, username='admin'):
self.login(username=username)
- dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
+ dash = db.session.query(models.Dashboard).filter_by(
+ slug="births").first()
positions = []
for i, slc in enumerate(dash.slices):
d = {
@@ -237,18 +266,24 @@ def test_save_dash(self, username='admin'):
def test_add_slices(self, username='admin'):
self.login(username=username)
- dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
- new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first()
- existing_slice = db.session.query(models.Slice).filter_by(slice_name="Name Cloud").first()
+ dash = db.session.query(models.Dashboard).filter_by(
+ slug="births").first()
+ new_slice = db.session.query(models.Slice).filter_by(
+ slice_name="Mapbox Long/Lat").first()
+ existing_slice = db.session.query(models.Slice).filter_by(
+ slice_name="Name Cloud").first()
data = {
- "slice_ids": [new_slice.data["slice_id"], existing_slice.data["slice_id"]]
+ "slice_ids": [new_slice.data["slice_id"],
+ existing_slice.data["slice_id"]]
}
url = '/caravel/add_slices/{}/'.format(dash.id)
resp = self.client.post(url, data=dict(data=json.dumps(data)))
assert "SLICES ADDED" in resp.data.decode('utf-8')
- dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
- new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first()
+ dash = db.session.query(models.Dashboard).filter_by(
+ slug="births").first()
+ new_slice = db.session.query(models.Slice).filter_by(
+ slice_name="Mapbox Long/Lat").first()
assert new_slice in dash.slices
assert len(set(dash.slices)) == len(dash.slices)
@@ -256,7 +291,10 @@ def test_add_slice_redirect_to_sqla(self, username='admin'):
self.login(username=username)
url = '/slicemodelview/add'
resp = self.client.get(url, follow_redirects=True)
- assert "Click on a table link to create a Slice" in resp.data.decode('utf-8')
+ assert (
+ "Click on a table link to create a Slice" in
+ resp.data.decode('utf-8')
+ )
def test_add_slice_redirect_to_druid(self, username='admin'):
datasource = DruidDatasource(
@@ -268,7 +306,10 @@ def test_add_slice_redirect_to_druid(self, username='admin'):
self.login(username=username)
url = '/slicemodelview/add'
resp = self.client.get(url, follow_redirects=True)
- assert "Click on a datasource link to create a Slice" in resp.data.decode('utf-8')
+ assert (
+ "Click on a datasource link to create a Slice"
+ in resp.data.decode('utf-8')
+ )
db.session.delete(datasource)
db.session.commit()
@@ -281,41 +322,48 @@ def test_gamma(self):
resp = self.client.get('/dashboardmodelview/list/')
assert "List Dashboard" in resp.data.decode('utf-8')
- def run_sql(self, sql, user_name):
+ def run_sql(self, sql, user_name, client_id='not_used'):
self.login(username=user_name)
dbid = (
db.session.query(models.Database)
- .filter_by(database_name="main")
+ .filter_by(database_name='main')
.first().id
)
resp = self.client.post(
'/caravel/sql_json/',
- data=dict(database_id=dbid, sql=sql),
+ data=dict(database_id=dbid, sql=sql, select_as_create_as=False, client_id=client_id),
)
self.logout()
return json.loads(resp.data.decode('utf-8'))
- def test_sql_json_no_access(self):
- self.assertRaises(
- utils.CaravelSecurityException,
- self.run_sql, "SELECT * FROM ab_user", 'gamma')
+ # TODO(bkyryliuk): fix the gamma access, they shouldn't be allowed running queries.
+ # def test_sql_json_no_access(self):
+ # result = self.run_sql('SELECT * FROM ab_user', 'gamma')
+ # self.assertEqual('SQL Lab requires the `all_datasource_access` or specific DB permission',
+ # result['error'])
+
+ def test_sql_json(self):
+ data = self.run_sql('SELECT * FROM ab_user', 'admin')
+ assert len(data['data']) > 0
+
+ data = self.run_sql('SELECT * FROM unexistant_table', 'admin')
+ assert len(data['error']) > 0
def test_sql_json_has_access(self):
main_db = (
- db.session.query(models.Database).filter_by(database_name="main")
- .first()
+ db.session.query(models.Database).filter_by(database_name="main").first()
)
utils.merge_perm(sm, 'database_access', main_db.perm)
db.session.commit()
main_db_permission_view = (
db.session.query(ab_models.PermissionView)
- .join(ab_models.ViewMenu)
- .filter(ab_models.ViewMenu.name == '[main].(id:1)')
- .first()
+ .join(ab_models.ViewMenu)
+ .filter(ab_models.ViewMenu.name == '[main].(id:1)')
+ .first()
)
astronaut = sm.add_role("Astronaut")
sm.add_permission_role(astronaut, main_db_permission_view)
- # Astronaut role is Gamme + main db permissions
+ # Astronaut role is Gamma + main db permissions
for gamma_perm in sm.find_role('Gamma').permissions:
sm.add_permission_role(astronaut, gamma_perm)
@@ -326,14 +374,62 @@ def test_sql_json_has_access(self):
appbuilder.sm.find_role('Astronaut'),
password='general')
data = self.run_sql('SELECT * FROM ab_user', 'gagarin')
+ db.session.query(models.Query).delete()
+ db.session.commit()
assert len(data['data']) > 0
- def test_sql_json(self):
- data = self.run_sql("SELECT * FROM ab_user", 'admin')
- assert len(data['data']) > 0
+ def test_csv_endpoint(self):
+ sql = "SELECT first_name, last_name FROM ab_user " \
+ "where first_name='admin'"
+ self.run_sql(sql, 'admin')
+
+ # TODO(bkyryliuk): fix the permissions
+ # No access if the user is not logged in.
+ # query1_id = self.get_query_by_sql(sql).id
+ # no_access_reps = self.client.get('/caravel/csv/{}'.format(query1_id))
+ # self.assertTrue(
+ # 'SQL Lab requires the `all_datasource_access` or specific DB permission'
+ # in no_access_reps.data.decode('utf-8'))
+
+ query1_id = self.get_query_by_sql(sql).id
+ self.login('admin')
+ resp = self.client.get('/caravel/csv/{}'.format(query1_id))
+ data = csv.reader(io.StringIO(resp.data.decode('utf-8')))
+ expected_data = csv.reader(io.StringIO(
+ "first_name,last_name\nadmin, user\n"))
+
+ self.assertEqual(list(expected_data), list(data))
+ self.logout()
- data = self.run_sql("SELECT * FROM unexistant_table", 'admin')
- assert len(data['error']) > 0
+ def test_queries_endpoint(self):
+ resp = self.client.get('/caravel/queries/{}'.format(0))
+ self.assertEquals(403, resp.status_code)
+
+ self.login('admin')
+ resp = self.client.get('/caravel/queries/{}'.format(0))
+ data = json.loads(resp.data.decode('utf-8'))
+ self.assertEquals(0, len(data))
+ self.logout()
+
+ self.run_sql("SELECT * FROM ab_user", 'admin', client_id='client_id_1')
+ self.run_sql("SELECT * FROM ab_user1", 'admin', client_id='client_id_2')
+ self.login('admin')
+ resp = self.client.get('/caravel/queries/{}'.format(0))
+ data = json.loads(resp.data.decode('utf-8'))
+ self.assertEquals(2, len(data))
+
+ query = db.session.query(models.Query).filter_by(
+ sql='SELECT * FROM ab_user').first()
+ query.changed_on = utils.EPOCH
+ db.session.commit()
+
+ resp = self.client.get('/caravel/queries/{}'.format(123456000))
+ data = json.loads(resp.data.decode('utf-8'))
+ self.assertEquals(1, len(data))
+
+ self.logout()
+ resp = self.client.get('/caravel/queries/{}'.format(0))
+ self.assertEquals(403, resp.status_code)
def test_public_user_dashboard_access(self):
# Try access before adding appropriate permissions.
@@ -365,6 +461,10 @@ def test_public_user_dashboard_access(self):
assert 'Births' in data
# Confirm that public doesn't have access to other datasets.
+ resp = self.client.get('/slicemodelview/list/')
+ data = resp.data.decode('utf-8')
+ assert 'wb_health_population' not in data
+
resp = self.client.get('/dashboardmodelview/list/')
data = resp.data.decode('utf-8')
assert "/caravel/dashboard/world_health/" not in data
@@ -400,26 +500,26 @@ def test_only_owners_can_save(self):
SEGMENT_METADATA = [{
"id": "some_id",
- "intervals": [ "2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z" ],
+ "intervals": ["2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z"],
"columns": {
"__time": {
"type": "LONG", "hasMultipleValues": False,
- "size": 407240380, "cardinality": None, "errorMessage": None },
+ "size": 407240380, "cardinality": None, "errorMessage": None},
"dim1": {
"type": "STRING", "hasMultipleValues": False,
- "size": 100000, "cardinality": 1944, "errorMessage": None },
+ "size": 100000, "cardinality": 1944, "errorMessage": None},
"dim2": {
"type": "STRING", "hasMultipleValues": True,
- "size": 100000, "cardinality": 1504, "errorMessage": None },
+ "size": 100000, "cardinality": 1504, "errorMessage": None},
"metric1": {
"type": "FLOAT", "hasMultipleValues": False,
- "size": 100000, "cardinality": None, "errorMessage": None }
+ "size": 100000, "cardinality": None, "errorMessage": None}
},
"aggregators": {
"metric1": {
"type": "longSum",
"name": "metric1",
- "fieldName": "metric1" }
+ "fieldName": "metric1"}
},
"size": 300000,
"numRows": 5000000
@@ -485,7 +585,8 @@ def test_client(self, PyDruid):
datasource_id = cluster.datasources[0].id
db.session.commit()
- resp = self.client.get('/caravel/explore/druid/{}/'.format(datasource_id))
+ resp = self.client.get('/caravel/explore/druid/{}/'.format(
+ datasource_id))
assert "[test_cluster].[test_datasource]" in resp.data.decode('utf-8')
nres = [
@@ -497,9 +598,15 @@ def test_client(self, PyDruid):
instance.export_pandas.return_value = df
instance.query_dict = {}
instance.query_builder.last_query.query_dict = {}
- resp = self.client.get('/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&include_search=false&metrics=count&groupby=name&flt_col_0=dim1&flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&action=&datasource_name=test_datasource&datasource_id={}&datasource_type=druid&previous_viz_type=table&json=true&force=true'.format(datasource_id, datasource_id))
+ resp = self.client.get(
+ '/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&'
+ 'druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&'
+ 'include_search=false&metrics=count&groupby=name&flt_col_0=dim1&'
+ 'flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&'
+ 'action=&datasource_name=test_datasource&datasource_id={}&'
+ 'datasource_type=druid&previous_viz_type=table&json=true&'
+ 'force=true'.format(datasource_id, datasource_id))
assert "Canada" in resp.data.decode('utf-8')
-
if __name__ == '__main__':
unittest.main()