From 963b9cbd72eba86275cc6ca24a420b5cebc6734b Mon Sep 17 00:00:00 2001 From: Jeff Niu Date: Sun, 17 Sep 2017 12:58:06 -0700 Subject: [PATCH 1/2] Feature: disable dashboard refresh staggering --- docs/faq.rst | 20 +++++++- .../javascripts/dashboard/Dashboard.jsx | 47 ++++++++++--------- .../dashboard/components/Controls.jsx | 5 +- .../assets/javascripts/modules/superset.js | 38 ++++++++------- 4 files changed, 68 insertions(+), 42 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 82280ed46bdf..21e23fcdde4c 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -143,6 +143,24 @@ To exclude specific slices from the timed refresh process, add the ``timed_refre In the example above, if a timed refresh is set for the dashboard, then every slice except 324 will be automatically requeried on schedule. +How to speed up/slow down dashboard loading? +-------------------------------------------- +Dashboards stagger slice rendering by default over a period of 5 seconds. This reduces loads on +some databases but slows down loading time. You can disable render staggering by setting the key +``stagger_refresh`` in the ``JSON Metadata`` to ``false`` or alter the stagger period by setting +``stagger_time`` to a value in milliseconds: + +..code:: + + { + "stagger_refresh": false, + "stagger_time": 2000 + } + +Here, whenever the dashboard refreshes slices, all requests will be issued simultaneously. The +stagger time of 2 seconds is ignored. + + Why does fabmanager or superset freezed/hung/not responding when started (my home directory is NFS mounted)? ----------------------------------------------------------------------------------------- By default, superset creates and uses an sqlite database at ``~/.superset/superset.db``. Sqlite is known to `don't work well if used on NFS`__ due to broken file locking implementation on NFS. @@ -188,7 +206,7 @@ Please note that pretty much any databases that have a SqlAlchemy integration sh How can i configure OAuth authentication and authorization? ----------------------------------------------------------- -You can take a look at this Flask-AppBuilder `configuration example +You can take a look at this Flask-AppBuilder `configuration example `_. How can I set a default filter on my dashboard? diff --git a/superset/assets/javascripts/dashboard/Dashboard.jsx b/superset/assets/javascripts/dashboard/Dashboard.jsx index 8a7000ea8b3f..6e07b983b353 100644 --- a/superset/assets/javascripts/dashboard/Dashboard.jsx +++ b/superset/assets/javascripts/dashboard/Dashboard.jsx @@ -255,25 +255,35 @@ export function dashboardContainer(dashboard, datasources, userid) { this.refreshTimer = null; } }, + renderSlices(slices, interval = 0) { + const meta = this.metadata; + const refreshTime = Math.max(interval, meta.stagger_time || 5000); + // Delay is zero if not staggerring slice refresh + if (typeof meta.stagger_refresh !== 'boolean') { + meta.stagger_refresh = meta.stagger_refresh === undefined ? + true : meta.stagger_refresh === 'true'; + } + const delay = meta.stagger_refresh ? refreshTime / (slices.length - 1) : 0; + slices.forEach((slice, i) => { + // Caller may pass array of slices or { slice, force } + if (slice.slice) { + slice.slice.render(slice.force, delay * i); + } else { + slice.render(false, delay * i); + } + }); + }, startPeriodicRender(interval) { this.stopPeriodicRender(); const dash = this; const immune = this.metadata.timed_refresh_immune_slices || []; - const maxRandomDelay = Math.max(interval * 0.2, 5000); const refreshAll = () => { - dash.sliceObjects.forEach((slice) => { - const force = !dash.firstLoad; - if (immune.indexOf(slice.data.slice_id) === -1) { - setTimeout(() => { - slice.render(force); - }, - // Randomize to prevent all widgets refreshing at the same time - maxRandomDelay * Math.random()); - } - }); + const slices = dash.sliceObjects + .filter(slice => immune.indexOf(slice.data.slice_id) === -1 || dash.firstLoad) + .map(slice => ({ slice, force: !dash.firstLoad })); dash.firstLoad = false; + dash.renderSlices(slices, interval * 0.2); }; - const fetchAndRender = function () { refreshAll(); if (interval > 0) { @@ -286,16 +296,9 @@ export function dashboardContainer(dashboard, datasources, userid) { }, refreshExcept(sliceId) { const immune = this.metadata.filter_immune_slices || []; - this.sliceObjects.forEach((slice) => { - if (slice.data.slice_id !== sliceId && immune.indexOf(slice.data.slice_id) === -1) { - slice.render(); - const sliceSeletor = $(`#${slice.data.slice_id}-cell`); - sliceSeletor.addClass('slice-cell-highlight'); - setTimeout(function () { - sliceSeletor.removeClass('slice-cell-highlight'); - }, 1200); - } - }); + const slices = this.sliceObjects.filter(slice => + slice.data.slice_id !== sliceId && immune.indexOf(slice.data.slice_id) === -1); + this.renderSlices(slices); }, clearFilters(sliceId) { delete this.filters[sliceId]; diff --git a/superset/assets/javascripts/dashboard/components/Controls.jsx b/superset/assets/javascripts/dashboard/components/Controls.jsx index 1169642ff60e..fd7689799a37 100644 --- a/superset/assets/javascripts/dashboard/components/Controls.jsx +++ b/superset/assets/javascripts/dashboard/components/Controls.jsx @@ -34,9 +34,8 @@ class Controls extends React.PureComponent { }); } refresh() { - this.props.dashboard.sliceObjects.forEach((slice) => { - slice.render(true); - }); + const slices = this.props.dashboard.sliceObjects.map(slice => ({ slice, force: true })); + this.props.dashboard.renderSlices(slices); } changeCss(css) { this.setState({ css }); diff --git a/superset/assets/javascripts/modules/superset.js b/superset/assets/javascripts/modules/superset.js index aaf1e8557355..641826252851 100644 --- a/superset/assets/javascripts/modules/superset.js +++ b/superset/assets/javascripts/modules/superset.js @@ -64,6 +64,7 @@ const px = function (state) { const container = $(selector); const sliceId = data.slice_id; const formData = applyDefaultFormData(data.form_data); + const sliceCell = $(`#${data.slice_id}-cell`); slice = { data, formData, @@ -113,6 +114,7 @@ const px = function (state) { token.find('img.loading').hide(); container.fadeTo(0.5, 1); + sliceCell.removeClass('slice-cell-highlight'); container.show(); $('.query-and-save button').removeAttr('disabled'); @@ -138,6 +140,7 @@ const px = function (state) { let errorMsg = msg; token.find('img.loading').hide(); container.fadeTo(0.5, 1); + sliceCell.removeClass('slice-cell-highlight'); let errHtml = ''; let o; try { @@ -197,7 +200,7 @@ const px = function (state) { }, 500); }); }, - render(force) { + render(force, delay = 0) { if (force === undefined) { this.force = false; } else { @@ -210,22 +213,25 @@ const px = function (state) { controls.find('a.exportCSV').attr('href', getExploreUrl(formDataExtra, 'csv')); token.find('img.loading').show(); container.fadeTo(0.5, 0.25); + sliceCell.addClass('slice-cell-highlight'); container.css('height', this.height()); - $.ajax({ - url: this.jsonEndpoint(formDataExtra), - timeout: timeout * 1000, - success: (queryResponse) => { - try { - vizMap[formData.viz_type](this, queryResponse); - this.done(queryResponse); - } catch (e) { - this.error('An error occurred while rendering the visualization: ' + e); - } - }, - error: (err) => { - this.error(err.responseText, err); - }, - }); + setTimeout(() => { + $.ajax({ + url: this.jsonEndpoint(formDataExtra), + timeout: timeout * 1000, + success: (queryResponse) => { + try { + vizMap[formData.viz_type](this, queryResponse); + this.done(queryResponse); + } catch (e) { + this.error('An error occurred while rendering the visualization: ' + e); + } + }, + error: (err) => { + this.error(err.responseText, err); + }, + }); + }, delay); }, resize() { this.render(); From 268804e65f6a6baa43290423cc8208c6179cb7f2 Mon Sep 17 00:00:00 2001 From: Jeff Niu Date: Wed, 20 Sep 2017 16:58:24 -0700 Subject: [PATCH 2/2] Removed refresh staggering everywhere except during periodic render --- docs/faq.rst | 29 ++++++++-------- .../javascripts/dashboard/Dashboard.jsx | 25 ++++++-------- .../dashboard/components/Controls.jsx | 4 +-- .../assets/javascripts/modules/superset.js | 34 +++++++++---------- 4 files changed, 43 insertions(+), 49 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 21e23fcdde4c..4a2a85e66c95 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -128,8 +128,11 @@ be applied, it's as simple as that. How to limit the timed refresh on a dashboard? ---------------------------------------------- -By default, the dashboard timed refresh feature allows you to automatically requery every slice on a dashboard according to a set schedule. Sometimes, however, you won't want all of the slices to be refreshed - especially if some data is slow moving, or run heavy queries. -To exclude specific slices from the timed refresh process, add the ``timed_refresh_immune_slices`` key to the dashboard ``JSON Metadata`` field: +By default, the dashboard timed refresh feature allows you to automatically re-query every slice +on a dashboard according to a set schedule. Sometimes, however, you won't want all of the slices +to be refreshed - especially if some data is slow moving, or run heavy queries. To exclude specific +slices from the timed refresh process, add the ``timed_refresh_immune_slices`` key to the dashboard +``JSON Metadata`` field: ..code:: @@ -140,26 +143,22 @@ To exclude specific slices from the timed refresh process, add the ``timed_refre "timed_refresh_immune_slices": [324] } -In the example above, if a timed refresh is set for the dashboard, then every slice except 324 will be automatically requeried on schedule. +In the example above, if a timed refresh is set for the dashboard, then every slice except 324 will +be automatically re-queried on schedule. - -How to speed up/slow down dashboard loading? --------------------------------------------- -Dashboards stagger slice rendering by default over a period of 5 seconds. This reduces loads on -some databases but slows down loading time. You can disable render staggering by setting the key -``stagger_refresh`` in the ``JSON Metadata`` to ``false`` or alter the stagger period by setting -``stagger_time`` to a value in milliseconds: +Slice refresh will also be staggered over the specified period. You can turn off this staggering +by setting the ``stagger_refresh`` to ``false`` and modify the stagger period by setting +``stagger_time`` to a value in milliseconds in the ``JSON Metadata`` field: ..code:: { - "stagger_refresh": false, - "stagger_time": 2000 + "stagger_refresh": false, + "stagger_time": 2500 } -Here, whenever the dashboard refreshes slices, all requests will be issued simultaneously. The -stagger time of 2 seconds is ignored. - +Here, the entire dashboard will refresh at once if periodic refresh is on. The stagger time of +2.5 seconds is ignored. Why does fabmanager or superset freezed/hung/not responding when started (my home directory is NFS mounted)? ----------------------------------------------------------------------------------------- diff --git a/superset/assets/javascripts/dashboard/Dashboard.jsx b/superset/assets/javascripts/dashboard/Dashboard.jsx index 6e07b983b353..c04a3b3b3973 100644 --- a/superset/assets/javascripts/dashboard/Dashboard.jsx +++ b/superset/assets/javascripts/dashboard/Dashboard.jsx @@ -126,7 +126,8 @@ export function dashboardContainer(dashboard, datasources, userid) { } }); this.loadPreSelectFilters(); - this.startPeriodicRender(0); + this.renderSlices(this.sliceObjects); + this.firstLoad = false; this.bindResizeToWindowResize(); }, onChange() { @@ -255,22 +256,20 @@ export function dashboardContainer(dashboard, datasources, userid) { this.refreshTimer = null; } }, - renderSlices(slices, interval = 0) { + renderSlices(slices, force = false, interval = 0) { + if (!interval) { + slices.forEach(slice => slice.render(force)); + return; + } const meta = this.metadata; - const refreshTime = Math.max(interval, meta.stagger_time || 5000); - // Delay is zero if not staggerring slice refresh + const refreshTime = Math.max(interval, meta.stagger_time || 5000); // default 5 seconds if (typeof meta.stagger_refresh !== 'boolean') { meta.stagger_refresh = meta.stagger_refresh === undefined ? true : meta.stagger_refresh === 'true'; } const delay = meta.stagger_refresh ? refreshTime / (slices.length - 1) : 0; slices.forEach((slice, i) => { - // Caller may pass array of slices or { slice, force } - if (slice.slice) { - slice.slice.render(slice.force, delay * i); - } else { - slice.render(false, delay * i); - } + setTimeout(() => slice.render(force), delay * i); }); }, startPeriodicRender(interval) { @@ -279,10 +278,8 @@ export function dashboardContainer(dashboard, datasources, userid) { const immune = this.metadata.timed_refresh_immune_slices || []; const refreshAll = () => { const slices = dash.sliceObjects - .filter(slice => immune.indexOf(slice.data.slice_id) === -1 || dash.firstLoad) - .map(slice => ({ slice, force: !dash.firstLoad })); - dash.firstLoad = false; - dash.renderSlices(slices, interval * 0.2); + .filter(slice => immune.indexOf(slice.data.slice_id) === -1); + dash.renderSlices(slices, true, interval * 0.2); }; const fetchAndRender = function () { refreshAll(); diff --git a/superset/assets/javascripts/dashboard/components/Controls.jsx b/superset/assets/javascripts/dashboard/components/Controls.jsx index fd7689799a37..bdf6843d9c93 100644 --- a/superset/assets/javascripts/dashboard/components/Controls.jsx +++ b/superset/assets/javascripts/dashboard/components/Controls.jsx @@ -34,8 +34,8 @@ class Controls extends React.PureComponent { }); } refresh() { - const slices = this.props.dashboard.sliceObjects.map(slice => ({ slice, force: true })); - this.props.dashboard.renderSlices(slices); + // Force refresh all slices + this.props.dashboard.renderSlices(this.props.dashboard.sliceObjects, true); } changeCss(css) { this.setState({ css }); diff --git a/superset/assets/javascripts/modules/superset.js b/superset/assets/javascripts/modules/superset.js index 641826252851..3a20bba79a3c 100644 --- a/superset/assets/javascripts/modules/superset.js +++ b/superset/assets/javascripts/modules/superset.js @@ -200,7 +200,7 @@ const px = function (state) { }, 500); }); }, - render(force, delay = 0) { + render(force) { if (force === undefined) { this.force = false; } else { @@ -215,23 +215,21 @@ const px = function (state) { container.fadeTo(0.5, 0.25); sliceCell.addClass('slice-cell-highlight'); container.css('height', this.height()); - setTimeout(() => { - $.ajax({ - url: this.jsonEndpoint(formDataExtra), - timeout: timeout * 1000, - success: (queryResponse) => { - try { - vizMap[formData.viz_type](this, queryResponse); - this.done(queryResponse); - } catch (e) { - this.error('An error occurred while rendering the visualization: ' + e); - } - }, - error: (err) => { - this.error(err.responseText, err); - }, - }); - }, delay); + $.ajax({ + url: this.jsonEndpoint(formDataExtra), + timeout: timeout * 1000, + success: (queryResponse) => { + try { + vizMap[formData.viz_type](this, queryResponse); + this.done(queryResponse); + } catch (e) { + this.error('An error occurred while rendering the visualization: ' + e); + } + }, + error: (err) => { + this.error(err.responseText, err); + }, + }); }, resize() { this.render();