From c94fc95ca87b33b11586087eaadb8a09320c1495 Mon Sep 17 00:00:00 2001 From: leocape <67806134+leocape@users.noreply.github.com> Date: Sat, 13 Feb 2021 11:09:09 +0200 Subject: [PATCH 01/18] update docs for new alerts and reporting feature --- .../pages/docs/installation/email_reports.mdx | 299 +++++++++++++++++- 1 file changed, 298 insertions(+), 1 deletion(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index 88e6986c60a8..a1c295942157 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -6,8 +6,305 @@ index: 10 version: 1 --- -## Scheduling and Emailing Reports +## Alerts and Reports +(version 1.0.1 and above) + +Users can configure automated alerts and reports to send charts and dashboards to an email recipient or slack channel. + +- Alerts are sent when a specified condition is passed. +- Reports are sent on a specified schedule + +### Turning on Alerts and reports +Alerts and reports are not on by default, and are currently behind a feature flag. They also require the addition of some extra running services. + +#### Requirements: + +- `Dockerfile` +-- webdriver to run a headless browser (for screenshots) +- `docker-compose.yaml` +-- redis message broker running +-- replacing SQLlite DB with Postgres DB +-- celery worker +-- celery beat +- `superset_config.py` +-- feature flag turned on +-- all configs as outlined in the template below +- At least one of these is needed to sent alerts and reports: +-- (optional) SMTP server for sending email +-- (optional) Slack app integration for sending to slack channels + +#### Summary of steps to turn on alerts and reporting: + +Using the templates below, +1. Create a new directory and create the Dockerfile +2. Build the extended image using the Dockerfile +3. Create the `docker-compose.yaml` file in the same directory +4. Create a new sub directory called `config` +5. Create the `superset_config.py` file in the `config` sub directory +6. Run the image using `docker-compose up` in the same directory as the `docker-compose.py` file + +(note: v 1.0.1 is current at time of writing, you can change the version number to the latest version if a newer version is available) +### Dockerfile + +A webdriver (and headless browser) is needed to capture screenshots of the charts and dashboards which are then sent to the recipient. As the base image does not have a webdriver installed by default, we need to extend the base image and install the webdriver (this template uses the Chrome webdriver). We are also adding in connectors for Mysql and Postgres, as well as Redis and Flower (Flower and Mysql are optional depending on your requirements) + +You can extend the image by running this Docker build command from the directory that contains the Dockerfile: +`docker build -t superset-1.0.1-extended -f Dockerfile` + +Config for `Dockerfile`: +```docker +FROM apache/superset:1.0.1 +USER root +RUN apt update +RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt install -y --no-install-recommends ./google-chrome-stable_current_amd64.deb && \ + wget https://chromedriver.storage.googleapis.com/88.0.4324.96/chromedriver_linux64.zip && \ + unzip chromedriver_linux64.zip && \ + chmod +x chromedriver && \ + mv chromedriver /usr/bin && \ + apt autoremove -yqq --purge && \ + apt clean && \ + rm -f google-chrome-stable_current_amd64.deb chromedriver_linux64.zip +RUN pip install --no-cache-dir gevent +RUN pip install --no-cache-dir mysqlclient +RUN pip install --no-cache-dir psycopg2 +RUN pip install --no-cache-dir redis +RUN pip install --no-cache-dir flower +USER superset + +``` +### Docker compose +The docker compose file lists the services that will be used when running the image. The specific services needed for alerts and reporting are outlined below. + +#### Redis message broker +To ferry requests between the celery worker and the Superset instance, we use a message broker. This template uses Redis. +#### Replacing SQLite with Postgres +While it might be possible to use SQLite for alerts and reporting, it is highly recommended to use a more production ready DB for Superset in general. Our template uses Postgres. + +#### Celery worker +The worker will process the tasks that need to be performed when an alert or report is fired. + +#### Celery beat +The beat is the scheduler that tells the worker when to perform its tasks. This schedule is defined when you create the alert or report. + +#### Full `docker-compose.yaml` configuration +The Redis, Postgres, Celery worker and Celery beat services are defined in the template: + +Config for `docker-compose.yaml`: +```docker +version: '3.6' +services: + redis-superset: + image: redis:6.0.9-buster + restart: on-failure + volumes: + - redis:/data + postgres: + image: postgres + restart: on-failure + environment: + POSTGRES_DB: superset + POSTGRES_PASSWORD: superset + POSTGRES_USER: superset + volumes: + - db:/var/lib/postgresql/data + worker: + image: superset-1.0.1-extended + restart: on-failure + healthcheck: + disable: true + depends_on: + - superset + - postgres + - redis-superset + command: "celery worker --app=superset.tasks.celery_app:app --pool=prefork --max-tasks-per-child=128 -O fair" + volumes: + - ./config/:/app/pythonpath/ + beat: + image: superset-1.0.1-extended + restart: on-failure + healthcheck: + disable: true + depends_on: + - superset + - postgres + - redis-superset + command: "celery beat --app=superset.tasks.celery_app:app --pidfile /tmp/celerybeat.pid --schedule /tmp/celerybeat-schedule" + volumes: + - ./config/:/app/pythonpath/ + superset: + image: superset-1.0.1-extended + restart: on-failure + environment: + - SUPERSET_PORT=8088 + ports: + - "8088:8088" + depends_on: + - postgres + - redis-superset + command: gunicorn --bind 0.0.0.0:8088 --access-logfile - --error-logfile - --workers 5 --worker-class gthread --threads 4 --timeout 200 --limit-request-line 4094 --limit-request-field_size 8190 superset.app:create_app() + volumes: + - ./config/:/app/pythonpath/ +volumes: + db: + external: false + redis: + external: false +``` + +### Superset_config.py + +The following configurations need to be added to the `superset_config.py` file. This file is loaded when the image runs, and any configurations in it will override the default configurations found in the `config.py`. + +You will need to add your custom SMTP settings, and or Slack APP token + +Config for `superset_config.py`: +```python +from superset_config import * +from celery.schedules import crontab +from cachelib import RedisCache +from superset.typing import CacheConfig +import os + +FEATURE_FLAGS = { + "ALERT_REPORTS": True +} + +# slack API token (optional) +SLACK_API_TOKEN = "xoxb-" +SLACK_PROXY = None + +POSTGRES_USER = "superset" +POSTGRES_PASS = "superset" +POSTGRES_HOST = "postgres" +POSTGRES_PORT = "5432" +POSTGRES_DATABASE = "superset" +REDIS_HOST = "redis-superset" +REDIS_PORT = "6379" +# The SQLAlchemy connection string. +SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://%s:%s@%s:%s/%s?client_encoding=utf8' % (POSTGRES_USER, + POSTGRES_PASS, + POSTGRES_HOST, + POSTGRES_PORT, + POSTGRES_DATABASE) +CACHE_CONFIG: CacheConfig = { + 'CACHE_TYPE': 'redis', + 'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day + 'CACHE_KEY_PREFIX': 'superset_', + 'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) +} +DATA_CACHE_CONFIG: CacheConfig = { + 'CACHE_TYPE': 'redis', + 'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day + 'CACHE_KEY_PREFIX': 'data_', + 'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) +} +THUMBNAIL_SELENIUM_USER = "admin" +THUMBNAIL_CACHE_CONFIG: CacheConfig = { + 'CACHE_TYPE': 'redis', + 'CACHE_DEFAULT_TIMEOUT': 24*60*60*30, + 'CACHE_KEY_PREFIX': 'thumbnail_', + 'CACHE_NO_NULL_WARNING': True, + 'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) +} +SCREENSHOT_LOCATE_WAIT = 100 +SCREENSHOT_LOAD_WAIT = 600 +RESULTS_BACKEND = RedisCache(host=REDIS_HOST, port=REDIS_PORT, key_prefix='superset_results') +class CeleryConfig(object): + BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) + CELERY_IMPORTS = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", ) + CELERY_RESULT_BACKEND = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) + CELERYD_PREFETCH_MULTIPLIER = 10 + CELERY_ACKS_LATE = True + CELERY_ANNOTATIONS = { + 'sql_lab.get_sql_results': { + 'rate_limit': '100/s', + }, + 'email_reports.send': { + 'rate_limit': '1/s', + 'time_limit': 600, + 'soft_time_limit': 600, + 'ignore_result': True, + }, + } + CELERYBEAT_SCHEDULE = { + 'reports.scheduler': { + 'task': 'reports.scheduler', + 'schedule': crontab(minute='*', hour='*'), + }, + 'reports.prune_log': { + 'task': 'reports.prune_log', + 'schedule': crontab(minute=0, hour=0), + }, + 'cache-warmup-hourly': { + 'task': 'cache-warmup', + 'schedule': crontab(minute='*/30', hour='*'), + 'kwargs': { + 'strategy_name': 'top_n_dashboards', + 'top_n': 10, + 'since': '7 days ago', + }, + }, + } +CELERY_CONFIG = CeleryConfig + +# SMTP email configuration +EMAIL_REPORTS_USER="admin" +EMAIL_PAGE_RENDER_WAIT=300 +EMAIL_NOTIFICATIONS = True + +SMTP_HOST = "smtp.sendgrid.net" +SMTP_STARTTLS = True +SMTP_SSL = False +SMTP_USER = "your_user" +SMTP_PORT = 2525 # your port eg. 587 +SMTP_PASSWORD = "your_password" +SMTP_MAIL_FROM = "noreply@youremail.com" + +WEBDRIVER_TYPE= "chrome" +WEBDRIVER_OPTION_ARGS = [ + "--force-device-scale-factor=2.0", + "--high-dpi-support=2.0", + "--headless", + "--disable-gpu", + "--disable-dev-shm-usage", + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-extensions", + ] + +WEBDRIVER_BASEURL="http://superset:8088" +WEBDRIVER_BASEURL_USER_FRIENDLY="http://localhost:8088" # change to your domain eg. https://superset.mydomain.com - this is the link that is sent to the recipient +SUPERSET_WEBSERVER_ADDRESS = "localhost" +SUPERSET_WEBSERVER_PORT = 8088 +SUPERSET_WEBSERVER_TIMEOUT=600 + +``` + +### Summary +With the extended image created by using the `Dockerfile`, and then running that image using `docker-compose.yaml`, plus the required configurations in the `superset_config.py` you should now have alerts and reporting working correctly. + +- For Kubernetes you can see the Helm chart here +- The above templates also work in a Docker swarm environment, you would just need to add `Deploy:` to the Superset, redis and Postgres services along with your specific configs for your swarm + +### Optional - Slack integration +To send alerts and reports to a Slack channel, you need to create a new Slack APP on your domain. +1. Head to https://api.slack.com/apps +2. Create a new APP, give it a name (eg. Superset) +3. Under the OAuth and Permissions section, give the following scopes to the app: + a) incoming-webhook + b) calls:write +4. At the top of the OAuth and Permissions section, click 'install to workspace' +5. Select a default channel for the app to post to and continue. (You can post to any channel by inviting your Superset app into that channel) +6. The app should now be installed on the workspace, and a 'Bot User OAuth Access Token' should be created. Copy the OAuth token and add it into the Slack section in the `superset_config.py` +7. Restart the service (or run `superset init`) to pull in the new configuration. +8. Note when sending to the channel from the alerts and reports UI, set the channel without the leading '#' eg. use `alerts` instead of `#alerts` + + +# +## Scheduling and Emailing Reports +(version 0.38 and below) ### Email Reports Email reports allow users to schedule email reports for: From 2f8c93790dde3dff93ed51bbbcbbff6640473628 Mon Sep 17 00:00:00 2001 From: leocape <67806134+leocape@users.noreply.github.com> Date: Sat, 13 Feb 2021 11:21:46 +0200 Subject: [PATCH 02/18] fix punctuation --- docs/src/pages/docs/installation/email_reports.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index a1c295942157..177555bd52c4 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -9,13 +9,13 @@ version: 1 ## Alerts and Reports (version 1.0.1 and above) -Users can configure automated alerts and reports to send charts and dashboards to an email recipient or slack channel. +Users can configure automated alerts and reports to send charts and dashboards to an email recipient or Slack channel. -- Alerts are sent when a specified condition is passed. +- Alerts are sent when a specified condition is passed - Reports are sent on a specified schedule ### Turning on Alerts and reports -Alerts and reports are not on by default, and are currently behind a feature flag. They also require the addition of some extra running services. +Alerts and reports are not on by default, they are currently behind a feature flag, and also require some additional services and configurations. #### Requirements: From 57a0343aaffbfc578bc046d271c887242f50d552 Mon Sep 17 00:00:00 2001 From: leocape <67806134+leocape@users.noreply.github.com> Date: Sat, 13 Feb 2021 11:27:36 +0200 Subject: [PATCH 03/18] fix punctuation --- docs/src/pages/docs/installation/email_reports.mdx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index 177555bd52c4..18a8cd948166 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -15,23 +15,23 @@ Users can configure automated alerts and reports to send charts and dashboards t - Reports are sent on a specified schedule ### Turning on Alerts and reports -Alerts and reports are not on by default, they are currently behind a feature flag, and also require some additional services and configurations. +Alerts and reports are not turned on by default. They are currently behind a feature flag, and require some additional services and configurations. #### Requirements: - `Dockerfile` --- webdriver to run a headless browser (for screenshots) +-- webdriver to run a headless browser (for taking screenshots of the charts and dahboards) - `docker-compose.yaml` --- redis message broker running +-- redis message broker -- replacing SQLlite DB with Postgres DB -- celery worker -- celery beat - `superset_config.py` --- feature flag turned on +-- feature flag turned to True -- all configs as outlined in the template below -- At least one of these is needed to sent alerts and reports: +- At least one of these is needed to send alerts and reports: -- (optional) SMTP server for sending email --- (optional) Slack app integration for sending to slack channels +-- (optional) Slack app integration for sending to Slack channels #### Summary of steps to turn on alerts and reporting: From 1a9e9e511e184e1d33787afba6173581d7238638 Mon Sep 17 00:00:00 2001 From: leocape <67806134+leocape@users.noreply.github.com> Date: Sat, 13 Feb 2021 11:40:12 +0200 Subject: [PATCH 04/18] fix punctuation fix bullets fix redis depends on name in docker-compose template --- .../pages/docs/installation/email_reports.mdx | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index 18a8cd948166..78063e24a41b 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -19,19 +19,19 @@ Alerts and reports are not turned on by default. They are currently behind a fea #### Requirements: -- `Dockerfile` --- webdriver to run a headless browser (for taking screenshots of the charts and dahboards) + `Dockerfile` + - webdriver to run a headless browser (for taking screenshots of the charts and dahboards) - `docker-compose.yaml` --- redis message broker --- replacing SQLlite DB with Postgres DB --- celery worker --- celery beat + - redis message broker + - replacing SQLlite DB with Postgres DB + - celery worker + - celery beat - `superset_config.py` --- feature flag turned to True --- all configs as outlined in the template below + - feature flag turned to True + - all configs as outlined in the template below - At least one of these is needed to send alerts and reports: --- (optional) SMTP server for sending email --- (optional) Slack app integration for sending to Slack channels + - (optional) SMTP server for sending email + - (optional) Slack app integration for sending to Slack channels #### Summary of steps to turn on alerts and reporting: @@ -117,7 +117,7 @@ services: depends_on: - superset - postgres - - redis-superset + - redis command: "celery worker --app=superset.tasks.celery_app:app --pool=prefork --max-tasks-per-child=128 -O fair" volumes: - ./config/:/app/pythonpath/ @@ -129,7 +129,7 @@ services: depends_on: - superset - postgres - - redis-superset + - redis command: "celery beat --app=superset.tasks.celery_app:app --pidfile /tmp/celerybeat.pid --schedule /tmp/celerybeat-schedule" volumes: - ./config/:/app/pythonpath/ @@ -142,7 +142,7 @@ services: - "8088:8088" depends_on: - postgres - - redis-superset + - redis command: gunicorn --bind 0.0.0.0:8088 --access-logfile - --error-logfile - --workers 5 --worker-class gthread --threads 4 --timeout 200 --limit-request-line 4094 --limit-request-field_size 8190 superset.app:create_app() volumes: - ./config/:/app/pythonpath/ @@ -286,15 +286,15 @@ SUPERSET_WEBSERVER_TIMEOUT=600 With the extended image created by using the `Dockerfile`, and then running that image using `docker-compose.yaml`, plus the required configurations in the `superset_config.py` you should now have alerts and reporting working correctly. - For Kubernetes you can see the Helm chart here -- The above templates also work in a Docker swarm environment, you would just need to add `Deploy:` to the Superset, redis and Postgres services along with your specific configs for your swarm +- The above templates also work in a Docker swarm environment, you would just need to add `Deploy:` to the Superset, Redis and Postgres services along with your specific configs for your swarm ### Optional - Slack integration To send alerts and reports to a Slack channel, you need to create a new Slack APP on your domain. 1. Head to https://api.slack.com/apps 2. Create a new APP, give it a name (eg. Superset) 3. Under the OAuth and Permissions section, give the following scopes to the app: - a) incoming-webhook - b) calls:write + 1. `incoming-webhook` + 2. `calls:write` 4. At the top of the OAuth and Permissions section, click 'install to workspace' 5. Select a default channel for the app to post to and continue. (You can post to any channel by inviting your Superset app into that channel) 6. The app should now be installed on the workspace, and a 'Bot User OAuth Access Token' should be created. Copy the OAuth token and add it into the Slack section in the `superset_config.py` From 4fa3b6c7185629b87c27fc2c0e5435d458f7b73d Mon Sep 17 00:00:00 2001 From: leocape <67806134+leocape@users.noreply.github.com> Date: Sat, 13 Feb 2021 11:43:13 +0200 Subject: [PATCH 05/18] fix bullets --- docs/src/pages/docs/installation/email_reports.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index 78063e24a41b..446a23b5a800 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -19,7 +19,7 @@ Alerts and reports are not turned on by default. They are currently behind a fea #### Requirements: - `Dockerfile` +- `Dockerfile` - webdriver to run a headless browser (for taking screenshots of the charts and dahboards) - `docker-compose.yaml` - redis message broker From 6b565707f03a0c592a85deb112c1d1d67903fc96 Mon Sep 17 00:00:00 2001 From: leocape <67806134+leocape@users.noreply.github.com> Date: Mon, 15 Feb 2021 09:13:04 +0200 Subject: [PATCH 06/18] Add extra steps for getting v1.0.1 running step added: Upgrade the db step added: init step added: setup admin fix redis superset naming add . to docker build command --- docs/src/pages/docs/installation/email_reports.mdx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index 446a23b5a800..817a97c696b5 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -42,6 +42,10 @@ Using the templates below, 4. Create a new sub directory called `config` 5. Create the `superset_config.py` file in the `config` sub directory 6. Run the image using `docker-compose up` in the same directory as the `docker-compose.py` file +7. In a new terminal window, upgrade the DB by running `docker exec -it superset-1.0.1-extended superset db upgrade` +8. Then run `docker exec -it superset-1.0.1-extended superset init` +9. Then setup your admin user if need be, `docker exec -it superset-1.0.1-extended superset fab create-admin` +10. Finally, restart the running instance - `CTRL-C`, then `docker-compose up` (note: v 1.0.1 is current at time of writing, you can change the version number to the latest version if a newer version is available) ### Dockerfile @@ -49,7 +53,7 @@ Using the templates below, A webdriver (and headless browser) is needed to capture screenshots of the charts and dashboards which are then sent to the recipient. As the base image does not have a webdriver installed by default, we need to extend the base image and install the webdriver (this template uses the Chrome webdriver). We are also adding in connectors for Mysql and Postgres, as well as Redis and Flower (Flower and Mysql are optional depending on your requirements) You can extend the image by running this Docker build command from the directory that contains the Dockerfile: -`docker build -t superset-1.0.1-extended -f Dockerfile` +`docker build -t superset-1.0.1-extended -f Dockerfile .` Config for `Dockerfile`: ```docker @@ -95,7 +99,7 @@ Config for `docker-compose.yaml`: ```docker version: '3.6' services: - redis-superset: + redis: image: redis:6.0.9-buster restart: on-failure volumes: From a8c27b92e59a6036e2d72e6fd18e1b0865d3fa18 Mon Sep 17 00:00:00 2001 From: leocape <67806134+leocape@users.noreply.github.com> Date: Mon, 15 Feb 2021 11:29:35 +0200 Subject: [PATCH 07/18] fix worker to use gevent instead of prefork change postgres to internal so it persists on docker-compose down --- docs/src/pages/docs/installation/email_reports.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index 817a97c696b5..e7475d7372a4 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -122,7 +122,7 @@ services: - superset - postgres - redis - command: "celery worker --app=superset.tasks.celery_app:app --pool=prefork --max-tasks-per-child=128 -O fair" + command: "celery worker --app=superset.tasks.celery_app:app --pool=gevent --concurrency=500" volumes: - ./config/:/app/pythonpath/ beat: @@ -152,7 +152,7 @@ services: - ./config/:/app/pythonpath/ volumes: db: - external: false + external: true redis: external: false ``` From 1f3674fcbe8661fd67385af3606225fd0afb1e86 Mon Sep 17 00:00:00 2001 From: leocape <67806134+leocape@users.noreply.github.com> Date: Mon, 15 Feb 2021 14:09:01 +0200 Subject: [PATCH 08/18] add comments to superset_config.py --- docs/src/pages/docs/installation/email_reports.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index e7475d7372a4..238a73d1e82c 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -194,13 +194,13 @@ SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://%s:%s@%s:%s/%s?client_encoding= POSTGRES_DATABASE) CACHE_CONFIG: CacheConfig = { 'CACHE_TYPE': 'redis', - 'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day + 'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day - if your alerts are running faster than 1 day, the cache should match 'CACHE_KEY_PREFIX': 'superset_', 'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) } DATA_CACHE_CONFIG: CacheConfig = { 'CACHE_TYPE': 'redis', - 'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day + 'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day - if your alerts are running faster than 1 day, the cache should match 'CACHE_KEY_PREFIX': 'data_', 'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) } @@ -254,11 +254,11 @@ class CeleryConfig(object): CELERY_CONFIG = CeleryConfig # SMTP email configuration -EMAIL_REPORTS_USER="admin" +EMAIL_REPORTS_USER="admin" # change to a user that has access to the chart / dashboard that you are sending EMAIL_PAGE_RENDER_WAIT=300 EMAIL_NOTIFICATIONS = True -SMTP_HOST = "smtp.sendgrid.net" +SMTP_HOST = "smtp.sendgrid.net" #change to your host SMTP_STARTTLS = True SMTP_SSL = False SMTP_USER = "your_user" From c778a2e9f2556fccb9ac019fa89877ece30ec18b Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Mon, 15 Feb 2021 16:35:38 +0100 Subject: [PATCH 09/18] ref: Re-organize a bit and make it easier to read --- .../pages/docs/installation/email_reports.mdx | 72 ++++++++++--------- 1 file changed, 40 insertions(+), 32 deletions(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index 238a73d1e82c..f454e8504cb2 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -9,32 +9,53 @@ version: 1 ## Alerts and Reports (version 1.0.1 and above) -Users can configure automated alerts and reports to send charts and dashboards to an email recipient or Slack channel. +Users can configure automated alerts and reports to send dashboards or charts to an email recipient or Slack channel. -- Alerts are sent when a specified condition is passed -- Reports are sent on a specified schedule +- Alerts are sent when a SQL condition is reached +- Reports are sent on a schedule ### Turning on Alerts and reports -Alerts and reports are not turned on by default. They are currently behind a feature flag, and require some additional services and configurations. +Alerts and reports are disabled by default. To turn them on, you need to do some setup, described here. -#### Requirements: +#### Requirements summary: -- `Dockerfile` - - webdriver to run a headless browser (for taking screenshots of the charts and dahboards) +##### Commons +- In your `superset_config.py`: + - `"ALERT_REPORTS"` feature flag must be turned to True. + - `CELERYBEAT_SCHEDULE` in CeleryConfig must contains schedule for `reports.scheduler`. + - At least one of those must be configured, depending of what you want to use: + - emails: `SMTP_*` settings + - slack messages: `SLACK_API_TOKEN` +- In your `Dockerfile`: + - You must install a headless browser, for taking screenshots of the charts and dahboards (only firefox and chrome are currently supported). + +##### Docker-compose specific - `docker-compose.yaml` - - redis message broker - - replacing SQLlite DB with Postgres DB - - celery worker - - celery beat -- `superset_config.py` - - feature flag turned to True - - all configs as outlined in the template below -- At least one of these is needed to send alerts and reports: - - (optional) SMTP server for sending email - - (optional) Slack app integration for sending to Slack channels - -#### Summary of steps to turn on alerts and reporting: + - redis message broker + - replacing SQLlite DB with Postgres DB + - celery worker + - celery beat + +##### Kubernetes specific +- you must have a celery beat pod + +##### Slack integration +To send alerts and reports to Slack channels, you need to create a new Slack Application on your workspace. + +1. Connect to your Slack workspace, then head to https://api.slack.com/apps. +2. Create a new app. +3. Go to `OAuth & Permissions` section, and give the following scopes to your app: + 1. `incoming-webhook` + 2. `files:write` +4. At the top of the OAuth and Permissions section, click 'install to workspace'. +5. Select a default channel for your app to post to and continue. +(You can post to any channel by inviting your Superset app into that channel). +6. The app should now be installed on your workspace, and a 'Bot User OAuth Access Token' should have been created. Copy that token in the `SLACK_API_TOKEN` variable of your `superset_config.py`. +7. Restart the service (or run `superset init`) to pull in the new configuration. +Note: when you configure an alert or a report, the Slack channel list take channel names without the leading '#' eg. use `alerts` instead of `#alerts`. + +#### Summary of steps to turn on alerts and reporting: Using the templates below, 1. Create a new directory and create the Dockerfile 2. Build the extended image using the Dockerfile @@ -292,19 +313,6 @@ With the extended image created by using the `Dockerfile`, and then running that - For Kubernetes you can see the Helm chart here - The above templates also work in a Docker swarm environment, you would just need to add `Deploy:` to the Superset, Redis and Postgres services along with your specific configs for your swarm -### Optional - Slack integration -To send alerts and reports to a Slack channel, you need to create a new Slack APP on your domain. -1. Head to https://api.slack.com/apps -2. Create a new APP, give it a name (eg. Superset) -3. Under the OAuth and Permissions section, give the following scopes to the app: - 1. `incoming-webhook` - 2. `calls:write` -4. At the top of the OAuth and Permissions section, click 'install to workspace' -5. Select a default channel for the app to post to and continue. (You can post to any channel by inviting your Superset app into that channel) -6. The app should now be installed on the workspace, and a 'Bot User OAuth Access Token' should be created. Copy the OAuth token and add it into the Slack section in the `superset_config.py` -7. Restart the service (or run `superset init`) to pull in the new configuration. -8. Note when sending to the channel from the alerts and reports UI, set the channel without the leading '#' eg. use `alerts` instead of `#alerts` - # ## Scheduling and Emailing Reports From b08a0f41e72b42f2b7a9c4531027b2332968717b Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Mon, 15 Feb 2021 17:15:30 +0100 Subject: [PATCH 10/18] ref: rm useless level --- .../src/pages/docs/installation/email_reports.mdx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index f454e8504cb2..e0982b962423 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -14,12 +14,11 @@ Users can configure automated alerts and reports to send dashboards or charts to - Alerts are sent when a SQL condition is reached - Reports are sent on a schedule -### Turning on Alerts and reports Alerts and reports are disabled by default. To turn them on, you need to do some setup, described here. -#### Requirements summary: +### Requirements summary: -##### Commons +#### Commons - In your `superset_config.py`: - `"ALERT_REPORTS"` feature flag must be turned to True. - `CELERYBEAT_SCHEDULE` in CeleryConfig must contains schedule for `reports.scheduler`. @@ -27,19 +26,19 @@ Alerts and reports are disabled by default. To turn them on, you need to do some - emails: `SMTP_*` settings - slack messages: `SLACK_API_TOKEN` - In your `Dockerfile`: - - You must install a headless browser, for taking screenshots of the charts and dahboards (only firefox and chrome are currently supported). + - You must install a headless browser, for taking screenshots of the charts and dashboards (only firefox and chrome are currently supported). -##### Docker-compose specific +#### Docker-compose specific - `docker-compose.yaml` - redis message broker - replacing SQLlite DB with Postgres DB - celery worker - celery beat -##### Kubernetes specific +#### Kubernetes specific - you must have a celery beat pod -##### Slack integration +#### Slack integration To send alerts and reports to Slack channels, you need to create a new Slack Application on your workspace. 1. Connect to your Slack workspace, then head to https://api.slack.com/apps. @@ -55,7 +54,7 @@ To send alerts and reports to Slack channels, you need to create a new Slack App Note: when you configure an alert or a report, the Slack channel list take channel names without the leading '#' eg. use `alerts` instead of `#alerts`. -#### Summary of steps to turn on alerts and reporting: +### Summary of steps to turn on alerts and reporting: Using the templates below, 1. Create a new directory and create the Dockerfile 2. Build the extended image using the Dockerfile From 7afbc4c6ef1f2bfa9dddbe3ba82d7c15119fd5c0 Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Fri, 19 Feb 2021 09:43:13 +0100 Subject: [PATCH 11/18] docs: md changes + more details + kubernetes --- .../pages/docs/installation/email_reports.mdx | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/email_reports.mdx index e0982b962423..e43ec2b6d4b4 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/email_reports.mdx @@ -19,37 +19,40 @@ Alerts and reports are disabled by default. To turn them on, you need to do some ### Requirements summary: #### Commons -- In your `superset_config.py`: - - `"ALERT_REPORTS"` feature flag must be turned to True. - - `CELERYBEAT_SCHEDULE` in CeleryConfig must contains schedule for `reports.scheduler`. - - At least one of those must be configured, depending of what you want to use: - - emails: `SMTP_*` settings - - slack messages: `SLACK_API_TOKEN` -- In your `Dockerfile`: - - You must install a headless browser, for taking screenshots of the charts and dashboards (only firefox and chrome are currently supported). + +##### In your `superset_config.py`: +- `"ALERT_REPORTS"` feature flag must be turned to True. +- `CELERYBEAT_SCHEDULE` in CeleryConfig must contain schedule for `reports.scheduler`. +- At least one of those must be configured, depending on what you want to use: + - emails: `SMTP_*` settings + - Slack messages: `SLACK_API_TOKEN` + +##### In your `Dockerfile`: +- You must install a headless browser, for taking screenshots of the charts and dashboards. Only Firefox and Chrome are currently supported. + >If you choose Chrome, you must also change the value of `WEBDRIVER_TYPE` to `"chrome"` in your `superset_config.py`. #### Docker-compose specific - `docker-compose.yaml` - - redis message broker - - replacing SQLlite DB with Postgres DB - - celery worker - - celery beat + - redis message broker + - replacing SQLlite DB with Postgres DB + - celery worker + - celery beat #### Kubernetes specific -- you must have a celery beat pod +- You must have a `celery beat` pod running. If you're using the chart included in the GitHub repository under [helm/superset](https://github.com/apache/superset/tree/master/helm/superset), you need to put `supersetCeleryBeat.enabled = true` in your values override. #### Slack integration To send alerts and reports to Slack channels, you need to create a new Slack Application on your workspace. 1. Connect to your Slack workspace, then head to https://api.slack.com/apps. 2. Create a new app. -3. Go to `OAuth & Permissions` section, and give the following scopes to your app: - 1. `incoming-webhook` - 2. `files:write` -4. At the top of the OAuth and Permissions section, click 'install to workspace'. -5. Select a default channel for your app to post to and continue. +3. Go to "OAuth & Permissions" section, and give the following scopes to your app: + 1. `incoming-webhook` + 2. `files:write` +4. At the top of the "OAuth and Permissions" section, click "install to workspace". +5. Select a default channel for your app and continue. (You can post to any channel by inviting your Superset app into that channel). -6. The app should now be installed on your workspace, and a 'Bot User OAuth Access Token' should have been created. Copy that token in the `SLACK_API_TOKEN` variable of your `superset_config.py`. +6. The app should now be installed in your workspace, and a "Bot User OAuth Access Token" should have been created. Copy that token in the `SLACK_API_TOKEN` variable of your `superset_config.py`. 7. Restart the service (or run `superset init`) to pull in the new configuration. Note: when you configure an alert or a report, the Slack channel list take channel names without the leading '#' eg. use `alerts` instead of `#alerts`. @@ -59,8 +62,8 @@ Using the templates below, 1. Create a new directory and create the Dockerfile 2. Build the extended image using the Dockerfile 3. Create the `docker-compose.yaml` file in the same directory -4. Create a new sub directory called `config` -5. Create the `superset_config.py` file in the `config` sub directory +4. Create a new subdirectory called `config` +5. Create the `superset_config.py` file in the `config` subdirectory 6. Run the image using `docker-compose up` in the same directory as the `docker-compose.py` file 7. In a new terminal window, upgrade the DB by running `docker exec -it superset-1.0.1-extended superset db upgrade` 8. Then run `docker exec -it superset-1.0.1-extended superset init` @@ -70,7 +73,7 @@ Using the templates below, (note: v 1.0.1 is current at time of writing, you can change the version number to the latest version if a newer version is available) ### Dockerfile -A webdriver (and headless browser) is needed to capture screenshots of the charts and dashboards which are then sent to the recipient. As the base image does not have a webdriver installed by default, we need to extend the base image and install the webdriver (this template uses the Chrome webdriver). We are also adding in connectors for Mysql and Postgres, as well as Redis and Flower (Flower and Mysql are optional depending on your requirements) +A webdriver (and headless browser) is needed to capture screenshots of the charts and dashboards which are then sent to the recipient. As the base image does not have a webdriver installed by default, we need to extend the base image and install the webdriver (this template uses the Chrome webdriver). We are also adding in connectors for MySQL and Postgres, as well as Redis and Flower (Flower and MySQL are optional depending on your requirements) You can extend the image by running this Docker build command from the directory that contains the Dockerfile: `docker build -t superset-1.0.1-extended -f Dockerfile .` From 196841ecbaea62803291bfc45b5882a3f74a55bc Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Fri, 19 Feb 2021 09:44:26 +0100 Subject: [PATCH 12/18] docs: Rename to Alerts and Reports --- .../installation/{email_reports.mdx => alerts_reports.mdx} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename docs/src/pages/docs/installation/{email_reports.mdx => alerts_reports.mdx} (99%) diff --git a/docs/src/pages/docs/installation/email_reports.mdx b/docs/src/pages/docs/installation/alerts_reports.mdx similarity index 99% rename from docs/src/pages/docs/installation/email_reports.mdx rename to docs/src/pages/docs/installation/alerts_reports.mdx index e43ec2b6d4b4..15259de55991 100644 --- a/docs/src/pages/docs/installation/email_reports.mdx +++ b/docs/src/pages/docs/installation/alerts_reports.mdx @@ -1,9 +1,9 @@ --- -name: Scheduling and Emailing Reports +name: Alerts and Reports menu: Installation and Configuration -route: /docs/installation/email-reports +route: /docs/installation/alerts-reports index: 10 -version: 1 +version: 2 --- ## Alerts and Reports From 9e0551da81b590caada44dcb9662513eef874c9d Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Fri, 19 Feb 2021 10:07:43 +0100 Subject: [PATCH 13/18] docs: Link to Kubernetes doc --- docs/src/pages/docs/installation/alerts_reports.mdx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/src/pages/docs/installation/alerts_reports.mdx b/docs/src/pages/docs/installation/alerts_reports.mdx index 15259de55991..2ed7e3453c83 100644 --- a/docs/src/pages/docs/installation/alerts_reports.mdx +++ b/docs/src/pages/docs/installation/alerts_reports.mdx @@ -40,6 +40,7 @@ Alerts and reports are disabled by default. To turn them on, you need to do some #### Kubernetes specific - You must have a `celery beat` pod running. If you're using the chart included in the GitHub repository under [helm/superset](https://github.com/apache/superset/tree/master/helm/superset), you need to put `supersetCeleryBeat.enabled = true` in your values override. +- You can see the dedicated docs about [Kubernetes installation](/docs/installation/running-on-kubernetes) for more generic details. #### Slack integration To send alerts and reports to Slack channels, you need to create a new Slack Application on your workspace. @@ -312,11 +313,11 @@ SUPERSET_WEBSERVER_TIMEOUT=600 ### Summary With the extended image created by using the `Dockerfile`, and then running that image using `docker-compose.yaml`, plus the required configurations in the `superset_config.py` you should now have alerts and reporting working correctly. -- For Kubernetes you can see the Helm chart here - The above templates also work in a Docker swarm environment, you would just need to add `Deploy:` to the Superset, Redis and Postgres services along with your specific configs for your swarm -# +# Old Reports feature + ## Scheduling and Emailing Reports (version 0.38 and below) ### Email Reports From 2758a2ebc122b7bfccda01d752b9030d01f1199f Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Fri, 19 Feb 2021 10:25:32 +0100 Subject: [PATCH 14/18] docs: details for docker-compose + minor md changes --- .../docs/installation/alerts_reports.mdx | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/src/pages/docs/installation/alerts_reports.mdx b/docs/src/pages/docs/installation/alerts_reports.mdx index 2ed7e3453c83..06c15b7e7b78 100644 --- a/docs/src/pages/docs/installation/alerts_reports.mdx +++ b/docs/src/pages/docs/installation/alerts_reports.mdx @@ -16,32 +16,21 @@ Users can configure automated alerts and reports to send dashboards or charts to Alerts and reports are disabled by default. To turn them on, you need to do some setup, described here. -### Requirements summary: +### Requirements #### Commons -##### In your `superset_config.py`: +##### In your `superset_config.py` - `"ALERT_REPORTS"` feature flag must be turned to True. - `CELERYBEAT_SCHEDULE` in CeleryConfig must contain schedule for `reports.scheduler`. - At least one of those must be configured, depending on what you want to use: - emails: `SMTP_*` settings - Slack messages: `SLACK_API_TOKEN` -##### In your `Dockerfile`: +##### In your `Dockerfile` - You must install a headless browser, for taking screenshots of the charts and dashboards. Only Firefox and Chrome are currently supported. >If you choose Chrome, you must also change the value of `WEBDRIVER_TYPE` to `"chrome"` in your `superset_config.py`. -#### Docker-compose specific -- `docker-compose.yaml` - - redis message broker - - replacing SQLlite DB with Postgres DB - - celery worker - - celery beat - -#### Kubernetes specific -- You must have a `celery beat` pod running. If you're using the chart included in the GitHub repository under [helm/superset](https://github.com/apache/superset/tree/master/helm/superset), you need to put `supersetCeleryBeat.enabled = true` in your values override. -- You can see the dedicated docs about [Kubernetes installation](/docs/installation/running-on-kubernetes) for more generic details. - #### Slack integration To send alerts and reports to Slack channels, you need to create a new Slack Application on your workspace. @@ -56,7 +45,18 @@ To send alerts and reports to Slack channels, you need to create a new Slack App 6. The app should now be installed in your workspace, and a "Bot User OAuth Access Token" should have been created. Copy that token in the `SLACK_API_TOKEN` variable of your `superset_config.py`. 7. Restart the service (or run `superset init`) to pull in the new configuration. -Note: when you configure an alert or a report, the Slack channel list take channel names without the leading '#' eg. use `alerts` instead of `#alerts`. +Note: when you configure an alert or a report, the Slack channel list take channel names without the leading '#' e.g. use `alerts` instead of `#alerts`. + +#### Kubernetes specific +- You must have a `celery beat` pod running. If you're using the chart included in the GitHub repository under [helm/superset](https://github.com/apache/superset/tree/master/helm/superset), you need to put `supersetCeleryBeat.enabled = true` in your values override. +- You can see the dedicated docs about [Kubernetes installation](/docs/installation/running-on-kubernetes) for more generic details. + +#### Docker-compose specific +##### You must have in your`docker-compose.yaml` + - a redis message broker + - PostgreSQL DB instead of SQLlite + - one or more `celery worker` + - a single `celery beat` ### Summary of steps to turn on alerts and reporting: Using the templates below, @@ -72,8 +72,8 @@ Using the templates below, 10. Finally, restart the running instance - `CTRL-C`, then `docker-compose up` (note: v 1.0.1 is current at time of writing, you can change the version number to the latest version if a newer version is available) -### Dockerfile +### Dockerfile A webdriver (and headless browser) is needed to capture screenshots of the charts and dashboards which are then sent to the recipient. As the base image does not have a webdriver installed by default, we need to extend the base image and install the webdriver (this template uses the Chrome webdriver). We are also adding in connectors for MySQL and Postgres, as well as Redis and Flower (Flower and MySQL are optional depending on your requirements) You can extend the image by running this Docker build command from the directory that contains the Dockerfile: From e859bbef9b5ed42337ed393f635951540e0768fc Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Fri, 19 Feb 2021 10:54:21 +0100 Subject: [PATCH 15/18] docs: fix lists indent --- docs/src/pages/docs/installation/alerts_reports.mdx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/src/pages/docs/installation/alerts_reports.mdx b/docs/src/pages/docs/installation/alerts_reports.mdx index 06c15b7e7b78..87324786dd37 100644 --- a/docs/src/pages/docs/installation/alerts_reports.mdx +++ b/docs/src/pages/docs/installation/alerts_reports.mdx @@ -1,3 +1,4 @@ + --- name: Alerts and Reports menu: Installation and Configuration @@ -24,12 +25,12 @@ Alerts and reports are disabled by default. To turn them on, you need to do some - `"ALERT_REPORTS"` feature flag must be turned to True. - `CELERYBEAT_SCHEDULE` in CeleryConfig must contain schedule for `reports.scheduler`. - At least one of those must be configured, depending on what you want to use: - - emails: `SMTP_*` settings - - Slack messages: `SLACK_API_TOKEN` + - emails: `SMTP_*` settings + - Slack messages: `SLACK_API_TOKEN` ##### In your `Dockerfile` - You must install a headless browser, for taking screenshots of the charts and dashboards. Only Firefox and Chrome are currently supported. - >If you choose Chrome, you must also change the value of `WEBDRIVER_TYPE` to `"chrome"` in your `superset_config.py`. + > If you choose Chrome, you must also change the value of `WEBDRIVER_TYPE` to `"chrome"` in your `superset_config.py`. #### Slack integration To send alerts and reports to Slack channels, you need to create a new Slack Application on your workspace. @@ -37,8 +38,8 @@ To send alerts and reports to Slack channels, you need to create a new Slack App 1. Connect to your Slack workspace, then head to https://api.slack.com/apps. 2. Create a new app. 3. Go to "OAuth & Permissions" section, and give the following scopes to your app: - 1. `incoming-webhook` - 2. `files:write` + - `incoming-webhook` + - `files:write` 4. At the top of the "OAuth and Permissions" section, click "install to workspace". 5. Select a default channel for your app and continue. (You can post to any channel by inviting your Superset app into that channel). From f81da929a35c0a6ecb9736ea1cd13f59b9b699b9 Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Fri, 19 Feb 2021 13:00:48 +0100 Subject: [PATCH 16/18] docs: remove unused config --- .../docs/installation/alerts_reports.mdx | 84 +++++-------------- 1 file changed, 19 insertions(+), 65 deletions(-) diff --git a/docs/src/pages/docs/installation/alerts_reports.mdx b/docs/src/pages/docs/installation/alerts_reports.mdx index 87324786dd37..856366e0ab89 100644 --- a/docs/src/pages/docs/installation/alerts_reports.mdx +++ b/docs/src/pages/docs/installation/alerts_reports.mdx @@ -71,7 +71,7 @@ Using the templates below, 8. Then run `docker exec -it superset-1.0.1-extended superset init` 9. Then setup your admin user if need be, `docker exec -it superset-1.0.1-extended superset fab create-admin` 10. Finally, restart the running instance - `CTRL-C`, then `docker-compose up` - + (note: v 1.0.1 is current at time of writing, you can change the version number to the latest version if a newer version is available) ### Dockerfile @@ -112,10 +112,10 @@ To ferry requests between the celery worker and the Superset instance, we use a While it might be possible to use SQLite for alerts and reporting, it is highly recommended to use a more production ready DB for Superset in general. Our template uses Postgres. #### Celery worker -The worker will process the tasks that need to be performed when an alert or report is fired. +The worker will process the tasks that need to be performed when an alert or report is fired. #### Celery beat -The beat is the scheduler that tells the worker when to perform its tasks. This schedule is defined when you create the alert or report. +The beat is the scheduler that tells the worker when to perform its tasks. This schedule is defined when you create the alert or report. #### Full `docker-compose.yaml` configuration The Redis, Postgres, Celery worker and Celery beat services are defined in the template: @@ -127,7 +127,7 @@ services: redis: image: redis:6.0.9-buster restart: on-failure - volumes: + volumes: - redis:/data postgres: image: postgres @@ -138,7 +138,7 @@ services: POSTGRES_USER: superset volumes: - db:/var/lib/postgresql/data - worker: + worker: image: superset-1.0.1-extended restart: on-failure healthcheck: @@ -186,61 +186,24 @@ volumes: The following configurations need to be added to the `superset_config.py` file. This file is loaded when the image runs, and any configurations in it will override the default configurations found in the `config.py`. -You will need to add your custom SMTP settings, and or Slack APP token +You need to replace default values with your custom Redis, Slack and/or SMTP config. + +As said before, in the`CeleryConfig`, only the `CELERYBEAT_SCHEDULE` is relative to this feature, the rest of the `CeleryConfig` can be changed for your needs. Config for `superset_config.py`: ```python -from superset_config import * from celery.schedules import crontab -from cachelib import RedisCache -from superset.typing import CacheConfig -import os FEATURE_FLAGS = { "ALERT_REPORTS": True } -# slack API token (optional) SLACK_API_TOKEN = "xoxb-" -SLACK_PROXY = None -POSTGRES_USER = "superset" -POSTGRES_PASS = "superset" -POSTGRES_HOST = "postgres" -POSTGRES_PORT = "5432" -POSTGRES_DATABASE = "superset" REDIS_HOST = "redis-superset" REDIS_PORT = "6379" -# The SQLAlchemy connection string. -SQLALCHEMY_DATABASE_URI = 'postgresql+psycopg2://%s:%s@%s:%s/%s?client_encoding=utf8' % (POSTGRES_USER, - POSTGRES_PASS, - POSTGRES_HOST, - POSTGRES_PORT, - POSTGRES_DATABASE) -CACHE_CONFIG: CacheConfig = { - 'CACHE_TYPE': 'redis', - 'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day - if your alerts are running faster than 1 day, the cache should match - 'CACHE_KEY_PREFIX': 'superset_', - 'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) -} -DATA_CACHE_CONFIG: CacheConfig = { - 'CACHE_TYPE': 'redis', - 'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day - if your alerts are running faster than 1 day, the cache should match - 'CACHE_KEY_PREFIX': 'data_', - 'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) -} -THUMBNAIL_SELENIUM_USER = "admin" -THUMBNAIL_CACHE_CONFIG: CacheConfig = { - 'CACHE_TYPE': 'redis', - 'CACHE_DEFAULT_TIMEOUT': 24*60*60*30, - 'CACHE_KEY_PREFIX': 'thumbnail_', - 'CACHE_NO_NULL_WARNING': True, - 'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT) -} -SCREENSHOT_LOCATE_WAIT = 100 -SCREENSHOT_LOAD_WAIT = 600 -RESULTS_BACKEND = RedisCache(host=REDIS_HOST, port=REDIS_PORT, key_prefix='superset_results') -class CeleryConfig(object): + +class CeleryConfig: BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) CELERY_IMPORTS = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", ) CELERY_RESULT_BACKEND = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) @@ -266,23 +229,13 @@ class CeleryConfig(object): 'task': 'reports.prune_log', 'schedule': crontab(minute=0, hour=0), }, - 'cache-warmup-hourly': { - 'task': 'cache-warmup', - 'schedule': crontab(minute='*/30', hour='*'), - 'kwargs': { - 'strategy_name': 'top_n_dashboards', - 'top_n': 10, - 'since': '7 days ago', - }, - }, } CELERY_CONFIG = CeleryConfig -# SMTP email configuration -EMAIL_REPORTS_USER="admin" # change to a user that has access to the chart / dashboard that you are sending -EMAIL_PAGE_RENDER_WAIT=300 -EMAIL_NOTIFICATIONS = True +SCREENSHOT_LOCATE_WAIT = 100 +SCREENSHOT_LOAD_WAIT = 600 +# SMTP email configuration SMTP_HOST = "smtp.sendgrid.net" #change to your host SMTP_STARTTLS = True SMTP_SSL = False @@ -291,6 +244,9 @@ SMTP_PORT = 2525 # your port eg. 587 SMTP_PASSWORD = "your_password" SMTP_MAIL_FROM = "noreply@youremail.com" +# WebDriver configuration +# If you use Firefox, you can stick with default values +# If you use Chrome, then add the following WEBDRIVER_TYPE and WEBDRIVER_OPTION_ARGS WEBDRIVER_TYPE= "chrome" WEBDRIVER_OPTION_ARGS = [ "--force-device-scale-factor=2.0", @@ -303,12 +259,10 @@ WEBDRIVER_OPTION_ARGS = [ "--disable-extensions", ] +# This is for internal use, you can keep http WEBDRIVER_BASEURL="http://superset:8088" -WEBDRIVER_BASEURL_USER_FRIENDLY="http://localhost:8088" # change to your domain eg. https://superset.mydomain.com - this is the link that is sent to the recipient -SUPERSET_WEBSERVER_ADDRESS = "localhost" -SUPERSET_WEBSERVER_PORT = 8088 -SUPERSET_WEBSERVER_TIMEOUT=600 - +# This is the link sent to the recipient, change to your domain eg. https://superset.mydomain.com +WEBDRIVER_BASEURL_USER_FRIENDLY="http://localhost:8088" ``` ### Summary From 70bd92b0227afed87d061b811c82c11153e0fd38 Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Fri, 19 Feb 2021 15:25:55 +0100 Subject: [PATCH 17/18] docs: Put detailed config and dockerfile before specific stuff --- .../docs/installation/alerts_reports.mdx | 242 ++++++++++-------- 1 file changed, 129 insertions(+), 113 deletions(-) diff --git a/docs/src/pages/docs/installation/alerts_reports.mdx b/docs/src/pages/docs/installation/alerts_reports.mdx index 856366e0ab89..6c6436d4089f 100644 --- a/docs/src/pages/docs/installation/alerts_reports.mdx +++ b/docs/src/pages/docs/installation/alerts_reports.mdx @@ -1,4 +1,3 @@ - --- name: Alerts and Reports menu: Installation and Configuration @@ -34,8 +33,7 @@ Alerts and reports are disabled by default. To turn them on, you need to do some #### Slack integration To send alerts and reports to Slack channels, you need to create a new Slack Application on your workspace. - -1. Connect to your Slack workspace, then head to https://api.slack.com/apps. +1. Connect to your Slack workspace, then head to . 2. Create a new app. 3. Go to "OAuth & Permissions" section, and give the following scopes to your app: - `incoming-webhook` @@ -59,33 +57,120 @@ Note: when you configure an alert or a report, the Slack channel list take chann - one or more `celery worker` - a single `celery beat` -### Summary of steps to turn on alerts and reporting: -Using the templates below, -1. Create a new directory and create the Dockerfile -2. Build the extended image using the Dockerfile -3. Create the `docker-compose.yaml` file in the same directory -4. Create a new subdirectory called `config` -5. Create the `superset_config.py` file in the `config` subdirectory -6. Run the image using `docker-compose up` in the same directory as the `docker-compose.py` file -7. In a new terminal window, upgrade the DB by running `docker exec -it superset-1.0.1-extended superset db upgrade` -8. Then run `docker exec -it superset-1.0.1-extended superset init` -9. Then setup your admin user if need be, `docker exec -it superset-1.0.1-extended superset fab create-admin` -10. Finally, restart the running instance - `CTRL-C`, then `docker-compose up` +### Detailed config +The following configurations need to be added to the `superset_config.py` file. This file is loaded when the image runs, and any configurations in it will override the default configurations found in the `config.py`. -(note: v 1.0.1 is current at time of writing, you can change the version number to the latest version if a newer version is available) +You can find documentation about each field in the default `config.py` in the GitHub repository under [superset/config.py](https://github.com/apache/superset/blob/master/superset/config.py). + +You need to replace default values with your custom Redis, Slack and/or SMTP config. + +In the `CeleryConfig`, only the `CELERYBEAT_SCHEDULE` is relative to this feature, the rest of the `CeleryConfig` can be changed for your needs. + +```python +from celery.schedules import crontab + +FEATURE_FLAGS = { + "ALERT_REPORTS": True +} + +REDIS_HOST = "redis-superset" +REDIS_PORT = "6379" + +class CeleryConfig: + BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) + CELERY_IMPORTS = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", ) + CELERY_RESULT_BACKEND = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) + CELERYD_PREFETCH_MULTIPLIER = 10 + CELERY_ACKS_LATE = True + CELERY_ANNOTATIONS = { + 'sql_lab.get_sql_results': { + 'rate_limit': '100/s', + }, + 'email_reports.send': { + 'rate_limit': '1/s', + 'time_limit': 600, + 'soft_time_limit': 600, + 'ignore_result': True, + }, + } + CELERYBEAT_SCHEDULE = { + 'reports.scheduler': { + 'task': 'reports.scheduler', + 'schedule': crontab(minute='*', hour='*'), + }, + 'reports.prune_log': { + 'task': 'reports.prune_log', + 'schedule': crontab(minute=0, hour=0), + }, + } +CELERY_CONFIG = CeleryConfig + +SCREENSHOT_LOCATE_WAIT = 100 +SCREENSHOT_LOAD_WAIT = 600 + +# Slack configuration +SLACK_API_TOKEN = "xoxb-" + +# Email configuration +SMTP_HOST = "smtp.sendgrid.net" #change to your host +SMTP_STARTTLS = True +SMTP_SSL = False +SMTP_USER = "your_user" +SMTP_PORT = 2525 # your port eg. 587 +SMTP_PASSWORD = "your_password" +SMTP_MAIL_FROM = "noreply@youremail.com" -### Dockerfile -A webdriver (and headless browser) is needed to capture screenshots of the charts and dashboards which are then sent to the recipient. As the base image does not have a webdriver installed by default, we need to extend the base image and install the webdriver (this template uses the Chrome webdriver). We are also adding in connectors for MySQL and Postgres, as well as Redis and Flower (Flower and MySQL are optional depending on your requirements) +# WebDriver configuration +# If you use Firefox, you can stick with default values +# If you use Chrome, then add the following WEBDRIVER_TYPE and WEBDRIVER_OPTION_ARGS +WEBDRIVER_TYPE= "chrome" +WEBDRIVER_OPTION_ARGS = [ + "--force-device-scale-factor=2.0", + "--high-dpi-support=2.0", + "--headless", + "--disable-gpu", + "--disable-dev-shm-usage", + "--no-sandbox", + "--disable-setuid-sandbox", + "--disable-extensions", +] -You can extend the image by running this Docker build command from the directory that contains the Dockerfile: -`docker build -t superset-1.0.1-extended -f Dockerfile .` +# This is for internal use, you can keep http +WEBDRIVER_BASEURL="http://superset:8088" +# This is the link sent to the recipient, change to your domain eg. https://superset.mydomain.com +WEBDRIVER_BASEURL_USER_FRIENDLY="http://localhost:8088" +``` -Config for `Dockerfile`: +### Custom Dockerfile +A webdriver (and headless browser) is needed to capture screenshots of the charts and dashboards which are then sent to the recipient. As the base superset image does not have a webdriver installed, we need to extend it and install the webdriver. + +#### Using Firefox ```docker FROM apache/superset:1.0.1 + USER root -RUN apt update -RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + +RUN apt-get update \ + && apt install --no-install-recommends -y firefox-esr + +ENV GECKODRIVER_VERSION=0.29.0 +RUN wget -q https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz \ + && tar -x geckodriver -zf geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz -O > /usr/bin/geckodriver \ + && chmod 755 /usr/bin/geckodriver \ + && rm geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz + +RUN pip install --no-cache gevent psycopg2 redis + +USER superset +``` +#### Using Chrome +```docker +FROM apache/superset:1.0.1 + +USER root + +RUN apt update && \ + wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ apt install -y --no-install-recommends ./google-chrome-stable_current_amd64.deb && \ wget https://chromedriver.storage.googleapis.com/88.0.4324.96/chromedriver_linux64.zip && \ unzip chromedriver_linux64.zip && \ @@ -94,14 +179,28 @@ RUN wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.d apt autoremove -yqq --purge && \ apt clean && \ rm -f google-chrome-stable_current_amd64.deb chromedriver_linux64.zip -RUN pip install --no-cache-dir gevent -RUN pip install --no-cache-dir mysqlclient -RUN pip install --no-cache-dir psycopg2 -RUN pip install --no-cache-dir redis -RUN pip install --no-cache-dir flower -USER superset +RUN pip install --no-cache gevent psycopg2 redis + +USER superset ``` +>Don't forget to set `WEBDRIVER_TYPE` and `WEBDRIVER_OPTION_ARGS` in your config if you use Chrome. + +### Summary of steps to turn on alerts and reporting: +Using the templates below, +1. Create a new directory and create the Dockerfile +2. Build the extended image using the Dockerfile +3. Create the `docker-compose.yaml` file in the same directory +4. Create a new subdirectory called `config` +5. Create the `superset_config.py` file in the `config` subdirectory +6. Run the image using `docker-compose up` in the same directory as the `docker-compose.py` file +7. In a new terminal window, upgrade the DB by running `docker exec -it superset-1.0.1-extended superset db upgrade` +8. Then run `docker exec -it superset-1.0.1-extended superset init` +9. Then setup your admin user if need be, `docker exec -it superset-1.0.1-extended superset fab create-admin` +10. Finally, restart the running instance - `CTRL-C`, then `docker-compose up` + +(note: v 1.0.1 is current at time of writing, you can change the version number to the latest version if a newer version is available) + ### Docker compose The docker compose file lists the services that will be used when running the image. The specific services needed for alerts and reporting are outlined below. @@ -109,7 +208,7 @@ The docker compose file lists the services that will be used when running the im To ferry requests between the celery worker and the Superset instance, we use a message broker. This template uses Redis. #### Replacing SQLite with Postgres -While it might be possible to use SQLite for alerts and reporting, it is highly recommended to use a more production ready DB for Superset in general. Our template uses Postgres. +While it might be possible to use SQLite for alerts and reporting, it is highly recommended using a more production ready DB for Superset in general. Our template uses Postgres. #### Celery worker The worker will process the tasks that need to be performed when an alert or report is fired. @@ -182,89 +281,6 @@ volumes: external: false ``` -### Superset_config.py - -The following configurations need to be added to the `superset_config.py` file. This file is loaded when the image runs, and any configurations in it will override the default configurations found in the `config.py`. - -You need to replace default values with your custom Redis, Slack and/or SMTP config. - -As said before, in the`CeleryConfig`, only the `CELERYBEAT_SCHEDULE` is relative to this feature, the rest of the `CeleryConfig` can be changed for your needs. - -Config for `superset_config.py`: -```python -from celery.schedules import crontab - -FEATURE_FLAGS = { - "ALERT_REPORTS": True -} - -SLACK_API_TOKEN = "xoxb-" - -REDIS_HOST = "redis-superset" -REDIS_PORT = "6379" - -class CeleryConfig: - BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) - CELERY_IMPORTS = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", ) - CELERY_RESULT_BACKEND = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) - CELERYD_PREFETCH_MULTIPLIER = 10 - CELERY_ACKS_LATE = True - CELERY_ANNOTATIONS = { - 'sql_lab.get_sql_results': { - 'rate_limit': '100/s', - }, - 'email_reports.send': { - 'rate_limit': '1/s', - 'time_limit': 600, - 'soft_time_limit': 600, - 'ignore_result': True, - }, - } - CELERYBEAT_SCHEDULE = { - 'reports.scheduler': { - 'task': 'reports.scheduler', - 'schedule': crontab(minute='*', hour='*'), - }, - 'reports.prune_log': { - 'task': 'reports.prune_log', - 'schedule': crontab(minute=0, hour=0), - }, - } -CELERY_CONFIG = CeleryConfig - -SCREENSHOT_LOCATE_WAIT = 100 -SCREENSHOT_LOAD_WAIT = 600 - -# SMTP email configuration -SMTP_HOST = "smtp.sendgrid.net" #change to your host -SMTP_STARTTLS = True -SMTP_SSL = False -SMTP_USER = "your_user" -SMTP_PORT = 2525 # your port eg. 587 -SMTP_PASSWORD = "your_password" -SMTP_MAIL_FROM = "noreply@youremail.com" - -# WebDriver configuration -# If you use Firefox, you can stick with default values -# If you use Chrome, then add the following WEBDRIVER_TYPE and WEBDRIVER_OPTION_ARGS -WEBDRIVER_TYPE= "chrome" -WEBDRIVER_OPTION_ARGS = [ - "--force-device-scale-factor=2.0", - "--high-dpi-support=2.0", - "--headless", - "--disable-gpu", - "--disable-dev-shm-usage", - "--no-sandbox", - "--disable-setuid-sandbox", - "--disable-extensions", - ] - -# This is for internal use, you can keep http -WEBDRIVER_BASEURL="http://superset:8088" -# This is the link sent to the recipient, change to your domain eg. https://superset.mydomain.com -WEBDRIVER_BASEURL_USER_FRIENDLY="http://localhost:8088" -``` - ### Summary With the extended image created by using the `Dockerfile`, and then running that image using `docker-compose.yaml`, plus the required configurations in the `superset_config.py` you should now have alerts and reporting working correctly. From 2e24a57fc88ec2d5b86ada22afce7c3257ba379e Mon Sep 17 00:00:00 2001 From: Valentin NOURDIN Date: Fri, 19 Feb 2021 17:02:07 +0100 Subject: [PATCH 18/18] docs: clean Dockerfiles --- .../docs/installation/alerts_reports.mdx | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/src/pages/docs/installation/alerts_reports.mdx b/docs/src/pages/docs/installation/alerts_reports.mdx index 6c6436d4089f..a8a4e4623874 100644 --- a/docs/src/pages/docs/installation/alerts_reports.mdx +++ b/docs/src/pages/docs/installation/alerts_reports.mdx @@ -123,7 +123,7 @@ SMTP_MAIL_FROM = "noreply@youremail.com" # WebDriver configuration # If you use Firefox, you can stick with default values # If you use Chrome, then add the following WEBDRIVER_TYPE and WEBDRIVER_OPTION_ARGS -WEBDRIVER_TYPE= "chrome" +WEBDRIVER_TYPE = "chrome" WEBDRIVER_OPTION_ARGS = [ "--force-device-scale-factor=2.0", "--high-dpi-support=2.0", @@ -150,14 +150,14 @@ FROM apache/superset:1.0.1 USER root -RUN apt-get update \ - && apt install --no-install-recommends -y firefox-esr +RUN apt-get update && \ + apt-get install --no-install-recommends -y firefox-esr ENV GECKODRIVER_VERSION=0.29.0 -RUN wget -q https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz \ - && tar -x geckodriver -zf geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz -O > /usr/bin/geckodriver \ - && chmod 755 /usr/bin/geckodriver \ - && rm geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz +RUN wget -q https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz && \ + tar -x geckodriver -zf geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz -O > /usr/bin/geckodriver && \ + chmod 755 /usr/bin/geckodriver && \ + rm geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz RUN pip install --no-cache gevent psycopg2 redis @@ -169,16 +169,16 @@ FROM apache/superset:1.0.1 USER root -RUN apt update && \ - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ - apt install -y --no-install-recommends ./google-chrome-stable_current_amd64.deb && \ - wget https://chromedriver.storage.googleapis.com/88.0.4324.96/chromedriver_linux64.zip && \ - unzip chromedriver_linux64.zip && \ - chmod +x chromedriver && \ - mv chromedriver /usr/bin && \ - apt autoremove -yqq --purge && \ - apt clean && \ - rm -f google-chrome-stable_current_amd64.deb chromedriver_linux64.zip +RUN apt-get update && \ + wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + apt-get install -y --no-install-recommends ./google-chrome-stable_current_amd64.deb && \ + rm -f google-chrome-stable_current_amd64.deb + +RUN export CHROMEDRIVER_VERSION=$(curl --silent https://chromedriver.storage.googleapis.com/LATEST_RELEASE_88) && \ + wget -q https://chromedriver.storage.googleapis.com/${CHROMEDRIVER_VERSION}/chromedriver_linux64.zip && \ + unzip chromedriver_linux64.zip -d /usr/bin && \ + chmod 755 /usr/bin/chromedriver && \ + rm -f chromedriver_linux64.zip RUN pip install --no-cache gevent psycopg2 redis