Merge pull request #35 from BasPH/feature/fix-ch12

Feature/fix ch12

Author: BasPH

chapters/chapter12/README.md

@@ -1,28 +1,38 @@
 # Chapter 12
 
-Code accompanying Chapter 12 of the book 'Data Pipelines with Apache Airflow'.
+Code accompanying Chapter 12 of the book [Data Pipelines with Apache Airflow](https://www.manning.com/books/data-pipelines-with-apache-airflow).
 
 ## Contents
 
-This code example contains the following:
+This folder contains DAGs from Chapter 12. Topics covered are monitoring, logging, scaling horizontal, etc. An
+accompanying Docker Compose setup was built for demonstration purposes. This includes:
 
-- /DAGs:
-    - dag_failure_callback.py - Trigger a callback function when a DAG run fails.
-    - dag_puller_dag.py - Fetches latest code from a Git master branch every 5 minutes (Git must be configured).
-    - task_failure_callback.py - Trigger a callback function when a task fails.
-    - task_failure_email.py - Send an email when a task fails (SMTP must be configured).
-    - task_sla.py - Send an SLA miss notification in case of a missed SLA.
-- /docker: An example (base) Dockerfile for use in a CI/CD pipeline.
-- /monitoring_docker_compose: A docker-compose setup for monitoring Airflow with StatsD, Prometheus, and Grafana.
+- Airflow (webserver, scheduler, and Celery workers)
+- PostgreSQL database for Airflow metastore
+- Redis for Celery queue
+- Flower, a Celery monitoring tool
+- Prometheus, for scraping and storing metrics
+- Grafana, for visualizing metrics
+- And a Redis & StatsD exporter to expose metrics
+
+Given the number of services, this can become a bit resource-heavy on your machine.
+
+Unfortunately, not everything can be scripted/pre-initialized, especially in Grafana. Therefore, you must add
+Prometheus as a datasource and create a dashboard yourself.
 
 ## Usage
 
-To get started with the code examples, start Airflow in docker using the following command:
+To get started with the code examples, start Airflow with Docker Compose with the following command:
 
-    docker-compose up -d --build
+```bash
+docker-compose up -d
+```
 
-Wait for a few seconds and you should be able to access the examples at http://localhost:8080/.
+The webserver initializes a few things, so wait for a few seconds, and you should be able to access the
+Airflow webserver at http://localhost:8080.
 
 To stop running the examples, run the following command:
 
-    docker-compose down
+```bash
+docker-compose down -v
+```

chapters/chapter12/monitoring_docker_compose/docker-compose.yml renamed to chapters/chapter12/docker-compose.yml

@@ -4,7 +4,7 @@ version: '3.7'
 x-environment: &airflow_environment
   - AIRFLOW__CELERY__BROKER_URL=redis://redis:6379/1
   - AIRFLOW__CELERY__RESULT_BACKEND=db+postgresql://airflow:airflow@postgres:5432/airflow
-  - AIRFLOW__CORE__EXECUTOR=CeleryExecutor
+  - AIRFLOW__CORE__EXECUTOR=LocalExecutor
   - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
   - AIRFLOW__CORE__LOAD_EXAMPLES=False
   - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql://airflow:airflow@postgres:5432/airflow
@@ -14,7 +14,9 @@ x-environment: &airflow_environment
   - AIRFLOW__SCHEDULER__STATSD_ON=True
   - AIRFLOW__SCHEDULER__STATSD_PORT=9125
   - AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
-  - AIRFLOW__WEBSERVER__RBAC=True
+  - AIRFLOW__WEBSERVER__RBAC=False
+
+x-airflow-image: &airflow_image apache/airflow:1.10.12-python3.8
 # ====================================== /AIRFLOW ENVIRONMENT VARIABLES ======================================
 
 services:
@@ -27,53 +29,51 @@ services:
     ports:
       - "5432:5432"
 
-  initdb_adduser:
-    image: apache/airflow:1.10.11-python3.7
+  init:
+    image: *airflow_image
     depends_on:
       - postgres
     environment: *airflow_environment
     entrypoint: /bin/bash
-    # The webserver initializes permissions, so sleep for that to (approximately) be finished
-    # No disaster if the webserver isn't finished by then, but create_user will start spitting out errors until the permissions exist
-    command: -c 'airflow initdb && sleep 5 && airflow create_user --role Admin --username airflow --password airflow -e [email protected] -f airflow -l airflow'
+    command: -c 'airflow upgradedb && sleep 5 && airflow create_user --username admin --password admin --firstname John --lastname Smith --role Admin --email [email protected]'
 
   webserver:
-    image: apache/airflow:1.10.11-python3.7
+    image: *airflow_image
     restart: always
     depends_on:
       - postgres
-    volumes:
-      - logs:/opt/airflow/logs
     ports:
       - "8080:8080"
+    volumes:
+      - logs:/opt/airflow/logs
     environment: *airflow_environment
     command: webserver
 
   scheduler:
-    image: apache/airflow:1.10.11-python3.7
+    image: *airflow_image
     restart: always
     depends_on:
       - postgres
     volumes:
-      - ./hello_airflow.py:/opt/airflow/dags/hello_airflow.py
       - logs:/opt/airflow/logs
+      - ./dags:/opt/airflow/dags
     environment: *airflow_environment
     command: scheduler
 
   # docker-compose -f docker-compose-celeryexecutor.yml up --scale worker=3 -d
   worker:
-    image: apache/airflow:1.10.11-python3.7
+    image: *airflow_image
     restart: always
     depends_on:
       - scheduler
     volumes:
-      - ./hello_airflow.py:/opt/airflow/dags/hello_airflow.py
       - logs:/opt/airflow/logs
+      - ./dags:/opt/airflow/dags
     environment: *airflow_environment
     command: worker
 
   flower:
-    image: apache/airflow:1.10.11-python3.7
+    image: *airflow_image
     restart: always
     depends_on:
       - worker
@@ -83,20 +83,20 @@ services:
     command: flower
 
   statsd_exporter:
-    image: prom/statsd-exporter
+    image: prom/statsd-exporter:v0.18.0
     restart: always
     volumes:
-      - ./statsd_mapping.yml:/tmp/statsd_mapping.yml
+      - ./files/statsd_mapping.yml:/tmp/statsd_mapping.yml
     ports:
       - "9102:9102"
       - "9125:9125/udp"
     command: --statsd.mapping-config=/tmp/statsd_mapping.yml
 
   prometheus:
-    image: prom/prometheus
+    image: prom/prometheus:v2.22.0
     restart: always
     volumes:
-      - ./prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./files/prometheus.yml:/etc/prometheus/prometheus.yml
     ports:
       - "9090:9090"
     command:
@@ -109,7 +109,7 @@ services:
       - --web.console.templates=/usr/share/prometheus/consoles
 
   grafana:
-    image: grafana/grafana
+    image: grafana/grafana:7.2.1
     restart: always
     ports:
       - "3000:3000"