Merge pull request #22 from BasPH/feature/update-chapter-5

Update chapter 5 for final review.

Author: Julian de Ruiter

.gitignore

@@ -150,3 +150,6 @@ logs
 
 # Mac
 .DS_Store
+
+# Data
+data

chapters/chapter5/dags/chapter5_01_start.py renamed to chapters/chapter05/dags/01_start.py

@@ -5,7 +5,7 @@
 
 
 with DAG(
-    dag_id="chapter5_01_start",
+    dag_id="01_start",
     start_date=airflow.utils.dates.days_ago(3),
     schedule_interval="@daily",
 ) as dag:

chapters/chapter5/dags/chapter5_02_branch_in_function.py renamed to chapters/chapter05/dags/02_branch_in_function.py

@@ -38,7 +38,7 @@ def _clean_sales_new(**context):
 
 
 with DAG(
-    dag_id="chapter5_02_branch_in_function",
+    dag_id="02_branch_in_function",
     start_date=airflow.utils.dates.days_ago(3),
     schedule_interval="@daily",
 ) as dag:

chapters/chapter5/dags/chapter5_03_branch_in_dag.py renamed to chapters/chapter05/dags/03_branch_in_dag.py

@@ -31,7 +31,7 @@ def _clean_sales_new(**context):
 
 
 with DAG(
-    dag_id="chapter5_03_branch_in_dag",
+    dag_id="03_branch_in_dag",
     start_date=airflow.utils.dates.days_ago(3),
     schedule_interval="@daily",
 ) as dag:

chapters/chapter5/dags/chapter5_04_branch_in_dag_explicit_join.py renamed to chapters/chapter05/dags/04_branch_in_dag_explicit_join.py

@@ -31,7 +31,7 @@ def _clean_sales_new(**context):
 
 
 with DAG(
-    dag_id="chapter5_04_branch_in_dag_explicit_join",
+    dag_id="04_branch_in_dag_explicit_join",
     start_date=airflow.utils.dates.days_ago(3),
     schedule_interval="@daily",
 ) as dag:

chapters/chapter5/dags/chapter5_05_condition_in_function.py renamed to chapters/chapter05/dags/05_condition_in_function.py

@@ -28,7 +28,7 @@ def _is_latest_run(**context):
 
 
 with DAG(
-    dag_id="chapter5_05_condition_in_function",
+    dag_id="05_condition_in_function",
     start_date=airflow.utils.dates.days_ago(3),
     schedule_interval="@daily",
 ) as dag:

chapters/chapter5/dags/chapter5_06_condition_in_dag.py renamed to chapters/chapter05/dags/06_condition_in_dag.py

@@ -26,7 +26,7 @@ def _latest_only(**context):
 
 
 with DAG(
-    dag_id="chapter5_06_condition_in_dag",
+    dag_id="06_condition_in_dag",
     start_date=airflow.utils.dates.days_ago(3),
     schedule_interval="@daily",
 ) as dag:

chapters/chapter5/dags/chapter5_07_trigger_rules.py renamed to chapters/chapter05/dags/07_trigger_rules.py

@@ -11,7 +11,7 @@ def _fetch_sales(**context):
 
 
 with DAG(
-    dag_id="chapter5_07_trigger_rules",
+    dag_id="07_trigger_rules",
     start_date=airflow.utils.dates.days_ago(3),
     schedule_interval="@daily",
 ) as dag:

chapters/chapter5/dags/chapter5_08_xcoms.py renamed to chapters/chapter05/dags/08_xcoms.py

@@ -20,7 +20,7 @@ def _deploy_model(**context):
 
 
 with DAG(
-    dag_id="chapter5_08_xcoms",
+    dag_id="08_xcoms",
     start_date=airflow.utils.dates.days_ago(3),
     schedule_interval="@daily",
 ) as dag:

chapters/chapter05/docker-compose.yml

@@ -0,0 +1,49 @@
+version: '3.7'
+# ====================================== AIRFLOW ENVIRONMENT VARIABLES =======================================
+x-environment: &airflow_environment
+  - AIRFLOW__CORE__EXECUTOR=LocalExecutor
+  - AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS=False
+  - AIRFLOW__CORE__LOAD_EXAMPLES=False
+  - AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql://airflow:airflow@postgres:5432/airflow
+  - AIRFLOW__CORE__STORE_DAG_CODE=True
+  - AIRFLOW__CORE__STORE_SERIALIZED_DAGS=True
+  - AIRFLOW__WEBSERVER__EXPOSE_CONFIG=True
+  - AIRFLOW__WEBSERVER__RBAC=False
+x-airflow-image: &airflow_image apache/airflow:1.10.12-python3.8
+# ====================================== /AIRFLOW ENVIRONMENT VARIABLES ======================================
+services:
+  postgres:
+    image: postgres:12-alpine
+    environment:
+      - POSTGRES_USER=airflow
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+    ports:
+      - "5432:5432"
+  init:
+    image: *airflow_image
+    depends_on:
+      - postgres
+    environment: *airflow_environment
+    command: upgradedb
+  webserver:
+    image: *airflow_image
+    restart: always
+    depends_on:
+      - postgres
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./logs:/opt/airflow/logs
+    environment: *airflow_environment
+    command: webserver
+  scheduler:
+    image: *airflow_image
+    restart: always
+    depends_on:
+      - postgres
+    volumes:
+      - ./dags:/opt/airflow/dags
+      - ./logs:/opt/airflow/logs
+    environment: *airflow_environment
+    command: scheduler