From 644b44024d77194529b96ce0fb6bd89ad0b0c53a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Christian=20Langfj=C3=A6ran?= Date: Mon, 28 Mar 2022 14:40:19 +0200 Subject: [PATCH 1/5] Use entrypoint for pytest --- plugins/airflow/airflow.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/airflow/airflow.mk b/plugins/airflow/airflow.mk index 96f3a81..2667a72 100644 --- a/plugins/airflow/airflow.mk +++ b/plugins/airflow/airflow.mk @@ -32,7 +32,7 @@ airflow.logs: $(AIRFLOW_INIT_CHECK_SENTINEL) ## Tail the local logs .PHONY: airflow.test airflow.test: $(AIRFLOW_BUILD_SENTINEL) $(AIRFLOW_INIT_CHECK_SENTINEL) ## Run the tests found in /test - set +e;docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm test pytest --html=/code/output/report.html --junitxml=/code/output/junit.xml -rA /code/tests; \ + set +e;docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm --entrypoint pytest test --html=/code/output/report.html --junitxml=/code/output/junit.xml -rA /code/tests; \ open $${PWD}/output/report.html .PHONY: airflow.flake8 From a79b457b35b959ca81fb09a8a4a955cd53251200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Christian=20Langfj=C3=A6ran?= Date: Mon, 28 Mar 2022 14:41:41 +0200 Subject: [PATCH 2/5] Use entrypoint for flake8 --- plugins/airflow/airflow.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/airflow/airflow.mk b/plugins/airflow/airflow.mk index 2667a72..39baab7 100644 --- a/plugins/airflow/airflow.mk +++ b/plugins/airflow/airflow.mk @@ -37,7 +37,7 @@ airflow.test: $(AIRFLOW_BUILD_SENTINEL) $(AIRFLOW_INIT_CHECK_SENTINEL) ## Run th .PHONY: airflow.flake8 airflow.flake8:$(AIRFLOW_BUILD_SENTINEL) $(AIRFLOW_INIT_CHECK_SENTINEL) ## Run the flake8 agains dags folder - docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run -v ${PWD}/.flake8:/code/.flake8 --rm test flake8 /code/dags + docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run -v ${PWD}/.flake8:/code/.flake8 --rm --entrypoint flake8 test /code/dags @echo Flake 8 OK!s .PHONY: airflow.clean From 0fe5a9a2161277cc61e8599747dc23ad74b4c2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Christian=20Langfj=C3=A6ran?= Date: Mon, 28 Mar 2022 14:52:12 +0200 Subject: [PATCH 3/5] Add missing \ --- plugins/airflow/airflow.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/airflow/airflow.mk b/plugins/airflow/airflow.mk index 39baab7..1513761 100644 --- a/plugins/airflow/airflow.mk +++ b/plugins/airflow/airflow.mk @@ -141,7 +141,7 @@ $(AIRFLOW_VARIABLES_SENTINEL): $(AIRFLOW_DB_INIT_SENTINEL) $(AIRFLOW_VARIABLES_J docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver variables import /code/airflow-variables.json; \ else \ docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver airflow variables -i /code/airflow-variables.json; \ - fi + fi \ fi echo Done importing variables touch $@ From 4978d8df746b947942e3127ea7ebc41e3731de8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Christian=20Langfj=C3=A6ran?= Date: Thu, 31 Mar 2022 14:09:49 +0200 Subject: [PATCH 4/5] Airflow 1 docker --- plugins/airflow/docker/Dockerfile.Airflow1 | 41 +++++-------------- .../airflow/docker/requirements.1.10.15.txt | 4 ++ 2 files changed, 14 insertions(+), 31 deletions(-) create mode 100644 plugins/airflow/docker/requirements.1.10.15.txt diff --git a/plugins/airflow/docker/Dockerfile.Airflow1 b/plugins/airflow/docker/Dockerfile.Airflow1 index 5d8978c..6ca77d5 100644 --- a/plugins/airflow/docker/Dockerfile.Airflow1 +++ b/plugins/airflow/docker/Dockerfile.Airflow1 @@ -1,36 +1,15 @@ -FROM docker:17.12.0-ce as static-docker-source +ARG AIRFLOW_IMAGE_NAME="1.10.15-python3.6" -FROM python:3.6-stretch -ARG CLOUD_SDK_VERSION=297.0.1 -ENV CLOUD_SDK_VERSION=$CLOUD_SDK_VERSION +FROM "apache/airflow:${AIRFLOW_IMAGE_NAME}" -COPY --from=static-docker-source /usr/local/bin/docker /usr/local/bin/docker -RUN apt-get -qqy update && apt-get install -qqy \ - curl \ - gcc \ - apt-transport-https \ - lsb-release \ - openssh-client \ - git \ - gnupg \ - && pip install -U crcmod && \ - export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ - echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get update && \ - apt-get install -y google-cloud-sdk=${CLOUD_SDK_VERSION}-0 kubectl && \ - gcloud config set core/disable_usage_reporting true && \ - gcloud config set component_manager/disable_update_check true && \ - gcloud config set metrics/environment github_docker_image && \ - gcloud --version && \ - docker --version && kubectl version --client +# - Install GCP util +RUN curl -sSL https://sdk.cloud.google.com | bash +ENV PATH $PATH:/home/airflow/google-cloud-sdk/bin + + +# - Install dependencies +CMD echo "Installing dependencies" -RUN apt-get install default-libmysqlclient-dev -qqy -ENV AIRFLOW_GPL_UNIDECODE="yes" -#ENV SLUGIFY_USES_TEXT_UNIDECODE="yes" -# Default requirements ADD requirements.txt /tmp/requirements.txt -RUN pip3 install -r /tmp/requirements.txt -# Extra requirements ADD requirements.extra.txt /tmp/requirements.extra.txt -RUN pip3 install -r /tmp/requirements.extra.txt \ No newline at end of file +RUN pip3 install --no-deps -r /tmp/requirements.txt -r /tmp/requirements.extra.txt \ No newline at end of file diff --git a/plugins/airflow/docker/requirements.1.10.15.txt b/plugins/airflow/docker/requirements.1.10.15.txt new file mode 100644 index 0000000..92ee0dd --- /dev/null +++ b/plugins/airflow/docker/requirements.1.10.15.txt @@ -0,0 +1,4 @@ +apache-airflow[gcp,postgres,sendgrid,kubernetes,aws,slack]==1.10.15 +# Testing +pytest==5.3.5 +flake8==3.7.9 \ No newline at end of file From 37ebae018c58e9d5e9ee2f9207c198fb41f24e86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ole=20Christian=20Langfj=C3=A6ran?= Date: Thu, 31 Mar 2022 14:10:02 +0200 Subject: [PATCH 5/5] Use correct airflow commands --- plugins/airflow/airflow.mk | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/plugins/airflow/airflow.mk b/plugins/airflow/airflow.mk index 1513761..08b2bad 100644 --- a/plugins/airflow/airflow.mk +++ b/plugins/airflow/airflow.mk @@ -51,7 +51,7 @@ airflow.clean: ## Removes .airflow folder and docker containers .PHONY: airflow.error airflow.error: $(AIRFLOW_INIT_CHECK_SENTINEL) ## List all dags, and filter errors - docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run webserver airflow list_dags | grep -B5000 "DAGS" + docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm webserver list_dags | grep -B5000 "DAGS" .PHONY: airflow.build airflow.build: $(AIRFLOW_INIT_CHECK_SENTINEL) ## Rebuild docker images with --no-cache. Useful for debugging @@ -66,8 +66,7 @@ airflow.venv: $(AIRFLOW_REQUIREMENTS_TXT) $(AIRFLOW_REQUIREMENTS_EXTRA_TXT) $(AI virtualenv -p python3 $(AIRFLOW_VIRTUAL_ENV_FOLDER); \ source $(AIRFLOW_VIRTUAL_ENV_FOLDER)/bin/activate; \ export AIRFLOW_GPL_UNIDECODE="yes"; \ - pip install -r $(AIRFLOW_REQUIREMENTS_TXT); \ - pip install -r $(AIRFLOW_REQUIREMENTS_EXTRA_TXT); + pip install -r $(AIRFLOW_REQUIREMENTS_TXT) -r $(AIRFLOW_REQUIREMENTS_EXTRA_TXT); .PHONY: airflow.pip_install airflow.pip_install: $(AIRFLOW_REQUIREMENTS_TXT) ## Run pip install -r requirements.txt. This is helpful in a CI environment, where we don't use Docker. @@ -140,7 +139,7 @@ $(AIRFLOW_VARIABLES_SENTINEL): $(AIRFLOW_DB_INIT_SENTINEL) $(AIRFLOW_VARIABLES_J if [ $(AIRFLOW_MAJOR_VERSION) -eq "2" ]; then \ docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver variables import /code/airflow-variables.json; \ else \ - docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver airflow variables -i /code/airflow-variables.json; \ + docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver variables -i /code/airflow-variables.json; \ fi \ fi echo Done importing variables