Merge pull request #26 from unacast/airflow-2-fixes

Airflow 2 fixes
unacast · Mar 31, 2022 · b911b52 · b911b52
2 parents 5a8e2ad + 37ebae0
commit b911b52
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 38 deletions.
diff --git a/plugins/airflow/airflow.mk b/plugins/airflow/airflow.mk
@@ -32,12 +32,12 @@ airflow.logs: $(AIRFLOW_INIT_CHECK_SENTINEL) ## Tail the local logs
 
 .PHONY: airflow.test
 airflow.test: $(AIRFLOW_BUILD_SENTINEL) $(AIRFLOW_INIT_CHECK_SENTINEL) ## Run the tests found in /test
-	set +e;docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm  test pytest --html=/code/output/report.html --junitxml=/code/output/junit.xml -rA /code/tests; \
+	set +e;docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm --entrypoint pytest test --html=/code/output/report.html --junitxml=/code/output/junit.xml -rA /code/tests; \
 	open $${PWD}/output/report.html
 
 .PHONY: airflow.flake8
 airflow.flake8:$(AIRFLOW_BUILD_SENTINEL) $(AIRFLOW_INIT_CHECK_SENTINEL) ## Run the flake8 agains dags folder
-	docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run -v ${PWD}/.flake8:/code/.flake8 --rm test flake8 /code/dags
+	docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run -v ${PWD}/.flake8:/code/.flake8 --rm --entrypoint flake8 test /code/dags
 	@echo Flake 8 OK!s
 
 .PHONY: airflow.clean
@@ -51,7 +51,7 @@ airflow.clean: ## Removes .airflow folder and docker containers
 
 .PHONY: airflow.error
 airflow.error: $(AIRFLOW_INIT_CHECK_SENTINEL) ## List all dags, and filter errors
-	docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run webserver airflow list_dags | grep -B5000 "DAGS"
+	docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm webserver list_dags | grep -B5000 "DAGS"
 
 .PHONY: airflow.build
 airflow.build: $(AIRFLOW_INIT_CHECK_SENTINEL) ## Rebuild docker images with --no-cache. Useful for debugging
@@ -66,8 +66,7 @@ airflow.venv: $(AIRFLOW_REQUIREMENTS_TXT) $(AIRFLOW_REQUIREMENTS_EXTRA_TXT) $(AI
 	virtualenv -p python3 $(AIRFLOW_VIRTUAL_ENV_FOLDER); \
 	source $(AIRFLOW_VIRTUAL_ENV_FOLDER)/bin/activate; \
 	export AIRFLOW_GPL_UNIDECODE="yes"; \
-	pip install -r $(AIRFLOW_REQUIREMENTS_TXT); \
-	pip install -r $(AIRFLOW_REQUIREMENTS_EXTRA_TXT);
+	pip install -r $(AIRFLOW_REQUIREMENTS_TXT) -r $(AIRFLOW_REQUIREMENTS_EXTRA_TXT);
 
 .PHONY: airflow.pip_install
 airflow.pip_install: $(AIRFLOW_REQUIREMENTS_TXT) ## Run pip install -r requirements.txt. This is helpful in a CI environment, where we don't use Docker.
@@ -140,8 +139,8 @@ $(AIRFLOW_VARIABLES_SENTINEL): $(AIRFLOW_DB_INIT_SENTINEL) $(AIRFLOW_VARIABLES_J
 		if [ $(AIRFLOW_MAJOR_VERSION) -eq "2" ]; then \
 			docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver variables import /code/airflow-variables.json; \
 		else \
-			docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver airflow variables -i /code/airflow-variables.json; \
-		fi
+			docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver variables -i /code/airflow-variables.json; \
+		fi \
 	fi
 	echo Done importing variables
 	touch $@

diff --git a/plugins/airflow/docker/Dockerfile.Airflow1 b/plugins/airflow/docker/Dockerfile.Airflow1
@@ -1,36 +1,15 @@
-FROM docker:17.12.0-ce as static-docker-source
+ARG AIRFLOW_IMAGE_NAME="1.10.15-python3.6"
 
-FROM python:3.6-stretch
-ARG CLOUD_SDK_VERSION=297.0.1
-ENV CLOUD_SDK_VERSION=$CLOUD_SDK_VERSION
+FROM "apache/airflow:${AIRFLOW_IMAGE_NAME}"
 
-COPY --from=static-docker-source /usr/local/bin/docker /usr/local/bin/docker
-RUN apt-get -qqy update && apt-get install -qqy \
-        curl \
-        gcc \
-        apt-transport-https \
-        lsb-release \
-        openssh-client \
-        git \
-        gnupg \
-    && pip install -U crcmod   && \
-    export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
-    echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \
-    curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
-    apt-get update && \
-    apt-get install -y google-cloud-sdk=${CLOUD_SDK_VERSION}-0 kubectl && \
-    gcloud config set core/disable_usage_reporting true && \
-    gcloud config set component_manager/disable_update_check true && \
-    gcloud config set metrics/environment github_docker_image && \
-    gcloud --version && \
-    docker --version && kubectl version --client
+# - Install GCP util
+RUN curl -sSL https://sdk.cloud.google.com | bash
+ENV PATH $PATH:/home/airflow/google-cloud-sdk/bin
+
+
+# - Install dependencies
+CMD echo "Installing dependencies"
 
-RUN apt-get install default-libmysqlclient-dev -qqy
-ENV AIRFLOW_GPL_UNIDECODE="yes"
-#ENV SLUGIFY_USES_TEXT_UNIDECODE="yes"
-# Default requirements
 ADD requirements.txt /tmp/requirements.txt
-RUN pip3 install -r /tmp/requirements.txt
-# Extra requirements
 ADD requirements.extra.txt /tmp/requirements.extra.txt
-RUN pip3 install -r /tmp/requirements.extra.txt
+RUN pip3 install --no-deps -r /tmp/requirements.txt -r /tmp/requirements.extra.txt
diff --git a/plugins/airflow/docker/requirements.1.10.15.txt b/plugins/airflow/docker/requirements.1.10.15.txt
@@ -0,0 +1,4 @@
+apache-airflow[gcp,postgres,sendgrid,kubernetes,aws,slack]==1.10.15
+# Testing
+pytest==5.3.5
+flake8==3.7.9