Skip to content

Commit

Permalink
Merge pull request #26 from unacast/airflow-2-fixes
Browse files Browse the repository at this point in the history
Airflow 2 fixes
  • Loading branch information
judoole authored Mar 31, 2022
2 parents 5a8e2ad + 37ebae0 commit b911b52
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 38 deletions.
13 changes: 6 additions & 7 deletions plugins/airflow/airflow.mk
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,12 @@ airflow.logs: $(AIRFLOW_INIT_CHECK_SENTINEL) ## Tail the local logs

.PHONY: airflow.test
airflow.test: $(AIRFLOW_BUILD_SENTINEL) $(AIRFLOW_INIT_CHECK_SENTINEL) ## Run the tests found in /test
set +e;docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm test pytest --html=/code/output/report.html --junitxml=/code/output/junit.xml -rA /code/tests; \
set +e;docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm --entrypoint pytest test --html=/code/output/report.html --junitxml=/code/output/junit.xml -rA /code/tests; \
open $${PWD}/output/report.html

.PHONY: airflow.flake8
airflow.flake8:$(AIRFLOW_BUILD_SENTINEL) $(AIRFLOW_INIT_CHECK_SENTINEL) ## Run the flake8 agains dags folder
docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run -v ${PWD}/.flake8:/code/.flake8 --rm test flake8 /code/dags
docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run -v ${PWD}/.flake8:/code/.flake8 --rm --entrypoint flake8 test /code/dags
@echo Flake 8 OK!s

.PHONY: airflow.clean
Expand All @@ -51,7 +51,7 @@ airflow.clean: ## Removes .airflow folder and docker containers

.PHONY: airflow.error
airflow.error: $(AIRFLOW_INIT_CHECK_SENTINEL) ## List all dags, and filter errors
docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run webserver airflow list_dags | grep -B5000 "DAGS"
docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm webserver list_dags | grep -B5000 "DAGS"

.PHONY: airflow.build
airflow.build: $(AIRFLOW_INIT_CHECK_SENTINEL) ## Rebuild docker images with --no-cache. Useful for debugging
Expand All @@ -66,8 +66,7 @@ airflow.venv: $(AIRFLOW_REQUIREMENTS_TXT) $(AIRFLOW_REQUIREMENTS_EXTRA_TXT) $(AI
virtualenv -p python3 $(AIRFLOW_VIRTUAL_ENV_FOLDER); \
source $(AIRFLOW_VIRTUAL_ENV_FOLDER)/bin/activate; \
export AIRFLOW_GPL_UNIDECODE="yes"; \
pip install -r $(AIRFLOW_REQUIREMENTS_TXT); \
pip install -r $(AIRFLOW_REQUIREMENTS_EXTRA_TXT);
pip install -r $(AIRFLOW_REQUIREMENTS_TXT) -r $(AIRFLOW_REQUIREMENTS_EXTRA_TXT);

.PHONY: airflow.pip_install
airflow.pip_install: $(AIRFLOW_REQUIREMENTS_TXT) ## Run pip install -r requirements.txt. This is helpful in a CI environment, where we don't use Docker.
Expand Down Expand Up @@ -140,8 +139,8 @@ $(AIRFLOW_VARIABLES_SENTINEL): $(AIRFLOW_DB_INIT_SENTINEL) $(AIRFLOW_VARIABLES_J
if [ $(AIRFLOW_MAJOR_VERSION) -eq "2" ]; then \
docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver variables import /code/airflow-variables.json; \
else \
docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver airflow variables -i /code/airflow-variables.json; \
fi
docker-compose -f $(AIRFLOW_DOCKER_COMPOSE_FILE) run --rm -v ${PWD}/$(AIRFLOW_VARIABLES_JSON):/code/airflow-variables.json webserver variables -i /code/airflow-variables.json; \
fi \
fi
echo Done importing variables
touch $@
Expand Down
41 changes: 10 additions & 31 deletions plugins/airflow/docker/Dockerfile.Airflow1
Original file line number Diff line number Diff line change
@@ -1,36 +1,15 @@
FROM docker:17.12.0-ce as static-docker-source
ARG AIRFLOW_IMAGE_NAME="1.10.15-python3.6"

FROM python:3.6-stretch
ARG CLOUD_SDK_VERSION=297.0.1
ENV CLOUD_SDK_VERSION=$CLOUD_SDK_VERSION
FROM "apache/airflow:${AIRFLOW_IMAGE_NAME}"

COPY --from=static-docker-source /usr/local/bin/docker /usr/local/bin/docker
RUN apt-get -qqy update && apt-get install -qqy \
curl \
gcc \
apt-transport-https \
lsb-release \
openssh-client \
git \
gnupg \
&& pip install -U crcmod && \
export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \
echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \
apt-get update && \
apt-get install -y google-cloud-sdk=${CLOUD_SDK_VERSION}-0 kubectl && \
gcloud config set core/disable_usage_reporting true && \
gcloud config set component_manager/disable_update_check true && \
gcloud config set metrics/environment github_docker_image && \
gcloud --version && \
docker --version && kubectl version --client
# - Install GCP util
RUN curl -sSL https://sdk.cloud.google.com | bash
ENV PATH $PATH:/home/airflow/google-cloud-sdk/bin


# - Install dependencies
CMD echo "Installing dependencies"

RUN apt-get install default-libmysqlclient-dev -qqy
ENV AIRFLOW_GPL_UNIDECODE="yes"
#ENV SLUGIFY_USES_TEXT_UNIDECODE="yes"
# Default requirements
ADD requirements.txt /tmp/requirements.txt
RUN pip3 install -r /tmp/requirements.txt
# Extra requirements
ADD requirements.extra.txt /tmp/requirements.extra.txt
RUN pip3 install -r /tmp/requirements.extra.txt
RUN pip3 install --no-deps -r /tmp/requirements.txt -r /tmp/requirements.extra.txt
4 changes: 4 additions & 0 deletions plugins/airflow/docker/requirements.1.10.15.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
apache-airflow[gcp,postgres,sendgrid,kubernetes,aws,slack]==1.10.15
# Testing
pytest==5.3.5
flake8==3.7.9

0 comments on commit b911b52

Please sign in to comment.