Merge pull request #133 from getindata/grouping-feature #73

Workflow file for this run

.github/workflows/test_and_publish.yml at ecba16f

	name: Test & Publish

	on:
	push:
	branches:
	- master
	- develop
	paths-ignore:
	- "docs/**"
	pull_request:
	branches:
	- master
	- develop
	paths-ignore:
	- "docs/**"
	jobs:
	unit_tests:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: [ '3.8', '3.9', '3.10']

	steps:
	- uses: actions/checkout@v4

	- name: Setup python ${{ matrix.python-version }}
	uses: actions/[email protected]
	with:
	python-version: ${{ matrix.python-version }}

	- name: Setup poetry
	run: \|
	python -V
	python -m pip install poetry

	- name: Check pre-commit status
	run: \|
	poetry install -v
	poetry run pre-commit run --all-files -c .github/pre-commit-config.yaml

	- name: Test with tox
	run: \|
	pip install tox-pip-version tox-gh-actions "tox<4.0.0"
	tox -v

	- name: Store coverage reports
	uses: actions/upload-artifact@v3
	with:
	name: coverage-${{ matrix.python-version }}
	path: coverage.xml
	if-no-files-found: error

	sonarcloud:
	runs-on: ubuntu-latest
	needs: unit_tests
	steps:

	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- uses: actions/download-artifact@v3
	with:
	name: coverage-3.9
	path: .

	- name: SonarCloud Scan
	uses: sonarsource/sonarcloud-github-action@master
	with:
	projectBaseDir: ${{ github.workspace }}
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}

	codeql:
	runs-on: ubuntu-latest
	needs: unit_tests
	permissions:
	actions: read
	contents: read
	security-events: write

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	# Initializes the CodeQL tools for scanning.
	- name: Initialize CodeQL
	uses: github/codeql-action/init@v2
	with:
	languages: python

	- name: Autobuild
	uses: github/codeql-action/autobuild@v2

	- name: Perform CodeQL Analysis
	uses: github/codeql-action/analyze@v2

	e2e_tests:
	runs-on: ubuntu-latest
	timeout-minutes: 60
	needs: [unit_tests, sonarcloud]
	strategy:
	matrix:
	e2e_case: ["standard", "grouping"]
	steps:
	- uses: actions/checkout@v4

	- name: Setup python
	uses: actions/[email protected]
	with:
	python-version: 3.8

	- name: Setup poetry
	run: \|
	python -V
	python -m pip install poetry

	- name: Build the package
	run: \|
	poetry build -f sdist

	- name: Initialize kedro spaceflights project
	# kedro 0.18.1 is on purpose here, due to https://github.com/kedro-org/kedro-starters/issues/99
	run: \|
	pip install $(find "./dist" -name "*.tar.gz")
	kedro new --starter spaceflights --config tests/e2e/${{ matrix.e2e_case }}/starter-config.yml --verbose

	- name: Install project dependencies
	working-directory: ./spaceflights
	run: \|
	find "../dist" -name "*.tar.gz" \| xargs -I@ cp @ kedro-vertexai.tar.gz
	echo -e "\n./kedro-vertexai.tar.gz\n" >> src/requirements.txt
	echo -e "kedro-docker\n" >> src/requirements.txt
	sed -i '/kedro-telemetry/d' src/requirements.txt
	echo $(cat src/requirements.txt)
	pip install -r src/requirements.txt

	- name: Init and update configuration
	working-directory: ./spaceflights
	run: \|
	kedro docker init
	sed -i 's/$COPY src\/requirements.txt.*$$/\1\nCOPY kedro-vertexai.tar.gz ./g' Dockerfile
	echo "!data/01_raw" >> .dockerignore
	kedro vertexai init gid-ml-ops-sandbox europe-west4
	cp ../tests/e2e/${{ matrix.e2e_case }}/catalog.yml conf/base/catalog.yml
	cp ../tests/e2e/${{ matrix.e2e_case }}/vertexai.yml conf/base/vertexai.yml
	# Introducing tagging to pipelines
	if [[ "${{ matrix.e2e_case }}" == "grouping" ]]; then
	mv ../tests/e2e/${{ matrix.e2e_case }}/pipeline_data_processing.py src/spaceflights/pipelines/data_processing/pipeline.py
	mv ../tests/e2e/${{ matrix.e2e_case }}/pipeline_data_science.py src/spaceflights/pipelines/data_science/pipeline.py
	fi

	- name: Prepare docker env
	uses: docker/setup-buildx-action@v3
	id: buildx
	with:
	install: true

	- name: Build pipeline docker image
	run: \|
	cd ./spaceflights
	docker pull gcr.io/gid-ml-ops-sandbox/kedro-vertexai-e2e:${{ matrix.e2e_case }} \|\| true
	docker build --build-arg BASE_IMAGE=python:3.8-buster --tag kedro-vertexai-e2e:${{ matrix.e2e_case }} --load --cache-from=gcr.io/gid-ml-ops-sandbox/kedro-vertexai-e2e:${{ matrix.e2e_case }} .

	- name: Publish docker image to GCR
	uses: mattes/gce-docker-push-action@v1
	with:
	creds: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
	src: kedro-vertexai-e2e:${{ matrix.e2e_case }}
	dst: gcr.io/gid-ml-ops-sandbox/kedro-vertexai-e2e:${{ matrix.e2e_case }}

	- name: Set up GCP Credentials
	uses: google-github-actions/[email protected]
	with:
	credentials_json: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
	create_credentials_file: true
	cleanup_credentials: true

	- name: Run project on vertex pipeline
	run: \|
	cd ./spaceflights
	export KEDRO_CONFIG_COMMIT_ID=$GITHUB_SHA
	kedro vertexai run-once --wait-for-completion

	publish:
	if: github.event.pull_request == null && github.ref == 'refs/heads/master'
	needs: [ e2e_tests, codeql ]
	runs-on: ubuntu-latest
	env:
	PYTHON_PACKAGE: kedro_vertexai
	steps:
	- name: Checkout the repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0 # necessary to enable merging, all the history is needed
	- name: Set up Python 3.8
	uses: actions/[email protected]
	with:
	python-version: "3.8"
	- name: Build package dist from source # A better way will be : https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ but pep 517 is still marked as experimental
	run: \|
	pip install poetry
	poetry build
	- name: Merge back to develop # we have to set the config first on a fresh machine
	run: \|
	git config user.name github-actions
	git config user.email [email protected]
	git checkout -b develop --track origin/develop
	git merge master
	git push
	- name: Set dynamically package version as output variable # see https://github.com/actions/create-release/issues/39
	# see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
	id: set_package_version
	run: \|
	echo "::set-output name=PACKAGE_VERSION::$(cat $PYTHON_PACKAGE/__init__.py \| grep -Po '\d+\.\d+\.\d+')"
	- name: Create temporary file with the body content for the release
	run: \|
	grep -Poz "## \[${{steps.set_package_version.outputs.PACKAGE_VERSION}}] - \d{4}-\d{2}-\d{2}[\S\s]+?(?=## \[\d+\.\d+\.\d+\]\|\[.+\]:)" CHANGELOG.md > release_body.md
	- name: Create Release # https://github.com/actions/create-release
	id: create_release
	uses: actions/[email protected]
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
	with:
	tag_name: ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
	release_name: Release ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
	body_path: ./release_body.md
	draft: false
	prerelease: false
	- name: Rollback Release in case of run failure
	if: failure() && steps.create_release.outputs.id != ''
	uses: author/action-rollback@stable
	with:
	# Using a known release ID
	release_id: ${{ steps.create_release.outputs.id }}
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- name: Publish distribution to PyPI # official action from python maintainers
	uses: pypa/gh-action-pypi-publish@master
	with:
	user: __token__
	password: ${{ secrets.PYPI_PASSWORD }}
	verbose: true # trace if the upload fails

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Merge pull request #133 from getindata/grouping-feature #73

Workflow file

Merge pull request #133 from getindata/grouping-feature #73

Jobs

Run details

Workflow file for this run