Custom dataset that registers Vertex AI Artifacts #165

Workflow file for this run

.github/workflows/test_and_publish.yml at d3f925a

	name: Test & Publish

	on:
	push:
	branches:
	- master
	- develop
	paths-ignore:
	- "docs/**"
	pull_request:
	branches:
	- master
	- develop
	paths-ignore:
	- "docs/**"
	jobs:
	unit_tests:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: ['3.8', '3.9', '3.10']

	steps:
	- uses: actions/checkout@v4

	- name: Setup python ${{ matrix.python-version }}
	uses: actions/[email protected]
	with:
	python-version: ${{ matrix.python-version }}

	- name: Setup poetry
	run: \|
	python -V
	python -m pip install poetry

	- name: Check pre-commit status
	run: \|
	poetry install -v
	poetry run pre-commit run --all-files

	- name: Test with tox
	run: \|
	pip install tox-pip-version tox-gh-actions "tox<4.0.0"
	tox -v

	- name: Store coverage reports
	uses: actions/upload-artifact@v3
	with:
	name: coverage-${{ matrix.python-version }}
	path: coverage.xml
	if-no-files-found: error

	sonarcloud:
	runs-on: ubuntu-latest
	needs: unit_tests
	steps:

	- uses: actions/checkout@v4
	with:
	fetch-depth: 0

	- uses: actions/download-artifact@v3
	with:
	name: coverage-3.9
	path: .

	- name: SonarCloud Scan
	uses: sonarsource/sonarcloud-github-action@master
	with:
	projectBaseDir: ${{ github.workspace }}
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }}

	codeql:
	runs-on: ubuntu-latest
	needs: unit_tests
	permissions:
	actions: read
	contents: read
	security-events: write

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	# Initializes the CodeQL tools for scanning.
	- name: Initialize CodeQL
	uses: github/codeql-action/init@v2
	with:
	languages: python

	- name: Autobuild
	uses: github/codeql-action/autobuild@v2

	- name: Perform CodeQL Analysis
	uses: github/codeql-action/analyze@v2

	### PROJECT SPECIFIC CONFIGURATION HERE
	e2e_tests:
	runs-on: ubuntu-latest
	timeout-minutes: 60
	if: (github.event.pull_request.head.repo.full_name == github.repository \|\| github.event.pull_request == null)
	needs: [unit_tests] #, sonarcloud]
	strategy:
	matrix:
	e2e_case: ["standard", "grouping"]
	python-version: ['3.10']
	env:
	PROJECT_NAME: kedro-vertexai
	IMAGE_REGISTRY: gcr.io/gid-ml-ops-sandbox
	steps:
	- uses: actions/checkout@v4

	- name: Setup python
	uses: actions/[email protected]
	with:
	python-version: ${{ matrix.python-version }}

	- name: Setup poetry
	run: \|
	python -V
	python -m pip install poetry

	- name: Build the package
	run: \|
	poetry build -f sdist

	- name: Initialize starter project
	run: \|
	pip install $(find "./dist" -name "*.tar.gz")
	kedro new --starter spaceflights-pandas --config tests/e2e/${{ matrix.e2e_case }}/starter-config.yml --verbose

	- name: Install starter requirements
	working-directory: ./spaceflights
	run: \|
	find "../dist" -name "*.tar.gz" \| xargs -I@ cp @ $PROJECT_NAME.tar.gz
	echo -e "\n./$PROJECT_NAME.tar.gz\n" >> requirements.txt
	echo -e "kedro-docker\n" >> requirements.txt
	sed -i '/kedro-telemetry/d' requirements.txt
	echo $(cat requirements.txt)
	pip install -r requirements.txt

	- name: Init and update starter configuration
	working-directory: ./spaceflights
	run: \|
	kedro docker init
	sed -i "s/$COPY requirements.txt.*$$/\1\nCOPY $PROJECT_NAME.tar.gz ./g" Dockerfile
	cat Dockerfile
	echo '!data/01_raw' >> .dockerignore
	kedro vertexai init gid-ml-ops-sandbox europe-west4
	cp -r ../tests/e2e/${{ matrix.e2e_case }}/overwrite/* .
	sed -i "s/$CONFIG_LOADER_ARGS.*$$/from omegaconf.resolvers import oc\n\1\n \"custom_resolvers\": { \"oc.env\": oc.env },/g" src/spaceflights/settings.py

	- name: Prepare docker env
	uses: docker/setup-buildx-action@v3
	id: buildx
	with:
	install: true

	- name: Set up GCP Credentials
	uses: google-github-actions/[email protected]
	with:
	credentials_json: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
	create_credentials_file: true
	cleanup_credentials: true

	- name: Build docker image
	working-directory: ./spaceflights # FIXME: pull throws unauthorized error - add docker login? - update in template
	run: \|
	docker pull $IMAGE_REGISTRY/$PROJECT_NAME-e2e:${{ matrix.e2e_case }} \|\| true
	docker build --build-arg BASE_IMAGE=python:${{ matrix.python-version }}-buster --tag $PROJECT_NAME-e2e:${{ matrix.e2e_case }} --load --cache-from=$IMAGE_REGISTRY/$PROJECT_NAME-e2e:${{ matrix.e2e_case }} .

	- name: Publish docker image to docker registry
	uses: mattes/gce-docker-push-action@v1
	with:
	creds: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
	src: ${{ env.PROJECT_NAME }}-e2e:${{ matrix.e2e_case }}
	dst: ${{ env.IMAGE_REGISTRY }}/${{ env.PROJECT_NAME }}-e2e:${{ matrix.e2e_case }}

	- name: Run project on vertex pipeline
	working-directory: ./spaceflights
	run: \|
	export KEDRO_CONFIG_COMMIT_ID=$GITHUB_SHA
	kedro vertexai run-once --wait-for-completion

	publish:
	# only run on push to master
	if: github.event.pull_request == null && github.ref == 'refs/heads/master'
	needs: [e2e_tests, codeql]
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version: ['3.10']
	env:
	PYTHON_PACKAGE: kedro_vertexai
	steps:
	- name: Checkout the repo
	uses: actions/checkout@v4
	with:
	fetch-depth: 0 # necessary to enable merging, all the history is needed

	- name: Set up Python ${{ matrix.python-version }}
	uses: actions/[email protected]
	with:
	python-version: ${{ matrix.python-version }}

	- name: Build package dist from source # A better way will be : https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ but pep 517 is still marked as experimental
	run: \|
	pip install poetry
	poetry build

	- name: Merge back to develop # we have to set the config first on a fresh machine
	run: \|
	git config user.name github-actions
	git config user.email [email protected]
	git checkout -B develop --track origin/develop
	git merge master
	git push

	- name: Set dynamically package version as output variable # see https://github.com/actions/create-release/issues/39
	# see https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable
	id: set_package_version
	run: \|
	echo "::set-output name=PACKAGE_VERSION::$(cat $PYTHON_PACKAGE/__init__.py \| grep -Po '\d+\.\d+\.\d+')"

	- name: Create temporary file with the body content for the release
	run: \|
	grep -Poz "## \[${{steps.set_package_version.outputs.PACKAGE_VERSION}}] - \d{4}-\d{2}-\d{2}[\S\s]+?(?=## \[\d+\.\d+\.\d+\]\|\[.+\]:)" CHANGELOG.md > release_body.md

	- name: Create Release # https://github.com/actions/create-release
	id: create_release
	uses: actions/[email protected]
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token
	with:
	tag_name: ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
	release_name: Release ${{ steps.set_package_version.outputs.PACKAGE_VERSION }}
	body_path: ./release_body.md
	draft: false
	prerelease: false

	- name: Rollback Release in case of run failure
	if: failure() && steps.create_release.outputs.id != ''
	uses: author/action-rollback@stable
	with:
	# Using a known release ID
	release_id: ${{ steps.create_release.outputs.id }}
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

	- name: Publish distribution to PyPI # official action from python maintainers
	uses: pypa/gh-action-pypi-publish@master
	with:
	user: __token__
	password: ${{ secrets.PYPI_PASSWORD }}
	verbose: true # trace if the upload fails

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Custom dataset that registers Vertex AI Artifacts #165

Workflow file

Custom dataset that registers Vertex AI Artifacts #165

Jobs

Run details

Workflow file for this run