From 769d685205d647b80fa8da65d0e8ed17dff933d0 Mon Sep 17 00:00:00 2001 From: ElenaKhaustova <157851531+ElenaKhaustova@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:51:50 +0100 Subject: [PATCH] Add `kedro package` as a CI check (#236) * Added e2e test for packaging and running astro-airflow-iris Signed-off-by: Elena Khaustova * Added the rest e2e project packaging tests Signed-off-by: Elena Khaustova * Added github action to run added e2e tests at the CI Signed-off-by: Elena Khaustova * Updated github action Signed-off-by: Elena Khaustova * Fix typo Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> --------- Signed-off-by: Elena Khaustova Co-authored-by: Merel Theisen <49397448+merelcht@users.noreply.github.com> --- .github/workflows/all-checks.yml | 36 +++++++++++++++++++++++++ features/package.feature | 46 ++++++++++++++++++++++++++++++++ features/steps/run_steps.py | 26 ++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 features/package.feature diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml index a059e2e9..1178f469 100644 --- a/.github/workflows/all-checks.yml +++ b/.github/workflows/all-checks.yml @@ -78,3 +78,39 @@ jobs: run: make install-test-requirements - name: Run linter for all starters run: behave features/lint.feature + + package: + strategy: + matrix: + os: [ windows-latest , ubuntu-latest ] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{matrix.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install test requirements + run: | + make install-test-requirements + - name: Add MSBuild to PATH + if: matrix.os == 'windows-latest' + uses: microsoft/setup-msbuild@v1 + - name: Install Microsoft Visual C++ Redistributable + if: matrix.os == 'windows-latest' + run: | + choco install vcredist-all + - name: Setup Hadoop binary + if: matrix.os == 'windows-latest' + run: | + Invoke-WebRequest "https://github.com/steveloughran/winutils/blob/master/hadoop-2.7.1/bin/winutils.exe?raw=true" -OutFile winutils.exe + Invoke-WebRequest "https://github.com/steveloughran/winutils/blob/master/hadoop-2.7.1/bin/hadoop.dll?raw=true" -OutFile hadoop.dll + Move-Item .\hadoop.dll C:\Windows\System32 + New-Item -ItemType directory -Path C:\hadoop\bin + Move-Item .\winutils.exe C:\hadoop\bin + echo "HADOOP_HOME=C:\hadoop" | Out-File -Append -Encoding ascii -FilePath $env:GITHUB_ENV + echo "PATH=$env:HADOOP_HOME\bin;$env:PATH" | Out-File -Append -Encoding ascii -FilePath $env:GITHUB_ENV + - name: Package and run all starters + run: behave features/package.feature diff --git a/features/package.feature b/features/package.feature new file mode 100644 index 00000000..0f67a2d9 --- /dev/null +++ b/features/package.feature @@ -0,0 +1,46 @@ +Feature: Package and run all starters + + Scenario: Package astro-airflow-iris project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter astro-airflow-iris + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project + Then I should get a successful exit code + + Scenario: Package spaceflights-pandas project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter spaceflights-pandas + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project + Then I should get a successful exit code + + Scenario: Package spaceflights-pandas-viz project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter spaceflights-pandas-viz + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project + Then I should get a successful exit code + + Scenario: Package spaceflights-pyspark project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter spaceflights-pyspark + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project + Then I should get a successful exit code + + Scenario: Package spaceflights-pyspark-viz project and run packaged project + Given I have prepared a config file + And I have run a non-interactive kedro new with the starter spaceflights-pyspark-viz + When I execute the kedro command "package" + Then I should get a successful exit code + When I install the project's python package + And I execute the installed project + Then I should get a successful exit code \ No newline at end of file diff --git a/features/steps/run_steps.py b/features/steps/run_steps.py index c0853e48..90f7f269 100644 --- a/features/steps/run_steps.py +++ b/features/steps/run_steps.py @@ -169,3 +169,29 @@ def check_status_code(context): assert False, "Expected exit code {}" " but got {}".format( OK_EXIT_CODE, context.result.returncode ) + + +@when('I execute the kedro command "{command}"') +def exec_kedro_target(context, command): + """Execute Kedro target.""" + split_command = command.split() + cmd = [context.kedro, *split_command] + context.result = subprocess.run(cmd, env=context.env, cwd=str(context.root_project_dir)) + + +@when("I install the project's python package") +def install_project_package_via_pip(context): + """Install a python package using pip.""" + dist_dir = context.root_project_dir / "dist" + (whl_file,) = dist_dir.glob("*.whl") + subprocess.run([context.pip, "install", str(whl_file)], env=context.env) + + +@when("I execute the installed project") +def exec_project(context): + """Execute installed Kedro project target.""" + context.result = subprocess.run( + [context.python, "-m", context.project_name.replace("-", "_")], + cwd=context.root_project_dir, + env=context.env + )