Pin Pip and fix lint e2e tests (#201)

* Remove the installation step for lint e2e Signed-off-by: Nok <[email protected]> * Pin pip version to avoid package installation issue Signed-off-by: Nok <[email protected]> * Pin pip in e2e test before scenario Signed-off-by: Nok <[email protected]> * fix order Signed-off-by: Nok <[email protected]> * bug fix Signed-off-by: Nok <[email protected]> * fix gitpod Signed-off-by: Nok <[email protected]> * remove docker, a simple python 3.10 image is enough Signed-off-by: Nok <[email protected]> --------- Signed-off-by: Nok <[email protected]>
kedro-org · Dec 20, 2023 · e5da357 · e5da357
1 parent 4e39dd1
commit e5da357
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 9 deletions.
diff --git a/.github/workflows/all-checks.yml b/.github/workflows/all-checks.yml
@@ -37,7 +37,8 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install test requirements
-        run: make install-test-requirements
+        run: |
+          make install-test-requirements        
       - name: Add MSBuild to PATH
         if: matrix.os == 'windows-latest'
         uses: microsoft/setup-msbuild@v1

diff --git a/.gitpod.yml b/.gitpod.yml
@@ -4,6 +4,8 @@
 
 # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
 
+image: gitpod/workspace-python-3.10
+
 tasks:
   - init: |
       make sign-off

diff --git a/features/environment.py b/features/environment.py
@@ -1,17 +1,63 @@
 """Behave environment setup commands"""
+from __future__ import annotations
 
 import os
+import shlex
 import shutil
 import subprocess
 import tempfile
 import venv
 from pathlib import Path
-
-from typing import Set
+from typing import Any, Set
 
 _PATHS_TO_REMOVE: Set[Path] = set()
 
 
+
+
+def run(
+    cmd: list | str, split: bool = True, print_output: bool = False, **kwargs: Any
+) -> subprocess.CompletedProcess:
+    """Run a shell command.
+
+    Args:
+        cmd: A command string, or a command followed by program
+            arguments that will be submitted to Popen to run.
+
+        split: Flag that splits command to provide as multiple *args
+            to Popen. Default is True.
+
+        print_output: If True will print previously captured stdout.
+            Default is False.
+
+        **kwargs: Extra options to pass to subprocess.
+
+    Example:
+    ::
+        "ls"
+        "ls -la"
+        "chmod 754 local/file"
+
+    Returns:
+        Result with attributes args, returncode, stdout and stderr.
+
+    """
+    if isinstance(cmd, str) and split:
+        cmd = shlex.split(cmd)
+    result = subprocess.run(cmd, input="", capture_output=True, **kwargs)  # noqa: PLW1510
+    result.stdout = result.stdout.decode("utf-8")
+    result.stderr = result.stderr.decode("utf-8")
+    if print_output:
+        print(result.stdout)
+    return result
+
+def call(cmd, env):
+    res = run(cmd, env=env)
+    if res.returncode:
+        print(res.stdout)
+        print(res.stderr)
+        assert False
+
 def create_new_venv() -> Path:
     """Create a new venv.
     Returns:
@@ -34,6 +80,7 @@ def before_scenario(context, scenario):
     """Environment preparation before each test is run."""
     kedro_install_venv_dir = create_new_venv()
     context.venv_dir = kedro_install_venv_dir
+    context.env = os.environ.copy()
 
     if os.name == "posix":
         bin_dir = context.venv_dir / "bin"
@@ -45,6 +92,21 @@ def before_scenario(context, scenario):
     context.kedro = str(bin_dir / "kedro")
     context.python = str(bin_dir / "python")
 
+    # Fix Pip version
+    call(
+        [
+            context.python,
+            "-m",
+            "pip",
+            "install",
+            "-U",
+            # pip==23.3 breaks dependency resolution
+            "pip==23.2",
+        ],
+        env=context.env,
+    )
+
+
     starters_root = Path(__file__).parents[1]
     starter_names = [
         "astro-airflow-iris",

diff --git a/features/lint.feature b/features/lint.feature
@@ -3,41 +3,35 @@ Feature: Lint all starters
   Scenario: Lint astro-airflow-iris starter
     Given I have prepared a config file
     And I have run a non-interactive kedro new with the starter astro-airflow-iris
-    And I have installed the Kedro project's dependencies
     When I lint the project
     Then I should get a successful exit code
 
   Scenario: Lint databricks-iris starter
     Given I have prepared a config file
     And I have run a non-interactive kedro new with the starter databricks-iris
-    And I have installed the Kedro project's dependencies
     When I lint the project
     Then I should get a successful exit code
 
   Scenario: Lint spaceflights-pandas starter
     Given I have prepared a config file
     And I have run a non-interactive kedro new with the starter spaceflights-pandas
-    And I have installed the Kedro project's dependencies
     When I lint the project
     Then I should get a successful exit code
 
   Scenario: Lint spaceflights-pandas-viz starter
     Given I have prepared a config file
     And I have run a non-interactive kedro new with the starter spaceflights-pandas-viz
-    And I have installed the Kedro project's dependencies
     When I lint the project
     Then I should get a successful exit code
 
   Scenario: Lint spaceflights-pyspark starter
     Given I have prepared a config file
     And I have run a non-interactive kedro new with the starter spaceflights-pyspark
-    And I have installed the Kedro project's dependencies
     When I lint the project
     Then I should get a successful exit code
 
   Scenario: Lint spaceflights-pyspark-viz starter
     Given I have prepared a config file
     And I have run a non-interactive kedro new with the starter spaceflights-pyspark-viz
-    And I have installed the Kedro project's dependencies
     When I lint the project
     Then I should get a successful exit code