Merge pull request #1269 from d-v-b/feat/pytest-container-management

use pytest to manage docker container startup for tests

Author: dimitri-yatsenko

.codespellrc

@@ -2,4 +2,4 @@
 skip = .git,*.pdf,*.svg,*.csv,*.ipynb,*.drawio
 # Rever -- nobody knows
 # numer -- numerator variable
-ignore-words-list = rever,numer
+ignore-words-list = rever,numer,astroid

.gitattributes

@@ -0,0 +1,2 @@
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true

.github/workflows/test.yaml

@@ -34,9 +34,7 @@ jobs:
           python-version: ${{matrix.py_ver}}
       - name: Integration test
         env:
-          PY_VER: ${{matrix.py_ver}}
           MYSQL_VER: ${{matrix.mysql_ver}}
-          # taking default variables set in docker-compose.yaml to sync with local test
         run: |
-          export HOST_UID=$(id -u)
-          docker compose --profile test up --quiet-pull --build --exit-code-from djtest djtest
+          pip install -e ".[test]"
+          pytest --cov-report term-missing --cov=datajoint tests

.gitignore

@@ -185,3 +185,5 @@ cython_debug/
 dj_local_conf.json
 *.env
 !.vscode/launch.json
+# pixi environments
+.pixi

docker-compose.yaml

@@ -1,4 +1,5 @@
-# HOST_UID=$(id -u) PY_VER=3.11 DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) docker compose --profile test up --build --exit-code-from djtest djtest
+# Development environment with MySQL and MinIO services
+# To run tests: pytest --cov-report term-missing --cov=datajoint tests
 services:
   db:
     image: datajoint/mysql:${MYSQL_VER:-8.0}

pixi.lock

@@ -78,11 +78,17 @@ Repository = "https://github.com/datajoint/datajoint-python"
 dj = "datajoint.cli:cli"
 datajoint = "datajoint.cli:cli"
 
-[project.optional-dependencies]
+[dependency-groups]
 test = [
   "pytest",
   "pytest-cov",
+  "pytest-env",
+  "docker",
+  "requests",
+  "graphviz"
 ]
+
+[project.optional-dependencies]
 dev = [
   "pre-commit",
   "black==24.2.0",
@@ -103,3 +109,33 @@ package-dir = {"" = "src"}
 
 [tool.setuptools.dynamic]
 version = { attr = "datajoint.version.__version__"}
+
+[tool.pytest_env]
+# Default values - pytest fixtures will override with actual container details
+DJ_USER="root"
+DJ_PASS="password"
+DJ_TEST_USER="datajoint"
+DJ_TEST_PASSWORD="datajoint"
+S3_ACCESS_KEY="datajoint"
+S3_SECRET_KEY="datajoint"
+S3_BUCKET="datajoint.test"
+PYTHON_USER="dja"
+JUPYTER_PASSWORD="datajoint"
+
+
+[tool.pixi.workspace]
+channels = ["conda-forge"]
+platforms = ["linux-64"]
+
+[tool.pixi.pypi-dependencies]
+datajoint = { path = ".", editable = true }
+
+[tool.pixi.environments]
+default = { solve-group = "default" }
+dev = { features = ["dev"], solve-group = "default" }
+test = { features = ["test"], solve-group = "default" }
+
+[tool.pixi.tasks]
+
+[tool.pixi.dependencies]
+graphviz = ">=13.1.2,<14"

pyproject.toml

@@ -278,6 +278,7 @@ def __setitem__(self, key, value):
             "database.host",
             "database.user",
             "database.password",
+            "database.port",
             "external.aws_access_key_id",
             "external.aws_secret_access_key",
             "loglevel",
@@ -288,6 +289,7 @@ def __setitem__(self, key, value):
                 "DJ_HOST",
                 "DJ_USER",
                 "DJ_PASS",
+                "DJ_PORT",
                 "DJ_AWS_ACCESS_KEY_ID",
                 "DJ_AWS_SECRET_ACCESS_KEY",
                 "DJ_LOG_LEVEL",
@@ -296,6 +298,14 @@ def __setitem__(self, key, value):
     )
     if v is not None
 }
+
+# Convert DJ_PORT from string to int if present
+if "database.port" in mapping and mapping["database.port"] is not None:
+    try:
+        mapping["database.port"] = int(mapping["database.port"])
+    except ValueError:
+        logger.warning(f"Invalid DJ_PORT value: {mapping['database.port']}, using default port 3306")
+        del mapping["database.port"]
 if mapping:
     logger.info(f"Overloaded settings {tuple(mapping)} from environment variables.")
     config.update(mapping)