Stateful tests with Dataset

.github/workflows/ci-additional.yaml

@@ -35,14 +35,13 @@ jobs:
     runs-on: "ubuntu-latest"
     needs: detect-ci-trigger
     if: needs.detect-ci-trigger.outputs.triggered == 'false'
+
     defaults:
       run:
         shell: bash -l {0}
-
     env:
       CONDA_ENV_FILE: ci/requirements/environment.yml
       PYTHON_VERSION: "3.11"
-
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/ci.yaml

@@ -127,6 +127,14 @@ jobs:
         run: |
           python -c "import xarray"
 
+      - name: Restore cached hypothesis directory
+        uses: actions/cache@v4
+        with:
+          path: .hypothesis/
+          key: cache-hypothesis
+          enableCrossOsArchive: true
+          save-always: true
+
       - name: Run tests
         run: python -m pytest -n 4
           --timeout 180

.github/workflows/hypothesis.yaml

@@ -0,0 +1,100 @@
+name: Slow Hypothesis CI
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+    branches:
+      - "main"
+    types: [opened, reopened, synchronize, labeled]
+  workflow_dispatch: # allows you to trigger manually
+
+jobs:
+  detect-ci-trigger:
+    name: detect ci trigger
+    runs-on: ubuntu-latest
+    if: |
+      github.repository == 'pydata/xarray'
+      && (github.event_name == 'push' || github.event_name == 'pull_request')
+    outputs:
+      triggered: ${{ steps.detect-trigger.outputs.trigger-found }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - uses: xarray-contrib/ci-trigger@v1
+        id: detect-trigger
+        with:
+          keyword: "[skip-ci]"
+
+  hypothesis:
+    name: Slow Hypothesis Tests
+    runs-on: "ubuntu-latest"
+    needs: detect-ci-trigger
+    if: |
+        always()
+        && (
+            (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+            || needs.detect-ci-trigger.outputs.triggered == 'true'
+            || contains( github.event.pull_request.labels.*.name, 'run-slow-hypothesis')
+        )
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    env:
+      CONDA_ENV_FILE: ci/requirements/environment.yml
+      PYTHON_VERSION: "3.12"
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch all history for all branches and tags.
+
+      - name: set environment variables
+        run: |
+          echo "TODAY=$(date  +'%Y-%m-%d')" >> $GITHUB_ENV
+
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ci/requirements/environment.yml
+          environment-name: xarray-tests
+          create-args: >-
+            python=${{env.PYTHON_VERSION}}
+            pytest-reportlog
+          cache-environment: true
+          cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{env.PYTHON_VERSION}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+
+      - name: Install xarray
+        run: |
+          python -m pip install --no-deps -e .
+      - name: Version info
+        run: |
+          conda info -a
+          conda list
+          python xarray/util/print_versions.py
+      - name: Restore cached hypothesis directory
+        uses: actions/cache@v4
+        with:
+          path: .hypothesis/
+          key: cache-hypothesis
+          enableCrossOsArchive: true
+          save-always: true
+      - name: Run slow Hypothesis tests
+        if: success()
+        id: status
+        run: |
+           python -m pytest --hypothesis-show-statistics --run-slow-hypothesis properties/*.py \
+             --report-log output-${{ matrix.python-version }}-log.jsonl
+      - name: Generate and publish the report
+        if: |
+          failure()
+          && steps.status.outcome == 'failure'
+          && github.event_name == 'schedule'
+          && github.repository_owner == 'pydata'
+        uses: xarray-contrib/issue-from-pytest-log@v1
+        with:
+          log-path: output-${{ matrix.python-version }}-log.jsonl
+          issue-title: "Nightly Hypothesis tests failed"
+          issue-label: "topic-hypothesis"

properties/conftest.py

@@ -1,3 +1,24 @@
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--run-slow-hypothesis",
+        action="store_true",
+        default=False,
+        help="run slow hypothesis tests",
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--run-slow-hypothesis"):
+        return
+    skip_slow_hyp = pytest.mark.skip(reason="need --run-slow-hypothesis option to run")
+    for item in items:
+        if "slow_hypothesis" in item.keywords:
+            item.add_marker(skip_slow_hyp)
+
+
 try:
     from hypothesis import settings
 except ImportError: