diff --git a/.circleci/config.yml b/.circleci/config.yml index c609a04a57..9ccc0a09c2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,18 +25,12 @@ commands: - run: name: Run tests command: | + mkdir -p test-reports . /opt/conda/etc/profile.d/conda.sh conda activate esmvaltool flake8 -j 4 - pytest -n 4 -m "not sequential" --junitxml=report-1.xml - pytest -n 0 -m "sequential" --cov-append --junitxml=report-2.xml + pytest -n 4 --junitxml=test-reports/report.xml esmvaltool version - - run: - name: Merge test reports - command: | - pip install junitparser - mkdir -p test-reports - junitparser merge report-*.xml test-reports/report.xml - store_test_results: path: test-reports/ - store_artifacts: diff --git a/.github/workflows/create-condalock-file.yml b/.github/workflows/create-condalock-file.yml index 80516c9c26..1f932e592b 100644 --- a/.github/workflows/create-condalock-file.yml +++ b/.github/workflows/create-condalock-file.yml @@ -66,9 +66,7 @@ jobs: - shell: bash -l {0} run: flake8 - shell: bash -l {0} - run: pytest -n 2 -m "not installation and not sequential" - - shell: bash -l {0} - run: pytest -n 0 -m "sequential" + run: pytest -n 2 -m "not installation" # Automated PR # see https://github.com/marketplace/actions/create-pull-request - name: Create Automated PR if conda lock file has changed diff --git a/.github/workflows/install-from-condalock-file.yml b/.github/workflows/install-from-condalock-file.yml index ec326ed96c..d4579f8537 100644 --- a/.github/workflows/install-from-condalock-file.yml +++ b/.github/workflows/install-from-condalock-file.yml @@ -61,9 +61,7 @@ jobs: - shell: bash -l {0} run: flake8 - shell: bash -l {0} - run: pytest -n 2 -m "not installation and not sequential" - - shell: bash -l {0} - run: pytest -n 0 -m "sequential" + run: pytest -n 2 -m "not installation" - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail uses: actions/upload-artifact@v2 diff --git a/.github/workflows/run-tests-comment.yml b/.github/workflows/run-tests-comment.yml index e9790badec..d5342971a7 100644 --- a/.github/workflows/run-tests-comment.yml +++ b/.github/workflows/run-tests-comment.yml @@ -36,9 +36,7 @@ jobs: - shell: bash -l {0} run: flake8 - shell: bash -l {0} - run: pytest -n 2 -m "not installation and not sequential" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - - shell: bash -l {0} - run: pytest -n 0 -m "sequential" + run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail uses: actions/upload-artifact@v2 @@ -77,9 +75,7 @@ jobs: - shell: bash -l {0} run: flake8 - shell: bash -l {0} - run: pytest -n 2 -m "not installation and not sequential" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - - shell: bash -l {0} - run: pytest -n 0 -m "sequential" + run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail uses: actions/upload-artifact@v2 diff --git a/.github/workflows/run-tests-monitor.yml b/.github/workflows/run-tests-monitor.yml index 6ab81fa704..b0abaf15d6 100644 --- a/.github/workflows/run-tests-monitor.yml +++ b/.github/workflows/run-tests-monitor.yml @@ -43,11 +43,7 @@ jobs: - shell: bash -l {0} run: pip install -e .[develop] 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/install.txt - shell: bash -l {0} - run: pytest -n 2 -m "not installation and not sequential" --db ../.pymon 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - - shell: bash -l {0} - run: python tests/parse_pymon.py - - shell: bash -l {0} - run: pytest -n 0 -m "sequential" + run: pytest -n 2 -m "not installation" --db ../.pymon 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - shell: bash -l {0} run: python tests/parse_pymon.py - name: Upload artifacts @@ -87,11 +83,7 @@ jobs: - shell: bash -l {0} run: pip install -e .[develop] 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/install.txt - shell: bash -l {0} - run: pytest -n 2 -m "not installation and not sequential" --db ../.pymon 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - - shell: bash -l {0} - run: python tests/parse_pymon.py - - shell: bash -l {0} - run: pytest -n 0 -m "sequential" + run: pytest -n 2 -m "not installation" --db ../.pymon 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - shell: bash -l {0} run: python tests/parse_pymon.py - name: Upload artifacts diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index a9322fe9e8..68068bff9c 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -60,9 +60,7 @@ jobs: - shell: bash -l {0} run: flake8 - shell: bash -l {0} - run: pytest -n 2 -m "not installation and not sequential" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - - shell: bash -l {0} - run: pytest -n 0 -m "sequential" + run: pytest -n 2 -m "not installation" 2>&1 | tee test_linux_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail uses: actions/upload-artifact@v2 @@ -100,9 +98,7 @@ jobs: - shell: bash -l {0} run: flake8 - shell: bash -l {0} - run: pytest -n 2 -m "not installation and not sequential" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - - shell: bash -l {0} - run: pytest -n 0 -m "sequential" + run: pytest -n 2 -m "not installation" 2>&1 | tee test_osx_artifacts_python_${{ matrix.python-version }}/test_report.txt - name: Upload artifacts if: ${{ always() }} # upload artifacts even if fail uses: actions/upload-artifact@v2 diff --git a/setup.cfg b/setup.cfg index 7fb5f9a5cf..6410153665 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,6 @@ log_level = WARNING markers = installation: Test requires installation of dependencies use_sample_data: Run functional tests using real data - sequential: Run these tests sequentially, see https://github.com/ESMValGroup/ESMValCore/issues/644 [flake8] exclude = diff --git a/tests/integration/cmor/_fixes/test_common.py b/tests/integration/cmor/_fixes/test_common.py index ff7e42a3e8..3a8bce5505 100644 --- a/tests/integration/cmor/_fixes/test_common.py +++ b/tests/integration/cmor/_fixes/test_common.py @@ -84,7 +84,6 @@ def hybrid_pressure_coord_fix_metadata(nc_path, short_name, fix): return var_names -@pytest.mark.sequential def test_cl_hybrid_pressure_coord_fix_metadata_with_a(test_data_path): """Test ``fix_metadata`` for ``cl``.""" vardef = get_var_info('CMIP6', 'Amon', 'cl') @@ -94,7 +93,6 @@ def test_cl_hybrid_pressure_coord_fix_metadata_with_a(test_data_path): assert 'a_bnds' in var_names -@pytest.mark.sequential def test_cl_hybrid_pressure_coord_fix_metadata_with_ap(test_data_path): """Test ``fix_metadata`` for ``cl``.""" vardef = get_var_info('CMIP6', 'Amon', 'cl') @@ -162,7 +160,6 @@ def hybrid_height_coord_fix_metadata(nc_path, short_name, fix): assert air_pressure_coord.units == 'Pa' -@pytest.mark.sequential def test_cl_hybrid_height_coord_fix_metadata(test_data_path): """Test ``fix_metadata`` for ``cl``.""" vardef = get_var_info('CMIP6', 'Amon', 'cl') diff --git a/tests/integration/cmor/_fixes/test_shared.py b/tests/integration/cmor/_fixes/test_shared.py index 5029c9306f..e82005f595 100644 --- a/tests/integration/cmor/_fixes/test_shared.py +++ b/tests/integration/cmor/_fixes/test_shared.py @@ -27,7 +27,6 @@ ) -@pytest.mark.sequential def test_altitude_to_pressure_func(): """Test altitude to pressure function.""" func = get_altitude_to_pressure_func() @@ -42,7 +41,6 @@ def test_altitude_to_pressure_func(): [101325.0, 100129.0]) -@pytest.mark.sequential def test_pressure_to_altitude_func(): """Test pressure to altitude function.""" func = get_pressure_to_altitude_func() @@ -71,7 +69,6 @@ def test_pressure_to_altitude_func(): ] -@pytest.mark.sequential @pytest.mark.parametrize('coord_dict,output', TEST_ADD_AUX_COORDS_FROM_CUBES) def test_add_aux_coords_from_cubes(coord_dict, output): """Test extraction of auxiliary coordinates from cubes.""" @@ -132,7 +129,6 @@ def test_add_aux_coords_from_cubes(coord_dict, output): ] -@pytest.mark.sequential @pytest.mark.parametrize('cube,output', TEST_ADD_PLEV_FROM_ALTITUDE) def test_add_plev_from_altitude(cube, output): """Test adding of pressure level coordinate.""" @@ -170,7 +166,6 @@ def test_add_plev_from_altitude(cube, output): ] -@pytest.mark.sequential @pytest.mark.parametrize('cube,output', TEST_ADD_ALTITUDE_FROM_PLEV) def test_add_altitude_from_plev(cube, output): """Test adding of altitude coordinate.""" @@ -216,7 +211,6 @@ def test_add_altitude_from_plev(cube, output): TEST_ADD_SCALAR_COORD_NO_VALS = [CUBE_1.copy(), CUBE_2.copy()] -@pytest.mark.sequential @pytest.mark.parametrize('cube_in,depth', TEST_ADD_SCALAR_COORD) def test_add_scalar_depth_coord(cube_in, depth): """Test adding of scalar depth coordinate.""" @@ -244,7 +238,6 @@ def test_add_scalar_depth_coord(cube_in, depth): assert coord == depth_coord -@pytest.mark.sequential @pytest.mark.parametrize('cube_in,height', TEST_ADD_SCALAR_COORD) def test_add_scalar_height_coord(cube_in, height): """Test adding of scalar height coordinate.""" @@ -272,7 +265,6 @@ def test_add_scalar_height_coord(cube_in, height): assert coord == height_coord -@pytest.mark.sequential @pytest.mark.parametrize('cube_in', TEST_ADD_SCALAR_COORD_NO_VALS) def test_add_scalar_lambda550nm_coord(cube_in): """Test adding of scalar lambda550nm coordinate.""" @@ -296,7 +288,6 @@ def test_add_scalar_lambda550nm_coord(cube_in): assert coord == lambda550nm_coord -@pytest.mark.sequential @pytest.mark.parametrize('cube_in,typeland', TEST_ADD_SCALAR_COORD) def test_add_scalar_typeland_coord(cube_in, typeland): """Test adding of scalar typeland coordinate.""" @@ -323,7 +314,6 @@ def test_add_scalar_typeland_coord(cube_in, typeland): assert coord == typeland_coord -@pytest.mark.sequential @pytest.mark.parametrize('cube_in,typesea', TEST_ADD_SCALAR_COORD) def test_add_scalar_typesea_coord(cube_in, typesea): """Test adding of scalar typesea coordinate.""" @@ -350,7 +340,6 @@ def test_add_scalar_typesea_coord(cube_in, typesea): assert coord == typesea_coord -@pytest.mark.sequential @pytest.mark.parametrize('cube_in,typesi', TEST_ADD_SCALAR_COORD) def test_add_scalar_typesi_coord(cube_in, typesi): """Test adding of scalar typesi coordinate.""" @@ -377,7 +366,6 @@ def test_add_scalar_typesi_coord(cube_in, typesi): assert coord == typesi_coord -@pytest.mark.sequential def test_cube_to_aux_coord(): """Test converting cube to auxiliary coordinate.""" cube = iris.cube.Cube( @@ -405,7 +393,6 @@ def test_cube_to_aux_coord(): ] -@pytest.mark.sequential @pytest.mark.parametrize('coord_name,output', TEST_GET_BOUNDS_CUBE) def test_get_bounds_cube(coord_name, output): """Test retrieving of bounds cube from list of cubes.""" @@ -443,7 +430,6 @@ def test_get_bounds_cube(coord_name, output): ] -@pytest.mark.sequential @pytest.mark.parametrize('var_names,output', TEST_FIX_BOUNDS) def test_fix_bounds(var_names, output): """Test retrieving of bounds cube from list of cubes.""" @@ -482,7 +468,6 @@ def test_fix_bounds(var_names, output): ] -@pytest.mark.sequential @pytest.mark.parametrize('cubes_in,decimals,out', TEST_ROUND) def test_round_coordinate(cubes_in, decimals, out): """Test rounding of coordinates.""" @@ -497,7 +482,6 @@ def test_round_coordinate(cubes_in, decimals, out): assert coords[0] == out[idx] -@pytest.mark.sequential def test_round_coordinates_single_coord(): """Test rounding of specified coordinate.""" coords, bounds = [10.0001], [[9.0001, 11.0001]]