diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cdf5a78db..db0d7e217 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,6 +7,10 @@ on: pull_request: branches: [main] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + jobs: changes: if: ${{ !startsWith(github.ref, 'refs/tags/') }} diff --git a/.github/workflows/test-cmake.yml b/.github/workflows/test-cmake.yml index f9ad986ce..6702130cc 100644 --- a/.github/workflows/test-cmake.yml +++ b/.github/workflows/test-cmake.yml @@ -96,9 +96,20 @@ jobs: if-no-files-found: error retention-days: 1 - - name: Run tests + - name: Unit tests if: ${{ !matrix.build_only }} - run: pixi run test ${{ matrix.build_type }} + timeout-minutes: 5 + run: pixi run unit-test ${{ matrix.build_type }} + + - name: Integration tests + if: ${{ !matrix.build_only }} + timeout-minutes: 20 + run: pixi run integration-test ${{ matrix.build_type }} + + - name: Smoke tests + if: ${{ !matrix.build_only }} + timeout-minutes: 15 + run: pixi run smoke-test ${{ matrix.build_type }} - name: Print cache stats and stop server if: always() @@ -146,5 +157,14 @@ jobs: if: runner.os != 'Windows' run: chmod +x build/${{ matrix.build_type }}/bin/* - - name: Run tests - run: pixi run -e test-run test ${{ matrix.build_type }} + - name: Unit tests + timeout-minutes: 5 + run: pixi run -e test-run unit-test ${{ matrix.build_type }} + + - name: Integration tests + timeout-minutes: 20 + run: pixi run -e test-run integration-test ${{ matrix.build_type }} + + - name: Smoke tests + timeout-minutes: 10 + run: pixi run -e test-run smoke-test ${{ matrix.build_type }} diff --git a/pixi.lock b/pixi.lock index 775ec1cdb..922bdcc9d 100644 --- a/pixi.lock +++ b/pixi.lock @@ -1078,6 +1078,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl linux-aarch64: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda @@ -1152,6 +1153,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl osx-64: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_9.conda @@ -1224,6 +1226,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda @@ -1289,6 +1292,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl win-64: - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda @@ -1343,6 +1347,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl format: channels: @@ -1704,6 +1709,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl linux-aarch64: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda @@ -1782,6 +1788,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl osx-64: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_9.conda @@ -1858,6 +1865,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_8.conda @@ -1926,6 +1934,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl win-64: - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_8.conda @@ -1982,6 +1991,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl test-run: channels: @@ -2025,6 +2035,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl linux-aarch64: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/_openmp_mutex-4.5-20_gnu.conda @@ -2058,6 +2069,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl osx-64: - conda: https://conda.anaconda.org/conda-forge/osx-64/bzip2-1.0.8-h500dc9f_9.conda @@ -2113,6 +2125,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl osx-arm64: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/bzip2-1.0.8-hd037594_9.conda @@ -2168,6 +2181,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl win-64: - conda: https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-h0ad9c76_9.conda @@ -2199,6 +2213,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl win-arm64: - conda: https://conda.anaconda.org/conda-forge/win-arm64/bzip2-1.0.8-h50b96f5_9.conda @@ -2229,6 +2244,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl packages: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -7795,6 +7811,13 @@ packages: - coverage>=6.2 ; extra == 'testing' - hypothesis>=5.7.1 ; extra == 'testing' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl + name: pytest-timeout + version: 2.4.0 + sha256: c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2 + requires_dist: + - pytest>=7.0.0 + requires_python: '>=3.7' - conda: https://conda.anaconda.org/conda-forge/linux-64/python-3.14.2-h32b2ec7_100_cp314.conda build_number: 100 sha256: a120fb2da4e4d51dd32918c149b04a08815fd2bd52099dad1334647984bb07f1 diff --git a/pixi.toml b/pixi.toml index d106509ce..bedbd7173 100644 --- a/pixi.toml +++ b/pixi.toml @@ -102,6 +102,7 @@ lld = "==20.1.8" [feature.test.pypi-dependencies] pytest = "*" pytest-asyncio = ">=1.1.0" +pytest-timeout = "*" pygls = ">=2.0.0" lsprotocol = ">=2024.0.0" @@ -165,8 +166,8 @@ cmd = './build/{{ type }}/bin/unit_tests --test-dir="./tests/data"' [feature.test.tasks.integration-test] args = [{ arg = "type", default = "RelWithDebInfo" }] cmd = """ -pytest -s --log-cli-level=INFO tests/integration \ - --executable=./build/{{ type }}/bin/clice +pytest -s --log-cli-level=INFO --timeout=300 --timeout-method=thread \ + tests/integration --executable=./build/{{ type }}/bin/clice """ [feature.test.tasks.smoke-test] diff --git a/src/semantic/ast_utility.cpp b/src/semantic/ast_utility.cpp index 0be1b7d4e..d09c6144d 100644 --- a/src/semantic/ast_utility.cpp +++ b/src/semantic/ast_utility.cpp @@ -308,6 +308,10 @@ const clang::NamedDecl* decl_of_impl(const void* T) { } auto decl_of(clang::QualType type) -> const clang::NamedDecl* { + if(type.isNull()) { + return nullptr; + } + // Strip type-sugar that wraps the underlying type without adding a decl // (e.g. ElaboratedType for "struct Foo" vs plain "Foo"). if(auto ET = type->getAs()) { diff --git a/src/server/config.cpp b/src/server/config.cpp index a2d8087ca..ed767be23 100644 --- a/src/server/config.cpp +++ b/src/server/config.cpp @@ -6,6 +6,7 @@ #include "support/glob_pattern.h" #include "support/logging.h" +#include "kota/async/io/system.h" #include "kota/codec/json/json.h" #include "kota/codec/toml/toml.h" #include "llvm/Support/FileSystem.h" @@ -65,8 +66,10 @@ void Config::apply_defaults(llvm::StringRef workspace_root) { if(p.stateful_worker_count == 0) p.stateful_worker_count = 2; - if(p.stateless_worker_count == 0) - p.stateless_worker_count = 3; + if(p.stateless_worker_count == 0) { + auto cores = kota::sys::parallelism(); + p.stateless_worker_count = std::max(cores / 2, 2u); + } if(p.worker_memory_limit == 0) p.worker_memory_limit = 4ULL * 1024 * 1024 * 1024; // 4GB diff --git a/src/server/indexer.cpp b/src/server/indexer.cpp index a192e06ba..f46aa386a 100644 --- a/src/server/indexer.cpp +++ b/src/server/indexer.cpp @@ -1,5 +1,6 @@ #include "server/indexer.h" +#include #include #include #include @@ -624,6 +625,23 @@ void Indexer::enqueue(std::uint32_t server_path_id) { index_queue.push_back(server_path_id); } +void Indexer::pause_indexing() { + ++pause_depth; + if(pause_depth == 1) { + resume_event.reset(); + LOG_DEBUG("Background indexing paused"); + } +} + +void Indexer::resume_indexing() { + if(pause_depth > 0) + --pause_depth; + if(pause_depth == 0) { + resume_event.set(); + LOG_DEBUG("Background indexing resumed"); + } +} + void Indexer::schedule() { if(!*workspace.config.project.enable_indexing || indexing_active || indexing_scheduled) return; @@ -636,6 +654,76 @@ void Indexer::schedule() { loop.schedule(run_background_indexing()); } +kota::task<> Indexer::index_one(std::uint32_t server_path_id) { + auto file_path = std::string(workspace.path_pool.resolve(server_path_id)); + + if(sessions.contains(server_path_id)) + co_return; + + if(!need_update(file_path)) + co_return; + + // For module interface units, compile their PCM (and transitive deps) + // first so the stateless worker has the artifacts it needs. + if(workspace.compile_graph && workspace.path_to_module.contains(server_path_id)) { + co_await workspace.compile_graph->compile(server_path_id); + } + + worker::BuildParams params; + params.kind = worker::BuildKind::Index; + params.file = file_path; + if(!compiler.fill_compile_args(file_path, params.directory, params.arguments, nullptr)) + co_return; + + workspace.fill_pcm_deps(params.pcms); + + LOG_INFO("Background indexing: {}", file_path); + + auto result = co_await pool.send_stateless(params); + if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) { + LOG_INFO("Background indexing got TUIndex for {}: {} bytes", + file_path, + result.value().tu_index_data.size()); + merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); + } else if(result.has_value() && !result.value().success) { + LOG_WARN("Background index failed for {}: {}", file_path, result.value().error); + } else if(result.has_value() && result.value().tu_index_data.empty()) { + LOG_WARN("Background index returned empty TUIndex for {}", file_path); + } else { + LOG_WARN("Background index IPC error for {}: {}", file_path, result.error().message); + } +} + +kota::task<> Indexer::monitor_resources(std::uint32_t generation) { + while(generation == monitor_generation) { + co_await kota::sleep(std::chrono::milliseconds(3000), loop); + + if(generation != monitor_generation) + break; + + auto mem = kota::sys::memory(); + if(mem.total == 0) + continue; + + // Respect cgroup/container limits when present. + auto effective_total = + (mem.constrained > 0 && mem.constrained < mem.total) ? mem.constrained : mem.total; + auto ratio = static_cast(mem.available) / static_cast(effective_total); + + if(ratio < 0.15 && max_concurrent > 1) { + --max_concurrent; + LOG_INFO("Index concurrency -> {} (memory pressure: {:.0f}% available)", + max_concurrent, + ratio * 100); + } else if(ratio > 0.30 && max_concurrent < baseline_concurrent) { + ++max_concurrent; + LOG_DEBUG("Index concurrency -> {} (memory OK: {:.0f}% available)", + max_concurrent, + ratio * 100); + } + } +} + kota::task<> Indexer::run_background_indexing() { if(index_idle_timer) { co_await index_idle_timer->wait(); @@ -648,48 +736,88 @@ kota::task<> Indexer::run_background_indexing() { } indexing_active = true; - std::size_t processed = 0; + ++monitor_generation; + loop.schedule(monitor_resources(monitor_generation)); + + // Put module interface units first so their PCMs are built before + // non-module files that might import them. + std::stable_partition( + index_queue.begin() + index_queue_pos, + index_queue.end(), + [this](std::uint32_t id) { return workspace.path_to_module.contains(id); }); + + auto batch = index_queue.size() - index_queue_pos; + std::size_t dispatched = 0; + std::size_t completed = 0; + finished = 0; + + // Progress reporting via LSP $/progress. + std::optional> progress; + if(peer) { + progress.emplace(*peer, protocol::ProgressToken(std::string("clice/backgroundIndex"))); + auto create_result = co_await progress->create(); + if(!create_result.has_error()) { + progress->begin("Indexing", std::format("0/{} files", batch), 0); + } else { + progress.reset(); + } + } - while(index_queue_pos < index_queue.size()) { - auto server_path_id = index_queue[index_queue_pos]; - index_queue_pos++; + while(index_queue_pos < index_queue.size() || inflight > 0) { + // Dispatch new tasks up to max_concurrent. + while(index_queue_pos < index_queue.size() && inflight < max_concurrent) { + // Wait if paused by a user request. + if(pause_depth > 0) { + co_await resume_event.wait(); + } - auto file_path = std::string(workspace.path_pool.resolve(server_path_id)); + auto server_path_id = index_queue[index_queue_pos++]; - if(sessions.contains(server_path_id)) - continue; + // Quick pre-filter: skip open files and fresh files without + // consuming a concurrency slot. + auto file_path = std::string(workspace.path_pool.resolve(server_path_id)); + if(sessions.contains(server_path_id) || !need_update(file_path)) { + ++completed; + continue; + } - if(!need_update(file_path)) - continue; + ++inflight; + ++dispatched; + + // Launch the index task. On completion it decrements + // inflight, bumps finished, and signals the event. + loop.schedule([](Indexer* self, std::uint32_t id, kota::event& done) -> kota::task<> { + co_await self->index_one(id); + --self->inflight; + ++self->finished; + done.set(); + }(this, server_path_id, completion_event)); + } - worker::BuildParams params; - params.kind = worker::BuildKind::Index; - params.file = file_path; - if(!compiler.fill_compile_args(file_path, params.directory, params.arguments, nullptr)) - continue; + if(inflight == 0) + break; - workspace.fill_pcm_deps(params.pcms); - - LOG_INFO("Background indexing: {}", file_path); - - auto result = co_await pool.send_stateless(params); - if(result.has_value() && result.value().success && !result.value().tu_index_data.empty()) { - LOG_INFO("Background indexing got TUIndex for {}: {} bytes", - file_path, - result.value().tu_index_data.size()); - merge(result.value().tu_index_data.data(), result.value().tu_index_data.size()); - ++processed; - } else if(result.has_value() && !result.value().success) { - LOG_WARN("Background index failed for {}: {}", file_path, result.value().error); - } else if(result.has_value() && result.value().tu_index_data.empty()) { - LOG_WARN("Background index returned empty TUIndex for {}", file_path); - } else { - LOG_WARN("Background index IPC error for {}: {}", file_path, result.error().message); + // Wait for at least one task to finish. + co_await completion_event.wait(); + completion_event.reset(); + + // Drain all completions that occurred since last wake. + completed += std::exchange(finished, 0); + + // Report progress. + if(progress) { + auto pct = batch > 0 ? static_cast(completed * 100 / batch) : 100; + progress->report(std::format("{}/{} files", completed, batch), pct); } } + if(progress) { + progress->end(std::format("Indexed {} files", dispatched)); + } + indexing_active = false; - LOG_INFO("Background indexing complete: {} files processed", processed); + ++monitor_generation; // Stop the monitor coroutine. + LOG_INFO("Background indexing complete: {} files dispatched", dispatched); save(workspace.config.project.index_dir); } diff --git a/src/server/indexer.h b/src/server/indexer.h index 07c177b6c..7b6a621b1 100644 --- a/src/server/indexer.h +++ b/src/server/indexer.h @@ -12,7 +12,9 @@ #include "server/workspace.h" #include "kota/async/async.h" +#include "kota/ipc/codec/json.h" #include "kota/ipc/lsp/position.h" +#include "kota/ipc/lsp/progress.h" #include "kota/ipc/lsp/protocol.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" @@ -62,6 +64,47 @@ class Indexer { loop(loop), workspace(workspace), sessions(sessions), pool(pool), compiler(compiler), is_file_open(std::move(is_file_open)) {} + /// Set the LSP peer for progress reporting. Must be called before + /// schedule() if progress notifications are desired. + void set_peer(kota::ipc::JsonPeer* p) { + peer = p; + } + + /// Temporarily pause background indexing to give priority to user + /// requests. Indexing tasks already dispatched to workers continue, + /// but no new tasks will be sent until resume_indexing() is called. + void pause_indexing(); + + /// Resume background indexing after a pause. + void resume_indexing(); + + /// RAII guard that pauses indexing for its lifetime. + struct [[nodiscard]] ScopedPause { + Indexer& indexer; + + explicit ScopedPause(Indexer& idx) : indexer(idx) { + indexer.pause_indexing(); + } + + ~ScopedPause() { + indexer.resume_indexing(); + } + + ScopedPause(const ScopedPause&) = delete; + ScopedPause& operator=(const ScopedPause&) = delete; + }; + + ScopedPause scoped_pause() { + return ScopedPause{*this}; + } + + /// Set the maximum number of concurrent index tasks. + /// Also sets the baseline that dynamic adjustment will restore to. + void set_max_concurrency(std::size_t n) { + max_concurrent = std::max(n, 1); + baseline_concurrent = max_concurrent; + } + /// Add a file to the background indexing queue. void enqueue(std::uint32_t server_path_id); @@ -175,6 +218,9 @@ class Indexer { /// server-path-id-keyed sessions map to project-level path_ids. std::function is_file_open; + /// LSP peer for progress reporting (optional, not owned). + kota::ipc::JsonPeer* peer = nullptr; + /// Background indexing queue and scheduling state. std::vector index_queue; std::size_t index_queue_pos = 0; @@ -182,7 +228,30 @@ class Indexer { bool indexing_scheduled = false; std::shared_ptr index_idle_timer; + /// Concurrency control for background indexing. + std::size_t max_concurrent = 2; + std::size_t baseline_concurrent = 2; + std::size_t inflight = 0; + std::size_t finished = 0; ///< Incremented by each completed dispatch task. + + /// Pause/resume: when paused, new index tasks wait on this event. + /// Uses a counter so nested pause/resume pairs work correctly. + std::size_t pause_depth = 0; + kota::event resume_event{true}; + + /// Completion event — signalled by each finished dispatch task so the + /// main loop can wake up. Must be a member (not local to the coroutine) + /// because inflight tasks capture it by reference and may outlive the + /// coroutine frame during server shutdown. + kota::event completion_event; + + /// Generation counter — incremented each run so a stale monitor_resources + /// coroutine can detect that its owning run has ended. + std::uint32_t monitor_generation = 0; + kota::task<> run_background_indexing(); + kota::task<> index_one(std::uint32_t server_path_id); + kota::task<> monitor_resources(std::uint32_t generation); }; } // namespace clice diff --git a/src/server/master_server.cpp b/src/server/master_server.cpp index f5a6bf853..6255cec6f 100644 --- a/src/server/master_server.cpp +++ b/src/server/master_server.cpp @@ -331,6 +331,9 @@ void MasterServer::register_handlers() { indexer.schedule(); }; + indexer.set_peer(&peer); + indexer.set_max_concurrency(cfg.stateless_worker_count.value); + load_workspace(); }); @@ -670,28 +673,33 @@ void MasterServer::register_handlers() { /// Feature requests — stateless forwarding. - peer.on_request([this](RequestContext& ctx, - const protocol::CompletionParams& params) -> RawResult { - auto path = uri_to_path(params.text_document_position_params.text_document.uri); - auto path_id = workspace.path_pool.intern(path); - auto sit = sessions.find(path_id); - if(sit == sessions.end()) - co_return serde_raw{"null"}; - co_return co_await compiler.handle_completion(params.text_document_position_params.position, - sit->second); - }); - peer.on_request( - [this](RequestContext& ctx, const protocol::SignatureHelpParams& params) -> RawResult { + [this](RequestContext& ctx, const protocol::CompletionParams& params) -> RawResult { auto path = uri_to_path(params.text_document_position_params.text_document.uri); auto path_id = workspace.path_pool.intern(path); auto sit = sessions.find(path_id); if(sit == sessions.end()) co_return serde_raw{"null"}; - co_return co_await compiler.forward_build(worker::BuildKind::SignatureHelp, + auto pause = indexer.scoped_pause(); + auto result = + co_await compiler.handle_completion(params.text_document_position_params.position, + sit->second); + co_return std::move(result); + }); + + peer.on_request([this](RequestContext& ctx, + const protocol::SignatureHelpParams& params) -> RawResult { + auto path = uri_to_path(params.text_document_position_params.text_document.uri); + auto path_id = workspace.path_pool.intern(path); + auto sit = sessions.find(path_id); + if(sit == sessions.end()) + co_return serde_raw{"null"}; + auto pause = indexer.scoped_pause(); + auto result = co_await compiler.forward_build(worker::BuildKind::SignatureHelp, params.text_document_position_params.position, sit->second); - }); + co_return std::move(result); + }); /// Hierarchy queries — index-based. diff --git a/src/server/stateless_worker.cpp b/src/server/stateless_worker.cpp index 64ad11fb1..7d85f8fcf 100644 --- a/src/server/stateless_worker.cpp +++ b/src/server/stateless_worker.cpp @@ -15,6 +15,22 @@ namespace clice { +/// RAII guard that lowers the current process's scheduling priority and +/// restores it on destruction. +struct ScopedNice { + int saved; + + explicit ScopedNice(int increment = 10) { + auto p = kota::sys::priority(); + saved = p ? *p : 0; + kota::sys::set_priority(saved + increment); + } + + ~ScopedNice() { + kota::sys::set_priority(saved); + } +}; + using kota::ipc::RequestResult; using RequestContext = kota::ipc::BincodePeer::RequestContext; @@ -283,7 +299,10 @@ int run_stateless_worker_mode(const std::string& worker_name, const std::string& switch(params.kind) { case K::BuildPCH: return handle_build_pch(params); case K::BuildPCM: return handle_build_pcm(params); - case K::Index: return handle_index(params); + case K::Index: { + ScopedNice guard; + return handle_index(params); + } case K::Completion: return handle_completion(params); case K::SignatureHelp: return handle_signature_help(params); } diff --git a/src/server/worker_pool.cpp b/src/server/worker_pool.cpp index f7f00bbc3..e4beb9725 100644 --- a/src/server/worker_pool.cpp +++ b/src/server/worker_pool.cpp @@ -13,14 +13,13 @@ namespace { /// Coroutine that drains a worker's stderr pipe. /// Workers write their own log files, so this only captures unexpected output -/// (crash stacktraces, assertion failures, etc.) that bypasses spdlog. +/// (crash stacktraces, assertion failures, sanitizer reports, etc.). kota::task<> drain_stderr(kota::pipe stderr_pipe, std::string prefix) { std::string buffer; while(true) { auto result = co_await stderr_pipe.read(); - if(!result.has_value()) { + if(!result.has_value()) break; - } auto& chunk = result.value(); if(chunk.empty()) break; @@ -34,7 +33,7 @@ kota::task<> drain_stderr(kota::pipe stderr_pipe, std::string prefix) { break; auto line = buffer.substr(pos, nl - pos); if(!line.empty()) { - LOG_DEBUG("{} {}", prefix, line); + LOG_WARN("{} {}", prefix, line); } pos = nl + 1; } @@ -42,7 +41,7 @@ kota::task<> drain_stderr(kota::pipe stderr_pipe, std::string prefix) { } if(!buffer.empty()) { - LOG_DEBUG("{} {}", prefix, buffer); + LOG_WARN("{} {}", prefix, buffer); } } @@ -108,24 +107,29 @@ bool WorkerPool::spawn_worker(const std::string& self_path, }); auto& w = workers.back(); + w.alive = true; + ++alive_count_; loop.schedule(w.peer->run()); return true; } bool WorkerPool::start(const WorkerPoolOptions& options) { + options_ = options; log_dir_ = options.log_dir; for(std::uint32_t i = 0; i < options.stateless_count; ++i) { if(!spawn_worker(options.self_path, false, 0)) { return false; } + loop.schedule(monitor_worker(stateless_workers.size() - 1, false)); } for(std::uint32_t i = 0; i < options.stateful_count; ++i) { if(!spawn_worker(options.self_path, true, options.worker_memory_limit)) { return false; } + loop.schedule(monitor_worker(stateful_workers.size() - 1, true)); } // Register evicted notification handler for each stateful worker @@ -145,29 +149,24 @@ bool WorkerPool::start(const WorkerPoolOptions& options) { kota::task<> WorkerPool::stop() { LOG_INFO("WorkerPool stopping..."); + shutting_down_ = true; - // Close output pipes to signal workers to exit gracefully - for(auto& w: stateless_workers) { + // Close output pipes to signal workers to exit gracefully. + for(auto& w: stateless_workers) w.peer->close_output(); - } - for(auto& w: stateful_workers) { + for(auto& w: stateful_workers) w.peer->close_output(); - } - // Send SIGTERM to all workers - for(auto& w: stateless_workers) { + // Send SIGTERM. monitor_worker coroutines handle the wait. + for(auto& w: stateless_workers) w.proc.kill(SIGTERM); - } - for(auto& w: stateful_workers) { + for(auto& w: stateful_workers) w.proc.kill(SIGTERM); - } - // Wait for all worker processes to exit - for(auto& w: stateless_workers) { - co_await w.proc.wait(); - } - for(auto& w: stateful_workers) { - co_await w.proc.wait(); + // Wait until all monitor_worker coroutines have finished. + if(alive_count_ > 0) { + all_exited_.reset(); + co_await all_exited_.wait(); } LOG_INFO("WorkerPool stopped"); @@ -198,7 +197,10 @@ std::size_t WorkerPool::assign_worker(std::uint32_t path_id) { std::size_t WorkerPool::pick_least_loaded() { std::size_t best = 0; for(std::size_t i = 1; i < stateful_workers.size(); ++i) { - if(stateful_workers[i].owned_documents < stateful_workers[best].owned_documents) { + if(!stateful_workers[i].alive) + continue; + if(!stateful_workers[best].alive || + stateful_workers[i].owned_documents < stateful_workers[best].owned_documents) { best = i; } } @@ -233,4 +235,127 @@ void WorkerPool::clear_owner(std::size_t worker_index) { } } +kota::task<> WorkerPool::monitor_worker(std::size_t index, bool stateful) { + auto& workers = stateful ? stateful_workers : stateless_workers; + auto& w = workers[index]; + auto name = std::string(stateful ? "SF-" : "SL-") + std::to_string(index); + + auto result = co_await w.proc.wait(); + w.alive = false; + --alive_count_; + + if(shutting_down_) { + if(alive_count_ == 0) + all_exited_.set(); + co_return; + } + + if(result.has_value()) { + auto& exit = result.value(); + if(exit.term_signal != 0) { + LOG_ERROR("Worker {} killed by signal {} (restarts: {})", + name, + exit.term_signal, + w.restart_count); + } else { + LOG_ERROR("Worker {} exited with code {} (restarts: {})", + name, + exit.status, + w.restart_count); + } + } else { + LOG_ERROR("Worker {} lost: {} (restarts: {})", + name, + result.error().message(), + w.restart_count); + } + + if(stateful) + clear_owner(index); + + constexpr unsigned max_restarts = 5; + if(w.restart_count >= max_restarts) { + LOG_ERROR("Worker {} exceeded max restarts ({}), giving up", name, max_restarts); + co_return; + } + + if(!respawn_worker(index, stateful)) { + LOG_ERROR("Worker {} respawn failed", name); + } +} + +bool WorkerPool::respawn_worker(std::size_t index, bool stateful) { + auto& workers = stateful ? stateful_workers : stateless_workers; + auto old_restart_count = workers[index].restart_count + 1; + auto worker_name = std::string(stateful ? "SF-" : "SL-") + std::to_string(index); + + // Close the old peer and retire it so its coroutines (run/write_loop) + // can finish naturally before the object is destroyed. + if(workers[index].peer) { + workers[index].peer->close(); + retired_peers.push_back(std::move(workers[index].peer)); + } + + kota::process::options opts; + opts.file = options_.self_path; + if(stateful) { + opts.args = {options_.self_path, + "--mode", + "stateful-worker", + "--worker-memory-limit", + std::to_string(options_.worker_memory_limit)}; + } else { + opts.args = {options_.self_path, "--mode", "stateless-worker"}; + } + opts.args.push_back("--worker-name"); + opts.args.push_back(worker_name); + if(!log_dir_.empty()) { + opts.args.push_back("--log-dir"); + opts.args.push_back(log_dir_); + } + opts.streams = { + kota::process::stdio::pipe(true, false), + kota::process::stdio::pipe(false, true), + kota::process::stdio::pipe(false, true), + }; + + auto result = kota::process::spawn(opts, loop); + if(!result) { + LOG_ERROR("Failed to respawn worker {}: {}", worker_name, result.error().message()); + return false; + } + + auto& spawn = *result; + auto transport = std::make_unique(std::move(spawn.stdout_pipe), + std::move(spawn.stdin_pipe)); + auto peer = std::make_unique(loop, std::move(transport)); + + std::string prefix = "[" + worker_name + "]"; + loop.schedule(drain_stderr(std::move(spawn.stderr_pipe), prefix)); + + workers[index] = WorkerProcess{ + .proc = std::move(spawn.proc), + .peer = std::move(peer), + .owned_documents = 0, + .alive = true, + .restart_count = old_restart_count, + }; + + auto& w = workers[index]; + ++alive_count_; + loop.schedule(w.peer->run()); + + if(stateful) { + w.peer->on_notification([this](const worker::EvictedParams& params) { + if(on_evicted) + on_evicted(params.path); + }); + } + + loop.schedule(monitor_worker(index, stateful)); + + LOG_INFO("Worker {} restarted (attempt {})", worker_name, old_restart_count); + return true; +} + } // namespace clice diff --git a/src/server/worker_pool.h b/src/server/worker_pool.h index 9151880b1..0ba58734c 100644 --- a/src/server/worker_pool.h +++ b/src/server/worker_pool.h @@ -64,6 +64,8 @@ class WorkerPool { kota::process proc; std::unique_ptr peer; std::size_t owned_documents = 0; + bool alive = true; + unsigned restart_count = 0; }; kota::event_loop& loop; @@ -80,8 +82,19 @@ class WorkerPool { void clear_owner(std::size_t worker_index); std::size_t pick_least_loaded(); + bool shutting_down_ = false; + std::size_t alive_count_ = 0; + kota::event all_exited_{true}; // Signalled when alive_count_ reaches 0. + WorkerPoolOptions options_; std::string log_dir_; + + /// Peers moved here during respawn so their coroutines can finish + /// before the object is destroyed. + llvm::SmallVector> retired_peers; + bool spawn_worker(const std::string& self_path, bool stateful, std::uint64_t memory_limit); + bool respawn_worker(std::size_t index, bool stateful); + kota::task<> monitor_worker(std::size_t index, bool stateful); }; template @@ -91,11 +104,10 @@ RequestResult WorkerPool::send_stateful(std::uint32_t path_id, if(stateful_workers.empty()) { co_return kota::outcome_error(kota::ipc::Error{"No stateful workers available"}); } - // No timeout: compile tasks run as detached tasks (loop.schedule) that - // are immune to LSP $/cancelRequest. Adding a timeout here would use - // kotatsu's with_token/when_any which has a spurious-cancellation bug - // that kills requests within milliseconds instead of the configured period. auto idx = assign_worker(path_id); + if(!stateful_workers[idx].alive) { + co_return kota::outcome_error(kota::ipc::Error{"Assigned stateful worker is down"}); + } co_return co_await stateful_workers[idx].peer->send_request(params, opts); } @@ -105,9 +117,16 @@ RequestResult WorkerPool::send_stateless(const Params& params, if(stateless_workers.empty()) { co_return kota::outcome_error(kota::ipc::Error{"No stateless workers available"}); } - auto idx = next_stateless; - next_stateless = (next_stateless + 1) % stateless_workers.size(); - co_return co_await stateless_workers[idx].peer->send_request(params, opts); + // Round-robin, skipping dead workers. + auto start = next_stateless; + for(std::size_t i = 0; i < stateless_workers.size(); ++i) { + auto idx = (start + i) % stateless_workers.size(); + if(stateless_workers[idx].alive) { + next_stateless = (idx + 1) % stateless_workers.size(); + co_return co_await stateless_workers[idx].peer->send_request(params, opts); + } + } + co_return kota::outcome_error(kota::ipc::Error{"All stateless workers are down"}); } template @@ -115,6 +134,8 @@ void WorkerPool::notify_stateful(std::uint32_t path_id, const Params& params) { auto it = owner.find(path_id); if(it == owner.end()) return; + if(!stateful_workers[it->second].alive) + return; stateful_workers[it->second].peer->send_notification(params); } diff --git a/tests/conftest.py b/tests/conftest.py index 4935e82fa..ada56698a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -110,7 +110,11 @@ async def client( if workspace is not None: init_options_marker = request.node.get_closest_marker("init_options") - init_options = init_options_marker.args[0] if init_options_marker else None + init_options = dict(init_options_marker.args[0]) if init_options_marker else {} + # Force cache_dir into the workspace so .clice/ cleanup prevents stale PCH. + project = dict(init_options.get("project", {})) + project.setdefault("cache_dir", str(workspace / ".clice")) + init_options["project"] = project await c.initialize(workspace, initialization_options=init_options) yield c @@ -165,12 +169,17 @@ async def _shutdown_client(c: CliceClient) -> None: try: server = getattr(c, "_server", None) - if server and server.stderr: - stderr_data = await asyncio.wait_for(server.stderr.read(), timeout=2.0) - if stderr_data: - for line in stderr_data.decode("utf-8", errors="replace").splitlines(): - if "[warn]" in line or "[error]" in line: - print(f"[server] {line}", flush=True) + if server: + if server.returncode is not None: + print(f"[server] exit code: {server.returncode}", flush=True) + if server.stderr: + stderr_data = await asyncio.wait_for(server.stderr.read(), timeout=2.0) + if stderr_data: + for line in stderr_data.decode( + "utf-8", errors="replace" + ).splitlines(): + if "[warn]" in line or "[error]" in line or "Sanitizer" in line: + print(f"[server] {line}", flush=True) except Exception: pass diff --git a/tests/replay.py b/tests/replay.py index 3738a94c1..91fe5cb2e 100644 --- a/tests/replay.py +++ b/tests/replay.py @@ -13,6 +13,9 @@ import signal import sys import time + +# Force line-buffered stdout so CI sees output immediately. +sys.stdout.reconfigure(line_buffering=True) from pathlib import Path from urllib.parse import quote, unquote @@ -109,7 +112,9 @@ async def write_lsp_message(writer: asyncio.StreamWriter, payload: str): await writer.drain() -async def replay_one(trace_path: Path, clice_bin: Path, timeout: int) -> bool | None: +async def replay_one( + trace_path: Path, clice_bin: Path, timeout: int, wall_timeout: int = 300 +) -> bool | None: """Replay a single trace. Returns True=PASS, False=FAIL, None=SKIP.""" records = load_trace(trace_path) if not records: @@ -179,8 +184,21 @@ async def reader_loop(): last_method = None sent_count = 0 + wall_deadline = wall_start + wall_timeout + + def remaining_wall(): + return max(0, wall_deadline - time.monotonic()) + try: for i, rec in enumerate(records): + if remaining_wall() <= 0: + elapsed = time.monotonic() - wall_start + print( + f" result: TIMEOUT (wall-clock {wall_timeout}s exceeded, {elapsed:.1f}s)" + ) + success = False + break + if i > 0: delay = rec["ts"] - records[i - 1]["ts"] if delay > 0: @@ -196,7 +214,7 @@ async def reader_loop(): try: await asyncio.wait_for( asyncio.gather(*pending.values(), return_exceptions=True), - timeout=timeout, + timeout=min(timeout, remaining_wall()), ) except asyncio.TimeoutError: elapsed = time.monotonic() - wall_start @@ -210,7 +228,19 @@ async def reader_loop(): if msg_id is not None and method is not None: pending[msg_id] = asyncio.get_event_loop().create_future() - await write_lsp_message(proc.stdin, rec["msg"]) + try: + await asyncio.wait_for( + write_lsp_message(proc.stdin, rec["msg"]), + timeout=min(30, remaining_wall()), + ) + except asyncio.TimeoutError: + elapsed = time.monotonic() - wall_start + print( + f" result: HANG (write blocked at {last_method}," + f" sent={sent_count}/{len(records)}, {elapsed:.1f}s)" + ) + success = False + break sent_count = i + 1 except (ConnectionError, BrokenPipeError): @@ -231,7 +261,7 @@ async def reader_loop(): try: await asyncio.wait_for( asyncio.gather(*pending.values(), return_exceptions=True), - timeout=timeout, + timeout=min(timeout, remaining_wall()), ) except asyncio.TimeoutError: elapsed = time.monotonic() - wall_start @@ -294,7 +324,7 @@ async def async_main(args): print(f"SKIP: {trace} (not found)") skipped += 1 continue - result = await replay_one(trace, args.clice, args.timeout) + result = await replay_one(trace, args.clice, args.timeout, args.wall_timeout) if result is None: skipped += 1 elif result: @@ -317,7 +347,16 @@ def main(): p.add_argument("traces", nargs="+", type=Path, help="JSONL trace files") p.add_argument("--clice", required=True, type=Path, help="Path to clice binary") p.add_argument( - "--timeout", type=int, default=120, help="Timeout in seconds (default: 120)" + "--timeout", + type=int, + default=120, + help="Per-request timeout in seconds (default: 120)", + ) + p.add_argument( + "--wall-timeout", + type=int, + default=300, + help="Max wall-clock time per trace in seconds (default: 300)", ) args = p.parse_args() sys.exit(asyncio.run(async_main(args))) diff --git a/tests/unit/server/config_tests.cpp b/tests/unit/server/config_tests.cpp index 1cfb550a0..d9181fa8a 100644 --- a/tests/unit/server/config_tests.cpp +++ b/tests/unit/server/config_tests.cpp @@ -148,7 +148,7 @@ TEST_CASE(ApplyDefaults) { EXPECT_EQ(*config.project.idle_timeout_ms, 3000); EXPECT_EQ(config.project.max_active_file.value, 8); EXPECT_EQ(config.project.stateful_worker_count.value, 2u); - EXPECT_EQ(config.project.stateless_worker_count.value, 3u); + EXPECT_GE(config.project.stateless_worker_count.value, 2u); EXPECT_FALSE(config.project.cache_dir.empty()); EXPECT_FALSE(config.project.index_dir.empty()); EXPECT_FALSE(config.project.logging_dir.empty());