From d2ac198bb3d79a0f3bd769faaf3e6bc6c44adcdf Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 13:40:10 -0400 Subject: [PATCH 01/40] Update yscope-dev-utils --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index e300d1bab..5c6bfbd00 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit e300d1bab4c2f33cbf9ddf9f0c08185faf035070 +Subproject commit 5c6bfbd00ad4e9cbfdfab9708298fb11bda4ae8c From cc8a0977836682e8ed9aa3e39aa3268ffbb2447c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 13:44:03 -0400 Subject: [PATCH 02/40] Use boost install task --- dep-tasks.yaml | 160 +------------------------------------------------ 1 file changed, 1 insertion(+), 159 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index a451ddc12..324a7124e 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -147,7 +147,7 @@ tasks: internal: true run: "once" cmds: - - task: "boost-download-and-install" + - task: ":utils:boost:download-and-install" vars: WORK_DIR: "{{.G_DEPS_DIR}}/boost" FILE_SHA256: "2128a4c96862b5c0970c1e34d76b1d57e4a1016b80df85ad39667f30b1deba26" @@ -161,161 +161,3 @@ tasks: - "program_options" - "regex" - "system" - - # Runs the bootstrap.sh generate step in the given source directory. Boost only supports - # in-source generation and building. - # - # @param {string} SOURCE_DIR Project source directory. - # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. - # @param {string[]} TARGETS Target libraries to build. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the generate command. - boost-generate: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./bootstrap.sh - --prefix="{{.INSTALL_PREFIX}}" - --exec-prefix="{{.INSTALL_PREFIX}}" - --with-libraries={{(join "," .TARGETS)}} - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - - # Runs the b2 build step for boost. The caller must have previously called `generate` on - # `SOURCE_DIR` for this task to succeed. - # - # @param {string} SOURCE_DIR Directory containing the boost source. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the build command. - # @param {int} [JOBS] The maximum number of concurrent processes to use when building. If - # omitted, the b2 default number is used. Before 1.76.0, the number was 1. Since 1.76.0, the - # default is the number of cores. - boost-build: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./b2 - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - {{- if .JOBS}} - "-j{{.JOBS}}" - {{- end}} - - # Runs the b2 install step for boost. The caller must have previously called `build` on - # `SOURCE_DIR` for this task to succeed. If `CMAKE_SETTINGS_DIR` is set, a settings file will be - # created in that directory, containing a `boost_ROOT` CMake variable that points to - # `INSTALL_PREFIX`. - # - # @param {string} SOURCE_DIR Directory containing the boost source. - # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. - # @param {string} [CMAKE_SETTINGS_DIR] If set, the directory where the project's CMake settings - # file should be stored. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the install command. - boost-install: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./b2 - install - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - - >- - {{- if .CMAKE_SETTINGS_DIR}} - echo "set(BOOST_ROOT - \"{{.INSTALL_PREFIX}}\" - CACHE PATH - \"Package root for boost.\" - )" >> "{{.CMAKE_SETTINGS_DIR}}/boost.cmake" - {{- end}} - - # Downloads boost from `URL` and installs boost. - # - # General parameters - # @param {string} [WORK_DIR={{.TASK_DIR}}] Base directory to store the install and src - # directories inside. - # @param {string} [SOURCE_DIR={{.WORK_DIR}}/boost-src] Directory in which to extract the tar - # file. - # - # Download parameters - # @param {string} FILE_SHA256 Content hash to verify the downloaded tar file against. - # @param {string} URL - # - # Boost generate parameters - # @param {string} [INSTALL_PREFIX={{.WORK_DIR}}/boost-install] Path prefix of where the project - # should be installed. - # @param {string[]} TARGETS Target libraries to build. - # @param {string[]} [GEN_ARGS] Any additional arguments to pass to the generate command. - # - # Boost build parameters - # @param {int} [JOBS] The maximum number of concurrent processes to use when building. If - # omitted, the b2 default number is used. Before 1.76.0, the number was 1. Since 1.76.0, the - # default is the number of cores. - # @param {string[]} [BUILD_ARGS] Any additional arguments to pass to the build command. - # - # Boost install parameters - # @param {string[]} [INSTALL_ARGS] Any additional arguments to pass to the install command. - # @param {string} [CMAKE_SETTINGS_DIR] If set, the directory where the project's CMake settings - # file should be stored. - boost-download-and-install: - internal: true - label: "{{.TASK}}:{{.URL}}-{{.INSTALL_PREFIX}}" - vars: - # General parameters - WORK_DIR: >- - {{default .ROOT_DIR .WORK_DIR}} - SOURCE_DIR: >- - {{default (printf "%s/boost-src" .WORK_DIR) .SOURCE_DIR}} - - # Boost generate parameters - INSTALL_PREFIX: >- - {{default (printf "%s/boost-install" .WORK_DIR) .INSTALL_PREFIX}} - TARGETS: - ref: "default (list) .TARGETS" - GEN_ARGS: - ref: "default (list) .GEN_ARGS" - - # Boost build parameters - BUILD_ARGS: - ref: "default (list) .BUILD_ARGS" - JOBS: >- - {{default "" .JOBS}} - - # Boost install parameters - INSTALL_ARGS: - ref: "default (list) .INSTALL_ARGS" - CMAKE_SETTINGS_DIR: >- - {{default "" .CMAKE_SETTINGS_DIR}} - requires: - vars: ["FILE_SHA256", "URL"] - deps: - - task: ":utils:remote:download-and-extract-tar" - vars: - FILE_SHA256: "{{.FILE_SHA256}}" - OUTPUT_DIR: "{{.SOURCE_DIR}}" - URL: "{{.URL}}" - cmds: - - task: "boost-generate" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - TARGETS: - ref: ".TARGETS" - EXTRA_ARGS: - ref: ".GEN_ARGS" - - task: "boost-build" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - JOBS: "{{.JOBS}}" - EXTRA_ARGS: - ref: ".BUILD_ARGS" - - task: "boost-install" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" - EXTRA_ARGS: - ref: ".INSTALL_ARGS" From 7f68bdc0d8ec884eaa14096282b680a531d482a2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:00:23 -0400 Subject: [PATCH 03/40] Update install task variable names --- dep-tasks.yaml | 52 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 324a7124e..3c7c05f8a 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -30,12 +30,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "absl" + CMAKE_PACKAGE_NAME: "absl" WORK_DIR: "{{.G_DEPS_DIR}}/absl" - FILE_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" - URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" + TAR_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" + TAR_URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" @@ -45,12 +45,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "Catch2" + CMAKE_PACKAGE_NAME: "Catch2" WORK_DIR: "{{.G_DEPS_DIR}}/Catch2" - FILE_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" - URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" + TAR_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" + TAR_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" @@ -76,12 +76,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "fmt" + CMAKE_PACKAGE_NAME: "fmt" WORK_DIR: "{{.G_DEPS_DIR}}/fmtlib" - FILE_SHA256: "6cb1e6d37bdcb756dbbe59be438790db409cdb4868c66e888d5df9f13f7c027f" - URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.0.2.tar.gz" + TAR_SHA256: "6cb1e6d37bdcb756dbbe59be438790db409cdb4868c66e888d5df9f13f7c027f" + TAR_URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.0.2.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" @@ -95,16 +95,16 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "spdlog" + CMAKE_PACKAGE_NAME: "spdlog" WORK_DIR: "{{.G_DEPS_DIR}}/spdlog" - FILE_SHA256: "9962648c9b4f1a7bbc76fd8d9172555bad1871fdb14ff4f842ef87949682caa5" - URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.0.tar.gz" + TAR_SHA256: "9962648c9b4f1a7bbc76fd8d9172555bad1871fdb14ff4f842ef87949682caa5" + TAR_URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DSPDLOG_BUILD_TESTS=OFF" - "-DUSE_EXTERNAL_FMT=ON" - - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" install-mariadb-connector-cpp: @@ -113,13 +113,13 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "mariadb-connector-cpp" + CMAKE_PACKAGE_NAME: "mariadb-connector-cpp" WORK_DIR: "{{.G_DEPS_DIR}}/mariadb-connector-cpp" - FILE_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" - URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/tags/\ + TAR_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" + TAR_URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/tags/\ 1.1.5.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" @@ -132,16 +132,16 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "msgpack-cxx" + CMAKE_PACKAGE_NAME: "msgpack-cxx" WORK_DIR: "{{.G_DEPS_DIR}}/msgpack" - FILE_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" - URL: "https://github.com/msgpack/msgpack-c/releases/download/\ + TAR_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" + TAR_URL: "https://github.com/msgpack/msgpack-c/releases/download/\ cpp-7.0.0/msgpack-cxx-7.0.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" install-boost: internal: true From 2394649ab8c6454733ff34d8736fb1b8960d2022 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:11:02 -0400 Subject: [PATCH 04/40] Set CMP0074 to NEW to find boost --- dep-tasks.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 3c7c05f8a..d58a61f0d 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -105,6 +105,7 @@ tasks: - "-DSPDLOG_BUILD_TESTS=OFF" - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" install-mariadb-connector-cpp: @@ -142,6 +143,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" install-boost: internal: true From 6ff3e33f9d5b36e3b54be01fbf6b263b4c9ca6a9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:40:52 -0400 Subject: [PATCH 05/40] Add uv to install script --- tools/scripts/lib_install/linux/install-dev.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/scripts/lib_install/linux/install-dev.sh b/tools/scripts/lib_install/linux/install-dev.sh index 73fd319bc..f88c9cc93 100755 --- a/tools/scripts/lib_install/linux/install-dev.sh +++ b/tools/scripts/lib_install/linux/install-dev.sh @@ -36,3 +36,6 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir="$script_dir/.." # TODO https://github.com/y-scope/spider/issues/86 "$lib_install_scripts_dir"/check-cmake-version.sh + +# Install uv +curl -LsSf https://astral.sh/uv/install.sh | sh From 7ae8a782aaf63043f4f1c4b31a47f3dc57bfb346 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 15:04:40 -0400 Subject: [PATCH 06/40] Fix cpp-lint root paths --- lint-tasks.yaml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index d81b7247d..85417f30c 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -45,7 +45,7 @@ tasks: - task: "cpp-static-fix" cpp-format-check: - sources: &cpp_source_files + sources: &cpp_lint_source_files - "{{.ROOT_DIR}}/.clang-format" - "{{.ROOT_DIR}}/.clang-tidy" - "{{.TASKFILE}}" @@ -59,11 +59,14 @@ tasks: vars: FLAGS: ["--dry-run"] INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: &cpp_source_files + - "{{.G_SRC_SPIDER_DIR}}" + - "{{.G_EXAMPLES_DIR}}" + - "{{.G_TEST_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" cpp-format-fix: - sources: *cpp_source_files + sources: *cpp_lint_source_files deps: ["cpp-configs", "venv"] cmds: - task: ":utils:cpp-lint:clang-format" @@ -152,18 +155,6 @@ tasks: taskfile.yaml \ test-tasks.yaml - clang-tidy: - internal: true - requires: - vars: ["FLAGS", "SRC_DIR"] - cmd: |- - . "{{.G_LINT_VENV_DIR}}/bin/activate" - find "{{.SRC_DIR}}" \ - -type f \ - \( -iname "*.cpp" -o -iname "*.h" -o -iname "*.hpp" \) \ - -print0 | \ - xargs -0 --no-run-if-empty clang-tidy {{.FLAGS}} - cmake: internal: true requires: From 56502f90d6df56d5e0d97ed0f872b7682743455b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 15:16:33 -0400 Subject: [PATCH 07/40] Fix clang-tidy file pattern --- lint-tasks.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 85417f30c..659480e50 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -82,7 +82,7 @@ tasks: # When we eventually determine which errors can be safely fixed, we'll allow clang-tidy to # fix them. aliases: ["cpp-static-fix"] - sources: *cpp_source_files + sources: *cpp_lint_source_files deps: [":config-cmake-project", "cpp-configs", "venv"] cmds: - task: ":utils:cpp-lint:clang-tidy-find" @@ -90,21 +90,21 @@ tasks: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_SPIDER_COMPILE_COMMANDS_DB}}'" - INCLUDE_PATTERNS: - - "{{.G_SRC_SPIDER_DIR}}/**" - - "{{.G_TEST_DIR}}/**" + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: + - "{{.G_SRC_SPIDER_DIR}}" + - "{{.G_TEST_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" - task: ":utils:cpp-lint:clang-tidy-find" vars: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_EXAMPLES_COMPILE_COMMANDS_DB}}'" - INCLUDE_PATTERNS: - - "{{.G_EXAMPLES_DIR}}/**" + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: + - "{{.G_EXAMPLES_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" py-check: From 6f6becfff0e90960d46fe07ce442012de70150b1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:19:38 -0400 Subject: [PATCH 08/40] Limit build parallelism --- .github/workflows/code-linting-checks.yaml | 2 ++ .github/workflows/unit-tests.yaml | 2 ++ dep-tasks.yaml | 6 ++++++ taskfile.yaml | 2 ++ 4 files changed, 12 insertions(+) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index c6b1e9117..4ee4e58e4 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -47,6 +47,8 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 + env: + SPIDER_DEP_BUILD_PARALLELISM: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 4b42a93cc..0e6089d37 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -50,6 +50,8 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 + env: + SPIDER_DEP_BUILD_PARALLELISM: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index a451ddc12..9652125e6 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,6 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-Catch2: internal: true @@ -53,6 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} download-ystdlib: internal: true @@ -85,6 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-spdlog: internal: true @@ -106,6 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-mariadb-connector-cpp: internal: true @@ -123,6 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-msgpack: internal: true @@ -142,6 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index 556ef3486..ecece7a09 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + G_DEP_BUILD_PARALLELISM: >- + {{default "" (env SPIDER_DEP_BUILD_PARALLELISM)}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From a74523daa93aef57b0d279863583f7b3b4a7240b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:34:32 -0400 Subject: [PATCH 09/40] Bug fix --- dep-tasks.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 9652125e6..9262da104 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-boost: internal: true From 77ee49460e2dfb7752f4416b0b595a16ced5fc16 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:37:22 -0400 Subject: [PATCH 10/40] Bug fix --- taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index ecece7a09..4a110c06c 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -21,7 +21,7 @@ vars: G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" G_DEP_BUILD_PARALLELISM: >- - {{default "" (env SPIDER_DEP_BUILD_PARALLELISM)}} + {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 3698339529129c883a0d1d927a873e2b879b4dce Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:47:19 -0400 Subject: [PATCH 11/40] Bug fix --- dep-tasks.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 4f45bd44c..fe90b3544 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -169,4 +169,3 @@ tasks: - "program_options" - "regex" - "system" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" From 369e9f1bbf876d7ef1b1ed209f953681d5f83211 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 22:57:18 -0400 Subject: [PATCH 12/40] Rename variables to mirror CLP core Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- taskfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 4a110c06c..d44c1a464 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,8 +20,11 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + # Build parameters + # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEP_BUILD_PARALLELISM: >- {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} + G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 90aa5a285b7068d245c52663dc82cbac3063ee67 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 22:57:45 -0400 Subject: [PATCH 13/40] Rename variables to mirror clp core --- .github/workflows/code-linting-checks.yaml | 2 +- .github/workflows/unit-tests.yaml | 2 +- dep-tasks.yaml | 12 ++++++------ taskfile.yaml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 4ee4e58e4..48f8eacd7 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -48,7 +48,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_BUILD_PARALLELISM: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 0e6089d37..1ced6a889 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -51,7 +51,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_BUILD_PARALLELISM: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 9262da104..cab98d77c 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index 4a110c06c..20da0f3b7 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,8 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" - G_DEP_BUILD_PARALLELISM: >- - {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} + G_DEPS_MAX_PARALLELISM_PER_TASK: >- + {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 7faac8fc6a1e2b196abb0853b23c8ea3153cdeb9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:07:53 -0400 Subject: [PATCH 14/40] Revert "Merge branch 'dep-concurrency' of github.com:sitaowang1998/spider into dep-concurrency" This reverts commit 1769c95feb3868e4a462ccaa694b97f880947f12, reversing changes made to 90aa5a285b7068d245c52663dc82cbac3063ee67. --- .github/workflows/code-linting-checks.yaml | 2 +- .github/workflows/unit-tests.yaml | 2 +- dep-tasks.yaml | 12 ++++++------ taskfile.yaml | 7 ++----- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 1178f0598..48f8eacd7 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -48,7 +48,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_MAX_PARALLELISM_PER_TASK: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 675e7c6ec..1ced6a889 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -51,7 +51,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_MAX_PARALLELISM_PER_TASK: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index ea282d430..cab98d77c 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index d44c1a464..20da0f3b7 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,11 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" - # Build parameters - # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. - G_DEP_BUILD_PARALLELISM: >- - {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} - + G_DEPS_MAX_PARALLELISM_PER_TASK: >- + {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From f494a90b85e24d4e9ee07875e01e7cdb65e5387c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:10:25 -0400 Subject: [PATCH 15/40] Add comment for deps parallelism default value --- taskfile.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 20da0f3b7..470edd582 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + # Build parameters + # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEPS_MAX_PARALLELISM_PER_TASK: >- {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" From ff2fe1c69a7ff4d952d546babba27413dcb0370f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:28:39 -0400 Subject: [PATCH 16/40] Update yscope-dev-utils --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index 5c6bfbd00..b965f19f3 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit 5c6bfbd00ad4e9cbfdfab9708298fb11bda4ae8c +Subproject commit b965f19f3e11c548f32bf7ab8e28ff38906be018 From 65841a019afaaf7aaaf56ef5b516743a482f9861 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 10:31:24 -0400 Subject: [PATCH 17/40] Add latest python lint config files --- mypy.ini | 8 ++++++++ ruff.toml | 29 +++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..4c66b1114 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true diff --git a/ruff.toml b/ruff.toml index 969421ff8..cb6ee887f 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,6 +1,27 @@ line-length = 100 -lint.select = ["I"] -[lint.isort] -case-sensitive = false -order-by-type = false \ No newline at end of file +[lint] +select = ["ALL"] +ignore = [ + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful +] +isort.order-by-type = false + +[format] +docstring-code-format = true +docstring-code-line-length = 100 From 8564fc259a54fdbe08ec27b39f33dc0ca3e4f593 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 11:03:48 -0400 Subject: [PATCH 18/40] Update ruff lint tasks --- lint-requirements.txt | 1 - lint-tasks.yaml | 16 +++++++--------- mypy.ini | 8 -------- 3 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 mypy.ini diff --git a/lint-requirements.txt b/lint-requirements.txt index 717adb508..690b9797f 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,4 +1,3 @@ -black>=24.4.2 clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 659480e50..04f8cef3a 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -111,29 +111,27 @@ tasks: cmds: - task: "py" vars: - BLACK_FLAGS: "--check" - RUFF_FLAGS: "" + RUFF_FORMAT_FLAGS: "--diff" py-fix: cmds: - task: "py" vars: - BLACK_FLAGS: "" - RUFF_FLAGS: "--fix" + RUFF_CHECK_FLAGS: "--fix" py: internal: true - requires: - vars: ["BLACK_FLAGS", "RUFF_FLAGS"] + vars: + RUFF_CHECK_FLAGS: "{{.RUFF_CHECK_FLAGS | default \"\"}}" + RUFF_FORMAT_FLAGS: "{{.RUFF_FORMAT_FLAGS | default \"\"}}" deps: ["venv"] cmds: - for: - "tests/integration" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" - cd "{{.ITEM}}" - black --color --line-length 100 {{.BLACK_FLAGS}} . - ruff check {{.RUFF_FLAGS}} . + ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" + ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" yml: aliases: diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 4c66b1114..000000000 --- a/mypy.ini +++ /dev/null @@ -1,8 +0,0 @@ -[mypy] -strict = true - -# Additional output -pretty = true -show_error_code_links = true -show_error_context = true -show_error_end = true From a6e7d29a435f66eeac173a1a28415c7d1ab3c820 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:13:55 -0400 Subject: [PATCH 19/40] Fix ruff lint --- ruff.toml | 8 ++ tests/integration/__init__.py | 1 + tests/integration/client.py | 118 +++++++++++++++++---- tests/integration/test_client.py | 27 ++++- tests/integration/test_scheduler_worker.py | 85 +++++++++++++-- tests/integration/test_signal.py | 52 ++++++--- 6 files changed, 244 insertions(+), 47 deletions(-) diff --git a/ruff.toml b/ruff.toml index cb6ee887f..a7a590d31 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,6 +2,7 @@ line-length = 100 [lint] select = ["ALL"] +extend-select = ["PT"] ignore = [ "COM812", # Redundant and conflicts with ruff format "D203", # No blank line before docstrings (D211) @@ -22,6 +23,13 @@ ignore = [ ] isort.order-by-type = false +[lint.per-file-ignores] +"tests/integration/test_*.py" = [ + "S101", # Allow use of `assert` (security warning) + "ANN001", # Ignore missing type annotations for arguments + "ANN201", # Ignore missing type annotations for parameters +] + [format] docstring-code-format = true docstring-code-line-length = 100 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index e69de29bb..4d2ed7264 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests for Spider C++ projects.""" diff --git a/tests/integration/client.py b/tests/integration/client.py index 1b658c5ac..834ae2b5e 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,7 +1,7 @@ import re import uuid from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from collections.abc import Generator import mysql.connector import pytest @@ -9,48 +9,73 @@ @dataclass class TaskInput: + """ + TaskInput represents an input to a task. + It can either be a direct value, a reference to another task's output, or a reference to data. + """ type: str - task_output: Optional[Tuple[uuid.UUID, int]] = None - value: Optional[str] = None - data_id: Optional[uuid.UUID] = None + task_output: tuple[uuid.UUID, int] | None = None + value: str | None = None + data_id: uuid.UUID | None = None @dataclass class TaskOutput: + """ + TaskOutput represents an output of a task. + It can either be a direct value or a reference to data. + """ type: str - value: Optional[str] = None - data_id: Optional[uuid.UUID] = None + value: str | None = None + data_id: uuid.UUID | None = None @dataclass class Task: + """ + Task represents a unit of work in the task graph. + """ id: uuid.UUID function_name: str - inputs: List[TaskInput] - outputs: List[TaskOutput] + inputs: list[TaskInput] + outputs: list[TaskOutput] timeout: float = 0.0 max_retries: int = 0 @dataclass class TaskGraph: + """ + TaskGraph represents a directed acyclic graph of tasks. + """ id: uuid.UUID - tasks: Dict[uuid.UUID, Task] - dependencies: List[Tuple[uuid.UUID, uuid.UUID]] + tasks: dict[uuid.UUID, Task] + dependencies: list[tuple[uuid.UUID, uuid.UUID]] @dataclass class Driver: + """ + Driver represents a client that can submit jobs to the task graph. + """ id: uuid.UUID @dataclass class Data: + """ + Data represents a Spider Data object. + """ id: uuid.UUID value: str -def create_connection(storage_url: str): +def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: + """ + Creation a MariaDB connection from a JDBC URL. + :param storage_url: JDBC URL for the MariaDB database. + :return: The created MySQL connection. + """ pattern = re.compile( r"jdbc:mariadb://(?P[^:/]+):(?P\d+)/(?P[^?]+)\?user=(?P[^&]+)&password=(?P[^&]+)" ) @@ -68,7 +93,13 @@ def create_connection(storage_url: str): ) -def is_head_task(task_id: uuid.UUID, dependencies: List[Tuple[uuid.UUID, uuid.UUID]]): +def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UUID]]) -> bool: + """ + Check if the task is a head task, meaning it has no parent. + :param task_id: the ID of the task to check. + :param dependencies: list of dependencies where each dependency is a tuple (parent_id, child_id). + :return: True if the task has no parent, False otherwise. + """ return not any(dependency[1] == task_id for dependency in dependencies) @@ -76,13 +107,25 @@ def is_head_task(task_id: uuid.UUID, dependencies: List[Tuple[uuid.UUID, uuid.UU @pytest.fixture(scope="session") -def storage(): +def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: + """ + Fixture to create a database connection for the test session. Yields a connection object + and ensures it is closed after the tests are done. + :return: + """ conn = create_connection(g_storage_url) yield conn conn.close() -def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph): +def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: + """ + Submit a job to the database. + :param conn: database connection object. + :param client_id: client ID of the driver submitting the job. + :param graph: task graph to be submitted. + :return: None + """ cursor = conn.cursor() cursor.execute( @@ -136,7 +179,13 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph): cursor.close() -def get_task_outputs(conn, task_id: uuid.UUID) -> List[TaskOutput]: +def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: + """ + Get the outputs of a task by its ID. + :param conn: database connection object. + :param task_id: the ID of the task whose outputs are to be retrieved. + :return: list of TaskOutput objects representing the outputs of the task. + """ cursor = conn.cursor() cursor.execute( @@ -158,6 +207,12 @@ def get_task_outputs(conn, task_id: uuid.UUID) -> List[TaskOutput]: def get_task_state(conn, task_id: uuid.UUID) -> str: + """ + Get the state of a task by its ID. + :param conn: database connection object. + :param task_id: the ID of the task whose state is to be retrieved. + :return: the state of the task as a string. + """ cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) @@ -168,7 +223,13 @@ def get_task_state(conn, task_id: uuid.UUID) -> str: return state -def remove_job(conn, job_id: uuid.UUID): +def remove_job(conn, job_id: uuid.UUID) -> None: + """ + Remove a job from the database by its ID. + :param conn: database connection object. + :param job_id: the ID of the job to be removed. + :return: None + """ cursor = conn.cursor() cursor.execute("DELETE FROM jobs WHERE id = %s", (job_id.bytes,)) @@ -176,7 +237,13 @@ def remove_job(conn, job_id: uuid.UUID): cursor.close() -def add_driver(conn, driver: Driver): +def add_driver(conn, driver: Driver) -> None: + """ + Register a new driver in the database. + :param conn: database connection object. + :param driver: driver object to be registered. + :return: None + """ cursor = conn.cursor() cursor.execute("INSERT INTO drivers (id) VALUES (%s)", (driver.id.bytes,)) @@ -185,7 +252,14 @@ def add_driver(conn, driver: Driver): cursor.close() -def add_driver_data(conn, driver: Driver, data: Data): +def add_driver_data(conn, driver: Driver, data: Data) -> None: + """ + Add a new data associated with a driver in the database. + :param conn: database connection object. + :param driver: driver object to which the data is associated. + :param data: data object to be added. + :return: None + """ cursor = conn.cursor() cursor.execute("INSERT INTO data (id, value) VALUES (%s, %s)", (data.id.bytes, data.value)) @@ -198,7 +272,13 @@ def add_driver_data(conn, driver: Driver, data: Data): cursor.close() -def remove_data(conn, data: Data): +def remove_data(conn, data: Data) -> None: + """ + Remove data from the database by its ID. + :param conn: database connection object. + :param data: data object to be removed. + :return: None + """ cursor = conn.cursor() cursor.execute("DELETE FROM data WHERE id = %s", (data.id.bytes,)) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 94b48afdf..7d6edeebe 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,7 +1,7 @@ import subprocess import time from pathlib import Path -from typing import Tuple +from collections.abc import Generator import pytest @@ -14,7 +14,13 @@ def start_scheduler_workers( storage_url: str, scheduler_port: int -) -> Tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: + """ + Start the scheduler and two worker processes. + :param storage_url: + :param scheduler_port: The port for the scheduler to listen on. + :return: scheduler_process, worker_process_0, worker_process_1 + """ # Start the scheduler dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." / ".." / "src" / "spider" @@ -43,7 +49,13 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -def scheduler_worker(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker() -> Generator[None, None, None]: + """ + Fixture to start the scheduler and two worker processes. Yields control to the test class, + and then kills the processes after the test class is done. + :return: + """ scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -56,7 +68,12 @@ def scheduler_worker(storage): class TestClient: - def test_client(self, scheduler_worker): + @pytest.mark.usefixtures("scheduler_worker") + def test_client(self) -> None: + """ + Test the client_test C++ program and check for successful execution. + :return: None + """ dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." client_cmds = [ @@ -64,5 +81,5 @@ def test_client(self, scheduler_worker): "--storage_url", g_storage_url, ] - p = subprocess.run(client_cmds, timeout=20) + p = subprocess.run(client_cmds, check=False, timeout=20) assert p.returncode == 0 diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 9dfc359d0..f138f9f81 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -2,11 +2,12 @@ import time import uuid from pathlib import Path -from typing import Tuple +from collections.abc import Generator import msgpack import pytest +from tests.integration.client import TaskGraph, Task from .client import ( add_driver, add_driver_data, @@ -29,7 +30,13 @@ def start_scheduler_worker( storage_url: str, scheduler_port: int -) -> Tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen, subprocess.Popen]: + """ + Start a scheduler and a worker process. + :param storage_url: JDBC storage URL + :param scheduler_port: the port for the scheduler to listen on + :return: scheduler_process, worker_process + """ # Start the scheduler dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." / ".." / "src" / "spider" @@ -57,7 +64,13 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -def scheduler_worker(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker() -> Generator[None, None, None]: + """ + Fixture to start a scheduler and a worker process. Yields control to the test function. + After the test function completes, it kills the scheduler and the worker process. + :return: + """ scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -69,7 +82,13 @@ def scheduler_worker(storage): @pytest.fixture(scope="function") -def success_job(storage): +def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: + """ + Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. + Cleans up the job after the test function completes. + :param storage: + :return: + """ parent_1 = Task( id=uuid.uuid4(), function_name="sum_test", @@ -127,7 +146,13 @@ def success_job(storage): @pytest.fixture(scope="function") -def fail_job(storage): +def fail_job(storage) -> Generator[Task, None, None]: + """ + Fixture to create a job that will fail. The task will raise an error when executed. + Yield the task. Cleanup the job after the test function completes. + :param storage: + :return: + """ task = Task( id=uuid.uuid4(), function_name="error_test", @@ -149,7 +174,13 @@ def fail_job(storage): @pytest.fixture(scope="function") -def data_job(storage): +def data_job(storage) -> Generator[Task, None, None]: + """ + Fixture to create a job that uses data. Yields the task that uses data. + Cleans up the job and data after the test function completes. + :param storage: + :return: + """ data = Data( id=uuid.uuid4(), value=msgpack.packb(2), @@ -181,6 +212,12 @@ def data_job(storage): @pytest.fixture(scope="function") def random_fail_job(storage): + """ + Fixture to create a job that randomly fails. The task will succeed after a few retries. + Yields the task. Cleans up the job after the test function completes. + :param storage: + :return: + """ data = Data( id=uuid.uuid4(), value=msgpack.packb(2), @@ -212,7 +249,14 @@ def random_fail_job(storage): class TestSchedulerWorker: - def test_job_success(self, scheduler_worker, storage, success_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_job_success(self, storage, success_job): + """ + Test the successful execution of a job with two parent tasks and one child task. + :param storage: + :param success_job: + :return: None + """ graph, parent_1, parent_2, child = success_job # Wait for 2 seconds and check task state and output time.sleep(2) @@ -232,14 +276,28 @@ def test_job_success(self, scheduler_worker, storage, success_job): assert len(outputs) == 1 assert outputs[0].value == msgpack.packb(10) - def test_job_failure(self, scheduler_worker, storage, fail_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_job_failure(self, storage, fail_job): + """ + Test the failure of a job that raise an error. + :param storage: + :param fail_job: + :return: None + """ task = fail_job # Wait for 2 seconds and check task output time.sleep(2) state = get_task_state(storage, task.id) assert state == "fail" - def test_data_job(self, scheduler_worker, storage, data_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_data_job(self, storage, data_job): + """ + Test the successful execution of a job that uses data. + :param storage: + :param data_job: + :return: None + """ task = data_job # Wait for 2 seconds and check task output time.sleep(2) @@ -249,7 +307,14 @@ def test_data_job(self, scheduler_worker, storage, data_job): assert len(outputs) == 1 assert outputs[0].value == msgpack.packb(2) - def test_random_fail_job(self, scheduler_worker, storage, random_fail_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_random_fail_job(self, storage, random_fail_job): + """ + Test the successful recovery and execution of a job that randomly fails. + :param storage: + :param random_fail_job: + :return: None + """ task = random_fail_job # Wait for 2 seconds and check task output time.sleep(2) diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index a32f1c3d8..012fac7b9 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -4,6 +4,7 @@ import time import uuid from pathlib import Path +from collections.abc import Generator import msgpack import pytest @@ -23,7 +24,14 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str): +def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> tuple[subprocess.Popen, subprocess.Popen]: + """ + Create a scheduler and a worker process. + :param storage_url: JDB storage URL. + :param scheduler_port: the port for the scheduler to listen on. + :param lib: Library to load in the worker. + :return: scheduler and worker processes. + """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -52,7 +60,12 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str): @pytest.fixture(scope="function") -def scheduler_worker_signal(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen], None, None]: + """ + Fixture to start a scheduler and a worker process for testing signal handling. + :return: + """ scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -65,12 +78,19 @@ def scheduler_worker_signal(storage): class TestWorkerSignal: - # Test that worker propagates the SIGTERM signal to the task executor. - # Submit a task that checks whether the task executor receives the SIGTERM signal. - # The task should return the SIGTERM signal number as the output. - # Later task should not be executed. - # Worker should exit with SIGTERM. - def test_task_signal(self, storage, scheduler_worker_signal): + + def test_task_signal(self, storage, scheduler_worker_signal) -> None: + """ + Test that worker propagates the SIGTERM signal to the task executor. + Submit a task that checks whether the task executor receives the SIGTERM signal. + The task should return the SIGTERM signal number as the output. + Later task should not be executed. + Worker should exit with SIGTERM. + + :param storage: + :param scheduler_worker_signal: + :return: None + """ _, worker_process = scheduler_worker_signal # Submit signal handler task to check for SIGTERM signal in task executor @@ -133,11 +153,17 @@ def test_task_signal(self, storage, scheduler_worker_signal): remove_job(storage, new_graph.id) remove_job(storage, graph.id) - # Test that worker propagates the SIGTERM signal to the task executor. - # Task executor exits immediately after receiving the signal. - # The running task should be marked as failed. - # The worker should exit with SIGTERM. - def test_task_exit(self, storage, scheduler_worker_signal): + def test_task_exit(self, storage, scheduler_worker_signal) -> None: + """ + Test that worker propagates the SIGTERM signal to the task executor. + Task executor exits immediately after receiving the signal. + The running task should be marked as failed. + The worker should exit with SIGTERM. + + :param storage: + :param scheduler_worker_signal: + :return: None + """ _, worker_process = scheduler_worker_signal # Submit a task to sleep for 10 seconds From 573b4487fc933e8cc852e90af321bcf338517a77 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:18:04 -0400 Subject: [PATCH 20/40] Fix ruff lint --- tests/integration/client.py | 8 +++++++- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 3 +-- tests/integration/test_signal.py | 6 ++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 834ae2b5e..99662b542 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,7 +1,7 @@ import re import uuid -from dataclasses import dataclass from collections.abc import Generator +from dataclasses import dataclass import mysql.connector import pytest @@ -13,6 +13,7 @@ class TaskInput: TaskInput represents an input to a task. It can either be a direct value, a reference to another task's output, or a reference to data. """ + type: str task_output: tuple[uuid.UUID, int] | None = None value: str | None = None @@ -25,6 +26,7 @@ class TaskOutput: TaskOutput represents an output of a task. It can either be a direct value or a reference to data. """ + type: str value: str | None = None data_id: uuid.UUID | None = None @@ -35,6 +37,7 @@ class Task: """ Task represents a unit of work in the task graph. """ + id: uuid.UUID function_name: str inputs: list[TaskInput] @@ -48,6 +51,7 @@ class TaskGraph: """ TaskGraph represents a directed acyclic graph of tasks. """ + id: uuid.UUID tasks: dict[uuid.UUID, Task] dependencies: list[tuple[uuid.UUID, uuid.UUID]] @@ -58,6 +62,7 @@ class Driver: """ Driver represents a client that can submit jobs to the task graph. """ + id: uuid.UUID @@ -66,6 +71,7 @@ class Data: """ Data represents a Spider Data object. """ + id: uuid.UUID value: str diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 7d6edeebe..c1d155043 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,7 +1,7 @@ import subprocess import time -from pathlib import Path from collections.abc import Generator +from pathlib import Path import pytest diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index f138f9f81..9c6ac27e4 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -1,13 +1,12 @@ import subprocess import time import uuid -from pathlib import Path from collections.abc import Generator +from pathlib import Path import msgpack import pytest -from tests.integration.client import TaskGraph, Task from .client import ( add_driver, add_driver_data, diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 012fac7b9..764bf8185 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -3,8 +3,8 @@ import subprocess import time import uuid -from pathlib import Path from collections.abc import Generator +from pathlib import Path import msgpack import pytest @@ -32,6 +32,7 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t :param lib: Library to load in the worker. :return: scheduler and worker processes. """ + root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -61,11 +62,12 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t @pytest.fixture(scope="function") @pytest.mark.usefixtures("storage") -def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: """ + scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) From 6ad72c3b93e9415b6fddd291956435fda79db005 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:34:04 -0400 Subject: [PATCH 21/40] Bug fix --- ruff.toml | 1 + tests/integration/client.py | 43 +++++++++------------- tests/integration/test_client.py | 4 ++ tests/integration/test_scheduler_worker.py | 3 ++ tests/integration/test_signal.py | 5 ++- tests/integration/utils.py | 2 + 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/ruff.toml b/ruff.toml index a7a590d31..75462c9cd 100644 --- a/ruff.toml +++ b/ruff.toml @@ -11,6 +11,7 @@ ignore = [ "D400", # First line of docstrings may not end in period "D401", # Docstrings should be written in present tense (not imperative) "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations "FBT", # Allow bool positional parameters since other value positions are allowed "FIX002", # Allow todo statements "PERF401", # Allow for loops when creating lists diff --git a/tests/integration/client.py b/tests/integration/client.py index 99662b542..682038e43 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,3 +1,5 @@ +"""Simple Spider client for testing purposes.""" + import re import uuid from collections.abc import Generator @@ -34,9 +36,7 @@ class TaskOutput: @dataclass class Task: - """ - Task represents a unit of work in the task graph. - """ + """Task represents a unit of work in the task graph.""" id: uuid.UUID function_name: str @@ -48,9 +48,7 @@ class Task: @dataclass class TaskGraph: - """ - TaskGraph represents a directed acyclic graph of tasks. - """ + """TaskGraph represents a directed acyclic graph of tasks.""" id: uuid.UUID tasks: dict[uuid.UUID, Task] @@ -59,18 +57,14 @@ class TaskGraph: @dataclass class Driver: - """ - Driver represents a client that can submit jobs to the task graph. - """ + """Driver represents a client that can submit jobs to the task graph.""" id: uuid.UUID @dataclass class Data: - """ - Data represents a Spider Data object. - """ + """Data represents a Spider Data object.""" id: uuid.UUID value: str @@ -87,7 +81,7 @@ def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: ) match = pattern.match(storage_url) if not match: - raise ValueError("Invalid JDBC URL format") + raise ValueError(storage_url) connection_params = match.groupdict() return mysql.connector.connect( @@ -103,7 +97,8 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU """ Check if the task is a head task, meaning it has no parent. :param task_id: the ID of the task to check. - :param dependencies: list of dependencies where each dependency is a tuple (parent_id, child_id). + :param dependencies: list of dependencies where each dependency is a tuple + (parent_id, child_id). :return: True if the task has no parent, False otherwise. """ return not any(dependency[1] == task_id for dependency in dependencies) @@ -124,7 +119,8 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: conn.close() -def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: +def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph)\ + -> None: """ Submit a job to the database. :param conn: database connection object. @@ -139,10 +135,7 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: ) for task_id, task in graph.tasks.items(): - if is_head_task(task_id, graph.dependencies): - state = "ready" - else: - state = "pending" + state = "ready" if is_head_task(task_id, graph.dependencies) else "pending" cursor.execute( "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry) VALUES (%s, %s, %s, %s, %s, %s)", ( @@ -185,7 +178,7 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: cursor.close() -def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: +def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: """ Get the outputs of a task by its ID. :param conn: database connection object. @@ -212,7 +205,7 @@ def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: return outputs -def get_task_state(conn, task_id: uuid.UUID) -> str: +def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> str: """ Get the state of a task by its ID. :param conn: database connection object. @@ -229,7 +222,7 @@ def get_task_state(conn, task_id: uuid.UUID) -> str: return state -def remove_job(conn, job_id: uuid.UUID) -> None: +def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None: """ Remove a job from the database by its ID. :param conn: database connection object. @@ -243,7 +236,7 @@ def remove_job(conn, job_id: uuid.UUID) -> None: cursor.close() -def add_driver(conn, driver: Driver) -> None: +def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: """ Register a new driver in the database. :param conn: database connection object. @@ -258,7 +251,7 @@ def add_driver(conn, driver: Driver) -> None: cursor.close() -def add_driver_data(conn, driver: Driver, data: Data) -> None: +def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: Data) -> None: """ Add a new data associated with a driver in the database. :param conn: database connection object. @@ -278,7 +271,7 @@ def add_driver_data(conn, driver: Driver, data: Data) -> None: cursor.close() -def remove_data(conn, data: Data) -> None: +def remove_data(conn: mysql.connector.MySQLConnection, data: Data) -> None: """ Remove data from the database by its ID. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index c1d155043..6fbbd6ad8 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,3 +1,5 @@ +"""Integration test for the client_test C++ program.""" + import subprocess import time from collections.abc import Generator @@ -68,6 +70,8 @@ def scheduler_worker() -> Generator[None, None, None]: class TestClient: + """Test class for the client_test C++ program.""" + @pytest.mark.usefixtures("scheduler_worker") def test_client(self) -> None: """ diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 9c6ac27e4..93f2cde2a 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -1,3 +1,5 @@ +"""Integration tests for the scheduler and worker processes.""" + import subprocess import time import uuid @@ -248,6 +250,7 @@ def random_fail_job(storage): class TestSchedulerWorker: + """Test class for the scheduler and worker integration tests.""" @pytest.mark.usefixtures("scheduler_worker") def test_job_success(self, storage, success_job): """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 764bf8185..c9605b898 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -1,3 +1,5 @@ +"""Integration tests for worker signal handling.""" + import os import signal import subprocess @@ -32,7 +34,6 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t :param lib: Library to load in the worker. :return: scheduler and worker processes. """ - root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -67,7 +68,6 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po Fixture to start a scheduler and a worker process for testing signal handling. :return: """ - scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -79,6 +79,7 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po class TestWorkerSignal: + """Test cases for worker signal handling.""" def test_task_signal(self, storage, scheduler_worker_signal) -> None: diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 33aada438..79e4bf950 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -1,3 +1,5 @@ +"""Utilities for the network port.""" + import socket From ba7c6e58e9d6936149bb1d5ba397e0c1c076ec2c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:57:00 -0400 Subject: [PATCH 22/40] Fix ruff --- ruff.toml | 4 +++- tests/integration/client.py | 6 ++++-- tests/integration/test_client.py | 5 ++--- tests/integration/test_scheduler_worker.py | 16 ++++++++-------- tests/integration/test_signal.py | 9 ++++----- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/ruff.toml b/ruff.toml index 75462c9cd..d2294f8d0 100644 --- a/ruff.toml +++ b/ruff.toml @@ -26,9 +26,11 @@ isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "S101", # Allow use of `assert` (security warning) "ANN001", # Ignore missing type annotations for arguments "ANN201", # Ignore missing type annotations for parameters + "S101", # Allow use of `assert` (security warning) + "S603", # Allow user of subprocess.Popen (security warning) + "T201", # Allow use of `print` (testing) ] [format] diff --git a/tests/integration/client.py b/tests/integration/client.py index 682038e43..4ed1258dd 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -137,7 +137,8 @@ def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, grap for task_id, task in graph.tasks.items(): state = "ready" if is_head_task(task_id, graph.dependencies) else "pending" cursor.execute( - "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry) VALUES (%s, %s, %s, %s, %s, %s)", + "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry)" + " VALUES (%s, %s, %s, %s, %s, %s)", ( task.id.bytes, graph.id.bytes, @@ -150,7 +151,8 @@ def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, grap for i, task_input in enumerate(task.inputs): cursor.execute( - "INSERT INTO task_inputs (type, task_id, position, output_task_id, output_task_position, value, data_id) VALUES (%s, %s, %s, %s, %s, %s, %s)", + "INSERT INTO task_inputs (type, task_id, position, output_task_id," + " output_task_position, value, data_id) VALUES (%s, %s, %s, %s, %s, %s, %s)", ( task_input.type, task.id.bytes, diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 6fbbd6ad8..a00dc8a6f 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -51,8 +51,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -@pytest.mark.usefixtures("storage") -def scheduler_worker() -> Generator[None, None, None]: +def scheduler_worker(storage) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, and then kills the processes after the test class is done. @@ -85,5 +84,5 @@ def test_client(self) -> None: "--storage_url", g_storage_url, ] - p = subprocess.run(client_cmds, check=False, timeout=20) + p = subprocess.run(client_cmds, check=True, timeout=20) assert p.returncode == 0 diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 93f2cde2a..1a2f88877 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -65,8 +65,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -@pytest.mark.usefixtures("storage") -def scheduler_worker() -> Generator[None, None, None]: +def scheduler_worker(storage) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. After the test function completes, it kills the scheduler and the worker process. @@ -82,11 +81,11 @@ def scheduler_worker() -> Generator[None, None, None]: worker_process.kill() -@pytest.fixture(scope="function") +@pytest.fixture def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ - Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. - Cleans up the job after the test function completes. + Fixture to create a job with two parent tasks and one child task. Yields the task graph and + tasks. Cleans up the job after the test function completes. :param storage: :return: """ @@ -146,7 +145,7 @@ def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, remove_job(storage, graph.id) -@pytest.fixture(scope="function") +@pytest.fixture def fail_job(storage) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -174,7 +173,7 @@ def fail_job(storage) -> Generator[Task, None, None]: remove_job(storage, graph.id) -@pytest.fixture(scope="function") +@pytest.fixture def data_job(storage) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -211,7 +210,7 @@ def data_job(storage) -> Generator[Task, None, None]: remove_data(storage, data) -@pytest.fixture(scope="function") +@pytest.fixture def random_fail_job(storage): """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -251,6 +250,7 @@ def random_fail_job(storage): class TestSchedulerWorker: """Test class for the scheduler and worker integration tests.""" + @pytest.mark.usefixtures("scheduler_worker") def test_job_success(self, storage, success_job): """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index c9605b898..0edb71645 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,7 +26,8 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> tuple[subprocess.Popen, subprocess.Popen]: +def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ + -> tuple[subprocess.Popen, subprocess.Popen]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -61,9 +62,8 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t return scheduler_process, worker_process -@pytest.fixture(scope="function") -@pytest.mark.usefixtures("storage") -def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +@pytest.fixture +def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -81,7 +81,6 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po class TestWorkerSignal: """Test cases for worker signal handling.""" - def test_task_signal(self, storage, scheduler_worker_signal) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From 209acb14a38a3f13c2622f2e9c0a210115f5038a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 14:56:59 -0400 Subject: [PATCH 23/40] Fix ruff --- ruff.toml | 2 - tests/integration/client.py | 7 +++- tests/integration/test_client.py | 7 +++- tests/integration/test_scheduler_worker.py | 47 +++++++++++++++++----- tests/integration/test_signal.py | 22 +++++++--- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/ruff.toml b/ruff.toml index d2294f8d0..6cdab7927 100644 --- a/ruff.toml +++ b/ruff.toml @@ -26,8 +26,6 @@ isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "ANN001", # Ignore missing type annotations for arguments - "ANN201", # Ignore missing type annotations for parameters "S101", # Allow use of `assert` (security warning) "S603", # Allow user of subprocess.Popen (security warning) "T201", # Allow use of `print` (testing) diff --git a/tests/integration/client.py b/tests/integration/client.py index 4ed1258dd..886a6c3c3 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -119,8 +119,11 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: conn.close() -def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph)\ - -> None: +def submit_job( + conn: mysql.connector.MySQLConnection, + client_id: uuid.UUID, + graph: TaskGraph +) -> None: """ Submit a job to the database. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index a00dc8a6f..aac10a9ce 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -5,11 +5,11 @@ from collections.abc import Generator from pathlib import Path +import mysql.connector import pytest from .client import ( g_storage_url, - storage, ) from .utils import g_scheduler_port @@ -51,12 +51,15 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -def scheduler_worker(storage) -> Generator[None, None, None]: +def scheduler_worker( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, and then kills the processes after the test class is done. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 1a2f88877..84293fc42 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -7,6 +7,7 @@ from pathlib import Path import msgpack +import mysql.connector import pytest from .client import ( @@ -19,7 +20,6 @@ get_task_state, remove_data, remove_job, - storage, submit_job, Task, TaskGraph, @@ -65,12 +65,15 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -def scheduler_worker(storage) -> Generator[None, None, None]: +def scheduler_worker( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. After the test function completes, it kills the scheduler and the worker process. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -82,7 +85,9 @@ def scheduler_worker(storage) -> Generator[None, None, None]: @pytest.fixture -def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: +def success_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +)-> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. Cleans up the job after the test function completes. @@ -146,7 +151,9 @@ def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, @pytest.fixture -def fail_job(storage) -> Generator[Task, None, None]: +def fail_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. Yield the task. Cleanup the job after the test function completes. @@ -174,7 +181,9 @@ def fail_job(storage) -> Generator[Task, None, None]: @pytest.fixture -def data_job(storage) -> Generator[Task, None, None]: +def data_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. Cleans up the job and data after the test function completes. @@ -211,7 +220,9 @@ def data_job(storage) -> Generator[Task, None, None]: @pytest.fixture -def random_fail_job(storage): +def random_fail_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. Yields the task. Cleans up the job after the test function completes. @@ -252,7 +263,11 @@ class TestSchedulerWorker: """Test class for the scheduler and worker integration tests.""" @pytest.mark.usefixtures("scheduler_worker") - def test_job_success(self, storage, success_job): + def test_job_success( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None] + ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. :param storage: @@ -279,7 +294,11 @@ def test_job_success(self, storage, success_job): assert outputs[0].value == msgpack.packb(10) @pytest.mark.usefixtures("scheduler_worker") - def test_job_failure(self, storage, fail_job): + def test_job_failure( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + fail_job: Generator[Task, None, None] + ) -> None: """ Test the failure of a job that raise an error. :param storage: @@ -293,7 +312,11 @@ def test_job_failure(self, storage, fail_job): assert state == "fail" @pytest.mark.usefixtures("scheduler_worker") - def test_data_job(self, storage, data_job): + def test_data_job( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + data_job: Generator[Task, None, None] + ) -> None: """ Test the successful execution of a job that uses data. :param storage: @@ -310,7 +333,11 @@ def test_data_job(self, storage, data_job): assert outputs[0].value == msgpack.packb(2) @pytest.mark.usefixtures("scheduler_worker") - def test_random_fail_job(self, storage, random_fail_job): + def test_random_fail_job( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + random_fail_job: Generator[Task, None, None] + ) -> None: """ Test the successful recovery and execution of a job that randomly fails. :param storage: diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 0edb71645..0da6466a4 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -9,6 +9,7 @@ from pathlib import Path import msgpack +import mysql.connector import pytest from .client import ( @@ -16,7 +17,6 @@ get_task_outputs, get_task_state, remove_job, - storage, submit_job, Task, TaskGraph, @@ -37,7 +37,7 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" - popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + popen_opts = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True} scheduler_cmds = [ str(bin_dir / "spider_scheduler"), "--host", @@ -63,11 +63,13 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ @pytest.fixture -def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal(storage: Generator[mysql.connector.MySQLConnection, None, None])\ + -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -81,7 +83,12 @@ def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subpro class TestWorkerSignal: """Test cases for worker signal handling.""" - def test_task_signal(self, storage, scheduler_worker_signal) -> None: + def test_task_signal( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[ + tuple[subprocess.Popen, subprocess.Popen], None, None] + ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. Submit a task that checks whether the task executor receives the SIGTERM signal. @@ -155,7 +162,12 @@ def test_task_signal(self, storage, scheduler_worker_signal) -> None: remove_job(storage, new_graph.id) remove_job(storage, graph.id) - def test_task_exit(self, storage, scheduler_worker_signal) -> None: + def test_task_exit( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[ + tuple[subprocess.Popen, subprocess.Popen], None, None] + ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. Task executor exits immediately after receiving the signal. From 406b514007a05e9701ddcacd0638dd5d20ad4e0e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 14:57:52 -0400 Subject: [PATCH 24/40] Reformat files --- tests/integration/client.py | 4 +-- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 36 +++++++++++----------- tests/integration/test_signal.py | 24 +++++++-------- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 886a6c3c3..721ec2444 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -120,9 +120,7 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: def submit_job( - conn: mysql.connector.MySQLConnection, - client_id: uuid.UUID, - graph: TaskGraph + conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph ) -> None: """ Submit a job to the database. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index aac10a9ce..f70f51987 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 84293fc42..2f0421c05 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,8 +86,8 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] -)-> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: + storage: Generator[mysql.connector.MySQLConnection, None, None], +) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. Cleans up the job after the test function completes. @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -264,9 +264,9 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. @@ -295,9 +295,9 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - fail_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + fail_job: Generator[Task, None, None], ) -> None: """ Test the failure of a job that raise an error. @@ -313,9 +313,9 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - data_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + data_job: Generator[Task, None, None], ) -> None: """ Test the successful execution of a job that uses data. @@ -334,9 +334,9 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - random_fail_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + random_fail_job: Generator[Task, None, None], ) -> None: """ Test the successful recovery and execution of a job that randomly fails. diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 0da6466a4..28b275346 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,8 +26,9 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ - -> tuple[subprocess.Popen, subprocess.Popen]: +def start_scheduler_worker( + storage_url: str, scheduler_port: int, lib: str +) -> tuple[subprocess.Popen, subprocess.Popen]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -63,8 +64,9 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ @pytest.fixture -def scheduler_worker_signal(storage: Generator[mysql.connector.MySQLConnection, None, None])\ - -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal( + storage: Generator[mysql.connector.MySQLConnection, None, None], +) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -84,10 +86,9 @@ class TestWorkerSignal: """Test cases for worker signal handling.""" def test_task_signal( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - scheduler_worker_signal: Generator[ - tuple[subprocess.Popen, subprocess.Popen], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. @@ -163,10 +164,9 @@ def test_task_signal( remove_job(storage, graph.id) def test_task_exit( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - scheduler_worker_signal: Generator[ - tuple[subprocess.Popen, subprocess.Popen], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From 66892f5e5194490d9dd8af79ecb9db2ac63740fe Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 16:08:45 -0400 Subject: [PATCH 25/40] Remove .inc from cpp linting --- lint-tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 659480e50..161c8a193 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -101,7 +101,7 @@ tasks: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_EXAMPLES_COMPILE_COMMANDS_DB}}'" - INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" ROOT_PATHS: - "{{.G_EXAMPLES_DIR}}" From 3186a429f9a79ab93750deaa47c3d542a3aaa0af Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 18:41:52 -0400 Subject: [PATCH 26/40] Add mypy and merge lint and test requirements.txt --- lint-tasks.yaml | 5 +++-- lint-requirements.txt => lint-test-requirements.txt | 7 +++++++ mypy.ini | 8 ++++++++ test-requirements.txt | 3 --- test-tasks.yaml | 4 ++-- 5 files changed, 20 insertions(+), 7 deletions(-) rename lint-requirements.txt => lint-test-requirements.txt (57%) create mode 100644 mypy.ini delete mode 100644 test-requirements.txt diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 7822bda7f..3739fa12d 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -130,6 +130,7 @@ tasks: - "tests/integration" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" + mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" @@ -177,7 +178,7 @@ tasks: sources: - "{{.ROOT_DIR}}/taskfile.yaml" - "{{.TASKFILE}}" - - "lint-requirements.txt" + - "lint-test-requirements.txt" generates: ["{{.CHECKSUM_FILE}}"] run: "once" deps: @@ -191,7 +192,7 @@ tasks: vars: LABEL: "lint" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-test-requirements.txt" # This command must be last - task: ":utils:checksum:compute" vars: diff --git a/lint-requirements.txt b/lint-test-requirements.txt similarity index 57% rename from lint-requirements.txt rename to lint-test-requirements.txt index 690b9797f..7ac75af21 100644 --- a/lint-requirements.txt +++ b/lint-test-requirements.txt @@ -1,6 +1,13 @@ +# Lint dependencies clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 ruff>=0.4.4 +mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 + +# Test dependencies +msgpack>=1.1.0 +mysql-connector-python>=8.0.26 +pytest>=8.3.4 diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..4c66b1114 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index ec24b0b20..000000000 --- a/test-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -msgpack>=1.1.0 -mysql-connector-python>=8.0.26 -pytest>=8.3.4 \ No newline at end of file diff --git a/test-tasks.yaml b/test-tasks.yaml index 116ae85b6..41dfc4578 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -56,7 +56,7 @@ tasks: sources: - "{{.ROOT_DIR}}/taskfile.yaml" - "{{.TASKFILE}}" - - "test-requirements.txt" + - "lint-test-requirements.txt" generates: ["{{.CHECKSUM_FILE}}"] run: "once" deps: @@ -70,7 +70,7 @@ tasks: vars: LABEL: "test" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "{{.ROOT_DIR}}/test-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-test-requirements.txt" # This command must be last - task: ":utils:checksum:compute" vars: From 092593844041d72e6da23797a7c8fb16fbf1d258 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:34:15 -0400 Subject: [PATCH 27/40] Fix mysql connection type --- tests/integration/client.py | 19 ++++++++++--------- tests/integration/test_client.py | 4 ++-- tests/integration/test_scheduler_worker.py | 20 ++++++++++---------- tests/integration/test_signal.py | 8 ++++---- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 721ec2444..f0b7f0cfe 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -69,8 +69,9 @@ class Data: id: uuid.UUID value: str +SQLConnection = mysql.connector.abstracts.MySQLConnectionAbstract | mysql.connector.pooling.PooledMySQLConnection -def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: +def create_connection(storage_url: str) -> SQLConnection : """ Creation a MariaDB connection from a JDBC URL. :param storage_url: JDBC URL for the MariaDB database. @@ -108,7 +109,7 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU @pytest.fixture(scope="session") -def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: +def storage() -> Generator[SQLConnection, None, None]: """ Fixture to create a database connection for the test session. Yields a connection object and ensures it is closed after the tests are done. @@ -120,7 +121,7 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: def submit_job( - conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph + conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph ) -> None: """ Submit a job to the database. @@ -181,7 +182,7 @@ def submit_job( cursor.close() -def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: +def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: """ Get the outputs of a task by its ID. :param conn: database connection object. @@ -208,7 +209,7 @@ def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) return outputs -def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> str: +def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: """ Get the state of a task by its ID. :param conn: database connection object. @@ -225,7 +226,7 @@ def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> return state -def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None: +def remove_job(conn: SQLConnection, job_id: uuid.UUID) -> None: """ Remove a job from the database by its ID. :param conn: database connection object. @@ -239,7 +240,7 @@ def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None cursor.close() -def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: +def add_driver(conn: SQLConnection, driver: Driver) -> None: """ Register a new driver in the database. :param conn: database connection object. @@ -254,7 +255,7 @@ def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: cursor.close() -def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: Data) -> None: +def add_driver_data(conn: SQLConnection, driver: Driver, data: Data) -> None: """ Add a new data associated with a driver in the database. :param conn: database connection object. @@ -274,7 +275,7 @@ def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: cursor.close() -def remove_data(conn: mysql.connector.MySQLConnection, data: Data) -> None: +def remove_data(conn: SQLConnection, data: Data) -> None: """ Remove data from the database by its ID. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index f70f51987..809085924 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -5,11 +5,11 @@ from collections.abc import Generator from pathlib import Path -import mysql.connector import pytest from .client import ( g_storage_url, + SQLConnection ) from .utils import g_scheduler_port @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 2f0421c05..190875688 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -7,7 +7,6 @@ from pathlib import Path import msgpack -import mysql.connector import pytest from .client import ( @@ -20,6 +19,7 @@ get_task_state, remove_data, remove_job, + SQLConnection, submit_job, Task, TaskGraph, @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,7 +86,7 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -265,7 +265,7 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], ) -> None: """ @@ -296,7 +296,7 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], fail_job: Generator[Task, None, None], ) -> None: """ @@ -314,7 +314,7 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], data_job: Generator[Task, None, None], ) -> None: """ @@ -335,7 +335,7 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], random_fail_job: Generator[Task, None, None], ) -> None: """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 28b275346..42eee6bac 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -9,7 +9,6 @@ from pathlib import Path import msgpack -import mysql.connector import pytest from .client import ( @@ -17,6 +16,7 @@ get_task_outputs, get_task_state, remove_job, + SQLConnection, submit_job, Task, TaskGraph, @@ -65,7 +65,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. @@ -87,7 +87,7 @@ class TestWorkerSignal: def test_task_signal( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ @@ -165,7 +165,7 @@ def test_task_signal( def test_task_exit( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ From 6936a9b448a6ce4ececbc97d53e23b67569850e4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:39:24 -0400 Subject: [PATCH 28/40] Fix socket name type --- tests/integration/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 79e4bf950..cae3f1730 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,12 +2,16 @@ import socket +IPv4Addr = tuple[str, int] +IPv6Addr = tuple[str, int, int, int] +AddrType = IPv4Addr | IPv6Addr def _get_free_tcp_port() -> int: """Returns a free TCP port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("127.0.0.1", 0)) - return s.getsockname()[1] + addr: AddrType = s.getsockname() + return addr[1] g_scheduler_port = _get_free_tcp_port() From 74070908318632b16e7dde6800994676d19e8982 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:43:07 -0400 Subject: [PATCH 29/40] Fix return type from db cursor --- tests/integration/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index f0b7f0cfe..b4369fe3a 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -196,7 +196,8 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - for output_type, value, data_id in cursor.fetchall(): + rows: list[tuple[str, str | None, bytes | None]] = cursor.fetchall() + for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) elif data_id is not None: @@ -219,7 +220,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state = cursor.fetchone()[0] + state: str = cursor.fetchone()[0] conn.commit() cursor.close() From 6a95b24912c69807c9cc66172f97a5441beea8e6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:50:34 -0400 Subject: [PATCH 30/40] Fix db cursor return type --- tests/integration/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index b4369fe3a..13f9666a9 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -4,6 +4,7 @@ import uuid from collections.abc import Generator from dataclasses import dataclass +from typing import cast import mysql.connector import pytest @@ -196,7 +197,7 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - rows: list[tuple[str, str | None, bytes | None]] = cursor.fetchall() + rows = cast(list[tuple[str, str | None, bytes | None]], cursor.fetchall()) for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) @@ -220,7 +221,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state: str = cursor.fetchone()[0] + state = cast(tuple[str], cursor.fetchone())[0] conn.commit() cursor.close() From f29dcba479f5f80d66c99aa94fbf7ab93ecfc4c2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:55:24 -0400 Subject: [PATCH 31/40] Fix mypy import untyped --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 4c66b1114..932c4e402 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,3 +6,6 @@ pretty = true show_error_code_links = true show_error_context = true show_error_end = true + +[mypy-msgpack.*] +ignore_missing_imports = true From 453f20908e8f566d7561e1a3c178c713e206e915 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:17:20 -0400 Subject: [PATCH 32/40] Fix mypy and Popen --- tests/integration/test_signal.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 42eee6bac..47848f636 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -7,6 +7,7 @@ import uuid from collections.abc import Generator from pathlib import Path +from typing import TypedDict import msgpack import pytest @@ -25,10 +26,16 @@ ) from .utils import g_scheduler_port +class PopenOpts(TypedDict, total=False): + stdin: int + stdout: int + stderr: int + text: bool + def start_scheduler_worker( storage_url: str, scheduler_port: int, lib: str -) -> tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -38,7 +45,11 @@ def start_scheduler_worker( """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" - popen_opts = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True} + popen_opts: PopenOpts = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "text": True, + } scheduler_cmds = [ str(bin_dir / "spider_scheduler"), "--host", @@ -66,7 +77,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( storage: Generator[SQLConnection, None, None], -) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +) -> Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -88,7 +99,7 @@ class TestWorkerSignal: def test_task_signal( self, storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From d1011c70eb908fad0247f84d845be67c89d7bbd4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:22:31 -0400 Subject: [PATCH 33/40] Fix generator type hint --- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 26 +++++++++++----------- tests/integration/test_signal.py | 10 ++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 809085924..c96b731df 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 190875688..2aa59ed50 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,7 +86,7 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -265,8 +265,8 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( self, - storage: Generator[SQLConnection, None, None], - success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], + storage: SQLConnection, + success_job: tuple[TaskGraph, Task, Task, Task], ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. @@ -296,8 +296,8 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( self, - storage: Generator[SQLConnection, None, None], - fail_job: Generator[Task, None, None], + storage: SQLConnection, + fail_job: Task, ) -> None: """ Test the failure of a job that raise an error. @@ -314,8 +314,8 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( self, - storage: Generator[SQLConnection, None, None], - data_job: Generator[Task, None, None], + storage: SQLConnection, + data_job: Task, ) -> None: """ Test the successful execution of a job that uses data. @@ -335,8 +335,8 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( self, - storage: Generator[SQLConnection, None, None], - random_fail_job: Generator[Task, None, None], + storage: SQLConnection, + random_fail_job: Task, ) -> None: """ Test the successful recovery and execution of a job that randomly fails. diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 47848f636..667e23d51 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -76,7 +76,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. @@ -98,8 +98,8 @@ class TestWorkerSignal: def test_task_signal( self, - storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None], + storage: SQLConnection, + scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. @@ -176,8 +176,8 @@ def test_task_signal( def test_task_exit( self, - storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], + storage: SQLConnection, + scheduler_worker_signal: tuple[subprocess.Popen, subprocess.Popen], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From a7dc642bcb68e83f52152b27bd811f358843ad2e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:24:47 -0400 Subject: [PATCH 34/40] Fix mypy --- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 2 +- tests/integration/test_signal.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index c96b731df..e5219835c 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -16,7 +16,7 @@ def start_scheduler_workers( storage_url: str, scheduler_port: int -) -> tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Start the scheduler and two worker processes. :param storage_url: diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 2aa59ed50..ae841332e 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -31,7 +31,7 @@ def start_scheduler_worker( storage_url: str, scheduler_port: int -) -> tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Start a scheduler and a worker process. :param storage_url: JDBC storage URL diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 667e23d51..f42fdefc3 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -177,7 +177,7 @@ def test_task_signal( def test_task_exit( self, storage: SQLConnection, - scheduler_worker_signal: tuple[subprocess.Popen, subprocess.Popen], + scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From a7d92c2255faf5c3b336ca90069aecec7d70b600 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:28:12 -0400 Subject: [PATCH 35/40] Fix ruff --- tests/integration/client.py | 17 ++++++++++------- tests/integration/test_client.py | 5 +---- tests/integration/test_signal.py | 3 +++ tests/integration/utils.py | 1 + 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 13f9666a9..b1e5a30ea 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -70,9 +70,14 @@ class Data: id: uuid.UUID value: str -SQLConnection = mysql.connector.abstracts.MySQLConnectionAbstract | mysql.connector.pooling.PooledMySQLConnection -def create_connection(storage_url: str) -> SQLConnection : +SQLConnection = ( + mysql.connector.abstracts.MySQLConnectionAbstract + | mysql.connector.pooling.PooledMySQLConnection +) + + +def create_connection(storage_url: str) -> SQLConnection: """ Creation a MariaDB connection from a JDBC URL. :param storage_url: JDBC URL for the MariaDB database. @@ -121,9 +126,7 @@ def storage() -> Generator[SQLConnection, None, None]: conn.close() -def submit_job( - conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph -) -> None: +def submit_job(conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph) -> None: """ Submit a job to the database. :param conn: database connection object. @@ -197,7 +200,7 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - rows = cast(list[tuple[str, str | None, bytes | None]], cursor.fetchall()) + rows = cast("list[tuple[str, str | None, bytes | None]]", cursor.fetchall()) for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) @@ -221,7 +224,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state = cast(tuple[str], cursor.fetchone())[0] + state = cast("tuple[str]", cursor.fetchone())[0] conn.commit() cursor.close() diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index e5219835c..2b2ca00a3 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -7,10 +7,7 @@ import pytest -from .client import ( - g_storage_url, - SQLConnection -) +from .client import g_storage_url, SQLConnection from .utils import g_scheduler_port diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index f42fdefc3..89191621c 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,7 +26,10 @@ ) from .utils import g_scheduler_port + class PopenOpts(TypedDict, total=False): + """Options for subprocess.Popen.""" + stdin: int stdout: int stderr: int diff --git a/tests/integration/utils.py b/tests/integration/utils.py index cae3f1730..7f1635390 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -6,6 +6,7 @@ IPv6Addr = tuple[str, int, int, int] AddrType = IPv4Addr | IPv6Addr + def _get_free_tcp_port() -> int: """Returns a free TCP port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: From 1c6213bb4a8d5d229bfacc89f84020c9463f2857 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:40:58 -0400 Subject: [PATCH 36/40] Simply socket return types. Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- tests/integration/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 7f1635390..64c134cec 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,9 +2,7 @@ import socket -IPv4Addr = tuple[str, int] -IPv6Addr = tuple[str, int, int, int] -AddrType = IPv4Addr | IPv6Addr +AddrType = tuple[str, int] def _get_free_tcp_port() -> int: From 55a727ae8012f724a95d25360079b76ad1353ae5 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 23:41:32 -0400 Subject: [PATCH 37/40] Add tombi lint tasks --- lint-tasks.yaml | 24 +++++++++++++++++++++++ lint-test-requirements.txt | 1 + ruff.toml | 40 +++++++++++++++++++------------------- tombi.toml | 34 ++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 tombi.toml diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 2bfe90649..40259b219 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -11,12 +11,14 @@ tasks: - task: "cpp-check" - task: "py-check" - task: "yml-check" + - task: "toml-check" fix: cmds: - task: "cmake-fix" - task: "cpp-fix" - task: "yml-fix" + - task: "toml-fix" cmake-check: deps: ["venv"] @@ -154,6 +156,28 @@ tasks: taskfile.yaml \ test-tasks.yaml + toml: + aliases: + - "toml-check" + - "toml-fix" + cmds: + - task: "toml-lint" + - task: "toml-format" + + toml-lint: + deps: ["venv"] + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + tombi lint "{{.ROOT_DIR}}" + + toml-format: + deps: ["venv"] + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + tombi format "{{.ROOT_DIR}}" + cmake: internal: true requires: diff --git a/lint-test-requirements.txt b/lint-test-requirements.txt index 7ac75af21..879c198b3 100644 --- a/lint-test-requirements.txt +++ b/lint-test-requirements.txt @@ -6,6 +6,7 @@ ruff>=0.4.4 mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 +tombi>=0.4.0 # Test dependencies msgpack>=1.1.0 diff --git a/ruff.toml b/ruff.toml index 6cdab7927..fde4359e1 100644 --- a/ruff.toml +++ b/ruff.toml @@ -4,31 +4,31 @@ line-length = 100 select = ["ALL"] extend-select = ["PT"] ignore = [ - "COM812", # Redundant and conflicts with ruff format - "D203", # No blank line before docstrings (D211) - "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists - "D212", # Enforce docstring summary line on the next line after quotes (D213) - "D400", # First line of docstrings may not end in period - "D401", # Docstrings should be written in present tense (not imperative) - "D415", # First line of docstrings may not end in a period, question mark, or exclamation point - "FA102", # Allow use of PEP 604 union in type annotations - "FBT", # Allow bool positional parameters since other value positions are allowed - "FIX002", # Allow todo statements - "PERF401", # Allow for loops when creating lists - "PERF403", # Allow for loops when creating dicts - "S311", # Allow usage of `random` package - "SIM102", # Allow collapsible if statements for readability - "TD002", # Author unnecessary for todo statement - "TD003", # Issue link unnecessary for todo statement - "UP015", # Explicit open modes are helpful + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful ] isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "S101", # Allow use of `assert` (security warning) - "S603", # Allow user of subprocess.Popen (security warning) - "T201", # Allow use of `print` (testing) + "S101", # Allow use of `assert` (security warning) + "S603", # Allow user of subprocess.Popen (security warning) + "T201", # Allow use of `print` (testing) ] [format] diff --git a/tombi.toml b/tombi.toml new file mode 100644 index 000000000..ad86d3c0c --- /dev/null +++ b/tombi.toml @@ -0,0 +1,34 @@ +toml-version = "v1.0.0" + +[files] +include = ["**/*.toml"] +exclude = ["tools/yscope-dev-utils/**", "build/**"] + +[format] + +[lint] +[lint.rules] +dotted-keys-out-of-order = "warn" +key-empty = "warn" +tables-out-of-order = "warn" + +[schema] +enabled = true +catalog = { + paths = [ + "tombi://json.schemastore.org/api/json/catalog.json", + "https://json.schemastore.org/api/json/catalog.json", + ], +} + +[[schemas]] +path = "tombi://json.schemastore.org/tombi.json" +include = ["tombi.toml"] + +[[schemas]] +path = "https://json.schemastore.org/pyproject.json" +include = ["pyproject.toml"] + +[[schemas]] +path = "https://json.schemastore.org/ruff.json" +include = ["ruff.toml"] From c98fea18e1a3f4134067c8e5648f970bdaba6fe2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 10:07:59 -0400 Subject: [PATCH 38/40] Rename cpp build tasks --- build-tasks.yaml | 4 ++-- test-tasks.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build-tasks.yaml b/build-tasks.yaml index 666576f26..53828027c 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -1,7 +1,7 @@ version: "3" tasks: - target: + cpp-target: internal: true vars: TARGETS: @@ -14,7 +14,7 @@ tasks: --parallel {{numCPU}} --target {{range .TARGETS}}{{.}} {{end}} - clean: + cpp-clean: internal: true deps: [":config-cmake-project"] cmds: diff --git a/test-tasks.yaml b/test-tasks.yaml index 41dfc4578..58a1ff190 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -27,7 +27,7 @@ tasks: build-unit-test: internal: true deps: - - task: ":build:target" + - task: ":build:cpp-target" vars: TARGETS: ["spider_task_executor", "unitTest", "worker_test"] @@ -35,7 +35,7 @@ tasks: dir: "{{.G_BUILD_SPIDER_DIR}}" deps: - "venv" - - task: ":build:target" + - task: ":build:cpp-target" vars: TARGETS: [ "spider_task_executor", From b09515c0d5a815c6e72a25e4275ddf1d464c4937 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 10:16:51 -0400 Subject: [PATCH 39/40] Increase min version of tombi --- lint-test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-test-requirements.txt b/lint-test-requirements.txt index 879c198b3..ffbf89925 100644 --- a/lint-test-requirements.txt +++ b/lint-test-requirements.txt @@ -6,7 +6,7 @@ ruff>=0.4.4 mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 -tombi>=0.4.0 +tombi>=0.4.35 # Test dependencies msgpack>=1.1.0 From f447614c9ba65d9e2ddd42e69a194120e7168d1f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 17:35:09 -0400 Subject: [PATCH 40/40] Fix redenduncy caused by merge --- lint-tasks.yaml | 2 -- requirements-dev.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 4e941803f..54ca4611c 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -12,7 +12,6 @@ tasks: - task: "py-check" - task: "toml-check" - task: "yml-check" - - task: "toml-check" fix: cmds: @@ -20,7 +19,6 @@ tasks: - task: "cpp-fix" - task: "toml-fix" - task: "yml-fix" - - task: "toml-fix" cmake-check: deps: ["venv"] diff --git a/requirements-dev.txt b/requirements-dev.txt index d1379209c..a5e8717f2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,6 @@ mypy>=1.12.0 ruff>=0.4.4 tombi>=0.4.35 yamllint>=1.35.1 -tombi>=0.4.35 # Test dependencies msgpack-types>=0.5.0