From d2ac198bb3d79a0f3bd769faaf3e6bc6c44adcdf Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 13:40:10 -0400 Subject: [PATCH 01/71] Update yscope-dev-utils --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index e300d1bab..5c6bfbd00 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit e300d1bab4c2f33cbf9ddf9f0c08185faf035070 +Subproject commit 5c6bfbd00ad4e9cbfdfab9708298fb11bda4ae8c From cc8a0977836682e8ed9aa3e39aa3268ffbb2447c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 13:44:03 -0400 Subject: [PATCH 02/71] Use boost install task --- dep-tasks.yaml | 160 +------------------------------------------------ 1 file changed, 1 insertion(+), 159 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index a451ddc12..324a7124e 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -147,7 +147,7 @@ tasks: internal: true run: "once" cmds: - - task: "boost-download-and-install" + - task: ":utils:boost:download-and-install" vars: WORK_DIR: "{{.G_DEPS_DIR}}/boost" FILE_SHA256: "2128a4c96862b5c0970c1e34d76b1d57e4a1016b80df85ad39667f30b1deba26" @@ -161,161 +161,3 @@ tasks: - "program_options" - "regex" - "system" - - # Runs the bootstrap.sh generate step in the given source directory. Boost only supports - # in-source generation and building. - # - # @param {string} SOURCE_DIR Project source directory. - # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. - # @param {string[]} TARGETS Target libraries to build. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the generate command. - boost-generate: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./bootstrap.sh - --prefix="{{.INSTALL_PREFIX}}" - --exec-prefix="{{.INSTALL_PREFIX}}" - --with-libraries={{(join "," .TARGETS)}} - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - - # Runs the b2 build step for boost. The caller must have previously called `generate` on - # `SOURCE_DIR` for this task to succeed. - # - # @param {string} SOURCE_DIR Directory containing the boost source. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the build command. - # @param {int} [JOBS] The maximum number of concurrent processes to use when building. If - # omitted, the b2 default number is used. Before 1.76.0, the number was 1. Since 1.76.0, the - # default is the number of cores. - boost-build: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./b2 - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - {{- if .JOBS}} - "-j{{.JOBS}}" - {{- end}} - - # Runs the b2 install step for boost. The caller must have previously called `build` on - # `SOURCE_DIR` for this task to succeed. If `CMAKE_SETTINGS_DIR` is set, a settings file will be - # created in that directory, containing a `boost_ROOT` CMake variable that points to - # `INSTALL_PREFIX`. - # - # @param {string} SOURCE_DIR Directory containing the boost source. - # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. - # @param {string} [CMAKE_SETTINGS_DIR] If set, the directory where the project's CMake settings - # file should be stored. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the install command. - boost-install: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./b2 - install - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - - >- - {{- if .CMAKE_SETTINGS_DIR}} - echo "set(BOOST_ROOT - \"{{.INSTALL_PREFIX}}\" - CACHE PATH - \"Package root for boost.\" - )" >> "{{.CMAKE_SETTINGS_DIR}}/boost.cmake" - {{- end}} - - # Downloads boost from `URL` and installs boost. - # - # General parameters - # @param {string} [WORK_DIR={{.TASK_DIR}}] Base directory to store the install and src - # directories inside. - # @param {string} [SOURCE_DIR={{.WORK_DIR}}/boost-src] Directory in which to extract the tar - # file. - # - # Download parameters - # @param {string} FILE_SHA256 Content hash to verify the downloaded tar file against. - # @param {string} URL - # - # Boost generate parameters - # @param {string} [INSTALL_PREFIX={{.WORK_DIR}}/boost-install] Path prefix of where the project - # should be installed. - # @param {string[]} TARGETS Target libraries to build. - # @param {string[]} [GEN_ARGS] Any additional arguments to pass to the generate command. - # - # Boost build parameters - # @param {int} [JOBS] The maximum number of concurrent processes to use when building. If - # omitted, the b2 default number is used. Before 1.76.0, the number was 1. Since 1.76.0, the - # default is the number of cores. - # @param {string[]} [BUILD_ARGS] Any additional arguments to pass to the build command. - # - # Boost install parameters - # @param {string[]} [INSTALL_ARGS] Any additional arguments to pass to the install command. - # @param {string} [CMAKE_SETTINGS_DIR] If set, the directory where the project's CMake settings - # file should be stored. - boost-download-and-install: - internal: true - label: "{{.TASK}}:{{.URL}}-{{.INSTALL_PREFIX}}" - vars: - # General parameters - WORK_DIR: >- - {{default .ROOT_DIR .WORK_DIR}} - SOURCE_DIR: >- - {{default (printf "%s/boost-src" .WORK_DIR) .SOURCE_DIR}} - - # Boost generate parameters - INSTALL_PREFIX: >- - {{default (printf "%s/boost-install" .WORK_DIR) .INSTALL_PREFIX}} - TARGETS: - ref: "default (list) .TARGETS" - GEN_ARGS: - ref: "default (list) .GEN_ARGS" - - # Boost build parameters - BUILD_ARGS: - ref: "default (list) .BUILD_ARGS" - JOBS: >- - {{default "" .JOBS}} - - # Boost install parameters - INSTALL_ARGS: - ref: "default (list) .INSTALL_ARGS" - CMAKE_SETTINGS_DIR: >- - {{default "" .CMAKE_SETTINGS_DIR}} - requires: - vars: ["FILE_SHA256", "URL"] - deps: - - task: ":utils:remote:download-and-extract-tar" - vars: - FILE_SHA256: "{{.FILE_SHA256}}" - OUTPUT_DIR: "{{.SOURCE_DIR}}" - URL: "{{.URL}}" - cmds: - - task: "boost-generate" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - TARGETS: - ref: ".TARGETS" - EXTRA_ARGS: - ref: ".GEN_ARGS" - - task: "boost-build" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - JOBS: "{{.JOBS}}" - EXTRA_ARGS: - ref: ".BUILD_ARGS" - - task: "boost-install" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" - EXTRA_ARGS: - ref: ".INSTALL_ARGS" From 7f68bdc0d8ec884eaa14096282b680a531d482a2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:00:23 -0400 Subject: [PATCH 03/71] Update install task variable names --- dep-tasks.yaml | 52 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 324a7124e..3c7c05f8a 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -30,12 +30,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "absl" + CMAKE_PACKAGE_NAME: "absl" WORK_DIR: "{{.G_DEPS_DIR}}/absl" - FILE_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" - URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" + TAR_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" + TAR_URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" @@ -45,12 +45,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "Catch2" + CMAKE_PACKAGE_NAME: "Catch2" WORK_DIR: "{{.G_DEPS_DIR}}/Catch2" - FILE_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" - URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" + TAR_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" + TAR_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" @@ -76,12 +76,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "fmt" + CMAKE_PACKAGE_NAME: "fmt" WORK_DIR: "{{.G_DEPS_DIR}}/fmtlib" - FILE_SHA256: "6cb1e6d37bdcb756dbbe59be438790db409cdb4868c66e888d5df9f13f7c027f" - URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.0.2.tar.gz" + TAR_SHA256: "6cb1e6d37bdcb756dbbe59be438790db409cdb4868c66e888d5df9f13f7c027f" + TAR_URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.0.2.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" @@ -95,16 +95,16 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "spdlog" + CMAKE_PACKAGE_NAME: "spdlog" WORK_DIR: "{{.G_DEPS_DIR}}/spdlog" - FILE_SHA256: "9962648c9b4f1a7bbc76fd8d9172555bad1871fdb14ff4f842ef87949682caa5" - URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.0.tar.gz" + TAR_SHA256: "9962648c9b4f1a7bbc76fd8d9172555bad1871fdb14ff4f842ef87949682caa5" + TAR_URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DSPDLOG_BUILD_TESTS=OFF" - "-DUSE_EXTERNAL_FMT=ON" - - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" install-mariadb-connector-cpp: @@ -113,13 +113,13 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "mariadb-connector-cpp" + CMAKE_PACKAGE_NAME: "mariadb-connector-cpp" WORK_DIR: "{{.G_DEPS_DIR}}/mariadb-connector-cpp" - FILE_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" - URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/tags/\ + TAR_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" + TAR_URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/tags/\ 1.1.5.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" @@ -132,16 +132,16 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "msgpack-cxx" + CMAKE_PACKAGE_NAME: "msgpack-cxx" WORK_DIR: "{{.G_DEPS_DIR}}/msgpack" - FILE_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" - URL: "https://github.com/msgpack/msgpack-c/releases/download/\ + TAR_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" + TAR_URL: "https://github.com/msgpack/msgpack-c/releases/download/\ cpp-7.0.0/msgpack-cxx-7.0.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" install-boost: internal: true From 2394649ab8c6454733ff34d8736fb1b8960d2022 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:11:02 -0400 Subject: [PATCH 04/71] Set CMP0074 to NEW to find boost --- dep-tasks.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 3c7c05f8a..d58a61f0d 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -105,6 +105,7 @@ tasks: - "-DSPDLOG_BUILD_TESTS=OFF" - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" install-mariadb-connector-cpp: @@ -142,6 +143,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" install-boost: internal: true From 6ff3e33f9d5b36e3b54be01fbf6b263b4c9ca6a9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:40:52 -0400 Subject: [PATCH 05/71] Add uv to install script --- tools/scripts/lib_install/linux/install-dev.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/scripts/lib_install/linux/install-dev.sh b/tools/scripts/lib_install/linux/install-dev.sh index 73fd319bc..f88c9cc93 100755 --- a/tools/scripts/lib_install/linux/install-dev.sh +++ b/tools/scripts/lib_install/linux/install-dev.sh @@ -36,3 +36,6 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir="$script_dir/.." # TODO https://github.com/y-scope/spider/issues/86 "$lib_install_scripts_dir"/check-cmake-version.sh + +# Install uv +curl -LsSf https://astral.sh/uv/install.sh | sh From 7ae8a782aaf63043f4f1c4b31a47f3dc57bfb346 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 15:04:40 -0400 Subject: [PATCH 06/71] Fix cpp-lint root paths --- lint-tasks.yaml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index d81b7247d..85417f30c 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -45,7 +45,7 @@ tasks: - task: "cpp-static-fix" cpp-format-check: - sources: &cpp_source_files + sources: &cpp_lint_source_files - "{{.ROOT_DIR}}/.clang-format" - "{{.ROOT_DIR}}/.clang-tidy" - "{{.TASKFILE}}" @@ -59,11 +59,14 @@ tasks: vars: FLAGS: ["--dry-run"] INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: &cpp_source_files + - "{{.G_SRC_SPIDER_DIR}}" + - "{{.G_EXAMPLES_DIR}}" + - "{{.G_TEST_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" cpp-format-fix: - sources: *cpp_source_files + sources: *cpp_lint_source_files deps: ["cpp-configs", "venv"] cmds: - task: ":utils:cpp-lint:clang-format" @@ -152,18 +155,6 @@ tasks: taskfile.yaml \ test-tasks.yaml - clang-tidy: - internal: true - requires: - vars: ["FLAGS", "SRC_DIR"] - cmd: |- - . "{{.G_LINT_VENV_DIR}}/bin/activate" - find "{{.SRC_DIR}}" \ - -type f \ - \( -iname "*.cpp" -o -iname "*.h" -o -iname "*.hpp" \) \ - -print0 | \ - xargs -0 --no-run-if-empty clang-tidy {{.FLAGS}} - cmake: internal: true requires: From 56502f90d6df56d5e0d97ed0f872b7682743455b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 15:16:33 -0400 Subject: [PATCH 07/71] Fix clang-tidy file pattern --- lint-tasks.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 85417f30c..659480e50 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -82,7 +82,7 @@ tasks: # When we eventually determine which errors can be safely fixed, we'll allow clang-tidy to # fix them. aliases: ["cpp-static-fix"] - sources: *cpp_source_files + sources: *cpp_lint_source_files deps: [":config-cmake-project", "cpp-configs", "venv"] cmds: - task: ":utils:cpp-lint:clang-tidy-find" @@ -90,21 +90,21 @@ tasks: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_SPIDER_COMPILE_COMMANDS_DB}}'" - INCLUDE_PATTERNS: - - "{{.G_SRC_SPIDER_DIR}}/**" - - "{{.G_TEST_DIR}}/**" + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: + - "{{.G_SRC_SPIDER_DIR}}" + - "{{.G_TEST_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" - task: ":utils:cpp-lint:clang-tidy-find" vars: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_EXAMPLES_COMPILE_COMMANDS_DB}}'" - INCLUDE_PATTERNS: - - "{{.G_EXAMPLES_DIR}}/**" + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: + - "{{.G_EXAMPLES_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" py-check: From 6f6becfff0e90960d46fe07ce442012de70150b1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:19:38 -0400 Subject: [PATCH 08/71] Limit build parallelism --- .github/workflows/code-linting-checks.yaml | 2 ++ .github/workflows/unit-tests.yaml | 2 ++ dep-tasks.yaml | 6 ++++++ taskfile.yaml | 2 ++ 4 files changed, 12 insertions(+) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index c6b1e9117..4ee4e58e4 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -47,6 +47,8 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 + env: + SPIDER_DEP_BUILD_PARALLELISM: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 4b42a93cc..0e6089d37 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -50,6 +50,8 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 + env: + SPIDER_DEP_BUILD_PARALLELISM: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index a451ddc12..9652125e6 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,6 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-Catch2: internal: true @@ -53,6 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} download-ystdlib: internal: true @@ -85,6 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-spdlog: internal: true @@ -106,6 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-mariadb-connector-cpp: internal: true @@ -123,6 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-msgpack: internal: true @@ -142,6 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index 556ef3486..ecece7a09 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + G_DEP_BUILD_PARALLELISM: >- + {{default "" (env SPIDER_DEP_BUILD_PARALLELISM)}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From a74523daa93aef57b0d279863583f7b3b4a7240b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:34:32 -0400 Subject: [PATCH 09/71] Bug fix --- dep-tasks.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 9652125e6..9262da104 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-boost: internal: true From 77ee49460e2dfb7752f4416b0b595a16ced5fc16 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:37:22 -0400 Subject: [PATCH 10/71] Bug fix --- taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index ecece7a09..4a110c06c 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -21,7 +21,7 @@ vars: G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" G_DEP_BUILD_PARALLELISM: >- - {{default "" (env SPIDER_DEP_BUILD_PARALLELISM)}} + {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 3698339529129c883a0d1d927a873e2b879b4dce Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:47:19 -0400 Subject: [PATCH 11/71] Bug fix --- dep-tasks.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 4f45bd44c..fe90b3544 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -169,4 +169,3 @@ tasks: - "program_options" - "regex" - "system" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" From 369e9f1bbf876d7ef1b1ed209f953681d5f83211 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 22:57:18 -0400 Subject: [PATCH 12/71] Rename variables to mirror CLP core Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- taskfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 4a110c06c..d44c1a464 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,8 +20,11 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + # Build parameters + # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEP_BUILD_PARALLELISM: >- {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} + G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 90aa5a285b7068d245c52663dc82cbac3063ee67 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 22:57:45 -0400 Subject: [PATCH 13/71] Rename variables to mirror clp core --- .github/workflows/code-linting-checks.yaml | 2 +- .github/workflows/unit-tests.yaml | 2 +- dep-tasks.yaml | 12 ++++++------ taskfile.yaml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 4ee4e58e4..48f8eacd7 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -48,7 +48,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_BUILD_PARALLELISM: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 0e6089d37..1ced6a889 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -51,7 +51,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_BUILD_PARALLELISM: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 9262da104..cab98d77c 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index 4a110c06c..20da0f3b7 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,8 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" - G_DEP_BUILD_PARALLELISM: >- - {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} + G_DEPS_MAX_PARALLELISM_PER_TASK: >- + {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 7faac8fc6a1e2b196abb0853b23c8ea3153cdeb9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:07:53 -0400 Subject: [PATCH 14/71] Revert "Merge branch 'dep-concurrency' of github.com:sitaowang1998/spider into dep-concurrency" This reverts commit 1769c95feb3868e4a462ccaa694b97f880947f12, reversing changes made to 90aa5a285b7068d245c52663dc82cbac3063ee67. --- .github/workflows/code-linting-checks.yaml | 2 +- .github/workflows/unit-tests.yaml | 2 +- dep-tasks.yaml | 12 ++++++------ taskfile.yaml | 7 ++----- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 1178f0598..48f8eacd7 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -48,7 +48,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_MAX_PARALLELISM_PER_TASK: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 675e7c6ec..1ced6a889 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -51,7 +51,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_MAX_PARALLELISM_PER_TASK: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index ea282d430..cab98d77c 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index d44c1a464..20da0f3b7 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,11 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" - # Build parameters - # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. - G_DEP_BUILD_PARALLELISM: >- - {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} - + G_DEPS_MAX_PARALLELISM_PER_TASK: >- + {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From f494a90b85e24d4e9ee07875e01e7cdb65e5387c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:10:25 -0400 Subject: [PATCH 15/71] Add comment for deps parallelism default value --- taskfile.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 20da0f3b7..470edd582 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + # Build parameters + # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEPS_MAX_PARALLELISM_PER_TASK: >- {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" From ff2fe1c69a7ff4d952d546babba27413dcb0370f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:28:39 -0400 Subject: [PATCH 16/71] Update yscope-dev-utils --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index 5c6bfbd00..b965f19f3 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit 5c6bfbd00ad4e9cbfdfab9708298fb11bda4ae8c +Subproject commit b965f19f3e11c548f32bf7ab8e28ff38906be018 From 65841a019afaaf7aaaf56ef5b516743a482f9861 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 10:31:24 -0400 Subject: [PATCH 17/71] Add latest python lint config files --- mypy.ini | 8 ++++++++ ruff.toml | 29 +++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..4c66b1114 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true diff --git a/ruff.toml b/ruff.toml index 969421ff8..cb6ee887f 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,6 +1,27 @@ line-length = 100 -lint.select = ["I"] -[lint.isort] -case-sensitive = false -order-by-type = false \ No newline at end of file +[lint] +select = ["ALL"] +ignore = [ + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful +] +isort.order-by-type = false + +[format] +docstring-code-format = true +docstring-code-line-length = 100 From 8564fc259a54fdbe08ec27b39f33dc0ca3e4f593 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 11:03:48 -0400 Subject: [PATCH 18/71] Update ruff lint tasks --- lint-requirements.txt | 1 - lint-tasks.yaml | 16 +++++++--------- mypy.ini | 8 -------- 3 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 mypy.ini diff --git a/lint-requirements.txt b/lint-requirements.txt index 717adb508..690b9797f 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,4 +1,3 @@ -black>=24.4.2 clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 659480e50..04f8cef3a 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -111,29 +111,27 @@ tasks: cmds: - task: "py" vars: - BLACK_FLAGS: "--check" - RUFF_FLAGS: "" + RUFF_FORMAT_FLAGS: "--diff" py-fix: cmds: - task: "py" vars: - BLACK_FLAGS: "" - RUFF_FLAGS: "--fix" + RUFF_CHECK_FLAGS: "--fix" py: internal: true - requires: - vars: ["BLACK_FLAGS", "RUFF_FLAGS"] + vars: + RUFF_CHECK_FLAGS: "{{.RUFF_CHECK_FLAGS | default \"\"}}" + RUFF_FORMAT_FLAGS: "{{.RUFF_FORMAT_FLAGS | default \"\"}}" deps: ["venv"] cmds: - for: - "tests/integration" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" - cd "{{.ITEM}}" - black --color --line-length 100 {{.BLACK_FLAGS}} . - ruff check {{.RUFF_FLAGS}} . + ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" + ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" yml: aliases: diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 4c66b1114..000000000 --- a/mypy.ini +++ /dev/null @@ -1,8 +0,0 @@ -[mypy] -strict = true - -# Additional output -pretty = true -show_error_code_links = true -show_error_context = true -show_error_end = true From a6e7d29a435f66eeac173a1a28415c7d1ab3c820 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:13:55 -0400 Subject: [PATCH 19/71] Fix ruff lint --- ruff.toml | 8 ++ tests/integration/__init__.py | 1 + tests/integration/client.py | 118 +++++++++++++++++---- tests/integration/test_client.py | 27 ++++- tests/integration/test_scheduler_worker.py | 85 +++++++++++++-- tests/integration/test_signal.py | 52 ++++++--- 6 files changed, 244 insertions(+), 47 deletions(-) diff --git a/ruff.toml b/ruff.toml index cb6ee887f..a7a590d31 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,6 +2,7 @@ line-length = 100 [lint] select = ["ALL"] +extend-select = ["PT"] ignore = [ "COM812", # Redundant and conflicts with ruff format "D203", # No blank line before docstrings (D211) @@ -22,6 +23,13 @@ ignore = [ ] isort.order-by-type = false +[lint.per-file-ignores] +"tests/integration/test_*.py" = [ + "S101", # Allow use of `assert` (security warning) + "ANN001", # Ignore missing type annotations for arguments + "ANN201", # Ignore missing type annotations for parameters +] + [format] docstring-code-format = true docstring-code-line-length = 100 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index e69de29bb..4d2ed7264 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests for Spider C++ projects.""" diff --git a/tests/integration/client.py b/tests/integration/client.py index 1b658c5ac..834ae2b5e 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,7 +1,7 @@ import re import uuid from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from collections.abc import Generator import mysql.connector import pytest @@ -9,48 +9,73 @@ @dataclass class TaskInput: + """ + TaskInput represents an input to a task. + It can either be a direct value, a reference to another task's output, or a reference to data. + """ type: str - task_output: Optional[Tuple[uuid.UUID, int]] = None - value: Optional[str] = None - data_id: Optional[uuid.UUID] = None + task_output: tuple[uuid.UUID, int] | None = None + value: str | None = None + data_id: uuid.UUID | None = None @dataclass class TaskOutput: + """ + TaskOutput represents an output of a task. + It can either be a direct value or a reference to data. + """ type: str - value: Optional[str] = None - data_id: Optional[uuid.UUID] = None + value: str | None = None + data_id: uuid.UUID | None = None @dataclass class Task: + """ + Task represents a unit of work in the task graph. + """ id: uuid.UUID function_name: str - inputs: List[TaskInput] - outputs: List[TaskOutput] + inputs: list[TaskInput] + outputs: list[TaskOutput] timeout: float = 0.0 max_retries: int = 0 @dataclass class TaskGraph: + """ + TaskGraph represents a directed acyclic graph of tasks. + """ id: uuid.UUID - tasks: Dict[uuid.UUID, Task] - dependencies: List[Tuple[uuid.UUID, uuid.UUID]] + tasks: dict[uuid.UUID, Task] + dependencies: list[tuple[uuid.UUID, uuid.UUID]] @dataclass class Driver: + """ + Driver represents a client that can submit jobs to the task graph. + """ id: uuid.UUID @dataclass class Data: + """ + Data represents a Spider Data object. + """ id: uuid.UUID value: str -def create_connection(storage_url: str): +def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: + """ + Creation a MariaDB connection from a JDBC URL. + :param storage_url: JDBC URL for the MariaDB database. + :return: The created MySQL connection. + """ pattern = re.compile( r"jdbc:mariadb://(?P[^:/]+):(?P\d+)/(?P[^?]+)\?user=(?P[^&]+)&password=(?P[^&]+)" ) @@ -68,7 +93,13 @@ def create_connection(storage_url: str): ) -def is_head_task(task_id: uuid.UUID, dependencies: List[Tuple[uuid.UUID, uuid.UUID]]): +def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UUID]]) -> bool: + """ + Check if the task is a head task, meaning it has no parent. + :param task_id: the ID of the task to check. + :param dependencies: list of dependencies where each dependency is a tuple (parent_id, child_id). + :return: True if the task has no parent, False otherwise. + """ return not any(dependency[1] == task_id for dependency in dependencies) @@ -76,13 +107,25 @@ def is_head_task(task_id: uuid.UUID, dependencies: List[Tuple[uuid.UUID, uuid.UU @pytest.fixture(scope="session") -def storage(): +def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: + """ + Fixture to create a database connection for the test session. Yields a connection object + and ensures it is closed after the tests are done. + :return: + """ conn = create_connection(g_storage_url) yield conn conn.close() -def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph): +def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: + """ + Submit a job to the database. + :param conn: database connection object. + :param client_id: client ID of the driver submitting the job. + :param graph: task graph to be submitted. + :return: None + """ cursor = conn.cursor() cursor.execute( @@ -136,7 +179,13 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph): cursor.close() -def get_task_outputs(conn, task_id: uuid.UUID) -> List[TaskOutput]: +def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: + """ + Get the outputs of a task by its ID. + :param conn: database connection object. + :param task_id: the ID of the task whose outputs are to be retrieved. + :return: list of TaskOutput objects representing the outputs of the task. + """ cursor = conn.cursor() cursor.execute( @@ -158,6 +207,12 @@ def get_task_outputs(conn, task_id: uuid.UUID) -> List[TaskOutput]: def get_task_state(conn, task_id: uuid.UUID) -> str: + """ + Get the state of a task by its ID. + :param conn: database connection object. + :param task_id: the ID of the task whose state is to be retrieved. + :return: the state of the task as a string. + """ cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) @@ -168,7 +223,13 @@ def get_task_state(conn, task_id: uuid.UUID) -> str: return state -def remove_job(conn, job_id: uuid.UUID): +def remove_job(conn, job_id: uuid.UUID) -> None: + """ + Remove a job from the database by its ID. + :param conn: database connection object. + :param job_id: the ID of the job to be removed. + :return: None + """ cursor = conn.cursor() cursor.execute("DELETE FROM jobs WHERE id = %s", (job_id.bytes,)) @@ -176,7 +237,13 @@ def remove_job(conn, job_id: uuid.UUID): cursor.close() -def add_driver(conn, driver: Driver): +def add_driver(conn, driver: Driver) -> None: + """ + Register a new driver in the database. + :param conn: database connection object. + :param driver: driver object to be registered. + :return: None + """ cursor = conn.cursor() cursor.execute("INSERT INTO drivers (id) VALUES (%s)", (driver.id.bytes,)) @@ -185,7 +252,14 @@ def add_driver(conn, driver: Driver): cursor.close() -def add_driver_data(conn, driver: Driver, data: Data): +def add_driver_data(conn, driver: Driver, data: Data) -> None: + """ + Add a new data associated with a driver in the database. + :param conn: database connection object. + :param driver: driver object to which the data is associated. + :param data: data object to be added. + :return: None + """ cursor = conn.cursor() cursor.execute("INSERT INTO data (id, value) VALUES (%s, %s)", (data.id.bytes, data.value)) @@ -198,7 +272,13 @@ def add_driver_data(conn, driver: Driver, data: Data): cursor.close() -def remove_data(conn, data: Data): +def remove_data(conn, data: Data) -> None: + """ + Remove data from the database by its ID. + :param conn: database connection object. + :param data: data object to be removed. + :return: None + """ cursor = conn.cursor() cursor.execute("DELETE FROM data WHERE id = %s", (data.id.bytes,)) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 94b48afdf..7d6edeebe 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,7 +1,7 @@ import subprocess import time from pathlib import Path -from typing import Tuple +from collections.abc import Generator import pytest @@ -14,7 +14,13 @@ def start_scheduler_workers( storage_url: str, scheduler_port: int -) -> Tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: + """ + Start the scheduler and two worker processes. + :param storage_url: + :param scheduler_port: The port for the scheduler to listen on. + :return: scheduler_process, worker_process_0, worker_process_1 + """ # Start the scheduler dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." / ".." / "src" / "spider" @@ -43,7 +49,13 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -def scheduler_worker(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker() -> Generator[None, None, None]: + """ + Fixture to start the scheduler and two worker processes. Yields control to the test class, + and then kills the processes after the test class is done. + :return: + """ scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -56,7 +68,12 @@ def scheduler_worker(storage): class TestClient: - def test_client(self, scheduler_worker): + @pytest.mark.usefixtures("scheduler_worker") + def test_client(self) -> None: + """ + Test the client_test C++ program and check for successful execution. + :return: None + """ dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." client_cmds = [ @@ -64,5 +81,5 @@ def test_client(self, scheduler_worker): "--storage_url", g_storage_url, ] - p = subprocess.run(client_cmds, timeout=20) + p = subprocess.run(client_cmds, check=False, timeout=20) assert p.returncode == 0 diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 9dfc359d0..f138f9f81 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -2,11 +2,12 @@ import time import uuid from pathlib import Path -from typing import Tuple +from collections.abc import Generator import msgpack import pytest +from tests.integration.client import TaskGraph, Task from .client import ( add_driver, add_driver_data, @@ -29,7 +30,13 @@ def start_scheduler_worker( storage_url: str, scheduler_port: int -) -> Tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen, subprocess.Popen]: + """ + Start a scheduler and a worker process. + :param storage_url: JDBC storage URL + :param scheduler_port: the port for the scheduler to listen on + :return: scheduler_process, worker_process + """ # Start the scheduler dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." / ".." / "src" / "spider" @@ -57,7 +64,13 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -def scheduler_worker(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker() -> Generator[None, None, None]: + """ + Fixture to start a scheduler and a worker process. Yields control to the test function. + After the test function completes, it kills the scheduler and the worker process. + :return: + """ scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -69,7 +82,13 @@ def scheduler_worker(storage): @pytest.fixture(scope="function") -def success_job(storage): +def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: + """ + Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. + Cleans up the job after the test function completes. + :param storage: + :return: + """ parent_1 = Task( id=uuid.uuid4(), function_name="sum_test", @@ -127,7 +146,13 @@ def success_job(storage): @pytest.fixture(scope="function") -def fail_job(storage): +def fail_job(storage) -> Generator[Task, None, None]: + """ + Fixture to create a job that will fail. The task will raise an error when executed. + Yield the task. Cleanup the job after the test function completes. + :param storage: + :return: + """ task = Task( id=uuid.uuid4(), function_name="error_test", @@ -149,7 +174,13 @@ def fail_job(storage): @pytest.fixture(scope="function") -def data_job(storage): +def data_job(storage) -> Generator[Task, None, None]: + """ + Fixture to create a job that uses data. Yields the task that uses data. + Cleans up the job and data after the test function completes. + :param storage: + :return: + """ data = Data( id=uuid.uuid4(), value=msgpack.packb(2), @@ -181,6 +212,12 @@ def data_job(storage): @pytest.fixture(scope="function") def random_fail_job(storage): + """ + Fixture to create a job that randomly fails. The task will succeed after a few retries. + Yields the task. Cleans up the job after the test function completes. + :param storage: + :return: + """ data = Data( id=uuid.uuid4(), value=msgpack.packb(2), @@ -212,7 +249,14 @@ def random_fail_job(storage): class TestSchedulerWorker: - def test_job_success(self, scheduler_worker, storage, success_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_job_success(self, storage, success_job): + """ + Test the successful execution of a job with two parent tasks and one child task. + :param storage: + :param success_job: + :return: None + """ graph, parent_1, parent_2, child = success_job # Wait for 2 seconds and check task state and output time.sleep(2) @@ -232,14 +276,28 @@ def test_job_success(self, scheduler_worker, storage, success_job): assert len(outputs) == 1 assert outputs[0].value == msgpack.packb(10) - def test_job_failure(self, scheduler_worker, storage, fail_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_job_failure(self, storage, fail_job): + """ + Test the failure of a job that raise an error. + :param storage: + :param fail_job: + :return: None + """ task = fail_job # Wait for 2 seconds and check task output time.sleep(2) state = get_task_state(storage, task.id) assert state == "fail" - def test_data_job(self, scheduler_worker, storage, data_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_data_job(self, storage, data_job): + """ + Test the successful execution of a job that uses data. + :param storage: + :param data_job: + :return: None + """ task = data_job # Wait for 2 seconds and check task output time.sleep(2) @@ -249,7 +307,14 @@ def test_data_job(self, scheduler_worker, storage, data_job): assert len(outputs) == 1 assert outputs[0].value == msgpack.packb(2) - def test_random_fail_job(self, scheduler_worker, storage, random_fail_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_random_fail_job(self, storage, random_fail_job): + """ + Test the successful recovery and execution of a job that randomly fails. + :param storage: + :param random_fail_job: + :return: None + """ task = random_fail_job # Wait for 2 seconds and check task output time.sleep(2) diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index a32f1c3d8..012fac7b9 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -4,6 +4,7 @@ import time import uuid from pathlib import Path +from collections.abc import Generator import msgpack import pytest @@ -23,7 +24,14 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str): +def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> tuple[subprocess.Popen, subprocess.Popen]: + """ + Create a scheduler and a worker process. + :param storage_url: JDB storage URL. + :param scheduler_port: the port for the scheduler to listen on. + :param lib: Library to load in the worker. + :return: scheduler and worker processes. + """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -52,7 +60,12 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str): @pytest.fixture(scope="function") -def scheduler_worker_signal(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen], None, None]: + """ + Fixture to start a scheduler and a worker process for testing signal handling. + :return: + """ scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -65,12 +78,19 @@ def scheduler_worker_signal(storage): class TestWorkerSignal: - # Test that worker propagates the SIGTERM signal to the task executor. - # Submit a task that checks whether the task executor receives the SIGTERM signal. - # The task should return the SIGTERM signal number as the output. - # Later task should not be executed. - # Worker should exit with SIGTERM. - def test_task_signal(self, storage, scheduler_worker_signal): + + def test_task_signal(self, storage, scheduler_worker_signal) -> None: + """ + Test that worker propagates the SIGTERM signal to the task executor. + Submit a task that checks whether the task executor receives the SIGTERM signal. + The task should return the SIGTERM signal number as the output. + Later task should not be executed. + Worker should exit with SIGTERM. + + :param storage: + :param scheduler_worker_signal: + :return: None + """ _, worker_process = scheduler_worker_signal # Submit signal handler task to check for SIGTERM signal in task executor @@ -133,11 +153,17 @@ def test_task_signal(self, storage, scheduler_worker_signal): remove_job(storage, new_graph.id) remove_job(storage, graph.id) - # Test that worker propagates the SIGTERM signal to the task executor. - # Task executor exits immediately after receiving the signal. - # The running task should be marked as failed. - # The worker should exit with SIGTERM. - def test_task_exit(self, storage, scheduler_worker_signal): + def test_task_exit(self, storage, scheduler_worker_signal) -> None: + """ + Test that worker propagates the SIGTERM signal to the task executor. + Task executor exits immediately after receiving the signal. + The running task should be marked as failed. + The worker should exit with SIGTERM. + + :param storage: + :param scheduler_worker_signal: + :return: None + """ _, worker_process = scheduler_worker_signal # Submit a task to sleep for 10 seconds From 573b4487fc933e8cc852e90af321bcf338517a77 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:18:04 -0400 Subject: [PATCH 20/71] Fix ruff lint --- tests/integration/client.py | 8 +++++++- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 3 +-- tests/integration/test_signal.py | 6 ++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 834ae2b5e..99662b542 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,7 +1,7 @@ import re import uuid -from dataclasses import dataclass from collections.abc import Generator +from dataclasses import dataclass import mysql.connector import pytest @@ -13,6 +13,7 @@ class TaskInput: TaskInput represents an input to a task. It can either be a direct value, a reference to another task's output, or a reference to data. """ + type: str task_output: tuple[uuid.UUID, int] | None = None value: str | None = None @@ -25,6 +26,7 @@ class TaskOutput: TaskOutput represents an output of a task. It can either be a direct value or a reference to data. """ + type: str value: str | None = None data_id: uuid.UUID | None = None @@ -35,6 +37,7 @@ class Task: """ Task represents a unit of work in the task graph. """ + id: uuid.UUID function_name: str inputs: list[TaskInput] @@ -48,6 +51,7 @@ class TaskGraph: """ TaskGraph represents a directed acyclic graph of tasks. """ + id: uuid.UUID tasks: dict[uuid.UUID, Task] dependencies: list[tuple[uuid.UUID, uuid.UUID]] @@ -58,6 +62,7 @@ class Driver: """ Driver represents a client that can submit jobs to the task graph. """ + id: uuid.UUID @@ -66,6 +71,7 @@ class Data: """ Data represents a Spider Data object. """ + id: uuid.UUID value: str diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 7d6edeebe..c1d155043 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,7 +1,7 @@ import subprocess import time -from pathlib import Path from collections.abc import Generator +from pathlib import Path import pytest diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index f138f9f81..9c6ac27e4 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -1,13 +1,12 @@ import subprocess import time import uuid -from pathlib import Path from collections.abc import Generator +from pathlib import Path import msgpack import pytest -from tests.integration.client import TaskGraph, Task from .client import ( add_driver, add_driver_data, diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 012fac7b9..764bf8185 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -3,8 +3,8 @@ import subprocess import time import uuid -from pathlib import Path from collections.abc import Generator +from pathlib import Path import msgpack import pytest @@ -32,6 +32,7 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t :param lib: Library to load in the worker. :return: scheduler and worker processes. """ + root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -61,11 +62,12 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t @pytest.fixture(scope="function") @pytest.mark.usefixtures("storage") -def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: """ + scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) From 6ad72c3b93e9415b6fddd291956435fda79db005 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:34:04 -0400 Subject: [PATCH 21/71] Bug fix --- ruff.toml | 1 + tests/integration/client.py | 43 +++++++++------------- tests/integration/test_client.py | 4 ++ tests/integration/test_scheduler_worker.py | 3 ++ tests/integration/test_signal.py | 5 ++- tests/integration/utils.py | 2 + 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/ruff.toml b/ruff.toml index a7a590d31..75462c9cd 100644 --- a/ruff.toml +++ b/ruff.toml @@ -11,6 +11,7 @@ ignore = [ "D400", # First line of docstrings may not end in period "D401", # Docstrings should be written in present tense (not imperative) "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations "FBT", # Allow bool positional parameters since other value positions are allowed "FIX002", # Allow todo statements "PERF401", # Allow for loops when creating lists diff --git a/tests/integration/client.py b/tests/integration/client.py index 99662b542..682038e43 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,3 +1,5 @@ +"""Simple Spider client for testing purposes.""" + import re import uuid from collections.abc import Generator @@ -34,9 +36,7 @@ class TaskOutput: @dataclass class Task: - """ - Task represents a unit of work in the task graph. - """ + """Task represents a unit of work in the task graph.""" id: uuid.UUID function_name: str @@ -48,9 +48,7 @@ class Task: @dataclass class TaskGraph: - """ - TaskGraph represents a directed acyclic graph of tasks. - """ + """TaskGraph represents a directed acyclic graph of tasks.""" id: uuid.UUID tasks: dict[uuid.UUID, Task] @@ -59,18 +57,14 @@ class TaskGraph: @dataclass class Driver: - """ - Driver represents a client that can submit jobs to the task graph. - """ + """Driver represents a client that can submit jobs to the task graph.""" id: uuid.UUID @dataclass class Data: - """ - Data represents a Spider Data object. - """ + """Data represents a Spider Data object.""" id: uuid.UUID value: str @@ -87,7 +81,7 @@ def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: ) match = pattern.match(storage_url) if not match: - raise ValueError("Invalid JDBC URL format") + raise ValueError(storage_url) connection_params = match.groupdict() return mysql.connector.connect( @@ -103,7 +97,8 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU """ Check if the task is a head task, meaning it has no parent. :param task_id: the ID of the task to check. - :param dependencies: list of dependencies where each dependency is a tuple (parent_id, child_id). + :param dependencies: list of dependencies where each dependency is a tuple + (parent_id, child_id). :return: True if the task has no parent, False otherwise. """ return not any(dependency[1] == task_id for dependency in dependencies) @@ -124,7 +119,8 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: conn.close() -def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: +def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph)\ + -> None: """ Submit a job to the database. :param conn: database connection object. @@ -139,10 +135,7 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: ) for task_id, task in graph.tasks.items(): - if is_head_task(task_id, graph.dependencies): - state = "ready" - else: - state = "pending" + state = "ready" if is_head_task(task_id, graph.dependencies) else "pending" cursor.execute( "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry) VALUES (%s, %s, %s, %s, %s, %s)", ( @@ -185,7 +178,7 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: cursor.close() -def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: +def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: """ Get the outputs of a task by its ID. :param conn: database connection object. @@ -212,7 +205,7 @@ def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: return outputs -def get_task_state(conn, task_id: uuid.UUID) -> str: +def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> str: """ Get the state of a task by its ID. :param conn: database connection object. @@ -229,7 +222,7 @@ def get_task_state(conn, task_id: uuid.UUID) -> str: return state -def remove_job(conn, job_id: uuid.UUID) -> None: +def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None: """ Remove a job from the database by its ID. :param conn: database connection object. @@ -243,7 +236,7 @@ def remove_job(conn, job_id: uuid.UUID) -> None: cursor.close() -def add_driver(conn, driver: Driver) -> None: +def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: """ Register a new driver in the database. :param conn: database connection object. @@ -258,7 +251,7 @@ def add_driver(conn, driver: Driver) -> None: cursor.close() -def add_driver_data(conn, driver: Driver, data: Data) -> None: +def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: Data) -> None: """ Add a new data associated with a driver in the database. :param conn: database connection object. @@ -278,7 +271,7 @@ def add_driver_data(conn, driver: Driver, data: Data) -> None: cursor.close() -def remove_data(conn, data: Data) -> None: +def remove_data(conn: mysql.connector.MySQLConnection, data: Data) -> None: """ Remove data from the database by its ID. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index c1d155043..6fbbd6ad8 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,3 +1,5 @@ +"""Integration test for the client_test C++ program.""" + import subprocess import time from collections.abc import Generator @@ -68,6 +70,8 @@ def scheduler_worker() -> Generator[None, None, None]: class TestClient: + """Test class for the client_test C++ program.""" + @pytest.mark.usefixtures("scheduler_worker") def test_client(self) -> None: """ diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 9c6ac27e4..93f2cde2a 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -1,3 +1,5 @@ +"""Integration tests for the scheduler and worker processes.""" + import subprocess import time import uuid @@ -248,6 +250,7 @@ def random_fail_job(storage): class TestSchedulerWorker: + """Test class for the scheduler and worker integration tests.""" @pytest.mark.usefixtures("scheduler_worker") def test_job_success(self, storage, success_job): """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 764bf8185..c9605b898 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -1,3 +1,5 @@ +"""Integration tests for worker signal handling.""" + import os import signal import subprocess @@ -32,7 +34,6 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t :param lib: Library to load in the worker. :return: scheduler and worker processes. """ - root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -67,7 +68,6 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po Fixture to start a scheduler and a worker process for testing signal handling. :return: """ - scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -79,6 +79,7 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po class TestWorkerSignal: + """Test cases for worker signal handling.""" def test_task_signal(self, storage, scheduler_worker_signal) -> None: diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 33aada438..79e4bf950 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -1,3 +1,5 @@ +"""Utilities for the network port.""" + import socket From ba7c6e58e9d6936149bb1d5ba397e0c1c076ec2c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:57:00 -0400 Subject: [PATCH 22/71] Fix ruff --- ruff.toml | 4 +++- tests/integration/client.py | 6 ++++-- tests/integration/test_client.py | 5 ++--- tests/integration/test_scheduler_worker.py | 16 ++++++++-------- tests/integration/test_signal.py | 9 ++++----- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/ruff.toml b/ruff.toml index 75462c9cd..d2294f8d0 100644 --- a/ruff.toml +++ b/ruff.toml @@ -26,9 +26,11 @@ isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "S101", # Allow use of `assert` (security warning) "ANN001", # Ignore missing type annotations for arguments "ANN201", # Ignore missing type annotations for parameters + "S101", # Allow use of `assert` (security warning) + "S603", # Allow user of subprocess.Popen (security warning) + "T201", # Allow use of `print` (testing) ] [format] diff --git a/tests/integration/client.py b/tests/integration/client.py index 682038e43..4ed1258dd 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -137,7 +137,8 @@ def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, grap for task_id, task in graph.tasks.items(): state = "ready" if is_head_task(task_id, graph.dependencies) else "pending" cursor.execute( - "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry) VALUES (%s, %s, %s, %s, %s, %s)", + "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry)" + " VALUES (%s, %s, %s, %s, %s, %s)", ( task.id.bytes, graph.id.bytes, @@ -150,7 +151,8 @@ def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, grap for i, task_input in enumerate(task.inputs): cursor.execute( - "INSERT INTO task_inputs (type, task_id, position, output_task_id, output_task_position, value, data_id) VALUES (%s, %s, %s, %s, %s, %s, %s)", + "INSERT INTO task_inputs (type, task_id, position, output_task_id," + " output_task_position, value, data_id) VALUES (%s, %s, %s, %s, %s, %s, %s)", ( task_input.type, task.id.bytes, diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 6fbbd6ad8..a00dc8a6f 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -51,8 +51,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -@pytest.mark.usefixtures("storage") -def scheduler_worker() -> Generator[None, None, None]: +def scheduler_worker(storage) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, and then kills the processes after the test class is done. @@ -85,5 +84,5 @@ def test_client(self) -> None: "--storage_url", g_storage_url, ] - p = subprocess.run(client_cmds, check=False, timeout=20) + p = subprocess.run(client_cmds, check=True, timeout=20) assert p.returncode == 0 diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 93f2cde2a..1a2f88877 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -65,8 +65,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -@pytest.mark.usefixtures("storage") -def scheduler_worker() -> Generator[None, None, None]: +def scheduler_worker(storage) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. After the test function completes, it kills the scheduler and the worker process. @@ -82,11 +81,11 @@ def scheduler_worker() -> Generator[None, None, None]: worker_process.kill() -@pytest.fixture(scope="function") +@pytest.fixture def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ - Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. - Cleans up the job after the test function completes. + Fixture to create a job with two parent tasks and one child task. Yields the task graph and + tasks. Cleans up the job after the test function completes. :param storage: :return: """ @@ -146,7 +145,7 @@ def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, remove_job(storage, graph.id) -@pytest.fixture(scope="function") +@pytest.fixture def fail_job(storage) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -174,7 +173,7 @@ def fail_job(storage) -> Generator[Task, None, None]: remove_job(storage, graph.id) -@pytest.fixture(scope="function") +@pytest.fixture def data_job(storage) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -211,7 +210,7 @@ def data_job(storage) -> Generator[Task, None, None]: remove_data(storage, data) -@pytest.fixture(scope="function") +@pytest.fixture def random_fail_job(storage): """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -251,6 +250,7 @@ def random_fail_job(storage): class TestSchedulerWorker: """Test class for the scheduler and worker integration tests.""" + @pytest.mark.usefixtures("scheduler_worker") def test_job_success(self, storage, success_job): """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index c9605b898..0edb71645 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,7 +26,8 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> tuple[subprocess.Popen, subprocess.Popen]: +def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ + -> tuple[subprocess.Popen, subprocess.Popen]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -61,9 +62,8 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t return scheduler_process, worker_process -@pytest.fixture(scope="function") -@pytest.mark.usefixtures("storage") -def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +@pytest.fixture +def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -81,7 +81,6 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po class TestWorkerSignal: """Test cases for worker signal handling.""" - def test_task_signal(self, storage, scheduler_worker_signal) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From 209acb14a38a3f13c2622f2e9c0a210115f5038a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 14:56:59 -0400 Subject: [PATCH 23/71] Fix ruff --- ruff.toml | 2 - tests/integration/client.py | 7 +++- tests/integration/test_client.py | 7 +++- tests/integration/test_scheduler_worker.py | 47 +++++++++++++++++----- tests/integration/test_signal.py | 22 +++++++--- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/ruff.toml b/ruff.toml index d2294f8d0..6cdab7927 100644 --- a/ruff.toml +++ b/ruff.toml @@ -26,8 +26,6 @@ isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "ANN001", # Ignore missing type annotations for arguments - "ANN201", # Ignore missing type annotations for parameters "S101", # Allow use of `assert` (security warning) "S603", # Allow user of subprocess.Popen (security warning) "T201", # Allow use of `print` (testing) diff --git a/tests/integration/client.py b/tests/integration/client.py index 4ed1258dd..886a6c3c3 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -119,8 +119,11 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: conn.close() -def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph)\ - -> None: +def submit_job( + conn: mysql.connector.MySQLConnection, + client_id: uuid.UUID, + graph: TaskGraph +) -> None: """ Submit a job to the database. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index a00dc8a6f..aac10a9ce 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -5,11 +5,11 @@ from collections.abc import Generator from pathlib import Path +import mysql.connector import pytest from .client import ( g_storage_url, - storage, ) from .utils import g_scheduler_port @@ -51,12 +51,15 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -def scheduler_worker(storage) -> Generator[None, None, None]: +def scheduler_worker( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, and then kills the processes after the test class is done. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 1a2f88877..84293fc42 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -7,6 +7,7 @@ from pathlib import Path import msgpack +import mysql.connector import pytest from .client import ( @@ -19,7 +20,6 @@ get_task_state, remove_data, remove_job, - storage, submit_job, Task, TaskGraph, @@ -65,12 +65,15 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -def scheduler_worker(storage) -> Generator[None, None, None]: +def scheduler_worker( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. After the test function completes, it kills the scheduler and the worker process. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -82,7 +85,9 @@ def scheduler_worker(storage) -> Generator[None, None, None]: @pytest.fixture -def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: +def success_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +)-> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. Cleans up the job after the test function completes. @@ -146,7 +151,9 @@ def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, @pytest.fixture -def fail_job(storage) -> Generator[Task, None, None]: +def fail_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. Yield the task. Cleanup the job after the test function completes. @@ -174,7 +181,9 @@ def fail_job(storage) -> Generator[Task, None, None]: @pytest.fixture -def data_job(storage) -> Generator[Task, None, None]: +def data_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. Cleans up the job and data after the test function completes. @@ -211,7 +220,9 @@ def data_job(storage) -> Generator[Task, None, None]: @pytest.fixture -def random_fail_job(storage): +def random_fail_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. Yields the task. Cleans up the job after the test function completes. @@ -252,7 +263,11 @@ class TestSchedulerWorker: """Test class for the scheduler and worker integration tests.""" @pytest.mark.usefixtures("scheduler_worker") - def test_job_success(self, storage, success_job): + def test_job_success( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None] + ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. :param storage: @@ -279,7 +294,11 @@ def test_job_success(self, storage, success_job): assert outputs[0].value == msgpack.packb(10) @pytest.mark.usefixtures("scheduler_worker") - def test_job_failure(self, storage, fail_job): + def test_job_failure( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + fail_job: Generator[Task, None, None] + ) -> None: """ Test the failure of a job that raise an error. :param storage: @@ -293,7 +312,11 @@ def test_job_failure(self, storage, fail_job): assert state == "fail" @pytest.mark.usefixtures("scheduler_worker") - def test_data_job(self, storage, data_job): + def test_data_job( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + data_job: Generator[Task, None, None] + ) -> None: """ Test the successful execution of a job that uses data. :param storage: @@ -310,7 +333,11 @@ def test_data_job(self, storage, data_job): assert outputs[0].value == msgpack.packb(2) @pytest.mark.usefixtures("scheduler_worker") - def test_random_fail_job(self, storage, random_fail_job): + def test_random_fail_job( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + random_fail_job: Generator[Task, None, None] + ) -> None: """ Test the successful recovery and execution of a job that randomly fails. :param storage: diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 0edb71645..0da6466a4 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -9,6 +9,7 @@ from pathlib import Path import msgpack +import mysql.connector import pytest from .client import ( @@ -16,7 +17,6 @@ get_task_outputs, get_task_state, remove_job, - storage, submit_job, Task, TaskGraph, @@ -37,7 +37,7 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" - popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + popen_opts = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True} scheduler_cmds = [ str(bin_dir / "spider_scheduler"), "--host", @@ -63,11 +63,13 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ @pytest.fixture -def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal(storage: Generator[mysql.connector.MySQLConnection, None, None])\ + -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -81,7 +83,12 @@ def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subpro class TestWorkerSignal: """Test cases for worker signal handling.""" - def test_task_signal(self, storage, scheduler_worker_signal) -> None: + def test_task_signal( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[ + tuple[subprocess.Popen, subprocess.Popen], None, None] + ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. Submit a task that checks whether the task executor receives the SIGTERM signal. @@ -155,7 +162,12 @@ def test_task_signal(self, storage, scheduler_worker_signal) -> None: remove_job(storage, new_graph.id) remove_job(storage, graph.id) - def test_task_exit(self, storage, scheduler_worker_signal) -> None: + def test_task_exit( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[ + tuple[subprocess.Popen, subprocess.Popen], None, None] + ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. Task executor exits immediately after receiving the signal. From 406b514007a05e9701ddcacd0638dd5d20ad4e0e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 14:57:52 -0400 Subject: [PATCH 24/71] Reformat files --- tests/integration/client.py | 4 +-- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 36 +++++++++++----------- tests/integration/test_signal.py | 24 +++++++-------- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 886a6c3c3..721ec2444 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -120,9 +120,7 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: def submit_job( - conn: mysql.connector.MySQLConnection, - client_id: uuid.UUID, - graph: TaskGraph + conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph ) -> None: """ Submit a job to the database. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index aac10a9ce..f70f51987 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 84293fc42..2f0421c05 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,8 +86,8 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] -)-> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: + storage: Generator[mysql.connector.MySQLConnection, None, None], +) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. Cleans up the job after the test function completes. @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -264,9 +264,9 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. @@ -295,9 +295,9 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - fail_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + fail_job: Generator[Task, None, None], ) -> None: """ Test the failure of a job that raise an error. @@ -313,9 +313,9 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - data_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + data_job: Generator[Task, None, None], ) -> None: """ Test the successful execution of a job that uses data. @@ -334,9 +334,9 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - random_fail_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + random_fail_job: Generator[Task, None, None], ) -> None: """ Test the successful recovery and execution of a job that randomly fails. diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 0da6466a4..28b275346 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,8 +26,9 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ - -> tuple[subprocess.Popen, subprocess.Popen]: +def start_scheduler_worker( + storage_url: str, scheduler_port: int, lib: str +) -> tuple[subprocess.Popen, subprocess.Popen]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -63,8 +64,9 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ @pytest.fixture -def scheduler_worker_signal(storage: Generator[mysql.connector.MySQLConnection, None, None])\ - -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal( + storage: Generator[mysql.connector.MySQLConnection, None, None], +) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -84,10 +86,9 @@ class TestWorkerSignal: """Test cases for worker signal handling.""" def test_task_signal( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - scheduler_worker_signal: Generator[ - tuple[subprocess.Popen, subprocess.Popen], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. @@ -163,10 +164,9 @@ def test_task_signal( remove_job(storage, graph.id) def test_task_exit( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - scheduler_worker_signal: Generator[ - tuple[subprocess.Popen, subprocess.Popen], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From 66892f5e5194490d9dd8af79ecb9db2ac63740fe Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 16:08:45 -0400 Subject: [PATCH 25/71] Remove .inc from cpp linting --- lint-tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 659480e50..161c8a193 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -101,7 +101,7 @@ tasks: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_EXAMPLES_COMPILE_COMMANDS_DB}}'" - INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" ROOT_PATHS: - "{{.G_EXAMPLES_DIR}}" From 3186a429f9a79ab93750deaa47c3d542a3aaa0af Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 18:41:52 -0400 Subject: [PATCH 26/71] Add mypy and merge lint and test requirements.txt --- lint-tasks.yaml | 5 +++-- lint-requirements.txt => lint-test-requirements.txt | 7 +++++++ mypy.ini | 8 ++++++++ test-requirements.txt | 3 --- test-tasks.yaml | 4 ++-- 5 files changed, 20 insertions(+), 7 deletions(-) rename lint-requirements.txt => lint-test-requirements.txt (57%) create mode 100644 mypy.ini delete mode 100644 test-requirements.txt diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 7822bda7f..3739fa12d 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -130,6 +130,7 @@ tasks: - "tests/integration" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" + mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" @@ -177,7 +178,7 @@ tasks: sources: - "{{.ROOT_DIR}}/taskfile.yaml" - "{{.TASKFILE}}" - - "lint-requirements.txt" + - "lint-test-requirements.txt" generates: ["{{.CHECKSUM_FILE}}"] run: "once" deps: @@ -191,7 +192,7 @@ tasks: vars: LABEL: "lint" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-test-requirements.txt" # This command must be last - task: ":utils:checksum:compute" vars: diff --git a/lint-requirements.txt b/lint-test-requirements.txt similarity index 57% rename from lint-requirements.txt rename to lint-test-requirements.txt index 690b9797f..7ac75af21 100644 --- a/lint-requirements.txt +++ b/lint-test-requirements.txt @@ -1,6 +1,13 @@ +# Lint dependencies clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 ruff>=0.4.4 +mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 + +# Test dependencies +msgpack>=1.1.0 +mysql-connector-python>=8.0.26 +pytest>=8.3.4 diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..4c66b1114 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index ec24b0b20..000000000 --- a/test-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -msgpack>=1.1.0 -mysql-connector-python>=8.0.26 -pytest>=8.3.4 \ No newline at end of file diff --git a/test-tasks.yaml b/test-tasks.yaml index 116ae85b6..41dfc4578 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -56,7 +56,7 @@ tasks: sources: - "{{.ROOT_DIR}}/taskfile.yaml" - "{{.TASKFILE}}" - - "test-requirements.txt" + - "lint-test-requirements.txt" generates: ["{{.CHECKSUM_FILE}}"] run: "once" deps: @@ -70,7 +70,7 @@ tasks: vars: LABEL: "test" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "{{.ROOT_DIR}}/test-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-test-requirements.txt" # This command must be last - task: ":utils:checksum:compute" vars: From 092593844041d72e6da23797a7c8fb16fbf1d258 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:34:15 -0400 Subject: [PATCH 27/71] Fix mysql connection type --- tests/integration/client.py | 19 ++++++++++--------- tests/integration/test_client.py | 4 ++-- tests/integration/test_scheduler_worker.py | 20 ++++++++++---------- tests/integration/test_signal.py | 8 ++++---- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 721ec2444..f0b7f0cfe 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -69,8 +69,9 @@ class Data: id: uuid.UUID value: str +SQLConnection = mysql.connector.abstracts.MySQLConnectionAbstract | mysql.connector.pooling.PooledMySQLConnection -def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: +def create_connection(storage_url: str) -> SQLConnection : """ Creation a MariaDB connection from a JDBC URL. :param storage_url: JDBC URL for the MariaDB database. @@ -108,7 +109,7 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU @pytest.fixture(scope="session") -def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: +def storage() -> Generator[SQLConnection, None, None]: """ Fixture to create a database connection for the test session. Yields a connection object and ensures it is closed after the tests are done. @@ -120,7 +121,7 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: def submit_job( - conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph + conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph ) -> None: """ Submit a job to the database. @@ -181,7 +182,7 @@ def submit_job( cursor.close() -def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: +def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: """ Get the outputs of a task by its ID. :param conn: database connection object. @@ -208,7 +209,7 @@ def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) return outputs -def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> str: +def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: """ Get the state of a task by its ID. :param conn: database connection object. @@ -225,7 +226,7 @@ def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> return state -def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None: +def remove_job(conn: SQLConnection, job_id: uuid.UUID) -> None: """ Remove a job from the database by its ID. :param conn: database connection object. @@ -239,7 +240,7 @@ def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None cursor.close() -def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: +def add_driver(conn: SQLConnection, driver: Driver) -> None: """ Register a new driver in the database. :param conn: database connection object. @@ -254,7 +255,7 @@ def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: cursor.close() -def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: Data) -> None: +def add_driver_data(conn: SQLConnection, driver: Driver, data: Data) -> None: """ Add a new data associated with a driver in the database. :param conn: database connection object. @@ -274,7 +275,7 @@ def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: cursor.close() -def remove_data(conn: mysql.connector.MySQLConnection, data: Data) -> None: +def remove_data(conn: SQLConnection, data: Data) -> None: """ Remove data from the database by its ID. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index f70f51987..809085924 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -5,11 +5,11 @@ from collections.abc import Generator from pathlib import Path -import mysql.connector import pytest from .client import ( g_storage_url, + SQLConnection ) from .utils import g_scheduler_port @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 2f0421c05..190875688 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -7,7 +7,6 @@ from pathlib import Path import msgpack -import mysql.connector import pytest from .client import ( @@ -20,6 +19,7 @@ get_task_state, remove_data, remove_job, + SQLConnection, submit_job, Task, TaskGraph, @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,7 +86,7 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -265,7 +265,7 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], ) -> None: """ @@ -296,7 +296,7 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], fail_job: Generator[Task, None, None], ) -> None: """ @@ -314,7 +314,7 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], data_job: Generator[Task, None, None], ) -> None: """ @@ -335,7 +335,7 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], random_fail_job: Generator[Task, None, None], ) -> None: """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 28b275346..42eee6bac 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -9,7 +9,6 @@ from pathlib import Path import msgpack -import mysql.connector import pytest from .client import ( @@ -17,6 +16,7 @@ get_task_outputs, get_task_state, remove_job, + SQLConnection, submit_job, Task, TaskGraph, @@ -65,7 +65,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. @@ -87,7 +87,7 @@ class TestWorkerSignal: def test_task_signal( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ @@ -165,7 +165,7 @@ def test_task_signal( def test_task_exit( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ From 6936a9b448a6ce4ececbc97d53e23b67569850e4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:39:24 -0400 Subject: [PATCH 28/71] Fix socket name type --- tests/integration/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 79e4bf950..cae3f1730 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,12 +2,16 @@ import socket +IPv4Addr = tuple[str, int] +IPv6Addr = tuple[str, int, int, int] +AddrType = IPv4Addr | IPv6Addr def _get_free_tcp_port() -> int: """Returns a free TCP port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("127.0.0.1", 0)) - return s.getsockname()[1] + addr: AddrType = s.getsockname() + return addr[1] g_scheduler_port = _get_free_tcp_port() From 74070908318632b16e7dde6800994676d19e8982 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:43:07 -0400 Subject: [PATCH 29/71] Fix return type from db cursor --- tests/integration/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index f0b7f0cfe..b4369fe3a 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -196,7 +196,8 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - for output_type, value, data_id in cursor.fetchall(): + rows: list[tuple[str, str | None, bytes | None]] = cursor.fetchall() + for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) elif data_id is not None: @@ -219,7 +220,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state = cursor.fetchone()[0] + state: str = cursor.fetchone()[0] conn.commit() cursor.close() From 6a95b24912c69807c9cc66172f97a5441beea8e6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:50:34 -0400 Subject: [PATCH 30/71] Fix db cursor return type --- tests/integration/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index b4369fe3a..13f9666a9 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -4,6 +4,7 @@ import uuid from collections.abc import Generator from dataclasses import dataclass +from typing import cast import mysql.connector import pytest @@ -196,7 +197,7 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - rows: list[tuple[str, str | None, bytes | None]] = cursor.fetchall() + rows = cast(list[tuple[str, str | None, bytes | None]], cursor.fetchall()) for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) @@ -220,7 +221,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state: str = cursor.fetchone()[0] + state = cast(tuple[str], cursor.fetchone())[0] conn.commit() cursor.close() From f29dcba479f5f80d66c99aa94fbf7ab93ecfc4c2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:55:24 -0400 Subject: [PATCH 31/71] Fix mypy import untyped --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 4c66b1114..932c4e402 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,3 +6,6 @@ pretty = true show_error_code_links = true show_error_context = true show_error_end = true + +[mypy-msgpack.*] +ignore_missing_imports = true From 453f20908e8f566d7561e1a3c178c713e206e915 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:17:20 -0400 Subject: [PATCH 32/71] Fix mypy and Popen --- tests/integration/test_signal.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 42eee6bac..47848f636 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -7,6 +7,7 @@ import uuid from collections.abc import Generator from pathlib import Path +from typing import TypedDict import msgpack import pytest @@ -25,10 +26,16 @@ ) from .utils import g_scheduler_port +class PopenOpts(TypedDict, total=False): + stdin: int + stdout: int + stderr: int + text: bool + def start_scheduler_worker( storage_url: str, scheduler_port: int, lib: str -) -> tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -38,7 +45,11 @@ def start_scheduler_worker( """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" - popen_opts = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True} + popen_opts: PopenOpts = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "text": True, + } scheduler_cmds = [ str(bin_dir / "spider_scheduler"), "--host", @@ -66,7 +77,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( storage: Generator[SQLConnection, None, None], -) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +) -> Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -88,7 +99,7 @@ class TestWorkerSignal: def test_task_signal( self, storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From d1011c70eb908fad0247f84d845be67c89d7bbd4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:22:31 -0400 Subject: [PATCH 33/71] Fix generator type hint --- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 26 +++++++++++----------- tests/integration/test_signal.py | 10 ++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 809085924..c96b731df 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 190875688..2aa59ed50 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,7 +86,7 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -265,8 +265,8 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( self, - storage: Generator[SQLConnection, None, None], - success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], + storage: SQLConnection, + success_job: tuple[TaskGraph, Task, Task, Task], ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. @@ -296,8 +296,8 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( self, - storage: Generator[SQLConnection, None, None], - fail_job: Generator[Task, None, None], + storage: SQLConnection, + fail_job: Task, ) -> None: """ Test the failure of a job that raise an error. @@ -314,8 +314,8 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( self, - storage: Generator[SQLConnection, None, None], - data_job: Generator[Task, None, None], + storage: SQLConnection, + data_job: Task, ) -> None: """ Test the successful execution of a job that uses data. @@ -335,8 +335,8 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( self, - storage: Generator[SQLConnection, None, None], - random_fail_job: Generator[Task, None, None], + storage: SQLConnection, + random_fail_job: Task, ) -> None: """ Test the successful recovery and execution of a job that randomly fails. diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 47848f636..667e23d51 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -76,7 +76,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. @@ -98,8 +98,8 @@ class TestWorkerSignal: def test_task_signal( self, - storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None], + storage: SQLConnection, + scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. @@ -176,8 +176,8 @@ def test_task_signal( def test_task_exit( self, - storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], + storage: SQLConnection, + scheduler_worker_signal: tuple[subprocess.Popen, subprocess.Popen], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From a7dc642bcb68e83f52152b27bd811f358843ad2e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:24:47 -0400 Subject: [PATCH 34/71] Fix mypy --- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 2 +- tests/integration/test_signal.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index c96b731df..e5219835c 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -16,7 +16,7 @@ def start_scheduler_workers( storage_url: str, scheduler_port: int -) -> tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Start the scheduler and two worker processes. :param storage_url: diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 2aa59ed50..ae841332e 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -31,7 +31,7 @@ def start_scheduler_worker( storage_url: str, scheduler_port: int -) -> tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Start a scheduler and a worker process. :param storage_url: JDBC storage URL diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 667e23d51..f42fdefc3 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -177,7 +177,7 @@ def test_task_signal( def test_task_exit( self, storage: SQLConnection, - scheduler_worker_signal: tuple[subprocess.Popen, subprocess.Popen], + scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From a7d92c2255faf5c3b336ca90069aecec7d70b600 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:28:12 -0400 Subject: [PATCH 35/71] Fix ruff --- tests/integration/client.py | 17 ++++++++++------- tests/integration/test_client.py | 5 +---- tests/integration/test_signal.py | 3 +++ tests/integration/utils.py | 1 + 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 13f9666a9..b1e5a30ea 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -70,9 +70,14 @@ class Data: id: uuid.UUID value: str -SQLConnection = mysql.connector.abstracts.MySQLConnectionAbstract | mysql.connector.pooling.PooledMySQLConnection -def create_connection(storage_url: str) -> SQLConnection : +SQLConnection = ( + mysql.connector.abstracts.MySQLConnectionAbstract + | mysql.connector.pooling.PooledMySQLConnection +) + + +def create_connection(storage_url: str) -> SQLConnection: """ Creation a MariaDB connection from a JDBC URL. :param storage_url: JDBC URL for the MariaDB database. @@ -121,9 +126,7 @@ def storage() -> Generator[SQLConnection, None, None]: conn.close() -def submit_job( - conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph -) -> None: +def submit_job(conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph) -> None: """ Submit a job to the database. :param conn: database connection object. @@ -197,7 +200,7 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - rows = cast(list[tuple[str, str | None, bytes | None]], cursor.fetchall()) + rows = cast("list[tuple[str, str | None, bytes | None]]", cursor.fetchall()) for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) @@ -221,7 +224,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state = cast(tuple[str], cursor.fetchone())[0] + state = cast("tuple[str]", cursor.fetchone())[0] conn.commit() cursor.close() diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index e5219835c..2b2ca00a3 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -7,10 +7,7 @@ import pytest -from .client import ( - g_storage_url, - SQLConnection -) +from .client import g_storage_url, SQLConnection from .utils import g_scheduler_port diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index f42fdefc3..89191621c 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,7 +26,10 @@ ) from .utils import g_scheduler_port + class PopenOpts(TypedDict, total=False): + """Options for subprocess.Popen.""" + stdin: int stdout: int stderr: int diff --git a/tests/integration/utils.py b/tests/integration/utils.py index cae3f1730..7f1635390 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -6,6 +6,7 @@ IPv6Addr = tuple[str, int, int, int] AddrType = IPv4Addr | IPv6Addr + def _get_free_tcp_port() -> int: """Returns a free TCP port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: From 1c6213bb4a8d5d229bfacc89f84020c9463f2857 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:40:58 -0400 Subject: [PATCH 36/71] Simply socket return types. Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- tests/integration/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 7f1635390..64c134cec 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,9 +2,7 @@ import socket -IPv4Addr = tuple[str, int] -IPv6Addr = tuple[str, int, int, int] -AddrType = IPv4Addr | IPv6Addr +AddrType = tuple[str, int] def _get_free_tcp_port() -> int: From 55a727ae8012f724a95d25360079b76ad1353ae5 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 23:41:32 -0400 Subject: [PATCH 37/71] Add tombi lint tasks --- lint-tasks.yaml | 24 +++++++++++++++++++++++ lint-test-requirements.txt | 1 + ruff.toml | 40 +++++++++++++++++++------------------- tombi.toml | 34 ++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 tombi.toml diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 2bfe90649..40259b219 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -11,12 +11,14 @@ tasks: - task: "cpp-check" - task: "py-check" - task: "yml-check" + - task: "toml-check" fix: cmds: - task: "cmake-fix" - task: "cpp-fix" - task: "yml-fix" + - task: "toml-fix" cmake-check: deps: ["venv"] @@ -154,6 +156,28 @@ tasks: taskfile.yaml \ test-tasks.yaml + toml: + aliases: + - "toml-check" + - "toml-fix" + cmds: + - task: "toml-lint" + - task: "toml-format" + + toml-lint: + deps: ["venv"] + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + tombi lint "{{.ROOT_DIR}}" + + toml-format: + deps: ["venv"] + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + tombi format "{{.ROOT_DIR}}" + cmake: internal: true requires: diff --git a/lint-test-requirements.txt b/lint-test-requirements.txt index 7ac75af21..879c198b3 100644 --- a/lint-test-requirements.txt +++ b/lint-test-requirements.txt @@ -6,6 +6,7 @@ ruff>=0.4.4 mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 +tombi>=0.4.0 # Test dependencies msgpack>=1.1.0 diff --git a/ruff.toml b/ruff.toml index 6cdab7927..fde4359e1 100644 --- a/ruff.toml +++ b/ruff.toml @@ -4,31 +4,31 @@ line-length = 100 select = ["ALL"] extend-select = ["PT"] ignore = [ - "COM812", # Redundant and conflicts with ruff format - "D203", # No blank line before docstrings (D211) - "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists - "D212", # Enforce docstring summary line on the next line after quotes (D213) - "D400", # First line of docstrings may not end in period - "D401", # Docstrings should be written in present tense (not imperative) - "D415", # First line of docstrings may not end in a period, question mark, or exclamation point - "FA102", # Allow use of PEP 604 union in type annotations - "FBT", # Allow bool positional parameters since other value positions are allowed - "FIX002", # Allow todo statements - "PERF401", # Allow for loops when creating lists - "PERF403", # Allow for loops when creating dicts - "S311", # Allow usage of `random` package - "SIM102", # Allow collapsible if statements for readability - "TD002", # Author unnecessary for todo statement - "TD003", # Issue link unnecessary for todo statement - "UP015", # Explicit open modes are helpful + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful ] isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "S101", # Allow use of `assert` (security warning) - "S603", # Allow user of subprocess.Popen (security warning) - "T201", # Allow use of `print` (testing) + "S101", # Allow use of `assert` (security warning) + "S603", # Allow user of subprocess.Popen (security warning) + "T201", # Allow use of `print` (testing) ] [format] diff --git a/tombi.toml b/tombi.toml new file mode 100644 index 000000000..ad86d3c0c --- /dev/null +++ b/tombi.toml @@ -0,0 +1,34 @@ +toml-version = "v1.0.0" + +[files] +include = ["**/*.toml"] +exclude = ["tools/yscope-dev-utils/**", "build/**"] + +[format] + +[lint] +[lint.rules] +dotted-keys-out-of-order = "warn" +key-empty = "warn" +tables-out-of-order = "warn" + +[schema] +enabled = true +catalog = { + paths = [ + "tombi://json.schemastore.org/api/json/catalog.json", + "https://json.schemastore.org/api/json/catalog.json", + ], +} + +[[schemas]] +path = "tombi://json.schemastore.org/tombi.json" +include = ["tombi.toml"] + +[[schemas]] +path = "https://json.schemastore.org/pyproject.json" +include = ["pyproject.toml"] + +[[schemas]] +path = "https://json.schemastore.org/ruff.json" +include = ["ruff.toml"] From c98fea18e1a3f4134067c8e5648f970bdaba6fe2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 10:07:59 -0400 Subject: [PATCH 38/71] Rename cpp build tasks --- build-tasks.yaml | 4 ++-- test-tasks.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build-tasks.yaml b/build-tasks.yaml index 666576f26..53828027c 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -1,7 +1,7 @@ version: "3" tasks: - target: + cpp-target: internal: true vars: TARGETS: @@ -14,7 +14,7 @@ tasks: --parallel {{numCPU}} --target {{range .TARGETS}}{{.}} {{end}} - clean: + cpp-clean: internal: true deps: [":config-cmake-project"] cmds: diff --git a/test-tasks.yaml b/test-tasks.yaml index 41dfc4578..58a1ff190 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -27,7 +27,7 @@ tasks: build-unit-test: internal: true deps: - - task: ":build:target" + - task: ":build:cpp-target" vars: TARGETS: ["spider_task_executor", "unitTest", "worker_test"] @@ -35,7 +35,7 @@ tasks: dir: "{{.G_BUILD_SPIDER_DIR}}" deps: - "venv" - - task: ":build:target" + - task: ":build:cpp-target" vars: TARGETS: [ "spider_task_executor", From 4e7237ea5775299f94728d674c4675d9b5dae6b2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 14:54:01 -0400 Subject: [PATCH 39/71] Add basic python structure --- python/.python-version | 1 + python/README.md | 0 python/pyproject.toml | 72 +++++++ python/src/__init__.py | 0 python/src/client/__init__.py | 0 python/src/core/__init__.py | 0 python/src/core/task.py | 0 python/src/task_executor/__init__.py | 0 python/src/task_executor/task_executor.py | 6 + python/uv.lock | 242 ++++++++++++++++++++++ 10 files changed, 321 insertions(+) create mode 100644 python/.python-version create mode 100644 python/README.md create mode 100644 python/pyproject.toml create mode 100644 python/src/__init__.py create mode 100644 python/src/client/__init__.py create mode 100644 python/src/core/__init__.py create mode 100644 python/src/core/task.py create mode 100644 python/src/task_executor/__init__.py create mode 100644 python/src/task_executor/task_executor.py create mode 100644 python/uv.lock diff --git a/python/.python-version b/python/.python-version new file mode 100644 index 000000000..24ee5b1be --- /dev/null +++ b/python/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 000000000..c8d875809 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,72 @@ +[project] +name = "spider" +version = "0.0.1" +description = "Spider is a distributed task execution framework" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "mariadb>=1.1.13", + "msgpack>=1.1.1", +] + +[project.scritps] +spider_task_executor = "src.task_executor.task_executor:main" + +[build-system] +requires = ["hatchling>=1.18.0"] +build-backend = "hatchling.build" + +[tool.hatch] + +[tool.hatch.build.targets.wheel] +packages = ["src/client", "src/task_executor"] + +[dependency-groups] +dev = [ + "mypy>=1.12.0", + "ruff>=0.4.4", +] + +[tool.mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true + +[[tool.mypy.overrides]] +module = "msgpack.*" +ignore_missing_imports = true + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["ALL"] +extend-select = ["PT"] +ignore = [ + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful +] +isort.order-by-type = false + +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 100 diff --git a/python/src/__init__.py b/python/src/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/client/__init__.py b/python/src/client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/core/__init__.py b/python/src/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/core/task.py b/python/src/core/task.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/task_executor/__init__.py b/python/src/task_executor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/task_executor/task_executor.py b/python/src/task_executor/task_executor.py new file mode 100644 index 000000000..b0c61da37 --- /dev/null +++ b/python/src/task_executor/task_executor.py @@ -0,0 +1,6 @@ +def main() -> None: + """Main function to execute the task.""" + + +if __name__ == "__main__": + main() diff --git a/python/uv.lock b/python/uv.lock new file mode 100644 index 000000000..31a72ace1 --- /dev/null +++ b/python/uv.lock @@ -0,0 +1,242 @@ +version = 1 +revision = 2 +requires-python = ">=3.10" + +[[package]] +name = "mariadb" +version = "1.1.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/a0/0498c90f46940b6fd7f5c76d22df9f011849964e94e0b22fde26c42191c7/mariadb-1.1.13.tar.gz", hash = "sha256:3a0fd24fae2b9990dc13a0a427a43d4a5434cc2a8c3b260a27f40b7824719037", size = 111126, upload-time = "2025-07-11T09:55:11.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e2/5e4068b225ccbd6fb7ea2b8f5ba52060e48bf2e63448206670cdfe73a1c5/mariadb-1.1.13-cp310-cp310-win32.whl", hash = "sha256:f49467e7076819ccf21a7e5ef030761af2160b23ce4484d88d260459809d90ff", size = 185138, upload-time = "2025-07-11T09:54:53.334Z" }, + { url = "https://files.pythonhosted.org/packages/33/8d/65b28ce61c422aba68c052a3cdb23772003267f47438a27a6e2cbf58c1c9/mariadb-1.1.13-cp310-cp310-win_amd64.whl", hash = "sha256:1c9aa9a31f943737c3a58bb4dc9deb9cf2b50b3ec807f90c5c99c7b35f8442bd", size = 202048, upload-time = "2025-07-11T09:54:54.978Z" }, + { url = "https://files.pythonhosted.org/packages/09/71/c23f6ffb06ba2c0ad2e90f4236b1541f9005e24eaeb166800b19c4f584f6/mariadb-1.1.13-cp311-cp311-win32.whl", hash = "sha256:c15ab872f1e951dd1bd4006d2ee2b942156f66c1cd724a1b2992dc02c47dff6a", size = 185136, upload-time = "2025-07-11T09:54:56.791Z" }, + { url = "https://files.pythonhosted.org/packages/c8/08/10308cb9bbccf83ff72caa1146014445a79dfeffb4eba9a85f73575c5959/mariadb-1.1.13-cp311-cp311-win_amd64.whl", hash = "sha256:d6fa7f2cdfae762d2d69b71e51f61415d0251118a622a6d06c7f05261579100a", size = 202046, upload-time = "2025-07-11T09:54:58.365Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b4/d961a5dfa8a7a04ede70f76f85b951b8d0ec28d79b86ced829a67af78bdd/mariadb-1.1.13-cp312-cp312-win32.whl", hash = "sha256:358ecdc8775f4a97357fc233bb84a1c8d8d52866ee5fe98f2d13e9e58a04b6fb", size = 185334, upload-time = "2025-07-11T09:54:59.901Z" }, + { url = "https://files.pythonhosted.org/packages/0a/3b/16cc69a9c90f91b74c096c0cb9634e14692486beb28b82ec5efd97866de7/mariadb-1.1.13-cp312-cp312-win_amd64.whl", hash = "sha256:0604c57e6f90ff7ef190a56894d2b32ca5f0182924b5159be9319ba243ffbf98", size = 202109, upload-time = "2025-07-11T09:55:01.846Z" }, + { url = "https://files.pythonhosted.org/packages/02/16/583719e127774294d4af085a063b38f20407801cd9194a676334fb121edd/mariadb-1.1.13-cp313-cp313-win32.whl", hash = "sha256:36c644c13ecf3e38cb89f37f9bd054703760b94c881476492fe789801a063ce3", size = 185301, upload-time = "2025-07-11T09:55:03.486Z" }, + { url = "https://files.pythonhosted.org/packages/1b/da/4170f5532813ffba2641f821ed23ed1f22fa3ff8d603e47c91b0776ff9b7/mariadb-1.1.13-cp313-cp313-win_amd64.whl", hash = "sha256:6ec51ad2e33312c397cfcc5e2bca59861f572d5e8880e025d43db9a5d472a3a9", size = 202105, upload-time = "2025-07-11T09:55:05.681Z" }, +] + +[[package]] +name = "msgpack" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/b1/ea4f68038a18c77c9467400d166d74c4ffa536f34761f7983a104357e614/msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd", size = 173555, upload-time = "2025-06-13T06:52:51.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/52/f30da112c1dc92cf64f57d08a273ac771e7b29dea10b4b30369b2d7e8546/msgpack-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:353b6fc0c36fde68b661a12949d7d49f8f51ff5fa019c1e47c87c4ff34b080ed", size = 81799, upload-time = "2025-06-13T06:51:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/35/7bfc0def2f04ab4145f7f108e3563f9b4abae4ab0ed78a61f350518cc4d2/msgpack-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:79c408fcf76a958491b4e3b103d1c417044544b68e96d06432a189b43d1215c8", size = 78278, upload-time = "2025-06-13T06:51:38.534Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c5/df5d6c1c39856bc55f800bf82778fd4c11370667f9b9e9d51b2f5da88f20/msgpack-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78426096939c2c7482bf31ef15ca219a9e24460289c00dd0b94411040bb73ad2", size = 402805, upload-time = "2025-06-13T06:51:39.538Z" }, + { url = "https://files.pythonhosted.org/packages/20/8e/0bb8c977efecfe6ea7116e2ed73a78a8d32a947f94d272586cf02a9757db/msgpack-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b17ba27727a36cb73aabacaa44b13090feb88a01d012c0f4be70c00f75048b4", size = 408642, upload-time = "2025-06-13T06:51:41.092Z" }, + { url = "https://files.pythonhosted.org/packages/59/a1/731d52c1aeec52006be6d1f8027c49fdc2cfc3ab7cbe7c28335b2910d7b6/msgpack-1.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a17ac1ea6ec3c7687d70201cfda3b1e8061466f28f686c24f627cae4ea8efd0", size = 395143, upload-time = "2025-06-13T06:51:42.575Z" }, + { url = "https://files.pythonhosted.org/packages/2b/92/b42911c52cda2ba67a6418ffa7d08969edf2e760b09015593c8a8a27a97d/msgpack-1.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:88d1e966c9235c1d4e2afac21ca83933ba59537e2e2727a999bf3f515ca2af26", size = 395986, upload-time = "2025-06-13T06:51:43.807Z" }, + { url = "https://files.pythonhosted.org/packages/61/dc/8ae165337e70118d4dab651b8b562dd5066dd1e6dd57b038f32ebc3e2f07/msgpack-1.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f6d58656842e1b2ddbe07f43f56b10a60f2ba5826164910968f5933e5178af75", size = 402682, upload-time = "2025-06-13T06:51:45.534Z" }, + { url = "https://files.pythonhosted.org/packages/58/27/555851cb98dcbd6ce041df1eacb25ac30646575e9cd125681aa2f4b1b6f1/msgpack-1.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96decdfc4adcbc087f5ea7ebdcfd3dee9a13358cae6e81d54be962efc38f6338", size = 406368, upload-time = "2025-06-13T06:51:46.97Z" }, + { url = "https://files.pythonhosted.org/packages/d4/64/39a26add4ce16f24e99eabb9005e44c663db00e3fce17d4ae1ae9d61df99/msgpack-1.1.1-cp310-cp310-win32.whl", hash = "sha256:6640fd979ca9a212e4bcdf6eb74051ade2c690b862b679bfcb60ae46e6dc4bfd", size = 65004, upload-time = "2025-06-13T06:51:48.582Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/73dfa3e9d5d7450d39debde5b0d848139f7de23bd637a4506e36c9800fd6/msgpack-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:8b65b53204fe1bd037c40c4148d00ef918eb2108d24c9aaa20bc31f9810ce0a8", size = 71548, upload-time = "2025-06-13T06:51:49.558Z" }, + { url = "https://files.pythonhosted.org/packages/7f/83/97f24bf9848af23fe2ba04380388216defc49a8af6da0c28cc636d722502/msgpack-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:71ef05c1726884e44f8b1d1773604ab5d4d17729d8491403a705e649116c9558", size = 82728, upload-time = "2025-06-13T06:51:50.68Z" }, + { url = "https://files.pythonhosted.org/packages/aa/7f/2eaa388267a78401f6e182662b08a588ef4f3de6f0eab1ec09736a7aaa2b/msgpack-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:36043272c6aede309d29d56851f8841ba907a1a3d04435e43e8a19928e243c1d", size = 79279, upload-time = "2025-06-13T06:51:51.72Z" }, + { url = "https://files.pythonhosted.org/packages/f8/46/31eb60f4452c96161e4dfd26dbca562b4ec68c72e4ad07d9566d7ea35e8a/msgpack-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a32747b1b39c3ac27d0670122b57e6e57f28eefb725e0b625618d1b59bf9d1e0", size = 423859, upload-time = "2025-06-13T06:51:52.749Z" }, + { url = "https://files.pythonhosted.org/packages/45/16/a20fa8c32825cc7ae8457fab45670c7a8996d7746ce80ce41cc51e3b2bd7/msgpack-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a8b10fdb84a43e50d38057b06901ec9da52baac6983d3f709d8507f3889d43f", size = 429975, upload-time = "2025-06-13T06:51:53.97Z" }, + { url = "https://files.pythonhosted.org/packages/86/ea/6c958e07692367feeb1a1594d35e22b62f7f476f3c568b002a5ea09d443d/msgpack-1.1.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0c325c3f485dc54ec298d8b024e134acf07c10d494ffa24373bea729acf704", size = 413528, upload-time = "2025-06-13T06:51:55.507Z" }, + { url = "https://files.pythonhosted.org/packages/75/05/ac84063c5dae79722bda9f68b878dc31fc3059adb8633c79f1e82c2cd946/msgpack-1.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:88daaf7d146e48ec71212ce21109b66e06a98e5e44dca47d853cbfe171d6c8d2", size = 413338, upload-time = "2025-06-13T06:51:57.023Z" }, + { url = "https://files.pythonhosted.org/packages/69/e8/fe86b082c781d3e1c09ca0f4dacd457ede60a13119b6ce939efe2ea77b76/msgpack-1.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8b55ea20dc59b181d3f47103f113e6f28a5e1c89fd5b67b9140edb442ab67f2", size = 422658, upload-time = "2025-06-13T06:51:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2b/bafc9924df52d8f3bb7c00d24e57be477f4d0f967c0a31ef5e2225e035c7/msgpack-1.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4a28e8072ae9779f20427af07f53bbb8b4aa81151054e882aee333b158da8752", size = 427124, upload-time = "2025-06-13T06:51:59.969Z" }, + { url = "https://files.pythonhosted.org/packages/a2/3b/1f717e17e53e0ed0b68fa59e9188f3f610c79d7151f0e52ff3cd8eb6b2dc/msgpack-1.1.1-cp311-cp311-win32.whl", hash = "sha256:7da8831f9a0fdb526621ba09a281fadc58ea12701bc709e7b8cbc362feabc295", size = 65016, upload-time = "2025-06-13T06:52:01.294Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/9d1780768d3b249accecc5a38c725eb1e203d44a191f7b7ff1941f7df60c/msgpack-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fd1b58e1431008a57247d6e7cc4faa41c3607e8e7d4aaf81f7c29ea013cb458", size = 72267, upload-time = "2025-06-13T06:52:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/389b9c593eda2b8551b2e7126ad3a06af6f9b44274eb3a4f054d48ff7e47/msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238", size = 82359, upload-time = "2025-06-13T06:52:03.909Z" }, + { url = "https://files.pythonhosted.org/packages/ab/65/7d1de38c8a22cf8b1551469159d4b6cf49be2126adc2482de50976084d78/msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157", size = 79172, upload-time = "2025-06-13T06:52:05.246Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bd/cacf208b64d9577a62c74b677e1ada005caa9b69a05a599889d6fc2ab20a/msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce", size = 425013, upload-time = "2025-06-13T06:52:06.341Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ec/fd869e2567cc9c01278a736cfd1697941ba0d4b81a43e0aa2e8d71dab208/msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a", size = 426905, upload-time = "2025-06-13T06:52:07.501Z" }, + { url = "https://files.pythonhosted.org/packages/55/2a/35860f33229075bce803a5593d046d8b489d7ba2fc85701e714fc1aaf898/msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c", size = 407336, upload-time = "2025-06-13T06:52:09.047Z" }, + { url = "https://files.pythonhosted.org/packages/8c/16/69ed8f3ada150bf92745fb4921bd621fd2cdf5a42e25eb50bcc57a5328f0/msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b", size = 409485, upload-time = "2025-06-13T06:52:10.382Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b6/0c398039e4c6d0b2e37c61d7e0e9d13439f91f780686deb8ee64ecf1ae71/msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef", size = 412182, upload-time = "2025-06-13T06:52:11.644Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/0cf4a6ecb9bc960d624c93effaeaae75cbf00b3bc4a54f35c8507273cda1/msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a", size = 419883, upload-time = "2025-06-13T06:52:12.806Z" }, + { url = "https://files.pythonhosted.org/packages/62/83/9697c211720fa71a2dfb632cad6196a8af3abea56eece220fde4674dc44b/msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c", size = 65406, upload-time = "2025-06-13T06:52:14.271Z" }, + { url = "https://files.pythonhosted.org/packages/c0/23/0abb886e80eab08f5e8c485d6f13924028602829f63b8f5fa25a06636628/msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4", size = 72558, upload-time = "2025-06-13T06:52:15.252Z" }, + { url = "https://files.pythonhosted.org/packages/a1/38/561f01cf3577430b59b340b51329803d3a5bf6a45864a55f4ef308ac11e3/msgpack-1.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3765afa6bd4832fc11c3749be4ba4b69a0e8d7b728f78e68120a157a4c5d41f0", size = 81677, upload-time = "2025-06-13T06:52:16.64Z" }, + { url = "https://files.pythonhosted.org/packages/09/48/54a89579ea36b6ae0ee001cba8c61f776451fad3c9306cd80f5b5c55be87/msgpack-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8ddb2bcfd1a8b9e431c8d6f4f7db0773084e107730ecf3472f1dfe9ad583f3d9", size = 78603, upload-time = "2025-06-13T06:52:17.843Z" }, + { url = "https://files.pythonhosted.org/packages/a0/60/daba2699b308e95ae792cdc2ef092a38eb5ee422f9d2fbd4101526d8a210/msgpack-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:196a736f0526a03653d829d7d4c5500a97eea3648aebfd4b6743875f28aa2af8", size = 420504, upload-time = "2025-06-13T06:52:18.982Z" }, + { url = "https://files.pythonhosted.org/packages/20/22/2ebae7ae43cd8f2debc35c631172ddf14e2a87ffcc04cf43ff9df9fff0d3/msgpack-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d592d06e3cc2f537ceeeb23d38799c6ad83255289bb84c2e5792e5a8dea268a", size = 423749, upload-time = "2025-06-13T06:52:20.211Z" }, + { url = "https://files.pythonhosted.org/packages/40/1b/54c08dd5452427e1179a40b4b607e37e2664bca1c790c60c442c8e972e47/msgpack-1.1.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4df2311b0ce24f06ba253fda361f938dfecd7b961576f9be3f3fbd60e87130ac", size = 404458, upload-time = "2025-06-13T06:52:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/2e/60/6bb17e9ffb080616a51f09928fdd5cac1353c9becc6c4a8abd4e57269a16/msgpack-1.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4141c5a32b5e37905b5940aacbc59739f036930367d7acce7a64e4dec1f5e0b", size = 405976, upload-time = "2025-06-13T06:52:22.995Z" }, + { url = "https://files.pythonhosted.org/packages/ee/97/88983e266572e8707c1f4b99c8fd04f9eb97b43f2db40e3172d87d8642db/msgpack-1.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b1ce7f41670c5a69e1389420436f41385b1aa2504c3b0c30620764b15dded2e7", size = 408607, upload-time = "2025-06-13T06:52:24.152Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/36c78af2efaffcc15a5a61ae0df53a1d025f2680122e2a9eb8442fed3ae4/msgpack-1.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4147151acabb9caed4e474c3344181e91ff7a388b888f1e19ea04f7e73dc7ad5", size = 424172, upload-time = "2025-06-13T06:52:25.704Z" }, + { url = "https://files.pythonhosted.org/packages/8c/87/a75eb622b555708fe0427fab96056d39d4c9892b0c784b3a721088c7ee37/msgpack-1.1.1-cp313-cp313-win32.whl", hash = "sha256:500e85823a27d6d9bba1d057c871b4210c1dd6fb01fbb764e37e4e8847376323", size = 65347, upload-time = "2025-06-13T06:52:26.846Z" }, + { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" }, +] + +[[package]] +name = "mypy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, + { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, + { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, + { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, + { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, + { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, + { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, + { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, + { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" }, + { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" }, + { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" }, + { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" }, + { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" }, + { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "ruff" +version = "0.12.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/81/0bd3594fa0f690466e41bd033bdcdf86cba8288345ac77ad4afbe5ec743a/ruff-0.12.7.tar.gz", hash = "sha256:1fc3193f238bc2d7968772c82831a4ff69252f673be371fb49663f0068b7ec71", size = 5197814, upload-time = "2025-07-29T22:32:35.877Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/d2/6cb35e9c85e7a91e8d22ab32ae07ac39cc34a71f1009a6f9e4a2a019e602/ruff-0.12.7-py3-none-linux_armv6l.whl", hash = "sha256:76e4f31529899b8c434c3c1dede98c4483b89590e15fb49f2d46183801565303", size = 11852189, upload-time = "2025-07-29T22:31:41.281Z" }, + { url = "https://files.pythonhosted.org/packages/63/5b/a4136b9921aa84638f1a6be7fb086f8cad0fde538ba76bda3682f2599a2f/ruff-0.12.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:789b7a03e72507c54fb3ba6209e4bb36517b90f1a3569ea17084e3fd295500fb", size = 12519389, upload-time = "2025-07-29T22:31:54.265Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c9/3e24a8472484269b6b1821794141f879c54645a111ded4b6f58f9ab0705f/ruff-0.12.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2e1c2a3b8626339bb6369116e7030a4cf194ea48f49b64bb505732a7fce4f4e3", size = 11743384, upload-time = "2025-07-29T22:31:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/26/7c/458dd25deeb3452c43eaee853c0b17a1e84169f8021a26d500ead77964fd/ruff-0.12.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32dec41817623d388e645612ec70d5757a6d9c035f3744a52c7b195a57e03860", size = 11943759, upload-time = "2025-07-29T22:32:01.95Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8b/658798472ef260ca050e400ab96ef7e85c366c39cf3dfbef4d0a46a528b6/ruff-0.12.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47ef751f722053a5df5fa48d412dbb54d41ab9b17875c6840a58ec63ff0c247c", size = 11654028, upload-time = "2025-07-29T22:32:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/a8/86/9c2336f13b2a3326d06d39178fd3448dcc7025f82514d1b15816fe42bfe8/ruff-0.12.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a828a5fc25a3efd3e1ff7b241fd392686c9386f20e5ac90aa9234a5faa12c423", size = 13225209, upload-time = "2025-07-29T22:32:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/76/69/df73f65f53d6c463b19b6b312fd2391dc36425d926ec237a7ed028a90fc1/ruff-0.12.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5726f59b171111fa6a69d82aef48f00b56598b03a22f0f4170664ff4d8298efb", size = 14182353, upload-time = "2025-07-29T22:32:10.053Z" }, + { url = "https://files.pythonhosted.org/packages/58/1e/de6cda406d99fea84b66811c189b5ea139814b98125b052424b55d28a41c/ruff-0.12.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74e6f5c04c4dd4aba223f4fe6e7104f79e0eebf7d307e4f9b18c18362124bccd", size = 13631555, upload-time = "2025-07-29T22:32:12.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ae/625d46d5164a6cc9261945a5e89df24457dc8262539ace3ac36c40f0b51e/ruff-0.12.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0bfe4e77fba61bf2ccadf8cf005d6133e3ce08793bbe870dd1c734f2699a3e", size = 12667556, upload-time = "2025-07-29T22:32:15.312Z" }, + { url = "https://files.pythonhosted.org/packages/55/bf/9cb1ea5e3066779e42ade8d0cd3d3b0582a5720a814ae1586f85014656b6/ruff-0.12.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06bfb01e1623bf7f59ea749a841da56f8f653d641bfd046edee32ede7ff6c606", size = 12939784, upload-time = "2025-07-29T22:32:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/7ead2663be5627c04be83754c4f3096603bf5e99ed856c7cd29618c691bd/ruff-0.12.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e41df94a957d50083fd09b916d6e89e497246698c3f3d5c681c8b3e7b9bb4ac8", size = 11771356, upload-time = "2025-07-29T22:32:20.134Z" }, + { url = "https://files.pythonhosted.org/packages/17/40/a95352ea16edf78cd3a938085dccc55df692a4d8ba1b3af7accbe2c806b0/ruff-0.12.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4000623300563c709458d0ce170c3d0d788c23a058912f28bbadc6f905d67afa", size = 11612124, upload-time = "2025-07-29T22:32:22.645Z" }, + { url = "https://files.pythonhosted.org/packages/4d/74/633b04871c669e23b8917877e812376827c06df866e1677f15abfadc95cb/ruff-0.12.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:69ffe0e5f9b2cf2b8e289a3f8945b402a1b19eff24ec389f45f23c42a3dd6fb5", size = 12479945, upload-time = "2025-07-29T22:32:24.765Z" }, + { url = "https://files.pythonhosted.org/packages/be/34/c3ef2d7799c9778b835a76189c6f53c179d3bdebc8c65288c29032e03613/ruff-0.12.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a07a5c8ffa2611a52732bdc67bf88e243abd84fe2d7f6daef3826b59abbfeda4", size = 12998677, upload-time = "2025-07-29T22:32:27.022Z" }, + { url = "https://files.pythonhosted.org/packages/77/ab/aca2e756ad7b09b3d662a41773f3edcbd262872a4fc81f920dc1ffa44541/ruff-0.12.7-py3-none-win32.whl", hash = "sha256:c928f1b2ec59fb77dfdf70e0419408898b63998789cc98197e15f560b9e77f77", size = 11756687, upload-time = "2025-07-29T22:32:29.381Z" }, + { url = "https://files.pythonhosted.org/packages/b4/71/26d45a5042bc71db22ddd8252ca9d01e9ca454f230e2996bb04f16d72799/ruff-0.12.7-py3-none-win_amd64.whl", hash = "sha256:9c18f3d707ee9edf89da76131956aba1270c6348bfee8f6c647de841eac7194f", size = 12912365, upload-time = "2025-07-29T22:32:31.517Z" }, + { url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" }, +] + +[[package]] +name = "spider" +version = "0.0.1" +source = { virtual = "." } +dependencies = [ + { name = "mariadb" }, + { name = "msgpack" }, +] + +[package.dev-dependencies] +dev = [ + { name = "mypy" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "mariadb", specifier = ">=1.1.13" }, + { name = "msgpack", specifier = ">=1.1.1" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "mypy", specifier = ">=1.12.0" }, + { name = "ruff", specifier = ">=0.4.4" }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, +] From 1f944d058288587c891c5910c41d4469a4d7f5cd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 14:56:46 -0400 Subject: [PATCH 40/71] Fix code structure --- python/src/__init__.py | 0 python/uv.lock | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 python/src/__init__.py diff --git a/python/src/__init__.py b/python/src/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/python/uv.lock b/python/uv.lock index 31a72ace1..330a5aaa8 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -169,7 +169,7 @@ wheels = [ [[package]] name = "spider" version = "0.0.1" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "mariadb" }, { name = "msgpack" }, From 68992fcd4daac3ca8e93b897aa8b8fa8f921e469 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 15:00:38 -0400 Subject: [PATCH 41/71] Fix ruff lint --- python/src/client/__init__.py | 1 + python/src/core/__init__.py | 1 + python/src/core/task.py | 5 +++++ python/src/task_executor/__init__.py | 1 + python/src/task_executor/task_executor.py | 3 +++ 5 files changed, 11 insertions(+) diff --git a/python/src/client/__init__.py b/python/src/client/__init__.py index e69de29bb..038456143 100644 --- a/python/src/client/__init__.py +++ b/python/src/client/__init__.py @@ -0,0 +1 @@ +"""Spider python client.""" diff --git a/python/src/core/__init__.py b/python/src/core/__init__.py index e69de29bb..e4e408948 100644 --- a/python/src/core/__init__.py +++ b/python/src/core/__init__.py @@ -0,0 +1 @@ +"""Spider core module.""" diff --git a/python/src/core/task.py b/python/src/core/task.py index e69de29bb..855f2a72b 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -0,0 +1,5 @@ +"""Task module for Spider.""" + + +class Task: + """Represents a task in Spider.""" diff --git a/python/src/task_executor/__init__.py b/python/src/task_executor/__init__.py index e69de29bb..03195404d 100644 --- a/python/src/task_executor/__init__.py +++ b/python/src/task_executor/__init__.py @@ -0,0 +1 @@ +"""Python task executor for running Python task function.""" diff --git a/python/src/task_executor/task_executor.py b/python/src/task_executor/task_executor.py index b0c61da37..c8a0593d3 100644 --- a/python/src/task_executor/task_executor.py +++ b/python/src/task_executor/task_executor.py @@ -1,3 +1,6 @@ +"""Executes a Spider Python task.""" + + def main() -> None: """Main function to execute the task.""" From 1127b2215bd735aa8b8205d5e1443d17efafcb1e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 15:06:00 -0400 Subject: [PATCH 42/71] Extend lint tasks to python directory --- lint-tasks.yaml | 5 +++++ taskfile.yaml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 40259b219..270a121cb 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -135,6 +135,11 @@ tasks: mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" + - cmd: |- + cd "{{.G_SRC_PYTHON_DIR}}" + uv run mypy . + uv run ruff check {{.RUFF_CHECK_FLAGS}} + uv run ruff format {{.RUFF_FORMAT_FLAGS}} yml: aliases: diff --git a/taskfile.yaml b/taskfile.yaml index 470edd582..4588d5bd4 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + G_SRC_PYTHON_DIR: "{{.ROOT_DIR}}/python" + # Build parameters # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEPS_MAX_PARALLELISM_PER_TASK: >- From 4c51c916abbc549b8b6a395b6d9085bc5bb7e587 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 15:13:01 -0400 Subject: [PATCH 43/71] Add python build tasks --- build-tasks.yaml | 4 ++++ taskfile.yaml | 1 + 2 files changed, 5 insertions(+) diff --git a/build-tasks.yaml b/build-tasks.yaml index 53828027c..7363d4d2a 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -19,3 +19,7 @@ tasks: deps: [":config-cmake-project"] cmds: - "cmake --build {{.G_BUILD_SPIDER_DIR}} --target clean --parallel {{numCPU}}" + + py-build: + cmds: + - uv build --directory "{{.G_SRC_PYTHON_DIR}}" -o "{{.G_BUILD_PYTHON_DIR}}" diff --git a/taskfile.yaml b/taskfile.yaml index 4588d5bd4..1a94fdc38 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -21,6 +21,7 @@ vars: G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" G_SRC_PYTHON_DIR: "{{.ROOT_DIR}}/python" + G_BUILD_PYTHON_DIR: "{{.G_BUILD_DIR}}/spider-py" # Build parameters # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. From 6004d6455b56ae4bdd1f30878254ddbe65fe1b9a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 17:51:53 -0400 Subject: [PATCH 44/71] Fix yaml lint error --- build-tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tasks.yaml b/build-tasks.yaml index 7363d4d2a..1ce1efb9b 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -22,4 +22,4 @@ tasks: py-build: cmds: - - uv build --directory "{{.G_SRC_PYTHON_DIR}}" -o "{{.G_BUILD_PYTHON_DIR}}" + - "uv build --directory {{.G_SRC_PYTHON_DIR}} -o {{.G_BUILD_PYTHON_DIR}}" From bcab5db89a55c3bbf7086718e1e1a89adcc7adba Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 18:33:58 -0400 Subject: [PATCH 45/71] Fix tombi lint --- python/pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index c8d875809..235370dd2 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -16,17 +16,17 @@ spider_task_executor = "src.task_executor.task_executor:main" requires = ["hatchling>=1.18.0"] build-backend = "hatchling.build" -[tool.hatch] - -[tool.hatch.build.targets.wheel] -packages = ["src/client", "src/task_executor"] - [dependency-groups] dev = [ "mypy>=1.12.0", "ruff>=0.4.4", ] +[tool.hatch] + +[tool.hatch.build.targets.wheel] +packages = ["src/client", "src/task_executor"] + [tool.mypy] strict = true @@ -43,6 +43,10 @@ ignore_missing_imports = true [tool.ruff] line-length = 100 +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 100 + [tool.ruff.lint] select = ["ALL"] extend-select = ["PT"] @@ -66,7 +70,3 @@ ignore = [ "UP015", # Explicit open modes are helpful ] isort.order-by-type = false - -[tool.ruff.format] -docstring-code-format = true -docstring-code-line-length = 100 From a50d25a36e71146abb374328dbf99c910e70d038 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 18:38:25 -0400 Subject: [PATCH 46/71] Remove wrong mypy config --- python/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 235370dd2..09c1e0f1a 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -34,7 +34,6 @@ strict = true pretty = true show_error_code_links = true show_error_context = true -show_error_end = true [[tool.mypy.overrides]] module = "msgpack.*" From fbebb963eecfef0b7073ae5aad6311b68fb5fd05 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 18:48:37 -0400 Subject: [PATCH 47/71] Fix typo and format file --- python/pyproject.toml | 54 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 09c1e0f1a..be9486a1f 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -5,31 +5,29 @@ description = "Spider is a distributed task execution framework" readme = "README.md" requires-python = ">=3.10" dependencies = [ - "mariadb>=1.1.13", - "msgpack>=1.1.1", + "mariadb>=1.1.13", + "msgpack>=1.1.1", ] -[project.scritps] +[project.scripts] spider_task_executor = "src.task_executor.task_executor:main" -[build-system] -requires = ["hatchling>=1.18.0"] -build-backend = "hatchling.build" - [dependency-groups] dev = [ - "mypy>=1.12.0", - "ruff>=0.4.4", + "mypy>=1.12.0", + "ruff>=0.4.4", ] -[tool.hatch] +[build-system] +requires = ["hatchling>=1.18.0"] +build-backend = "hatchling.build" +[tool.hatch] [tool.hatch.build.targets.wheel] packages = ["src/client", "src/task_executor"] [tool.mypy] strict = true - # Additional output pretty = true show_error_code_links = true @@ -50,22 +48,22 @@ docstring-code-line-length = 100 select = ["ALL"] extend-select = ["PT"] ignore = [ - "COM812", # Redundant and conflicts with ruff format - "D203", # No blank line before docstrings (D211) - "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists - "D212", # Enforce docstring summary line on the next line after quotes (D213) - "D400", # First line of docstrings may not end in period - "D401", # Docstrings should be written in present tense (not imperative) - "D415", # First line of docstrings may not end in a period, question mark, or exclamation point - "FA102", # Allow use of PEP 604 union in type annotations - "FBT", # Allow bool positional parameters since other value positions are allowed - "FIX002", # Allow todo statements - "PERF401", # Allow for loops when creating lists - "PERF403", # Allow for loops when creating dicts - "S311", # Allow usage of `random` package - "SIM102", # Allow collapsible if statements for readability - "TD002", # Author unnecessary for todo statement - "TD003", # Issue link unnecessary for todo statement - "UP015", # Explicit open modes are helpful + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful ] isort.order-by-type = false From 0322bd528424d820a755284e9f8d86a42788c854 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 21:07:18 -0400 Subject: [PATCH 48/71] Use typed msgpack and remove mypy config for msgpack from pyproject --- python/pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index be9486a1f..9b69cfdeb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "mariadb>=1.1.13", - "msgpack>=1.1.1", + "msgpack-types>=0.5.0", ] [project.scripts] @@ -33,10 +33,6 @@ pretty = true show_error_code_links = true show_error_context = true -[[tool.mypy.overrides]] -module = "msgpack.*" -ignore_missing_imports = true - [tool.ruff] line-length = 100 From 2a0815558de44be469e2e4a1581b91683d621b3a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 23:55:16 -0400 Subject: [PATCH 49/71] Update uv lock --- python/uv.lock | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/python/uv.lock b/python/uv.lock index 330a5aaa8..6dc97a77e 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -69,6 +69,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" }, ] +[[package]] +name = "msgpack-types" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/26/a15707f2af5681333cd598724bedd1948844ac2af45eafc4175af0671a8d/msgpack_types-0.5.0.tar.gz", hash = "sha256:aebd1b8da23f8f9966d66ebb1a43bd261b95751c6a267bd21a124d2ccac84201", size = 6702, upload-time = "2024-09-21T13:55:05.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/dd/cd9d2b0ef506f6164cd81d4e92e408095041f28523d751b9f7dabdc244eb/msgpack_types-0.5.0-py3-none-any.whl", hash = "sha256:8b633ed75e495a555fa0615843de559a74b1d176828d59bb393d266e51f6bda7", size = 8182, upload-time = "2024-09-21T13:55:04.232Z" }, +] + [[package]] name = "mypy" version = "1.17.1" @@ -172,7 +185,7 @@ version = "0.0.1" source = { editable = "." } dependencies = [ { name = "mariadb" }, - { name = "msgpack" }, + { name = "msgpack-types" }, ] [package.dev-dependencies] @@ -184,7 +197,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "mariadb", specifier = ">=1.1.13" }, - { name = "msgpack", specifier = ">=1.1.1" }, + { name = "msgpack-types", specifier = ">=0.5.0" }, ] [package.metadata.requires-dev] From dd29c239f499c77c9a33f2b2528236b13dce133c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 23:55:54 -0400 Subject: [PATCH 50/71] Add task module --- python/src/core/task.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/python/src/core/task.py b/python/src/core/task.py index 855f2a72b..5d1c20b6f 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -1,5 +1,46 @@ """Task module for Spider.""" +from dataclasses import dataclass, field +from enum import IntEnum +from uuid import UUID + +@dataclass +class TaskInputOutput: + """Represents a task input that points to output of another task""" + + task_id: UUID + position: int + + +TaskInputValue = bytes +TaskInputData = UUID +TaskInput = TaskInputOutput | TaskInputValue | TaskInputData + +TaskOutputValue = bytes +TaskOutputData = UUID +TaskOutput = TaskOutputValue | TaskOutputData + + +class TaskState(IntEnum): + """Represents state of a task""" + + Pending = 0 + Ready = 1 + Running = 2 + Succeeded = 3 + Failed = 4 + Cancelled = 5 + + +@dataclass class Task: """Represents a task in Spider.""" + + task_id: UUID + function_name: str + state: TaskState + timeout: float + max_retires: int + task_input: list[TaskInputData] = field(default_factory=list) + task_output: list[TaskOutputData] = field(default_factory=list) From 9b2a1752a69c6a53b9fc1332b1a74f8b259240a9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 00:01:03 -0400 Subject: [PATCH 51/71] Add data and improve type alias --- python/src/core/data.py | 12 ++++++++++++ python/src/core/task.py | 11 +++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 python/src/core/data.py diff --git a/python/src/core/data.py b/python/src/core/data.py new file mode 100644 index 000000000..5c632551b --- /dev/null +++ b/python/src/core/data.py @@ -0,0 +1,12 @@ +"""Data module for Spider.""" +from dataclasses import dataclass +from uuid import UUID + +DataId = UUID + +@dataclass +class Data: + """Represents a data object.""" + + id: DataId + value: bytes diff --git a/python/src/core/task.py b/python/src/core/task.py index 5d1c20b6f..63b156db9 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -4,21 +4,24 @@ from enum import IntEnum from uuid import UUID +from core.data import DataId + +TaskId = UUID @dataclass class TaskInputOutput: """Represents a task input that points to output of another task""" - task_id: UUID + task_id: TaskId position: int TaskInputValue = bytes -TaskInputData = UUID +TaskInputData = DataId TaskInput = TaskInputOutput | TaskInputValue | TaskInputData TaskOutputValue = bytes -TaskOutputData = UUID +TaskOutputData = DataId TaskOutput = TaskOutputValue | TaskOutputData @@ -37,7 +40,7 @@ class TaskState(IntEnum): class Task: """Represents a task in Spider.""" - task_id: UUID + task_id: TaskId function_name: str state: TaskState timeout: float From 32afe0a4dbe282be84ab5f748570d062dd1bd26f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 00:18:34 -0400 Subject: [PATCH 52/71] Add task graph and reformat files --- python/src/core/data.py | 2 ++ python/src/core/task.py | 1 + python/src/core/taskgraph.py | 56 ++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 python/src/core/taskgraph.py diff --git a/python/src/core/data.py b/python/src/core/data.py index 5c632551b..7a0586712 100644 --- a/python/src/core/data.py +++ b/python/src/core/data.py @@ -1,9 +1,11 @@ """Data module for Spider.""" + from dataclasses import dataclass from uuid import UUID DataId = UUID + @dataclass class Data: """Represents a data object.""" diff --git a/python/src/core/task.py b/python/src/core/task.py index 63b156db9..1590f1b1c 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -8,6 +8,7 @@ TaskId = UUID + @dataclass class TaskInputOutput: """Represents a task input that points to output of another task""" diff --git a/python/src/core/taskgraph.py b/python/src/core/taskgraph.py new file mode 100644 index 000000000..e0bf91e96 --- /dev/null +++ b/python/src/core/taskgraph.py @@ -0,0 +1,56 @@ +"""TaskGraph module for Spider.""" + +from core.task import Task, TaskId + + +class TaskGraph: + """Represents a task graph in Spider.""" + + def __init__(self) -> None: + """Initializes an empty task graph.""" + self.tasks: dict[TaskId, Task] = {} + # Dependency list consists of a list of tuples of + # - parent task id + # - child task id + self.dependencies: list[tuple[TaskId, TaskId]] = [] + self.input_tasks: set[TaskId] = set() + self.output_tasks: set[TaskId] = set() + + def add_task( + self, task: Task, parents: list[TaskId] | None = None, children: list[TaskId] | None = None + ) -> None: + """ + Adds a task to the graph. + :param task: The task to add. + :param parents: The parent ids of the task. Must be already in the task graph. + :param children: The children ids of the task. Must be already in the task graph. + """ + self.tasks[task.task_id] = task + if parents: + for parent in parents: + self.dependencies.append((parent, task.task_id)) + self.output_tasks.remove(parent) + else: + self.input_tasks.add(task.task_id) + if children: + for child in children: + self.dependencies.append((task.task_id, child)) + self.input_tasks.remove(child) + else: + self.output_tasks.add(task.task_id) + + def get_parents(self, task_id: TaskId) -> list[Task]: + """ + Gets parent tasks of task. + :param task_id: ID of the task. + :return: List of parent tasks. + """ + return [self.tasks[parent] for (parent, child) in self.dependencies if child == task_id] + + def get_children(self, task_id: TaskId) -> list[Task]: + """ + Gets child tasks of task. + :param task_id: ID of the task. + :return: List of children tasks. + """ + return [self.tasks[child] for (parent, child) in self.dependencies if parent == task_id] From 2f9b4a072cc8aadf41e2b4351aabd7c53180fad0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 09:23:41 -0400 Subject: [PATCH 53/71] Fix typo Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- python/src/core/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/core/task.py b/python/src/core/task.py index 1590f1b1c..2deff2208 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -45,6 +45,6 @@ class Task: function_name: str state: TaskState timeout: float - max_retires: int + max_retries: int task_input: list[TaskInputData] = field(default_factory=list) task_output: list[TaskOutputData] = field(default_factory=list) From 4e080ac28e86d4ac2c5b698ae6f4aa8bd3330088 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 09:41:27 -0400 Subject: [PATCH 54/71] Fix set remove key error --- python/src/core/taskgraph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/core/taskgraph.py b/python/src/core/taskgraph.py index e0bf91e96..d209bc378 100644 --- a/python/src/core/taskgraph.py +++ b/python/src/core/taskgraph.py @@ -29,13 +29,13 @@ def add_task( if parents: for parent in parents: self.dependencies.append((parent, task.task_id)) - self.output_tasks.remove(parent) + self.output_tasks.discard(parent) else: self.input_tasks.add(task.task_id) if children: for child in children: self.dependencies.append((task.task_id, child)) - self.input_tasks.remove(child) + self.input_tasks.discard(child) else: self.output_tasks.add(task.task_id) From b09515c0d5a815c6e72a25e4275ddf1d464c4937 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 10:16:51 -0400 Subject: [PATCH 55/71] Increase min version of tombi --- lint-test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-test-requirements.txt b/lint-test-requirements.txt index 879c198b3..ffbf89925 100644 --- a/lint-test-requirements.txt +++ b/lint-test-requirements.txt @@ -6,7 +6,7 @@ ruff>=0.4.4 mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 -tombi>=0.4.0 +tombi>=0.4.35 # Test dependencies msgpack>=1.1.0 From b4d657637fa5da39f8dd1cdd25f67348371eec43 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 16:34:55 -0400 Subject: [PATCH 56/71] Add uv in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2ccfff88f..54cf6bd69 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Follow the steps below to develop and contribute to the project. ## Requirements * Python 3.10 or higher * [Task] 3.40.0 or higher +* [uv] 0.7.0 or higher ## Set up Run dependency installation task: From f447614c9ba65d9e2ddd42e69a194120e7168d1f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 17:35:09 -0400 Subject: [PATCH 57/71] Fix redenduncy caused by merge --- lint-tasks.yaml | 2 -- requirements-dev.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 4e941803f..54ca4611c 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -12,7 +12,6 @@ tasks: - task: "py-check" - task: "toml-check" - task: "yml-check" - - task: "toml-check" fix: cmds: @@ -20,7 +19,6 @@ tasks: - task: "cpp-fix" - task: "toml-fix" - task: "yml-fix" - - task: "toml-fix" cmake-check: deps: ["venv"] diff --git a/requirements-dev.txt b/requirements-dev.txt index d1379209c..a5e8717f2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,6 @@ mypy>=1.12.0 ruff>=0.4.4 tombi>=0.4.35 yamllint>=1.35.1 -tombi>=0.4.35 # Test dependencies msgpack-types>=0.5.0 From de149ee82fcad48b70f5f128c01a0d860449663c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 19:51:06 -0400 Subject: [PATCH 58/71] Add integral types --- python/src/type/__init__.py | 1 + python/src/type/type.py | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 python/src/type/__init__.py create mode 100644 python/src/type/type.py diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py new file mode 100644 index 000000000..f147ce3ed --- /dev/null +++ b/python/src/type/__init__.py @@ -0,0 +1 @@ +"""Spider type module.""" diff --git a/python/src/type/type.py b/python/src/type/type.py new file mode 100644 index 000000000..1a14dfdaf --- /dev/null +++ b/python/src/type/type.py @@ -0,0 +1,42 @@ +"""Custom type module for Spider.""" + +from typing import cast + + +class BoundedInt(int): + """Bounded integer type.""" + + def __new__(cls, value: int, bits: int = 32) -> "BoundedInt": + """Creates a bounded integer.""" + min_val = -(1 << (bits - 1)) + max_val = (1 << (bits - 1)) - 1 + + if not (min_val <= value <= max_val): + msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" + raise ValueError(msg) + + return super().__new__(cls, value) + + +class Int8(BoundedInt): + """8 bits integer type.""" + + def __new__(cls, value: int) -> "Int8": + """Creates an int8 integer.""" + return cast("Int8", super().__new__(cls, value, bits=8)) + + +class Int16(BoundedInt): + """16 bits integer type.""" + + def __new__(cls, value: int) -> "Int16": + """Creates an int16 integer.""" + return cast("Int16", super().__new__(cls, value, bits=16)) + + +class Int32(BoundedInt): + """32 bits integer type.""" + + def __new__(cls, value: int) -> "Int32": + """Creates an int32 integer.""" + return cast("Int32", super().__new__(cls, value, bits=8)) From d332dbe3ed95b3a4a422d44248c054f3200aeb6b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 19:52:13 -0400 Subject: [PATCH 59/71] Revert "Add integral types" This reverts commit de149ee82fcad48b70f5f128c01a0d860449663c. --- python/src/type/__init__.py | 1 - python/src/type/type.py | 42 ------------------------------------- 2 files changed, 43 deletions(-) delete mode 100644 python/src/type/__init__.py delete mode 100644 python/src/type/type.py diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py deleted file mode 100644 index f147ce3ed..000000000 --- a/python/src/type/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider type module.""" diff --git a/python/src/type/type.py b/python/src/type/type.py deleted file mode 100644 index 1a14dfdaf..000000000 --- a/python/src/type/type.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Custom type module for Spider.""" - -from typing import cast - - -class BoundedInt(int): - """Bounded integer type.""" - - def __new__(cls, value: int, bits: int = 32) -> "BoundedInt": - """Creates a bounded integer.""" - min_val = -(1 << (bits - 1)) - max_val = (1 << (bits - 1)) - 1 - - if not (min_val <= value <= max_val): - msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" - raise ValueError(msg) - - return super().__new__(cls, value) - - -class Int8(BoundedInt): - """8 bits integer type.""" - - def __new__(cls, value: int) -> "Int8": - """Creates an int8 integer.""" - return cast("Int8", super().__new__(cls, value, bits=8)) - - -class Int16(BoundedInt): - """16 bits integer type.""" - - def __new__(cls, value: int) -> "Int16": - """Creates an int16 integer.""" - return cast("Int16", super().__new__(cls, value, bits=16)) - - -class Int32(BoundedInt): - """32 bits integer type.""" - - def __new__(cls, value: int) -> "Int32": - """Creates an int32 integer.""" - return cast("Int32", super().__new__(cls, value, bits=8)) From 8c27b1696ede2bb099693d1536e94930de2bac9f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 20:57:38 -0400 Subject: [PATCH 60/71] Restructure under src/spider --- python/pyproject.toml | 6 +----- python/src/core/__init__.py | 1 - python/src/spider/__init__.py | 1 + python/src/{ => spider}/client/__init__.py | 0 python/src/spider/core/__init__.py | 1 + python/src/{ => spider}/core/task.py | 0 python/src/{ => spider}/task_executor/__init__.py | 0 python/src/{ => spider}/task_executor/task_executor.py | 0 8 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 python/src/core/__init__.py create mode 100644 python/src/spider/__init__.py rename python/src/{ => spider}/client/__init__.py (100%) create mode 100644 python/src/spider/core/__init__.py rename python/src/{ => spider}/core/task.py (100%) rename python/src/{ => spider}/task_executor/__init__.py (100%) rename python/src/{ => spider}/task_executor/task_executor.py (100%) diff --git a/python/pyproject.toml b/python/pyproject.toml index 9b69cfdeb..658b21433 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ ] [project.scripts] -spider_task_executor = "src.task_executor.task_executor:main" +spider_task_executor = "src.spider.task_executor.task_executor:main" [dependency-groups] dev = [ @@ -22,10 +22,6 @@ dev = [ requires = ["hatchling>=1.18.0"] build-backend = "hatchling.build" -[tool.hatch] -[tool.hatch.build.targets.wheel] -packages = ["src/client", "src/task_executor"] - [tool.mypy] strict = true # Additional output diff --git a/python/src/core/__init__.py b/python/src/core/__init__.py deleted file mode 100644 index e4e408948..000000000 --- a/python/src/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider core module.""" diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py new file mode 100644 index 000000000..c3628976d --- /dev/null +++ b/python/src/spider/__init__.py @@ -0,0 +1 @@ +"""Spider package root.""" diff --git a/python/src/client/__init__.py b/python/src/spider/client/__init__.py similarity index 100% rename from python/src/client/__init__.py rename to python/src/spider/client/__init__.py diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py new file mode 100644 index 000000000..d417d95e2 --- /dev/null +++ b/python/src/spider/core/__init__.py @@ -0,0 +1 @@ +"""Spider core package.""" diff --git a/python/src/core/task.py b/python/src/spider/core/task.py similarity index 100% rename from python/src/core/task.py rename to python/src/spider/core/task.py diff --git a/python/src/task_executor/__init__.py b/python/src/spider/task_executor/__init__.py similarity index 100% rename from python/src/task_executor/__init__.py rename to python/src/spider/task_executor/__init__.py diff --git a/python/src/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py similarity index 100% rename from python/src/task_executor/task_executor.py rename to python/src/spider/task_executor/task_executor.py From 8ffbd039011fb2e5a04267bf49e2f1369d1d818a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 21:07:14 -0400 Subject: [PATCH 61/71] Fix import path --- python/src/spider/core/task.py | 6 +++--- python/src/spider/core/taskgraph.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index 2deff2208..8a6c79d31 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -4,7 +4,7 @@ from enum import IntEnum from uuid import UUID -from core.data import DataId +from spider.core.data import DataId TaskId = UUID @@ -46,5 +46,5 @@ class Task: state: TaskState timeout: float max_retries: int - task_input: list[TaskInputData] = field(default_factory=list) - task_output: list[TaskOutputData] = field(default_factory=list) + task_input: list[TaskInput] = field(default_factory=list) + task_output: list[TaskOutput] = field(default_factory=list) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index d209bc378..12de9feb3 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -1,6 +1,6 @@ """TaskGraph module for Spider.""" -from core.task import Task, TaskId +from spider.core.task import Task, TaskId class TaskGraph: From 854cd11dbfcc009b144c374b493973a52c66524b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 17:19:25 -0400 Subject: [PATCH 62/71] Fix missing link --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8ac64e9e3..11aec22fd 100644 --- a/README.md +++ b/README.md @@ -68,3 +68,4 @@ in the table below. [spider-docs]: https://docs.yscope.com/spider/main/ [Task]: https://taskfile.dev +[uv]: https://docs.astral.sh/uv/ \ No newline at end of file From 01dd0c6b6fb4ca99f7b045bfb13d70c697c0d8ca Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 17:26:30 -0400 Subject: [PATCH 63/71] Fix task executor path --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 658b21433..0c9da379e 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ ] [project.scripts] -spider_task_executor = "src.spider.task_executor.task_executor:main" +spider_task_executor = "spider.task_executor.task_executor:main" [dependency-groups] dev = [ From 008c6fb4a6dd78ac5b655016c3ad987446dffd1d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 12:20:49 -0400 Subject: [PATCH 64/71] Export core --- python/src/spider/core/__init__.py | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py index d417d95e2..d463958c8 100644 --- a/python/src/spider/core/__init__.py +++ b/python/src/spider/core/__init__.py @@ -1 +1,32 @@ """Spider core package.""" + +from .data import Data, DataId +from .task import ( + Task, + TaskId, + TaskInput, + TaskInputData, + TaskInputOutput, + TaskInputValue, + TaskOutput, + TaskOutputData, + TaskOutputValue, + TaskState, +) +from .taskgraph import TaskGraph + +__all__ = [ + "Data", + "DataId", + "Task", + "TaskGraph", + "TaskId", + "TaskInput", + "TaskInputData", + "TaskInputOutput", + "TaskInputValue", + "TaskOutput", + "TaskOutputData", + "TaskOutputValue", + "TaskState", +] From 4d10f88284932ac363b5f74e6b8c340feb002b13 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 13:02:21 -0400 Subject: [PATCH 65/71] "Add default values for tasks" --- python/src/spider/core/task.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index 8a6c79d31..f4c72911f 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from enum import IntEnum -from uuid import UUID +from uuid import UUID, uuid4 from spider.core.data import DataId @@ -41,10 +41,10 @@ class TaskState(IntEnum): class Task: """Represents a task in Spider.""" - task_id: TaskId - function_name: str - state: TaskState - timeout: float - max_retries: int - task_input: list[TaskInput] = field(default_factory=list) - task_output: list[TaskOutput] = field(default_factory=list) + task_id: TaskId = field(default_factory=uuid4) + function_name: str = "" + state: TaskState = TaskState.Pending + timeout: float = 0 + max_retries: int = 0 + task_inputs: list[TaskInput] = field(default_factory=list) + task_outputs: list[TaskOutput] = field(default_factory=list) From ea764640373ee284f491bee2a36e4257595dbefb Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 13 Aug 2025 21:46:04 -0400 Subject: [PATCH 66/71] Fix merge --- .../spider-py/src/spider_py/core/__init__.py | 31 ++++++++++ python/spider-py/src/spider_py/core/data.py | 14 +++++ python/spider-py/src/spider_py/core/task.py | 45 +++++++++++++++ .../spider-py/src/spider_py/core/taskgraph.py | 56 +++++++++++++++++++ 4 files changed, 146 insertions(+) create mode 100644 python/spider-py/src/spider_py/core/data.py create mode 100644 python/spider-py/src/spider_py/core/taskgraph.py diff --git a/python/spider-py/src/spider_py/core/__init__.py b/python/spider-py/src/spider_py/core/__init__.py index d417d95e2..d463958c8 100644 --- a/python/spider-py/src/spider_py/core/__init__.py +++ b/python/spider-py/src/spider_py/core/__init__.py @@ -1 +1,32 @@ """Spider core package.""" + +from .data import Data, DataId +from .task import ( + Task, + TaskId, + TaskInput, + TaskInputData, + TaskInputOutput, + TaskInputValue, + TaskOutput, + TaskOutputData, + TaskOutputValue, + TaskState, +) +from .taskgraph import TaskGraph + +__all__ = [ + "Data", + "DataId", + "Task", + "TaskGraph", + "TaskId", + "TaskInput", + "TaskInputData", + "TaskInputOutput", + "TaskInputValue", + "TaskOutput", + "TaskOutputData", + "TaskOutputValue", + "TaskState", +] diff --git a/python/spider-py/src/spider_py/core/data.py b/python/spider-py/src/spider_py/core/data.py new file mode 100644 index 000000000..7a0586712 --- /dev/null +++ b/python/spider-py/src/spider_py/core/data.py @@ -0,0 +1,14 @@ +"""Data module for Spider.""" + +from dataclasses import dataclass +from uuid import UUID + +DataId = UUID + + +@dataclass +class Data: + """Represents a data object.""" + + id: DataId + value: bytes diff --git a/python/spider-py/src/spider_py/core/task.py b/python/spider-py/src/spider_py/core/task.py index 855f2a72b..e02bf52f9 100644 --- a/python/spider-py/src/spider_py/core/task.py +++ b/python/spider-py/src/spider_py/core/task.py @@ -1,5 +1,50 @@ """Task module for Spider.""" +from dataclasses import dataclass, field +from enum import IntEnum +from uuid import UUID, uuid4 +from spider_py.core.data import DataId + +TaskId = UUID + + +@dataclass +class TaskInputOutput: + """Represents a task input that points to output of another task""" + + task_id: TaskId + position: int + + +TaskInputValue = bytes +TaskInputData = DataId +TaskInput = TaskInputOutput | TaskInputValue | TaskInputData + +TaskOutputValue = bytes +TaskOutputData = DataId +TaskOutput = TaskOutputValue | TaskOutputData + + +class TaskState(IntEnum): + """Represents state of a task""" + + Pending = 0 + Ready = 1 + Running = 2 + Succeeded = 3 + Failed = 4 + Cancelled = 5 + + +@dataclass class Task: """Represents a task in Spider.""" + + task_id: TaskId = field(default_factory=uuid4) + function_name: str = "" + state: TaskState = TaskState.Pending + timeout: float = 0 + max_retries: int = 0 + task_inputs: list[TaskInput] = field(default_factory=list) + task_outputs: list[TaskOutput] = field(default_factory=list) diff --git a/python/spider-py/src/spider_py/core/taskgraph.py b/python/spider-py/src/spider_py/core/taskgraph.py new file mode 100644 index 000000000..c40490af5 --- /dev/null +++ b/python/spider-py/src/spider_py/core/taskgraph.py @@ -0,0 +1,56 @@ +"""TaskGraph module for Spider.""" + +from spider_py.core.task import Task, TaskId + + +class TaskGraph: + """Represents a task graph in Spider.""" + + def __init__(self) -> None: + """Initializes an empty task graph.""" + self.tasks: dict[TaskId, Task] = {} + # Dependency list consists of a list of tuples of + # - parent task id + # - child task id + self.dependencies: list[tuple[TaskId, TaskId]] = [] + self.input_tasks: set[TaskId] = set() + self.output_tasks: set[TaskId] = set() + + def add_task( + self, task: Task, parents: list[TaskId] | None = None, children: list[TaskId] | None = None + ) -> None: + """ + Adds a task to the graph. + :param task: The task to add. + :param parents: The parent ids of the task. Must be already in the task graph. + :param children: The children ids of the task. Must be already in the task graph. + """ + self.tasks[task.task_id] = task + if parents: + for parent in parents: + self.dependencies.append((parent, task.task_id)) + self.output_tasks.discard(parent) + else: + self.input_tasks.add(task.task_id) + if children: + for child in children: + self.dependencies.append((task.task_id, child)) + self.input_tasks.discard(child) + else: + self.output_tasks.add(task.task_id) + + def get_parents(self, task_id: TaskId) -> list[Task]: + """ + Gets parent tasks of task. + :param task_id: ID of the task. + :return: List of parent tasks. + """ + return [self.tasks[parent] for (parent, child) in self.dependencies if child == task_id] + + def get_children(self, task_id: TaskId) -> list[Task]: + """ + Gets child tasks of task. + :param task_id: ID of the task. + :return: List of children tasks. + """ + return [self.tasks[child] for (parent, child) in self.dependencies if parent == task_id] From 9448189b2d207e09fde016c44eaec89b52a07465 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 13 Aug 2025 21:48:46 -0400 Subject: [PATCH 67/71] Fix merge --- lint-tasks.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 28394313e..2e0e9e3b3 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -138,11 +138,6 @@ tasks: mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" - - cmd: |- - cd "{{.G_SRC_PYTHON_DIR}}" - uv run mypy . - uv run ruff check {{.RUFF_CHECK_FLAGS}} - uv run ruff format {{.RUFF_FORMAT_FLAGS}} spider-py-check: cmds: From be143a7d88ddc18811a5738111aaef79f8ea1549 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 20:06:47 -0400 Subject: [PATCH 68/71] Rename taskgraph.py to task_graph.py --- python/spider-py/src/spider_py/core/__init__.py | 2 +- .../src/spider_py/core/{taskgraph.py => task_graph.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename python/spider-py/src/spider_py/core/{taskgraph.py => task_graph.py} (100%) diff --git a/python/spider-py/src/spider_py/core/__init__.py b/python/spider-py/src/spider_py/core/__init__.py index d463958c8..26e822ca3 100644 --- a/python/spider-py/src/spider_py/core/__init__.py +++ b/python/spider-py/src/spider_py/core/__init__.py @@ -13,7 +13,7 @@ TaskOutputValue, TaskState, ) -from .taskgraph import TaskGraph +from .task_graph import TaskGraph __all__ = [ "Data", diff --git a/python/spider-py/src/spider_py/core/taskgraph.py b/python/spider-py/src/spider_py/core/task_graph.py similarity index 100% rename from python/spider-py/src/spider_py/core/taskgraph.py rename to python/spider-py/src/spider_py/core/task_graph.py From c4c035a150100d3b4505eb03437b7d3f7c6bf5a2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 20:09:32 -0400 Subject: [PATCH 69/71] Move class members to class docstring --- .../spider-py/src/spider_py/core/task_graph.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/python/spider-py/src/spider_py/core/task_graph.py b/python/spider-py/src/spider_py/core/task_graph.py index c40490af5..b58ff9c62 100644 --- a/python/spider-py/src/spider_py/core/task_graph.py +++ b/python/spider-py/src/spider_py/core/task_graph.py @@ -4,14 +4,22 @@ class TaskGraph: - """Represents a task graph in Spider.""" + """ + Represents a task graph in Spider. + TaskGraph represents a directed acyclic graph (DAG) of tasks. + It stores: + - tasks: A dictionary mapping task ids to Task objects. + - dependencies: A list of tuples representing the dependencies between tasks. Each tuple + contains: + - parent task id + - child task id + - input_tasks: A set of task ids that have no parents (input tasks). + - output_tasks: A set of task ids that have no children (output tasks). + """ def __init__(self) -> None: """Initializes an empty task graph.""" self.tasks: dict[TaskId, Task] = {} - # Dependency list consists of a list of tuples of - # - parent task id - # - child task id self.dependencies: list[tuple[TaskId, TaskId]] = [] self.input_tasks: set[TaskId] = set() self.output_tasks: set[TaskId] = set() From fa935d1b76d7c7fd34359fe21cc3e2b8e83fa423 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 20:18:40 -0400 Subject: [PATCH 70/71] Fix docstring --- python/spider-py/src/spider_py/core/task.py | 4 ++-- python/spider-py/src/spider_py/core/task_graph.py | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/python/spider-py/src/spider_py/core/task.py b/python/spider-py/src/spider_py/core/task.py index e02bf52f9..d9c81e68d 100644 --- a/python/spider-py/src/spider_py/core/task.py +++ b/python/spider-py/src/spider_py/core/task.py @@ -11,7 +11,7 @@ @dataclass class TaskInputOutput: - """Represents a task input that points to output of another task""" + """Represents a task input that references the output of another task by its ID and position.""" task_id: TaskId position: int @@ -27,7 +27,7 @@ class TaskInputOutput: class TaskState(IntEnum): - """Represents state of a task""" + """Represents the state of a task""" Pending = 0 Ready = 1 diff --git a/python/spider-py/src/spider_py/core/task_graph.py b/python/spider-py/src/spider_py/core/task_graph.py index b58ff9c62..fb0f03841 100644 --- a/python/spider-py/src/spider_py/core/task_graph.py +++ b/python/spider-py/src/spider_py/core/task_graph.py @@ -49,16 +49,14 @@ def add_task( def get_parents(self, task_id: TaskId) -> list[Task]: """ - Gets parent tasks of task. - :param task_id: ID of the task. - :return: List of parent tasks. + :param task_id: + :return: Parent tasks of the task identified by `task_id`. """ return [self.tasks[parent] for (parent, child) in self.dependencies if child == task_id] def get_children(self, task_id: TaskId) -> list[Task]: """ - Gets child tasks of task. - :param task_id: ID of the task. - :return: List of children tasks. + :param task_id: + :return: Child tasks of the task identified by `task_id`. """ return [self.tasks[child] for (parent, child) in self.dependencies if parent == task_id] From 08b79654eb2926560e6038a959d4235e45770b2b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 21:28:09 -0400 Subject: [PATCH 71/71] Explicity check list not none and not empty --- python/spider-py/src/spider_py/core/task_graph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/spider-py/src/spider_py/core/task_graph.py b/python/spider-py/src/spider_py/core/task_graph.py index fb0f03841..845bbd7c0 100644 --- a/python/spider-py/src/spider_py/core/task_graph.py +++ b/python/spider-py/src/spider_py/core/task_graph.py @@ -34,13 +34,13 @@ def add_task( :param children: The children ids of the task. Must be already in the task graph. """ self.tasks[task.task_id] = task - if parents: + if parents is not None and len(parents) > 0: for parent in parents: self.dependencies.append((parent, task.task_id)) self.output_tasks.discard(parent) else: self.input_tasks.add(task.task_id) - if children: + if children is not None and len(children) > 0: for child in children: self.dependencies.append((task.task_id, child)) self.input_tasks.discard(child)