From d2ac198bb3d79a0f3bd769faaf3e6bc6c44adcdf Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 13:40:10 -0400 Subject: [PATCH 001/198] Update yscope-dev-utils --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index e300d1bab..5c6bfbd00 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit e300d1bab4c2f33cbf9ddf9f0c08185faf035070 +Subproject commit 5c6bfbd00ad4e9cbfdfab9708298fb11bda4ae8c From cc8a0977836682e8ed9aa3e39aa3268ffbb2447c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 13:44:03 -0400 Subject: [PATCH 002/198] Use boost install task --- dep-tasks.yaml | 160 +------------------------------------------------ 1 file changed, 1 insertion(+), 159 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index a451ddc12..324a7124e 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -147,7 +147,7 @@ tasks: internal: true run: "once" cmds: - - task: "boost-download-and-install" + - task: ":utils:boost:download-and-install" vars: WORK_DIR: "{{.G_DEPS_DIR}}/boost" FILE_SHA256: "2128a4c96862b5c0970c1e34d76b1d57e4a1016b80df85ad39667f30b1deba26" @@ -161,161 +161,3 @@ tasks: - "program_options" - "regex" - "system" - - # Runs the bootstrap.sh generate step in the given source directory. Boost only supports - # in-source generation and building. - # - # @param {string} SOURCE_DIR Project source directory. - # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. - # @param {string[]} TARGETS Target libraries to build. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the generate command. - boost-generate: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./bootstrap.sh - --prefix="{{.INSTALL_PREFIX}}" - --exec-prefix="{{.INSTALL_PREFIX}}" - --with-libraries={{(join "," .TARGETS)}} - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - - # Runs the b2 build step for boost. The caller must have previously called `generate` on - # `SOURCE_DIR` for this task to succeed. - # - # @param {string} SOURCE_DIR Directory containing the boost source. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the build command. - # @param {int} [JOBS] The maximum number of concurrent processes to use when building. If - # omitted, the b2 default number is used. Before 1.76.0, the number was 1. Since 1.76.0, the - # default is the number of cores. - boost-build: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./b2 - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - {{- if .JOBS}} - "-j{{.JOBS}}" - {{- end}} - - # Runs the b2 install step for boost. The caller must have previously called `build` on - # `SOURCE_DIR` for this task to succeed. If `CMAKE_SETTINGS_DIR` is set, a settings file will be - # created in that directory, containing a `boost_ROOT` CMake variable that points to - # `INSTALL_PREFIX`. - # - # @param {string} SOURCE_DIR Directory containing the boost source. - # @param {string} INSTALL_PREFIX Path prefix of where the project should be installed. - # @param {string} [CMAKE_SETTINGS_DIR] If set, the directory where the project's CMake settings - # file should be stored. - # @param {string[]} [EXTRA_ARGS] Any additional arguments to pass to the install command. - boost-install: - internal: true - dir: "{{.SOURCE_DIR}}" - cmds: - - >- - ./b2 - install - {{- range .EXTRA_ARGS}} - "{{.}}" - {{- end}} - - >- - {{- if .CMAKE_SETTINGS_DIR}} - echo "set(BOOST_ROOT - \"{{.INSTALL_PREFIX}}\" - CACHE PATH - \"Package root for boost.\" - )" >> "{{.CMAKE_SETTINGS_DIR}}/boost.cmake" - {{- end}} - - # Downloads boost from `URL` and installs boost. - # - # General parameters - # @param {string} [WORK_DIR={{.TASK_DIR}}] Base directory to store the install and src - # directories inside. - # @param {string} [SOURCE_DIR={{.WORK_DIR}}/boost-src] Directory in which to extract the tar - # file. - # - # Download parameters - # @param {string} FILE_SHA256 Content hash to verify the downloaded tar file against. - # @param {string} URL - # - # Boost generate parameters - # @param {string} [INSTALL_PREFIX={{.WORK_DIR}}/boost-install] Path prefix of where the project - # should be installed. - # @param {string[]} TARGETS Target libraries to build. - # @param {string[]} [GEN_ARGS] Any additional arguments to pass to the generate command. - # - # Boost build parameters - # @param {int} [JOBS] The maximum number of concurrent processes to use when building. If - # omitted, the b2 default number is used. Before 1.76.0, the number was 1. Since 1.76.0, the - # default is the number of cores. - # @param {string[]} [BUILD_ARGS] Any additional arguments to pass to the build command. - # - # Boost install parameters - # @param {string[]} [INSTALL_ARGS] Any additional arguments to pass to the install command. - # @param {string} [CMAKE_SETTINGS_DIR] If set, the directory where the project's CMake settings - # file should be stored. - boost-download-and-install: - internal: true - label: "{{.TASK}}:{{.URL}}-{{.INSTALL_PREFIX}}" - vars: - # General parameters - WORK_DIR: >- - {{default .ROOT_DIR .WORK_DIR}} - SOURCE_DIR: >- - {{default (printf "%s/boost-src" .WORK_DIR) .SOURCE_DIR}} - - # Boost generate parameters - INSTALL_PREFIX: >- - {{default (printf "%s/boost-install" .WORK_DIR) .INSTALL_PREFIX}} - TARGETS: - ref: "default (list) .TARGETS" - GEN_ARGS: - ref: "default (list) .GEN_ARGS" - - # Boost build parameters - BUILD_ARGS: - ref: "default (list) .BUILD_ARGS" - JOBS: >- - {{default "" .JOBS}} - - # Boost install parameters - INSTALL_ARGS: - ref: "default (list) .INSTALL_ARGS" - CMAKE_SETTINGS_DIR: >- - {{default "" .CMAKE_SETTINGS_DIR}} - requires: - vars: ["FILE_SHA256", "URL"] - deps: - - task: ":utils:remote:download-and-extract-tar" - vars: - FILE_SHA256: "{{.FILE_SHA256}}" - OUTPUT_DIR: "{{.SOURCE_DIR}}" - URL: "{{.URL}}" - cmds: - - task: "boost-generate" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - TARGETS: - ref: ".TARGETS" - EXTRA_ARGS: - ref: ".GEN_ARGS" - - task: "boost-build" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - JOBS: "{{.JOBS}}" - EXTRA_ARGS: - ref: ".BUILD_ARGS" - - task: "boost-install" - vars: - SOURCE_DIR: "{{.SOURCE_DIR}}" - INSTALL_PREFIX: "{{.INSTALL_PREFIX}}" - CMAKE_SETTINGS_DIR: "{{.CMAKE_SETTINGS_DIR}}" - EXTRA_ARGS: - ref: ".INSTALL_ARGS" From 7f68bdc0d8ec884eaa14096282b680a531d482a2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:00:23 -0400 Subject: [PATCH 003/198] Update install task variable names --- dep-tasks.yaml | 52 +++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 324a7124e..3c7c05f8a 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -30,12 +30,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "absl" + CMAKE_PACKAGE_NAME: "absl" WORK_DIR: "{{.G_DEPS_DIR}}/absl" - FILE_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" - URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" + TAR_SHA256: "b396401fd29e2e679cace77867481d388c807671dc2acc602a0259eeb79b7811" + TAR_URL: "https://github.com/abseil/abseil-cpp/archive/refs/tags/20250127.1.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" @@ -45,12 +45,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "Catch2" + CMAKE_PACKAGE_NAME: "Catch2" WORK_DIR: "{{.G_DEPS_DIR}}/Catch2" - FILE_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" - URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" + TAR_SHA256: "1ab2de20460d4641553addfdfe6acd4109d871d5531f8f519a52ea4926303087" + TAR_URL: "https://github.com/catchorg/Catch2/archive/refs/tags/v3.8.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" @@ -76,12 +76,12 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "fmt" + CMAKE_PACKAGE_NAME: "fmt" WORK_DIR: "{{.G_DEPS_DIR}}/fmtlib" - FILE_SHA256: "6cb1e6d37bdcb756dbbe59be438790db409cdb4868c66e888d5df9f13f7c027f" - URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.0.2.tar.gz" + TAR_SHA256: "6cb1e6d37bdcb756dbbe59be438790db409cdb4868c66e888d5df9f13f7c027f" + TAR_URL: "https://github.com/fmtlib/fmt/archive/refs/tags/11.0.2.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" @@ -95,16 +95,16 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "spdlog" + CMAKE_PACKAGE_NAME: "spdlog" WORK_DIR: "{{.G_DEPS_DIR}}/spdlog" - FILE_SHA256: "9962648c9b4f1a7bbc76fd8d9172555bad1871fdb14ff4f842ef87949682caa5" - URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.0.tar.gz" + TAR_SHA256: "9962648c9b4f1a7bbc76fd8d9172555bad1871fdb14ff4f842ef87949682caa5" + TAR_URL: "https://github.com/gabime/spdlog/archive/refs/tags/v1.15.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DSPDLOG_BUILD_TESTS=OFF" - "-DUSE_EXTERNAL_FMT=ON" - - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" install-mariadb-connector-cpp: @@ -113,13 +113,13 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "mariadb-connector-cpp" + CMAKE_PACKAGE_NAME: "mariadb-connector-cpp" WORK_DIR: "{{.G_DEPS_DIR}}/mariadb-connector-cpp" - FILE_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" - URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/tags/\ + TAR_SHA256: "0e3dfe9f2bc3f7bb6f7c159009556290064a7c23402ea08019fa8aebfc3ff2c9" + TAR_URL: "https://github.com/mariadb-corporation/mariadb-connector-cpp/archive/refs/tags/\ 1.1.5.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" @@ -132,16 +132,16 @@ tasks: cmds: - task: ":utils:cmake:install-remote-tar" vars: - NAME: "msgpack-cxx" + CMAKE_PACKAGE_NAME: "msgpack-cxx" WORK_DIR: "{{.G_DEPS_DIR}}/msgpack" - FILE_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" - URL: "https://github.com/msgpack/msgpack-c/releases/download/\ + TAR_SHA256: "7504b7af7e7b9002ce529d4f941e1b7fb1fb435768780ce7da4abaac79bb156f" + TAR_URL: "https://github.com/msgpack/msgpack-c/releases/download/\ cpp-7.0.0/msgpack-cxx-7.0.0.tar.gz" CMAKE_SETTINGS_DIR: "{{.G_DEPS_CMAKE_SETTINGS_DIR}}" - GEN_ARGS: + CMAKE_GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" install-boost: internal: true From 2394649ab8c6454733ff34d8736fb1b8960d2022 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:11:02 -0400 Subject: [PATCH 004/198] Set CMP0074 to NEW to find boost --- dep-tasks.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 3c7c05f8a..d58a61f0d 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -105,6 +105,7 @@ tasks: - "-DSPDLOG_BUILD_TESTS=OFF" - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" install-mariadb-connector-cpp: @@ -142,6 +143,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/Boost.cmake" + - "-DCMAKE_POLICY_DEFAULT_CMP0074=NEW" install-boost: internal: true From 6ff3e33f9d5b36e3b54be01fbf6b263b4c9ca6a9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 14:40:52 -0400 Subject: [PATCH 005/198] Add uv to install script --- tools/scripts/lib_install/linux/install-dev.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/scripts/lib_install/linux/install-dev.sh b/tools/scripts/lib_install/linux/install-dev.sh index 73fd319bc..f88c9cc93 100755 --- a/tools/scripts/lib_install/linux/install-dev.sh +++ b/tools/scripts/lib_install/linux/install-dev.sh @@ -36,3 +36,6 @@ script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir="$script_dir/.." # TODO https://github.com/y-scope/spider/issues/86 "$lib_install_scripts_dir"/check-cmake-version.sh + +# Install uv +curl -LsSf https://astral.sh/uv/install.sh | sh From 7ae8a782aaf63043f4f1c4b31a47f3dc57bfb346 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 15:04:40 -0400 Subject: [PATCH 006/198] Fix cpp-lint root paths --- lint-tasks.yaml | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index d81b7247d..85417f30c 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -45,7 +45,7 @@ tasks: - task: "cpp-static-fix" cpp-format-check: - sources: &cpp_source_files + sources: &cpp_lint_source_files - "{{.ROOT_DIR}}/.clang-format" - "{{.ROOT_DIR}}/.clang-tidy" - "{{.TASKFILE}}" @@ -59,11 +59,14 @@ tasks: vars: FLAGS: ["--dry-run"] INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: &cpp_source_files + - "{{.G_SRC_SPIDER_DIR}}" + - "{{.G_EXAMPLES_DIR}}" + - "{{.G_TEST_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" cpp-format-fix: - sources: *cpp_source_files + sources: *cpp_lint_source_files deps: ["cpp-configs", "venv"] cmds: - task: ":utils:cpp-lint:clang-format" @@ -152,18 +155,6 @@ tasks: taskfile.yaml \ test-tasks.yaml - clang-tidy: - internal: true - requires: - vars: ["FLAGS", "SRC_DIR"] - cmd: |- - . "{{.G_LINT_VENV_DIR}}/bin/activate" - find "{{.SRC_DIR}}" \ - -type f \ - \( -iname "*.cpp" -o -iname "*.h" -o -iname "*.hpp" \) \ - -print0 | \ - xargs -0 --no-run-if-empty clang-tidy {{.FLAGS}} - cmake: internal: true requires: From 56502f90d6df56d5e0d97ed0f872b7682743455b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 15 Jul 2025 15:16:33 -0400 Subject: [PATCH 007/198] Fix clang-tidy file pattern --- lint-tasks.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 85417f30c..659480e50 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -82,7 +82,7 @@ tasks: # When we eventually determine which errors can be safely fixed, we'll allow clang-tidy to # fix them. aliases: ["cpp-static-fix"] - sources: *cpp_source_files + sources: *cpp_lint_source_files deps: [":config-cmake-project", "cpp-configs", "venv"] cmds: - task: ":utils:cpp-lint:clang-tidy-find" @@ -90,21 +90,21 @@ tasks: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_SPIDER_COMPILE_COMMANDS_DB}}'" - INCLUDE_PATTERNS: - - "{{.G_SRC_SPIDER_DIR}}/**" - - "{{.G_TEST_DIR}}/**" + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: + - "{{.G_SRC_SPIDER_DIR}}" + - "{{.G_TEST_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" - task: ":utils:cpp-lint:clang-tidy-find" vars: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_EXAMPLES_COMPILE_COMMANDS_DB}}'" - INCLUDE_PATTERNS: - - "{{.G_EXAMPLES_DIR}}/**" + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" - ROOT_PATHS: *cpp_source_files + ROOT_PATHS: + - "{{.G_EXAMPLES_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" py-check: From 6f6becfff0e90960d46fe07ce442012de70150b1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:19:38 -0400 Subject: [PATCH 008/198] Limit build parallelism --- .github/workflows/code-linting-checks.yaml | 2 ++ .github/workflows/unit-tests.yaml | 2 ++ dep-tasks.yaml | 6 ++++++ taskfile.yaml | 2 ++ 4 files changed, 12 insertions(+) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index c6b1e9117..4ee4e58e4 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -47,6 +47,8 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 + env: + SPIDER_DEP_BUILD_PARALLELISM: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 4b42a93cc..0e6089d37 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -50,6 +50,8 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 + env: + SPIDER_DEP_BUILD_PARALLELISM: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index a451ddc12..9652125e6 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,6 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-Catch2: internal: true @@ -53,6 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} download-ystdlib: internal: true @@ -85,6 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-spdlog: internal: true @@ -106,6 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-mariadb-connector-cpp: internal: true @@ -123,6 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-msgpack: internal: true @@ -142,6 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" + JOBS: {{.G_DEP_BUILD_PARALLELISM}} install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index 556ef3486..ecece7a09 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + G_DEP_BUILD_PARALLELISM: >- + {{default "" (env SPIDER_DEP_BUILD_PARALLELISM)}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From a74523daa93aef57b0d279863583f7b3b4a7240b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:34:32 -0400 Subject: [PATCH 009/198] Bug fix --- dep-tasks.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 9652125e6..9262da104 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: {{.G_DEP_BUILD_PARALLELISM}} + JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" install-boost: internal: true From 77ee49460e2dfb7752f4416b0b595a16ced5fc16 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:37:22 -0400 Subject: [PATCH 010/198] Bug fix --- taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index ecece7a09..4a110c06c 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -21,7 +21,7 @@ vars: G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" G_DEP_BUILD_PARALLELISM: >- - {{default "" (env SPIDER_DEP_BUILD_PARALLELISM)}} + {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 3698339529129c883a0d1d927a873e2b879b4dce Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 17 Jul 2025 13:47:19 -0400 Subject: [PATCH 011/198] Bug fix --- dep-tasks.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 4f45bd44c..fe90b3544 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -169,4 +169,3 @@ tasks: - "program_options" - "regex" - "system" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" From 369e9f1bbf876d7ef1b1ed209f953681d5f83211 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 22:57:18 -0400 Subject: [PATCH 012/198] Rename variables to mirror CLP core Co-authored-by: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> --- taskfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 4a110c06c..d44c1a464 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,8 +20,11 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + # Build parameters + # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEP_BUILD_PARALLELISM: >- {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} + G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 90aa5a285b7068d245c52663dc82cbac3063ee67 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 22:57:45 -0400 Subject: [PATCH 013/198] Rename variables to mirror clp core --- .github/workflows/code-linting-checks.yaml | 2 +- .github/workflows/unit-tests.yaml | 2 +- dep-tasks.yaml | 12 ++++++------ taskfile.yaml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 4ee4e58e4..48f8eacd7 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -48,7 +48,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_BUILD_PARALLELISM: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 0e6089d37..1ced6a889 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -51,7 +51,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_BUILD_PARALLELISM: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index 9262da104..cab98d77c 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: "{{.G_DEP_BUILD_PARALLELISM}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index 4a110c06c..20da0f3b7 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,8 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" - G_DEP_BUILD_PARALLELISM: >- - {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} + G_DEPS_MAX_PARALLELISM_PER_TASK: >- + {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From 7faac8fc6a1e2b196abb0853b23c8ea3153cdeb9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:07:53 -0400 Subject: [PATCH 014/198] Revert "Merge branch 'dep-concurrency' of github.com:sitaowang1998/spider into dep-concurrency" This reverts commit 1769c95feb3868e4a462ccaa694b97f880947f12, reversing changes made to 90aa5a285b7068d245c52663dc82cbac3063ee67. --- .github/workflows/code-linting-checks.yaml | 2 +- .github/workflows/unit-tests.yaml | 2 +- dep-tasks.yaml | 12 ++++++------ taskfile.yaml | 7 ++----- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 1178f0598..48f8eacd7 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -48,7 +48,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_MAX_PARALLELISM_PER_TASK: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task lint:check -C $(nproc)" diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 675e7c6ec..1ced6a889 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -51,7 +51,7 @@ jobs: - name: "Install project dependencies " timeout-minutes: 10 env: - SPIDER_DEP_MAX_PARALLELISM_PER_TASK: "1" + SPIDER_DEPS_MAX_PARALLELISM_PER_TASK: "1" run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" diff --git a/dep-tasks.yaml b/dep-tasks.yaml index ea282d430..cab98d77c 100644 --- a/dep-tasks.yaml +++ b/dep-tasks.yaml @@ -38,7 +38,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DABSL_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-Catch2: internal: true @@ -54,7 +54,7 @@ tasks: GEN_ARGS: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DCATCH_BUILD_TESTING=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" download-ystdlib: internal: true @@ -87,7 +87,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DFMT_TEST=OFF" - "-DFMT_DOC=OFF" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-spdlog: internal: true @@ -109,7 +109,7 @@ tasks: - "-DUSE_EXTERNAL_FMT=ON" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/fmt.cmake" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-mariadb-connector-cpp: internal: true @@ -127,7 +127,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DUSE_SYSTEM_INSTALLED_LIB=ON" - "-DINSTALL_LAYOUT=RPM" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-msgpack: internal: true @@ -147,7 +147,7 @@ tasks: - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-DMSGPACK_BUILD_TESTS=OFF" - "-C {{.G_DEPS_CMAKE_SETTINGS_DIR}}/boost.cmake" - JOBS: "{{.G_DEP_MAX_PARALLELISM_PER_TASK}}" + JOBS: "{{.G_DEPS_MAX_PARALLELISM_PER_TASK}}" install-boost: internal: true diff --git a/taskfile.yaml b/taskfile.yaml index d44c1a464..20da0f3b7 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,11 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" - # Build parameters - # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. - G_DEP_BUILD_PARALLELISM: >- - {{default "" (env "SPIDER_DEP_BUILD_PARALLELISM")}} - + G_DEPS_MAX_PARALLELISM_PER_TASK: >- + {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" # These should be kept in-sync with its usage in CMakeLists.txt G_DEPS_CMAKE_SETTINGS_DIR: "{{.G_DEPS_DIR}}/cmake-settings" From f494a90b85e24d4e9ee07875e01e7cdb65e5387c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:10:25 -0400 Subject: [PATCH 015/198] Add comment for deps parallelism default value --- taskfile.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfile.yaml b/taskfile.yaml index 20da0f3b7..470edd582 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + # Build parameters + # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEPS_MAX_PARALLELISM_PER_TASK: >- {{default "" (env "SPIDER_DEPS_MAX_PARALLELISM_PER_TASK")}} G_DEPS_DIR: "{{.G_BUILD_DIR}}/deps" From ff2fe1c69a7ff4d952d546babba27413dcb0370f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 3 Aug 2025 23:28:39 -0400 Subject: [PATCH 016/198] Update yscope-dev-utils --- tools/yscope-dev-utils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils index 5c6bfbd00..b965f19f3 160000 --- a/tools/yscope-dev-utils +++ b/tools/yscope-dev-utils @@ -1 +1 @@ -Subproject commit 5c6bfbd00ad4e9cbfdfab9708298fb11bda4ae8c +Subproject commit b965f19f3e11c548f32bf7ab8e28ff38906be018 From 65841a019afaaf7aaaf56ef5b516743a482f9861 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 10:31:24 -0400 Subject: [PATCH 017/198] Add latest python lint config files --- mypy.ini | 8 ++++++++ ruff.toml | 29 +++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..4c66b1114 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true diff --git a/ruff.toml b/ruff.toml index 969421ff8..cb6ee887f 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,6 +1,27 @@ line-length = 100 -lint.select = ["I"] -[lint.isort] -case-sensitive = false -order-by-type = false \ No newline at end of file +[lint] +select = ["ALL"] +ignore = [ + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful +] +isort.order-by-type = false + +[format] +docstring-code-format = true +docstring-code-line-length = 100 From 8564fc259a54fdbe08ec27b39f33dc0ca3e4f593 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 11:03:48 -0400 Subject: [PATCH 018/198] Update ruff lint tasks --- lint-requirements.txt | 1 - lint-tasks.yaml | 16 +++++++--------- mypy.ini | 8 -------- 3 files changed, 7 insertions(+), 18 deletions(-) delete mode 100644 mypy.ini diff --git a/lint-requirements.txt b/lint-requirements.txt index 717adb508..690b9797f 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,4 +1,3 @@ -black>=24.4.2 clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 659480e50..04f8cef3a 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -111,29 +111,27 @@ tasks: cmds: - task: "py" vars: - BLACK_FLAGS: "--check" - RUFF_FLAGS: "" + RUFF_FORMAT_FLAGS: "--diff" py-fix: cmds: - task: "py" vars: - BLACK_FLAGS: "" - RUFF_FLAGS: "--fix" + RUFF_CHECK_FLAGS: "--fix" py: internal: true - requires: - vars: ["BLACK_FLAGS", "RUFF_FLAGS"] + vars: + RUFF_CHECK_FLAGS: "{{.RUFF_CHECK_FLAGS | default \"\"}}" + RUFF_FORMAT_FLAGS: "{{.RUFF_FORMAT_FLAGS | default \"\"}}" deps: ["venv"] cmds: - for: - "tests/integration" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" - cd "{{.ITEM}}" - black --color --line-length 100 {{.BLACK_FLAGS}} . - ruff check {{.RUFF_FLAGS}} . + ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" + ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" yml: aliases: diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 4c66b1114..000000000 --- a/mypy.ini +++ /dev/null @@ -1,8 +0,0 @@ -[mypy] -strict = true - -# Additional output -pretty = true -show_error_code_links = true -show_error_context = true -show_error_end = true From a6e7d29a435f66eeac173a1a28415c7d1ab3c820 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:13:55 -0400 Subject: [PATCH 019/198] Fix ruff lint --- ruff.toml | 8 ++ tests/integration/__init__.py | 1 + tests/integration/client.py | 118 +++++++++++++++++---- tests/integration/test_client.py | 27 ++++- tests/integration/test_scheduler_worker.py | 85 +++++++++++++-- tests/integration/test_signal.py | 52 ++++++--- 6 files changed, 244 insertions(+), 47 deletions(-) diff --git a/ruff.toml b/ruff.toml index cb6ee887f..a7a590d31 100644 --- a/ruff.toml +++ b/ruff.toml @@ -2,6 +2,7 @@ line-length = 100 [lint] select = ["ALL"] +extend-select = ["PT"] ignore = [ "COM812", # Redundant and conflicts with ruff format "D203", # No blank line before docstrings (D211) @@ -22,6 +23,13 @@ ignore = [ ] isort.order-by-type = false +[lint.per-file-ignores] +"tests/integration/test_*.py" = [ + "S101", # Allow use of `assert` (security warning) + "ANN001", # Ignore missing type annotations for arguments + "ANN201", # Ignore missing type annotations for parameters +] + [format] docstring-code-format = true docstring-code-line-length = 100 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index e69de29bb..4d2ed7264 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -0,0 +1 @@ +"""Integration tests for Spider C++ projects.""" diff --git a/tests/integration/client.py b/tests/integration/client.py index 1b658c5ac..834ae2b5e 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,7 +1,7 @@ import re import uuid from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from collections.abc import Generator import mysql.connector import pytest @@ -9,48 +9,73 @@ @dataclass class TaskInput: + """ + TaskInput represents an input to a task. + It can either be a direct value, a reference to another task's output, or a reference to data. + """ type: str - task_output: Optional[Tuple[uuid.UUID, int]] = None - value: Optional[str] = None - data_id: Optional[uuid.UUID] = None + task_output: tuple[uuid.UUID, int] | None = None + value: str | None = None + data_id: uuid.UUID | None = None @dataclass class TaskOutput: + """ + TaskOutput represents an output of a task. + It can either be a direct value or a reference to data. + """ type: str - value: Optional[str] = None - data_id: Optional[uuid.UUID] = None + value: str | None = None + data_id: uuid.UUID | None = None @dataclass class Task: + """ + Task represents a unit of work in the task graph. + """ id: uuid.UUID function_name: str - inputs: List[TaskInput] - outputs: List[TaskOutput] + inputs: list[TaskInput] + outputs: list[TaskOutput] timeout: float = 0.0 max_retries: int = 0 @dataclass class TaskGraph: + """ + TaskGraph represents a directed acyclic graph of tasks. + """ id: uuid.UUID - tasks: Dict[uuid.UUID, Task] - dependencies: List[Tuple[uuid.UUID, uuid.UUID]] + tasks: dict[uuid.UUID, Task] + dependencies: list[tuple[uuid.UUID, uuid.UUID]] @dataclass class Driver: + """ + Driver represents a client that can submit jobs to the task graph. + """ id: uuid.UUID @dataclass class Data: + """ + Data represents a Spider Data object. + """ id: uuid.UUID value: str -def create_connection(storage_url: str): +def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: + """ + Creation a MariaDB connection from a JDBC URL. + :param storage_url: JDBC URL for the MariaDB database. + :return: The created MySQL connection. + """ pattern = re.compile( r"jdbc:mariadb://(?P[^:/]+):(?P\d+)/(?P[^?]+)\?user=(?P[^&]+)&password=(?P[^&]+)" ) @@ -68,7 +93,13 @@ def create_connection(storage_url: str): ) -def is_head_task(task_id: uuid.UUID, dependencies: List[Tuple[uuid.UUID, uuid.UUID]]): +def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UUID]]) -> bool: + """ + Check if the task is a head task, meaning it has no parent. + :param task_id: the ID of the task to check. + :param dependencies: list of dependencies where each dependency is a tuple (parent_id, child_id). + :return: True if the task has no parent, False otherwise. + """ return not any(dependency[1] == task_id for dependency in dependencies) @@ -76,13 +107,25 @@ def is_head_task(task_id: uuid.UUID, dependencies: List[Tuple[uuid.UUID, uuid.UU @pytest.fixture(scope="session") -def storage(): +def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: + """ + Fixture to create a database connection for the test session. Yields a connection object + and ensures it is closed after the tests are done. + :return: + """ conn = create_connection(g_storage_url) yield conn conn.close() -def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph): +def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: + """ + Submit a job to the database. + :param conn: database connection object. + :param client_id: client ID of the driver submitting the job. + :param graph: task graph to be submitted. + :return: None + """ cursor = conn.cursor() cursor.execute( @@ -136,7 +179,13 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph): cursor.close() -def get_task_outputs(conn, task_id: uuid.UUID) -> List[TaskOutput]: +def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: + """ + Get the outputs of a task by its ID. + :param conn: database connection object. + :param task_id: the ID of the task whose outputs are to be retrieved. + :return: list of TaskOutput objects representing the outputs of the task. + """ cursor = conn.cursor() cursor.execute( @@ -158,6 +207,12 @@ def get_task_outputs(conn, task_id: uuid.UUID) -> List[TaskOutput]: def get_task_state(conn, task_id: uuid.UUID) -> str: + """ + Get the state of a task by its ID. + :param conn: database connection object. + :param task_id: the ID of the task whose state is to be retrieved. + :return: the state of the task as a string. + """ cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) @@ -168,7 +223,13 @@ def get_task_state(conn, task_id: uuid.UUID) -> str: return state -def remove_job(conn, job_id: uuid.UUID): +def remove_job(conn, job_id: uuid.UUID) -> None: + """ + Remove a job from the database by its ID. + :param conn: database connection object. + :param job_id: the ID of the job to be removed. + :return: None + """ cursor = conn.cursor() cursor.execute("DELETE FROM jobs WHERE id = %s", (job_id.bytes,)) @@ -176,7 +237,13 @@ def remove_job(conn, job_id: uuid.UUID): cursor.close() -def add_driver(conn, driver: Driver): +def add_driver(conn, driver: Driver) -> None: + """ + Register a new driver in the database. + :param conn: database connection object. + :param driver: driver object to be registered. + :return: None + """ cursor = conn.cursor() cursor.execute("INSERT INTO drivers (id) VALUES (%s)", (driver.id.bytes,)) @@ -185,7 +252,14 @@ def add_driver(conn, driver: Driver): cursor.close() -def add_driver_data(conn, driver: Driver, data: Data): +def add_driver_data(conn, driver: Driver, data: Data) -> None: + """ + Add a new data associated with a driver in the database. + :param conn: database connection object. + :param driver: driver object to which the data is associated. + :param data: data object to be added. + :return: None + """ cursor = conn.cursor() cursor.execute("INSERT INTO data (id, value) VALUES (%s, %s)", (data.id.bytes, data.value)) @@ -198,7 +272,13 @@ def add_driver_data(conn, driver: Driver, data: Data): cursor.close() -def remove_data(conn, data: Data): +def remove_data(conn, data: Data) -> None: + """ + Remove data from the database by its ID. + :param conn: database connection object. + :param data: data object to be removed. + :return: None + """ cursor = conn.cursor() cursor.execute("DELETE FROM data WHERE id = %s", (data.id.bytes,)) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 94b48afdf..7d6edeebe 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,7 +1,7 @@ import subprocess import time from pathlib import Path -from typing import Tuple +from collections.abc import Generator import pytest @@ -14,7 +14,13 @@ def start_scheduler_workers( storage_url: str, scheduler_port: int -) -> Tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: + """ + Start the scheduler and two worker processes. + :param storage_url: + :param scheduler_port: The port for the scheduler to listen on. + :return: scheduler_process, worker_process_0, worker_process_1 + """ # Start the scheduler dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." / ".." / "src" / "spider" @@ -43,7 +49,13 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -def scheduler_worker(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker() -> Generator[None, None, None]: + """ + Fixture to start the scheduler and two worker processes. Yields control to the test class, + and then kills the processes after the test class is done. + :return: + """ scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -56,7 +68,12 @@ def scheduler_worker(storage): class TestClient: - def test_client(self, scheduler_worker): + @pytest.mark.usefixtures("scheduler_worker") + def test_client(self) -> None: + """ + Test the client_test C++ program and check for successful execution. + :return: None + """ dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." client_cmds = [ @@ -64,5 +81,5 @@ def test_client(self, scheduler_worker): "--storage_url", g_storage_url, ] - p = subprocess.run(client_cmds, timeout=20) + p = subprocess.run(client_cmds, check=False, timeout=20) assert p.returncode == 0 diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 9dfc359d0..f138f9f81 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -2,11 +2,12 @@ import time import uuid from pathlib import Path -from typing import Tuple +from collections.abc import Generator import msgpack import pytest +from tests.integration.client import TaskGraph, Task from .client import ( add_driver, add_driver_data, @@ -29,7 +30,13 @@ def start_scheduler_worker( storage_url: str, scheduler_port: int -) -> Tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen, subprocess.Popen]: + """ + Start a scheduler and a worker process. + :param storage_url: JDBC storage URL + :param scheduler_port: the port for the scheduler to listen on + :return: scheduler_process, worker_process + """ # Start the scheduler dir_path = Path(__file__).resolve().parent dir_path = dir_path / ".." / ".." / "src" / "spider" @@ -57,7 +64,13 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -def scheduler_worker(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker() -> Generator[None, None, None]: + """ + Fixture to start a scheduler and a worker process. Yields control to the test function. + After the test function completes, it kills the scheduler and the worker process. + :return: + """ scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -69,7 +82,13 @@ def scheduler_worker(storage): @pytest.fixture(scope="function") -def success_job(storage): +def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: + """ + Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. + Cleans up the job after the test function completes. + :param storage: + :return: + """ parent_1 = Task( id=uuid.uuid4(), function_name="sum_test", @@ -127,7 +146,13 @@ def success_job(storage): @pytest.fixture(scope="function") -def fail_job(storage): +def fail_job(storage) -> Generator[Task, None, None]: + """ + Fixture to create a job that will fail. The task will raise an error when executed. + Yield the task. Cleanup the job after the test function completes. + :param storage: + :return: + """ task = Task( id=uuid.uuid4(), function_name="error_test", @@ -149,7 +174,13 @@ def fail_job(storage): @pytest.fixture(scope="function") -def data_job(storage): +def data_job(storage) -> Generator[Task, None, None]: + """ + Fixture to create a job that uses data. Yields the task that uses data. + Cleans up the job and data after the test function completes. + :param storage: + :return: + """ data = Data( id=uuid.uuid4(), value=msgpack.packb(2), @@ -181,6 +212,12 @@ def data_job(storage): @pytest.fixture(scope="function") def random_fail_job(storage): + """ + Fixture to create a job that randomly fails. The task will succeed after a few retries. + Yields the task. Cleans up the job after the test function completes. + :param storage: + :return: + """ data = Data( id=uuid.uuid4(), value=msgpack.packb(2), @@ -212,7 +249,14 @@ def random_fail_job(storage): class TestSchedulerWorker: - def test_job_success(self, scheduler_worker, storage, success_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_job_success(self, storage, success_job): + """ + Test the successful execution of a job with two parent tasks and one child task. + :param storage: + :param success_job: + :return: None + """ graph, parent_1, parent_2, child = success_job # Wait for 2 seconds and check task state and output time.sleep(2) @@ -232,14 +276,28 @@ def test_job_success(self, scheduler_worker, storage, success_job): assert len(outputs) == 1 assert outputs[0].value == msgpack.packb(10) - def test_job_failure(self, scheduler_worker, storage, fail_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_job_failure(self, storage, fail_job): + """ + Test the failure of a job that raise an error. + :param storage: + :param fail_job: + :return: None + """ task = fail_job # Wait for 2 seconds and check task output time.sleep(2) state = get_task_state(storage, task.id) assert state == "fail" - def test_data_job(self, scheduler_worker, storage, data_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_data_job(self, storage, data_job): + """ + Test the successful execution of a job that uses data. + :param storage: + :param data_job: + :return: None + """ task = data_job # Wait for 2 seconds and check task output time.sleep(2) @@ -249,7 +307,14 @@ def test_data_job(self, scheduler_worker, storage, data_job): assert len(outputs) == 1 assert outputs[0].value == msgpack.packb(2) - def test_random_fail_job(self, scheduler_worker, storage, random_fail_job): + @pytest.mark.usefixtures("scheduler_worker") + def test_random_fail_job(self, storage, random_fail_job): + """ + Test the successful recovery and execution of a job that randomly fails. + :param storage: + :param random_fail_job: + :return: None + """ task = random_fail_job # Wait for 2 seconds and check task output time.sleep(2) diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index a32f1c3d8..012fac7b9 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -4,6 +4,7 @@ import time import uuid from pathlib import Path +from collections.abc import Generator import msgpack import pytest @@ -23,7 +24,14 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str): +def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> tuple[subprocess.Popen, subprocess.Popen]: + """ + Create a scheduler and a worker process. + :param storage_url: JDB storage URL. + :param scheduler_port: the port for the scheduler to listen on. + :param lib: Library to load in the worker. + :return: scheduler and worker processes. + """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -52,7 +60,12 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str): @pytest.fixture(scope="function") -def scheduler_worker_signal(storage): +@pytest.mark.usefixtures("storage") +def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen], None, None]: + """ + Fixture to start a scheduler and a worker process for testing signal handling. + :return: + """ scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -65,12 +78,19 @@ def scheduler_worker_signal(storage): class TestWorkerSignal: - # Test that worker propagates the SIGTERM signal to the task executor. - # Submit a task that checks whether the task executor receives the SIGTERM signal. - # The task should return the SIGTERM signal number as the output. - # Later task should not be executed. - # Worker should exit with SIGTERM. - def test_task_signal(self, storage, scheduler_worker_signal): + + def test_task_signal(self, storage, scheduler_worker_signal) -> None: + """ + Test that worker propagates the SIGTERM signal to the task executor. + Submit a task that checks whether the task executor receives the SIGTERM signal. + The task should return the SIGTERM signal number as the output. + Later task should not be executed. + Worker should exit with SIGTERM. + + :param storage: + :param scheduler_worker_signal: + :return: None + """ _, worker_process = scheduler_worker_signal # Submit signal handler task to check for SIGTERM signal in task executor @@ -133,11 +153,17 @@ def test_task_signal(self, storage, scheduler_worker_signal): remove_job(storage, new_graph.id) remove_job(storage, graph.id) - # Test that worker propagates the SIGTERM signal to the task executor. - # Task executor exits immediately after receiving the signal. - # The running task should be marked as failed. - # The worker should exit with SIGTERM. - def test_task_exit(self, storage, scheduler_worker_signal): + def test_task_exit(self, storage, scheduler_worker_signal) -> None: + """ + Test that worker propagates the SIGTERM signal to the task executor. + Task executor exits immediately after receiving the signal. + The running task should be marked as failed. + The worker should exit with SIGTERM. + + :param storage: + :param scheduler_worker_signal: + :return: None + """ _, worker_process = scheduler_worker_signal # Submit a task to sleep for 10 seconds From 573b4487fc933e8cc852e90af321bcf338517a77 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:18:04 -0400 Subject: [PATCH 020/198] Fix ruff lint --- tests/integration/client.py | 8 +++++++- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 3 +-- tests/integration/test_signal.py | 6 ++++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 834ae2b5e..99662b542 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,7 +1,7 @@ import re import uuid -from dataclasses import dataclass from collections.abc import Generator +from dataclasses import dataclass import mysql.connector import pytest @@ -13,6 +13,7 @@ class TaskInput: TaskInput represents an input to a task. It can either be a direct value, a reference to another task's output, or a reference to data. """ + type: str task_output: tuple[uuid.UUID, int] | None = None value: str | None = None @@ -25,6 +26,7 @@ class TaskOutput: TaskOutput represents an output of a task. It can either be a direct value or a reference to data. """ + type: str value: str | None = None data_id: uuid.UUID | None = None @@ -35,6 +37,7 @@ class Task: """ Task represents a unit of work in the task graph. """ + id: uuid.UUID function_name: str inputs: list[TaskInput] @@ -48,6 +51,7 @@ class TaskGraph: """ TaskGraph represents a directed acyclic graph of tasks. """ + id: uuid.UUID tasks: dict[uuid.UUID, Task] dependencies: list[tuple[uuid.UUID, uuid.UUID]] @@ -58,6 +62,7 @@ class Driver: """ Driver represents a client that can submit jobs to the task graph. """ + id: uuid.UUID @@ -66,6 +71,7 @@ class Data: """ Data represents a Spider Data object. """ + id: uuid.UUID value: str diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 7d6edeebe..c1d155043 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,7 +1,7 @@ import subprocess import time -from pathlib import Path from collections.abc import Generator +from pathlib import Path import pytest diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index f138f9f81..9c6ac27e4 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -1,13 +1,12 @@ import subprocess import time import uuid -from pathlib import Path from collections.abc import Generator +from pathlib import Path import msgpack import pytest -from tests.integration.client import TaskGraph, Task from .client import ( add_driver, add_driver_data, diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 012fac7b9..764bf8185 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -3,8 +3,8 @@ import subprocess import time import uuid -from pathlib import Path from collections.abc import Generator +from pathlib import Path import msgpack import pytest @@ -32,6 +32,7 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t :param lib: Library to load in the worker. :return: scheduler and worker processes. """ + root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -61,11 +62,12 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t @pytest.fixture(scope="function") @pytest.mark.usefixtures("storage") -def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: """ + scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) From 6ad72c3b93e9415b6fddd291956435fda79db005 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:34:04 -0400 Subject: [PATCH 021/198] Bug fix --- ruff.toml | 1 + tests/integration/client.py | 43 +++++++++------------- tests/integration/test_client.py | 4 ++ tests/integration/test_scheduler_worker.py | 3 ++ tests/integration/test_signal.py | 5 ++- tests/integration/utils.py | 2 + 6 files changed, 31 insertions(+), 27 deletions(-) diff --git a/ruff.toml b/ruff.toml index a7a590d31..75462c9cd 100644 --- a/ruff.toml +++ b/ruff.toml @@ -11,6 +11,7 @@ ignore = [ "D400", # First line of docstrings may not end in period "D401", # Docstrings should be written in present tense (not imperative) "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations "FBT", # Allow bool positional parameters since other value positions are allowed "FIX002", # Allow todo statements "PERF401", # Allow for loops when creating lists diff --git a/tests/integration/client.py b/tests/integration/client.py index 99662b542..682038e43 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -1,3 +1,5 @@ +"""Simple Spider client for testing purposes.""" + import re import uuid from collections.abc import Generator @@ -34,9 +36,7 @@ class TaskOutput: @dataclass class Task: - """ - Task represents a unit of work in the task graph. - """ + """Task represents a unit of work in the task graph.""" id: uuid.UUID function_name: str @@ -48,9 +48,7 @@ class Task: @dataclass class TaskGraph: - """ - TaskGraph represents a directed acyclic graph of tasks. - """ + """TaskGraph represents a directed acyclic graph of tasks.""" id: uuid.UUID tasks: dict[uuid.UUID, Task] @@ -59,18 +57,14 @@ class TaskGraph: @dataclass class Driver: - """ - Driver represents a client that can submit jobs to the task graph. - """ + """Driver represents a client that can submit jobs to the task graph.""" id: uuid.UUID @dataclass class Data: - """ - Data represents a Spider Data object. - """ + """Data represents a Spider Data object.""" id: uuid.UUID value: str @@ -87,7 +81,7 @@ def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: ) match = pattern.match(storage_url) if not match: - raise ValueError("Invalid JDBC URL format") + raise ValueError(storage_url) connection_params = match.groupdict() return mysql.connector.connect( @@ -103,7 +97,8 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU """ Check if the task is a head task, meaning it has no parent. :param task_id: the ID of the task to check. - :param dependencies: list of dependencies where each dependency is a tuple (parent_id, child_id). + :param dependencies: list of dependencies where each dependency is a tuple + (parent_id, child_id). :return: True if the task has no parent, False otherwise. """ return not any(dependency[1] == task_id for dependency in dependencies) @@ -124,7 +119,8 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: conn.close() -def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: +def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph)\ + -> None: """ Submit a job to the database. :param conn: database connection object. @@ -139,10 +135,7 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: ) for task_id, task in graph.tasks.items(): - if is_head_task(task_id, graph.dependencies): - state = "ready" - else: - state = "pending" + state = "ready" if is_head_task(task_id, graph.dependencies) else "pending" cursor.execute( "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry) VALUES (%s, %s, %s, %s, %s, %s)", ( @@ -185,7 +178,7 @@ def submit_job(conn, client_id: uuid.UUID, graph: TaskGraph) -> None: cursor.close() -def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: +def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: """ Get the outputs of a task by its ID. :param conn: database connection object. @@ -212,7 +205,7 @@ def get_task_outputs(conn, task_id: uuid.UUID) -> list[TaskOutput]: return outputs -def get_task_state(conn, task_id: uuid.UUID) -> str: +def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> str: """ Get the state of a task by its ID. :param conn: database connection object. @@ -229,7 +222,7 @@ def get_task_state(conn, task_id: uuid.UUID) -> str: return state -def remove_job(conn, job_id: uuid.UUID) -> None: +def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None: """ Remove a job from the database by its ID. :param conn: database connection object. @@ -243,7 +236,7 @@ def remove_job(conn, job_id: uuid.UUID) -> None: cursor.close() -def add_driver(conn, driver: Driver) -> None: +def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: """ Register a new driver in the database. :param conn: database connection object. @@ -258,7 +251,7 @@ def add_driver(conn, driver: Driver) -> None: cursor.close() -def add_driver_data(conn, driver: Driver, data: Data) -> None: +def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: Data) -> None: """ Add a new data associated with a driver in the database. :param conn: database connection object. @@ -278,7 +271,7 @@ def add_driver_data(conn, driver: Driver, data: Data) -> None: cursor.close() -def remove_data(conn, data: Data) -> None: +def remove_data(conn: mysql.connector.MySQLConnection, data: Data) -> None: """ Remove data from the database by its ID. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index c1d155043..6fbbd6ad8 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -1,3 +1,5 @@ +"""Integration test for the client_test C++ program.""" + import subprocess import time from collections.abc import Generator @@ -68,6 +70,8 @@ def scheduler_worker() -> Generator[None, None, None]: class TestClient: + """Test class for the client_test C++ program.""" + @pytest.mark.usefixtures("scheduler_worker") def test_client(self) -> None: """ diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 9c6ac27e4..93f2cde2a 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -1,3 +1,5 @@ +"""Integration tests for the scheduler and worker processes.""" + import subprocess import time import uuid @@ -248,6 +250,7 @@ def random_fail_job(storage): class TestSchedulerWorker: + """Test class for the scheduler and worker integration tests.""" @pytest.mark.usefixtures("scheduler_worker") def test_job_success(self, storage, success_job): """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 764bf8185..c9605b898 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -1,3 +1,5 @@ +"""Integration tests for worker signal handling.""" + import os import signal import subprocess @@ -32,7 +34,6 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t :param lib: Library to load in the worker. :return: scheduler and worker processes. """ - root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) @@ -67,7 +68,6 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po Fixture to start a scheduler and a worker process for testing signal handling. :return: """ - scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -79,6 +79,7 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po class TestWorkerSignal: + """Test cases for worker signal handling.""" def test_task_signal(self, storage, scheduler_worker_signal) -> None: diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 33aada438..79e4bf950 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -1,3 +1,5 @@ +"""Utilities for the network port.""" + import socket From ba7c6e58e9d6936149bb1d5ba397e0c1c076ec2c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 13:57:00 -0400 Subject: [PATCH 022/198] Fix ruff --- ruff.toml | 4 +++- tests/integration/client.py | 6 ++++-- tests/integration/test_client.py | 5 ++--- tests/integration/test_scheduler_worker.py | 16 ++++++++-------- tests/integration/test_signal.py | 9 ++++----- 5 files changed, 21 insertions(+), 19 deletions(-) diff --git a/ruff.toml b/ruff.toml index 75462c9cd..d2294f8d0 100644 --- a/ruff.toml +++ b/ruff.toml @@ -26,9 +26,11 @@ isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "S101", # Allow use of `assert` (security warning) "ANN001", # Ignore missing type annotations for arguments "ANN201", # Ignore missing type annotations for parameters + "S101", # Allow use of `assert` (security warning) + "S603", # Allow user of subprocess.Popen (security warning) + "T201", # Allow use of `print` (testing) ] [format] diff --git a/tests/integration/client.py b/tests/integration/client.py index 682038e43..4ed1258dd 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -137,7 +137,8 @@ def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, grap for task_id, task in graph.tasks.items(): state = "ready" if is_head_task(task_id, graph.dependencies) else "pending" cursor.execute( - "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry) VALUES (%s, %s, %s, %s, %s, %s)", + "INSERT INTO tasks (id, job_id, func_name, state, timeout, max_retry)" + " VALUES (%s, %s, %s, %s, %s, %s)", ( task.id.bytes, graph.id.bytes, @@ -150,7 +151,8 @@ def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, grap for i, task_input in enumerate(task.inputs): cursor.execute( - "INSERT INTO task_inputs (type, task_id, position, output_task_id, output_task_position, value, data_id) VALUES (%s, %s, %s, %s, %s, %s, %s)", + "INSERT INTO task_inputs (type, task_id, position, output_task_id," + " output_task_position, value, data_id) VALUES (%s, %s, %s, %s, %s, %s, %s)", ( task_input.type, task.id.bytes, diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 6fbbd6ad8..a00dc8a6f 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -51,8 +51,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -@pytest.mark.usefixtures("storage") -def scheduler_worker() -> Generator[None, None, None]: +def scheduler_worker(storage) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, and then kills the processes after the test class is done. @@ -85,5 +84,5 @@ def test_client(self) -> None: "--storage_url", g_storage_url, ] - p = subprocess.run(client_cmds, check=False, timeout=20) + p = subprocess.run(client_cmds, check=True, timeout=20) assert p.returncode == 0 diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 93f2cde2a..1a2f88877 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -65,8 +65,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -@pytest.mark.usefixtures("storage") -def scheduler_worker() -> Generator[None, None, None]: +def scheduler_worker(storage) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. After the test function completes, it kills the scheduler and the worker process. @@ -82,11 +81,11 @@ def scheduler_worker() -> Generator[None, None, None]: worker_process.kill() -@pytest.fixture(scope="function") +@pytest.fixture def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ - Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. - Cleans up the job after the test function completes. + Fixture to create a job with two parent tasks and one child task. Yields the task graph and + tasks. Cleans up the job after the test function completes. :param storage: :return: """ @@ -146,7 +145,7 @@ def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, remove_job(storage, graph.id) -@pytest.fixture(scope="function") +@pytest.fixture def fail_job(storage) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -174,7 +173,7 @@ def fail_job(storage) -> Generator[Task, None, None]: remove_job(storage, graph.id) -@pytest.fixture(scope="function") +@pytest.fixture def data_job(storage) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -211,7 +210,7 @@ def data_job(storage) -> Generator[Task, None, None]: remove_data(storage, data) -@pytest.fixture(scope="function") +@pytest.fixture def random_fail_job(storage): """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -251,6 +250,7 @@ def random_fail_job(storage): class TestSchedulerWorker: """Test class for the scheduler and worker integration tests.""" + @pytest.mark.usefixtures("scheduler_worker") def test_job_success(self, storage, success_job): """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index c9605b898..0edb71645 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,7 +26,8 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> tuple[subprocess.Popen, subprocess.Popen]: +def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ + -> tuple[subprocess.Popen, subprocess.Popen]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -61,9 +62,8 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str) -> t return scheduler_process, worker_process -@pytest.fixture(scope="function") -@pytest.mark.usefixtures("storage") -def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +@pytest.fixture +def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -81,7 +81,6 @@ def scheduler_worker_signal() -> Generator[tuple[subprocess.Popen, subprocess.Po class TestWorkerSignal: """Test cases for worker signal handling.""" - def test_task_signal(self, storage, scheduler_worker_signal) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From 209acb14a38a3f13c2622f2e9c0a210115f5038a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 14:56:59 -0400 Subject: [PATCH 023/198] Fix ruff --- ruff.toml | 2 - tests/integration/client.py | 7 +++- tests/integration/test_client.py | 7 +++- tests/integration/test_scheduler_worker.py | 47 +++++++++++++++++----- tests/integration/test_signal.py | 22 +++++++--- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/ruff.toml b/ruff.toml index d2294f8d0..6cdab7927 100644 --- a/ruff.toml +++ b/ruff.toml @@ -26,8 +26,6 @@ isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "ANN001", # Ignore missing type annotations for arguments - "ANN201", # Ignore missing type annotations for parameters "S101", # Allow use of `assert` (security warning) "S603", # Allow user of subprocess.Popen (security warning) "T201", # Allow use of `print` (testing) diff --git a/tests/integration/client.py b/tests/integration/client.py index 4ed1258dd..886a6c3c3 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -119,8 +119,11 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: conn.close() -def submit_job(conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph)\ - -> None: +def submit_job( + conn: mysql.connector.MySQLConnection, + client_id: uuid.UUID, + graph: TaskGraph +) -> None: """ Submit a job to the database. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index a00dc8a6f..aac10a9ce 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -5,11 +5,11 @@ from collections.abc import Generator from pathlib import Path +import mysql.connector import pytest from .client import ( g_storage_url, - storage, ) from .utils import g_scheduler_port @@ -51,12 +51,15 @@ def start_scheduler_workers( @pytest.fixture(scope="class") -def scheduler_worker(storage) -> Generator[None, None, None]: +def scheduler_worker( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, and then kills the processes after the test class is done. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process_0, worker_process_1 = start_scheduler_workers( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 1a2f88877..84293fc42 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -7,6 +7,7 @@ from pathlib import Path import msgpack +import mysql.connector import pytest from .client import ( @@ -19,7 +20,6 @@ get_task_state, remove_data, remove_job, - storage, submit_job, Task, TaskGraph, @@ -65,12 +65,15 @@ def start_scheduler_worker( @pytest.fixture(scope="class") -def scheduler_worker(storage) -> Generator[None, None, None]: +def scheduler_worker( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. After the test function completes, it kills the scheduler and the worker process. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port ) @@ -82,7 +85,9 @@ def scheduler_worker(storage) -> Generator[None, None, None]: @pytest.fixture -def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: +def success_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +)-> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. Cleans up the job after the test function completes. @@ -146,7 +151,9 @@ def success_job(storage) -> Generator[tuple[TaskGraph, Task, Task, Task], None, @pytest.fixture -def fail_job(storage) -> Generator[Task, None, None]: +def fail_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. Yield the task. Cleanup the job after the test function completes. @@ -174,7 +181,9 @@ def fail_job(storage) -> Generator[Task, None, None]: @pytest.fixture -def data_job(storage) -> Generator[Task, None, None]: +def data_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. Cleans up the job and data after the test function completes. @@ -211,7 +220,9 @@ def data_job(storage) -> Generator[Task, None, None]: @pytest.fixture -def random_fail_job(storage): +def random_fail_job( + storage: Generator[mysql.connector.MySQLConnection, None, None] +) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. Yields the task. Cleans up the job after the test function completes. @@ -252,7 +263,11 @@ class TestSchedulerWorker: """Test class for the scheduler and worker integration tests.""" @pytest.mark.usefixtures("scheduler_worker") - def test_job_success(self, storage, success_job): + def test_job_success( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None] + ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. :param storage: @@ -279,7 +294,11 @@ def test_job_success(self, storage, success_job): assert outputs[0].value == msgpack.packb(10) @pytest.mark.usefixtures("scheduler_worker") - def test_job_failure(self, storage, fail_job): + def test_job_failure( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + fail_job: Generator[Task, None, None] + ) -> None: """ Test the failure of a job that raise an error. :param storage: @@ -293,7 +312,11 @@ def test_job_failure(self, storage, fail_job): assert state == "fail" @pytest.mark.usefixtures("scheduler_worker") - def test_data_job(self, storage, data_job): + def test_data_job( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + data_job: Generator[Task, None, None] + ) -> None: """ Test the successful execution of a job that uses data. :param storage: @@ -310,7 +333,11 @@ def test_data_job(self, storage, data_job): assert outputs[0].value == msgpack.packb(2) @pytest.mark.usefixtures("scheduler_worker") - def test_random_fail_job(self, storage, random_fail_job): + def test_random_fail_job( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + random_fail_job: Generator[Task, None, None] + ) -> None: """ Test the successful recovery and execution of a job that randomly fails. :param storage: diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 0edb71645..0da6466a4 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -9,6 +9,7 @@ from pathlib import Path import msgpack +import mysql.connector import pytest from .client import ( @@ -16,7 +17,6 @@ get_task_outputs, get_task_state, remove_job, - storage, submit_job, Task, TaskGraph, @@ -37,7 +37,7 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" - popen_opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + popen_opts = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True} scheduler_cmds = [ str(bin_dir / "spider_scheduler"), "--host", @@ -63,11 +63,13 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ @pytest.fixture -def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal(storage: Generator[mysql.connector.MySQLConnection, None, None])\ + -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: """ + _ = storage # Avoid ARG001 scheduler_process, worker_process = start_scheduler_worker( storage_url=g_storage_url, scheduler_port=g_scheduler_port, lib="tests/libsignal_test.so" ) @@ -81,7 +83,12 @@ def scheduler_worker_signal(storage) -> Generator[tuple[subprocess.Popen, subpro class TestWorkerSignal: """Test cases for worker signal handling.""" - def test_task_signal(self, storage, scheduler_worker_signal) -> None: + def test_task_signal( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[ + tuple[subprocess.Popen, subprocess.Popen], None, None] + ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. Submit a task that checks whether the task executor receives the SIGTERM signal. @@ -155,7 +162,12 @@ def test_task_signal(self, storage, scheduler_worker_signal) -> None: remove_job(storage, new_graph.id) remove_job(storage, graph.id) - def test_task_exit(self, storage, scheduler_worker_signal) -> None: + def test_task_exit( + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[ + tuple[subprocess.Popen, subprocess.Popen], None, None] + ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. Task executor exits immediately after receiving the signal. From 406b514007a05e9701ddcacd0638dd5d20ad4e0e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 14:57:52 -0400 Subject: [PATCH 024/198] Reformat files --- tests/integration/client.py | 4 +-- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 36 +++++++++++----------- tests/integration/test_signal.py | 24 +++++++-------- 4 files changed, 32 insertions(+), 34 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 886a6c3c3..721ec2444 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -120,9 +120,7 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: def submit_job( - conn: mysql.connector.MySQLConnection, - client_id: uuid.UUID, - graph: TaskGraph + conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph ) -> None: """ Submit a job to the database. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index aac10a9ce..f70f51987 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 84293fc42..2f0421c05 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,8 +86,8 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] -)-> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: + storage: Generator[mysql.connector.MySQLConnection, None, None], +) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and tasks. Cleans up the job after the test function completes. @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None] + storage: Generator[mysql.connector.MySQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -264,9 +264,9 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. @@ -295,9 +295,9 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - fail_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + fail_job: Generator[Task, None, None], ) -> None: """ Test the failure of a job that raise an error. @@ -313,9 +313,9 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - data_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + data_job: Generator[Task, None, None], ) -> None: """ Test the successful execution of a job that uses data. @@ -334,9 +334,9 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - random_fail_job: Generator[Task, None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + random_fail_job: Generator[Task, None, None], ) -> None: """ Test the successful recovery and execution of a job that randomly fails. diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 0da6466a4..28b275346 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,8 +26,9 @@ from .utils import g_scheduler_port -def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ - -> tuple[subprocess.Popen, subprocess.Popen]: +def start_scheduler_worker( + storage_url: str, scheduler_port: int, lib: str +) -> tuple[subprocess.Popen, subprocess.Popen]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -63,8 +64,9 @@ def start_scheduler_worker(storage_url: str, scheduler_port: int, lib: str)\ @pytest.fixture -def scheduler_worker_signal(storage: Generator[mysql.connector.MySQLConnection, None, None])\ - -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +def scheduler_worker_signal( + storage: Generator[mysql.connector.MySQLConnection, None, None], +) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -84,10 +86,9 @@ class TestWorkerSignal: """Test cases for worker signal handling.""" def test_task_signal( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - scheduler_worker_signal: Generator[ - tuple[subprocess.Popen, subprocess.Popen], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. @@ -163,10 +164,9 @@ def test_task_signal( remove_job(storage, graph.id) def test_task_exit( - self, - storage: Generator[mysql.connector.MySQLConnection, None, None], - scheduler_worker_signal: Generator[ - tuple[subprocess.Popen, subprocess.Popen], None, None] + self, + storage: Generator[mysql.connector.MySQLConnection, None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From 66892f5e5194490d9dd8af79ecb9db2ac63740fe Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 16:08:45 -0400 Subject: [PATCH 025/198] Remove .inc from cpp linting --- lint-tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 659480e50..161c8a193 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -101,7 +101,7 @@ tasks: FLAGS: - "--config-file '{{.ROOT_DIR}}/.clang-tidy'" - "-p '{{.G_EXAMPLES_COMPILE_COMMANDS_DB}}'" - INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp", "*.inc"] + INCLUDE_FILENAME_PATTERNS: ["*.cpp", "*.h", "*.hpp"] OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" ROOT_PATHS: - "{{.G_EXAMPLES_DIR}}" From 3186a429f9a79ab93750deaa47c3d542a3aaa0af Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 18:41:52 -0400 Subject: [PATCH 026/198] Add mypy and merge lint and test requirements.txt --- lint-tasks.yaml | 5 +++-- lint-requirements.txt => lint-test-requirements.txt | 7 +++++++ mypy.ini | 8 ++++++++ test-requirements.txt | 3 --- test-tasks.yaml | 4 ++-- 5 files changed, 20 insertions(+), 7 deletions(-) rename lint-requirements.txt => lint-test-requirements.txt (57%) create mode 100644 mypy.ini delete mode 100644 test-requirements.txt diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 7822bda7f..3739fa12d 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -130,6 +130,7 @@ tasks: - "tests/integration" cmd: |- . "{{.G_LINT_VENV_DIR}}/bin/activate" + mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" @@ -177,7 +178,7 @@ tasks: sources: - "{{.ROOT_DIR}}/taskfile.yaml" - "{{.TASKFILE}}" - - "lint-requirements.txt" + - "lint-test-requirements.txt" generates: ["{{.CHECKSUM_FILE}}"] run: "once" deps: @@ -191,7 +192,7 @@ tasks: vars: LABEL: "lint" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-test-requirements.txt" # This command must be last - task: ":utils:checksum:compute" vars: diff --git a/lint-requirements.txt b/lint-test-requirements.txt similarity index 57% rename from lint-requirements.txt rename to lint-test-requirements.txt index 690b9797f..7ac75af21 100644 --- a/lint-requirements.txt +++ b/lint-test-requirements.txt @@ -1,6 +1,13 @@ +# Lint dependencies clang-format>=20.1.0 # Lock to v19.x until we can upgrade our code to fix new v20 issues. clang-tidy~=19.1 ruff>=0.4.4 +mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 + +# Test dependencies +msgpack>=1.1.0 +mysql-connector-python>=8.0.26 +pytest>=8.3.4 diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..4c66b1114 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,8 @@ +[mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true diff --git a/test-requirements.txt b/test-requirements.txt deleted file mode 100644 index ec24b0b20..000000000 --- a/test-requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -msgpack>=1.1.0 -mysql-connector-python>=8.0.26 -pytest>=8.3.4 \ No newline at end of file diff --git a/test-tasks.yaml b/test-tasks.yaml index 116ae85b6..41dfc4578 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -56,7 +56,7 @@ tasks: sources: - "{{.ROOT_DIR}}/taskfile.yaml" - "{{.TASKFILE}}" - - "test-requirements.txt" + - "lint-test-requirements.txt" generates: ["{{.CHECKSUM_FILE}}"] run: "once" deps: @@ -70,7 +70,7 @@ tasks: vars: LABEL: "test" OUTPUT_DIR: "{{.OUTPUT_DIR}}" - REQUIREMENTS_FILE: "{{.ROOT_DIR}}/test-requirements.txt" + REQUIREMENTS_FILE: "{{.ROOT_DIR}}/lint-test-requirements.txt" # This command must be last - task: ":utils:checksum:compute" vars: From 092593844041d72e6da23797a7c8fb16fbf1d258 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:34:15 -0400 Subject: [PATCH 027/198] Fix mysql connection type --- tests/integration/client.py | 19 ++++++++++--------- tests/integration/test_client.py | 4 ++-- tests/integration/test_scheduler_worker.py | 20 ++++++++++---------- tests/integration/test_signal.py | 8 ++++---- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 721ec2444..f0b7f0cfe 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -69,8 +69,9 @@ class Data: id: uuid.UUID value: str +SQLConnection = mysql.connector.abstracts.MySQLConnectionAbstract | mysql.connector.pooling.PooledMySQLConnection -def create_connection(storage_url: str) -> mysql.connector.MySQLConnection: +def create_connection(storage_url: str) -> SQLConnection : """ Creation a MariaDB connection from a JDBC URL. :param storage_url: JDBC URL for the MariaDB database. @@ -108,7 +109,7 @@ def is_head_task(task_id: uuid.UUID, dependencies: list[tuple[uuid.UUID, uuid.UU @pytest.fixture(scope="session") -def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: +def storage() -> Generator[SQLConnection, None, None]: """ Fixture to create a database connection for the test session. Yields a connection object and ensures it is closed after the tests are done. @@ -120,7 +121,7 @@ def storage() -> Generator[mysql.connector.MySQLConnection, None, None]: def submit_job( - conn: mysql.connector.MySQLConnection, client_id: uuid.UUID, graph: TaskGraph + conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph ) -> None: """ Submit a job to the database. @@ -181,7 +182,7 @@ def submit_job( cursor.close() -def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: +def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput]: """ Get the outputs of a task by its ID. :param conn: database connection object. @@ -208,7 +209,7 @@ def get_task_outputs(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) return outputs -def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> str: +def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: """ Get the state of a task by its ID. :param conn: database connection object. @@ -225,7 +226,7 @@ def get_task_state(conn: mysql.connector.MySQLConnection, task_id: uuid.UUID) -> return state -def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None: +def remove_job(conn: SQLConnection, job_id: uuid.UUID) -> None: """ Remove a job from the database by its ID. :param conn: database connection object. @@ -239,7 +240,7 @@ def remove_job(conn: mysql.connector.MySQLConnection, job_id: uuid.UUID) -> None cursor.close() -def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: +def add_driver(conn: SQLConnection, driver: Driver) -> None: """ Register a new driver in the database. :param conn: database connection object. @@ -254,7 +255,7 @@ def add_driver(conn: mysql.connector.MySQLConnection, driver: Driver) -> None: cursor.close() -def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: Data) -> None: +def add_driver_data(conn: SQLConnection, driver: Driver, data: Data) -> None: """ Add a new data associated with a driver in the database. :param conn: database connection object. @@ -274,7 +275,7 @@ def add_driver_data(conn: mysql.connector.MySQLConnection, driver: Driver, data: cursor.close() -def remove_data(conn: mysql.connector.MySQLConnection, data: Data) -> None: +def remove_data(conn: SQLConnection, data: Data) -> None: """ Remove data from the database by its ID. :param conn: database connection object. diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index f70f51987..809085924 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -5,11 +5,11 @@ from collections.abc import Generator from pathlib import Path -import mysql.connector import pytest from .client import ( g_storage_url, + SQLConnection ) from .utils import g_scheduler_port @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 2f0421c05..190875688 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -7,7 +7,6 @@ from pathlib import Path import msgpack -import mysql.connector import pytest from .client import ( @@ -20,6 +19,7 @@ get_task_state, remove_data, remove_job, + SQLConnection, submit_job, Task, TaskGraph, @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,7 +86,7 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -265,7 +265,7 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], ) -> None: """ @@ -296,7 +296,7 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], fail_job: Generator[Task, None, None], ) -> None: """ @@ -314,7 +314,7 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], data_job: Generator[Task, None, None], ) -> None: """ @@ -335,7 +335,7 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], random_fail_job: Generator[Task, None, None], ) -> None: """ diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 28b275346..42eee6bac 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -9,7 +9,6 @@ from pathlib import Path import msgpack -import mysql.connector import pytest from .client import ( @@ -17,6 +16,7 @@ get_task_outputs, get_task_state, remove_job, + SQLConnection, submit_job, Task, TaskGraph, @@ -65,7 +65,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], ) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. @@ -87,7 +87,7 @@ class TestWorkerSignal: def test_task_signal( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ @@ -165,7 +165,7 @@ def test_task_signal( def test_task_exit( self, - storage: Generator[mysql.connector.MySQLConnection, None, None], + storage: Generator[SQLConnection, None, None], scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], ) -> None: """ From 6936a9b448a6ce4ececbc97d53e23b67569850e4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:39:24 -0400 Subject: [PATCH 028/198] Fix socket name type --- tests/integration/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 79e4bf950..cae3f1730 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,12 +2,16 @@ import socket +IPv4Addr = tuple[str, int] +IPv6Addr = tuple[str, int, int, int] +AddrType = IPv4Addr | IPv6Addr def _get_free_tcp_port() -> int: """Returns a free TCP port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(("127.0.0.1", 0)) - return s.getsockname()[1] + addr: AddrType = s.getsockname() + return addr[1] g_scheduler_port = _get_free_tcp_port() From 74070908318632b16e7dde6800994676d19e8982 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:43:07 -0400 Subject: [PATCH 029/198] Fix return type from db cursor --- tests/integration/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index f0b7f0cfe..b4369fe3a 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -196,7 +196,8 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - for output_type, value, data_id in cursor.fetchall(): + rows: list[tuple[str, str | None, bytes | None]] = cursor.fetchall() + for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) elif data_id is not None: @@ -219,7 +220,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state = cursor.fetchone()[0] + state: str = cursor.fetchone()[0] conn.commit() cursor.close() From 6a95b24912c69807c9cc66172f97a5441beea8e6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:50:34 -0400 Subject: [PATCH 030/198] Fix db cursor return type --- tests/integration/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index b4369fe3a..13f9666a9 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -4,6 +4,7 @@ import uuid from collections.abc import Generator from dataclasses import dataclass +from typing import cast import mysql.connector import pytest @@ -196,7 +197,7 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - rows: list[tuple[str, str | None, bytes | None]] = cursor.fetchall() + rows = cast(list[tuple[str, str | None, bytes | None]], cursor.fetchall()) for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) @@ -220,7 +221,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state: str = cursor.fetchone()[0] + state = cast(tuple[str], cursor.fetchone())[0] conn.commit() cursor.close() From f29dcba479f5f80d66c99aa94fbf7ab93ecfc4c2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 19:55:24 -0400 Subject: [PATCH 031/198] Fix mypy import untyped --- mypy.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mypy.ini b/mypy.ini index 4c66b1114..932c4e402 100644 --- a/mypy.ini +++ b/mypy.ini @@ -6,3 +6,6 @@ pretty = true show_error_code_links = true show_error_context = true show_error_end = true + +[mypy-msgpack.*] +ignore_missing_imports = true From 453f20908e8f566d7561e1a3c178c713e206e915 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:17:20 -0400 Subject: [PATCH 032/198] Fix mypy and Popen --- tests/integration/test_signal.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 42eee6bac..47848f636 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -7,6 +7,7 @@ import uuid from collections.abc import Generator from pathlib import Path +from typing import TypedDict import msgpack import pytest @@ -25,10 +26,16 @@ ) from .utils import g_scheduler_port +class PopenOpts(TypedDict, total=False): + stdin: int + stdout: int + stderr: int + text: bool + def start_scheduler_worker( storage_url: str, scheduler_port: int, lib: str -) -> tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Create a scheduler and a worker process. :param storage_url: JDB storage URL. @@ -38,7 +45,11 @@ def start_scheduler_worker( """ root_dir = Path(__file__).resolve().parents[2] bin_dir = root_dir / "src" / "spider" - popen_opts = {"stdout": subprocess.PIPE, "stderr": subprocess.PIPE, "text": True} + popen_opts: PopenOpts = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "text": True, + } scheduler_cmds = [ str(bin_dir / "spider_scheduler"), "--host", @@ -66,7 +77,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( storage: Generator[SQLConnection, None, None], -) -> Generator[tuple[subprocess.Popen, subprocess.Popen], None, None]: +) -> Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. :return: @@ -88,7 +99,7 @@ class TestWorkerSignal: def test_task_signal( self, storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], + scheduler_worker_signal: Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From d1011c70eb908fad0247f84d845be67c89d7bbd4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:22:31 -0400 Subject: [PATCH 033/198] Fix generator type hint --- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 26 +++++++++++----------- tests/integration/test_signal.py | 10 ++++----- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index 809085924..c96b731df 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -52,7 +52,7 @@ def start_scheduler_workers( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[None, None, None]: """ Fixture to start the scheduler and two worker processes. Yields control to the test class, diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 190875688..2aa59ed50 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -66,7 +66,7 @@ def start_scheduler_worker( @pytest.fixture(scope="class") def scheduler_worker( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[None, None, None]: """ Fixture to start a scheduler and a worker process. Yields control to the test function. @@ -86,7 +86,7 @@ def scheduler_worker( @pytest.fixture def success_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[tuple[TaskGraph, Task, Task, Task], None, None]: """ Fixture to create a job with two parent tasks and one child task. Yields the task graph and @@ -152,7 +152,7 @@ def success_job( @pytest.fixture def fail_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that will fail. The task will raise an error when executed. @@ -182,7 +182,7 @@ def fail_job( @pytest.fixture def data_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that uses data. Yields the task that uses data. @@ -221,7 +221,7 @@ def data_job( @pytest.fixture def random_fail_job( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[Task, None, None]: """ Fixture to create a job that randomly fails. The task will succeed after a few retries. @@ -265,8 +265,8 @@ class TestSchedulerWorker: @pytest.mark.usefixtures("scheduler_worker") def test_job_success( self, - storage: Generator[SQLConnection, None, None], - success_job: Generator[tuple[TaskGraph, Task, Task, Task], None, None], + storage: SQLConnection, + success_job: tuple[TaskGraph, Task, Task, Task], ) -> None: """ Test the successful execution of a job with two parent tasks and one child task. @@ -296,8 +296,8 @@ def test_job_success( @pytest.mark.usefixtures("scheduler_worker") def test_job_failure( self, - storage: Generator[SQLConnection, None, None], - fail_job: Generator[Task, None, None], + storage: SQLConnection, + fail_job: Task, ) -> None: """ Test the failure of a job that raise an error. @@ -314,8 +314,8 @@ def test_job_failure( @pytest.mark.usefixtures("scheduler_worker") def test_data_job( self, - storage: Generator[SQLConnection, None, None], - data_job: Generator[Task, None, None], + storage: SQLConnection, + data_job: Task, ) -> None: """ Test the successful execution of a job that uses data. @@ -335,8 +335,8 @@ def test_data_job( @pytest.mark.usefixtures("scheduler_worker") def test_random_fail_job( self, - storage: Generator[SQLConnection, None, None], - random_fail_job: Generator[Task, None, None], + storage: SQLConnection, + random_fail_job: Task, ) -> None: """ Test the successful recovery and execution of a job that randomly fails. diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 47848f636..667e23d51 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -76,7 +76,7 @@ def start_scheduler_worker( @pytest.fixture def scheduler_worker_signal( - storage: Generator[SQLConnection, None, None], + storage: SQLConnection, ) -> Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None]: """ Fixture to start a scheduler and a worker process for testing signal handling. @@ -98,8 +98,8 @@ class TestWorkerSignal: def test_task_signal( self, - storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], None, None], + storage: SQLConnection, + scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. @@ -176,8 +176,8 @@ def test_task_signal( def test_task_exit( self, - storage: Generator[SQLConnection, None, None], - scheduler_worker_signal: Generator[tuple[subprocess.Popen, subprocess.Popen], None, None], + storage: SQLConnection, + scheduler_worker_signal: tuple[subprocess.Popen, subprocess.Popen], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From a7dc642bcb68e83f52152b27bd811f358843ad2e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:24:47 -0400 Subject: [PATCH 034/198] Fix mypy --- tests/integration/test_client.py | 2 +- tests/integration/test_scheduler_worker.py | 2 +- tests/integration/test_signal.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index c96b731df..e5219835c 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -16,7 +16,7 @@ def start_scheduler_workers( storage_url: str, scheduler_port: int -) -> tuple[subprocess.Popen, subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Start the scheduler and two worker processes. :param storage_url: diff --git a/tests/integration/test_scheduler_worker.py b/tests/integration/test_scheduler_worker.py index 2aa59ed50..ae841332e 100644 --- a/tests/integration/test_scheduler_worker.py +++ b/tests/integration/test_scheduler_worker.py @@ -31,7 +31,7 @@ def start_scheduler_worker( storage_url: str, scheduler_port: int -) -> tuple[subprocess.Popen, subprocess.Popen]: +) -> tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]: """ Start a scheduler and a worker process. :param storage_url: JDBC storage URL diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index 667e23d51..f42fdefc3 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -177,7 +177,7 @@ def test_task_signal( def test_task_exit( self, storage: SQLConnection, - scheduler_worker_signal: tuple[subprocess.Popen, subprocess.Popen], + scheduler_worker_signal: tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]], ) -> None: """ Test that worker propagates the SIGTERM signal to the task executor. From a7d92c2255faf5c3b336ca90069aecec7d70b600 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:28:12 -0400 Subject: [PATCH 035/198] Fix ruff --- tests/integration/client.py | 17 ++++++++++------- tests/integration/test_client.py | 5 +---- tests/integration/test_signal.py | 3 +++ tests/integration/utils.py | 1 + 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/integration/client.py b/tests/integration/client.py index 13f9666a9..b1e5a30ea 100644 --- a/tests/integration/client.py +++ b/tests/integration/client.py @@ -70,9 +70,14 @@ class Data: id: uuid.UUID value: str -SQLConnection = mysql.connector.abstracts.MySQLConnectionAbstract | mysql.connector.pooling.PooledMySQLConnection -def create_connection(storage_url: str) -> SQLConnection : +SQLConnection = ( + mysql.connector.abstracts.MySQLConnectionAbstract + | mysql.connector.pooling.PooledMySQLConnection +) + + +def create_connection(storage_url: str) -> SQLConnection: """ Creation a MariaDB connection from a JDBC URL. :param storage_url: JDBC URL for the MariaDB database. @@ -121,9 +126,7 @@ def storage() -> Generator[SQLConnection, None, None]: conn.close() -def submit_job( - conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph -) -> None: +def submit_job(conn: SQLConnection, client_id: uuid.UUID, graph: TaskGraph) -> None: """ Submit a job to the database. :param conn: database connection object. @@ -197,7 +200,7 @@ def get_task_outputs(conn: SQLConnection, task_id: uuid.UUID) -> list[TaskOutput (task_id.bytes,), ) outputs = [] - rows = cast(list[tuple[str, str | None, bytes | None]], cursor.fetchall()) + rows = cast("list[tuple[str, str | None, bytes | None]]", cursor.fetchall()) for output_type, value, data_id in rows: if value is not None: outputs.append(TaskOutput(type=output_type, value=value)) @@ -221,7 +224,7 @@ def get_task_state(conn: SQLConnection, task_id: uuid.UUID) -> str: cursor = conn.cursor() cursor.execute("SELECT state FROM tasks WHERE id = %s", (task_id.bytes,)) - state = cast(tuple[str], cursor.fetchone())[0] + state = cast("tuple[str]", cursor.fetchone())[0] conn.commit() cursor.close() diff --git a/tests/integration/test_client.py b/tests/integration/test_client.py index e5219835c..2b2ca00a3 100644 --- a/tests/integration/test_client.py +++ b/tests/integration/test_client.py @@ -7,10 +7,7 @@ import pytest -from .client import ( - g_storage_url, - SQLConnection -) +from .client import g_storage_url, SQLConnection from .utils import g_scheduler_port diff --git a/tests/integration/test_signal.py b/tests/integration/test_signal.py index f42fdefc3..89191621c 100644 --- a/tests/integration/test_signal.py +++ b/tests/integration/test_signal.py @@ -26,7 +26,10 @@ ) from .utils import g_scheduler_port + class PopenOpts(TypedDict, total=False): + """Options for subprocess.Popen.""" + stdin: int stdout: int stderr: int diff --git a/tests/integration/utils.py b/tests/integration/utils.py index cae3f1730..7f1635390 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -6,6 +6,7 @@ IPv6Addr = tuple[str, int, int, int] AddrType = IPv4Addr | IPv6Addr + def _get_free_tcp_port() -> int: """Returns a free TCP port.""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: From 1c6213bb4a8d5d229bfacc89f84020c9463f2857 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 20:40:58 -0400 Subject: [PATCH 036/198] Simply socket return types. Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- tests/integration/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/utils.py b/tests/integration/utils.py index 7f1635390..64c134cec 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -2,9 +2,7 @@ import socket -IPv4Addr = tuple[str, int] -IPv6Addr = tuple[str, int, int, int] -AddrType = IPv4Addr | IPv6Addr +AddrType = tuple[str, int] def _get_free_tcp_port() -> int: From 55a727ae8012f724a95d25360079b76ad1353ae5 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 4 Aug 2025 23:41:32 -0400 Subject: [PATCH 037/198] Add tombi lint tasks --- lint-tasks.yaml | 24 +++++++++++++++++++++++ lint-test-requirements.txt | 1 + ruff.toml | 40 +++++++++++++++++++------------------- tombi.toml | 34 ++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 tombi.toml diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 2bfe90649..40259b219 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -11,12 +11,14 @@ tasks: - task: "cpp-check" - task: "py-check" - task: "yml-check" + - task: "toml-check" fix: cmds: - task: "cmake-fix" - task: "cpp-fix" - task: "yml-fix" + - task: "toml-fix" cmake-check: deps: ["venv"] @@ -154,6 +156,28 @@ tasks: taskfile.yaml \ test-tasks.yaml + toml: + aliases: + - "toml-check" + - "toml-fix" + cmds: + - task: "toml-lint" + - task: "toml-format" + + toml-lint: + deps: ["venv"] + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + tombi lint "{{.ROOT_DIR}}" + + toml-format: + deps: ["venv"] + cmds: + - |- + . "{{.G_LINT_VENV_DIR}}/bin/activate" + tombi format "{{.ROOT_DIR}}" + cmake: internal: true requires: diff --git a/lint-test-requirements.txt b/lint-test-requirements.txt index 7ac75af21..879c198b3 100644 --- a/lint-test-requirements.txt +++ b/lint-test-requirements.txt @@ -6,6 +6,7 @@ ruff>=0.4.4 mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 +tombi>=0.4.0 # Test dependencies msgpack>=1.1.0 diff --git a/ruff.toml b/ruff.toml index 6cdab7927..fde4359e1 100644 --- a/ruff.toml +++ b/ruff.toml @@ -4,31 +4,31 @@ line-length = 100 select = ["ALL"] extend-select = ["PT"] ignore = [ - "COM812", # Redundant and conflicts with ruff format - "D203", # No blank line before docstrings (D211) - "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists - "D212", # Enforce docstring summary line on the next line after quotes (D213) - "D400", # First line of docstrings may not end in period - "D401", # Docstrings should be written in present tense (not imperative) - "D415", # First line of docstrings may not end in a period, question mark, or exclamation point - "FA102", # Allow use of PEP 604 union in type annotations - "FBT", # Allow bool positional parameters since other value positions are allowed - "FIX002", # Allow todo statements - "PERF401", # Allow for loops when creating lists - "PERF403", # Allow for loops when creating dicts - "S311", # Allow usage of `random` package - "SIM102", # Allow collapsible if statements for readability - "TD002", # Author unnecessary for todo statement - "TD003", # Issue link unnecessary for todo statement - "UP015", # Explicit open modes are helpful + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful ] isort.order-by-type = false [lint.per-file-ignores] "tests/integration/test_*.py" = [ - "S101", # Allow use of `assert` (security warning) - "S603", # Allow user of subprocess.Popen (security warning) - "T201", # Allow use of `print` (testing) + "S101", # Allow use of `assert` (security warning) + "S603", # Allow user of subprocess.Popen (security warning) + "T201", # Allow use of `print` (testing) ] [format] diff --git a/tombi.toml b/tombi.toml new file mode 100644 index 000000000..ad86d3c0c --- /dev/null +++ b/tombi.toml @@ -0,0 +1,34 @@ +toml-version = "v1.0.0" + +[files] +include = ["**/*.toml"] +exclude = ["tools/yscope-dev-utils/**", "build/**"] + +[format] + +[lint] +[lint.rules] +dotted-keys-out-of-order = "warn" +key-empty = "warn" +tables-out-of-order = "warn" + +[schema] +enabled = true +catalog = { + paths = [ + "tombi://json.schemastore.org/api/json/catalog.json", + "https://json.schemastore.org/api/json/catalog.json", + ], +} + +[[schemas]] +path = "tombi://json.schemastore.org/tombi.json" +include = ["tombi.toml"] + +[[schemas]] +path = "https://json.schemastore.org/pyproject.json" +include = ["pyproject.toml"] + +[[schemas]] +path = "https://json.schemastore.org/ruff.json" +include = ["ruff.toml"] From c98fea18e1a3f4134067c8e5648f970bdaba6fe2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 10:07:59 -0400 Subject: [PATCH 038/198] Rename cpp build tasks --- build-tasks.yaml | 4 ++-- test-tasks.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build-tasks.yaml b/build-tasks.yaml index 666576f26..53828027c 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -1,7 +1,7 @@ version: "3" tasks: - target: + cpp-target: internal: true vars: TARGETS: @@ -14,7 +14,7 @@ tasks: --parallel {{numCPU}} --target {{range .TARGETS}}{{.}} {{end}} - clean: + cpp-clean: internal: true deps: [":config-cmake-project"] cmds: diff --git a/test-tasks.yaml b/test-tasks.yaml index 41dfc4578..58a1ff190 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -27,7 +27,7 @@ tasks: build-unit-test: internal: true deps: - - task: ":build:target" + - task: ":build:cpp-target" vars: TARGETS: ["spider_task_executor", "unitTest", "worker_test"] @@ -35,7 +35,7 @@ tasks: dir: "{{.G_BUILD_SPIDER_DIR}}" deps: - "venv" - - task: ":build:target" + - task: ":build:cpp-target" vars: TARGETS: [ "spider_task_executor", From 4e7237ea5775299f94728d674c4675d9b5dae6b2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 14:54:01 -0400 Subject: [PATCH 039/198] Add basic python structure --- python/.python-version | 1 + python/README.md | 0 python/pyproject.toml | 72 +++++++ python/src/__init__.py | 0 python/src/client/__init__.py | 0 python/src/core/__init__.py | 0 python/src/core/task.py | 0 python/src/task_executor/__init__.py | 0 python/src/task_executor/task_executor.py | 6 + python/uv.lock | 242 ++++++++++++++++++++++ 10 files changed, 321 insertions(+) create mode 100644 python/.python-version create mode 100644 python/README.md create mode 100644 python/pyproject.toml create mode 100644 python/src/__init__.py create mode 100644 python/src/client/__init__.py create mode 100644 python/src/core/__init__.py create mode 100644 python/src/core/task.py create mode 100644 python/src/task_executor/__init__.py create mode 100644 python/src/task_executor/task_executor.py create mode 100644 python/uv.lock diff --git a/python/.python-version b/python/.python-version new file mode 100644 index 000000000..24ee5b1be --- /dev/null +++ b/python/.python-version @@ -0,0 +1 @@ +3.13 diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 000000000..c8d875809 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,72 @@ +[project] +name = "spider" +version = "0.0.1" +description = "Spider is a distributed task execution framework" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "mariadb>=1.1.13", + "msgpack>=1.1.1", +] + +[project.scritps] +spider_task_executor = "src.task_executor.task_executor:main" + +[build-system] +requires = ["hatchling>=1.18.0"] +build-backend = "hatchling.build" + +[tool.hatch] + +[tool.hatch.build.targets.wheel] +packages = ["src/client", "src/task_executor"] + +[dependency-groups] +dev = [ + "mypy>=1.12.0", + "ruff>=0.4.4", +] + +[tool.mypy] +strict = true + +# Additional output +pretty = true +show_error_code_links = true +show_error_context = true +show_error_end = true + +[[tool.mypy.overrides]] +module = "msgpack.*" +ignore_missing_imports = true + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["ALL"] +extend-select = ["PT"] +ignore = [ + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful +] +isort.order-by-type = false + +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 100 diff --git a/python/src/__init__.py b/python/src/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/client/__init__.py b/python/src/client/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/core/__init__.py b/python/src/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/core/task.py b/python/src/core/task.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/task_executor/__init__.py b/python/src/task_executor/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/src/task_executor/task_executor.py b/python/src/task_executor/task_executor.py new file mode 100644 index 000000000..b0c61da37 --- /dev/null +++ b/python/src/task_executor/task_executor.py @@ -0,0 +1,6 @@ +def main() -> None: + """Main function to execute the task.""" + + +if __name__ == "__main__": + main() diff --git a/python/uv.lock b/python/uv.lock new file mode 100644 index 000000000..31a72ace1 --- /dev/null +++ b/python/uv.lock @@ -0,0 +1,242 @@ +version = 1 +revision = 2 +requires-python = ">=3.10" + +[[package]] +name = "mariadb" +version = "1.1.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/a0/0498c90f46940b6fd7f5c76d22df9f011849964e94e0b22fde26c42191c7/mariadb-1.1.13.tar.gz", hash = "sha256:3a0fd24fae2b9990dc13a0a427a43d4a5434cc2a8c3b260a27f40b7824719037", size = 111126, upload-time = "2025-07-11T09:55:11.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/e2/5e4068b225ccbd6fb7ea2b8f5ba52060e48bf2e63448206670cdfe73a1c5/mariadb-1.1.13-cp310-cp310-win32.whl", hash = "sha256:f49467e7076819ccf21a7e5ef030761af2160b23ce4484d88d260459809d90ff", size = 185138, upload-time = "2025-07-11T09:54:53.334Z" }, + { url = "https://files.pythonhosted.org/packages/33/8d/65b28ce61c422aba68c052a3cdb23772003267f47438a27a6e2cbf58c1c9/mariadb-1.1.13-cp310-cp310-win_amd64.whl", hash = "sha256:1c9aa9a31f943737c3a58bb4dc9deb9cf2b50b3ec807f90c5c99c7b35f8442bd", size = 202048, upload-time = "2025-07-11T09:54:54.978Z" }, + { url = "https://files.pythonhosted.org/packages/09/71/c23f6ffb06ba2c0ad2e90f4236b1541f9005e24eaeb166800b19c4f584f6/mariadb-1.1.13-cp311-cp311-win32.whl", hash = "sha256:c15ab872f1e951dd1bd4006d2ee2b942156f66c1cd724a1b2992dc02c47dff6a", size = 185136, upload-time = "2025-07-11T09:54:56.791Z" }, + { url = "https://files.pythonhosted.org/packages/c8/08/10308cb9bbccf83ff72caa1146014445a79dfeffb4eba9a85f73575c5959/mariadb-1.1.13-cp311-cp311-win_amd64.whl", hash = "sha256:d6fa7f2cdfae762d2d69b71e51f61415d0251118a622a6d06c7f05261579100a", size = 202046, upload-time = "2025-07-11T09:54:58.365Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b4/d961a5dfa8a7a04ede70f76f85b951b8d0ec28d79b86ced829a67af78bdd/mariadb-1.1.13-cp312-cp312-win32.whl", hash = "sha256:358ecdc8775f4a97357fc233bb84a1c8d8d52866ee5fe98f2d13e9e58a04b6fb", size = 185334, upload-time = "2025-07-11T09:54:59.901Z" }, + { url = "https://files.pythonhosted.org/packages/0a/3b/16cc69a9c90f91b74c096c0cb9634e14692486beb28b82ec5efd97866de7/mariadb-1.1.13-cp312-cp312-win_amd64.whl", hash = "sha256:0604c57e6f90ff7ef190a56894d2b32ca5f0182924b5159be9319ba243ffbf98", size = 202109, upload-time = "2025-07-11T09:55:01.846Z" }, + { url = "https://files.pythonhosted.org/packages/02/16/583719e127774294d4af085a063b38f20407801cd9194a676334fb121edd/mariadb-1.1.13-cp313-cp313-win32.whl", hash = "sha256:36c644c13ecf3e38cb89f37f9bd054703760b94c881476492fe789801a063ce3", size = 185301, upload-time = "2025-07-11T09:55:03.486Z" }, + { url = "https://files.pythonhosted.org/packages/1b/da/4170f5532813ffba2641f821ed23ed1f22fa3ff8d603e47c91b0776ff9b7/mariadb-1.1.13-cp313-cp313-win_amd64.whl", hash = "sha256:6ec51ad2e33312c397cfcc5e2bca59861f572d5e8880e025d43db9a5d472a3a9", size = 202105, upload-time = "2025-07-11T09:55:05.681Z" }, +] + +[[package]] +name = "msgpack" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/b1/ea4f68038a18c77c9467400d166d74c4ffa536f34761f7983a104357e614/msgpack-1.1.1.tar.gz", hash = "sha256:77b79ce34a2bdab2594f490c8e80dd62a02d650b91a75159a63ec413b8d104cd", size = 173555, upload-time = "2025-06-13T06:52:51.324Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/52/f30da112c1dc92cf64f57d08a273ac771e7b29dea10b4b30369b2d7e8546/msgpack-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:353b6fc0c36fde68b661a12949d7d49f8f51ff5fa019c1e47c87c4ff34b080ed", size = 81799, upload-time = "2025-06-13T06:51:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/e4/35/7bfc0def2f04ab4145f7f108e3563f9b4abae4ab0ed78a61f350518cc4d2/msgpack-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:79c408fcf76a958491b4e3b103d1c417044544b68e96d06432a189b43d1215c8", size = 78278, upload-time = "2025-06-13T06:51:38.534Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c5/df5d6c1c39856bc55f800bf82778fd4c11370667f9b9e9d51b2f5da88f20/msgpack-1.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78426096939c2c7482bf31ef15ca219a9e24460289c00dd0b94411040bb73ad2", size = 402805, upload-time = "2025-06-13T06:51:39.538Z" }, + { url = "https://files.pythonhosted.org/packages/20/8e/0bb8c977efecfe6ea7116e2ed73a78a8d32a947f94d272586cf02a9757db/msgpack-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b17ba27727a36cb73aabacaa44b13090feb88a01d012c0f4be70c00f75048b4", size = 408642, upload-time = "2025-06-13T06:51:41.092Z" }, + { url = "https://files.pythonhosted.org/packages/59/a1/731d52c1aeec52006be6d1f8027c49fdc2cfc3ab7cbe7c28335b2910d7b6/msgpack-1.1.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a17ac1ea6ec3c7687d70201cfda3b1e8061466f28f686c24f627cae4ea8efd0", size = 395143, upload-time = "2025-06-13T06:51:42.575Z" }, + { url = "https://files.pythonhosted.org/packages/2b/92/b42911c52cda2ba67a6418ffa7d08969edf2e760b09015593c8a8a27a97d/msgpack-1.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:88d1e966c9235c1d4e2afac21ca83933ba59537e2e2727a999bf3f515ca2af26", size = 395986, upload-time = "2025-06-13T06:51:43.807Z" }, + { url = "https://files.pythonhosted.org/packages/61/dc/8ae165337e70118d4dab651b8b562dd5066dd1e6dd57b038f32ebc3e2f07/msgpack-1.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f6d58656842e1b2ddbe07f43f56b10a60f2ba5826164910968f5933e5178af75", size = 402682, upload-time = "2025-06-13T06:51:45.534Z" }, + { url = "https://files.pythonhosted.org/packages/58/27/555851cb98dcbd6ce041df1eacb25ac30646575e9cd125681aa2f4b1b6f1/msgpack-1.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:96decdfc4adcbc087f5ea7ebdcfd3dee9a13358cae6e81d54be962efc38f6338", size = 406368, upload-time = "2025-06-13T06:51:46.97Z" }, + { url = "https://files.pythonhosted.org/packages/d4/64/39a26add4ce16f24e99eabb9005e44c663db00e3fce17d4ae1ae9d61df99/msgpack-1.1.1-cp310-cp310-win32.whl", hash = "sha256:6640fd979ca9a212e4bcdf6eb74051ade2c690b862b679bfcb60ae46e6dc4bfd", size = 65004, upload-time = "2025-06-13T06:51:48.582Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/73dfa3e9d5d7450d39debde5b0d848139f7de23bd637a4506e36c9800fd6/msgpack-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:8b65b53204fe1bd037c40c4148d00ef918eb2108d24c9aaa20bc31f9810ce0a8", size = 71548, upload-time = "2025-06-13T06:51:49.558Z" }, + { url = "https://files.pythonhosted.org/packages/7f/83/97f24bf9848af23fe2ba04380388216defc49a8af6da0c28cc636d722502/msgpack-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:71ef05c1726884e44f8b1d1773604ab5d4d17729d8491403a705e649116c9558", size = 82728, upload-time = "2025-06-13T06:51:50.68Z" }, + { url = "https://files.pythonhosted.org/packages/aa/7f/2eaa388267a78401f6e182662b08a588ef4f3de6f0eab1ec09736a7aaa2b/msgpack-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:36043272c6aede309d29d56851f8841ba907a1a3d04435e43e8a19928e243c1d", size = 79279, upload-time = "2025-06-13T06:51:51.72Z" }, + { url = "https://files.pythonhosted.org/packages/f8/46/31eb60f4452c96161e4dfd26dbca562b4ec68c72e4ad07d9566d7ea35e8a/msgpack-1.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a32747b1b39c3ac27d0670122b57e6e57f28eefb725e0b625618d1b59bf9d1e0", size = 423859, upload-time = "2025-06-13T06:51:52.749Z" }, + { url = "https://files.pythonhosted.org/packages/45/16/a20fa8c32825cc7ae8457fab45670c7a8996d7746ce80ce41cc51e3b2bd7/msgpack-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a8b10fdb84a43e50d38057b06901ec9da52baac6983d3f709d8507f3889d43f", size = 429975, upload-time = "2025-06-13T06:51:53.97Z" }, + { url = "https://files.pythonhosted.org/packages/86/ea/6c958e07692367feeb1a1594d35e22b62f7f476f3c568b002a5ea09d443d/msgpack-1.1.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba0c325c3f485dc54ec298d8b024e134acf07c10d494ffa24373bea729acf704", size = 413528, upload-time = "2025-06-13T06:51:55.507Z" }, + { url = "https://files.pythonhosted.org/packages/75/05/ac84063c5dae79722bda9f68b878dc31fc3059adb8633c79f1e82c2cd946/msgpack-1.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:88daaf7d146e48ec71212ce21109b66e06a98e5e44dca47d853cbfe171d6c8d2", size = 413338, upload-time = "2025-06-13T06:51:57.023Z" }, + { url = "https://files.pythonhosted.org/packages/69/e8/fe86b082c781d3e1c09ca0f4dacd457ede60a13119b6ce939efe2ea77b76/msgpack-1.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:d8b55ea20dc59b181d3f47103f113e6f28a5e1c89fd5b67b9140edb442ab67f2", size = 422658, upload-time = "2025-06-13T06:51:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/3b/2b/bafc9924df52d8f3bb7c00d24e57be477f4d0f967c0a31ef5e2225e035c7/msgpack-1.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4a28e8072ae9779f20427af07f53bbb8b4aa81151054e882aee333b158da8752", size = 427124, upload-time = "2025-06-13T06:51:59.969Z" }, + { url = "https://files.pythonhosted.org/packages/a2/3b/1f717e17e53e0ed0b68fa59e9188f3f610c79d7151f0e52ff3cd8eb6b2dc/msgpack-1.1.1-cp311-cp311-win32.whl", hash = "sha256:7da8831f9a0fdb526621ba09a281fadc58ea12701bc709e7b8cbc362feabc295", size = 65016, upload-time = "2025-06-13T06:52:01.294Z" }, + { url = "https://files.pythonhosted.org/packages/48/45/9d1780768d3b249accecc5a38c725eb1e203d44a191f7b7ff1941f7df60c/msgpack-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:5fd1b58e1431008a57247d6e7cc4faa41c3607e8e7d4aaf81f7c29ea013cb458", size = 72267, upload-time = "2025-06-13T06:52:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/389b9c593eda2b8551b2e7126ad3a06af6f9b44274eb3a4f054d48ff7e47/msgpack-1.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ae497b11f4c21558d95de9f64fff7053544f4d1a17731c866143ed6bb4591238", size = 82359, upload-time = "2025-06-13T06:52:03.909Z" }, + { url = "https://files.pythonhosted.org/packages/ab/65/7d1de38c8a22cf8b1551469159d4b6cf49be2126adc2482de50976084d78/msgpack-1.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:33be9ab121df9b6b461ff91baac6f2731f83d9b27ed948c5b9d1978ae28bf157", size = 79172, upload-time = "2025-06-13T06:52:05.246Z" }, + { url = "https://files.pythonhosted.org/packages/0f/bd/cacf208b64d9577a62c74b677e1ada005caa9b69a05a599889d6fc2ab20a/msgpack-1.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f64ae8fe7ffba251fecb8408540c34ee9df1c26674c50c4544d72dbf792e5ce", size = 425013, upload-time = "2025-06-13T06:52:06.341Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ec/fd869e2567cc9c01278a736cfd1697941ba0d4b81a43e0aa2e8d71dab208/msgpack-1.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a494554874691720ba5891c9b0b39474ba43ffb1aaf32a5dac874effb1619e1a", size = 426905, upload-time = "2025-06-13T06:52:07.501Z" }, + { url = "https://files.pythonhosted.org/packages/55/2a/35860f33229075bce803a5593d046d8b489d7ba2fc85701e714fc1aaf898/msgpack-1.1.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cb643284ab0ed26f6957d969fe0dd8bb17beb567beb8998140b5e38a90974f6c", size = 407336, upload-time = "2025-06-13T06:52:09.047Z" }, + { url = "https://files.pythonhosted.org/packages/8c/16/69ed8f3ada150bf92745fb4921bd621fd2cdf5a42e25eb50bcc57a5328f0/msgpack-1.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d275a9e3c81b1093c060c3837e580c37f47c51eca031f7b5fb76f7b8470f5f9b", size = 409485, upload-time = "2025-06-13T06:52:10.382Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b6/0c398039e4c6d0b2e37c61d7e0e9d13439f91f780686deb8ee64ecf1ae71/msgpack-1.1.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4fd6b577e4541676e0cc9ddc1709d25014d3ad9a66caa19962c4f5de30fc09ef", size = 412182, upload-time = "2025-06-13T06:52:11.644Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/0cf4a6ecb9bc960d624c93effaeaae75cbf00b3bc4a54f35c8507273cda1/msgpack-1.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb29aaa613c0a1c40d1af111abf025f1732cab333f96f285d6a93b934738a68a", size = 419883, upload-time = "2025-06-13T06:52:12.806Z" }, + { url = "https://files.pythonhosted.org/packages/62/83/9697c211720fa71a2dfb632cad6196a8af3abea56eece220fde4674dc44b/msgpack-1.1.1-cp312-cp312-win32.whl", hash = "sha256:870b9a626280c86cff9c576ec0d9cbcc54a1e5ebda9cd26dab12baf41fee218c", size = 65406, upload-time = "2025-06-13T06:52:14.271Z" }, + { url = "https://files.pythonhosted.org/packages/c0/23/0abb886e80eab08f5e8c485d6f13924028602829f63b8f5fa25a06636628/msgpack-1.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:5692095123007180dca3e788bb4c399cc26626da51629a31d40207cb262e67f4", size = 72558, upload-time = "2025-06-13T06:52:15.252Z" }, + { url = "https://files.pythonhosted.org/packages/a1/38/561f01cf3577430b59b340b51329803d3a5bf6a45864a55f4ef308ac11e3/msgpack-1.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3765afa6bd4832fc11c3749be4ba4b69a0e8d7b728f78e68120a157a4c5d41f0", size = 81677, upload-time = "2025-06-13T06:52:16.64Z" }, + { url = "https://files.pythonhosted.org/packages/09/48/54a89579ea36b6ae0ee001cba8c61f776451fad3c9306cd80f5b5c55be87/msgpack-1.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8ddb2bcfd1a8b9e431c8d6f4f7db0773084e107730ecf3472f1dfe9ad583f3d9", size = 78603, upload-time = "2025-06-13T06:52:17.843Z" }, + { url = "https://files.pythonhosted.org/packages/a0/60/daba2699b308e95ae792cdc2ef092a38eb5ee422f9d2fbd4101526d8a210/msgpack-1.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:196a736f0526a03653d829d7d4c5500a97eea3648aebfd4b6743875f28aa2af8", size = 420504, upload-time = "2025-06-13T06:52:18.982Z" }, + { url = "https://files.pythonhosted.org/packages/20/22/2ebae7ae43cd8f2debc35c631172ddf14e2a87ffcc04cf43ff9df9fff0d3/msgpack-1.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d592d06e3cc2f537ceeeb23d38799c6ad83255289bb84c2e5792e5a8dea268a", size = 423749, upload-time = "2025-06-13T06:52:20.211Z" }, + { url = "https://files.pythonhosted.org/packages/40/1b/54c08dd5452427e1179a40b4b607e37e2664bca1c790c60c442c8e972e47/msgpack-1.1.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4df2311b0ce24f06ba253fda361f938dfecd7b961576f9be3f3fbd60e87130ac", size = 404458, upload-time = "2025-06-13T06:52:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/2e/60/6bb17e9ffb080616a51f09928fdd5cac1353c9becc6c4a8abd4e57269a16/msgpack-1.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e4141c5a32b5e37905b5940aacbc59739f036930367d7acce7a64e4dec1f5e0b", size = 405976, upload-time = "2025-06-13T06:52:22.995Z" }, + { url = "https://files.pythonhosted.org/packages/ee/97/88983e266572e8707c1f4b99c8fd04f9eb97b43f2db40e3172d87d8642db/msgpack-1.1.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b1ce7f41670c5a69e1389420436f41385b1aa2504c3b0c30620764b15dded2e7", size = 408607, upload-time = "2025-06-13T06:52:24.152Z" }, + { url = "https://files.pythonhosted.org/packages/bc/66/36c78af2efaffcc15a5a61ae0df53a1d025f2680122e2a9eb8442fed3ae4/msgpack-1.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4147151acabb9caed4e474c3344181e91ff7a388b888f1e19ea04f7e73dc7ad5", size = 424172, upload-time = "2025-06-13T06:52:25.704Z" }, + { url = "https://files.pythonhosted.org/packages/8c/87/a75eb622b555708fe0427fab96056d39d4c9892b0c784b3a721088c7ee37/msgpack-1.1.1-cp313-cp313-win32.whl", hash = "sha256:500e85823a27d6d9bba1d057c871b4210c1dd6fb01fbb764e37e4e8847376323", size = 65347, upload-time = "2025-06-13T06:52:26.846Z" }, + { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" }, +] + +[[package]] +name = "mypy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mypy-extensions" }, + { name = "pathspec" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/a9/3d7aa83955617cdf02f94e50aab5c830d205cfa4320cf124ff64acce3a8e/mypy-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3fbe6d5555bf608c47203baa3e72dbc6ec9965b3d7c318aa9a4ca76f465bd972", size = 11003299, upload-time = "2025-07-31T07:54:06.425Z" }, + { url = "https://files.pythonhosted.org/packages/83/e8/72e62ff837dd5caaac2b4a5c07ce769c8e808a00a65e5d8f94ea9c6f20ab/mypy-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80ef5c058b7bce08c83cac668158cb7edea692e458d21098c7d3bce35a5d43e7", size = 10125451, upload-time = "2025-07-31T07:53:52.974Z" }, + { url = "https://files.pythonhosted.org/packages/7d/10/f3f3543f6448db11881776f26a0ed079865926b0c841818ee22de2c6bbab/mypy-1.17.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4a580f8a70c69e4a75587bd925d298434057fe2a428faaf927ffe6e4b9a98df", size = 11916211, upload-time = "2025-07-31T07:53:18.879Z" }, + { url = "https://files.pythonhosted.org/packages/06/bf/63e83ed551282d67bb3f7fea2cd5561b08d2bb6eb287c096539feb5ddbc5/mypy-1.17.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd86bb649299f09d987a2eebb4d52d10603224500792e1bee18303bbcc1ce390", size = 12652687, upload-time = "2025-07-31T07:53:30.544Z" }, + { url = "https://files.pythonhosted.org/packages/69/66/68f2eeef11facf597143e85b694a161868b3b006a5fbad50e09ea117ef24/mypy-1.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a76906f26bd8d51ea9504966a9c25419f2e668f012e0bdf3da4ea1526c534d94", size = 12896322, upload-time = "2025-07-31T07:53:50.74Z" }, + { url = "https://files.pythonhosted.org/packages/a3/87/8e3e9c2c8bd0d7e071a89c71be28ad088aaecbadf0454f46a540bda7bca6/mypy-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:e79311f2d904ccb59787477b7bd5d26f3347789c06fcd7656fa500875290264b", size = 9507962, upload-time = "2025-07-31T07:53:08.431Z" }, + { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, + { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, + { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, + { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, + { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, + { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, + { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, + { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, + { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, + { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, + { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, + { url = "https://files.pythonhosted.org/packages/5b/82/aec2fc9b9b149f372850291827537a508d6c4d3664b1750a324b91f71355/mypy-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93378d3203a5c0800c6b6d850ad2f19f7a3cdf1a3701d3416dbf128805c6a6a7", size = 11075338, upload-time = "2025-07-31T07:53:38.873Z" }, + { url = "https://files.pythonhosted.org/packages/07/ac/ee93fbde9d2242657128af8c86f5d917cd2887584cf948a8e3663d0cd737/mypy-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:15d54056f7fe7a826d897789f53dd6377ec2ea8ba6f776dc83c2902b899fee81", size = 10113066, upload-time = "2025-07-31T07:54:14.707Z" }, + { url = "https://files.pythonhosted.org/packages/5a/68/946a1e0be93f17f7caa56c45844ec691ca153ee8b62f21eddda336a2d203/mypy-1.17.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:209a58fed9987eccc20f2ca94afe7257a8f46eb5df1fb69958650973230f91e6", size = 11875473, upload-time = "2025-07-31T07:53:14.504Z" }, + { url = "https://files.pythonhosted.org/packages/9f/0f/478b4dce1cb4f43cf0f0d00fba3030b21ca04a01b74d1cd272a528cf446f/mypy-1.17.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:099b9a5da47de9e2cb5165e581f158e854d9e19d2e96b6698c0d64de911dd849", size = 12744296, upload-time = "2025-07-31T07:53:03.896Z" }, + { url = "https://files.pythonhosted.org/packages/ca/70/afa5850176379d1b303f992a828de95fc14487429a7139a4e0bdd17a8279/mypy-1.17.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa6ffadfbe6994d724c5a1bb6123a7d27dd68fc9c059561cd33b664a79578e14", size = 12914657, upload-time = "2025-07-31T07:54:08.576Z" }, + { url = "https://files.pythonhosted.org/packages/53/f9/4a83e1c856a3d9c8f6edaa4749a4864ee98486e9b9dbfbc93842891029c2/mypy-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:9a2b7d9180aed171f033c9f2fc6c204c1245cf60b0cb61cf2e7acc24eea78e0a", size = 9593320, upload-time = "2025-07-31T07:53:01.341Z" }, + { url = "https://files.pythonhosted.org/packages/38/56/79c2fac86da57c7d8c48622a05873eaab40b905096c33597462713f5af90/mypy-1.17.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:15a83369400454c41ed3a118e0cc58bd8123921a602f385cb6d6ea5df050c733", size = 11040037, upload-time = "2025-07-31T07:54:10.942Z" }, + { url = "https://files.pythonhosted.org/packages/4d/c3/adabe6ff53638e3cad19e3547268482408323b1e68bf082c9119000cd049/mypy-1.17.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:55b918670f692fc9fba55c3298d8a3beae295c5cded0a55dccdc5bbead814acd", size = 10131550, upload-time = "2025-07-31T07:53:41.307Z" }, + { url = "https://files.pythonhosted.org/packages/b8/c5/2e234c22c3bdeb23a7817af57a58865a39753bde52c74e2c661ee0cfc640/mypy-1.17.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:62761474061feef6f720149d7ba876122007ddc64adff5ba6f374fda35a018a0", size = 11872963, upload-time = "2025-07-31T07:53:16.878Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/c13c130f35ca8caa5f2ceab68a247775648fdcd6c9a18f158825f2bc2410/mypy-1.17.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c49562d3d908fd49ed0938e5423daed8d407774a479b595b143a3d7f87cdae6a", size = 12710189, upload-time = "2025-07-31T07:54:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/c7d79d09f6de8383fe800521d066d877e54d30b4fb94281c262be2df84ef/mypy-1.17.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:397fba5d7616a5bc60b45c7ed204717eaddc38f826e3645402c426057ead9a91", size = 12900322, upload-time = "2025-07-31T07:53:10.551Z" }, + { url = "https://files.pythonhosted.org/packages/b8/98/3d5a48978b4f708c55ae832619addc66d677f6dc59f3ebad71bae8285ca6/mypy-1.17.1-cp314-cp314-win_amd64.whl", hash = "sha256:9d6b20b97d373f41617bd0708fd46aa656059af57f2ef72aa8c7d6a2b73b74ed", size = 9751879, upload-time = "2025-07-31T07:52:56.683Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "ruff" +version = "0.12.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/81/0bd3594fa0f690466e41bd033bdcdf86cba8288345ac77ad4afbe5ec743a/ruff-0.12.7.tar.gz", hash = "sha256:1fc3193f238bc2d7968772c82831a4ff69252f673be371fb49663f0068b7ec71", size = 5197814, upload-time = "2025-07-29T22:32:35.877Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/d2/6cb35e9c85e7a91e8d22ab32ae07ac39cc34a71f1009a6f9e4a2a019e602/ruff-0.12.7-py3-none-linux_armv6l.whl", hash = "sha256:76e4f31529899b8c434c3c1dede98c4483b89590e15fb49f2d46183801565303", size = 11852189, upload-time = "2025-07-29T22:31:41.281Z" }, + { url = "https://files.pythonhosted.org/packages/63/5b/a4136b9921aa84638f1a6be7fb086f8cad0fde538ba76bda3682f2599a2f/ruff-0.12.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:789b7a03e72507c54fb3ba6209e4bb36517b90f1a3569ea17084e3fd295500fb", size = 12519389, upload-time = "2025-07-29T22:31:54.265Z" }, + { url = "https://files.pythonhosted.org/packages/a8/c9/3e24a8472484269b6b1821794141f879c54645a111ded4b6f58f9ab0705f/ruff-0.12.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:2e1c2a3b8626339bb6369116e7030a4cf194ea48f49b64bb505732a7fce4f4e3", size = 11743384, upload-time = "2025-07-29T22:31:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/26/7c/458dd25deeb3452c43eaee853c0b17a1e84169f8021a26d500ead77964fd/ruff-0.12.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32dec41817623d388e645612ec70d5757a6d9c035f3744a52c7b195a57e03860", size = 11943759, upload-time = "2025-07-29T22:32:01.95Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8b/658798472ef260ca050e400ab96ef7e85c366c39cf3dfbef4d0a46a528b6/ruff-0.12.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47ef751f722053a5df5fa48d412dbb54d41ab9b17875c6840a58ec63ff0c247c", size = 11654028, upload-time = "2025-07-29T22:32:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/a8/86/9c2336f13b2a3326d06d39178fd3448dcc7025f82514d1b15816fe42bfe8/ruff-0.12.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a828a5fc25a3efd3e1ff7b241fd392686c9386f20e5ac90aa9234a5faa12c423", size = 13225209, upload-time = "2025-07-29T22:32:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/76/69/df73f65f53d6c463b19b6b312fd2391dc36425d926ec237a7ed028a90fc1/ruff-0.12.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5726f59b171111fa6a69d82aef48f00b56598b03a22f0f4170664ff4d8298efb", size = 14182353, upload-time = "2025-07-29T22:32:10.053Z" }, + { url = "https://files.pythonhosted.org/packages/58/1e/de6cda406d99fea84b66811c189b5ea139814b98125b052424b55d28a41c/ruff-0.12.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:74e6f5c04c4dd4aba223f4fe6e7104f79e0eebf7d307e4f9b18c18362124bccd", size = 13631555, upload-time = "2025-07-29T22:32:12.644Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ae/625d46d5164a6cc9261945a5e89df24457dc8262539ace3ac36c40f0b51e/ruff-0.12.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5d0bfe4e77fba61bf2ccadf8cf005d6133e3ce08793bbe870dd1c734f2699a3e", size = 12667556, upload-time = "2025-07-29T22:32:15.312Z" }, + { url = "https://files.pythonhosted.org/packages/55/bf/9cb1ea5e3066779e42ade8d0cd3d3b0582a5720a814ae1586f85014656b6/ruff-0.12.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06bfb01e1623bf7f59ea749a841da56f8f653d641bfd046edee32ede7ff6c606", size = 12939784, upload-time = "2025-07-29T22:32:17.69Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/7ead2663be5627c04be83754c4f3096603bf5e99ed856c7cd29618c691bd/ruff-0.12.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e41df94a957d50083fd09b916d6e89e497246698c3f3d5c681c8b3e7b9bb4ac8", size = 11771356, upload-time = "2025-07-29T22:32:20.134Z" }, + { url = "https://files.pythonhosted.org/packages/17/40/a95352ea16edf78cd3a938085dccc55df692a4d8ba1b3af7accbe2c806b0/ruff-0.12.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:4000623300563c709458d0ce170c3d0d788c23a058912f28bbadc6f905d67afa", size = 11612124, upload-time = "2025-07-29T22:32:22.645Z" }, + { url = "https://files.pythonhosted.org/packages/4d/74/633b04871c669e23b8917877e812376827c06df866e1677f15abfadc95cb/ruff-0.12.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:69ffe0e5f9b2cf2b8e289a3f8945b402a1b19eff24ec389f45f23c42a3dd6fb5", size = 12479945, upload-time = "2025-07-29T22:32:24.765Z" }, + { url = "https://files.pythonhosted.org/packages/be/34/c3ef2d7799c9778b835a76189c6f53c179d3bdebc8c65288c29032e03613/ruff-0.12.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a07a5c8ffa2611a52732bdc67bf88e243abd84fe2d7f6daef3826b59abbfeda4", size = 12998677, upload-time = "2025-07-29T22:32:27.022Z" }, + { url = "https://files.pythonhosted.org/packages/77/ab/aca2e756ad7b09b3d662a41773f3edcbd262872a4fc81f920dc1ffa44541/ruff-0.12.7-py3-none-win32.whl", hash = "sha256:c928f1b2ec59fb77dfdf70e0419408898b63998789cc98197e15f560b9e77f77", size = 11756687, upload-time = "2025-07-29T22:32:29.381Z" }, + { url = "https://files.pythonhosted.org/packages/b4/71/26d45a5042bc71db22ddd8252ca9d01e9ca454f230e2996bb04f16d72799/ruff-0.12.7-py3-none-win_amd64.whl", hash = "sha256:9c18f3d707ee9edf89da76131956aba1270c6348bfee8f6c647de841eac7194f", size = 12912365, upload-time = "2025-07-29T22:32:31.517Z" }, + { url = "https://files.pythonhosted.org/packages/4c/9b/0b8aa09817b63e78d94b4977f18b1fcaead3165a5ee49251c5d5c245bb2d/ruff-0.12.7-py3-none-win_arm64.whl", hash = "sha256:dfce05101dbd11833a0776716d5d1578641b7fddb537fe7fa956ab85d1769b69", size = 11982083, upload-time = "2025-07-29T22:32:33.881Z" }, +] + +[[package]] +name = "spider" +version = "0.0.1" +source = { virtual = "." } +dependencies = [ + { name = "mariadb" }, + { name = "msgpack" }, +] + +[package.dev-dependencies] +dev = [ + { name = "mypy" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "mariadb", specifier = ">=1.1.13" }, + { name = "msgpack", specifier = ">=1.1.1" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "mypy", specifier = ">=1.12.0" }, + { name = "ruff", specifier = ">=0.4.4" }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175, upload-time = "2024-11-27T22:38:36.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077, upload-time = "2024-11-27T22:37:54.956Z" }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429, upload-time = "2024-11-27T22:37:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067, upload-time = "2024-11-27T22:37:57.63Z" }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030, upload-time = "2024-11-27T22:37:59.344Z" }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898, upload-time = "2024-11-27T22:38:00.429Z" }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894, upload-time = "2024-11-27T22:38:02.094Z" }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319, upload-time = "2024-11-27T22:38:03.206Z" }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273, upload-time = "2024-11-27T22:38:04.217Z" }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310, upload-time = "2024-11-27T22:38:05.908Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309, upload-time = "2024-11-27T22:38:06.812Z" }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762, upload-time = "2024-11-27T22:38:07.731Z" }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453, upload-time = "2024-11-27T22:38:09.384Z" }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486, upload-time = "2024-11-27T22:38:10.329Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349, upload-time = "2024-11-27T22:38:11.443Z" }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159, upload-time = "2024-11-27T22:38:13.099Z" }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243, upload-time = "2024-11-27T22:38:14.766Z" }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645, upload-time = "2024-11-27T22:38:15.843Z" }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584, upload-time = "2024-11-27T22:38:17.645Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875, upload-time = "2024-11-27T22:38:19.159Z" }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418, upload-time = "2024-11-27T22:38:20.064Z" }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708, upload-time = "2024-11-27T22:38:21.659Z" }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582, upload-time = "2024-11-27T22:38:22.693Z" }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543, upload-time = "2024-11-27T22:38:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691, upload-time = "2024-11-27T22:38:26.081Z" }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170, upload-time = "2024-11-27T22:38:27.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530, upload-time = "2024-11-27T22:38:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666, upload-time = "2024-11-27T22:38:30.639Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954, upload-time = "2024-11-27T22:38:31.702Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724, upload-time = "2024-11-27T22:38:32.837Z" }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383, upload-time = "2024-11-27T22:38:34.455Z" }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257, upload-time = "2024-11-27T22:38:35.385Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/98/5a/da40306b885cc8c09109dc2e1abd358d5684b1425678151cdaed4731c822/typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36", size = 107673, upload-time = "2025-07-04T13:28:34.16Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, +] From 1f944d058288587c891c5910c41d4469a4d7f5cd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 14:56:46 -0400 Subject: [PATCH 040/198] Fix code structure --- python/src/__init__.py | 0 python/uv.lock | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 python/src/__init__.py diff --git a/python/src/__init__.py b/python/src/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/python/uv.lock b/python/uv.lock index 31a72ace1..330a5aaa8 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -169,7 +169,7 @@ wheels = [ [[package]] name = "spider" version = "0.0.1" -source = { virtual = "." } +source = { editable = "." } dependencies = [ { name = "mariadb" }, { name = "msgpack" }, From 68992fcd4daac3ca8e93b897aa8b8fa8f921e469 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 15:00:38 -0400 Subject: [PATCH 041/198] Fix ruff lint --- python/src/client/__init__.py | 1 + python/src/core/__init__.py | 1 + python/src/core/task.py | 5 +++++ python/src/task_executor/__init__.py | 1 + python/src/task_executor/task_executor.py | 3 +++ 5 files changed, 11 insertions(+) diff --git a/python/src/client/__init__.py b/python/src/client/__init__.py index e69de29bb..038456143 100644 --- a/python/src/client/__init__.py +++ b/python/src/client/__init__.py @@ -0,0 +1 @@ +"""Spider python client.""" diff --git a/python/src/core/__init__.py b/python/src/core/__init__.py index e69de29bb..e4e408948 100644 --- a/python/src/core/__init__.py +++ b/python/src/core/__init__.py @@ -0,0 +1 @@ +"""Spider core module.""" diff --git a/python/src/core/task.py b/python/src/core/task.py index e69de29bb..855f2a72b 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -0,0 +1,5 @@ +"""Task module for Spider.""" + + +class Task: + """Represents a task in Spider.""" diff --git a/python/src/task_executor/__init__.py b/python/src/task_executor/__init__.py index e69de29bb..03195404d 100644 --- a/python/src/task_executor/__init__.py +++ b/python/src/task_executor/__init__.py @@ -0,0 +1 @@ +"""Python task executor for running Python task function.""" diff --git a/python/src/task_executor/task_executor.py b/python/src/task_executor/task_executor.py index b0c61da37..c8a0593d3 100644 --- a/python/src/task_executor/task_executor.py +++ b/python/src/task_executor/task_executor.py @@ -1,3 +1,6 @@ +"""Executes a Spider Python task.""" + + def main() -> None: """Main function to execute the task.""" From 1127b2215bd735aa8b8205d5e1443d17efafcb1e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 15:06:00 -0400 Subject: [PATCH 042/198] Extend lint tasks to python directory --- lint-tasks.yaml | 5 +++++ taskfile.yaml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 40259b219..270a121cb 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -135,6 +135,11 @@ tasks: mypy "{{.ITEM}}" ruff check {{.RUFF_CHECK_FLAGS}} "{{.ITEM}}" ruff format {{.RUFF_FORMAT_FLAGS}} "{{.ITEM}}" + - cmd: |- + cd "{{.G_SRC_PYTHON_DIR}}" + uv run mypy . + uv run ruff check {{.RUFF_CHECK_FLAGS}} + uv run ruff format {{.RUFF_FORMAT_FLAGS}} yml: aliases: diff --git a/taskfile.yaml b/taskfile.yaml index 470edd582..4588d5bd4 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -20,6 +20,8 @@ vars: G_TEST_DIR: "{{.ROOT_DIR}}/tests" G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" + G_SRC_PYTHON_DIR: "{{.ROOT_DIR}}/python" + # Build parameters # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. G_DEPS_MAX_PARALLELISM_PER_TASK: >- From 4c51c916abbc549b8b6a395b6d9085bc5bb7e587 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 15:13:01 -0400 Subject: [PATCH 043/198] Add python build tasks --- build-tasks.yaml | 4 ++++ taskfile.yaml | 1 + 2 files changed, 5 insertions(+) diff --git a/build-tasks.yaml b/build-tasks.yaml index 53828027c..7363d4d2a 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -19,3 +19,7 @@ tasks: deps: [":config-cmake-project"] cmds: - "cmake --build {{.G_BUILD_SPIDER_DIR}} --target clean --parallel {{numCPU}}" + + py-build: + cmds: + - uv build --directory "{{.G_SRC_PYTHON_DIR}}" -o "{{.G_BUILD_PYTHON_DIR}}" diff --git a/taskfile.yaml b/taskfile.yaml index 4588d5bd4..1a94fdc38 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -21,6 +21,7 @@ vars: G_EXAMPLES_DIR: "{{.ROOT_DIR}}/examples" G_SRC_PYTHON_DIR: "{{.ROOT_DIR}}/python" + G_BUILD_PYTHON_DIR: "{{.G_BUILD_DIR}}/spider-py" # Build parameters # NOTE: Defaulting to an empty string is safe since CMake ignores an empty string. From 6004d6455b56ae4bdd1f30878254ddbe65fe1b9a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 17:51:53 -0400 Subject: [PATCH 044/198] Fix yaml lint error --- build-tasks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tasks.yaml b/build-tasks.yaml index 7363d4d2a..1ce1efb9b 100644 --- a/build-tasks.yaml +++ b/build-tasks.yaml @@ -22,4 +22,4 @@ tasks: py-build: cmds: - - uv build --directory "{{.G_SRC_PYTHON_DIR}}" -o "{{.G_BUILD_PYTHON_DIR}}" + - "uv build --directory {{.G_SRC_PYTHON_DIR}} -o {{.G_BUILD_PYTHON_DIR}}" From bcab5db89a55c3bbf7086718e1e1a89adcc7adba Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 18:33:58 -0400 Subject: [PATCH 045/198] Fix tombi lint --- python/pyproject.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index c8d875809..235370dd2 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -16,17 +16,17 @@ spider_task_executor = "src.task_executor.task_executor:main" requires = ["hatchling>=1.18.0"] build-backend = "hatchling.build" -[tool.hatch] - -[tool.hatch.build.targets.wheel] -packages = ["src/client", "src/task_executor"] - [dependency-groups] dev = [ "mypy>=1.12.0", "ruff>=0.4.4", ] +[tool.hatch] + +[tool.hatch.build.targets.wheel] +packages = ["src/client", "src/task_executor"] + [tool.mypy] strict = true @@ -43,6 +43,10 @@ ignore_missing_imports = true [tool.ruff] line-length = 100 +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 100 + [tool.ruff.lint] select = ["ALL"] extend-select = ["PT"] @@ -66,7 +70,3 @@ ignore = [ "UP015", # Explicit open modes are helpful ] isort.order-by-type = false - -[tool.ruff.format] -docstring-code-format = true -docstring-code-line-length = 100 From a50d25a36e71146abb374328dbf99c910e70d038 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 18:38:25 -0400 Subject: [PATCH 046/198] Remove wrong mypy config --- python/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 235370dd2..09c1e0f1a 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -34,7 +34,6 @@ strict = true pretty = true show_error_code_links = true show_error_context = true -show_error_end = true [[tool.mypy.overrides]] module = "msgpack.*" From fbebb963eecfef0b7073ae5aad6311b68fb5fd05 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 18:48:37 -0400 Subject: [PATCH 047/198] Fix typo and format file --- python/pyproject.toml | 54 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 09c1e0f1a..be9486a1f 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -5,31 +5,29 @@ description = "Spider is a distributed task execution framework" readme = "README.md" requires-python = ">=3.10" dependencies = [ - "mariadb>=1.1.13", - "msgpack>=1.1.1", + "mariadb>=1.1.13", + "msgpack>=1.1.1", ] -[project.scritps] +[project.scripts] spider_task_executor = "src.task_executor.task_executor:main" -[build-system] -requires = ["hatchling>=1.18.0"] -build-backend = "hatchling.build" - [dependency-groups] dev = [ - "mypy>=1.12.0", - "ruff>=0.4.4", + "mypy>=1.12.0", + "ruff>=0.4.4", ] -[tool.hatch] +[build-system] +requires = ["hatchling>=1.18.0"] +build-backend = "hatchling.build" +[tool.hatch] [tool.hatch.build.targets.wheel] packages = ["src/client", "src/task_executor"] [tool.mypy] strict = true - # Additional output pretty = true show_error_code_links = true @@ -50,22 +48,22 @@ docstring-code-line-length = 100 select = ["ALL"] extend-select = ["PT"] ignore = [ - "COM812", # Redundant and conflicts with ruff format - "D203", # No blank line before docstrings (D211) - "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists - "D212", # Enforce docstring summary line on the next line after quotes (D213) - "D400", # First line of docstrings may not end in period - "D401", # Docstrings should be written in present tense (not imperative) - "D415", # First line of docstrings may not end in a period, question mark, or exclamation point - "FA102", # Allow use of PEP 604 union in type annotations - "FBT", # Allow bool positional parameters since other value positions are allowed - "FIX002", # Allow todo statements - "PERF401", # Allow for loops when creating lists - "PERF403", # Allow for loops when creating dicts - "S311", # Allow usage of `random` package - "SIM102", # Allow collapsible if statements for readability - "TD002", # Author unnecessary for todo statement - "TD003", # Issue link unnecessary for todo statement - "UP015", # Explicit open modes are helpful + "COM812", # Redundant and conflicts with ruff format + "D203", # No blank line before docstrings (D211) + "D205", # Breaks if summary is larger than one line due to wrapping or if no summary exists + "D212", # Enforce docstring summary line on the next line after quotes (D213) + "D400", # First line of docstrings may not end in period + "D401", # Docstrings should be written in present tense (not imperative) + "D415", # First line of docstrings may not end in a period, question mark, or exclamation point + "FA102", # Allow use of PEP 604 union in type annotations + "FBT", # Allow bool positional parameters since other value positions are allowed + "FIX002", # Allow todo statements + "PERF401", # Allow for loops when creating lists + "PERF403", # Allow for loops when creating dicts + "S311", # Allow usage of `random` package + "SIM102", # Allow collapsible if statements for readability + "TD002", # Author unnecessary for todo statement + "TD003", # Issue link unnecessary for todo statement + "UP015", # Explicit open modes are helpful ] isort.order-by-type = false From 0322bd528424d820a755284e9f8d86a42788c854 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 21:07:18 -0400 Subject: [PATCH 048/198] Use typed msgpack and remove mypy config for msgpack from pyproject --- python/pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index be9486a1f..9b69cfdeb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -6,7 +6,7 @@ readme = "README.md" requires-python = ">=3.10" dependencies = [ "mariadb>=1.1.13", - "msgpack>=1.1.1", + "msgpack-types>=0.5.0", ] [project.scripts] @@ -33,10 +33,6 @@ pretty = true show_error_code_links = true show_error_context = true -[[tool.mypy.overrides]] -module = "msgpack.*" -ignore_missing_imports = true - [tool.ruff] line-length = 100 From 2a0815558de44be469e2e4a1581b91683d621b3a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 23:55:16 -0400 Subject: [PATCH 049/198] Update uv lock --- python/uv.lock | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/python/uv.lock b/python/uv.lock index 330a5aaa8..6dc97a77e 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -69,6 +69,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/91/7dc28d5e2a11a5ad804cf2b7f7a5fcb1eb5a4966d66a5d2b41aee6376543/msgpack-1.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:6d489fba546295983abd142812bda76b57e33d0b9f5d5b71c09a583285506f69", size = 72341, upload-time = "2025-06-13T06:52:27.835Z" }, ] +[[package]] +name = "msgpack-types" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "msgpack" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/26/a15707f2af5681333cd598724bedd1948844ac2af45eafc4175af0671a8d/msgpack_types-0.5.0.tar.gz", hash = "sha256:aebd1b8da23f8f9966d66ebb1a43bd261b95751c6a267bd21a124d2ccac84201", size = 6702, upload-time = "2024-09-21T13:55:05.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/dd/cd9d2b0ef506f6164cd81d4e92e408095041f28523d751b9f7dabdc244eb/msgpack_types-0.5.0-py3-none-any.whl", hash = "sha256:8b633ed75e495a555fa0615843de559a74b1d176828d59bb393d266e51f6bda7", size = 8182, upload-time = "2024-09-21T13:55:04.232Z" }, +] + [[package]] name = "mypy" version = "1.17.1" @@ -172,7 +185,7 @@ version = "0.0.1" source = { editable = "." } dependencies = [ { name = "mariadb" }, - { name = "msgpack" }, + { name = "msgpack-types" }, ] [package.dev-dependencies] @@ -184,7 +197,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "mariadb", specifier = ">=1.1.13" }, - { name = "msgpack", specifier = ">=1.1.1" }, + { name = "msgpack-types", specifier = ">=0.5.0" }, ] [package.metadata.requires-dev] From dd29c239f499c77c9a33f2b2528236b13dce133c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 5 Aug 2025 23:55:54 -0400 Subject: [PATCH 050/198] Add task module --- python/src/core/task.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/python/src/core/task.py b/python/src/core/task.py index 855f2a72b..5d1c20b6f 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -1,5 +1,46 @@ """Task module for Spider.""" +from dataclasses import dataclass, field +from enum import IntEnum +from uuid import UUID + +@dataclass +class TaskInputOutput: + """Represents a task input that points to output of another task""" + + task_id: UUID + position: int + + +TaskInputValue = bytes +TaskInputData = UUID +TaskInput = TaskInputOutput | TaskInputValue | TaskInputData + +TaskOutputValue = bytes +TaskOutputData = UUID +TaskOutput = TaskOutputValue | TaskOutputData + + +class TaskState(IntEnum): + """Represents state of a task""" + + Pending = 0 + Ready = 1 + Running = 2 + Succeeded = 3 + Failed = 4 + Cancelled = 5 + + +@dataclass class Task: """Represents a task in Spider.""" + + task_id: UUID + function_name: str + state: TaskState + timeout: float + max_retires: int + task_input: list[TaskInputData] = field(default_factory=list) + task_output: list[TaskOutputData] = field(default_factory=list) From 9b2a1752a69c6a53b9fc1332b1a74f8b259240a9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 00:01:03 -0400 Subject: [PATCH 051/198] Add data and improve type alias --- python/src/core/data.py | 12 ++++++++++++ python/src/core/task.py | 11 +++++++---- 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 python/src/core/data.py diff --git a/python/src/core/data.py b/python/src/core/data.py new file mode 100644 index 000000000..5c632551b --- /dev/null +++ b/python/src/core/data.py @@ -0,0 +1,12 @@ +"""Data module for Spider.""" +from dataclasses import dataclass +from uuid import UUID + +DataId = UUID + +@dataclass +class Data: + """Represents a data object.""" + + id: DataId + value: bytes diff --git a/python/src/core/task.py b/python/src/core/task.py index 5d1c20b6f..63b156db9 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -4,21 +4,24 @@ from enum import IntEnum from uuid import UUID +from core.data import DataId + +TaskId = UUID @dataclass class TaskInputOutput: """Represents a task input that points to output of another task""" - task_id: UUID + task_id: TaskId position: int TaskInputValue = bytes -TaskInputData = UUID +TaskInputData = DataId TaskInput = TaskInputOutput | TaskInputValue | TaskInputData TaskOutputValue = bytes -TaskOutputData = UUID +TaskOutputData = DataId TaskOutput = TaskOutputValue | TaskOutputData @@ -37,7 +40,7 @@ class TaskState(IntEnum): class Task: """Represents a task in Spider.""" - task_id: UUID + task_id: TaskId function_name: str state: TaskState timeout: float From 32afe0a4dbe282be84ab5f748570d062dd1bd26f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 00:18:34 -0400 Subject: [PATCH 052/198] Add task graph and reformat files --- python/src/core/data.py | 2 ++ python/src/core/task.py | 1 + python/src/core/taskgraph.py | 56 ++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 python/src/core/taskgraph.py diff --git a/python/src/core/data.py b/python/src/core/data.py index 5c632551b..7a0586712 100644 --- a/python/src/core/data.py +++ b/python/src/core/data.py @@ -1,9 +1,11 @@ """Data module for Spider.""" + from dataclasses import dataclass from uuid import UUID DataId = UUID + @dataclass class Data: """Represents a data object.""" diff --git a/python/src/core/task.py b/python/src/core/task.py index 63b156db9..1590f1b1c 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -8,6 +8,7 @@ TaskId = UUID + @dataclass class TaskInputOutput: """Represents a task input that points to output of another task""" diff --git a/python/src/core/taskgraph.py b/python/src/core/taskgraph.py new file mode 100644 index 000000000..e0bf91e96 --- /dev/null +++ b/python/src/core/taskgraph.py @@ -0,0 +1,56 @@ +"""TaskGraph module for Spider.""" + +from core.task import Task, TaskId + + +class TaskGraph: + """Represents a task graph in Spider.""" + + def __init__(self) -> None: + """Initializes an empty task graph.""" + self.tasks: dict[TaskId, Task] = {} + # Dependency list consists of a list of tuples of + # - parent task id + # - child task id + self.dependencies: list[tuple[TaskId, TaskId]] = [] + self.input_tasks: set[TaskId] = set() + self.output_tasks: set[TaskId] = set() + + def add_task( + self, task: Task, parents: list[TaskId] | None = None, children: list[TaskId] | None = None + ) -> None: + """ + Adds a task to the graph. + :param task: The task to add. + :param parents: The parent ids of the task. Must be already in the task graph. + :param children: The children ids of the task. Must be already in the task graph. + """ + self.tasks[task.task_id] = task + if parents: + for parent in parents: + self.dependencies.append((parent, task.task_id)) + self.output_tasks.remove(parent) + else: + self.input_tasks.add(task.task_id) + if children: + for child in children: + self.dependencies.append((task.task_id, child)) + self.input_tasks.remove(child) + else: + self.output_tasks.add(task.task_id) + + def get_parents(self, task_id: TaskId) -> list[Task]: + """ + Gets parent tasks of task. + :param task_id: ID of the task. + :return: List of parent tasks. + """ + return [self.tasks[parent] for (parent, child) in self.dependencies if child == task_id] + + def get_children(self, task_id: TaskId) -> list[Task]: + """ + Gets child tasks of task. + :param task_id: ID of the task. + :return: List of children tasks. + """ + return [self.tasks[child] for (parent, child) in self.dependencies if parent == task_id] From 2f9b4a072cc8aadf41e2b4351aabd7c53180fad0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 09:23:41 -0400 Subject: [PATCH 053/198] Fix typo Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- python/src/core/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/core/task.py b/python/src/core/task.py index 1590f1b1c..2deff2208 100644 --- a/python/src/core/task.py +++ b/python/src/core/task.py @@ -45,6 +45,6 @@ class Task: function_name: str state: TaskState timeout: float - max_retires: int + max_retries: int task_input: list[TaskInputData] = field(default_factory=list) task_output: list[TaskOutputData] = field(default_factory=list) From 4e080ac28e86d4ac2c5b698ae6f4aa8bd3330088 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 09:41:27 -0400 Subject: [PATCH 054/198] Fix set remove key error --- python/src/core/taskgraph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/core/taskgraph.py b/python/src/core/taskgraph.py index e0bf91e96..d209bc378 100644 --- a/python/src/core/taskgraph.py +++ b/python/src/core/taskgraph.py @@ -29,13 +29,13 @@ def add_task( if parents: for parent in parents: self.dependencies.append((parent, task.task_id)) - self.output_tasks.remove(parent) + self.output_tasks.discard(parent) else: self.input_tasks.add(task.task_id) if children: for child in children: self.dependencies.append((task.task_id, child)) - self.input_tasks.remove(child) + self.input_tasks.discard(child) else: self.output_tasks.add(task.task_id) From b09515c0d5a815c6e72a25e4275ddf1d464c4937 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 10:16:51 -0400 Subject: [PATCH 055/198] Increase min version of tombi --- lint-test-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lint-test-requirements.txt b/lint-test-requirements.txt index 879c198b3..ffbf89925 100644 --- a/lint-test-requirements.txt +++ b/lint-test-requirements.txt @@ -6,7 +6,7 @@ ruff>=0.4.4 mypy>=1.12.0 gersemi>=0.16.2 yamllint>=1.35.1 -tombi>=0.4.0 +tombi>=0.4.35 # Test dependencies msgpack>=1.1.0 From b4d657637fa5da39f8dd1cdd25f67348371eec43 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 16:34:55 -0400 Subject: [PATCH 056/198] Add uv in README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2ccfff88f..54cf6bd69 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Follow the steps below to develop and contribute to the project. ## Requirements * Python 3.10 or higher * [Task] 3.40.0 or higher +* [uv] 0.7.0 or higher ## Set up Run dependency installation task: From f447614c9ba65d9e2ddd42e69a194120e7168d1f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 6 Aug 2025 17:35:09 -0400 Subject: [PATCH 057/198] Fix redenduncy caused by merge --- lint-tasks.yaml | 2 -- requirements-dev.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/lint-tasks.yaml b/lint-tasks.yaml index 4e941803f..54ca4611c 100644 --- a/lint-tasks.yaml +++ b/lint-tasks.yaml @@ -12,7 +12,6 @@ tasks: - task: "py-check" - task: "toml-check" - task: "yml-check" - - task: "toml-check" fix: cmds: @@ -20,7 +19,6 @@ tasks: - task: "cpp-fix" - task: "toml-fix" - task: "yml-fix" - - task: "toml-fix" cmake-check: deps: ["venv"] diff --git a/requirements-dev.txt b/requirements-dev.txt index d1379209c..a5e8717f2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,6 @@ mypy>=1.12.0 ruff>=0.4.4 tombi>=0.4.35 yamllint>=1.35.1 -tombi>=0.4.35 # Test dependencies msgpack-types>=0.5.0 From de149ee82fcad48b70f5f128c01a0d860449663c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 19:51:06 -0400 Subject: [PATCH 058/198] Add integral types --- python/src/type/__init__.py | 1 + python/src/type/type.py | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 python/src/type/__init__.py create mode 100644 python/src/type/type.py diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py new file mode 100644 index 000000000..f147ce3ed --- /dev/null +++ b/python/src/type/__init__.py @@ -0,0 +1 @@ +"""Spider type module.""" diff --git a/python/src/type/type.py b/python/src/type/type.py new file mode 100644 index 000000000..1a14dfdaf --- /dev/null +++ b/python/src/type/type.py @@ -0,0 +1,42 @@ +"""Custom type module for Spider.""" + +from typing import cast + + +class BoundedInt(int): + """Bounded integer type.""" + + def __new__(cls, value: int, bits: int = 32) -> "BoundedInt": + """Creates a bounded integer.""" + min_val = -(1 << (bits - 1)) + max_val = (1 << (bits - 1)) - 1 + + if not (min_val <= value <= max_val): + msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" + raise ValueError(msg) + + return super().__new__(cls, value) + + +class Int8(BoundedInt): + """8 bits integer type.""" + + def __new__(cls, value: int) -> "Int8": + """Creates an int8 integer.""" + return cast("Int8", super().__new__(cls, value, bits=8)) + + +class Int16(BoundedInt): + """16 bits integer type.""" + + def __new__(cls, value: int) -> "Int16": + """Creates an int16 integer.""" + return cast("Int16", super().__new__(cls, value, bits=16)) + + +class Int32(BoundedInt): + """32 bits integer type.""" + + def __new__(cls, value: int) -> "Int32": + """Creates an int32 integer.""" + return cast("Int32", super().__new__(cls, value, bits=8)) From d332dbe3ed95b3a4a422d44248c054f3200aeb6b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 19:52:13 -0400 Subject: [PATCH 059/198] Revert "Add integral types" This reverts commit de149ee82fcad48b70f5f128c01a0d860449663c. --- python/src/type/__init__.py | 1 - python/src/type/type.py | 42 ------------------------------------- 2 files changed, 43 deletions(-) delete mode 100644 python/src/type/__init__.py delete mode 100644 python/src/type/type.py diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py deleted file mode 100644 index f147ce3ed..000000000 --- a/python/src/type/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider type module.""" diff --git a/python/src/type/type.py b/python/src/type/type.py deleted file mode 100644 index 1a14dfdaf..000000000 --- a/python/src/type/type.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Custom type module for Spider.""" - -from typing import cast - - -class BoundedInt(int): - """Bounded integer type.""" - - def __new__(cls, value: int, bits: int = 32) -> "BoundedInt": - """Creates a bounded integer.""" - min_val = -(1 << (bits - 1)) - max_val = (1 << (bits - 1)) - 1 - - if not (min_val <= value <= max_val): - msg = f"Bounded integer value ({value}) must be between {min_val} and {max_val}" - raise ValueError(msg) - - return super().__new__(cls, value) - - -class Int8(BoundedInt): - """8 bits integer type.""" - - def __new__(cls, value: int) -> "Int8": - """Creates an int8 integer.""" - return cast("Int8", super().__new__(cls, value, bits=8)) - - -class Int16(BoundedInt): - """16 bits integer type.""" - - def __new__(cls, value: int) -> "Int16": - """Creates an int16 integer.""" - return cast("Int16", super().__new__(cls, value, bits=16)) - - -class Int32(BoundedInt): - """32 bits integer type.""" - - def __new__(cls, value: int) -> "Int32": - """Creates an int32 integer.""" - return cast("Int32", super().__new__(cls, value, bits=8)) From 80b67726082556e8c02d768ab54778d92a7b7a59 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 19:57:46 -0400 Subject: [PATCH 060/198] Add package export control for type --- python/src/type/__init__.py | 4 ++++ python/src/type/type.py | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py index f147ce3ed..f0f38119e 100644 --- a/python/src/type/__init__.py +++ b/python/src/type/__init__.py @@ -1 +1,5 @@ """Spider type module.""" + +from type.type import Int8, Int16, Int32, Int64 + +__all__ = ["Int8", "Int16", "Int32", "Int64"] diff --git a/python/src/type/type.py b/python/src/type/type.py index 1a14dfdaf..a7d9ad41f 100644 --- a/python/src/type/type.py +++ b/python/src/type/type.py @@ -39,4 +39,12 @@ class Int32(BoundedInt): def __new__(cls, value: int) -> "Int32": """Creates an int32 integer.""" - return cast("Int32", super().__new__(cls, value, bits=8)) + return cast("Int32", super().__new__(cls, value, bits=32)) + + +class Int64(BoundedInt): + """64 bits integer type.""" + + def __new__(cls, value: int) -> "Int64": + """Creates an int64 integer.""" + return cast("Int64", super().__new__(cls, value, bits=64)) From f2ebaa67020d3525cdf705470cb2be5f7884270c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 20:02:45 -0400 Subject: [PATCH 061/198] Add floating point type --- python/src/type/__init__.py | 4 ++-- python/src/type/type.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/src/type/__init__.py b/python/src/type/__init__.py index f0f38119e..f8dba98e5 100644 --- a/python/src/type/__init__.py +++ b/python/src/type/__init__.py @@ -1,5 +1,5 @@ """Spider type module.""" -from type.type import Int8, Int16, Int32, Int64 +from type.type import Double, Float, Int8, Int16, Int32, Int64 -__all__ = ["Int8", "Int16", "Int32", "Int64"] +__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64"] diff --git a/python/src/type/type.py b/python/src/type/type.py index a7d9ad41f..318cc90ca 100644 --- a/python/src/type/type.py +++ b/python/src/type/type.py @@ -48,3 +48,7 @@ class Int64(BoundedInt): def __new__(cls, value: int) -> "Int64": """Creates an int64 integer.""" return cast("Int64", super().__new__(cls, value, bits=64)) + + +Float = float +Double = float From 8c27b1696ede2bb099693d1536e94930de2bac9f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 20:57:38 -0400 Subject: [PATCH 062/198] Restructure under src/spider --- python/pyproject.toml | 6 +----- python/src/core/__init__.py | 1 - python/src/spider/__init__.py | 1 + python/src/{ => spider}/client/__init__.py | 0 python/src/spider/core/__init__.py | 1 + python/src/{ => spider}/core/task.py | 0 python/src/{ => spider}/task_executor/__init__.py | 0 python/src/{ => spider}/task_executor/task_executor.py | 0 8 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 python/src/core/__init__.py create mode 100644 python/src/spider/__init__.py rename python/src/{ => spider}/client/__init__.py (100%) create mode 100644 python/src/spider/core/__init__.py rename python/src/{ => spider}/core/task.py (100%) rename python/src/{ => spider}/task_executor/__init__.py (100%) rename python/src/{ => spider}/task_executor/task_executor.py (100%) diff --git a/python/pyproject.toml b/python/pyproject.toml index 9b69cfdeb..658b21433 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ ] [project.scripts] -spider_task_executor = "src.task_executor.task_executor:main" +spider_task_executor = "src.spider.task_executor.task_executor:main" [dependency-groups] dev = [ @@ -22,10 +22,6 @@ dev = [ requires = ["hatchling>=1.18.0"] build-backend = "hatchling.build" -[tool.hatch] -[tool.hatch.build.targets.wheel] -packages = ["src/client", "src/task_executor"] - [tool.mypy] strict = true # Additional output diff --git a/python/src/core/__init__.py b/python/src/core/__init__.py deleted file mode 100644 index e4e408948..000000000 --- a/python/src/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider core module.""" diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py new file mode 100644 index 000000000..c3628976d --- /dev/null +++ b/python/src/spider/__init__.py @@ -0,0 +1 @@ +"""Spider package root.""" diff --git a/python/src/client/__init__.py b/python/src/spider/client/__init__.py similarity index 100% rename from python/src/client/__init__.py rename to python/src/spider/client/__init__.py diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py new file mode 100644 index 000000000..d417d95e2 --- /dev/null +++ b/python/src/spider/core/__init__.py @@ -0,0 +1 @@ +"""Spider core package.""" diff --git a/python/src/core/task.py b/python/src/spider/core/task.py similarity index 100% rename from python/src/core/task.py rename to python/src/spider/core/task.py diff --git a/python/src/task_executor/__init__.py b/python/src/spider/task_executor/__init__.py similarity index 100% rename from python/src/task_executor/__init__.py rename to python/src/spider/task_executor/__init__.py diff --git a/python/src/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py similarity index 100% rename from python/src/task_executor/task_executor.py rename to python/src/spider/task_executor/task_executor.py From a7aa6b096bbf69ac99d97d331ca65ea4bca4ae1f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 21:02:07 -0400 Subject: [PATCH 063/198] Re-export spider.type under spider --- python/src/spider/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py index c3628976d..929a7aaa1 100644 --- a/python/src/spider/__init__.py +++ b/python/src/spider/__init__.py @@ -1 +1,12 @@ """Spider package root.""" + +from spider.type import Double, Float, Int8, Int16, Int32, Int64 + +__all__ = [ + "Double", + "Float", + "Int8", + "Int16", + "Int32", + "Int64", +] From 8ffbd039011fb2e5a04267bf49e2f1369d1d818a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 7 Aug 2025 21:07:14 -0400 Subject: [PATCH 064/198] Fix import path --- python/src/spider/core/task.py | 6 +++--- python/src/spider/core/taskgraph.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index 2deff2208..8a6c79d31 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -4,7 +4,7 @@ from enum import IntEnum from uuid import UUID -from core.data import DataId +from spider.core.data import DataId TaskId = UUID @@ -46,5 +46,5 @@ class Task: state: TaskState timeout: float max_retries: int - task_input: list[TaskInputData] = field(default_factory=list) - task_output: list[TaskOutputData] = field(default_factory=list) + task_input: list[TaskInput] = field(default_factory=list) + task_output: list[TaskOutput] = field(default_factory=list) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index d209bc378..12de9feb3 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -1,6 +1,6 @@ """TaskGraph module for Spider.""" -from core.task import Task, TaskId +from spider.core.task import Task, TaskId class TaskGraph: From 5765acf6b059e05c036f5c7eb1a337f73c60a82b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 00:50:38 -0400 Subject: [PATCH 065/198] Add tdl types; Allow no docstring for override function --- python/pyproject.toml | 3 + python/src/spider/type/tdl_type.py | 130 +++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 python/src/spider/type/tdl_type.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 658b21433..5da3702d2 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -59,3 +59,6 @@ ignore = [ "UP015", # Explicit open modes are helpful ] isort.order-by-type = false + +[tool.ruff.lint.pydocstyle] +ignore-decorators = ["typing.override"] diff --git a/python/src/spider/type/tdl_type.py b/python/src/spider/type/tdl_type.py new file mode 100644 index 000000000..5f484ca8a --- /dev/null +++ b/python/src/spider/type/tdl_type.py @@ -0,0 +1,130 @@ +"""Spider TDL types.""" + +from abc import ABC, abstractmethod + +from typing_extensions import override + + +class TdlType(ABC): + """Abstract base class for all TDL types.""" + + @abstractmethod + def type_str(self) -> str: + """:return: String representation of the TDL type.""" + + +class DoubleType(TdlType): + """TDL double type.""" + + @override + def type_str(self) -> str: + return "double" + + +class FloatType(TdlType): + """TDL float type.""" + + @override + def type_str(self) -> str: + return "float" + + +class Int8Type(TdlType): + """TDL int8 type.""" + + @override + def type_str(self) -> str: + return "int8" + + +class Int16Type(TdlType): + """TDL int16 type.""" + + @override + def type_str(self) -> str: + return "int16" + + +class Int32Type(TdlType): + """TDL int32 type.""" + + @override + def type_str(self) -> str: + return "int32" + + +class Int64Type(TdlType): + """TDL int64 type.""" + + @override + def type_str(self) -> str: + return "int64" + + +class BoolType(TdlType): + """TDL bool type.""" + + @override + def type_str(self) -> str: + return "bool" + + +class ClassType(TdlType): + """TDL Custom class type.""" + + def __init__(self, name: str) -> None: + """ + Creates a TDL custom class type. + :param name: The name of the class. + """ + self.name = name + + @override + def type_str(self) -> str: + return self.name + + +class ListType(TdlType): + """TDL List type.""" + + def __init__(self, key: TdlType) -> None: + """Creates a TDL list type.""" + self.key = key + + @override + def type_str(self) -> str: + return f"List<{self.key.type_str()}>" + + +def is_integral(tdl_type: TdlType) -> bool: + """:return: If TDL type is an integral type.""" + return isinstance(tdl_type, (Int8Type, Int16Type, Int32Type, Int64Type)) + + +def is_string(tdl_type: TdlType) -> bool: + """:return: If TDL type is a string type, i.e. List.""" + return isinstance(tdl_type, ListType) and isinstance(tdl_type.key, Int8Type) + + +def is_map_key(tdl_type: TdlType) -> bool: + """:return: If TDL type is a valid type for map key.""" + return is_integral(tdl_type) or is_string(tdl_type) + + +class MapType(TdlType): + """TDL Map type.""" + + def __init__(self, key: TdlType, value: TdlType) -> None: + """ + Creates a TDL map type. + :raises TypeError: If key is not a supported type. + """ + if not is_map_key(key): + msg = f"{key} is not a supported type for map key." + raise TypeError(msg) + self.key = key + self.value = value + + @override + def type_str(self) -> str: + return f"Map<{self.key.type_str()},{self.value.type_str()}>" From 213d2fcd5e69099f68d43953ebc40e0797156232 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 13:28:19 -0400 Subject: [PATCH 066/198] Add type conversion to tdl type --- python/src/spider/type/tdl_convert.py | 84 +++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 python/src/spider/type/tdl_convert.py diff --git a/python/src/spider/type/tdl_convert.py b/python/src/spider/type/tdl_convert.py new file mode 100644 index 000000000..82c99d89e --- /dev/null +++ b/python/src/spider/type/tdl_convert.py @@ -0,0 +1,84 @@ +"""Converts native types to TDL types.""" + +from collections.abc import Collection +from typing import get_args + +from spider import Double, Float, Int8, Int16, Int32, Int64 +from spider.type.tdl_type import ( + BoolType, + ClassType, + DoubleType, + FloatType, + Int8Type, + Int16Type, + Int32Type, + Int64Type, + ListType, + MapType, + TdlType, +) + + +def to_primitive_tdl_type(native_type: type) -> TdlType | None: + """ + Converts a native type to primitive TDL type. + :param native_type: + :return: Converted TDL primitive. None if `native_type` is not a supported primitive type. + """ + tdl_type: TdlType | None = None + if native_type is Int8: + tdl_type = Int8Type() + elif native_type is Int16: + tdl_type = Int16Type() + elif native_type is Int32: + tdl_type = Int32Type() + elif native_type is Int64: + tdl_type = Int64Type() + elif native_type is Float: + tdl_type = FloatType() + elif native_type is Double: + tdl_type = DoubleType() + elif native_type is bool: + tdl_type = BoolType() + return tdl_type + + +def to_tdl_type(native_type: type) -> TdlType: + """ + Converts a Python type to TDL type. + :param native_type: + :return: + :raise: TypeError if `native_type` is not a valid TDL type. + """ + primitive_tdl_type = to_primitive_tdl_type(native_type) + if primitive_tdl_type is not None: + return primitive_tdl_type + + if native_type is list: + arg = get_args(native_type) + if arg is None: + msg = "List does not have a key type." + raise TypeError(msg) + arg = arg[0] + return ListType(to_tdl_type(arg)) + + if native_type is dict: + arg = get_args(native_type) + msg = "Dict does not have a key/value type." + if arg is None: + raise TypeError(msg) + if len(arg) != 2: # noqa: PLR2004 + raise TypeError(msg) + key = arg[0] + value = arg[1] + return MapType(to_tdl_type(key), to_tdl_type(value)) + + if native_type in (int, float, str, complex, bytes): + msg = f"{native_type} is not a valid TDL type." + raise TypeError(msg) + + if issubclass(native_type, Collection): + msg = f"{native_type} is not a valid TDL type." + raise TypeError(msg) + + return ClassType(native_type.__name__) From a1a86f329c370a8bc1b9063276e4af56e9a5ecc2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 13:31:34 -0400 Subject: [PATCH 067/198] Add to_tdl_str --- python/src/spider/type/tdl_convert.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/src/spider/type/tdl_convert.py b/python/src/spider/type/tdl_convert.py index 82c99d89e..eb2a8ad8a 100644 --- a/python/src/spider/type/tdl_convert.py +++ b/python/src/spider/type/tdl_convert.py @@ -82,3 +82,13 @@ def to_tdl_type(native_type: type) -> TdlType: raise TypeError(msg) return ClassType(native_type.__name__) + + +def to_tdl_type_str(native_type: type) -> str: + """ + Converts a Python type to TDL type string. + :param native_type: + :return: + :raise: TypeError if `native_type` is not a valid TDL type. + """ + return to_tdl_type(native_type).type_str() From 97af71bb7d0e0e86a7a34ec2a3dd7b45a1cb9898 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 13:33:10 -0400 Subject: [PATCH 068/198] Add to_tdl_type_str to type package export --- python/src/spider/type/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/src/spider/type/__init__.py b/python/src/spider/type/__init__.py index 70ff91d90..e0c873db6 100644 --- a/python/src/spider/type/__init__.py +++ b/python/src/spider/type/__init__.py @@ -1,5 +1,6 @@ """Spider type package.""" +from spider.type.tdl_convert import to_tdl_type_str from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 -__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64"] +__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64", "to_tdl_type_str"] From 74f31e26e4e50f9b09253e769b1eb501b718e741 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 16:28:35 -0400 Subject: [PATCH 069/198] Add pytest and basic test structure --- python/pyproject.toml | 11 ++++++ python/tests/__init__.py | 1 + python/tests/core/__init__.py | 1 + python/tests/core/test_task.py | 9 +++++ python/uv.lock | 68 ++++++++++++++++++++++++++++++++++ 5 files changed, 90 insertions(+) create mode 100644 python/tests/__init__.py create mode 100644 python/tests/core/__init__.py create mode 100644 python/tests/core/test_task.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 658b21433..f0c2b7680 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -15,6 +15,7 @@ spider_task_executor = "src.spider.task_executor.task_executor:main" [dependency-groups] dev = [ "mypy>=1.12.0", + "pytest>=8.4.1", "ruff>=0.4.4", ] @@ -29,6 +30,9 @@ pretty = true show_error_code_links = true show_error_context = true +[tool.pytest.ini_options] +testpaths = ["tests"] + [tool.ruff] line-length = 100 @@ -59,3 +63,10 @@ ignore = [ "UP015", # Explicit open modes are helpful ] isort.order-by-type = false + +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "S101", # Allow use of `assert` (security warning) + "S603", # Allow use of `subprocess.Popen` (security warning) + "T201", # Allow use of `print` (testing) +] diff --git a/python/tests/__init__.py b/python/tests/__init__.py new file mode 100644 index 000000000..ab1be478c --- /dev/null +++ b/python/tests/__init__.py @@ -0,0 +1 @@ +"""Spider tests.""" diff --git a/python/tests/core/__init__.py b/python/tests/core/__init__.py new file mode 100644 index 000000000..4835acd8d --- /dev/null +++ b/python/tests/core/__init__.py @@ -0,0 +1 @@ +"""Spider core package tests.""" diff --git a/python/tests/core/test_task.py b/python/tests/core/test_task.py new file mode 100644 index 000000000..a6d4834ec --- /dev/null +++ b/python/tests/core/test_task.py @@ -0,0 +1,9 @@ +"""Test core task.""" + +from spider.core.task import Task + + +def test_task() -> None: + """Tests task created is not None.""" + task = Task() + assert task is not None diff --git a/python/uv.lock b/python/uv.lock index 6dc97a77e..ae2f0e96f 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -2,6 +2,36 @@ version = 1 revision = 2 requires-python = ">=3.10" +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/f4/c6e662dade71f56cd2f3735141b265c3c79293c109549c1e6933b0651ffc/exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10", size = 16674, upload-time = "2025-05-10T17:42:49.33Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + [[package]] name = "mariadb" version = "1.1.13" @@ -154,6 +184,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, ] +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/ba/45911d754e8eba3d5a841a5ce61a65a685ff1798421ac054f85aa8747dfb/pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c", size = 1517714, upload-time = "2025-06-18T05:48:06.109Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/16/c8a903f4c4dffe7a12843191437d7cd8e32751d5de349d45d3fe69544e87/pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7", size = 365474, upload-time = "2025-06-18T05:48:03.955Z" }, +] + [[package]] name = "ruff" version = "0.12.7" @@ -191,6 +257,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "mypy" }, + { name = "pytest" }, { name = "ruff" }, ] @@ -203,6 +270,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "mypy", specifier = ">=1.12.0" }, + { name = "pytest", specifier = ">=8.4.1" }, { name = "ruff", specifier = ">=0.4.4" }, ] From 7d9570663a0e6452b40f7fa50eb1169c926ea7a8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 16:28:57 -0400 Subject: [PATCH 070/198] Add python test tasks --- test-tasks.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test-tasks.yaml b/test-tasks.yaml index 936211e5b..0b6e4aa47 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -76,3 +76,8 @@ tasks: vars: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + + python-tests: + dir: "{{.G_SRC_PYTHON_DIR}}" + cmd: |- + uv run pytest From 96b5324a642087d75ec757b2908e489b6cd87d1d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 16:33:24 -0400 Subject: [PATCH 071/198] Rename some cpp tests and add python tests to GH workflow and doc --- .github/workflows/code-linting-checks.yaml | 1 + .github/workflows/unit-tests.yaml | 3 +++ docs/src/dev-docs/testing.md | 17 +++++++++-------- test-tasks.yaml | 4 ++-- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/code-linting-checks.yaml b/.github/workflows/code-linting-checks.yaml index 48f8eacd7..f8dd495ae 100644 --- a/.github/workflows/code-linting-checks.yaml +++ b/.github/workflows/code-linting-checks.yaml @@ -44,6 +44,7 @@ jobs: python --version tar --version task --version + uv --version - name: "Install project dependencies " timeout-minutes: 10 diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 1ced6a889..058779260 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -47,6 +47,7 @@ jobs: python --version tar --version task --version + uv --version - name: "Install project dependencies " timeout-minutes: 10 @@ -55,3 +56,5 @@ jobs: run: "task deps:lib_install" - run: "task test:non-storage-unit-tests" + + - run: "task test:python-tests" diff --git a/docs/src/dev-docs/testing.md b/docs/src/dev-docs/testing.md index 873ef7b67..759cf5023 100644 --- a/docs/src/dev-docs/testing.md +++ b/docs/src/dev-docs/testing.md @@ -30,11 +30,12 @@ require this storage backend. You can use the following tasks to run the set of unit tests that's appropriate. -| Task | Description | -|-------------------------------|-------------------------------------------------------------------| -| `test:all` | Runs all unit tests. | -| `test:non-storage-unit-tests` | Runs all unit tests which don't require a storage backend to run. | -| `test:storage-unit-tests` | Runs all unit tests which require a storage backend to run. | +| Task | Description | +|-------------------------------|-----------------------------------------------------------------------| +| `test:cpp-all` | Runs all C++ unit tests. | +| `test:non-storage-unit-tests` | Runs all C++ unit tests which don't require a storage backend to run. | +| `test:storage-unit-tests` | Runs all C++ unit tests which require a storage backend to run. | +| `test:python-tests` | Runs all Python tests. | If any tests show error messages for the connection function below, revisit the [setup section](#set-up-mysql-as-storage-backend) and verify that `cStorageUrl` was set correctly. @@ -52,9 +53,9 @@ pull requests, and daily. Currently, it only runs unit tests that don't require You can use the following tasks to run integration tests. -| Task | Description | -|-------------------------------|-------------------------------------------------------------------| -| `test:integration` | Runs all integration tests. | +| Task | Description | +|------------------------|---------------------------------| +| `test:cpp-integration` | Runs all C++ integration tests. | [gh-workflow-unit-tests]: https://github.com/y-scope/spider/blob/main/.github/workflows/unit-tests.yaml diff --git a/test-tasks.yaml b/test-tasks.yaml index 0b6e4aa47..1d4c88fb3 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -18,7 +18,7 @@ tasks: cmds: - "{{.G_UNIT_TEST_BINARY}} \"[storage]\"" - all: + cpp-all: deps: - "build-unit-test" cmds: @@ -31,7 +31,7 @@ tasks: vars: TARGETS: ["spider_task_executor", "unitTest", "worker_test"] - integration: + cpp-integration: dir: "{{.G_BUILD_SPIDER_DIR}}" deps: - "venv" From 14ea6fb93a7eebda34b29920b245897715f059c6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 16:44:22 -0400 Subject: [PATCH 072/198] Don't create __pycache__ when running pytest --- test-tasks.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-tasks.yaml b/test-tasks.yaml index 1d4c88fb3..cb8bbdde6 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -79,5 +79,6 @@ tasks: python-tests: dir: "{{.G_SRC_PYTHON_DIR}}" - cmd: |- - uv run pytest + cmds: + # Don't create __pycache__ directories in source tree. + - "uv run env PYTHONDONTWRITEBYTECODE=1 pytest" From 854cd11dbfcc009b144c374b493973a52c66524b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 17:19:25 -0400 Subject: [PATCH 073/198] Fix missing link --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8ac64e9e3..11aec22fd 100644 --- a/README.md +++ b/README.md @@ -68,3 +68,4 @@ in the table below. [spider-docs]: https://docs.yscope.com/spider/main/ [Task]: https://taskfile.dev +[uv]: https://docs.astral.sh/uv/ \ No newline at end of file From 7ea49287700fbab6d7d1ac0344e42e4f5297a60c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 17:21:16 -0400 Subject: [PATCH 074/198] Use task env --- test-tasks.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-tasks.yaml b/test-tasks.yaml index cb8bbdde6..88a2c8380 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -79,6 +79,8 @@ tasks: python-tests: dir: "{{.G_SRC_PYTHON_DIR}}" + env: + PYTHONDONTWRITEBYTECODE: "1" cmds: # Don't create __pycache__ directories in source tree. - - "uv run env PYTHONDONTWRITEBYTECODE=1 pytest" + - "uv run env pytest" From 01dd0c6b6fb4ca99f7b045bfb13d70c697c0d8ca Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 17:26:30 -0400 Subject: [PATCH 075/198] Fix task executor path --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 658b21433..0c9da379e 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ ] [project.scripts] -spider_task_executor = "src.spider.task_executor.task_executor:main" +spider_task_executor = "spider.task_executor.task_executor:main" [dependency-groups] dev = [ From 438f6e770c4a92492e1208144fb2e5f211fabe08 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 22:11:10 -0400 Subject: [PATCH 076/198] Remove unnecessary __init__.py files --- python/tests/__init__.py | 1 - python/tests/core/__init__.py | 1 - 2 files changed, 2 deletions(-) delete mode 100644 python/tests/__init__.py delete mode 100644 python/tests/core/__init__.py diff --git a/python/tests/__init__.py b/python/tests/__init__.py deleted file mode 100644 index ab1be478c..000000000 --- a/python/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider tests.""" diff --git a/python/tests/core/__init__.py b/python/tests/core/__init__.py deleted file mode 100644 index 4835acd8d..000000000 --- a/python/tests/core/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Spider core package tests.""" From 61e0651d0d74e49bfc3ae10658ea0c5b6b33a2e7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 22:51:30 -0400 Subject: [PATCH 077/198] Fix bugs --- python/src/spider/type/tdl_convert.py | 46 +++++++++++++++------------ python/src/spider/type/type.py | 16 ++++++++-- python/tests/core/test_task.py | 9 ------ 3 files changed, 40 insertions(+), 31 deletions(-) delete mode 100644 python/tests/core/test_task.py diff --git a/python/src/spider/type/tdl_convert.py b/python/src/spider/type/tdl_convert.py index eb2a8ad8a..35bff2667 100644 --- a/python/src/spider/type/tdl_convert.py +++ b/python/src/spider/type/tdl_convert.py @@ -1,9 +1,9 @@ """Converts native types to TDL types.""" +import types from collections.abc import Collection -from typing import get_args +from typing import get_args, get_origin -from spider import Double, Float, Int8, Int16, Int32, Int64 from spider.type.tdl_type import ( BoolType, ClassType, @@ -17,6 +17,7 @@ MapType, TdlType, ) +from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 def to_primitive_tdl_type(native_type: type) -> TdlType | None: @@ -54,26 +55,31 @@ def to_tdl_type(native_type: type) -> TdlType: if primitive_tdl_type is not None: return primitive_tdl_type - if native_type is list: - arg = get_args(native_type) - if arg is None: - msg = "List does not have a key type." - raise TypeError(msg) - arg = arg[0] - return ListType(to_tdl_type(arg)) + if native_type in (int, float, str, complex, bytes): + msg = f"{native_type} is not a valid TDL type." + raise TypeError(msg) - if native_type is dict: - arg = get_args(native_type) - msg = "Dict does not have a key/value type." - if arg is None: - raise TypeError(msg) - if len(arg) != 2: # noqa: PLR2004 - raise TypeError(msg) - key = arg[0] - value = arg[1] - return MapType(to_tdl_type(key), to_tdl_type(value)) + if isinstance(native_type, types.GenericAlias): + origin = get_origin(native_type) + if origin is list: + arg = get_args(native_type) + if arg is None: + msg = "List does not have a key type." + raise TypeError(msg) + arg = arg[0] + return ListType(to_tdl_type(arg)) + + if origin is dict: + arg = get_args(native_type) + msg = "Dict does not have a key/value type." + if arg is None: + raise TypeError(msg) + if len(arg) != 2: # noqa: PLR2004 + raise TypeError(msg) + key = arg[0] + value = arg[1] + return MapType(to_tdl_type(key), to_tdl_type(value)) - if native_type in (int, float, str, complex, bytes): msg = f"{native_type} is not a valid TDL type." raise TypeError(msg) diff --git a/python/src/spider/type/type.py b/python/src/spider/type/type.py index 318cc90ca..903a9972c 100644 --- a/python/src/spider/type/type.py +++ b/python/src/spider/type/type.py @@ -50,5 +50,17 @@ def __new__(cls, value: int) -> "Int64": return cast("Int64", super().__new__(cls, value, bits=64)) -Float = float -Double = float +class Float(float): + """Float type.""" + + def __new__(cls, value: float) -> "Float": + """Creates a float number.""" + return super().__new__(cls, value) + + +class Double(float): + """Double type.""" + + def __new__(cls, value: float) -> "Double": + """Creates a double number.""" + return super().__new__(cls, value) diff --git a/python/tests/core/test_task.py b/python/tests/core/test_task.py deleted file mode 100644 index a6d4834ec..000000000 --- a/python/tests/core/test_task.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Test core task.""" - -from spider.core.task import Task - - -def test_task() -> None: - """Tests task created is not None.""" - task = Task() - assert task is not None From 8b13abdb591703623ef9d39dfabfc377db29c1d0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 22:55:59 -0400 Subject: [PATCH 078/198] Add some type convertion tests --- python/pyproject.toml | 1 + python/tests/type/test_to_tdl.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 python/tests/type/test_to_tdl.py diff --git a/python/pyproject.toml b/python/pyproject.toml index ab0c7ae3c..74c43b235 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -66,6 +66,7 @@ isort.order-by-type = false [tool.ruff.lint.per-file-ignores] "tests/**" = [ + "INP001", # Allow implicit namespace package for tests "S101", # Allow use of `assert` (security warning) "S603", # Allow use of `subprocess.Popen` (security warning) "T201", # Allow use of `print` (testing) diff --git a/python/tests/type/test_to_tdl.py b/python/tests/type/test_to_tdl.py new file mode 100644 index 000000000..a0d2db3c2 --- /dev/null +++ b/python/tests/type/test_to_tdl.py @@ -0,0 +1,23 @@ +"""Unit tests for converting to TDL.""" + +from spider import Double, Float, Int8, Int16, Int32, Int64 +from spider.type import to_tdl_type_str + + +class TestToTDL: + """Unit tests for converting to TDL.""" + + def test_to_tdl_primitive(self) -> None: + """Test converting primitive types to TDL Types.""" + assert to_tdl_type_str(bool) == "bool" + assert to_tdl_type_str(Double) == "double" + assert to_tdl_type_str(Float) == "float" + assert to_tdl_type_str(Int8) == "int8" + assert to_tdl_type_str(Int16) == "int16" + assert to_tdl_type_str(Int32) == "int32" + assert to_tdl_type_str(Int64) == "int64" + + def test_to_tdl_list(self) -> None: + """Test converting lists to TDL Types.""" + assert to_tdl_type_str(list[Int8]) == "List" + assert to_tdl_type_str(list[list[Int8]]) == "List>" From 50b2b25cd13bb3236bf8617ef731f50a41624ac4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 23:02:20 -0400 Subject: [PATCH 079/198] Bug fix --- python/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index e3308eb50..db8a99329 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -66,6 +66,7 @@ isort.order-by-type = false [tool.ruff.lint.per-file-ignores] "tests/**" = [ + "INP001", # Allow implicit namespace package for tests "S101", # Allow use of `assert` (security warning) "S603", # Allow use of `subprocess.Popen` (security warning) "T201", # Allow use of `print` (testing) From 8e2786bda5bab979967e5037e1765cf96b3fedb0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 8 Aug 2025 23:40:25 -0400 Subject: [PATCH 080/198] Add more tests --- python/tests/type/test_to_tdl.py | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/python/tests/type/test_to_tdl.py b/python/tests/type/test_to_tdl.py index a0d2db3c2..6ffb98a5a 100644 --- a/python/tests/type/test_to_tdl.py +++ b/python/tests/type/test_to_tdl.py @@ -1,5 +1,7 @@ """Unit tests for converting to TDL.""" +import pytest + from spider import Double, Float, Int8, Int16, Int32, Int64 from spider.type import to_tdl_type_str @@ -21,3 +23,47 @@ def test_to_tdl_list(self) -> None: """Test converting lists to TDL Types.""" assert to_tdl_type_str(list[Int8]) == "List" assert to_tdl_type_str(list[list[Int8]]) == "List>" + + def test_to_tdl_map(self) -> None: + """Test converting maps to TDL Types.""" + assert to_tdl_type_str(dict[Int8, dict[Int16, Float]]) == "Map>" + assert to_tdl_type_str(dict[list[Int8], Double]) == "Map,double>" + + def test_to_tdl_class(self) -> None: + """Test converting class to TDL Types.""" + assert to_tdl_type_str(TestToTDL) == "TestToTDL" + + def test_to_tdl_primitive_exception(self) -> None: + """Test converting unsupported primitive types to TDL Types.""" + with pytest.raises(TypeError): + to_tdl_type_str(int) + with pytest.raises(TypeError): + to_tdl_type_str(float) + with pytest.raises(TypeError): + to_tdl_type_str(str) + with pytest.raises(TypeError): + to_tdl_type_str(bytes) + with pytest.raises(TypeError): + to_tdl_type_str(list) + with pytest.raises(TypeError): + to_tdl_type_str(dict) + with pytest.raises(TypeError): + to_tdl_type_str(tuple) + + def test_to_tdl_list_exception(self) -> None: + """Test converting unsupported lists to TDL Types.""" + with pytest.raises(TypeError): + to_tdl_type_str(list[int]) + + def test_to_tdl_map_exception(self) -> None: + """Test converting unsupported maps to TDL Types.""" + with pytest.raises(TypeError): + to_tdl_type_str(dict[Int8, int]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[int, Int8]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[list[Int16], Int8]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[Float, Int8]) + with pytest.raises(TypeError): + to_tdl_type_str(dict[dict[Int8, Float], Int8]) From 41d5de28e79876be75dd548da7d6f49350fd6415 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 14:09:47 -0400 Subject: [PATCH 081/198] Add parsing of tdl type string --- python/pyproject.toml | 1 + python/src/spider/type/tdl_parse.py | 61 +++++++++++++++++++++++++++++ python/uv.lock | 11 ++++++ 3 files changed, 73 insertions(+) create mode 100644 python/src/spider/type/tdl_parse.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 74c43b235..b9614a302 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -5,6 +5,7 @@ description = "Spider is a distributed task execution framework" readme = "README.md" requires-python = ">=3.10" dependencies = [ + "lark>=1.2.2", "mariadb>=1.1.13", "msgpack-types>=0.5.0", ] diff --git a/python/src/spider/type/tdl_parse.py b/python/src/spider/type/tdl_parse.py new file mode 100644 index 000000000..24cc5cc89 --- /dev/null +++ b/python/src/spider/type/tdl_parse.py @@ -0,0 +1,61 @@ +"""Parse TDL type string.""" + +from copy import copy + +from lark import Token, Transformer, Tree, v_args + +from spider.type.tdl_type import ( + BoolType, + ClassType, + DoubleType, + Int8Type, + Int16Type, + Int32Type, + Int64Type, + ListType, + MapType, + TdlType, +) + +grammar = r""" +type: map_type | list_type | base_type + +map_type: "Map<" type_type "," type_type ">" +list_type: "List<" type_type ">" +base_type: ID + +ID: /[A-Za-z_][A-Za-z0-9_]*/ + +%import common.WS +%ignore WS +""" + +primitive_type_map = { + "bool": BoolType(), + "double": DoubleType(), + "int8": Int8Type(), + "int16": Int16Type(), + "int32": Int32Type(), + "int64": Int64Type(), +} + + +class TypeTransformer(Transformer[Tree[Token], TdlType]): + """Transform Lark tree into TDL type.""" + + @v_args(inline=True) + def map_type(self, key: TdlType, value: TdlType) -> TdlType: + """Transforms map node into Map type.""" + return MapType(key, value) + + @v_args(inline=True) + def list_type(self, key: TdlType) -> TdlType: + """Transforms list node into Map type.""" + return ListType(key) + + def base_type(self, children: list[Token]) -> TdlType: + """Transforms primitive node into primitive type.""" + name = str(children[0]) + if name in primitive_type_map: + return copy(primitive_type_map[name]) + return ClassType(name) diff --git a/python/uv.lock b/python/uv.lock index ae2f0e96f..f3a9e6630 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -32,6 +32,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, ] +[[package]] +name = "lark" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132, upload-time = "2024-08-13T19:49:00.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" }, +] + [[package]] name = "mariadb" version = "1.1.13" @@ -250,6 +259,7 @@ name = "spider" version = "0.0.1" source = { editable = "." } dependencies = [ + { name = "lark" }, { name = "mariadb" }, { name = "msgpack-types" }, ] @@ -263,6 +273,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "lark", specifier = ">=1.2.2" }, { name = "mariadb", specifier = ">=1.1.13" }, { name = "msgpack-types", specifier = ">=0.5.0" }, ] From 0a377f5bbfd1ee07e7830a81fd73c84dffd04339 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 17:43:15 -0400 Subject: [PATCH 082/198] Add tests and fix parser --- python/src/spider/type/tdl_parse.py | 39 +++++++++++----- python/src/spider/type/tdl_type.py | 70 ++++++++++++++++++++++++++++- python/tests/type/test_to_native.py | 47 +++++++++++++++++++ 3 files changed, 144 insertions(+), 12 deletions(-) create mode 100644 python/tests/type/test_to_native.py diff --git a/python/src/spider/type/tdl_parse.py b/python/src/spider/type/tdl_parse.py index 24cc5cc89..82c99c7a4 100644 --- a/python/src/spider/type/tdl_parse.py +++ b/python/src/spider/type/tdl_parse.py @@ -2,29 +2,30 @@ from copy import copy -from lark import Token, Transformer, Tree, v_args +from lark import Lark, Token, Transformer, v_args, Tree from spider.type.tdl_type import ( BoolType, ClassType, DoubleType, + FloatType, Int8Type, Int16Type, Int32Type, Int64Type, ListType, MapType, - TdlType, + TdlType ) grammar = r""" type: map_type | list_type | base_type -map_type: "Map<" type_type "," type_type ">" -list_type: "List<" type_type ">" +map_type: "Map" "<" type "," type ">" +list_type: "List" "<" type ">" base_type: ID -ID: /[A-Za-z_][A-Za-z0-9_]*/ +ID: /[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*/ %import common.WS %ignore WS @@ -33,6 +34,7 @@ primitive_type_map = { "bool": BoolType(), "double": DoubleType(), + "float": FloatType(), "int8": Int8Type(), "int16": Int16Type(), "int32": Int32Type(), @@ -40,18 +42,18 @@ } -class TypeTransformer(Transformer[Tree[Token], TdlType]): - """Transform Lark tree into TDL type.""" +class TypeTransformer(Transformer[Token, TdlType]): + """Transform Lark parse tree into TDL type.""" @v_args(inline=True) - def map_type(self, key: TdlType, value: TdlType) -> TdlType: + def map_type(self, key: Tree, value: Tree) -> TdlType: """Transforms map node into Map type.""" - return MapType(key, value) + return MapType(key.children[0], value.children[0]) @v_args(inline=True) - def list_type(self, key: TdlType) -> TdlType: + def list_type(self, key: Tree) -> TdlType: """Transforms list node into Map type.""" - return ListType(key) + return ListType(key.children[0]) def base_type(self, children: list[Token]) -> TdlType: """Transforms primitive node into primitive type.""" @@ -59,3 +61,18 @@ def base_type(self, children: list[Token]) -> TdlType: if name in primitive_type_map: return copy(primitive_type_map[name]) return ClassType(name) + +parser = Lark(grammar, start="type", parser="lalr") + +def parse_tdl_type(string: str) -> TdlType: + """ + Parses TDL type string into TDL type. + :param string: TDL type string. + :return: Parsed TDL type. + :raise: TypeError if string is not a valid TDL type. + """ + tree = parser.parse(string) + try: + return TypeTransformer(visit_tokens=False).transform(tree).children[0] + except IndexError: + raise TypeError(f"'{string}' is not a valid TDL type.") diff --git a/python/src/spider/type/tdl_type.py b/python/src/spider/type/tdl_type.py index 5f484ca8a..dae5930be 100644 --- a/python/src/spider/type/tdl_type.py +++ b/python/src/spider/type/tdl_type.py @@ -1,9 +1,13 @@ """Spider TDL types.""" - +import importlib from abc import ABC, abstractmethod +from types import GenericAlias +from typing import cast from typing_extensions import override +from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 + class TdlType(ABC): """Abstract base class for all TDL types.""" @@ -12,6 +16,10 @@ class TdlType(ABC): def type_str(self) -> str: """:return: String representation of the TDL type.""" + @abstractmethod + def native_type(self) -> type | GenericAlias: + """:return: Native Python type of the TDL type.""" + class DoubleType(TdlType): """TDL double type.""" @@ -20,6 +28,9 @@ class DoubleType(TdlType): def type_str(self) -> str: return "double" + @override + def native_type(self) -> type | GenericAlias: + return Double class FloatType(TdlType): """TDL float type.""" @@ -28,6 +39,9 @@ class FloatType(TdlType): def type_str(self) -> str: return "float" + @override + def native_type(self) -> type | GenericAlias: + return Float class Int8Type(TdlType): """TDL int8 type.""" @@ -36,6 +50,10 @@ class Int8Type(TdlType): def type_str(self) -> str: return "int8" + @override + def native_type(self) -> type | GenericAlias: + return Int8 + class Int16Type(TdlType): """TDL int16 type.""" @@ -44,6 +62,9 @@ class Int16Type(TdlType): def type_str(self) -> str: return "int16" + @override + def native_type(self) -> type | GenericAlias: + return Int16 class Int32Type(TdlType): """TDL int32 type.""" @@ -52,6 +73,9 @@ class Int32Type(TdlType): def type_str(self) -> str: return "int32" + @override + def native_type(self) -> type | GenericAlias: + return Int32 class Int64Type(TdlType): """TDL int64 type.""" @@ -60,6 +84,9 @@ class Int64Type(TdlType): def type_str(self) -> str: return "int64" + @override + def native_type(self) -> type | GenericAlias: + return Int64 class BoolType(TdlType): """TDL bool type.""" @@ -68,6 +95,33 @@ class BoolType(TdlType): def type_str(self) -> str: return "bool" + @override + def native_type(self) -> type | GenericAlias: + return bool + +def get_class_name(cls: type) -> str: + """ + Gets Full class name. + :param cls: + :return: + """ + return f"{cls.__module__}.{cls.__qualname__}" + +def get_class_by_name(name: str) -> type: + """ + Gets class by name. + :param name: + :return: + :raise: TypeError if `class_name` is not a valid class. + """ + parts = name.split(".") + module_name = ".".join(parts[:-1]) + class_name = parts[-1] + try: + module = importlib.import_module(module_name) + return cast("type", getattr(module, class_name)) + except Exception: + raise TypeError(f"'{name}' is not a valid class.") class ClassType(TdlType): """TDL Custom class type.""" @@ -83,6 +137,13 @@ def __init__(self, name: str) -> None: def type_str(self) -> str: return self.name + @override + def native_type(self) -> type | GenericAlias: + """ + :return: Native Python type of the class. + :raise: TypeError if `class_name` is not a valid class. + """ + return get_class_by_name(self.name) class ListType(TdlType): """TDL List type.""" @@ -95,6 +156,9 @@ def __init__(self, key: TdlType) -> None: def type_str(self) -> str: return f"List<{self.key.type_str()}>" + @override + def native_type(self) -> type | GenericAlias: + return list[self.key.native_type()] def is_integral(tdl_type: TdlType) -> bool: """:return: If TDL type is an integral type.""" @@ -128,3 +192,7 @@ def __init__(self, key: TdlType, value: TdlType) -> None: @override def type_str(self) -> str: return f"Map<{self.key.type_str()},{self.value.type_str()}>" + + @override + def native_type(self) -> type | GenericAlias: + return dict[self.key.native_type(), self.value.native_type()] diff --git a/python/tests/type/test_to_native.py b/python/tests/type/test_to_native.py new file mode 100644 index 000000000..944065319 --- /dev/null +++ b/python/tests/type/test_to_native.py @@ -0,0 +1,47 @@ +"""Test converting TDL type to native type.""" +from types import GenericAlias + +import pytest + +import spider +from spider.type.tdl_parse import parse_tdl_type +from spider.type.tdl_type import get_class_name + + +def string_to_native(s: str) -> type | GenericAlias: + """ + Converts TDL string to native type. + :param s: TDL string to convert. + :return: The converted native type. + :raises: TypeError forwards from `parse_tdl_type` and `native_type`. + """ + return parse_tdl_type(s).native_type() + + +class TestToNativeType: + """Test converting TDL type to native type.""" + + def test_to_primitive_native_type(self) -> None: + """Test converting primitive TDL type to native type.""" + assert string_to_native("double") == spider.Double + assert string_to_native("float") == spider.Float + assert string_to_native("int8") == spider.Int8 + assert string_to_native("int16") == spider.Int16 + assert string_to_native("int32") == spider.Int32 + assert string_to_native("int64") == spider.Int64 + + def test_to_class_type(self) -> None: + """Test converting class TDL type to native type.""" + assert string_to_native(get_class_name(TestToNativeType)) == TestToNativeType + with pytest.raises(TypeError): + string_to_native("NonExistType") + + def test_to_list_type(self) -> None: + """Test converting list TDL type to native type.""" + assert string_to_native("List") == list[spider.Int8] + assert string_to_native("List>") == list[list[spider.Int8]] + + def test_to_map_type(self) -> None: + """Test converting map TDL type to native type.""" + assert string_to_native("Map") == dict[spider.Int8, spider.Int8] + assert string_to_native("Map,Map>") == dict[list[spider.Int8], dict[spider.Int8, spider.Double]] From a8c8641fc8f524902d784395bfc737e6600a1d0a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 17:47:36 -0400 Subject: [PATCH 083/198] Fix pytest --- test-tasks.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-tasks.yaml b/test-tasks.yaml index 88a2c8380..ca635301b 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -80,7 +80,7 @@ tasks: python-tests: dir: "{{.G_SRC_PYTHON_DIR}}" env: + # Don't create __pycache__ directories in source tree. PYTHONDONTWRITEBYTECODE: "1" cmds: - # Don't create __pycache__ directories in source tree. - - "uv run env pytest" + - "uv run pytest" From 0b18170b24b61e3ca56a0e0c93943242553d2dd0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 18:12:32 -0400 Subject: [PATCH 084/198] Satisfy linter --- python/src/spider/type/tdl_parse.py | 22 +++++++++++++--------- python/src/spider/type/tdl_type.py | 24 ++++++++++++++++++------ python/tests/type/test_to_native.py | 8 +++++++- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/python/src/spider/type/tdl_parse.py b/python/src/spider/type/tdl_parse.py index 82c99c7a4..bc8f04498 100644 --- a/python/src/spider/type/tdl_parse.py +++ b/python/src/spider/type/tdl_parse.py @@ -1,8 +1,9 @@ """Parse TDL type string.""" from copy import copy +from typing import cast -from lark import Lark, Token, Transformer, v_args, Tree +from lark import Lark, Token, Transformer, Tree, v_args from spider.type.tdl_type import ( BoolType, @@ -15,7 +16,7 @@ Int64Type, ListType, MapType, - TdlType + TdlType, ) grammar = r""" @@ -46,14 +47,14 @@ class TypeTransformer(Transformer[Token, TdlType]): """Transform Lark parse tree into TDL type.""" @v_args(inline=True) - def map_type(self, key: Tree, value: Tree) -> TdlType: + def map_type(self, key: Tree[str], value: Tree[str]) -> TdlType: """Transforms map node into Map type.""" - return MapType(key.children[0], value.children[0]) + return MapType(cast("TdlType", key.children[0]), cast("TdlType", value.children[0])) @v_args(inline=True) - def list_type(self, key: Tree) -> TdlType: + def list_type(self, key: Tree[str]) -> TdlType: """Transforms list node into Map type.""" - return ListType(key.children[0]) + return ListType(cast("TdlType", key.children[0])) def base_type(self, children: list[Token]) -> TdlType: """Transforms primitive node into primitive type.""" @@ -62,8 +63,10 @@ def base_type(self, children: list[Token]) -> TdlType: return copy(primitive_type_map[name]) return ClassType(name) + parser = Lark(grammar, start="type", parser="lalr") + def parse_tdl_type(string: str) -> TdlType: """ Parses TDL type string into TDL type. @@ -73,6 +76,7 @@ def parse_tdl_type(string: str) -> TdlType: """ tree = parser.parse(string) try: - return TypeTransformer(visit_tokens=False).transform(tree).children[0] - except IndexError: - raise TypeError(f"'{string}' is not a valid TDL type.") + return cast("TdlType", TypeTransformer(visit_tokens=False).transform(tree).children[0]) # type: ignore[attr-defined] + except IndexError as exc: + msg = f"{string} is not a valid TDL type." + raise TypeError(msg) from exc diff --git a/python/src/spider/type/tdl_type.py b/python/src/spider/type/tdl_type.py index dae5930be..84318d8ad 100644 --- a/python/src/spider/type/tdl_type.py +++ b/python/src/spider/type/tdl_type.py @@ -1,6 +1,7 @@ """Spider TDL types.""" -import importlib + from abc import ABC, abstractmethod +from importlib import import_module from types import GenericAlias from typing import cast @@ -32,6 +33,7 @@ def type_str(self) -> str: def native_type(self) -> type | GenericAlias: return Double + class FloatType(TdlType): """TDL float type.""" @@ -43,6 +45,7 @@ def type_str(self) -> str: def native_type(self) -> type | GenericAlias: return Float + class Int8Type(TdlType): """TDL int8 type.""" @@ -66,6 +69,7 @@ def type_str(self) -> str: def native_type(self) -> type | GenericAlias: return Int16 + class Int32Type(TdlType): """TDL int32 type.""" @@ -77,6 +81,7 @@ def type_str(self) -> str: def native_type(self) -> type | GenericAlias: return Int32 + class Int64Type(TdlType): """TDL int64 type.""" @@ -88,6 +93,7 @@ def type_str(self) -> str: def native_type(self) -> type | GenericAlias: return Int64 + class BoolType(TdlType): """TDL bool type.""" @@ -99,6 +105,7 @@ def type_str(self) -> str: def native_type(self) -> type | GenericAlias: return bool + def get_class_name(cls: type) -> str: """ Gets Full class name. @@ -107,6 +114,7 @@ def get_class_name(cls: type) -> str: """ return f"{cls.__module__}.{cls.__qualname__}" + def get_class_by_name(name: str) -> type: """ Gets class by name. @@ -118,10 +126,12 @@ def get_class_by_name(name: str) -> type: module_name = ".".join(parts[:-1]) class_name = parts[-1] try: - module = importlib.import_module(module_name) + module = import_module(module_name) return cast("type", getattr(module, class_name)) - except Exception: - raise TypeError(f"'{name}' is not a valid class.") + except (ValueError, ModuleNotFoundError, AttributeError) as exc: + msg = f"{name} is not a valid TDL type." + raise TypeError(msg) from exc + class ClassType(TdlType): """TDL Custom class type.""" @@ -145,6 +155,7 @@ def native_type(self) -> type | GenericAlias: """ return get_class_by_name(self.name) + class ListType(TdlType): """TDL List type.""" @@ -158,7 +169,8 @@ def type_str(self) -> str: @override def native_type(self) -> type | GenericAlias: - return list[self.key.native_type()] + return list[self.key.native_type()] # type: ignore[misc] + def is_integral(tdl_type: TdlType) -> bool: """:return: If TDL type is an integral type.""" @@ -195,4 +207,4 @@ def type_str(self) -> str: @override def native_type(self) -> type | GenericAlias: - return dict[self.key.native_type(), self.value.native_type()] + return dict[self.key.native_type(), self.value.native_type()] # type: ignore[misc] diff --git a/python/tests/type/test_to_native.py b/python/tests/type/test_to_native.py index 944065319..fb7afb4d9 100644 --- a/python/tests/type/test_to_native.py +++ b/python/tests/type/test_to_native.py @@ -1,4 +1,5 @@ """Test converting TDL type to native type.""" + from types import GenericAlias import pytest @@ -35,6 +36,8 @@ def test_to_class_type(self) -> None: assert string_to_native(get_class_name(TestToNativeType)) == TestToNativeType with pytest.raises(TypeError): string_to_native("NonExistType") + with pytest.raises(TypeError): + string_to_native("NonExistModule.NonExistType") def test_to_list_type(self) -> None: """Test converting list TDL type to native type.""" @@ -44,4 +47,7 @@ def test_to_list_type(self) -> None: def test_to_map_type(self) -> None: """Test converting map TDL type to native type.""" assert string_to_native("Map") == dict[spider.Int8, spider.Int8] - assert string_to_native("Map,Map>") == dict[list[spider.Int8], dict[spider.Int8, spider.Double]] + assert ( + string_to_native("Map,Map>") + == dict[list[spider.Int8], dict[spider.Int8, spider.Double]] + ) From 0bfbdb5b1044ea568f2bb216d43b351bc4988b2c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 18:37:08 -0400 Subject: [PATCH 085/198] Fix comment --- python/src/spider/type/tdl_parse.py | 19 +++++++++---------- python/tests/type/test_to_native.py | 13 +++++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/python/src/spider/type/tdl_parse.py b/python/src/spider/type/tdl_parse.py index bc8f04498..e47a78889 100644 --- a/python/src/spider/type/tdl_parse.py +++ b/python/src/spider/type/tdl_parse.py @@ -1,6 +1,5 @@ """Parse TDL type string.""" -from copy import copy from typing import cast from lark import Lark, Token, Transformer, Tree, v_args @@ -33,13 +32,13 @@ """ primitive_type_map = { - "bool": BoolType(), - "double": DoubleType(), - "float": FloatType(), - "int8": Int8Type(), - "int16": Int16Type(), - "int32": Int32Type(), - "int64": Int64Type(), + "bool": BoolType, + "double": DoubleType, + "float": FloatType, + "int8": Int8Type, + "int16": Int16Type, + "int32": Int32Type, + "int64": Int64Type, } @@ -53,14 +52,14 @@ def map_type(self, key: Tree[str], value: Tree[str]) -> TdlType: @v_args(inline=True) def list_type(self, key: Tree[str]) -> TdlType: - """Transforms list node into Map type.""" + """Transforms list node into List type.""" return ListType(cast("TdlType", key.children[0])) def base_type(self, children: list[Token]) -> TdlType: """Transforms primitive node into primitive type.""" name = str(children[0]) if name in primitive_type_map: - return copy(primitive_type_map[name]) + return primitive_type_map[name]() # type: ignore[abstract] return ClassType(name) diff --git a/python/tests/type/test_to_native.py b/python/tests/type/test_to_native.py index fb7afb4d9..cb599d3b6 100644 --- a/python/tests/type/test_to_native.py +++ b/python/tests/type/test_to_native.py @@ -24,12 +24,13 @@ class TestToNativeType: def test_to_primitive_native_type(self) -> None: """Test converting primitive TDL type to native type.""" - assert string_to_native("double") == spider.Double - assert string_to_native("float") == spider.Float - assert string_to_native("int8") == spider.Int8 - assert string_to_native("int16") == spider.Int16 - assert string_to_native("int32") == spider.Int32 - assert string_to_native("int64") == spider.Int64 + assert string_to_native("bool") is bool + assert string_to_native("double") is spider.Double + assert string_to_native("float") is spider.Float + assert string_to_native("int8") is spider.Int8 + assert string_to_native("int16") is spider.Int16 + assert string_to_native("int32") is spider.Int32 + assert string_to_native("int64") is spider.Int64 def test_to_class_type(self) -> None: """Test converting class TDL type to native type.""" From e558f67a068fff8ef135a4006ee644970c97d393 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 18:46:12 -0400 Subject: [PATCH 086/198] Improve code according to coderabbit --- python/src/spider/type/tdl_convert.py | 13 +++++++------ python/src/spider/type/utils.py | 9 +++++++++ python/tests/type/test_to_tdl.py | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 python/src/spider/type/utils.py diff --git a/python/src/spider/type/tdl_convert.py b/python/src/spider/type/tdl_convert.py index 35bff2667..09a2f416d 100644 --- a/python/src/spider/type/tdl_convert.py +++ b/python/src/spider/type/tdl_convert.py @@ -1,7 +1,7 @@ """Converts native types to TDL types.""" -import types from collections.abc import Collection +from types import GenericAlias from typing import get_args, get_origin from spider.type.tdl_type import ( @@ -18,9 +18,10 @@ TdlType, ) from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 +from spider.type.utils import get_class_name -def to_primitive_tdl_type(native_type: type) -> TdlType | None: +def to_primitive_tdl_type(native_type: type | GenericAlias) -> TdlType | None: """ Converts a native type to primitive TDL type. :param native_type: @@ -44,7 +45,7 @@ def to_primitive_tdl_type(native_type: type) -> TdlType | None: return tdl_type -def to_tdl_type(native_type: type) -> TdlType: +def to_tdl_type(native_type: type | GenericAlias) -> TdlType: """ Converts a Python type to TDL type. :param native_type: @@ -59,7 +60,7 @@ def to_tdl_type(native_type: type) -> TdlType: msg = f"{native_type} is not a valid TDL type." raise TypeError(msg) - if isinstance(native_type, types.GenericAlias): + if isinstance(native_type, GenericAlias): origin = get_origin(native_type) if origin is list: arg = get_args(native_type) @@ -87,10 +88,10 @@ def to_tdl_type(native_type: type) -> TdlType: msg = f"{native_type} is not a valid TDL type." raise TypeError(msg) - return ClassType(native_type.__name__) + return ClassType(get_class_name(native_type)) -def to_tdl_type_str(native_type: type) -> str: +def to_tdl_type_str(native_type: type | GenericAlias) -> str: """ Converts a Python type to TDL type string. :param native_type: diff --git a/python/src/spider/type/utils.py b/python/src/spider/type/utils.py new file mode 100644 index 000000000..f1d3ae113 --- /dev/null +++ b/python/src/spider/type/utils.py @@ -0,0 +1,9 @@ +"""Utility for TDL types.""" + + +def get_class_name(cls: type) -> str: + """ + :param cls: + :return: Full name of `cls`. + """ + return f"{cls.__module__}.{cls.__qualname__}" diff --git a/python/tests/type/test_to_tdl.py b/python/tests/type/test_to_tdl.py index 6ffb98a5a..c0fa734d9 100644 --- a/python/tests/type/test_to_tdl.py +++ b/python/tests/type/test_to_tdl.py @@ -31,7 +31,7 @@ def test_to_tdl_map(self) -> None: def test_to_tdl_class(self) -> None: """Test converting class to TDL Types.""" - assert to_tdl_type_str(TestToTDL) == "TestToTDL" + assert to_tdl_type_str(TestToTDL) == "test_to_tdl.TestToTDL" def test_to_tdl_primitive_exception(self) -> None: """Test converting unsupported primitive types to TDL Types.""" From 6b58ff0c60f6918161c383d02bb2f07b21db7fc2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 18:48:43 -0400 Subject: [PATCH 087/198] Move functions to util --- python/src/spider/type/tdl_type.py | 30 +---------------------------- python/src/spider/type/utils.py | 21 ++++++++++++++++++++ python/tests/type/test_to_native.py | 2 +- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/python/src/spider/type/tdl_type.py b/python/src/spider/type/tdl_type.py index 84318d8ad..cd0256b49 100644 --- a/python/src/spider/type/tdl_type.py +++ b/python/src/spider/type/tdl_type.py @@ -1,13 +1,12 @@ """Spider TDL types.""" from abc import ABC, abstractmethod -from importlib import import_module from types import GenericAlias -from typing import cast from typing_extensions import override from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 +from spider.type.utils import get_class_by_name class TdlType(ABC): @@ -106,33 +105,6 @@ def native_type(self) -> type | GenericAlias: return bool -def get_class_name(cls: type) -> str: - """ - Gets Full class name. - :param cls: - :return: - """ - return f"{cls.__module__}.{cls.__qualname__}" - - -def get_class_by_name(name: str) -> type: - """ - Gets class by name. - :param name: - :return: - :raise: TypeError if `class_name` is not a valid class. - """ - parts = name.split(".") - module_name = ".".join(parts[:-1]) - class_name = parts[-1] - try: - module = import_module(module_name) - return cast("type", getattr(module, class_name)) - except (ValueError, ModuleNotFoundError, AttributeError) as exc: - msg = f"{name} is not a valid TDL type." - raise TypeError(msg) from exc - - class ClassType(TdlType): """TDL Custom class type.""" diff --git a/python/src/spider/type/utils.py b/python/src/spider/type/utils.py index f1d3ae113..023ceed74 100644 --- a/python/src/spider/type/utils.py +++ b/python/src/spider/type/utils.py @@ -1,5 +1,8 @@ """Utility for TDL types.""" +from importlib import import_module +from typing import cast + def get_class_name(cls: type) -> str: """ @@ -7,3 +10,21 @@ def get_class_name(cls: type) -> str: :return: Full name of `cls`. """ return f"{cls.__module__}.{cls.__qualname__}" + + +def get_class_by_name(name: str) -> type: + """ + Gets class by name. + :param name: + :return: + :raise: TypeError if `class_name` is not a valid class. + """ + parts = name.split(".") + module_name = ".".join(parts[:-1]) + class_name = parts[-1] + try: + module = import_module(module_name) + return cast("type", getattr(module, class_name)) + except (ValueError, ModuleNotFoundError, AttributeError) as exc: + msg = f"{name} is not a valid TDL type." + raise TypeError(msg) from exc diff --git a/python/tests/type/test_to_native.py b/python/tests/type/test_to_native.py index cb599d3b6..0f5a9c5ae 100644 --- a/python/tests/type/test_to_native.py +++ b/python/tests/type/test_to_native.py @@ -6,7 +6,7 @@ import spider from spider.type.tdl_parse import parse_tdl_type -from spider.type.tdl_type import get_class_name +from spider.type.utils import get_class_name def string_to_native(s: str) -> type | GenericAlias: From 90ab3345f1c1dfbc834049c156a9f07d3283b8a4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 20:11:22 -0400 Subject: [PATCH 088/198] Unwarp the type node to get TdlType --- python/src/spider/type/tdl_parse.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/python/src/spider/type/tdl_parse.py b/python/src/spider/type/tdl_parse.py index e47a78889..28416a82d 100644 --- a/python/src/spider/type/tdl_parse.py +++ b/python/src/spider/type/tdl_parse.py @@ -1,8 +1,7 @@ """Parse TDL type string.""" -from typing import cast -from lark import Lark, Token, Transformer, Tree, v_args +from lark import Lark, Token, Transformer, v_args from spider.type.tdl_type import ( BoolType, @@ -46,14 +45,19 @@ class TypeTransformer(Transformer[Token, TdlType]): """Transform Lark parse tree into TDL type.""" @v_args(inline=True) - def map_type(self, key: Tree[str], value: Tree[str]) -> TdlType: + def type(self, value: TdlType) -> TdlType: + """Unwraps the type node to return the TdlType.""" + return value + + @v_args(inline=True) + def map_type(self, key: TdlType, value: TdlType) -> TdlType: """Transforms map node into Map type.""" - return MapType(cast("TdlType", key.children[0]), cast("TdlType", value.children[0])) + return MapType(key, value) @v_args(inline=True) - def list_type(self, key: Tree[str]) -> TdlType: + def list_type(self, key: TdlType) -> TdlType: """Transforms list node into List type.""" - return ListType(cast("TdlType", key.children[0])) + return ListType(key) def base_type(self, children: list[Token]) -> TdlType: """Transforms primitive node into primitive type.""" @@ -74,8 +78,4 @@ def parse_tdl_type(string: str) -> TdlType: :raise: TypeError if string is not a valid TDL type. """ tree = parser.parse(string) - try: - return cast("TdlType", TypeTransformer(visit_tokens=False).transform(tree).children[0]) # type: ignore[attr-defined] - except IndexError as exc: - msg = f"{string} is not a valid TDL type." - raise TypeError(msg) from exc + return TypeTransformer(visit_tokens=False).transform(tree) From 49acaee4b56c6f733ff6deb1506527a051d31a5e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 20:13:40 -0400 Subject: [PATCH 089/198] Catch and rethrow lark error --- python/src/spider/type/tdl_parse.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/src/spider/type/tdl_parse.py b/python/src/spider/type/tdl_parse.py index 28416a82d..eebbe55ce 100644 --- a/python/src/spider/type/tdl_parse.py +++ b/python/src/spider/type/tdl_parse.py @@ -1,7 +1,6 @@ """Parse TDL type string.""" - -from lark import Lark, Token, Transformer, v_args +from lark import Lark, LarkError, Token, Transformer, v_args from spider.type.tdl_type import ( BoolType, @@ -77,5 +76,9 @@ def parse_tdl_type(string: str) -> TdlType: :return: Parsed TDL type. :raise: TypeError if string is not a valid TDL type. """ - tree = parser.parse(string) - return TypeTransformer(visit_tokens=False).transform(tree) + try: + tree = parser.parse(string) + return TypeTransformer(visit_tokens=False).transform(tree) + except LarkError as ecx: + msg = f"Cannot parse TDL type '{string}'" + raise TypeError(msg) from ecx From ae2a93570b526cc8015a6e527e6640db4625b728 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 9 Aug 2025 23:23:38 -0400 Subject: [PATCH 090/198] Add reset id --- python/src/spider/core/task.py | 4 ++-- python/src/spider/core/taskgraph.py | 37 +++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index 8a6c79d31..4a645bd3d 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -46,5 +46,5 @@ class Task: state: TaskState timeout: float max_retries: int - task_input: list[TaskInput] = field(default_factory=list) - task_output: list[TaskOutput] = field(default_factory=list) + task_inputs: list[TaskInput] = field(default_factory=list) + task_outputs: list[TaskOutput] = field(default_factory=list) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index 12de9feb3..44e723402 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -1,6 +1,9 @@ """TaskGraph module for Spider.""" -from spider.core.task import Task, TaskId +from copy import deepcopy +from uuid import uuid4 + +from spider.core.task import Task, TaskId, TaskInputOutput class TaskGraph: @@ -25,7 +28,7 @@ def add_task( :param parents: The parent ids of the task. Must be already in the task graph. :param children: The children ids of the task. Must be already in the task graph. """ - self.tasks[task.task_id] = task + self.tasks[task.task_id] = deepcopy(task) if parents: for parent in parents: self.dependencies.append((parent, task.task_id)) @@ -54,3 +57,33 @@ def get_children(self, task_id: TaskId) -> list[Task]: :return: List of children tasks. """ return [self.tasks[child] for (parent, child) in self.dependencies if parent == task_id] + + def reset_ids(self) -> None: + """Resets task ids.""" + id_map = {} + for task_id in self.tasks: + id_map[task_id] = uuid4() + + new_tasks = {} + for task_id in self.tasks: + new_task_id = id_map[task_id] + new_tasks[new_task_id] = deepcopy(self.tasks[task_id]) + for task_input in new_tasks[new_task_id].task_inputs: + if isinstance(task_input, TaskInputOutput): + task_input.task_id = id_map[task_input.task_id] + self.tasks = new_tasks + + new_dependencies = [] + for parent, child in self.dependencies: + new_dependencies.append((id_map[parent], id_map[child])) + self.dependencies = new_dependencies + + new_input_tasks = set() + for task_id in self.input_tasks: + new_input_tasks.add(id_map[task_id]) + self.input_tasks = new_input_tasks + + new_output_tasks = set() + for task_id in self.output_tasks: + new_output_tasks.add(id_map[task_id]) + self.output_tasks = new_output_tasks From 008c6fb4a6dd78ac5b655016c3ad987446dffd1d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 12:20:49 -0400 Subject: [PATCH 091/198] Export core --- python/src/spider/core/__init__.py | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py index d417d95e2..d463958c8 100644 --- a/python/src/spider/core/__init__.py +++ b/python/src/spider/core/__init__.py @@ -1 +1,32 @@ """Spider core package.""" + +from .data import Data, DataId +from .task import ( + Task, + TaskId, + TaskInput, + TaskInputData, + TaskInputOutput, + TaskInputValue, + TaskOutput, + TaskOutputData, + TaskOutputValue, + TaskState, +) +from .taskgraph import TaskGraph + +__all__ = [ + "Data", + "DataId", + "Task", + "TaskGraph", + "TaskId", + "TaskInput", + "TaskInputData", + "TaskInputOutput", + "TaskInputValue", + "TaskOutput", + "TaskOutputData", + "TaskOutputValue", + "TaskState", +] From 720265fe6674f274cc5a8430f8f64cba558050cd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 12:22:08 -0400 Subject: [PATCH 092/198] Add client task graph --- python/src/spider/client/__init__.py | 6 ++++++ python/src/spider/client/taskgraph.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 python/src/spider/client/taskgraph.py diff --git a/python/src/spider/client/__init__.py b/python/src/spider/client/__init__.py index 038456143..f74a2cf68 100644 --- a/python/src/spider/client/__init__.py +++ b/python/src/spider/client/__init__.py @@ -1 +1,7 @@ """Spider python client.""" + +from .taskgraph import TaskGraph + +__all__ = [ + "TaskGraph", +] diff --git a/python/src/spider/client/taskgraph.py b/python/src/spider/client/taskgraph.py new file mode 100644 index 000000000..992e164f1 --- /dev/null +++ b/python/src/spider/client/taskgraph.py @@ -0,0 +1,14 @@ +"""Spider client TaskGraph module.""" + +from spider import core + + +class TaskGraph: + """ + Spider client TaskGraph class. + Warps around the core TaskGraph class. + """ + + def __init__(self) -> None: + """Initialize TaskGraph.""" + self._impl = core.TaskGraph() From ebe64c6ece76580fdf8e40557b9a22526ab5e119 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 12:22:57 -0400 Subject: [PATCH 093/198] Add client taskgraph to spider export --- python/src/spider/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py index 929a7aaa1..01043548e 100644 --- a/python/src/spider/__init__.py +++ b/python/src/spider/__init__.py @@ -1,5 +1,6 @@ """Spider package root.""" +from spider.client import TaskGraph from spider.type import Double, Float, Int8, Int16, Int32, Int64 __all__ = [ @@ -9,4 +10,5 @@ "Int16", "Int32", "Int64", + "TaskGraph", ] From 404cf3bea7c5c2dd83a160aea618211b3bae1f65 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 12:26:23 -0400 Subject: [PATCH 094/198] Excempt _impl from private access check --- python/pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index 74c43b235..adb9f8d19 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -72,5 +72,8 @@ isort.order-by-type = false "T201", # Allow use of `print` (testing) ] +[tool.ruff.lint.flake8-self] +ignore-names = ["_impl"] + [tool.ruff.lint.pydocstyle] ignore-decorators = ["typing.override"] From 3f9fe049f1cad8bd34217bb256c92b3bb74cd0bd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 12:53:45 -0400 Subject: [PATCH 095/198] Add basic TaskFunction defintion --- python/src/spider/client/task.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 python/src/spider/client/task.py diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py new file mode 100644 index 000000000..b2fe99f11 --- /dev/null +++ b/python/src/spider/client/task.py @@ -0,0 +1,21 @@ +"""Spider client task module.""" + +from typing import ParamSpec, Protocol, TypeVar + + +class TaskContext: + """Spider task context.""" + + # TODO: Implement task context for use in task executor + + +P = ParamSpec("P") +R_co = TypeVar("R_co", covariant=True) + + +class TaskFunction(Protocol[P, R_co]): + """Task function accepts TaskContext as its first argument.""" + + def __call__(self, context: TaskContext, *args: P.args, **kwargs: P.kwargs) -> R_co: + """Task function accepts TaskContext as its first argument.""" + ... From 4d10f88284932ac363b5f74e6b8c340feb002b13 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 13:02:21 -0400 Subject: [PATCH 096/198] "Add default values for tasks" --- python/src/spider/core/task.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index 8a6c79d31..f4c72911f 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from enum import IntEnum -from uuid import UUID +from uuid import UUID, uuid4 from spider.core.data import DataId @@ -41,10 +41,10 @@ class TaskState(IntEnum): class Task: """Represents a task in Spider.""" - task_id: TaskId - function_name: str - state: TaskState - timeout: float - max_retries: int - task_input: list[TaskInput] = field(default_factory=list) - task_output: list[TaskOutput] = field(default_factory=list) + task_id: TaskId = field(default_factory=uuid4) + function_name: str = "" + state: TaskState = TaskState.Pending + timeout: float = 0 + max_retries: int = 0 + task_inputs: list[TaskInput] = field(default_factory=list) + task_outputs: list[TaskOutput] = field(default_factory=list) From f3ac001ea9c5c89fe60505c30e707dd8e60f82af Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 16:08:18 -0400 Subject: [PATCH 097/198] Fix task IO type and add create_task from function --- python/src/spider/client/data.py | 5 +++ python/src/spider/client/task.py | 64 +++++++++++++++++++++++---- python/src/spider/client/taskgraph.py | 16 +++++++ python/src/spider/core/task.py | 19 +++++++- 4 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 python/src/spider/client/data.py diff --git a/python/src/spider/client/data.py b/python/src/spider/client/data.py new file mode 100644 index 000000000..a5dffba18 --- /dev/null +++ b/python/src/spider/client/data.py @@ -0,0 +1,5 @@ +"""Spider client Data module.""" + + +class Data: + """Represents a spider client data.""" diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index b2fe99f11..7ec8b9c3b 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -1,6 +1,13 @@ """Spider client task module.""" -from typing import ParamSpec, Protocol, TypeVar +import inspect +from types import FunctionType +from typing import Protocol, runtime_checkable + +from spider import core +from spider.client.data import Data +from spider.core import TaskInput, TaskOutput, TaskOutputValue +from spider.type import to_tdl_type_str class TaskContext: @@ -9,13 +16,54 @@ class TaskContext: # TODO: Implement task context for use in task executor -P = ParamSpec("P") -R_co = TypeVar("R_co", covariant=True) +@runtime_checkable +class TaskFunction(Protocol): + """Task function must accept a TaskContext as its first argument.""" + + def __call__(self, context: TaskContext, *args: object) -> object: + """Task function must accept TaskContext as its first argument.""" + ... -class TaskFunction(Protocol[P, R_co]): - """Task function accepts TaskContext as its first argument.""" +def create_task(func: TaskFunction) -> core.Task: + """ + Creates a core Task object from the task function. + :param func: + :return: + :raise TypeError: If the function signature contains unsupported types. + """ + task = core.Task() + if not isinstance(func, FunctionType): + msg = "`func` is not a function." + raise TypeError(msg) + task.function_name = func.__qualname__ + signature = inspect.signature(func) + params = list(signature.parameters.values()) + if params[0].annotation is not TaskContext: + msg = "First argument is not a TaskContext." + raise TypeError(msg) + for param in params[1:]: + if param.annotation == inspect.Parameter.empty: + msg = "Argument must has type annotation" + raise TypeError(msg) + tdl_type_str = to_tdl_type_str(param.annotation) + task.task_inputs.append(TaskInput(tdl_type_str, None)) + returns = signature.return_annotation + if returns == inspect.Parameter.empty: + msg = "Return type must has type annotation" + raise TypeError(msg) + if type(returns) is tuple: + for r in returns: + tdl_type_str = to_tdl_type_str(r) + if r is Data: + task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) + else: + task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) + else: + tdl_type_str = to_tdl_type_str(returns) + if returns is Data: + task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) + else: + task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) - def __call__(self, context: TaskContext, *args: P.args, **kwargs: P.kwargs) -> R_co: - """Task function accepts TaskContext as its first argument.""" - ... + return task diff --git a/python/src/spider/client/taskgraph.py b/python/src/spider/client/taskgraph.py index 992e164f1..700ade201 100644 --- a/python/src/spider/client/taskgraph.py +++ b/python/src/spider/client/taskgraph.py @@ -1,6 +1,7 @@ """Spider client TaskGraph module.""" from spider import core +from spider.client.task import create_task, TaskFunction class TaskGraph: @@ -12,3 +13,18 @@ class TaskGraph: def __init__(self) -> None: """Initialize TaskGraph.""" self._impl = core.TaskGraph() + + +def group(tasks: list[TaskFunction | TaskGraph]) -> TaskGraph: + """ + Groups task functions and task graph into a single task graph. + :param tasks: List of task functions or task graphs. + :return: The new task graph. + """ + graph = TaskGraph() + for task in tasks: + if isinstance(task, TaskFunction): + graph._impl.add_task(create_task(task)) + # TODO: Add task graph + + return graph diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index f4c72911f..bf0edf0f6 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -19,11 +19,26 @@ class TaskInputOutput: TaskInputValue = bytes TaskInputData = DataId -TaskInput = TaskInputOutput | TaskInputValue | TaskInputData + + +@dataclass +class TaskInput: + """Represents a task input""" + + type: str + value: TaskInputData | TaskInputOutput | None + TaskOutputValue = bytes TaskOutputData = DataId -TaskOutput = TaskOutputValue | TaskOutputData + + +@dataclass +class TaskOutput: + """Represents a task output""" + + type: str + value: TaskOutputData | TaskOutputValue class TaskState(IntEnum): From 95b77734ab4a69179274b44b66c8d445ec3b5b75 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 17:16:36 -0400 Subject: [PATCH 098/198] Add client task group --- python/src/spider/client/taskgraph.py | 4 +++- python/src/spider/core/taskgraph.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/src/spider/client/taskgraph.py b/python/src/spider/client/taskgraph.py index 700ade201..40b12cb7c 100644 --- a/python/src/spider/client/taskgraph.py +++ b/python/src/spider/client/taskgraph.py @@ -25,6 +25,8 @@ def group(tasks: list[TaskFunction | TaskGraph]) -> TaskGraph: for task in tasks: if isinstance(task, TaskFunction): graph._impl.add_task(create_task(task)) - # TODO: Add task graph + else: + graph._impl.merge_graph(task._impl) + graph._impl.reset_ids() return graph diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index 44e723402..078cd6955 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -87,3 +87,14 @@ def reset_ids(self) -> None: for task_id in self.output_tasks: new_output_tasks.add(id_map[task_id]) self.output_tasks = new_output_tasks + + def merge_graph(self, graph: "TaskGraph") -> None: + """ + Merges another task graph into this task graph. + :param graph: The task graph to merge. + :return: + """ + self.tasks.update(graph.tasks) + self.dependencies.extend(graph.dependencies) + self.input_tasks.update(graph.input_tasks) + self.output_tasks.update(graph.output_tasks) From a72e4a62697c404fe4b5c363ff661bbaefe6c6ae Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 21:13:17 -0400 Subject: [PATCH 099/198] Add task graph chain --- python/src/spider/client/taskgraph.py | 30 +++++++++++++ python/src/spider/core/taskgraph.py | 62 +++++++++++++++++++++------ 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/python/src/spider/client/taskgraph.py b/python/src/spider/client/taskgraph.py index 40b12cb7c..c8ae280a8 100644 --- a/python/src/spider/client/taskgraph.py +++ b/python/src/spider/client/taskgraph.py @@ -14,6 +14,17 @@ def __init__(self) -> None: """Initialize TaskGraph.""" self._impl = core.TaskGraph() + def chain_graph(self, child: "TaskGraph") -> "TaskGraph": + """ + Chains another task graph with this task graph. + :param child: The task graph to be chained as child. + :return: The chained task graph. + :raise TypeError: If the outputs and the inputs of `graph` do not match. + """ + graph = TaskGraph() + graph._impl = self._impl.chain_graph(child._impl) + return graph + def group(tasks: list[TaskFunction | TaskGraph]) -> TaskGraph: """ @@ -30,3 +41,22 @@ def group(tasks: list[TaskFunction | TaskGraph]) -> TaskGraph: graph._impl.reset_ids() return graph + + +def chain(parent: TaskFunction | TaskGraph, child: TaskFunction | TaskGraph) -> TaskGraph: + """ + Chains two task functions or task graphs into a single task graph. + :param parent: + :param child: + :return: + :raises TypeError: If the parent outputs and child inputs do not match. + """ + if isinstance(parent, TaskFunction): + task = create_task(parent) + parent = TaskGraph() + parent._impl.add_task(task) + if isinstance(child, TaskFunction): + task = create_task(child) + child = TaskGraph() + child._impl.add_task(task) + return parent.chain_graph(child) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index 078cd6955..2f340f3f7 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -16,8 +16,8 @@ def __init__(self) -> None: # - parent task id # - child task id self.dependencies: list[tuple[TaskId, TaskId]] = [] - self.input_tasks: set[TaskId] = set() - self.output_tasks: set[TaskId] = set() + self.input_tasks: list[TaskId] = [] + self.output_tasks: list[TaskId] = [] def add_task( self, task: Task, parents: list[TaskId] | None = None, children: list[TaskId] | None = None @@ -32,15 +32,15 @@ def add_task( if parents: for parent in parents: self.dependencies.append((parent, task.task_id)) - self.output_tasks.discard(parent) + self.output_tasks.append(parent) else: - self.input_tasks.add(task.task_id) + self.input_tasks.append(task.task_id) if children: for child in children: self.dependencies.append((task.task_id, child)) - self.input_tasks.discard(child) + self.input_tasks.append(child) else: - self.output_tasks.add(task.task_id) + self.output_tasks.append(task.task_id) def get_parents(self, task_id: TaskId) -> list[Task]: """ @@ -78,14 +78,14 @@ def reset_ids(self) -> None: new_dependencies.append((id_map[parent], id_map[child])) self.dependencies = new_dependencies - new_input_tasks = set() + new_input_tasks = [] for task_id in self.input_tasks: - new_input_tasks.add(id_map[task_id]) + new_input_tasks.append(id_map[task_id]) self.input_tasks = new_input_tasks - new_output_tasks = set() + new_output_tasks = [] for task_id in self.output_tasks: - new_output_tasks.add(id_map[task_id]) + new_output_tasks.append(id_map[task_id]) self.output_tasks = new_output_tasks def merge_graph(self, graph: "TaskGraph") -> None: @@ -96,5 +96,43 @@ def merge_graph(self, graph: "TaskGraph") -> None: """ self.tasks.update(graph.tasks) self.dependencies.extend(graph.dependencies) - self.input_tasks.update(graph.input_tasks) - self.output_tasks.update(graph.output_tasks) + self.input_tasks.extend(graph.input_tasks) + self.output_tasks.extend(graph.output_tasks) + + def chain_graph(self, child: "TaskGraph") -> "TaskGraph": + """ + Chains another task graph with this task graph. + :param child: The task graph to be chained as child. + :return: The chained task graph. + :raise TypeError: If the outputs and the inputs of `graph` do not match. + """ + graph = deepcopy(self) + parent_output_tasks = graph.output_tasks + graph.tasks.update(child.tasks) + graph.dependencies.extend(child.dependencies) + graph.output_tasks = deepcopy(child.output_tasks) + + size_mismatch_msg = "Parent outputs size and child inputs size do not match." + + task_index, output_position = 0, 0 + for task_id in child.input_tasks: + input_task = graph.tasks[task_id] + for i in range(len(input_task.task_inputs)): + output_task_id = parent_output_tasks[task_index] + input_type = input_task.task_inputs[i].type + output_type = graph.tasks[output_task_id].task_outputs[output_position].type + if input_type != output_type: + msg = f"Output type {output_type} does not match input type {input_type}" + raise TypeError(msg) + input_task.task_inputs[i].value = TaskInputOutput(output_task_id, output_position) + output_position += 1 + if len(graph.tasks[output_task_id].task_outputs) > output_position: + output_position = 0 + task_index += 1 + if task_index >= len(graph.tasks[output_task_id].task_outputs): + raise TypeError(size_mismatch_msg) + + if task_index != len(parent_output_tasks) or output_position != 0: + raise TypeError(size_mismatch_msg) + + return graph From f289e5a43b31afaf2aaa213c9045a11b6d9707a3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 23:36:37 -0400 Subject: [PATCH 100/198] Reset ids after chain --- python/src/spider/core/taskgraph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index 2f340f3f7..58e9406d9 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -135,4 +135,5 @@ def chain_graph(self, child: "TaskGraph") -> "TaskGraph": if task_index != len(parent_output_tasks) or output_position != 0: raise TypeError(size_mismatch_msg) + graph.reset_ids() return graph From 2e829bc00428d80ff0d683010a3520950db3b517 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 23:39:23 -0400 Subject: [PATCH 101/198] Add export --- python/src/spider/__init__.py | 4 +++- python/src/spider/client/__init__.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py index 01043548e..4303f6488 100644 --- a/python/src/spider/__init__.py +++ b/python/src/spider/__init__.py @@ -1,6 +1,6 @@ """Spider package root.""" -from spider.client import TaskGraph +from spider.client import chain, group, TaskGraph from spider.type import Double, Float, Int8, Int16, Int32, Int64 __all__ = [ @@ -11,4 +11,6 @@ "Int32", "Int64", "TaskGraph", + "chain", + "group", ] diff --git a/python/src/spider/client/__init__.py b/python/src/spider/client/__init__.py index f74a2cf68..d76332b1d 100644 --- a/python/src/spider/client/__init__.py +++ b/python/src/spider/client/__init__.py @@ -1,7 +1,9 @@ """Spider python client.""" -from .taskgraph import TaskGraph +from .taskgraph import chain, group, TaskGraph __all__ = [ "TaskGraph", + "chain", + "group", ] From ec6802d5c8d8cd2254b32c9d9ef2d3b11cd1b64d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 23:42:43 -0400 Subject: [PATCH 102/198] Export TaskContext --- python/src/spider/__init__.py | 3 ++- python/src/spider/client/__init__.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py index 4303f6488..b0ac7f542 100644 --- a/python/src/spider/__init__.py +++ b/python/src/spider/__init__.py @@ -1,6 +1,6 @@ """Spider package root.""" -from spider.client import chain, group, TaskGraph +from spider.client import chain, group, TaskContext, TaskGraph from spider.type import Double, Float, Int8, Int16, Int32, Int64 __all__ = [ @@ -10,6 +10,7 @@ "Int16", "Int32", "Int64", + "TaskContext", "TaskGraph", "chain", "group", diff --git a/python/src/spider/client/__init__.py b/python/src/spider/client/__init__.py index d76332b1d..37ab0dd7a 100644 --- a/python/src/spider/client/__init__.py +++ b/python/src/spider/client/__init__.py @@ -1,8 +1,10 @@ """Spider python client.""" +from .task import TaskContext from .taskgraph import chain, group, TaskGraph __all__ = [ + "TaskContext", "TaskGraph", "chain", "group", From 3685dd7e092c6db7857f8bcea3434ff40ffbc94b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 10 Aug 2025 23:54:40 -0400 Subject: [PATCH 103/198] Add unit tests --- python/tests/client/test_task_graph.py | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 python/tests/client/test_task_graph.py diff --git a/python/tests/client/test_task_graph.py b/python/tests/client/test_task_graph.py new file mode 100644 index 000000000..6ae69e542 --- /dev/null +++ b/python/tests/client/test_task_graph.py @@ -0,0 +1,51 @@ +"""Unit tests for Spider client TaskGraph""" +import pytest + +from spider import TaskContext, Int8, group, chain + +def no_context(x: Int8, y: Int8) -> Int8: + """Invalid task function with no context.""" + return x + y + +def invalid_type(_: TaskContext, x: int) -> int: + return x + x + +def add(_: TaskContext, x: Int8, y: Int8) -> Int8: + """Adds two numbers.""" + return x + y + +def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: + """Swaps two numbers.""" + return y, x + +class TestTaskGraph: + + def test_group(self) -> None: + """Tests task grouping.""" + graph = group([add, add]) + graph = group([graph, graph]) + assert len(graph._impl.tasks) == 4 + assert len(graph._impl.dependencies) == 0 + assert len(graph._impl.input_tasks) == 4 + assert len(graph._impl.output_tasks) == 4 + + def test_task_fail(self) -> None: + """Tests task failing.""" + with pytest.raises(TypeError): + group([no_context]) + with pytest.raises(TypeError): + group([invalid_type]) + + def test_chain(self) -> None: + """Tests task chaining.""" + graph = group([add, add]) + graph = chain(graph, swap) + assert len(graph._impl.tasks) == 3 + assert len(graph._impl.dependencies) == 2 + assert len(graph._impl.input_tasks) == 2 + assert len(graph._impl.output_tasks) == 1 + + def test_chain_fail(self) -> None: + """Tests task chaining failure.""" + with pytest.raises(TypeError): + chain([add, add]) From 2862f01be75dd76ec0cd18dc818ffe2dbb2aaca9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 00:06:14 -0400 Subject: [PATCH 104/198] Fix tuple check --- python/src/spider/client/task.py | 18 ++++++++++++++---- python/tests/client/test_task_graph.py | 7 ++++++- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index 7ec8b9c3b..e83a8d80c 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -1,8 +1,8 @@ """Spider client task module.""" import inspect -from types import FunctionType -from typing import Protocol, runtime_checkable +from types import FunctionType, GenericAlias +from typing import Protocol, runtime_checkable, get_origin, get_args from spider import core from spider.client.data import Data @@ -25,6 +25,16 @@ def __call__(self, context: TaskContext, *args: object) -> object: ... +def is_tuple(t: type | GenericAlias) -> bool: + """ + :param t: + :return: Whether t is a tuple. + """ + if not isinstance(t, GenericAlias): + return False + return get_origin(t) is tuple + + def create_task(func: TaskFunction) -> core.Task: """ Creates a core Task object from the task function. @@ -52,8 +62,8 @@ def create_task(func: TaskFunction) -> core.Task: if returns == inspect.Parameter.empty: msg = "Return type must has type annotation" raise TypeError(msg) - if type(returns) is tuple: - for r in returns: + if is_tuple(returns): + for r in get_args(returns): tdl_type_str = to_tdl_type_str(r) if r is Data: task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) diff --git a/python/tests/client/test_task_graph.py b/python/tests/client/test_task_graph.py index 6ae69e542..7106418fb 100644 --- a/python/tests/client/test_task_graph.py +++ b/python/tests/client/test_task_graph.py @@ -1,25 +1,30 @@ """Unit tests for Spider client TaskGraph""" + import pytest from spider import TaskContext, Int8, group, chain + def no_context(x: Int8, y: Int8) -> Int8: """Invalid task function with no context.""" return x + y + def invalid_type(_: TaskContext, x: int) -> int: return x + x + def add(_: TaskContext, x: Int8, y: Int8) -> Int8: """Adds two numbers.""" return x + y + def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: """Swaps two numbers.""" return y, x -class TestTaskGraph: +class TestTaskGraph: def test_group(self) -> None: """Tests task grouping.""" graph = group([add, add]) From 03af08d5a32fbb42340d49d822716c21680c277c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 00:13:03 -0400 Subject: [PATCH 105/198] Fix size check in chain --- python/src/spider/core/taskgraph.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index 58e9406d9..f0a25baaa 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -118,6 +118,8 @@ def chain_graph(self, child: "TaskGraph") -> "TaskGraph": for task_id in child.input_tasks: input_task = graph.tasks[task_id] for i in range(len(input_task.task_inputs)): + if task_index >= len(parent_output_tasks): + raise TypeError(size_mismatch_msg) output_task_id = parent_output_tasks[task_index] input_type = input_task.task_inputs[i].type output_type = graph.tasks[output_task_id].task_outputs[output_position].type @@ -126,11 +128,9 @@ def chain_graph(self, child: "TaskGraph") -> "TaskGraph": raise TypeError(msg) input_task.task_inputs[i].value = TaskInputOutput(output_task_id, output_position) output_position += 1 - if len(graph.tasks[output_task_id].task_outputs) > output_position: + if output_position >= len(graph.tasks[output_task_id].task_outputs): output_position = 0 task_index += 1 - if task_index >= len(graph.tasks[output_task_id].task_outputs): - raise TypeError(size_mismatch_msg) if task_index != len(parent_output_tasks) or output_position != 0: raise TypeError(size_mismatch_msg) From c3dda181f5a3c5d5f1ac2c0a960485d9d8fc9af1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 00:15:07 -0400 Subject: [PATCH 106/198] Fix chain dependencies --- python/src/spider/core/taskgraph.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index f0a25baaa..f83a68f78 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -121,6 +121,10 @@ def chain_graph(self, child: "TaskGraph") -> "TaskGraph": if task_index >= len(parent_output_tasks): raise TypeError(size_mismatch_msg) output_task_id = parent_output_tasks[task_index] + + if [output_task_id, task_id] not in graph.dependencies: + graph.dependencies.append((output_task_id, task_id)) + input_type = input_task.task_inputs[i].type output_type = graph.tasks[output_task_id].task_outputs[output_position].type if input_type != output_type: From 2a1f33e33354cab16d06ce9960476192a1b98625 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 00:19:47 -0400 Subject: [PATCH 107/198] Fix id collision --- python/src/spider/core/taskgraph.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index f83a68f78..e8f0f80ad 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -94,7 +94,9 @@ def merge_graph(self, graph: "TaskGraph") -> None: :param graph: The task graph to merge. :return: """ - self.tasks.update(graph.tasks) + new_graph = deepcopy(graph) + new_graph.reset_ids() + self.tasks.update(new_graph) self.dependencies.extend(graph.dependencies) self.input_tasks.extend(graph.input_tasks) self.output_tasks.extend(graph.output_tasks) @@ -107,6 +109,7 @@ def chain_graph(self, child: "TaskGraph") -> "TaskGraph": :raise TypeError: If the outputs and the inputs of `graph` do not match. """ graph = deepcopy(self) + graph.reset_ids() parent_output_tasks = graph.output_tasks graph.tasks.update(child.tasks) graph.dependencies.extend(child.dependencies) From 0e4452c5430b004a1cdbe7b4f83a0139a62af958 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 00:25:05 -0400 Subject: [PATCH 108/198] Add more tests and more fixes --- python/src/spider/core/taskgraph.py | 10 +++++----- python/tests/client/test_task_graph.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index e8f0f80ad..08e4f4f02 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -96,10 +96,10 @@ def merge_graph(self, graph: "TaskGraph") -> None: """ new_graph = deepcopy(graph) new_graph.reset_ids() - self.tasks.update(new_graph) - self.dependencies.extend(graph.dependencies) - self.input_tasks.extend(graph.input_tasks) - self.output_tasks.extend(graph.output_tasks) + self.tasks.update(new_graph.tasks) + self.dependencies.extend(new_graph.dependencies) + self.input_tasks.extend(new_graph.input_tasks) + self.output_tasks.extend(new_graph.output_tasks) def chain_graph(self, child: "TaskGraph") -> "TaskGraph": """ @@ -125,7 +125,7 @@ def chain_graph(self, child: "TaskGraph") -> "TaskGraph": raise TypeError(size_mismatch_msg) output_task_id = parent_output_tasks[task_index] - if [output_task_id, task_id] not in graph.dependencies: + if (output_task_id, task_id) not in graph.dependencies: graph.dependencies.append((output_task_id, task_id)) input_type = input_task.task_inputs[i].type diff --git a/python/tests/client/test_task_graph.py b/python/tests/client/test_task_graph.py index 7106418fb..b504cab73 100644 --- a/python/tests/client/test_task_graph.py +++ b/python/tests/client/test_task_graph.py @@ -19,6 +19,11 @@ def add(_: TaskContext, x: Int8, y: Int8) -> Int8: return x + y +def double(_: TaskContext, x: Int8) -> Int8: + """Double a number.""" + return x * 2 + + def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: """Swaps two numbers.""" return y, x @@ -49,6 +54,16 @@ def test_chain(self) -> None: assert len(graph._impl.dependencies) == 2 assert len(graph._impl.input_tasks) == 2 assert len(graph._impl.output_tasks) == 1 + graph = chain(graph, add) + assert len(graph._impl.tasks) == 4 + assert len(graph._impl.dependencies) == 3 + assert len(graph._impl.input_tasks) == 2 + assert len(graph._impl.output_tasks) == 1 + graph = chain(swap, group([double, double])) + assert len(graph._impl.tasks) == 3 + assert len(graph._impl.dependencies) == 2 + assert len(graph._impl.input_tasks) == 1 + assert len(graph._impl.output_tasks) == 2 def test_chain_fail(self) -> None: """Tests task chaining failure.""" From 3705ef3016dcea96590710a8802aa0c63352fce1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 00:28:08 -0400 Subject: [PATCH 109/198] Satisfy ruff --- python/pyproject.toml | 1 + python/src/spider/client/task.py | 2 +- python/tests/client/test_task_graph.py | 5 ++++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index adb9f8d19..2bcf21c2d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -67,6 +67,7 @@ isort.order-by-type = false [tool.ruff.lint.per-file-ignores] "tests/**" = [ "INP001", # Allow implicit namespace package for tests + "PLR2004", # Allow use of magic value "S101", # Allow use of `assert` (security warning) "S603", # Allow use of `subprocess.Popen` (security warning) "T201", # Allow use of `print` (testing) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index e83a8d80c..95d1b2166 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -2,7 +2,7 @@ import inspect from types import FunctionType, GenericAlias -from typing import Protocol, runtime_checkable, get_origin, get_args +from typing import get_args, get_origin, Protocol, runtime_checkable from spider import core from spider.client.data import Data diff --git a/python/tests/client/test_task_graph.py b/python/tests/client/test_task_graph.py index b504cab73..3ceab5e18 100644 --- a/python/tests/client/test_task_graph.py +++ b/python/tests/client/test_task_graph.py @@ -2,7 +2,7 @@ import pytest -from spider import TaskContext, Int8, group, chain +from spider import chain, group, Int8, TaskContext def no_context(x: Int8, y: Int8) -> Int8: @@ -11,6 +11,7 @@ def no_context(x: Int8, y: Int8) -> Int8: def invalid_type(_: TaskContext, x: int) -> int: + """Invalid task function with unsupported type.""" return x + x @@ -30,6 +31,8 @@ def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: class TestTaskGraph: + """Test task graph composition.""" + def test_group(self) -> None: """Tests task grouping.""" graph = group([add, add]) From fb8938a71ba202806570dffde5f7e1a4215ce99b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 00:58:46 -0400 Subject: [PATCH 110/198] Satisfy mypy --- python/src/spider/client/task.py | 14 ++++++-------- python/src/spider/client/taskgraph.py | 10 ++++++---- python/tests/client/test_task_graph.py | 10 +++++----- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index 95d1b2166..f347c9ff8 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -1,8 +1,9 @@ """Spider client task module.""" import inspect +from collections.abc import Callable from types import FunctionType, GenericAlias -from typing import get_args, get_origin, Protocol, runtime_checkable +from typing import get_args, get_origin from spider import core from spider.client.data import Data @@ -16,13 +17,10 @@ class TaskContext: # TODO: Implement task context for use in task executor -@runtime_checkable -class TaskFunction(Protocol): - """Task function must accept a TaskContext as its first argument.""" - - def __call__(self, context: TaskContext, *args: object) -> object: - """Task function must accept TaskContext as its first argument.""" - ... +# Check the TaskFunction signature at runtime. +# Enforcing static check for first argument requires the use of Protocol. However, functions, which +# are Callable, are not considered a Protocol without explicit cast. +TaskFunction = Callable[..., object] def is_tuple(t: type | GenericAlias) -> bool: diff --git a/python/src/spider/client/taskgraph.py b/python/src/spider/client/taskgraph.py index c8ae280a8..863282f7c 100644 --- a/python/src/spider/client/taskgraph.py +++ b/python/src/spider/client/taskgraph.py @@ -1,5 +1,7 @@ """Spider client TaskGraph module.""" +from collections.abc import Sequence + from spider import core from spider.client.task import create_task, TaskFunction @@ -26,7 +28,7 @@ def chain_graph(self, child: "TaskGraph") -> "TaskGraph": return graph -def group(tasks: list[TaskFunction | TaskGraph]) -> TaskGraph: +def group(tasks: Sequence[TaskFunction | TaskGraph]) -> TaskGraph: """ Groups task functions and task graph into a single task graph. :param tasks: List of task functions or task graphs. @@ -34,7 +36,7 @@ def group(tasks: list[TaskFunction | TaskGraph]) -> TaskGraph: """ graph = TaskGraph() for task in tasks: - if isinstance(task, TaskFunction): + if callable(task): graph._impl.add_task(create_task(task)) else: graph._impl.merge_graph(task._impl) @@ -51,11 +53,11 @@ def chain(parent: TaskFunction | TaskGraph, child: TaskFunction | TaskGraph) -> :return: :raises TypeError: If the parent outputs and child inputs do not match. """ - if isinstance(parent, TaskFunction): + if callable(parent): task = create_task(parent) parent = TaskGraph() parent._impl.add_task(task) - if isinstance(child, TaskFunction): + if callable(child): task = create_task(child) child = TaskGraph() child._impl.add_task(task) diff --git a/python/tests/client/test_task_graph.py b/python/tests/client/test_task_graph.py index 3ceab5e18..6477bdee3 100644 --- a/python/tests/client/test_task_graph.py +++ b/python/tests/client/test_task_graph.py @@ -7,22 +7,22 @@ def no_context(x: Int8, y: Int8) -> Int8: """Invalid task function with no context.""" - return x + y + return Int8(x + y) def invalid_type(_: TaskContext, x: int) -> int: """Invalid task function with unsupported type.""" - return x + x + return Int8(x + x) def add(_: TaskContext, x: Int8, y: Int8) -> Int8: """Adds two numbers.""" - return x + y + return Int8(x + y) def double(_: TaskContext, x: Int8) -> Int8: """Double a number.""" - return x * 2 + return Int8(x * 2) def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: @@ -71,4 +71,4 @@ def test_chain(self) -> None: def test_chain_fail(self) -> None: """Tests task chaining failure.""" with pytest.raises(TypeError): - chain([add, add]) + chain(add, add) From 9ed09beb44a5f2cca76a1700d2d5998b5101d603 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 09:51:34 -0400 Subject: [PATCH 111/198] Bug fix --- python/src/spider/client/task.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index f347c9ff8..1d1c5e40d 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -7,7 +7,7 @@ from spider import core from spider.client.data import Data -from spider.core import TaskInput, TaskOutput, TaskOutputValue +from spider.core import TaskInput, TaskOutput, TaskOutputValue, TaskOutputData from spider.type import to_tdl_type_str @@ -64,13 +64,13 @@ def create_task(func: TaskFunction) -> core.Task: for r in get_args(returns): tdl_type_str = to_tdl_type_str(r) if r is Data: - task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) + task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputData())) else: task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) else: tdl_type_str = to_tdl_type_str(returns) if returns is Data: - task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) + task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputData())) else: task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) From ef827a36b53389bbd8e6cc9730cba9203a936a3a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 09:55:46 -0400 Subject: [PATCH 112/198] Remove unnecessary reset_ids --- python/src/spider/client/taskgraph.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/src/spider/client/taskgraph.py b/python/src/spider/client/taskgraph.py index 863282f7c..3546a80f0 100644 --- a/python/src/spider/client/taskgraph.py +++ b/python/src/spider/client/taskgraph.py @@ -40,7 +40,6 @@ def group(tasks: Sequence[TaskFunction | TaskGraph]) -> TaskGraph: graph._impl.add_task(create_task(task)) else: graph._impl.merge_graph(task._impl) - graph._impl.reset_ids() return graph From 220a2b00da1a9be9d0a8f2010c2385ceaeafa37d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 09:58:09 -0400 Subject: [PATCH 113/198] Fix task input id reset --- python/src/spider/core/taskgraph.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index 08e4f4f02..b89b105aa 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -69,8 +69,8 @@ def reset_ids(self) -> None: new_task_id = id_map[task_id] new_tasks[new_task_id] = deepcopy(self.tasks[task_id]) for task_input in new_tasks[new_task_id].task_inputs: - if isinstance(task_input, TaskInputOutput): - task_input.task_id = id_map[task_input.task_id] + if isinstance(task_input.value, TaskInputOutput): + task_input.value.task_id = id_map[task_input.value.task_id] self.tasks = new_tasks new_dependencies = [] From 56f78662e0555109ea53931aac56d5c99ece8f30 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 10:01:56 -0400 Subject: [PATCH 114/198] Fix comment grammar --- python/src/spider/client/task.py | 4 ++-- python/src/spider/client/taskgraph.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index 1d1c5e40d..e96ae0e66 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -52,13 +52,13 @@ def create_task(func: TaskFunction) -> core.Task: raise TypeError(msg) for param in params[1:]: if param.annotation == inspect.Parameter.empty: - msg = "Argument must has type annotation" + msg = "Argument must have type annotation" raise TypeError(msg) tdl_type_str = to_tdl_type_str(param.annotation) task.task_inputs.append(TaskInput(tdl_type_str, None)) returns = signature.return_annotation if returns == inspect.Parameter.empty: - msg = "Return type must has type annotation" + msg = "Return type must have type annotation" raise TypeError(msg) if is_tuple(returns): for r in get_args(returns): diff --git a/python/src/spider/client/taskgraph.py b/python/src/spider/client/taskgraph.py index 3546a80f0..3098f45c0 100644 --- a/python/src/spider/client/taskgraph.py +++ b/python/src/spider/client/taskgraph.py @@ -9,7 +9,7 @@ class TaskGraph: """ Spider client TaskGraph class. - Warps around the core TaskGraph class. + Wraps around the core TaskGraph class. """ def __init__(self) -> None: From 638d577eac747f2ef1d169c8acfc0f4049c71588 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 10:23:26 -0400 Subject: [PATCH 115/198] Fix tuple check --- python/src/spider/client/task.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index e96ae0e66..7ee4a639a 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -28,8 +28,6 @@ def is_tuple(t: type | GenericAlias) -> bool: :param t: :return: Whether t is a tuple. """ - if not isinstance(t, GenericAlias): - return False return get_origin(t) is tuple From 2bcc4a4b6b0fbc43e5d6a45fc9f3638c86ea6b77 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 10:24:46 -0400 Subject: [PATCH 116/198] Add guard for function with no argument --- python/src/spider/client/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index 7ee4a639a..0047ef0a0 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -45,7 +45,7 @@ def create_task(func: TaskFunction) -> core.Task: task.function_name = func.__qualname__ signature = inspect.signature(func) params = list(signature.parameters.values()) - if params[0].annotation is not TaskContext: + if not params or params[0].annotation is not TaskContext: msg = "First argument is not a TaskContext." raise TypeError(msg) for param in params[1:]: From aa1723ec0503e213ae2600f50c650090441417b3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 10:26:29 -0400 Subject: [PATCH 117/198] Disallow varargs in task function --- python/src/spider/client/task.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index 0047ef0a0..d1a04a66b 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -49,6 +49,8 @@ def create_task(func: TaskFunction) -> core.Task: msg = "First argument is not a TaskContext." raise TypeError(msg) for param in params[1:]: + if param.kind in {inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD}: + raise TypeError("Variadic parameters are not supported in task functions.") if param.annotation == inspect.Parameter.empty: msg = "Argument must have type annotation" raise TypeError(msg) From 24690084f99042961ed0fe0a1a856d9088027419 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 10:28:42 -0400 Subject: [PATCH 118/198] Disallow variable tuple return in task function --- python/src/spider/client/task.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index d1a04a66b..a7261c684 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -61,7 +61,10 @@ def create_task(func: TaskFunction) -> core.Task: msg = "Return type must have type annotation" raise TypeError(msg) if is_tuple(returns): - for r in get_args(returns): + args = get_args(returns) + if Ellipsis in args: + raise TypeError("Variable-length tuple return types (tuple[T, ...]) are not supported.") + for r in args: tdl_type_str = to_tdl_type_str(r) if r is Data: task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputData())) From 082476dd513d067d8bdd45ed78f7383c5dcae10f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 11:32:09 -0400 Subject: [PATCH 119/198] Use identity check for Parameter.empty sentinel --- python/src/spider/client/task.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index a7261c684..636f0423d 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -7,7 +7,7 @@ from spider import core from spider.client.data import Data -from spider.core import TaskInput, TaskOutput, TaskOutputValue, TaskOutputData +from spider.core import TaskInput, TaskOutput, TaskOutputData, TaskOutputValue from spider.type import to_tdl_type_str @@ -51,13 +51,13 @@ def create_task(func: TaskFunction) -> core.Task: for param in params[1:]: if param.kind in {inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD}: raise TypeError("Variadic parameters are not supported in task functions.") - if param.annotation == inspect.Parameter.empty: + if param.annotation is inspect.Parameter.empty: msg = "Argument must have type annotation" raise TypeError(msg) tdl_type_str = to_tdl_type_str(param.annotation) task.task_inputs.append(TaskInput(tdl_type_str, None)) returns = signature.return_annotation - if returns == inspect.Parameter.empty: + if returns is inspect.Parameter.empty: msg = "Return type must have type annotation" raise TypeError(msg) if is_tuple(returns): From 72239f7439f486fd0e959c18dbbe9cd45526f33c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 11:39:55 -0400 Subject: [PATCH 120/198] Fix error msg for ruff --- python/src/spider/client/task.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index 636f0423d..0d5fc8031 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -50,7 +50,8 @@ def create_task(func: TaskFunction) -> core.Task: raise TypeError(msg) for param in params[1:]: if param.kind in {inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD}: - raise TypeError("Variadic parameters are not supported in task functions.") + msg = "Variadic parameters are not supported." + raise TypeError(msg) if param.annotation is inspect.Parameter.empty: msg = "Argument must have type annotation" raise TypeError(msg) @@ -63,7 +64,8 @@ def create_task(func: TaskFunction) -> core.Task: if is_tuple(returns): args = get_args(returns) if Ellipsis in args: - raise TypeError("Variable-length tuple return types (tuple[T, ...]) are not supported.") + msg = "Variable-length tuple return types are not supported." + raise TypeError(msg) for r in args: tdl_type_str = to_tdl_type_str(r) if r is Data: From e13acec0d18e7d538e429e1dda9f60d29cb1a23d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 11:41:31 -0400 Subject: [PATCH 121/198] Make create_task private --- python/src/spider/client/task.py | 2 +- python/src/spider/client/taskgraph.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index 0d5fc8031..c32a67d13 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -31,7 +31,7 @@ def is_tuple(t: type | GenericAlias) -> bool: return get_origin(t) is tuple -def create_task(func: TaskFunction) -> core.Task: +def _create_task(func: TaskFunction) -> core.Task: """ Creates a core Task object from the task function. :param func: diff --git a/python/src/spider/client/taskgraph.py b/python/src/spider/client/taskgraph.py index 3098f45c0..bd3c43c24 100644 --- a/python/src/spider/client/taskgraph.py +++ b/python/src/spider/client/taskgraph.py @@ -3,7 +3,7 @@ from collections.abc import Sequence from spider import core -from spider.client.task import create_task, TaskFunction +from spider.client.task import _create_task, TaskFunction class TaskGraph: @@ -37,7 +37,7 @@ def group(tasks: Sequence[TaskFunction | TaskGraph]) -> TaskGraph: graph = TaskGraph() for task in tasks: if callable(task): - graph._impl.add_task(create_task(task)) + graph._impl.add_task(_create_task(task)) else: graph._impl.merge_graph(task._impl) @@ -53,11 +53,11 @@ def chain(parent: TaskFunction | TaskGraph, child: TaskFunction | TaskGraph) -> :raises TypeError: If the parent outputs and child inputs do not match. """ if callable(parent): - task = create_task(parent) + task = _create_task(parent) parent = TaskGraph() parent._impl.add_task(task) if callable(child): - task = create_task(child) + task = _create_task(child) child = TaskGraph() child._impl.add_task(task) return parent.chain_graph(child) From 84a6256ca6fc4c22312ee27e72ca2b9a4cfea5ff Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 11:49:25 -0400 Subject: [PATCH 122/198] Split create_task into multiple functions --- python/src/spider/client/task.py | 41 +++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index c32a67d13..6b27e5d4d 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -31,19 +31,13 @@ def is_tuple(t: type | GenericAlias) -> bool: return get_origin(t) is tuple -def _create_task(func: TaskFunction) -> core.Task: +def _process_parameters(task: core.Task, signature: inspect.Signature) -> None: """ - Creates a core Task object from the task function. - :param func: - :return: - :raise TypeError: If the function signature contains unsupported types. + Checks the parameters validity and add them to the task. + :param task: + :param signature: + :raises TypeError: If the parameters are invalid. """ - task = core.Task() - if not isinstance(func, FunctionType): - msg = "`func` is not a function." - raise TypeError(msg) - task.function_name = func.__qualname__ - signature = inspect.signature(func) params = list(signature.parameters.values()) if not params or params[0].annotation is not TaskContext: msg = "First argument is not a TaskContext." @@ -57,6 +51,15 @@ def _create_task(func: TaskFunction) -> core.Task: raise TypeError(msg) tdl_type_str = to_tdl_type_str(param.annotation) task.task_inputs.append(TaskInput(tdl_type_str, None)) + + +def _process_return(task: core.Task, signature: inspect.Signature) -> None: + """ + Checks the return type validity and add them to the task. + :param task: + :param signature: + :raises TypeError: If the return type is invalid. + """ returns = signature.return_annotation if returns is inspect.Parameter.empty: msg = "Return type must have type annotation" @@ -79,4 +82,20 @@ def _create_task(func: TaskFunction) -> core.Task: else: task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) + +def _create_task(func: TaskFunction) -> core.Task: + """ + Creates a core Task object from the task function. + :param func: + :return: + :raise TypeError: If the function signature contains unsupported types. + """ + task = core.Task() + if not isinstance(func, FunctionType): + msg = "`func` is not a function." + raise TypeError(msg) + task.function_name = func.__qualname__ + signature = inspect.signature(func) + _process_parameters(task, signature) + _process_return(task, signature) return task From c45eb0dbbf499ea6bafd31a5d124e30b3f88724c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 21:02:11 -0400 Subject: [PATCH 123/198] Add storage cursor --- python/src/spider/storage/__init__.py | 1 + .../src/spider/storage/storage_connection.py | 97 +++++++++++++++++++ 2 files changed, 98 insertions(+) create mode 100644 python/src/spider/storage/__init__.py create mode 100644 python/src/spider/storage/storage_connection.py diff --git a/python/src/spider/storage/__init__.py b/python/src/spider/storage/__init__.py new file mode 100644 index 000000000..f5e70d95e --- /dev/null +++ b/python/src/spider/storage/__init__.py @@ -0,0 +1 @@ +"""Spider Storage package.""" diff --git a/python/src/spider/storage/storage_connection.py b/python/src/spider/storage/storage_connection.py new file mode 100644 index 000000000..0ba6611ce --- /dev/null +++ b/python/src/spider/storage/storage_connection.py @@ -0,0 +1,97 @@ +"""Storage Connection Interface module.""" + +from abc import ABC, abstractmethod +from collections.abc import Sequence +from types import TracebackType + + +class StorageError(Exception): + """Storage Exception.""" + + def __init__(self, msg: str) -> None: + """Creates a storage exception.""" + super().__init__(msg) + + +class StorageCursor(ABC): + """Abstract base class for storage cursor.""" + + @abstractmethod + def execute(self, query: str, params: tuple[object]) -> None: + """ + Executes a query on the storage cursor. + To get the result, call fetch* functions. + :param query: The query to execute. + :param params: The query parameters. + :raise StorageError: If query execution fails. + """ + + @abstractmethod + def executemany(self, query: str, params: Sequence[tuple[object]]) -> None: + """ + Executes a batch query on the storage cursor. + :param query: The query to execute. + :param params: Batch of query parameters. + :raise StorageError: If query execution fails. + """ + + @abstractmethod + def fetchone(self) -> tuple[object]: + """ + :return: The next result from the previous query. + :raise StorageError: If query execution fails. + """ + + @abstractmethod + def fetchmany(self, size: int) -> Sequence[tuple[object]]: + """ + :param size: The number of results to fetch. + :return: The results from the previous query. At most `size` results are returned. + :raise StorageError: If query execution fails. + """ + + @abstractmethod + def fetchall(self) -> Sequence[tuple[object]]: + """ + :return: All results from the previous query. + :raise StorageError: If query execution fails. + """ + + @abstractmethod + def open(self) -> None: + """Opens the storage cursor.""" + + @abstractmethod + def close( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool: + """ + Closes the storage cursor. + :param exc_type: The exception type. + :param exc_value: The exception object. + :param traceback: The exception traceback. + :return: Whether the exception is suppressed. + """ + + def __enter__(self) -> "StorageCursor": + """Opens the storage cursor.""" + self.open() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> bool: + """ + Closes the storage cursor. + :param exc_type: The exception type. + :param exc_value: The exception object. + :param traceback: The exception traceback. + :return: Whether the exception is suppressed. + """ + return self.close(exc_type, exc_value, traceback) From 1a27b458df905f3f26d2a684e56397254400878b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 21:11:54 -0400 Subject: [PATCH 124/198] Add storage connection interface --- python/src/spider/storage/storage_connection.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/python/src/spider/storage/storage_connection.py b/python/src/spider/storage/storage_connection.py index 0ba6611ce..8e7f54ff5 100644 --- a/python/src/spider/storage/storage_connection.py +++ b/python/src/spider/storage/storage_connection.py @@ -95,3 +95,15 @@ def __exit__( :return: Whether the exception is suppressed. """ return self.close(exc_type, exc_value, traceback) + + +class StorageConnection(ABC): + """Abstract base class for storage connection.""" + + @abstractmethod + def open_cursor(self) -> StorageCursor: + """ + Opens the storage cursor. + :return: + :raise StorageError: If open cursor fails. + """ From b505e064dc4057ff8d49ce639ca85319e7901ea5 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 21:43:28 -0400 Subject: [PATCH 125/198] Use functionality as storage interface --- python/src/spider/storage/storage.py | 26 +++++ .../src/spider/storage/storage_connection.py | 109 ------------------ 2 files changed, 26 insertions(+), 109 deletions(-) create mode 100644 python/src/spider/storage/storage.py delete mode 100644 python/src/spider/storage/storage_connection.py diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py new file mode 100644 index 000000000..cd09b792e --- /dev/null +++ b/python/src/spider/storage/storage.py @@ -0,0 +1,26 @@ +"""Storage backend interface module.""" + +from abc import ABC, abstractmethod +from collections.abc import Sequence + +from spider.client.taskgraph import TaskGraph + + +class StorageError(Exception): + """Storage error.""" + + def __init__(self, message: str) -> None: + """Initializes storage error.""" + super().__init__(message) + + +class Storage(ABC): + """Storage backend interface.""" + + @abstractmethod + def submit_job(self, task_graphs: TaskGraph | Sequence[TaskGraph]) -> None: + """ + Submit jobs to the storage. + :param task_graphs: Task graphs to submit. + :raises StorageError: If the storage operations fail. + """ diff --git a/python/src/spider/storage/storage_connection.py b/python/src/spider/storage/storage_connection.py deleted file mode 100644 index 8e7f54ff5..000000000 --- a/python/src/spider/storage/storage_connection.py +++ /dev/null @@ -1,109 +0,0 @@ -"""Storage Connection Interface module.""" - -from abc import ABC, abstractmethod -from collections.abc import Sequence -from types import TracebackType - - -class StorageError(Exception): - """Storage Exception.""" - - def __init__(self, msg: str) -> None: - """Creates a storage exception.""" - super().__init__(msg) - - -class StorageCursor(ABC): - """Abstract base class for storage cursor.""" - - @abstractmethod - def execute(self, query: str, params: tuple[object]) -> None: - """ - Executes a query on the storage cursor. - To get the result, call fetch* functions. - :param query: The query to execute. - :param params: The query parameters. - :raise StorageError: If query execution fails. - """ - - @abstractmethod - def executemany(self, query: str, params: Sequence[tuple[object]]) -> None: - """ - Executes a batch query on the storage cursor. - :param query: The query to execute. - :param params: Batch of query parameters. - :raise StorageError: If query execution fails. - """ - - @abstractmethod - def fetchone(self) -> tuple[object]: - """ - :return: The next result from the previous query. - :raise StorageError: If query execution fails. - """ - - @abstractmethod - def fetchmany(self, size: int) -> Sequence[tuple[object]]: - """ - :param size: The number of results to fetch. - :return: The results from the previous query. At most `size` results are returned. - :raise StorageError: If query execution fails. - """ - - @abstractmethod - def fetchall(self) -> Sequence[tuple[object]]: - """ - :return: All results from the previous query. - :raise StorageError: If query execution fails. - """ - - @abstractmethod - def open(self) -> None: - """Opens the storage cursor.""" - - @abstractmethod - def close( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> bool: - """ - Closes the storage cursor. - :param exc_type: The exception type. - :param exc_value: The exception object. - :param traceback: The exception traceback. - :return: Whether the exception is suppressed. - """ - - def __enter__(self) -> "StorageCursor": - """Opens the storage cursor.""" - self.open() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> bool: - """ - Closes the storage cursor. - :param exc_type: The exception type. - :param exc_value: The exception object. - :param traceback: The exception traceback. - :return: Whether the exception is suppressed. - """ - return self.close(exc_type, exc_value, traceback) - - -class StorageConnection(ABC): - """Abstract base class for storage connection.""" - - @abstractmethod - def open_cursor(self) -> StorageCursor: - """ - Opens the storage cursor. - :return: - :raise StorageError: If open cursor fails. - """ From 44067abf79564a2a223273f4c4be7cf1cf6c9549 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 22:52:39 -0400 Subject: [PATCH 126/198] Add jdbc url --- python/src/spider/storage/jdbc_url.py | 64 +++++++++++++++++++++++++++ python/tests/storage/test_jdbc_url.py | 37 ++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 python/src/spider/storage/jdbc_url.py create mode 100644 python/tests/storage/test_jdbc_url.py diff --git a/python/src/spider/storage/jdbc_url.py b/python/src/spider/storage/jdbc_url.py new file mode 100644 index 000000000..0ca8c9754 --- /dev/null +++ b/python/src/spider/storage/jdbc_url.py @@ -0,0 +1,64 @@ +"""JDBC URL module.""" + +import re +from dataclasses import dataclass + + +@dataclass +class JdbcParameters: + """JDBC url parameters.""" + + protocol: str + host: str + database: str + port: int | None = None + user: str | None = None + password: str | None = None + + +pattern = re.compile( + r"^(?P[a-zA-Z][a-zA-Z0-9+.-]*(::[a-zA-Z0-9+.-]*)?)://" + r"(?P[^:/?]+)" + r"(?::(?P\d+))?" + r"/(?P[^?]+)" + r"(?:\?(?P[^#]+))?" +) + + +def parse_jdbc_url(url: str) -> JdbcParameters: + """ + Parses a JDBC URL. + :param url: + :return: + :raises ValueError: If the JDBC URL is invalid. + """ + match = pattern.match(url) + if not match: + msg = f"Invalid JDBC URL: {url}" + raise ValueError(msg) + + groups = match.groupdict() + query = groups.get("query") or "" + query_params = dict(param.split("=", 1) for param in query.split("&") if "=" in param) + + protocol = groups.get("protocol") + if not protocol: + msg = f"Protocol is required in JDBC URL: {url}" + raise ValueError(msg) + host = groups.get("host") + if not host: + msg = f"Host is required in JDBC URL: {url}" + raise ValueError(msg) + database = groups.get("database") + if not database: + msg = f"Database is required in JDBC URL: {url}" + raise ValueError(msg) + + return JdbcParameters( + protocol=protocol, + host=host, + port=int(groups["port"]) if groups.get("port") else None, + database=database, + user=query_params.get("user"), + password=query_params.get("password"), + ) diff --git a/python/tests/storage/test_jdbc_url.py b/python/tests/storage/test_jdbc_url.py new file mode 100644 index 000000000..65cb7bf00 --- /dev/null +++ b/python/tests/storage/test_jdbc_url.py @@ -0,0 +1,37 @@ +"""Tests for the jdbc url module.""" + +import pytest + +from spider.storage.jdbc_url import parse_jdbc_url + + +class TestJdbcUrl: + """Tests for the jdbc url module.""" + + def test_full_jdbc_url(self) -> None: + """Tests parsing a full JDBC URL with all fields.""" + url = "jdbc::mariadb://localhost:3306/dbname?user=root&password=secret" + params = parse_jdbc_url(url) + assert params.protocol == "jdbc::mariadb" + assert params.host == "localhost" + assert params.port == 3306 + assert params.database == "dbname" + assert params.user == "root" + assert params.password == "secret" # noqa: S105 + + def test_jdbc_url_simple(self) -> None: + """Tests parsing a simple JDBC URL without port, user and password.""" + url = "jdbc::postgresql://localhost/dbname" + params = parse_jdbc_url(url) + assert params.protocol == "jdbc::postgresql" + assert params.host == "localhost" + assert params.port is None + assert params.database == "dbname" + assert params.user is None + assert params.password is None + + def test_invalid_jdbc_url(self) -> None: + """Tests parsing an invalid JDBC URL.""" + url = "invalid_jdbc_url" + with pytest.raises(ValueError, match=f"Invalid JDBC URL: {url}"): + parse_jdbc_url(url) From c81719b961394f3d6200a9d357d7266c4ad8dc79 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 11 Aug 2025 23:58:48 -0400 Subject: [PATCH 127/198] Add mariadb connection --- python/pyproject.toml | 4 +++ python/src/spider/storage/mariadb_storage.py | 29 ++++++++++++++++++++ python/src/spider/storage/storage.py | 4 +-- 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 python/src/spider/storage/mariadb_storage.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 2bcf21c2d..bd2523a0f 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -30,6 +30,10 @@ pretty = true show_error_code_links = true show_error_context = true +[[tool.mypy.overrides]] +module = ["mariadb.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py new file mode 100644 index 000000000..aaac1d407 --- /dev/null +++ b/python/src/spider/storage/mariadb_storage.py @@ -0,0 +1,29 @@ +"""MariaDB Storage module.""" + +from collections.abc import Sequence + +import mariadb +from typing_extensions import override + +from spider import core +from spider.storage.jdbc_url import JdbcParameters +from spider.storage.storage import Storage, StorageError + + +class MariaDBStorage(Storage): + """MairaDB Storage class.""" + + def __init__(self, params: JdbcParameters) -> None: + """ + Connects to the MariaDB database. + :param params: The JDBC parameters for connecting to the database. + :raises StorageError: If the connection to the database fails. + """ + try: + self._conn = mariadb.connect(**params.__dict__) + except mariadb.Error as e: + raise StorageError(str(e)) from e + + @override + def submit_jobs(self, task_graphs: Sequence[core.TaskGraph]) -> None: + pass diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index cd09b792e..1cacacdaa 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from collections.abc import Sequence -from spider.client.taskgraph import TaskGraph +from spider import core class StorageError(Exception): @@ -18,7 +18,7 @@ class Storage(ABC): """Storage backend interface.""" @abstractmethod - def submit_job(self, task_graphs: TaskGraph | Sequence[TaskGraph]) -> None: + def submit_jobs(self, task_graphs: Sequence[core.TaskGraph]) -> None: """ Submit jobs to the storage. :param task_graphs: Task graphs to submit. From 899189e49dce4ffe1f44a391a4b810d8d2ed6849 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 08:59:05 -0400 Subject: [PATCH 128/198] Add submit job sql --- python/src/spider/storage/mariadb_storage.py | 54 ++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index aaac1d407..f455cc298 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -9,6 +9,60 @@ from spider.storage.jdbc_url import JdbcParameters from spider.storage.storage import Storage, StorageError +InsertJob = """ +INSERT INTO + `jobs` (`id`, `client_id`) +VALUES + (?, ?)""" + +InsertTask = """ +INSERT INTO + `tasks` (`id`, `job_id`, `func_name`, `state`, `timeout`, `max_retry`) +VALUES + (?, ?, ?, ?, ?, ?)""" + +InsertTaskInputOutput = """ +INSERT INTO + `task_inputs` (`task_id`, `position`, `type`, `output_task_id`, `output_task_position`) +VALUES + (?, ?, ?, ?, ?)""" + +InsertTaskInputData = """ +INSERT INTO + `task_inputs` (`task_id`, `position`, `type`, `data_id`) +VALUES + (?, ?, ?, ?)""" + +InsertTaskInputValue = """ +INSERT INTO + `task_inputs` (`task_id`, `position`, `type`, `value`) +VALUES + (?, ?, ?, ?)""" + +InsertTaskOutput = """ +INSERT INTO + `task_outputs` (`task_id`, `position`, `type`) +VALUES + (?, ?, ?)""" + +InsertTaskDependency = """ +INSERT INTO + `task_dependencies` (parent, child) +VALUES + (?, ?)""" + +InsertInputTask = """ +INSERT INTO + `input_tasks` (`job_id`, `task_id`, `position`) +VALUES + (?, ?, ?)""" + +InsertOutputTask = """ +INSERT INTO + `output_tasks` (`job_id`, `task_id`, `position`) +VALUES + (?, ?, ?)""" + class MariaDBStorage(Storage): """MairaDB Storage class.""" From ba1fd69b5c989892daad3f55f74b9fb311b63e54 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 09:04:04 -0400 Subject: [PATCH 129/198] Add driver id --- python/src/spider/core/driver.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 python/src/spider/core/driver.py diff --git a/python/src/spider/core/driver.py b/python/src/spider/core/driver.py new file mode 100644 index 000000000..f67abd865 --- /dev/null +++ b/python/src/spider/core/driver.py @@ -0,0 +1,5 @@ +"""Spider core Driver module.""" + +from uuid import UUID + +DriverID = UUID From fe4b24b777776ee4db5f85752045f0d33005cc7d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 09:05:34 -0400 Subject: [PATCH 130/198] Add export of driver id --- python/src/spider/core/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py index d463958c8..a811dc8ed 100644 --- a/python/src/spider/core/__init__.py +++ b/python/src/spider/core/__init__.py @@ -1,6 +1,7 @@ """Spider core package.""" from .data import Data, DataId +from .driver import DriverID from .task import ( Task, TaskId, @@ -18,6 +19,7 @@ __all__ = [ "Data", "DataId", + "DriverID", "Task", "TaskGraph", "TaskId", From 8c0125a90cd1dc040cab86c6e814c2f6af3f53cc Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 11:58:44 -0400 Subject: [PATCH 131/198] Add submit job db function --- python/src/spider/storage/mariadb_storage.py | 98 +++++++++++++++++++- python/src/spider/storage/storage.py | 3 +- 2 files changed, 98 insertions(+), 3 deletions(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index f455cc298..b41618847 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -1,8 +1,10 @@ """MariaDB Storage module.""" from collections.abc import Sequence +from uuid import uuid4 import mariadb +import msgpack from typing_extensions import override from spider import core @@ -79,5 +81,97 @@ def __init__(self, params: JdbcParameters) -> None: raise StorageError(str(e)) from e @override - def submit_jobs(self, task_graphs: Sequence[core.TaskGraph]) -> None: - pass + def submit_jobs(self, driver_id: core.DriverID, task_graphs: Sequence[core.TaskGraph]) -> None: + try: + job_ids = [uuid4() for _ in task_graphs] + with self._conn.cursor() as cursor: + cursor.executemany(InsertJob, [(job_id, driver_id) for job_id in job_ids]) + cursor.executemany( + InsertTask, + [ + ( + task.task_id, + job_id, + task.function_name, + task.state.value, + task.timeout, + task.max_retries, + ) + for job_id, task_graph in zip(job_ids, task_graphs, strict=True) + for task in task_graph.tasks.values() + ], + ) + cursor.executemany( + InsertTaskDependency, + [ + (parent, child) + for task_graph in task_graphs + for parent, child in task_graph.dependencies + ], + ) + cursor.executemany( + InsertInputTask, + [ + (job_id, task_id, position) + for job_id, task_graph in zip(job_ids, task_graphs, strict=True) + for position, task_id in enumerate(task_graph.input_tasks) + ], + ) + cursor.executemany( + InsertOutputTask, + [ + (job_id, task_id, position) + for job_id, task_graph in zip(job_ids, task_graphs, strict=True) + for position, task_id in enumerate(task_graph.output_tasks) + ], + ) + cursor.executemany( + InsertTaskOutput, + [ + (task.task_id, position, task_output.type) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_output in enumerate(task.task_outputs) + ], + ) + cursor.executemany( + InsertTaskInputData, + [ + (task.task_id, position, task_input.type, msgpack.packb(task_input.value)) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_input in enumerate(task.task_inputs) + if isinstance(task_input.value, core.TaskInputData) + ], + ) + cursor.executemany( + InsertTaskInputValue, + [ + (task.task_id, position, task_input.type, msgpack.packb(task_input.value)) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_input in enumerate(task.task_inputs) + if isinstance(task_input.value, core.TaskInputValue) + ], + ) + cursor.executemany( + InsertTaskInputOutput, + [ + ( + task.task_id, + position, + task_input.type, + task_input.value.task_id, + task_input.value.position, + ) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_input in enumerate(task.task_inputs) + if isinstance(task_input.value, core.TaskInputOutput) + ], + ) + cursor.executemany() + self._conn.commit() + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index 1cacacdaa..8b56ee05d 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -18,9 +18,10 @@ class Storage(ABC): """Storage backend interface.""" @abstractmethod - def submit_jobs(self, task_graphs: Sequence[core.TaskGraph]) -> None: + def submit_jobs(self, driver_id: core.DriverID, task_graphs: Sequence[core.TaskGraph]) -> None: """ Submit jobs to the storage. + :param driver_id: Driver ID. :param task_graphs: Task graphs to submit. :raises StorageError: If the storage operations fail. """ From beb4b5fc0699afa9fff24267d2be6e774e57d39e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 13:06:45 -0400 Subject: [PATCH 132/198] Add tests for mariadb --- python/pyproject.toml | 4 ++ python/src/spider/core/__init__.py | 7 +-- python/src/spider/core/driver.py | 2 +- python/src/spider/core/task.py | 2 +- python/src/spider/core/taskgraph.py | 4 +- python/src/spider/storage/__init__.py | 12 +++++ python/src/spider/storage/mariadb_storage.py | 10 ++-- python/src/spider/storage/storage.py | 7 ++- python/tests/storage/test_jdbc_url.py | 2 +- python/tests/storage/test_mariadb.py | 52 ++++++++++++++++++++ 10 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 python/tests/storage/test_mariadb.py diff --git a/python/pyproject.toml b/python/pyproject.toml index bd2523a0f..1435df991 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -36,6 +36,10 @@ ignore_missing_imports = true [tool.pytest.ini_options] testpaths = ["tests"] +markers = [ + "integration: mark test as an integration test", + "storage: mark unit tests needing storage", +] [tool.ruff] line-length = 100 diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py index a811dc8ed..36843b816 100644 --- a/python/src/spider/core/__init__.py +++ b/python/src/spider/core/__init__.py @@ -1,7 +1,7 @@ """Spider core package.""" from .data import Data, DataId -from .driver import DriverID +from .driver import DriverId from .task import ( Task, TaskId, @@ -14,12 +14,13 @@ TaskOutputValue, TaskState, ) -from .taskgraph import TaskGraph +from .taskgraph import JobId, TaskGraph __all__ = [ "Data", "DataId", - "DriverID", + "DriverId", + "JobId", "Task", "TaskGraph", "TaskId", diff --git a/python/src/spider/core/driver.py b/python/src/spider/core/driver.py index f67abd865..876c9b00b 100644 --- a/python/src/spider/core/driver.py +++ b/python/src/spider/core/driver.py @@ -2,4 +2,4 @@ from uuid import UUID -DriverID = UUID +DriverId = UUID diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index bf0edf0f6..f0b2e6d83 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -26,7 +26,7 @@ class TaskInput: """Represents a task input""" type: str - value: TaskInputData | TaskInputOutput | None + value: TaskInputData | TaskInputOutput | TaskInputValue | None TaskOutputValue = bytes diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index b89b105aa..d6fe29b19 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -1,10 +1,12 @@ """TaskGraph module for Spider.""" from copy import deepcopy -from uuid import uuid4 +from uuid import UUID, uuid4 from spider.core.task import Task, TaskId, TaskInputOutput +JobId = UUID + class TaskGraph: """Represents a task graph in Spider.""" diff --git a/python/src/spider/storage/__init__.py b/python/src/spider/storage/__init__.py index f5e70d95e..b96d8dd12 100644 --- a/python/src/spider/storage/__init__.py +++ b/python/src/spider/storage/__init__.py @@ -1 +1,13 @@ """Spider Storage package.""" + +from .jdbc_url import JdbcParameters, parse_jdbc_url +from .mariadb_storage import MariaDBStorage +from .storage import Storage, StorageError + +__all__ = [ + "JdbcParameters", + "MariaDBStorage", + "Storage", + "StorageError", + "parse_jdbc_url", +] diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index b41618847..805965371 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -4,7 +4,6 @@ from uuid import uuid4 import mariadb -import msgpack from typing_extensions import override from spider import core @@ -81,7 +80,9 @@ def __init__(self, params: JdbcParameters) -> None: raise StorageError(str(e)) from e @override - def submit_jobs(self, driver_id: core.DriverID, task_graphs: Sequence[core.TaskGraph]) -> None: + def submit_jobs( + self, driver_id: core.DriverId, task_graphs: Sequence[core.TaskGraph] + ) -> Sequence[core.JobId]: try: job_ids = [uuid4() for _ in task_graphs] with self._conn.cursor() as cursor: @@ -137,7 +138,7 @@ def submit_jobs(self, driver_id: core.DriverID, task_graphs: Sequence[core.TaskG cursor.executemany( InsertTaskInputData, [ - (task.task_id, position, task_input.type, msgpack.packb(task_input.value)) + (task.task_id, position, task_input.type, task_input.value) for task_graph in task_graphs for task in task_graph.tasks.values() for position, task_input in enumerate(task.task_inputs) @@ -147,7 +148,7 @@ def submit_jobs(self, driver_id: core.DriverID, task_graphs: Sequence[core.TaskG cursor.executemany( InsertTaskInputValue, [ - (task.task_id, position, task_input.type, msgpack.packb(task_input.value)) + (task.task_id, position, task_input.type, task_input.value) for task_graph in task_graphs for task in task_graph.tasks.values() for position, task_input in enumerate(task.task_inputs) @@ -172,6 +173,7 @@ def submit_jobs(self, driver_id: core.DriverID, task_graphs: Sequence[core.TaskG ) cursor.executemany() self._conn.commit() + return job_ids except mariadb.Error as e: self._conn.rollback() raise StorageError(str(e)) from e diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index 8b56ee05d..ac7e5f7e7 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -18,10 +18,13 @@ class Storage(ABC): """Storage backend interface.""" @abstractmethod - def submit_jobs(self, driver_id: core.DriverID, task_graphs: Sequence[core.TaskGraph]) -> None: + def submit_jobs( + self, driver_id: core.DriverId, task_graphs: Sequence[core.TaskGraph] + ) -> Sequence[core.JobId]: """ Submit jobs to the storage. - :param driver_id: Driver ID. + :param driver_id: Driver id. :param task_graphs: Task graphs to submit. + :return: List of job ids representing the submitted jobs. :raises StorageError: If the storage operations fail. """ diff --git a/python/tests/storage/test_jdbc_url.py b/python/tests/storage/test_jdbc_url.py index 65cb7bf00..fcc6f9ced 100644 --- a/python/tests/storage/test_jdbc_url.py +++ b/python/tests/storage/test_jdbc_url.py @@ -2,7 +2,7 @@ import pytest -from spider.storage.jdbc_url import parse_jdbc_url +from spider.storage import parse_jdbc_url class TestJdbcUrl: diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py new file mode 100644 index 000000000..14b4ecc15 --- /dev/null +++ b/python/tests/storage/test_mariadb.py @@ -0,0 +1,52 @@ +"""Tests for the MariaDB storage backend.""" + +from uuid import uuid4 + +import msgpack +import pytest + +from spider import chain, group, Int8, TaskContext +from spider.core import TaskInputValue +from spider.storage import MariaDBStorage, parse_jdbc_url + +MariaDBTestUrl = "jdbc:mariadb://localhost:3306/spider_test?user=root&password=password" + + +@pytest.fixture(scope="session") +def mariadb_storage() -> MariaDBStorage: + """Fixture to create a MariaDB storage instance.""" + params = parse_jdbc_url(MariaDBTestUrl) + return MariaDBStorage(params) + + +def add(_: TaskContext, x: Int8, y: Int8) -> Int8: + """Adds two numbers.""" + return Int8(x + y) + + +def double(_: TaskContext, x: Int8) -> Int8: + """Double a number.""" + return Int8(x * 2) + + +def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: + """Swaps two numbers.""" + return y, x + + +class TestMariaDBStorage: + """Test class for the MariaDB storage backend.""" + + @pytest.mark.storage + def test_job_submission(self, mariadb_storage: MariaDBStorage) -> None: + """Test job submission to the MariaDB storage backend.""" + graph = chain(group([double, double, double, double]), group([swap, swap]))._impl + # Fill input data + for task_id in graph.input_tasks: + task = graph.tasks[task_id] + task.task_inputs[0].value = TaskInputValue(msgpack.packb(1)) + task.task_inputs[1].value = TaskInputValue(msgpack.packb(2)) + + driver_id = uuid4() + job_ids = mariadb_storage.submit_jobs(driver_id, [graph]) + assert len(job_ids) == 1 From ff8bb01d5890e4f6f60ff4ba33fe5a049e4d98d2 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 13:33:26 -0400 Subject: [PATCH 133/198] Fix jdbc url --- python/src/spider/storage/jdbc_url.py | 8 ++++---- python/tests/storage/test_jdbc_url.py | 10 +++++----- python/tests/storage/test_mariadb.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/python/src/spider/storage/jdbc_url.py b/python/src/spider/storage/jdbc_url.py index 0ca8c9754..c180077e4 100644 --- a/python/src/spider/storage/jdbc_url.py +++ b/python/src/spider/storage/jdbc_url.py @@ -17,11 +17,11 @@ class JdbcParameters: pattern = re.compile( - r"^(?P[a-zA-Z][a-zA-Z0-9+.-]*(::[a-zA-Z0-9+.-]*)?)://" - r"(?P[^:/?]+)" + r"^(?P[a-zA-Z][a-zA-Z0-9+.-]*(:[a-zA-Z0-9+.-]*)?)://" + r"(?P([a-zA-Z0-9.-]+|\d{1,3}(?:\.\d{1,3}){3}))" r"(?::(?P\d+))?" - r"/(?P[^?]+)" - r"(?:\?(?P[^#]+))?" + r"/(?P[a-zA-Z0-9_\-]+)" + r"(?:\?(?P[a-zA-Z0-9_\-=&]+))?" ) diff --git a/python/tests/storage/test_jdbc_url.py b/python/tests/storage/test_jdbc_url.py index fcc6f9ced..680c02185 100644 --- a/python/tests/storage/test_jdbc_url.py +++ b/python/tests/storage/test_jdbc_url.py @@ -10,9 +10,9 @@ class TestJdbcUrl: def test_full_jdbc_url(self) -> None: """Tests parsing a full JDBC URL with all fields.""" - url = "jdbc::mariadb://localhost:3306/dbname?user=root&password=secret" + url = "jdbc:mariadb://localhost:3306/dbname?user=root&password=secret" params = parse_jdbc_url(url) - assert params.protocol == "jdbc::mariadb" + assert params.protocol == "jdbc:mariadb" assert params.host == "localhost" assert params.port == 3306 assert params.database == "dbname" @@ -21,10 +21,10 @@ def test_full_jdbc_url(self) -> None: def test_jdbc_url_simple(self) -> None: """Tests parsing a simple JDBC URL without port, user and password.""" - url = "jdbc::postgresql://localhost/dbname" + url = "jdbc:postgresql://127.0.0.1/dbname" params = parse_jdbc_url(url) - assert params.protocol == "jdbc::postgresql" - assert params.host == "localhost" + assert params.protocol == "jdbc:postgresql" + assert params.host == "127.0.0.1" assert params.port is None assert params.database == "dbname" assert params.user is None diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py index 14b4ecc15..11ca51509 100644 --- a/python/tests/storage/test_mariadb.py +++ b/python/tests/storage/test_mariadb.py @@ -9,7 +9,7 @@ from spider.core import TaskInputValue from spider.storage import MariaDBStorage, parse_jdbc_url -MariaDBTestUrl = "jdbc:mariadb://localhost:3306/spider_test?user=root&password=password" +MariaDBTestUrl = "jdbc:mariadb://127.0.0.1:3306/spider-storage?user=spider&password=password" @pytest.fixture(scope="session") From 43de3422e5cd77788722493003f16aaae735edea Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 13:58:58 -0400 Subject: [PATCH 134/198] Add mariadb test --- python/src/spider/core/__init__.py | 4 + python/src/spider/core/task.py | 35 ++++++ python/src/spider/storage/mariadb_storage.py | 124 +++++++++++-------- python/tests/storage/test_mariadb.py | 14 +-- 4 files changed, 114 insertions(+), 63 deletions(-) diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py index 36843b816..793bcfec2 100644 --- a/python/src/spider/core/__init__.py +++ b/python/src/spider/core/__init__.py @@ -3,6 +3,8 @@ from .data import Data, DataId from .driver import DriverId from .task import ( + get_state_from_str, + get_state_str, Task, TaskId, TaskInput, @@ -32,4 +34,6 @@ "TaskOutputData", "TaskOutputValue", "TaskState", + "get_state_from_str", + "get_state_str", ] diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index f0b2e6d83..721e76553 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -52,6 +52,41 @@ class TaskState(IntEnum): Cancelled = 5 +_StateStrMap = { + TaskState.Pending: "pending", + TaskState.Ready: "ready", + TaskState.Running: "running", + TaskState.Succeeded: "success", + TaskState.Failed: "fail", + TaskState.Cancelled: "cancel", +} + +_StrStateMap = {v: k for k, v in _StateStrMap.items()} + + +def get_state_str(state: TaskState) -> str: + """ + Returns string representation of task state. + :param state: The task state. + :return: The string representation of task state. + """ + return _StateStrMap[state] + + +def get_state_from_str(state_str: str) -> TaskState: + """ + Returns task state from string representation. + :param state_str: The string representation of task state. + :return: The task state from string representation. + :raises ValueError: If the state string is not recognized. + """ + state = _StrStateMap.get(state_str, TaskState.Pending) + if state is not None: + return state + msg = f"Invalid task state string: {state_str}" + raise ValueError(msg) + + @dataclass class Task: """Represents a task in Spider.""" diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 805965371..f5c850abc 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -7,6 +7,7 @@ from typing_extensions import override from spider import core +from spider.core import get_state_str from spider.storage.jdbc_url import JdbcParameters from spider.storage.storage import Storage, StorageError @@ -75,7 +76,13 @@ def __init__(self, params: JdbcParameters) -> None: :raises StorageError: If the connection to the database fails. """ try: - self._conn = mariadb.connect(**params.__dict__) + self._conn = mariadb.connect( + host=params.host, + port=params.port, + user=params.user, + password=params.password, + database=params.database, + ) except mariadb.Error as e: raise StorageError(str(e)) from e @@ -83,18 +90,22 @@ def __init__(self, params: JdbcParameters) -> None: def submit_jobs( self, driver_id: core.DriverId, task_graphs: Sequence[core.TaskGraph] ) -> Sequence[core.JobId]: + if len(task_graphs) == 0: + return [] try: job_ids = [uuid4() for _ in task_graphs] with self._conn.cursor() as cursor: - cursor.executemany(InsertJob, [(job_id, driver_id) for job_id in job_ids]) + cursor.executemany( + InsertJob, [(job_id.bytes, driver_id.bytes) for job_id in job_ids] + ) cursor.executemany( InsertTask, [ ( - task.task_id, - job_id, + task.task_id.bytes, + job_id.bytes, task.function_name, - task.state.value, + get_state_str(task.state), task.timeout, task.max_retries, ) @@ -102,18 +113,20 @@ def submit_jobs( for task in task_graph.tasks.values() ], ) - cursor.executemany( - InsertTaskDependency, - [ - (parent, child) - for task_graph in task_graphs - for parent, child in task_graph.dependencies - ], - ) + dep_params = [ + (parent.bytes, child.bytes) + for task_graph in task_graphs + for parent, child in task_graph.dependencies + ] + if len(dep_params) > 0: + cursor.executemany( + InsertTaskDependency, + dep_params, + ) cursor.executemany( InsertInputTask, [ - (job_id, task_id, position) + (job_id.bytes, task_id.bytes, position) for job_id, task_graph in zip(job_ids, task_graphs, strict=True) for position, task_id in enumerate(task_graph.input_tasks) ], @@ -121,7 +134,7 @@ def submit_jobs( cursor.executemany( InsertOutputTask, [ - (job_id, task_id, position) + (job_id.bytes, task_id.bytes, position) for job_id, task_graph in zip(job_ids, task_graphs, strict=True) for position, task_id in enumerate(task_graph.output_tasks) ], @@ -129,49 +142,54 @@ def submit_jobs( cursor.executemany( InsertTaskOutput, [ - (task.task_id, position, task_output.type) + (task.task_id.bytes, position, task_output.type) for task_graph in task_graphs for task in task_graph.tasks.values() for position, task_output in enumerate(task.task_outputs) ], ) - cursor.executemany( - InsertTaskInputData, - [ - (task.task_id, position, task_input.type, task_input.value) - for task_graph in task_graphs - for task in task_graph.tasks.values() - for position, task_input in enumerate(task.task_inputs) - if isinstance(task_input.value, core.TaskInputData) - ], - ) - cursor.executemany( - InsertTaskInputValue, - [ - (task.task_id, position, task_input.type, task_input.value) - for task_graph in task_graphs - for task in task_graph.tasks.values() - for position, task_input in enumerate(task.task_inputs) - if isinstance(task_input.value, core.TaskInputValue) - ], - ) - cursor.executemany( - InsertTaskInputOutput, - [ - ( - task.task_id, - position, - task_input.type, - task_input.value.task_id, - task_input.value.position, - ) - for task_graph in task_graphs - for task in task_graph.tasks.values() - for position, task_input in enumerate(task.task_inputs) - if isinstance(task_input.value, core.TaskInputOutput) - ], - ) - cursor.executemany() + input_data_params = [ + (task.task_id.bytes, position, task_input.type, task_input.value.bytes) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_input in enumerate(task.task_inputs) + if isinstance(task_input.value, core.TaskInputData) + ] + if len(input_data_params) > 0: + cursor.executemany( + InsertTaskInputData, + input_data_params, + ) + input_value_params = [ + (task.task_id.bytes, position, task_input.type, task_input.value) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_input in enumerate(task.task_inputs) + if isinstance(task_input.value, core.TaskInputValue) + ] + if len(input_value_params) > 0: + cursor.executemany( + InsertTaskInputValue, + input_value_params, + ) + input_output_params = [ + ( + task.task_id.bytes, + position, + task_input.type, + task_input.value.task_id.bytes, + task_input.value.position, + ) + for task_graph in task_graphs + for task in task_graph.tasks.values() + for position, task_input in enumerate(task.task_inputs) + if isinstance(task_input.value, core.TaskInputOutput) + ] + if len(input_output_params) > 0: + cursor.executemany( + InsertTaskInputOutput, + input_output_params, + ) self._conn.commit() return job_ids except mariadb.Error as e: diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py index 11ca51509..b0a377cb3 100644 --- a/python/tests/storage/test_mariadb.py +++ b/python/tests/storage/test_mariadb.py @@ -5,7 +5,7 @@ import msgpack import pytest -from spider import chain, group, Int8, TaskContext +from spider import group, Int8, TaskContext from spider.core import TaskInputValue from spider.storage import MariaDBStorage, parse_jdbc_url @@ -19,11 +19,6 @@ def mariadb_storage() -> MariaDBStorage: return MariaDBStorage(params) -def add(_: TaskContext, x: Int8, y: Int8) -> Int8: - """Adds two numbers.""" - return Int8(x + y) - - def double(_: TaskContext, x: Int8) -> Int8: """Double a number.""" return Int8(x * 2) @@ -40,12 +35,11 @@ class TestMariaDBStorage: @pytest.mark.storage def test_job_submission(self, mariadb_storage: MariaDBStorage) -> None: """Test job submission to the MariaDB storage backend.""" - graph = chain(group([double, double, double, double]), group([swap, swap]))._impl + graph = group([double, double, double, double])._impl # Fill input data - for task_id in graph.input_tasks: + for i, task_id in enumerate(graph.input_tasks): task = graph.tasks[task_id] - task.task_inputs[0].value = TaskInputValue(msgpack.packb(1)) - task.task_inputs[1].value = TaskInputValue(msgpack.packb(2)) + task.task_inputs[0].value = TaskInputValue(msgpack.packb(i)) driver_id = uuid4() job_ids = mariadb_storage.submit_jobs(driver_id, [graph]) From 6b06a304ae90b03838827f396641d982ea2cf376 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 15:48:32 -0400 Subject: [PATCH 135/198] Split python tests into storage and non-storage --- .github/workflows/unit-tests.yaml | 2 +- docs/src/dev-docs/testing.md | 14 ++++++++------ test-tasks.yaml | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 058779260..2b8b0754c 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -57,4 +57,4 @@ jobs: - run: "task test:non-storage-unit-tests" - - run: "task test:python-tests" + - run: "task test:python-non-storage-tests" diff --git a/docs/src/dev-docs/testing.md b/docs/src/dev-docs/testing.md index 759cf5023..e2bd98501 100644 --- a/docs/src/dev-docs/testing.md +++ b/docs/src/dev-docs/testing.md @@ -30,12 +30,14 @@ require this storage backend. You can use the following tasks to run the set of unit tests that's appropriate. -| Task | Description | -|-------------------------------|-----------------------------------------------------------------------| -| `test:cpp-all` | Runs all C++ unit tests. | -| `test:non-storage-unit-tests` | Runs all C++ unit tests which don't require a storage backend to run. | -| `test:storage-unit-tests` | Runs all C++ unit tests which require a storage backend to run. | -| `test:python-tests` | Runs all Python tests. | +| Task | Description | +|---------------------------------|-----------------------------------------------------------------------| +| `test:cpp-all` | Runs all C++ unit tests. | +| `test:non-storage-unit-tests` | Runs all C++ unit tests which don't require a storage backend to run. | +| `test:storage-unit-tests` | Runs all C++ unit tests which require a storage backend to run. | +| `test:python-tests` | Runs all Python tests. | +| `test:python-non-storage-tests` | Runs all Python tests which don't require a storage backend to run. | +| `test:python-storage-tests` | Runs all Python tests which require a storage backend to run. | If any tests show error messages for the connection function below, revisit the [setup section](#set-up-mysql-as-storage-backend) and verify that `cStorageUrl` was set correctly. diff --git a/test-tasks.yaml b/test-tasks.yaml index ca635301b..60b0724ef 100644 --- a/test-tasks.yaml +++ b/test-tasks.yaml @@ -84,3 +84,19 @@ tasks: PYTHONDONTWRITEBYTECODE: "1" cmds: - "uv run pytest" + + python-non-storage-tests: + dir: "{{.G_SRC_PYTHON_DIR}}" + env: + # Don't create __pycache__ directories in source tree. + PYTHONDONTWRITEBYTECODE: "1" + cmds: + - "uv run pytest -m \"not storage\"" + + python-storage-tests: + dir: "{{.G_SRC_PYTHON_DIR}}" + env: + # Don't create __pycache__ directories in source tree. + PYTHONDONTWRITEBYTECODE: "1" + cmds: + - "uv run pytest -m \"storage\"" From 8faa0c69e7ab7c2eb65cb4541c674c45e081ffc6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 15:53:41 -0400 Subject: [PATCH 136/198] Add more complex graph --- python/tests/storage/test_mariadb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py index b0a377cb3..55ba965d9 100644 --- a/python/tests/storage/test_mariadb.py +++ b/python/tests/storage/test_mariadb.py @@ -5,7 +5,7 @@ import msgpack import pytest -from spider import group, Int8, TaskContext +from spider import chain, group, Int8, TaskContext from spider.core import TaskInputValue from spider.storage import MariaDBStorage, parse_jdbc_url @@ -35,7 +35,7 @@ class TestMariaDBStorage: @pytest.mark.storage def test_job_submission(self, mariadb_storage: MariaDBStorage) -> None: """Test job submission to the MariaDB storage backend.""" - graph = group([double, double, double, double])._impl + graph = chain(group([double, double, double, double]), group([swap, swap]))._impl # Fill input data for i, task_id in enumerate(graph.input_tasks): task = graph.tasks[task_id] From fe904b0415f2f40f017cb36c4585ee8f45628732 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 16:04:40 -0400 Subject: [PATCH 137/198] Bug fix --- python/src/spider/core/taskgraph.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index b89b105aa..914c52ccf 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -29,16 +29,16 @@ def add_task( :param children: The children ids of the task. Must be already in the task graph. """ self.tasks[task.task_id] = deepcopy(task) - if parents: + if parents is not None and len(parents) > 0: for parent in parents: self.dependencies.append((parent, task.task_id)) - self.output_tasks.append(parent) + self.output_tasks.remove(parent) else: self.input_tasks.append(task.task_id) - if children: + if children is not None and len(children) > 0: for child in children: self.dependencies.append((task.task_id, child)) - self.input_tasks.append(child) + self.input_tasks.remove(child) else: self.output_tasks.append(task.task_id) @@ -68,6 +68,7 @@ def reset_ids(self) -> None: for task_id in self.tasks: new_task_id = id_map[task_id] new_tasks[new_task_id] = deepcopy(self.tasks[task_id]) + new_tasks[new_task_id].task_id = new_task_id for task_input in new_tasks[new_task_id].task_inputs: if isinstance(task_input.value, TaskInputOutput): task_input.value.task_id = id_map[task_input.value.task_id] From e1b1ac38ed09960c491c43f6e9bb4770a0e7a2ca Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 16:19:35 -0400 Subject: [PATCH 138/198] Bug fix --- python/src/spider/core/taskgraph.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index c1e3e778d..2a95e678d 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -34,13 +34,15 @@ def add_task( if parents is not None and len(parents) > 0: for parent in parents: self.dependencies.append((parent, task.task_id)) - self.output_tasks.remove(parent) + if parent in self.output_tasks: + self.output_tasks.remove(parent) else: self.input_tasks.append(task.task_id) if children is not None and len(children) > 0: for child in children: self.dependencies.append((task.task_id, child)) - self.input_tasks.remove(child) + if child in self.output_tasks: + self.input_tasks.remove(child) else: self.output_tasks.append(task.task_id) From 2b5405ac2797f0aa8e5a2b7dba2ec8ef6bf22184 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 16:30:25 -0400 Subject: [PATCH 139/198] Bug fix --- python/src/spider/core/taskgraph.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index 2a95e678d..50e319779 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -41,7 +41,7 @@ def add_task( if children is not None and len(children) > 0: for child in children: self.dependencies.append((task.task_id, child)) - if child in self.output_tasks: + if child in self.input_Tasks: self.input_tasks.remove(child) else: self.output_tasks.append(task.task_id) @@ -97,7 +97,6 @@ def merge_graph(self, graph: "TaskGraph") -> None: """ Merges another task graph into this task graph. :param graph: The task graph to merge. - :return: """ new_graph = deepcopy(graph) new_graph.reset_ids() From 3938934e5b27ca7603ed62518361a120cc159f75 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 16:35:40 -0400 Subject: [PATCH 140/198] Bug fix --- python/src/spider/core/taskgraph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index 50e319779..330651b5b 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -41,7 +41,7 @@ def add_task( if children is not None and len(children) > 0: for child in children: self.dependencies.append((task.task_id, child)) - if child in self.input_Tasks: + if child in self.input_tasks: self.input_tasks.remove(child) else: self.output_tasks.append(task.task_id) From eebcab719533c7592ec63b6b9f47f0a3a6c0fdbf Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 16:37:30 -0400 Subject: [PATCH 141/198] Fix docstring typo Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/src/spider/storage/mariadb_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index f5c850abc..98d9bd1dd 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -67,7 +67,7 @@ class MariaDBStorage(Storage): - """MairaDB Storage class.""" + """MariaDB Storage class.""" def __init__(self, params: JdbcParameters) -> None: """ From 5e2175a31869113d8506cf05a23cdff62281b988 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 16:37:58 -0400 Subject: [PATCH 142/198] Fix dict get default value Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/src/spider/core/task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index 721e76553..25c8ccd4f 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -80,7 +80,7 @@ def get_state_from_str(state_str: str) -> TaskState: :return: The task state from string representation. :raises ValueError: If the state string is not recognized. """ - state = _StrStateMap.get(state_str, TaskState.Pending) + state = _StrStateMap.get(state_str) if state is not None: return state msg = f"Invalid task state string: {state_str}" From bff3a95643651c16edfd9ad03e39ee04302b27f9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 17:29:37 -0400 Subject: [PATCH 143/198] Reformat pyporject.toml --- python/pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 1435df991..3e573dab3 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -35,11 +35,11 @@ module = ["mariadb.*"] ignore_missing_imports = true [tool.pytest.ini_options] -testpaths = ["tests"] markers = [ - "integration: mark test as an integration test", - "storage: mark unit tests needing storage", + "integration: mark test as an integration test", + "storage: mark unit tests needing storage", ] +testpaths = ["tests"] [tool.ruff] line-length = 100 From e0fd98199aeda01549b022cb7750f6fd42dd08f0 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 18:29:25 -0400 Subject: [PATCH 144/198] Use not list instead of len(list) > 0 Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/src/spider/storage/mariadb_storage.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 98d9bd1dd..ee4580cf3 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -90,7 +90,7 @@ def __init__(self, params: JdbcParameters) -> None: def submit_jobs( self, driver_id: core.DriverId, task_graphs: Sequence[core.TaskGraph] ) -> Sequence[core.JobId]: - if len(task_graphs) == 0: + if not task_graphs: return [] try: job_ids = [uuid4() for _ in task_graphs] @@ -118,7 +118,7 @@ def submit_jobs( for task_graph in task_graphs for parent, child in task_graph.dependencies ] - if len(dep_params) > 0: + if dep_params: cursor.executemany( InsertTaskDependency, dep_params, @@ -155,7 +155,7 @@ def submit_jobs( for position, task_input in enumerate(task.task_inputs) if isinstance(task_input.value, core.TaskInputData) ] - if len(input_data_params) > 0: + if input_data_params: cursor.executemany( InsertTaskInputData, input_data_params, @@ -167,7 +167,7 @@ def submit_jobs( for position, task_input in enumerate(task.task_inputs) if isinstance(task_input.value, core.TaskInputValue) ] - if len(input_value_params) > 0: + if input_value_params: cursor.executemany( InsertTaskInputValue, input_value_params, @@ -185,7 +185,7 @@ def submit_jobs( for position, task_input in enumerate(task.task_inputs) if isinstance(task_input.value, core.TaskInputOutput) ] - if len(input_output_params) > 0: + if input_output_params: cursor.executemany( InsertTaskInputOutput, input_output_params, From 22e45478d6a1dddcdca9e14e724ca4fcd9b9a03b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 22:24:15 -0400 Subject: [PATCH 145/198] Add client job submission --- python/src/spider/client/__init__.py | 4 ++ python/src/spider/client/data.py | 7 +++ python/src/spider/client/driver.py | 65 ++++++++++++++++++++++++++++ python/src/spider/client/job.py | 17 ++++++++ 4 files changed, 93 insertions(+) create mode 100644 python/src/spider/client/driver.py create mode 100644 python/src/spider/client/job.py diff --git a/python/src/spider/client/__init__.py b/python/src/spider/client/__init__.py index 37ab0dd7a..2dad4bb67 100644 --- a/python/src/spider/client/__init__.py +++ b/python/src/spider/client/__init__.py @@ -1,9 +1,13 @@ """Spider python client.""" +from .data import Data +from .driver import Driver from .task import TaskContext from .taskgraph import chain, group, TaskGraph __all__ = [ + "Data", + "Driver", "TaskContext", "TaskGraph", "chain", diff --git a/python/src/spider/client/data.py b/python/src/spider/client/data.py index a5dffba18..479afda76 100644 --- a/python/src/spider/client/data.py +++ b/python/src/spider/client/data.py @@ -1,5 +1,12 @@ """Spider client Data module.""" +from spider import core + class Data: """Represents a spider client data.""" + + def __init__(self, value: bytes) -> None: + """Initialize the Data object with the given value.""" + self.data_id = core.DataId() + self.value = value diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py new file mode 100644 index 000000000..de8f4f028 --- /dev/null +++ b/python/src/spider/client/driver.py @@ -0,0 +1,65 @@ +"""Spider client driver module.""" + +from collections.abc import Sequence + +import msgpack + +from spider import core +from spider.client.data import Data +from spider.client.job import Job +from spider.client.taskgraph import TaskGraph +from spider.storage import MariaDBStorage, parse_jdbc_url +from spider.type import to_tdl_type_str + + +class Driver: + """Spider client driver class.""" + + def __init__(self, storage_url: str) -> None: + """ + Creates a new Spider client driver and connects to the storage. + :param storage_url: The URL of the storage to connect to. + :raises StorageError: If the storage cannot be connected to. + """ + self.driver_id = core.DriverId() + self.storage = MariaDBStorage(parse_jdbc_url(storage_url)) + + def submit_jobs( + self, jobs: Sequence[TaskGraph], args: Sequence[Sequence[object]] + ) -> Sequence[Job]: + """ + Submits a list of jobs to the storage. + :param jobs: The list of task graphs to submit. + :param args: The arguments for each job. + :return: A sequence of Job objects representing the submitted jobs. + :raises StorageError: If the jobs cannot be submitted to the storage. + :raises ValueError: If the number of job inputs does not match the number of arguments. + :raises TypeError: If the arguments are not of the expected type. + :raises MsgpackError: If the arguments cannot be serialized with msgpack. + """ + msg = "Number of job inputs does not match number of arguments" + if len(jobs) != len(args): + raise ValueError(msg) + + if not jobs: + return [] + for task_graph, task_args in zip(jobs, args, strict=False): + arg_index = 0 + for task_id in task_graph._impl.input_tasks: + task = task_graph._impl.tasks[task_id] + for task_input in task.task_inputs: + if arg_index >= len(task_args): + raise ValueError(msg) + arg = task_args[arg_index] + if isinstance(arg, Data): + task_input.type = to_tdl_type_str(Data) + task_input.value = arg.data_id + else: + task_input.type = to_tdl_type_str(type(arg)) + task_input.value = core.TaskInputValue(msgpack.packb(arg)) + arg_index += 1 + if arg_index != len(task_args): + raise ValueError(msg) + + job_ids = self.storage.submit_jobs(self.driver_id, [job._impl for job in jobs]) + return [Job(job_id, self.storage) for job_id in job_ids] diff --git a/python/src/spider/client/job.py b/python/src/spider/client/job.py new file mode 100644 index 000000000..e8182b9c7 --- /dev/null +++ b/python/src/spider/client/job.py @@ -0,0 +1,17 @@ +"""Spider job module.""" + +from spider import core +from spider.storage import Storage + + +class Job: + """Represents Spider job.""" + + def __init__(self, job_id: core.JobId, storage: Storage) -> None: + """ + Creates a new Spider job. + :param job_id: + :param storage: The storage backend. + """ + self.job_id = job_id + self.storage = storage From e50e892fff127146d79b2958824c98408eea6b20 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 22:28:50 -0400 Subject: [PATCH 146/198] Export driver --- python/src/spider/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py index b0ac7f542..c03ec9172 100644 --- a/python/src/spider/__init__.py +++ b/python/src/spider/__init__.py @@ -1,10 +1,12 @@ """Spider package root.""" -from spider.client import chain, group, TaskContext, TaskGraph +from spider.client import chain, Data, Driver, group, TaskContext, TaskGraph from spider.type import Double, Float, Int8, Int16, Int32, Int64 __all__ = [ + "Data", "Double", + "Driver", "Float", "Int8", "Int16", From 573f01c859e3289fe26027631946e7a44cd9ee50 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 22:42:01 -0400 Subject: [PATCH 147/198] Add job submission unit test and fix bugs --- python/src/spider/client/driver.py | 3 +- python/tests/client/test_driver.py | 67 ++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 python/tests/client/test_driver.py diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py index de8f4f028..08e4e4e43 100644 --- a/python/src/spider/client/driver.py +++ b/python/src/spider/client/driver.py @@ -1,6 +1,7 @@ """Spider client driver module.""" from collections.abc import Sequence +from uuid import uuid4 import msgpack @@ -21,7 +22,7 @@ def __init__(self, storage_url: str) -> None: :param storage_url: The URL of the storage to connect to. :raises StorageError: If the storage cannot be connected to. """ - self.driver_id = core.DriverId() + self.driver_id = uuid4() self.storage = MariaDBStorage(parse_jdbc_url(storage_url)) def submit_jobs( diff --git a/python/tests/client/test_driver.py b/python/tests/client/test_driver.py new file mode 100644 index 000000000..99a42783f --- /dev/null +++ b/python/tests/client/test_driver.py @@ -0,0 +1,67 @@ +"""Tests for the driver module.""" + +import pytest + +from spider import chain, Driver, group, Int8, TaskContext + +MariaDBTestUrl = "jdbc:mariadb://127.0.0.1:3306/spider-storage?user=spider&password=password" + + +@pytest.fixture(scope="session") +def driver() -> Driver: + """Fixture for the driver.""" + return Driver(MariaDBTestUrl) + + +def double(_: TaskContext, x: Int8) -> Int8: + """Double a number.""" + return Int8(x * 2) + + +def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: + """Swaps two numbers.""" + return y, x + + +@pytest.mark.storage +class TestDriver: + """Test class for the driver module.""" + + def test_job_submission(self, driver: Driver) -> None: + """Tests successful job submission.""" + jobs = driver.submit_jobs( + [ + group([double]), + group([double, double]), + chain(group([double, double]), swap), + ], + [ + (Int8(1),), + (Int8(1), Int8(2)), + (Int8(1), Int8(2)), + ], + ) + assert len(jobs) == 3 + + def test_job_submission_fail(self, driver: Driver) -> None: + """Tests job submission failure.""" + with pytest.raises( + ValueError, match="Number of job inputs does not match number of arguments" + ): + driver.submit_jobs( + [ + group([double]), + ], + [ + (Int8(1), Int8(2)), + ], + ) + with pytest.raises(TypeError): + driver.submit_jobs( + [ + group([double]), + ], + [ + (1,), + ], + ) From 5a999ff14b71fa7ac9b4a21c4204e5a41354b0e3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 22:47:07 -0400 Subject: [PATCH 148/198] Set zip to strict Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/src/spider/client/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py index 08e4e4e43..d6fa8eb8d 100644 --- a/python/src/spider/client/driver.py +++ b/python/src/spider/client/driver.py @@ -44,7 +44,7 @@ def submit_jobs( if not jobs: return [] - for task_graph, task_args in zip(jobs, args, strict=False): + for task_graph, task_args in zip(jobs, args, strict=True): arg_index = 0 for task_id in task_graph._impl.input_tasks: task = task_graph._impl.tasks[task_id] From 8cbebc9b6cfbd96113feef506f3056b57d88b763 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 22:50:54 -0400 Subject: [PATCH 149/198] Remove unnecessary zip check Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- python/src/spider/client/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py index d6fa8eb8d..bd18ede1c 100644 --- a/python/src/spider/client/driver.py +++ b/python/src/spider/client/driver.py @@ -44,7 +44,7 @@ def submit_jobs( if not jobs: return [] - for task_graph, task_args in zip(jobs, args, strict=True): + for task_graph, task_args in zip(jobs, args): arg_index = 0 for task_id in task_graph._impl.input_tasks: task = task_graph._impl.tasks[task_id] From 2b29782195ab1e4318b166e807ee2a14d4d0c7ab Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 22:54:17 -0400 Subject: [PATCH 150/198] Remove msgpack exception in docstring --- python/src/spider/client/driver.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py index bd18ede1c..51a7131aa 100644 --- a/python/src/spider/client/driver.py +++ b/python/src/spider/client/driver.py @@ -36,7 +36,6 @@ def submit_jobs( :raises StorageError: If the jobs cannot be submitted to the storage. :raises ValueError: If the number of job inputs does not match the number of arguments. :raises TypeError: If the arguments are not of the expected type. - :raises MsgpackError: If the arguments cannot be serialized with msgpack. """ msg = "Number of job inputs does not match number of arguments" if len(jobs) != len(args): From 44129a6a9718e747a405ec728e096bf69f982673 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 12 Aug 2025 23:44:56 -0400 Subject: [PATCH 151/198] Fix ruff --- python/src/spider/client/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py index 51a7131aa..82a8d010e 100644 --- a/python/src/spider/client/driver.py +++ b/python/src/spider/client/driver.py @@ -43,7 +43,7 @@ def submit_jobs( if not jobs: return [] - for task_graph, task_args in zip(jobs, args): + for task_graph, task_args in zip(jobs, args, strict=True): arg_index = 0 for task_id in task_graph._impl.input_tasks: task = task_graph._impl.tasks[task_id] From 41cdb3e15a7003032a7efb27f2fb953ccede4b80 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 09:53:43 -0400 Subject: [PATCH 152/198] Add core job --- python/src/spider/client/driver.py | 14 ++++----- python/src/spider/client/job.py | 6 ++-- python/src/spider/core/__init__.py | 4 ++- python/src/spider/core/job.py | 30 ++++++++++++++++++++ python/src/spider/core/taskgraph.py | 4 +-- python/src/spider/storage/mariadb_storage.py | 4 +-- python/src/spider/storage/storage.py | 4 +-- 7 files changed, 48 insertions(+), 18 deletions(-) create mode 100644 python/src/spider/core/job.py diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py index 82a8d010e..c719de9eb 100644 --- a/python/src/spider/client/driver.py +++ b/python/src/spider/client/driver.py @@ -26,11 +26,11 @@ def __init__(self, storage_url: str) -> None: self.storage = MariaDBStorage(parse_jdbc_url(storage_url)) def submit_jobs( - self, jobs: Sequence[TaskGraph], args: Sequence[Sequence[object]] + self, graphs: Sequence[TaskGraph], args: Sequence[Sequence[object]] ) -> Sequence[Job]: """ Submits a list of jobs to the storage. - :param jobs: The list of task graphs to submit. + :param graphs: The list of task graphs to submit. :param args: The arguments for each job. :return: A sequence of Job objects representing the submitted jobs. :raises StorageError: If the jobs cannot be submitted to the storage. @@ -38,12 +38,12 @@ def submit_jobs( :raises TypeError: If the arguments are not of the expected type. """ msg = "Number of job inputs does not match number of arguments" - if len(jobs) != len(args): + if len(graphs) != len(args): raise ValueError(msg) - if not jobs: + if not graphs: return [] - for task_graph, task_args in zip(jobs, args, strict=True): + for task_graph, task_args in zip(graphs, args, strict=True): arg_index = 0 for task_id in task_graph._impl.input_tasks: task = task_graph._impl.tasks[task_id] @@ -61,5 +61,5 @@ def submit_jobs( if arg_index != len(task_args): raise ValueError(msg) - job_ids = self.storage.submit_jobs(self.driver_id, [job._impl for job in jobs]) - return [Job(job_id, self.storage) for job_id in job_ids] + jobs = self.storage.submit_jobs(self.driver_id, [graph._impl for graph in graphs]) + return [Job(job, self.storage) for job in jobs] diff --git a/python/src/spider/client/job.py b/python/src/spider/client/job.py index e8182b9c7..0ea67e55a 100644 --- a/python/src/spider/client/job.py +++ b/python/src/spider/client/job.py @@ -7,11 +7,11 @@ class Job: """Represents Spider job.""" - def __init__(self, job_id: core.JobId, storage: Storage) -> None: + def __init__(self, job: core.Job, storage: Storage) -> None: """ Creates a new Spider job. - :param job_id: + :param job: Core job object. :param storage: The storage backend. """ - self.job_id = job_id + self._impl = job self.storage = storage diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py index 793bcfec2..c18207551 100644 --- a/python/src/spider/core/__init__.py +++ b/python/src/spider/core/__init__.py @@ -2,6 +2,7 @@ from .data import Data, DataId from .driver import DriverId +from .job import Job, JobId from .task import ( get_state_from_str, get_state_str, @@ -16,12 +17,13 @@ TaskOutputValue, TaskState, ) -from .taskgraph import JobId, TaskGraph +from .taskgraph import TaskGraph __all__ = [ "Data", "DataId", "DriverId", + "Job", "JobId", "Task", "TaskGraph", diff --git a/python/src/spider/core/job.py b/python/src/spider/core/job.py new file mode 100644 index 000000000..a7c3ad7a5 --- /dev/null +++ b/python/src/spider/core/job.py @@ -0,0 +1,30 @@ +"""Job module for Spider.""" + +from enum import IntEnum +from uuid import UUID + +JobId = UUID + + +class JobStatus(IntEnum): + """Job status.""" + + Running = 0 + Succeeded = 1 + Failed = 2 + Cancelled = 3 + + +class Job: + """Represents a submitted job.""" + + def __init__(self, job_id: JobId) -> None: + """ + Initializes a running job. + Status and results are cache for the data in storage. + Once a job completes, the status and results won't change. + :param job_id: + """ + self.job_id = job_id + self.status = JobStatus.Running + self.results: object | None = None diff --git a/python/src/spider/core/taskgraph.py b/python/src/spider/core/taskgraph.py index 330651b5b..e4d03ffe6 100644 --- a/python/src/spider/core/taskgraph.py +++ b/python/src/spider/core/taskgraph.py @@ -1,12 +1,10 @@ """TaskGraph module for Spider.""" from copy import deepcopy -from uuid import UUID, uuid4 +from uuid import uuid4 from spider.core.task import Task, TaskId, TaskInputOutput -JobId = UUID - class TaskGraph: """Represents a task graph in Spider.""" diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index ee4580cf3..afc73085e 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -89,7 +89,7 @@ def __init__(self, params: JdbcParameters) -> None: @override def submit_jobs( self, driver_id: core.DriverId, task_graphs: Sequence[core.TaskGraph] - ) -> Sequence[core.JobId]: + ) -> Sequence[core.Job]: if not task_graphs: return [] try: @@ -191,7 +191,7 @@ def submit_jobs( input_output_params, ) self._conn.commit() - return job_ids + return [core.Job(job_id) for job_id in job_ids] except mariadb.Error as e: self._conn.rollback() raise StorageError(str(e)) from e diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index ac7e5f7e7..ed526e5b5 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -20,11 +20,11 @@ class Storage(ABC): @abstractmethod def submit_jobs( self, driver_id: core.DriverId, task_graphs: Sequence[core.TaskGraph] - ) -> Sequence[core.JobId]: + ) -> Sequence[core.Job]: """ Submit jobs to the storage. :param driver_id: Driver id. :param task_graphs: Task graphs to submit. - :return: List of job ids representing the submitted jobs. + :return: List of jobs representing the submitted jobs. :raises StorageError: If the storage operations fail. """ From a36ac49f66d04cfbe064925dde374183574e4c54 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 10:29:39 -0400 Subject: [PATCH 153/198] Fix docstring --- python/src/spider/storage/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index ed526e5b5..15393eec4 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -22,7 +22,7 @@ def submit_jobs( self, driver_id: core.DriverId, task_graphs: Sequence[core.TaskGraph] ) -> Sequence[core.Job]: """ - Submit jobs to the storage. + Submits jobs to the storage. :param driver_id: Driver id. :param task_graphs: Task graphs to submit. :return: List of jobs representing the submitted jobs. From f89ebc6b9df807a5d9afba69ac481807db09f159 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 10:34:58 -0400 Subject: [PATCH 154/198] Export JobStatus in core --- python/src/spider/core/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py index c18207551..2b3fa6fb2 100644 --- a/python/src/spider/core/__init__.py +++ b/python/src/spider/core/__init__.py @@ -2,7 +2,7 @@ from .data import Data, DataId from .driver import DriverId -from .job import Job, JobId +from .job import Job, JobId, JobStatus from .task import ( get_state_from_str, get_state_str, @@ -25,6 +25,7 @@ "DriverId", "Job", "JobId", + "JobStatus", "Task", "TaskGraph", "TaskId", From bd43d5874ebbcc19b8ecb9561c8b531d1d454bb3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 11:19:48 -0400 Subject: [PATCH 155/198] Add get job status --- python/src/spider/storage/mariadb_storage.py | 38 ++++++++++++++++++++ python/src/spider/storage/storage.py | 19 ++++++++++ 2 files changed, 57 insertions(+) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index afc73085e..8e83c571e 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -66,6 +66,16 @@ (?, ?, ?)""" +GetJobStatus = """ +SELECT + `state` +FROM + `jobs` +WHERE + `id` = ? +""" + + class MariaDBStorage(Storage): """MariaDB Storage class.""" @@ -195,3 +205,31 @@ def submit_jobs( except mariadb.Error as e: self._conn.rollback() raise StorageError(str(e)) from e + + @override + def get_job_status(self, job: core.Job) -> core.JobStatus: + try: + with self._conn.cursor() as cursor: + cursor.execute(GetJobStatus, (job.job_id.bytes,)) + job_str = cursor.fetchone()[0] + match job_str: + case "running": + status = core.JobStatus.Running + case "success": + status = core.JobStatus.Succeeded + case "fail": + status = core.JobStatus.Failed + case "cancel": + status = core.JobStatus.Cancelled + case _: + msg = "Unknown job status" + raise StorageError(msg) + self._conn.commit() + return status + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e + + @override + def get_job_results(self, job: core.Job) -> object: + pass diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index 15393eec4..8881957b7 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -4,6 +4,7 @@ from collections.abc import Sequence from spider import core +from spider.core import JobStatus class StorageError(Exception): @@ -28,3 +29,21 @@ def submit_jobs( :return: List of jobs representing the submitted jobs. :raises StorageError: If the storage operations fail. """ + + @abstractmethod + def get_job_status(self, job: core.Job) -> JobStatus: + """ + Gets the job status. This function does not set the `status` field in jobs. + :param job: + :return: + :raises StorageError: If the storage operations fail. + """ + + @abstractmethod + def get_job_results(self, job: core.Job) -> object: + """ + Gets the job's results. This function does not set the `results` field in the job. + :param job: + :return: + :raises StorageError: If the storage operations fail. + """ From d3f3eb1c2286aa0173694a73cc565bc65d7e05cf Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 11:38:31 -0400 Subject: [PATCH 156/198] Export tdl parse function --- python/src/spider/type/__init__.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/python/src/spider/type/__init__.py b/python/src/spider/type/__init__.py index e0c873db6..fda34ee9b 100644 --- a/python/src/spider/type/__init__.py +++ b/python/src/spider/type/__init__.py @@ -1,6 +1,16 @@ """Spider type package.""" from spider.type.tdl_convert import to_tdl_type_str +from spider.type.tdl_parse import parse_tdl_type from spider.type.type import Double, Float, Int8, Int16, Int32, Int64 -__all__ = ["Double", "Float", "Int8", "Int16", "Int32", "Int64", "to_tdl_type_str"] +__all__ = [ + "Double", + "Float", + "Int8", + "Int16", + "Int32", + "Int64", + "parse_tdl_type", + "to_tdl_type_str", +] From 9a6fb16814c565393b91d5d90a4b65e86369dbf9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 11:45:46 -0400 Subject: [PATCH 157/198] Add get result storage function --- python/src/spider/storage/mariadb_storage.py | 57 ++++++++++++++++++-- python/src/spider/storage/storage.py | 1 + 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 8e83c571e..497cef6c9 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -4,6 +4,7 @@ from uuid import uuid4 import mariadb +import msgpack from typing_extensions import override from spider import core @@ -72,8 +73,29 @@ FROM `jobs` WHERE - `id` = ? -""" + `id` = ?""" + +GetOutputTasks = """ +SELECT + `task_id` +FROM + `output_tasks` +WHERE + `job_id` = ? +ORDER BY + `position`""" + +GetTaskOutputs = """ +SELECT + `type`, + `value`, + `data_id` +FROM + `task_outputs` +WHERE + `task_id` = ? +ORDER BY + `position`""" class MariaDBStorage(Storage): @@ -232,4 +254,33 @@ def get_job_status(self, job: core.Job) -> core.JobStatus: @override def get_job_results(self, job: core.Job) -> object: - pass + try: + with self._conn.cursor() as cursor: + cursor.execute(GetOutputTasks, (job.job_id.bytes,)) + task_ids = [task_id for (task_id,) in cursor.fetchall()] + + cursor.executemany(GetTaskOutputs, [(task_id,) for task_id in task_ids]) + results = [] + for output_type, value, data_id in cursor: + if value is not None: + results.append( + core.TaskOutput( + type=output_type, + value=core.TaskOutputValue(msgpack.unpackb(value)), + ) + ) + if data_id is not None: + results.append( + core.TaskOutput( + type=output_type, + value=core.TaskOutputValue(msgpack.unpackb(data_id)), + ) + ) + self._conn.commit() + return results + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e + except msgpack.exceptions.UnpackValueError: + self._conn.rollback() + raise diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index 8881957b7..4780f6d6d 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -46,4 +46,5 @@ def get_job_results(self, job: core.Job) -> object: :param job: :return: :raises StorageError: If the storage operations fail. + :raises msgpack.exceptions.UnPackError: If deserialization of the result fails. """ From 4b755a42d33c89164ab3a0b8c7a1509709f8d8ec Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 12:03:51 -0400 Subject: [PATCH 158/198] Fix test var name --- python/tests/storage/test_mariadb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py index 55ba965d9..591a4994e 100644 --- a/python/tests/storage/test_mariadb.py +++ b/python/tests/storage/test_mariadb.py @@ -42,5 +42,5 @@ def test_job_submission(self, mariadb_storage: MariaDBStorage) -> None: task.task_inputs[0].value = TaskInputValue(msgpack.packb(i)) driver_id = uuid4() - job_ids = mariadb_storage.submit_jobs(driver_id, [graph]) - assert len(job_ids) == 1 + jobs = mariadb_storage.submit_jobs(driver_id, [graph]) + assert len(jobs) == 1 From af124fd446180fb64943750ef68568483258eb19 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 12:12:45 -0400 Subject: [PATCH 159/198] Add unit tests for getting status --- python/tests/storage/test_mariadb.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py index 591a4994e..bc46c40be 100644 --- a/python/tests/storage/test_mariadb.py +++ b/python/tests/storage/test_mariadb.py @@ -6,7 +6,7 @@ import pytest from spider import chain, group, Int8, TaskContext -from spider.core import TaskInputValue +from spider.core import Job, JobStatus, TaskInputValue from spider.storage import MariaDBStorage, parse_jdbc_url MariaDBTestUrl = "jdbc:mariadb://127.0.0.1:3306/spider-storage?user=spider&password=password" @@ -29,6 +29,19 @@ def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: return y, x +@pytest.fixture +def submit_job(mariadb_storage: MariaDBStorage) -> Job: + graph = chain(group([double, double]), group([swap]))._impl + # Fill input data + for i, task_id in enumerate(graph.input_tasks): + task = graph.tasks[task_id] + task.task_inputs[0].value = TaskInputValue(msgpack.packb(i)) + + driver_id = uuid4() + jobs = mariadb_storage.submit_jobs(driver_id, [graph]) + return jobs[0] + + class TestMariaDBStorage: """Test class for the MariaDB storage backend.""" @@ -44,3 +57,9 @@ def test_job_submission(self, mariadb_storage: MariaDBStorage) -> None: driver_id = uuid4() jobs = mariadb_storage.submit_jobs(driver_id, [graph]) assert len(jobs) == 1 + + @pytest.mark.storage + def test_job_status(self, mariadb_storage: MariaDBStorage, submit_job) -> None: + """Test job status of the MariaDB storage backend.""" + status = mariadb_storage.get_job_status(submit_job) + assert status == JobStatus.Running From 24108296c876ecf09bf15cdba4e7002b6cd17ee9 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 12:14:57 -0400 Subject: [PATCH 160/198] Fix lint --- python/tests/storage/test_mariadb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py index bc46c40be..12470bbc3 100644 --- a/python/tests/storage/test_mariadb.py +++ b/python/tests/storage/test_mariadb.py @@ -31,6 +31,7 @@ def swap(_: TaskContext, x: Int8, y: Int8) -> tuple[Int8, Int8]: @pytest.fixture def submit_job(mariadb_storage: MariaDBStorage) -> Job: + """Submits a simple job.""" graph = chain(group([double, double]), group([swap]))._impl # Fill input data for i, task_id in enumerate(graph.input_tasks): @@ -59,7 +60,7 @@ def test_job_submission(self, mariadb_storage: MariaDBStorage) -> None: assert len(jobs) == 1 @pytest.mark.storage - def test_job_status(self, mariadb_storage: MariaDBStorage, submit_job) -> None: + def test_job_status(self, mariadb_storage: MariaDBStorage, submit_job: Job) -> None: """Test job status of the MariaDB storage backend.""" status = mariadb_storage.get_job_status(submit_job) assert status == JobStatus.Running From 97fcd364ff77e4d6822eeaa8bc24b035f631cd08 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 12:46:32 -0400 Subject: [PATCH 161/198] Bug fix --- python/src/spider/storage/mariadb_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 497cef6c9..320ef177d 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -261,7 +261,7 @@ def get_job_results(self, job: core.Job) -> object: cursor.executemany(GetTaskOutputs, [(task_id,) for task_id in task_ids]) results = [] - for output_type, value, data_id in cursor: + for output_type, value, data_id in cursor.fetchall(): if value is not None: results.append( core.TaskOutput( @@ -273,7 +273,7 @@ def get_job_results(self, job: core.Job) -> object: results.append( core.TaskOutput( type=output_type, - value=core.TaskOutputValue(msgpack.unpackb(data_id)), + value=core.TaskOutputData(data_id), ) ) self._conn.commit() From 3019cb67bdb200f9fa1fdd35677f3ddf530d68cd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 12:54:04 -0400 Subject: [PATCH 162/198] Check for job existence --- python/src/spider/storage/mariadb_storage.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 320ef177d..842536155 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -233,8 +233,12 @@ def get_job_status(self, job: core.Job) -> core.JobStatus: try: with self._conn.cursor() as cursor: cursor.execute(GetJobStatus, (job.job_id.bytes,)) - job_str = cursor.fetchone()[0] - match job_str: + row = cursor.fetchone() + if row is None: + msg = f"No job found with id {job.job_id}" + raise StorageError(msg) + status_str = row[0] + match status_str: case "running": status = core.JobStatus.Running case "success": From 1a12604d803f2ea719beaa3b19020b68850b872f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 12:57:47 -0400 Subject: [PATCH 163/198] Bug fix --- python/src/spider/storage/mariadb_storage.py | 31 ++++++++++---------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 842536155..24b1b20fe 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -263,28 +263,29 @@ def get_job_results(self, job: core.Job) -> object: cursor.execute(GetOutputTasks, (job.job_id.bytes,)) task_ids = [task_id for (task_id,) in cursor.fetchall()] - cursor.executemany(GetTaskOutputs, [(task_id,) for task_id in task_ids]) results = [] - for output_type, value, data_id in cursor.fetchall(): - if value is not None: - results.append( - core.TaskOutput( - type=output_type, - value=core.TaskOutputValue(msgpack.unpackb(value)), + for task_id in task_ids: + cursor.execute(GetTaskOutputs, (task_id,)) + for output_type, value, data_id in cursor.fetchall(): + if value is not None: + results.append( + core.TaskOutput( + type=output_type, + value=core.TaskOutputValue(msgpack.unpackb(value)), + ) ) - ) - if data_id is not None: - results.append( - core.TaskOutput( - type=output_type, - value=core.TaskOutputData(data_id), + if data_id is not None: + results.append( + core.TaskOutput( + type=output_type, + value=core.TaskOutputData(data_id), + ) ) - ) self._conn.commit() return results except mariadb.Error as e: self._conn.rollback() raise StorageError(str(e)) from e - except msgpack.exceptions.UnpackValueError: + except msgpack.exceptions.UnpackException: self._conn.rollback() raise From cf280657bae9c1f95b7cc50f0c566ee6cb024a75 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 19:09:16 -0400 Subject: [PATCH 164/198] Add core.Data in task I/O --- python/src/spider/client/task.py | 6 ++--- python/src/spider/core/task.py | 6 ++--- python/src/spider/storage/mariadb_storage.py | 24 ++++++++++++++++++-- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index 6b27e5d4d..d1479db3a 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -7,7 +7,7 @@ from spider import core from spider.client.data import Data -from spider.core import TaskInput, TaskOutput, TaskOutputData, TaskOutputValue +from spider.core import TaskInput, TaskOutput, TaskOutputValue from spider.type import to_tdl_type_str @@ -72,13 +72,13 @@ def _process_return(task: core.Task, signature: inspect.Signature) -> None: for r in args: tdl_type_str = to_tdl_type_str(r) if r is Data: - task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputData())) + task.task_outputs.append(TaskOutput(tdl_type_str, core.DataId())) else: task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) else: tdl_type_str = to_tdl_type_str(returns) if returns is Data: - task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputData())) + task.task_outputs.append(TaskOutput(tdl_type_str, core.DataId())) else: task.task_outputs.append(TaskOutput(tdl_type_str, TaskOutputValue())) diff --git a/python/src/spider/core/task.py b/python/src/spider/core/task.py index 25c8ccd4f..0cd31bb9c 100644 --- a/python/src/spider/core/task.py +++ b/python/src/spider/core/task.py @@ -4,7 +4,7 @@ from enum import IntEnum from uuid import UUID, uuid4 -from spider.core.data import DataId +from spider.core.data import Data, DataId TaskId = UUID @@ -18,7 +18,7 @@ class TaskInputOutput: TaskInputValue = bytes -TaskInputData = DataId +TaskInputData = Data | DataId @dataclass @@ -30,7 +30,7 @@ class TaskInput: TaskOutputValue = bytes -TaskOutputData = DataId +TaskOutputData = Data | DataId @dataclass diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 24b1b20fe..fb8ff7a14 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -97,6 +97,14 @@ ORDER BY `position`""" +GetData = """ +SELECT + `value`, +FROM + `data` +WHERE + `id` = ?""" + class MariaDBStorage(Storage): """MariaDB Storage class.""" @@ -181,7 +189,14 @@ def submit_jobs( ], ) input_data_params = [ - (task.task_id.bytes, position, task_input.type, task_input.value.bytes) + ( + task.task_id.bytes, + position, + task_input.type, + task_input.value.id.bytes + if isinstance(task_input.value, core.Data) + else task_input.value.bytes, + ) for task_graph in task_graphs for task in task_graph.tasks.values() for position, task_input in enumerate(task.task_inputs) @@ -275,10 +290,15 @@ def get_job_results(self, job: core.Job) -> object: ) ) if data_id is not None: + cursor.execute(GetData, (data_id,)) + row = cursor.fetchone() + if row is None: + msg = f"No data found with id {data_id}" + raise StorageError(msg) results.append( core.TaskOutput( type=output_type, - value=core.TaskOutputData(data_id), + value=core.Data(data_id, row[0]), ) ) self._conn.commit() From 936fbf99e478a26cbf2731490c9f6e36f83705f4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 19:37:00 -0400 Subject: [PATCH 165/198] Use core data in client data --- python/src/spider/client/data.py | 5 +++-- python/src/spider/client/driver.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/src/spider/client/data.py b/python/src/spider/client/data.py index 479afda76..f13ab5279 100644 --- a/python/src/spider/client/data.py +++ b/python/src/spider/client/data.py @@ -1,5 +1,7 @@ """Spider client Data module.""" +from uuid import uuid4 + from spider import core @@ -8,5 +10,4 @@ class Data: def __init__(self, value: bytes) -> None: """Initialize the Data object with the given value.""" - self.data_id = core.DataId() - self.value = value + self._impl = core.Data(uuid4(), value) diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py index c719de9eb..edbb7d88f 100644 --- a/python/src/spider/client/driver.py +++ b/python/src/spider/client/driver.py @@ -53,7 +53,7 @@ def submit_jobs( arg = task_args[arg_index] if isinstance(arg, Data): task_input.type = to_tdl_type_str(Data) - task_input.value = arg.data_id + task_input.value = arg._impl.id else: task_input.type = to_tdl_type_str(type(arg)) task_input.value = core.TaskInputValue(msgpack.packb(arg)) From ab1c8b082bd4119e7304c1457cd883a157a3448e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 19:42:47 -0400 Subject: [PATCH 166/198] Add more fields to data --- python/src/spider/core/data.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/src/spider/core/data.py b/python/src/spider/core/data.py index 7a0586712..faf54df79 100644 --- a/python/src/spider/core/data.py +++ b/python/src/spider/core/data.py @@ -1,14 +1,23 @@ """Data module for Spider.""" -from dataclasses import dataclass +from dataclasses import dataclass, field from uuid import UUID DataId = UUID +@dataclass +class DataLocality: + """Represents the locality of a data object.""" + + address: str + @dataclass class Data: """Represents a data object.""" id: DataId value: bytes + localities: list[DataLocality] = field(default_factory=list) + hard_locality: bool = False + persisted: bool = False From 3822892f394cfcad2aefb76ba51f9b23edf690dd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 20:02:00 -0400 Subject: [PATCH 167/198] Add create driver data interface --- python/src/spider/storage/mariadb_storage.py | 8 ++++++++ python/src/spider/storage/storage.py | 9 +++++++++ 2 files changed, 17 insertions(+) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index fb8ff7a14..a7f76079b 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -309,3 +309,11 @@ def get_job_results(self, job: core.Job) -> object: except msgpack.exceptions.UnpackException: self._conn.rollback() raise + + @override + def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: + try: + pass + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index 4780f6d6d..d98b365db 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -48,3 +48,12 @@ def get_job_results(self, job: core.Job) -> object: :raises StorageError: If the storage operations fail. :raises msgpack.exceptions.UnPackError: If deserialization of the result fails. """ + + @abstractmethod + def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: + """ + Creates data from a driver in the storage. + :param driver_id: The driver id. + :param data: Data to create. + :raises StorageError: If the storage operations fail. + """ \ No newline at end of file From 8cd9c08c2b3f690031a9b509b37798db55c9acd1 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 20:52:56 -0400 Subject: [PATCH 168/198] Add storage function to get data --- python/src/spider/core/data.py | 1 + python/src/spider/storage/mariadb_storage.py | 55 +++++++++++++++++--- python/src/spider/storage/storage.py | 11 +++- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/python/src/spider/core/data.py b/python/src/spider/core/data.py index faf54df79..306afb2d5 100644 --- a/python/src/spider/core/data.py +++ b/python/src/spider/core/data.py @@ -12,6 +12,7 @@ class DataLocality: address: str + @dataclass class Data: """Represents a data object.""" diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index a7f76079b..784fe8143 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -97,14 +97,41 @@ ORDER BY `position`""" +InsertData = """ +INSERT INTO + `data` (`id`, `value`, `hard_locality`) +VALUES + (?, ?, ?)""" + +InsertDataLocality = """ +INSERT INTO + `data_locality` (`id`, `address`) +VALUES + (?, ?)""" + +InsertDataRefDriver = """ +INSERT INTO + `data_ref_driver` (`id`, `driver_id`) +VALUES + (?, ?)""" + GetData = """ SELECT `value`, + `hard_locality` FROM `data` WHERE `id` = ?""" +GetDataLocality = """ +SELECT + `address` +FROM + `data_locality` +WHERE + `id` = ?""" + class MariaDBStorage(Storage): """MariaDB Storage class.""" @@ -290,15 +317,11 @@ def get_job_results(self, job: core.Job) -> object: ) ) if data_id is not None: - cursor.execute(GetData, (data_id,)) - row = cursor.fetchone() - if row is None: - msg = f"No data found with id {data_id}" - raise StorageError(msg) + data = self.get_data(core.DataId(data_id)) results.append( core.TaskOutput( type=output_type, - value=core.Data(data_id, row[0]), + value=data, ) ) self._conn.commit() @@ -317,3 +340,23 @@ def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: except mariadb.Error as e: self._conn.rollback() raise StorageError(str(e)) from e + + @override + def get_data(self, data_id: core.DataId) -> core.Data: + try: + with self._conn.cursor() as cursor: + cursor.execute(GetData, (data_id.bytes,)) + row = cursor.fetchone() + if row is None: + msg = f"No data found with id {data_id}" + raise StorageError(msg) + value, hard_locality = row + data = core.Data(data_id, value, hard_locality) + cursor.execute(GetDataLocality, (data_id.bytes,)) + for (address,) in cursor.fetchall(): + data.localities.append(address) + self._conn.commit() + return data + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index d98b365db..edbc25543 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -56,4 +56,13 @@ def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: :param driver_id: The driver id. :param data: Data to create. :raises StorageError: If the storage operations fail. - """ \ No newline at end of file + """ + + @abstractmethod + def get_data(self, data_id: core.DataId) -> core.Data: + """ + Gets data from the storage. + :param data_id: + :return: The Data object associated with `data_id`. + :raises StorageError: If the storage operations fail. + """ From 77cd6ed74c349bd4441130c592dd57dec27aed3e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 20:57:58 -0400 Subject: [PATCH 169/198] Add mariadb insert driver data --- python/src/spider/storage/mariadb_storage.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 784fe8143..3847f194d 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -336,7 +336,21 @@ def get_job_results(self, job: core.Job) -> object: @override def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: try: - pass + with self._conn.cursor() as cursor: + cursor.execute( + InsertData, + (data.id.bytes, msgpack.packb(data.value), data.hard_locality), + ) + if data.localities: + cursor.executemany( + InsertDataLocality, + [(data.id.bytes, address) for address in data.localities], + ) + cursor.execute( + InsertDataRefDriver, + (data.id.bytes, driver_id.bytes), + ) + self._conn.commit() except mariadb.Error as e: self._conn.rollback() raise StorageError(str(e)) from e From c58eb8b8a9443533b2362905ab36631fcf7077b8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 21:26:03 -0400 Subject: [PATCH 170/198] Add storage driver creation; Add tests for data; Bug fixes --- python/src/spider/core/__init__.py | 3 ++- python/src/spider/storage/mariadb_storage.py | 24 ++++++++++++++++---- python/src/spider/storage/storage.py | 8 +++++++ python/tests/storage/test_mariadb.py | 23 ++++++++++++++++++- 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/python/src/spider/core/__init__.py b/python/src/spider/core/__init__.py index 2b3fa6fb2..f4fbd3a6b 100644 --- a/python/src/spider/core/__init__.py +++ b/python/src/spider/core/__init__.py @@ -1,6 +1,6 @@ """Spider core package.""" -from .data import Data, DataId +from .data import Data, DataId, DataLocality from .driver import DriverId from .job import Job, JobId, JobStatus from .task import ( @@ -22,6 +22,7 @@ __all__ = [ "Data", "DataId", + "DataLocality", "DriverId", "Job", "JobId", diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 3847f194d..7231ba564 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -132,6 +132,12 @@ WHERE `id` = ?""" +InsertDriver = """ +INSERT INTO + `drivers` (`id`) +VALUES + (?)""" + class MariaDBStorage(Storage): """MariaDB Storage class.""" @@ -339,12 +345,12 @@ def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: with self._conn.cursor() as cursor: cursor.execute( InsertData, - (data.id.bytes, msgpack.packb(data.value), data.hard_locality), + (data.id.bytes, data.value, data.hard_locality), ) if data.localities: cursor.executemany( InsertDataLocality, - [(data.id.bytes, address) for address in data.localities], + [(data.id.bytes, locality.address) for locality in data.localities], ) cursor.execute( InsertDataRefDriver, @@ -365,12 +371,22 @@ def get_data(self, data_id: core.DataId) -> core.Data: msg = f"No data found with id {data_id}" raise StorageError(msg) value, hard_locality = row - data = core.Data(data_id, value, hard_locality) + data = core.Data(id=data_id, value=value, hard_locality=hard_locality) cursor.execute(GetDataLocality, (data_id.bytes,)) for (address,) in cursor.fetchall(): - data.localities.append(address) + data.localities.append(core.DataLocality(address)) self._conn.commit() return data except mariadb.Error as e: self._conn.rollback() raise StorageError(str(e)) from e + + @override + def create_driver(self, driver_id: core.DriverId) -> None: + try: + with self._conn.cursor() as cursor: + cursor.execute(InsertDriver, (driver_id.bytes,)) + self._conn.commit() + except mariadb.Error as e: + self._conn.rollback() + raise StorageError(str(e)) from e diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index edbc25543..2bc0e1be5 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -66,3 +66,11 @@ def get_data(self, data_id: core.DataId) -> core.Data: :return: The Data object associated with `data_id`. :raises StorageError: If the storage operations fail. """ + + @abstractmethod + def create_driver(self, driver_id: core.DriverId) -> None: + """ + Creates a driver in the storage. + :param driver_id: + :raises StorageError: If the storage operations fail. + """ diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py index 12470bbc3..2c27d0905 100644 --- a/python/tests/storage/test_mariadb.py +++ b/python/tests/storage/test_mariadb.py @@ -6,7 +6,7 @@ import pytest from spider import chain, group, Int8, TaskContext -from spider.core import Job, JobStatus, TaskInputValue +from spider.core import Data, DataLocality, DriverId, Job, JobStatus, TaskInputValue from spider.storage import MariaDBStorage, parse_jdbc_url MariaDBTestUrl = "jdbc:mariadb://127.0.0.1:3306/spider-storage?user=spider&password=password" @@ -43,6 +43,14 @@ def submit_job(mariadb_storage: MariaDBStorage) -> Job: return jobs[0] +@pytest.fixture +def driver(mariadb_storage: MariaDBStorage) -> DriverId: + """Fixture to create a driver.""" + driver_id = uuid4() + mariadb_storage.create_driver(driver_id) + return driver_id + + class TestMariaDBStorage: """Test class for the MariaDB storage backend.""" @@ -64,3 +72,16 @@ def test_job_status(self, mariadb_storage: MariaDBStorage, submit_job: Job) -> N """Test job status of the MariaDB storage backend.""" status = mariadb_storage.get_job_status(submit_job) assert status == JobStatus.Running + + @pytest.mark.storage + def test_data(self, mariadb_storage: MariaDBStorage, driver: DriverId) -> None: + """Test data storage and retrieval.""" + value = b"test data" + data = Data(id=uuid4(), value=value, localities=[DataLocality("localhost")]) + mariadb_storage.create_driver_data(driver, data) + retrieved_data = mariadb_storage.get_data(data.id) + assert retrieved_data is not None + assert retrieved_data.id == data.id + assert retrieved_data.value == value + assert retrieved_data.hard_locality == data.hard_locality + assert retrieved_data.localities == data.localities From 7aefb7bac4aab831c329f1e849c01fd556c54ccb Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 22:26:29 -0400 Subject: [PATCH 171/198] Add client data, driver and job --- python/src/spider/__init__.py | 2 ++ python/src/spider/client/data.py | 18 ++++++++++++++++++ python/src/spider/client/driver.py | 17 +++++++++++++---- python/src/spider/client/job.py | 28 +++++++++++++++++++++++++++- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py index c03ec9172..3b6c79b7c 100644 --- a/python/src/spider/__init__.py +++ b/python/src/spider/__init__.py @@ -1,6 +1,7 @@ """Spider package root.""" from spider.client import chain, Data, Driver, group, TaskContext, TaskGraph +from spider.core import JobStatus from spider.type import Double, Float, Int8, Int16, Int32, Int64 __all__ = [ @@ -12,6 +13,7 @@ "Int16", "Int32", "Int64", + "JobStatus", "TaskContext", "TaskGraph", "chain", diff --git a/python/src/spider/client/data.py b/python/src/spider/client/data.py index f13ab5279..ba6677f14 100644 --- a/python/src/spider/client/data.py +++ b/python/src/spider/client/data.py @@ -11,3 +11,21 @@ class Data: def __init__(self, value: bytes) -> None: """Initialize the Data object with the given value.""" self._impl = core.Data(uuid4(), value) + + @property + def value(self) -> bytes: + """Property to get the value of the data.""" + return self._impl.value + + @property + def hard_locality(self) -> bool: + """Property to check if the data has hard locality.""" + return self._impl.hard_locality + + def get_localities(self) -> list[str]: + """Gets the list of localities where the data is stored.""" + return [locality.address for locality in self._impl.localities] + + def add_locality(self, address: str) -> None: + """Adds a new locality to the data.""" + self._impl.localities.append(core.DataLocality(address)) diff --git a/python/src/spider/client/driver.py b/python/src/spider/client/driver.py index edbb7d88f..95731f185 100644 --- a/python/src/spider/client/driver.py +++ b/python/src/spider/client/driver.py @@ -22,8 +22,9 @@ def __init__(self, storage_url: str) -> None: :param storage_url: The URL of the storage to connect to. :raises StorageError: If the storage cannot be connected to. """ - self.driver_id = uuid4() - self.storage = MariaDBStorage(parse_jdbc_url(storage_url)) + self._driver_id = uuid4() + self._storage = MariaDBStorage(parse_jdbc_url(storage_url)) + self._storage.create_driver(self._driver_id) def submit_jobs( self, graphs: Sequence[TaskGraph], args: Sequence[Sequence[object]] @@ -61,5 +62,13 @@ def submit_jobs( if arg_index != len(task_args): raise ValueError(msg) - jobs = self.storage.submit_jobs(self.driver_id, [graph._impl for graph in graphs]) - return [Job(job, self.storage) for job in jobs] + jobs = self._storage.submit_jobs(self._driver_id, [graph._impl for graph in graphs]) + return [Job(job, self._storage) for job in jobs] + + def create_data(self, data: Data) -> None: + """ + Registers a Data object in the storage. + :param data: The Data object to register. + :raises StorageError: If the Data object cannot be registered in the storage. + """ + self._storage.create_driver_data(self._driver_id, data._impl) diff --git a/python/src/spider/client/job.py b/python/src/spider/client/job.py index 0ea67e55a..83dcd0310 100644 --- a/python/src/spider/client/job.py +++ b/python/src/spider/client/job.py @@ -14,4 +14,30 @@ def __init__(self, job: core.Job, storage: Storage) -> None: :param storage: The storage backend. """ self._impl = job - self.storage = storage + self._storage = storage + + def get_status(self) -> core.JobStatus: + """ + :return: The current job status. + :raises StorageError: If there was an error retrieving the job status from storage. + """ + if self._impl.status != core.JobStatus.Running: + return self._impl.status + + status = self._storage.get_job_status(self._impl) + self._impl.status = status + return status + + def get_results(self) -> object | None: + """ + :return: The job results or None if the status is not Running. + :raises StorageError: If there was an error retrieving the job results from storage. + :raises msgpack.exceptions.UnpackException: If there was an error deserializing the job + results. + """ + if self._impl.results is not None: + return self._impl.results + + results = self._storage.get_job_results(self._impl) + self._impl.results = results + return results From 710f42d57d7f3d0cc6b6e9a4ade96f5c96716b94 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 22:30:24 -0400 Subject: [PATCH 172/198] Export job --- python/src/spider/__init__.py | 3 ++- python/src/spider/client/__init__.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py index 3b6c79b7c..4f52adb28 100644 --- a/python/src/spider/__init__.py +++ b/python/src/spider/__init__.py @@ -1,6 +1,6 @@ """Spider package root.""" -from spider.client import chain, Data, Driver, group, TaskContext, TaskGraph +from spider.client import chain, Data, Driver, Job, group, TaskContext, TaskGraph from spider.core import JobStatus from spider.type import Double, Float, Int8, Int16, Int32, Int64 @@ -8,6 +8,7 @@ "Data", "Double", "Driver", + "Job", "Float", "Int8", "Int16", diff --git a/python/src/spider/client/__init__.py b/python/src/spider/client/__init__.py index 2dad4bb67..cb8a477b5 100644 --- a/python/src/spider/client/__init__.py +++ b/python/src/spider/client/__init__.py @@ -2,12 +2,14 @@ from .data import Data from .driver import Driver +from .job import Job from .task import TaskContext from .taskgraph import chain, group, TaskGraph __all__ = [ "Data", "Driver", + "Job", "TaskContext", "TaskGraph", "chain", From 5f915cd7a0627bfabba5c7695edfcd763befeaa7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 22:48:19 -0400 Subject: [PATCH 173/198] Add data tests and bug fixes --- python/src/spider/__init__.py | 4 ++-- python/src/spider/client/data.py | 5 +++++ python/tests/client/test_data.py | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 python/tests/client/test_data.py diff --git a/python/src/spider/__init__.py b/python/src/spider/__init__.py index 4f52adb28..d5264f86c 100644 --- a/python/src/spider/__init__.py +++ b/python/src/spider/__init__.py @@ -1,6 +1,6 @@ """Spider package root.""" -from spider.client import chain, Data, Driver, Job, group, TaskContext, TaskGraph +from spider.client import chain, Data, Driver, group, Job, TaskContext, TaskGraph from spider.core import JobStatus from spider.type import Double, Float, Int8, Int16, Int32, Int64 @@ -8,12 +8,12 @@ "Data", "Double", "Driver", - "Job", "Float", "Int8", "Int16", "Int32", "Int64", + "Job", "JobStatus", "TaskContext", "TaskGraph", diff --git a/python/src/spider/client/data.py b/python/src/spider/client/data.py index ba6677f14..04c89b8fc 100644 --- a/python/src/spider/client/data.py +++ b/python/src/spider/client/data.py @@ -22,6 +22,11 @@ def hard_locality(self) -> bool: """Property to check if the data has hard locality.""" return self._impl.hard_locality + @hard_locality.setter + def hard_locality(self, value: bool) -> None: + """Sets the hard locality for the data.""" + self._impl.hard_locality = value + def get_localities(self) -> list[str]: """Gets the list of localities where the data is stored.""" return [locality.address for locality in self._impl.localities] diff --git a/python/tests/client/test_data.py b/python/tests/client/test_data.py new file mode 100644 index 000000000..d4f23080c --- /dev/null +++ b/python/tests/client/test_data.py @@ -0,0 +1,23 @@ +"""Tests for client data module.""" + +import pytest +from test_driver import driver # noqa: F401 + +from spider import Data, Driver + + +class TestData: + """Test data class for client data module.""" + + @pytest.mark.storage + def test_data(self, driver: Driver) -> None: # noqa: F811 + """Tests data creation""" + data = Data(b"test_data") + data.hard_locality = True + data.add_locality("localhost") + + assert data.value == b"test_data" + assert data.hard_locality is True + assert data.get_localities() == ["localhost"] + + driver.create_data(data) From 87606a12f0ebc6d5cd3124f0b2f019db4c63d029 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 23:05:13 -0400 Subject: [PATCH 174/198] Update get job result interface --- python/src/spider/storage/mariadb_storage.py | 6 ++++-- python/src/spider/storage/storage.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 24b1b20fe..8cff30e1e 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -257,7 +257,7 @@ def get_job_status(self, job: core.Job) -> core.JobStatus: raise StorageError(str(e)) from e @override - def get_job_results(self, job: core.Job) -> object: + def get_job_results(self, job: core.Job) -> list[core.TaskOutput] | None: try: with self._conn.cursor() as cursor: cursor.execute(GetOutputTasks, (job.job_id.bytes,)) @@ -274,13 +274,15 @@ def get_job_results(self, job: core.Job) -> object: value=core.TaskOutputValue(msgpack.unpackb(value)), ) ) - if data_id is not None: + elif data_id is not None: results.append( core.TaskOutput( type=output_type, value=core.TaskOutputData(data_id), ) ) + else: + return None self._conn.commit() return results except mariadb.Error as e: diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index 4780f6d6d..d649e5d43 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -40,11 +40,11 @@ def get_job_status(self, job: core.Job) -> JobStatus: """ @abstractmethod - def get_job_results(self, job: core.Job) -> object: + def get_job_results(self, job: core.Job) -> list[core.TaskOutput] | None: """ Gets the job's results. This function does not set the `results` field in the job. :param job: - :return: + :return: List of task outputs or None if the job has no results. :raises StorageError: If the storage operations fail. :raises msgpack.exceptions.UnPackError: If deserialization of the result fails. """ From 2eb9a096f835a8c3fd7169709c639a9cc7420665 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 23:08:35 -0400 Subject: [PATCH 175/198] Add tests for getting results of a running job --- python/tests/storage/test_mariadb.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py index 12470bbc3..27402e736 100644 --- a/python/tests/storage/test_mariadb.py +++ b/python/tests/storage/test_mariadb.py @@ -48,7 +48,7 @@ class TestMariaDBStorage: @pytest.mark.storage def test_job_submission(self, mariadb_storage: MariaDBStorage) -> None: - """Test job submission to the MariaDB storage backend.""" + """Tests job submission to the MariaDB storage backend.""" graph = chain(group([double, double, double, double]), group([swap, swap]))._impl # Fill input data for i, task_id in enumerate(graph.input_tasks): @@ -60,7 +60,13 @@ def test_job_submission(self, mariadb_storage: MariaDBStorage) -> None: assert len(jobs) == 1 @pytest.mark.storage - def test_job_status(self, mariadb_storage: MariaDBStorage, submit_job: Job) -> None: - """Test job status of the MariaDB storage backend.""" + def test_running_job_status(self, mariadb_storage: MariaDBStorage, submit_job: Job) -> None: + """Tests getting status of a running job.""" status = mariadb_storage.get_job_status(submit_job) assert status == JobStatus.Running + + @pytest.mark.storage + def test_running_job_result(self, mariadb_storage: MariaDBStorage, submit_job: Job) -> None: + """Tests getting results of a running job.""" + results = mariadb_storage.get_job_results(submit_job) + assert results is None From f234241ebb8e1c35637df0b49b17882d828f6db8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 23:17:56 -0400 Subject: [PATCH 176/198] Do not deserialize value in getting result --- python/src/spider/storage/mariadb_storage.py | 6 +----- python/src/spider/storage/storage.py | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 8cff30e1e..7ff6d916c 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -4,7 +4,6 @@ from uuid import uuid4 import mariadb -import msgpack from typing_extensions import override from spider import core @@ -271,7 +270,7 @@ def get_job_results(self, job: core.Job) -> list[core.TaskOutput] | None: results.append( core.TaskOutput( type=output_type, - value=core.TaskOutputValue(msgpack.unpackb(value)), + value=core.TaskOutputValue(value), ) ) elif data_id is not None: @@ -288,6 +287,3 @@ def get_job_results(self, job: core.Job) -> list[core.TaskOutput] | None: except mariadb.Error as e: self._conn.rollback() raise StorageError(str(e)) from e - except msgpack.exceptions.UnpackException: - self._conn.rollback() - raise diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index d649e5d43..1ed21cc6e 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -46,5 +46,4 @@ def get_job_results(self, job: core.Job) -> list[core.TaskOutput] | None: :param job: :return: List of task outputs or None if the job has no results. :raises StorageError: If the storage operations fail. - :raises msgpack.exceptions.UnPackError: If deserialization of the result fails. """ From 25d7012357f3ab03a6ac8e89df80d79f0060c399 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 23:32:15 -0400 Subject: [PATCH 177/198] Fix type annotation --- python/src/spider/core/job.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/src/spider/core/job.py b/python/src/spider/core/job.py index a7c3ad7a5..851149e7f 100644 --- a/python/src/spider/core/job.py +++ b/python/src/spider/core/job.py @@ -1,8 +1,12 @@ """Job module for Spider.""" from enum import IntEnum +from typing import TYPE_CHECKING from uuid import UUID +if TYPE_CHECKING: + from spider.core.task import TaskOutput + JobId = UUID @@ -27,4 +31,4 @@ def __init__(self, job_id: JobId) -> None: """ self.job_id = job_id self.status = JobStatus.Running - self.results: object | None = None + self.results: list[TaskOutput] | None = None From 563a337e219f04eba2dfa2e8e354ac9d4805fc5f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 23:36:07 -0400 Subject: [PATCH 178/198] Add job tests and fix client get results --- python/pyproject.toml | 2 +- python/src/spider/client/data.py | 7 +++++++ python/src/spider/client/job.py | 33 ++++++++++++++++++++++++++++---- python/tests/client/test_job.py | 30 +++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 python/tests/client/test_job.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 04860844d..8c846c944 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -83,7 +83,7 @@ isort.order-by-type = false ] [tool.ruff.lint.flake8-self] -ignore-names = ["_impl"] +ignore-names = ["_from_impl", "_impl"] [tool.ruff.lint.pydocstyle] ignore-decorators = ["typing.override"] diff --git a/python/src/spider/client/data.py b/python/src/spider/client/data.py index 04c89b8fc..bf274c661 100644 --- a/python/src/spider/client/data.py +++ b/python/src/spider/client/data.py @@ -12,6 +12,13 @@ def __init__(self, value: bytes) -> None: """Initialize the Data object with the given value.""" self._impl = core.Data(uuid4(), value) + @staticmethod + def _from_impl(impl: core.Data) -> "Data": + """Creates a Data instance from an existing core.Data implementation.""" + data = Data(impl.value) + data._impl = impl + return data + @property def value(self) -> bytes: """Property to get the value of the data.""" diff --git a/python/src/spider/client/job.py b/python/src/spider/client/job.py index 83dcd0310..9ba8ccd32 100644 --- a/python/src/spider/client/job.py +++ b/python/src/spider/client/job.py @@ -1,7 +1,31 @@ """Spider job module.""" +import msgpack + from spider import core -from spider.storage import Storage +from spider.client.data import Data +from spider.storage import Storage, StorageError + + +def _convert_outputs(outputs: list[core.TaskOutput]) -> tuple[object, ...]: + """ + Converts a list of TaskOutput objects to a tuple of their values. + :param outputs: The list of TaskOutput objects. + :return: A tuple containing the values of the TaskOutput objects. + :raises msgpack.exceptions.UnpackException: If there was an error deserializing the TaskOutput + values. + :raises StorageError: If there was an error in the TaskOutput values. + """ + results = [] + for output in outputs: + if isinstance(output.value, core.TaskOutputValue): + results.append(msgpack.unpackb(output.value)) + elif isinstance(output, core.Data): + results.append(Data._from_impl(output.value)) + else: + msg = "Fail to get data from storage." + raise StorageError(msg) + return tuple(results) class Job: @@ -36,8 +60,9 @@ def get_results(self) -> object | None: results. """ if self._impl.results is not None: - return self._impl.results + return _convert_outputs(self._impl.results) results = self._storage.get_job_results(self._impl) - self._impl.results = results - return results + if results is None: + return None + return _convert_outputs(results) diff --git a/python/tests/client/test_job.py b/python/tests/client/test_job.py new file mode 100644 index 000000000..013e533e4 --- /dev/null +++ b/python/tests/client/test_job.py @@ -0,0 +1,30 @@ +"""Tests for client job module.""" + +import pytest +from test_driver import driver # noqa: F401 +from test_task_graph import double, swap + +from spider import chain, Driver, group, Int8, JobStatus + + +class TestJob: + """Tests for client job module.""" + + @pytest.mark.storage + def test_job(self, driver: Driver) -> None: # noqa: F811 + """Test getting running job status and results.""" + jobs = driver.submit_jobs( + [ + group([double, double]), + chain(group([double, double]), swap), + ], + [ + (Int8(1), Int8(2)), + (Int8(1), Int8(2)), + ], + ) + + assert jobs[0].get_status() == JobStatus.Running + assert jobs[1].get_status() == JobStatus.Running + assert jobs[0].get_results() is None + assert jobs[1].get_results() is None From 4765d5dd131810b7f10ed51b46e9773775c8ee32 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 14 Aug 2025 23:42:24 -0400 Subject: [PATCH 179/198] Add missing commit --- python/src/spider/storage/mariadb_storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 7ff6d916c..6eafe2750 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -281,6 +281,7 @@ def get_job_results(self, job: core.Job) -> list[core.TaskOutput] | None: ) ) else: + self._conn.commit() return None self._conn.commit() return results From 0cd522ced540b411c26e5cfd466bd284ce464896 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 15 Aug 2025 00:10:05 -0400 Subject: [PATCH 180/198] Bug fix --- python/src/spider/client/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/client/job.py b/python/src/spider/client/job.py index 9ba8ccd32..5babebcb7 100644 --- a/python/src/spider/client/job.py +++ b/python/src/spider/client/job.py @@ -20,7 +20,7 @@ def _convert_outputs(outputs: list[core.TaskOutput]) -> tuple[object, ...]: for output in outputs: if isinstance(output.value, core.TaskOutputValue): results.append(msgpack.unpackb(output.value)) - elif isinstance(output, core.Data): + elif isinstance(output.value, core.Data): results.append(Data._from_impl(output.value)) else: msg = "Fail to get data from storage." From dfac70915287988e2c2eafb4129cd2288ffdb017 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 15 Aug 2025 00:14:10 -0400 Subject: [PATCH 181/198] Fix docstring --- python/tests/client/test_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/client/test_data.py b/python/tests/client/test_data.py index d4f23080c..5ea7d9b43 100644 --- a/python/tests/client/test_data.py +++ b/python/tests/client/test_data.py @@ -11,7 +11,7 @@ class TestData: @pytest.mark.storage def test_data(self, driver: Driver) -> None: # noqa: F811 - """Tests data creation""" + """Tests data creation.""" data = Data(b"test_data") data.hard_locality = True data.add_locality("localhost") From 4d7fc446d460240d396175e331cee13c8a3d6794 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 17 Aug 2025 20:28:16 -0400 Subject: [PATCH 182/198] Add create task data --- python/src/spider/storage/mariadb_storage.py | 30 ++++++++++++++++---- python/src/spider/storage/storage.py | 9 ++++++ 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index 7e73809b2..a8ecf5e82 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -1,7 +1,7 @@ """MariaDB Storage module.""" from collections.abc import Sequence -from uuid import uuid4 +from uuid import uuid4, UUID import mariadb from typing_extensions import override @@ -114,6 +114,12 @@ VALUES (?, ?)""" +InsertDataRefTask = """ +INSERT INTO + `data_ref_task` (`id`, `task_id`) +VALUES + (?, ?)""" + GetData = """ SELECT `value`, @@ -338,8 +344,14 @@ def get_job_results(self, job: core.Job) -> list[core.TaskOutput] | None: self._conn.rollback() raise StorageError(str(e)) from e - @override - def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: + def _create_data_with_ref(self, data: core.Data, insert_ref: str, ref_id: UUID) -> None: + """ + Inserts data object into the database with reference. + :param data: The data object to insert. + :param insert_ref: The SQL statement to insert the reference. + :param ref_id: The reference ID (DriverId or TaskId). + :raises StorageError: If the insertion fails. + """ try: with self._conn.cursor() as cursor: cursor.execute( @@ -352,14 +364,22 @@ def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: [(data.id.bytes, locality.address) for locality in data.localities], ) cursor.execute( - InsertDataRefDriver, - (data.id.bytes, driver_id.bytes), + insert_ref, + (data.id.bytes, ref_id.bytes), ) self._conn.commit() except mariadb.Error as e: self._conn.rollback() raise StorageError(str(e)) from e + @override + def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: + self._create_data_with_ref(data,InsertDataRefDriver, driver_id) + + @override + def create_task_data(self, task_id: core.TaskId, data: core.Data) -> None: + self._create_data_with_ref(data, InsertDataRefTask, task_id) + @override def get_data(self, data_id: core.DataId) -> core.Data: try: diff --git a/python/src/spider/storage/storage.py b/python/src/spider/storage/storage.py index a3a1b1d19..6cf852e12 100644 --- a/python/src/spider/storage/storage.py +++ b/python/src/spider/storage/storage.py @@ -57,6 +57,15 @@ def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: :raises StorageError: If the storage operations fail. """ + @abstractmethod + def create_task_data(self, task_id: core.TaskId, data: core.Data) -> None: + """ + Creates data from a task in the storage. + :param task_id: The task id. + :param data: Data to create. + :raises StorageError: If the storage operations fail. + """ + @abstractmethod def get_data(self, data_id: core.DataId) -> core.Data: """ From 8eef082680c90e4515975baa60cd01912662b0a8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 17 Aug 2025 20:33:03 -0400 Subject: [PATCH 183/198] Add unit test for creating data with no driver id --- python/tests/storage/test_mariadb.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/tests/storage/test_mariadb.py b/python/tests/storage/test_mariadb.py index 657faddd9..7ec909a10 100644 --- a/python/tests/storage/test_mariadb.py +++ b/python/tests/storage/test_mariadb.py @@ -7,7 +7,7 @@ from spider import chain, group, Int8, TaskContext from spider.core import Data, DataLocality, DriverId, Job, JobStatus, TaskInputValue -from spider.storage import MariaDBStorage, parse_jdbc_url +from spider.storage import MariaDBStorage, parse_jdbc_url, StorageError MariaDBTestUrl = "jdbc:mariadb://127.0.0.1:3306/spider-storage?user=spider&password=password" @@ -81,7 +81,7 @@ def test_running_job_result(self, mariadb_storage: MariaDBStorage, submit_job: J @pytest.mark.storage def test_data(self, mariadb_storage: MariaDBStorage, driver: DriverId) -> None: - """Test data storage and retrieval.""" + """Tests data storage and retrieval.""" value = b"test data" data = Data(id=uuid4(), value=value, localities=[DataLocality("localhost")]) mariadb_storage.create_driver_data(driver, data) @@ -91,3 +91,11 @@ def test_data(self, mariadb_storage: MariaDBStorage, driver: DriverId) -> None: assert retrieved_data.value == value assert retrieved_data.hard_locality == data.hard_locality assert retrieved_data.localities == data.localities + + @pytest.mark.storage + def test_create_data_fail(self, mariadb_storage: MariaDBStorage) -> None: + """Tests creating data without a driver fails.""" + value = b"test data" + data = Data(id=uuid4(), value=value, localities=[DataLocality("localhost")]) + with pytest.raises(StorageError): + mariadb_storage.create_driver_data(uuid4(), data) From e2f7a3824136689ca036da53982e8e282f4388d6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sun, 17 Aug 2025 20:44:14 -0400 Subject: [PATCH 184/198] Add task context impl --- python/src/spider/client/__init__.py | 2 +- python/src/spider/client/task.py | 8 +----- python/src/spider/client/task_context.py | 26 ++++++++++++++++++++ python/src/spider/storage/mariadb_storage.py | 4 +-- 4 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 python/src/spider/client/task_context.py diff --git a/python/src/spider/client/__init__.py b/python/src/spider/client/__init__.py index cb8a477b5..b0405dcdc 100644 --- a/python/src/spider/client/__init__.py +++ b/python/src/spider/client/__init__.py @@ -3,7 +3,7 @@ from .data import Data from .driver import Driver from .job import Job -from .task import TaskContext +from .task_context import TaskContext from .taskgraph import chain, group, TaskGraph __all__ = [ diff --git a/python/src/spider/client/task.py b/python/src/spider/client/task.py index d1479db3a..ecc450648 100644 --- a/python/src/spider/client/task.py +++ b/python/src/spider/client/task.py @@ -7,16 +7,10 @@ from spider import core from spider.client.data import Data +from spider.client.task_context import TaskContext from spider.core import TaskInput, TaskOutput, TaskOutputValue from spider.type import to_tdl_type_str - -class TaskContext: - """Spider task context.""" - - # TODO: Implement task context for use in task executor - - # Check the TaskFunction signature at runtime. # Enforcing static check for first argument requires the use of Protocol. However, functions, which # are Callable, are not considered a Protocol without explicit cast. diff --git a/python/src/spider/client/task_context.py b/python/src/spider/client/task_context.py new file mode 100644 index 000000000..cf1aaf58f --- /dev/null +++ b/python/src/spider/client/task_context.py @@ -0,0 +1,26 @@ +"""Spider client task context module.""" + +from spider import core, storage +from spider.client.data import Data + + +class TaskContext: + """Spider task context provides access to the task id and data creation for tasks.""" + + def __init__(self, task_id: core.TaskId, storage: storage.Storage) -> None: + """Initializes the task context.""" + self._task_id = task_id + self._storage = storage + + @property + def task_id(self) -> core.TaskId: + """:return: The task id.""" + return self._task_id + + def create_data(self, data: Data) -> None: + """ + Creates a new data object in the storage associated with the task. + :param data: The data object to be created. + :raises StorageError: If there is an error during storage operation. + """ + self._storage.create_task_data(self._task_id, data._impl) diff --git a/python/src/spider/storage/mariadb_storage.py b/python/src/spider/storage/mariadb_storage.py index a8ecf5e82..b5ea27c92 100644 --- a/python/src/spider/storage/mariadb_storage.py +++ b/python/src/spider/storage/mariadb_storage.py @@ -1,7 +1,7 @@ """MariaDB Storage module.""" from collections.abc import Sequence -from uuid import uuid4, UUID +from uuid import UUID, uuid4 import mariadb from typing_extensions import override @@ -374,7 +374,7 @@ def _create_data_with_ref(self, data: core.Data, insert_ref: str, ref_id: UUID) @override def create_driver_data(self, driver_id: core.DriverId, data: core.Data) -> None: - self._create_data_with_ref(data,InsertDataRefDriver, driver_id) + self._create_data_with_ref(data, InsertDataRefDriver, driver_id) @override def create_task_data(self, task_id: core.TaskId, data: core.Data) -> None: From 8a383c515418a0519d1efe71e14dac9911a936de Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 18 Aug 2025 10:27:43 -0400 Subject: [PATCH 185/198] Add argument parsing --- python/src/spider/task_executor/task_executor.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/python/src/spider/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py index c8a0593d3..6e7c08a4c 100644 --- a/python/src/spider/task_executor/task_executor.py +++ b/python/src/spider/task_executor/task_executor.py @@ -1,8 +1,24 @@ """Executes a Spider Python task.""" +import argparse + + +def parse_args() -> argparse.Namespace: + """ + Parses task executor arguments. + :return: Parsed arguments. + """ + parser = argparse.ArgumentParser() + parser.add_argument("func", type=str, required=True) + parser.add_argument("libs", nargs="+", type=str, required=True) + parser.add_argument("task_id", type=str, required=True) + parser.add_argument("storage_url", type=str, required=True) + return parser.parse_args() + def main() -> None: """Main function to execute the task.""" + args = parse_args() if __name__ == "__main__": From e78e4c47163951095e28a42db5df78152b75ed6e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 18 Aug 2025 11:22:20 -0400 Subject: [PATCH 186/198] Fix argparse --- .../src/spider/task_executor/task_executor.py | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/python/src/spider/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py index 6e7c08a4c..532b050d8 100644 --- a/python/src/spider/task_executor/task_executor.py +++ b/python/src/spider/task_executor/task_executor.py @@ -1,6 +1,13 @@ """Executes a Spider Python task.""" import argparse +import logging +from uuid import UUID + +from src.spider.storage import MariaDBStorage + +# Set up logger +logger = logging.getLogger(__name__) def parse_args() -> argparse.Namespace: @@ -9,16 +16,32 @@ def parse_args() -> argparse.Namespace: :return: Parsed arguments. """ parser = argparse.ArgumentParser() - parser.add_argument("func", type=str, required=True) - parser.add_argument("libs", nargs="+", type=str, required=True) - parser.add_argument("task_id", type=str, required=True) - parser.add_argument("storage_url", type=str, required=True) + parser.add_argument("--func", type=str, required=True) + parser.add_argument("--libs", nargs="+", type=str, required=True) + parser.add_argument("--storage_url", type=str, required=True) + parser.add_argument("--task_id", type=str, required=True) + parser.add_argument("--input-pipe", type=int, required=True) + parser.add_argument("--output-pipe", type=int, required=True) return parser.parse_args() def main() -> None: - """Main function to execute the task.""" + """ + Main function to execute the task. + :raises StorageError: If a storage operation fails. + """ + # Parses arguments args = parse_args() + func = args.func + libs = args.libs + task_id = args.task_id + task_id = UUID(task_id) + storage_url = args.storage_url + + logger.debug(f"Function to run: {func}") + + # Sets up storage + storage = MariaDBStorage(storage_url) if __name__ == "__main__": From 168683b0a9f14fd5c3191bb566ea7a5a573232dd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 18 Aug 2025 13:02:44 -0400 Subject: [PATCH 187/198] Fix ruff --- .../src/spider/task_executor/task_executor.py | 33 +++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/python/src/spider/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py index 532b050d8..1dfea8aeb 100644 --- a/python/src/spider/task_executor/task_executor.py +++ b/python/src/spider/task_executor/task_executor.py @@ -2,9 +2,11 @@ import argparse import logging +from io import BufferedReader +from os import fdopen from uuid import UUID -from src.spider.storage import MariaDBStorage +from spider.storage import MariaDBStorage # Set up logger logger = logging.getLogger(__name__) @@ -25,6 +27,27 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() +HeaderSize = 16 + + +def receive_message(pipe: BufferedReader) -> bytes: + """ + Receives message from the pipe with a size header. + :param pipe: Pipe to receive message from. + :return: Received message body. + :raises IOError: If read from pipe fails. + :raises UnicodeDecodeError: If parsing header fails. + :raises EOFError: If the message body size does not match header size. + """ + body_size_str = pipe.read(HeaderSize).decode() + body_size = int(body_size_str, base=10) + body = pipe.read(body_size) + if len(body) != body_size: + msg = "Received message body size does not match header size." + raise EOFError(msg) + return body + + def main() -> None: """ Main function to execute the task. @@ -37,12 +60,18 @@ def main() -> None: task_id = args.task_id task_id = UUID(task_id) storage_url = args.storage_url + input_pipe = args.input_pipe + output_pipe = args.output_pipe - logger.debug(f"Function to run: {func}") + logger.debug("Function to run: %s", func) # Sets up storage storage = MariaDBStorage(storage_url) + input_pipe = fdopen(input_pipe, "rb") + output_pipe = fdopen(output_pipe, "wb") + input_data = receive_message(input_pipe) + if __name__ == "__main__": main() From be731cb26fe36f932ed55c42a1966b3ea7f633fe Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 18 Aug 2025 22:37:46 -0400 Subject: [PATCH 188/198] Add task executor message --- .../task_executor/task_executor_message.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 python/src/spider/task_executor/task_executor_message.py diff --git a/python/src/spider/task_executor/task_executor_message.py b/python/src/spider/task_executor/task_executor_message.py new file mode 100644 index 000000000..c01eca870 --- /dev/null +++ b/python/src/spider/task_executor/task_executor_message.py @@ -0,0 +1,22 @@ +"""Task executor message module.""" + +from enum import IntEnum + + +class TaskExecutorResponseType(IntEnum): + """Task executor response type.""" + + Unknown = 0 + Result = 1 + Error = 2 + Block = 3 + Ready = 4 + Cancel = 5 + +class TaskExecutorRequestType(IntEnum): + """Task executor request type.""" + + Unknown = 0 + Arguments = 1 + Resume = 2 + From 34a314b34404b810078f41e522bb913363ba1b98 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 19 Aug 2025 15:21:57 -0400 Subject: [PATCH 189/198] WIP --- .../src/spider/task_executor/task_executor.py | 25 ++++++++++++++-- .../task_executor/task_executor_message.py | 29 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/python/src/spider/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py index 1dfea8aeb..3ebd90d0b 100644 --- a/python/src/spider/task_executor/task_executor.py +++ b/python/src/spider/task_executor/task_executor.py @@ -1,12 +1,17 @@ """Executes a Spider Python task.""" import argparse +import importlib +import inspect import logging from io import BufferedReader from os import fdopen from uuid import UUID from spider.storage import MariaDBStorage +from spider.task_executor.task_executor_message import get_request_body + +from spider.client import TaskContext # Set up logger logger = logging.getLogger(__name__) @@ -56,7 +61,6 @@ def main() -> None: # Parses arguments args = parse_args() func = args.func - libs = args.libs task_id = args.task_id task_id = UUID(task_id) storage_url = args.storage_url @@ -70,7 +74,24 @@ def main() -> None: input_pipe = fdopen(input_pipe, "rb") output_pipe = fdopen(output_pipe, "wb") - input_data = receive_message(input_pipe) + input_message = receive_message(input_pipe) + arguments = get_request_body(input_message) + logger.debug("Args buffer parsed") + + # Get the function to run + module_name, function_name = func.rsplit(".", 1) + module = importlib.import_module(module_name) + function = getattr(module, function_name) + logger.debug("Function %s imported from module %s", function_name, module_name) + + signature = inspect.signature(function) + if len(signature.parameters) != len(arguments) + 1: + msg = ( + f"Function {function_name} expects {len(signature.parameters) - 1} " + f"arguments, but {len(arguments)} were provided." + ) + raise ValueError(msg) + task_context = TaskContext(task_id, storage) if __name__ == "__main__": diff --git a/python/src/spider/task_executor/task_executor_message.py b/python/src/spider/task_executor/task_executor_message.py index c01eca870..a4f2c51a0 100644 --- a/python/src/spider/task_executor/task_executor_message.py +++ b/python/src/spider/task_executor/task_executor_message.py @@ -2,6 +2,8 @@ from enum import IntEnum +import msgpack + class TaskExecutorResponseType(IntEnum): """Task executor response type.""" @@ -20,3 +22,30 @@ class TaskExecutorRequestType(IntEnum): Arguments = 1 Resume = 2 +class InvalidRequestTypeError(Exception): + """Exception raised for invalid request types.""" + + def __init__(self, message: str): + super().__init__(message) + +def get_request_body(message: bytes) -> list[object]: + """ + Gets the request body from the request message. + :param message: The msgpack serialized request message. + :return: The request body as a list of objects. + :raises msgpack.exceptions.UnpackException: If the data is not a valid msgpack serialized list. + :raises TypeError: If the data is not a msgpack list or the list is too short. + :raises InvalidRequestTypeError: If the message header is not an `Arguments`. + """ + data = msgpack.unpackb(message) + if not isinstance(data, list): + msg = "Message is not a list." + raise TypeError(msg) + if len(data) < 2: + msg = "Message is too short." + raise TypeError(msg) + message_header = int(data[0]) + if TaskExecutorRequestType.Arguments != message_header: + msg = f"Message header is not an `Arguments`: {message_header}" + raise InvalidRequestTypeError(msg) + return data[1:] From 6e0dc175f4d69752599b4ed530583bae350acfb6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Tue, 19 Aug 2025 20:29:39 -0400 Subject: [PATCH 190/198] Add arg parse. WIP --- .../src/spider/task_executor/task_executor.py | 42 +++++++++++++++++-- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/python/src/spider/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py index 3ebd90d0b..ce976779b 100644 --- a/python/src/spider/task_executor/task_executor.py +++ b/python/src/spider/task_executor/task_executor.py @@ -6,12 +6,12 @@ import logging from io import BufferedReader from os import fdopen +from types import GenericAlias from uuid import UUID -from spider.storage import MariaDBStorage +from spider import client, core, storage from spider.task_executor.task_executor_message import get_request_body -from spider.client import TaskContext # Set up logger logger = logging.getLogger(__name__) @@ -52,6 +52,33 @@ def receive_message(pipe: BufferedReader) -> bytes: raise EOFError(msg) return body +def parse_arguments(store: storage.Storage, params: list[inspect.Parameter], arguments: list[object]) -> list[object]: + """ + Parses arguments for the function to be executed. + :param store: Storage instance to use to get Data. + :param params: List of parameters in the function signature. + :param arguments: List of arguments to parse. + :return: Parsed arguments. + :raises TypeError: If a parameter has no type annotation or if an argument cannot be parsed. + """ + parsed_args = [] + for i, param in enumerate(params): + arg = arguments[i] + cls = param.annotation + if param.annotation is inspect.Parameter.empty: + msg = f"Parameter {param.name} has no type annotation." + raise TypeError(msg) + if cls is bool: + parsed_args.append(arg) + elif cls is client.Data: + core_data = store.get_data(UUID(arg)) + parsed_args.append(client.Data._from_impl(core_data)) + else: + if isinstance(arg, list) or isinstance(arg, GenericAlias): + parsed_args.append(cls(*arg)) + else: + parsed_args.append(cls(arg)) + return parsed_args def main() -> None: """ @@ -70,7 +97,7 @@ def main() -> None: logger.debug("Function to run: %s", func) # Sets up storage - storage = MariaDBStorage(storage_url) + store = storage.MariaDBStorage(storage_url) input_pipe = fdopen(input_pipe, "rb") output_pipe = fdopen(output_pipe, "wb") @@ -91,7 +118,14 @@ def main() -> None: f"arguments, but {len(arguments)} were provided." ) raise ValueError(msg) - task_context = TaskContext(task_id, storage) + task_context = client.TaskContext(task_id, store) + args = [task_context] + for i, arg in enumerate(arguments): + param = list(signature.parameters.values())[i] + if param.annotation == inspect.Parameter.empty: + raise + results = function(*args) + logger.debug("Function %s executed", function_name) if __name__ == "__main__": From 55811831318ce15c8f15853c114a3ff67929dea6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 20 Aug 2025 18:19:48 -0400 Subject: [PATCH 191/198] Add msgpack class serde --- python/src/spider/utils/__init__.py | 8 +++ python/src/spider/utils/msgpack_serde.py | 79 ++++++++++++++++++++++++ python/tests/utils/test_msgpack_serde.py | 55 +++++++++++++++++ 3 files changed, 142 insertions(+) create mode 100644 python/src/spider/utils/__init__.py create mode 100644 python/src/spider/utils/msgpack_serde.py create mode 100644 python/tests/utils/test_msgpack_serde.py diff --git a/python/src/spider/utils/__init__.py b/python/src/spider/utils/__init__.py new file mode 100644 index 000000000..8d99c82a7 --- /dev/null +++ b/python/src/spider/utils/__init__.py @@ -0,0 +1,8 @@ +"""Spider utils package.""" + +from .msgpack_serde import msgpack_decoder, msgpack_encoder + +__all__ = [ + "msgpack_decoder", + "msgpack_encoder", +] diff --git a/python/src/spider/utils/msgpack_serde.py b/python/src/spider/utils/msgpack_serde.py new file mode 100644 index 000000000..655944377 --- /dev/null +++ b/python/src/spider/utils/msgpack_serde.py @@ -0,0 +1,79 @@ +"""Serialization and deserialization using msgpack.""" + +from dataclasses import fields, is_dataclass +from typing import get_args, get_origin + + +def msgpack_encoder(obj: object) -> list[object] | object: + """ + Encodes an object into a list of its field values. + This function recursively encodes nested dataclasses, lists, and dictionaries. + :param obj: Dataclass instance to serialize. + :return: List of field values. + """ + if is_dataclass(obj): + return [msgpack_encoder(getattr(obj, f.name)) for f in fields(obj)] + if isinstance(obj, list): + return [msgpack_encoder(item) for item in obj] + if isinstance(obj, dict): + return {msgpack_encoder(k): msgpack_encoder(v) for k, v in obj.items()} + return obj + + +def _decode_class(cls: type, data: object) -> object: + """ + Decodes data into an instance of a `cls`. + This function only works for non-container classes (not lists or dicts). + :param cls: Class to deserialize into. + :param data: Data to decode. + :return: Instance of `cls`. + :raise: TypeError if `data` is not compatible with `cls`. + """ + msg = f"Cannot create instance of {cls} with {data!r}" + if is_dataclass(cls): + if not isinstance(data, list): + raise TypeError(msg) + parameters = fields(cls) + if len(data) != len(parameters): + raise TypeError(msg) + args = [] + for param, value in zip(parameters, data, strict=False): + if not isinstance(param.type, type): + raise TypeError(msg) + args.append(msgpack_decoder(param.type, value)) + return cls(*args) + + return cls(data) + + +def msgpack_decoder(cls: type, data: object) -> object: + """ + Decodes data into an instance of `cls`. + This function recursively decodes nested dataclasses, lists, and dictionaries. + :param cls: Class to deserialize into. + :param data: Data to decode. + :return: Instance of `cls`. + :raise: TypeError if `data` is not compatible with `cls`. + """ + msg = f"Cannot create instance of {cls} with {data!r}" + + origin = get_origin(cls) + if origin is None: + return _decode_class(cls, data) + + if origin is list: + (key_type,) = get_args(cls) + if not isinstance(data, list): + raise TypeError(msg) + return [msgpack_decoder(key_type, item) for item in data] + + if origin is dict: + key_type, value_type = get_args(cls) + if not isinstance(data, dict): + raise TypeError(msg) + return { + msgpack_decoder(key_type, k): msgpack_decoder(value_type, v) for k, v in data.items() + } + + # Fall back + return cls(data) diff --git a/python/tests/utils/test_msgpack_serde.py b/python/tests/utils/test_msgpack_serde.py new file mode 100644 index 000000000..4049f7ee0 --- /dev/null +++ b/python/tests/utils/test_msgpack_serde.py @@ -0,0 +1,55 @@ +"""Tests for msgpack serialization and deserialization.""" + +from dataclasses import dataclass + +import msgpack + +import spider +from spider.utils.msgpack_serde import msgpack_decoder, msgpack_encoder + + +def compare_serde(obj: object) -> None: + """ + Serializes and then deserializes an object, and checks if the result matches the original + object. + """ + serialized = msgpack.packb(obj, default=msgpack_encoder) + unpacked_data = msgpack.unpackb(serialized, raw=False, strict_map_key=False) + deserialized = msgpack_decoder(type(obj), unpacked_data) + assert obj == deserialized + + +@dataclass +class Address: + """A simple address class for testing""" + + city: str + zipcode: str + + +@dataclass +class User: + """A simple user class for testing""" + + id: int + name: str + address: Address + + +class TestMsgpackSerde: + """Test class for msgpack serialization and deserialization.""" + + def test_primitives(self) -> None: + """Tests serialization and deserialization of primitive types.""" + compare_serde(True) + compare_serde([1, 2, 3]) + compare_serde({"key": "value"}) + compare_serde(spider.Int8(1)) + compare_serde(spider.Float(0.0)) + compare_serde([spider.Int8(1), spider.Int8(2)]) + compare_serde({spider.Int8(1): spider.Int8(3)}) + + def test_class(self) -> None: + """Tests serialization and deserialization of a custom class.""" + user = User(id=1, name="Alice", address=Address(city="Wonderland", zipcode="12345")) + compare_serde(user) From 42bbda01f7cdc716a5193544bd85e207c85eb98c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 20 Aug 2025 20:07:43 -0400 Subject: [PATCH 192/198] Expand the decoder to accept genric --- python/src/spider/utils/msgpack_serde.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/python/src/spider/utils/msgpack_serde.py b/python/src/spider/utils/msgpack_serde.py index 655944377..58af4dcc4 100644 --- a/python/src/spider/utils/msgpack_serde.py +++ b/python/src/spider/utils/msgpack_serde.py @@ -1,7 +1,8 @@ """Serialization and deserialization using msgpack.""" from dataclasses import fields, is_dataclass -from typing import get_args, get_origin +from types import GenericAlias +from typing import cast, get_args, get_origin def msgpack_encoder(obj: object) -> list[object] | object: @@ -46,7 +47,7 @@ def _decode_class(cls: type, data: object) -> object: return cls(data) -def msgpack_decoder(cls: type, data: object) -> object: +def msgpack_decoder(cls: type | GenericAlias, data: object) -> object: """ Decodes data into an instance of `cls`. This function recursively decodes nested dataclasses, lists, and dictionaries. @@ -59,7 +60,8 @@ def msgpack_decoder(cls: type, data: object) -> object: origin = get_origin(cls) if origin is None: - return _decode_class(cls, data) + # If `cls` does not have an origin, it is a concrete type. + return _decode_class(cast("type", cls), data) if origin is list: (key_type,) = get_args(cls) @@ -75,5 +77,4 @@ def msgpack_decoder(cls: type, data: object) -> object: msgpack_decoder(key_type, k): msgpack_decoder(value_type, v) for k, v in data.items() } - # Fall back - return cls(data) + raise TypeError(msg) From 9c07b8878dffd1557b15214c20ef51608731083f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 20 Aug 2025 20:07:58 -0400 Subject: [PATCH 193/198] Fix job deserialize --- python/src/spider/client/job.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/src/spider/client/job.py b/python/src/spider/client/job.py index 5babebcb7..2c430ff48 100644 --- a/python/src/spider/client/job.py +++ b/python/src/spider/client/job.py @@ -5,6 +5,8 @@ from spider import core from spider.client.data import Data from spider.storage import Storage, StorageError +from spider.type import parse_tdl_type +from spider.utils import msgpack_decoder def _convert_outputs(outputs: list[core.TaskOutput]) -> tuple[object, ...]: @@ -19,7 +21,9 @@ def _convert_outputs(outputs: list[core.TaskOutput]) -> tuple[object, ...]: results = [] for output in outputs: if isinstance(output.value, core.TaskOutputValue): - results.append(msgpack.unpackb(output.value)) + cls = parse_tdl_type(output.type).native_type() + unpacked = msgpack.unpackb(output.value, raw=False, strict_map_key=False) + results.append(msgpack_decoder(cls, unpacked)) elif isinstance(output.value, core.Data): results.append(Data._from_impl(output.value)) else: From a53920a8c8621820300fe43862381eec92f61a1b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 20 Aug 2025 20:31:24 -0400 Subject: [PATCH 194/198] Use strict zip --- python/src/spider/utils/msgpack_serde.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/spider/utils/msgpack_serde.py b/python/src/spider/utils/msgpack_serde.py index 58af4dcc4..b2b95f1d1 100644 --- a/python/src/spider/utils/msgpack_serde.py +++ b/python/src/spider/utils/msgpack_serde.py @@ -38,7 +38,7 @@ def _decode_class(cls: type, data: object) -> object: if len(data) != len(parameters): raise TypeError(msg) args = [] - for param, value in zip(parameters, data, strict=False): + for param, value in zip(parameters, data, strict=True): if not isinstance(param.type, type): raise TypeError(msg) args.append(msgpack_decoder(param.type, value)) From 5314e50523c1fa68c6c58e58a2226c1356b45fa3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 20 Aug 2025 20:31:32 -0400 Subject: [PATCH 195/198] Add nested list test --- python/tests/utils/test_msgpack_serde.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/tests/utils/test_msgpack_serde.py b/python/tests/utils/test_msgpack_serde.py index 4049f7ee0..92c267bf4 100644 --- a/python/tests/utils/test_msgpack_serde.py +++ b/python/tests/utils/test_msgpack_serde.py @@ -48,6 +48,7 @@ def test_primitives(self) -> None: compare_serde(spider.Float(0.0)) compare_serde([spider.Int8(1), spider.Int8(2)]) compare_serde({spider.Int8(1): spider.Int8(3)}) + compare_serde([[spider.Int8(1), spider.Int8(2)], [spider.Int8(3), spider.Int8(4)]]) def test_class(self) -> None: """Tests serialization and deserialization of a custom class.""" From e2c4610f5b5ea5f86fd3c3dcc356e960a16053de Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 20 Aug 2025 21:30:40 -0400 Subject: [PATCH 196/198] Add task executor --- .../src/spider/task_executor/task_executor.py | 48 ++++++++++++------- .../task_executor/task_executor_message.py | 3 ++ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/python/src/spider/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py index ce976779b..ad6052bd0 100644 --- a/python/src/spider/task_executor/task_executor.py +++ b/python/src/spider/task_executor/task_executor.py @@ -6,12 +6,13 @@ import logging from io import BufferedReader from os import fdopen -from types import GenericAlias from uuid import UUID -from spider import client, core, storage -from spider.task_executor.task_executor_message import get_request_body +import msgpack +from spider import client, core, storage +from spider.task_executor.task_executor_message import get_request_body, TaskExecutorResponseType +from spider.utils import msgpack_decoder, msgpack_encoder # Set up logger logger = logging.getLogger(__name__) @@ -52,7 +53,10 @@ def receive_message(pipe: BufferedReader) -> bytes: raise EOFError(msg) return body -def parse_arguments(store: storage.Storage, params: list[inspect.Parameter], arguments: list[object]) -> list[object]: + +def parse_arguments( + store: storage.Storage, params: list[inspect.Parameter], arguments: list[object] +) -> list[object]: """ Parses arguments for the function to be executed. :param store: Storage instance to use to get Data. @@ -68,18 +72,31 @@ def parse_arguments(store: storage.Storage, params: list[inspect.Parameter], arg if param.annotation is inspect.Parameter.empty: msg = f"Parameter {param.name} has no type annotation." raise TypeError(msg) - if cls is bool: - parsed_args.append(arg) - elif cls is client.Data: + if cls is client.Data: core_data = store.get_data(UUID(arg)) parsed_args.append(client.Data._from_impl(core_data)) else: - if isinstance(arg, list) or isinstance(arg, GenericAlias): - parsed_args.append(cls(*arg)) - else: - parsed_args.append(cls(arg)) + parsed_args.append(msgpack_decoder(cls, arg)) return parsed_args + +def parse_results(results: object) -> list[object]: + """ + Parses results from the function execution. + :param results: Results to parse. + :return: Parsed results. + :raises TypeError: If a result cannot be parsed. + """ + response_messages = [TaskExecutorResponseType.Result] + if isinstance(results, tuple): + for result in results: + if isinstance(result, client.Data): + response_messages.append(result.value) + else: + response_messages.append(msgpack_encoder(result)) + return response_messages + + def main() -> None: """ Main function to execute the task. @@ -119,14 +136,13 @@ def main() -> None: ) raise ValueError(msg) task_context = client.TaskContext(task_id, store) - args = [task_context] - for i, arg in enumerate(arguments): - param = list(signature.parameters.values())[i] - if param.annotation == inspect.Parameter.empty: - raise + args = [task_context] + parse_arguments(store, list(signature.parameters.values()), arguments) results = function(*args) logger.debug("Function %s executed", function_name) + responses = parse_results(results) + output_pipe.write(msgpack.packb(responses)) + if __name__ == "__main__": main() diff --git a/python/src/spider/task_executor/task_executor_message.py b/python/src/spider/task_executor/task_executor_message.py index a4f2c51a0..d0e2b8f20 100644 --- a/python/src/spider/task_executor/task_executor_message.py +++ b/python/src/spider/task_executor/task_executor_message.py @@ -15,6 +15,7 @@ class TaskExecutorResponseType(IntEnum): Ready = 4 Cancel = 5 + class TaskExecutorRequestType(IntEnum): """Task executor request type.""" @@ -22,12 +23,14 @@ class TaskExecutorRequestType(IntEnum): Arguments = 1 Resume = 2 + class InvalidRequestTypeError(Exception): """Exception raised for invalid request types.""" def __init__(self, message: str): super().__init__(message) + def get_request_body(message: bytes) -> list[object]: """ Gets the request body from the request message. From 0cb8e6f08e4adee9a2f132f0d36d3b49b5ed709a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 20 Aug 2025 21:34:28 -0400 Subject: [PATCH 197/198] Fix mypy --- python/src/spider/task_executor/task_executor.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python/src/spider/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py index ad6052bd0..67328db4d 100644 --- a/python/src/spider/task_executor/task_executor.py +++ b/python/src/spider/task_executor/task_executor.py @@ -65,7 +65,7 @@ def parse_arguments( :return: Parsed arguments. :raises TypeError: If a parameter has no type annotation or if an argument cannot be parsed. """ - parsed_args = [] + parsed_args: list[object] = [] for i, param in enumerate(params): arg = arguments[i] cls = param.annotation @@ -73,7 +73,10 @@ def parse_arguments( msg = f"Parameter {param.name} has no type annotation." raise TypeError(msg) if cls is client.Data: - core_data = store.get_data(UUID(arg)) + if not isinstance(arg, bytes): + msg = f"Argument {i} for spider.Data is not bytes." + raise TypeError(msg) + core_data = store.get_data(UUID(bytes=arg)) parsed_args.append(client.Data._from_impl(core_data)) else: parsed_args.append(msgpack_decoder(cls, arg)) @@ -87,7 +90,7 @@ def parse_results(results: object) -> list[object]: :return: Parsed results. :raises TypeError: If a result cannot be parsed. """ - response_messages = [TaskExecutorResponseType.Result] + response_messages: list[object] = [TaskExecutorResponseType.Result] if isinstance(results, tuple): for result in results: if isinstance(result, client.Data): @@ -136,8 +139,10 @@ def main() -> None: ) raise ValueError(msg) task_context = client.TaskContext(task_id, store) - args = [task_context] + parse_arguments(store, list(signature.parameters.values()), arguments) - results = function(*args) + arguments = [task_context] + parse_arguments( + store, list(signature.parameters.values()), arguments + ) + results = function(*arguments) logger.debug("Function %s executed", function_name) responses = parse_results(results) From 4d59367ccb7858f63af5235151843df75ea205ac Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 20 Aug 2025 21:45:01 -0400 Subject: [PATCH 198/198] Fix ruff --- python/src/spider/task_executor/task_executor.py | 9 +++++---- python/src/spider/task_executor/task_executor_message.py | 8 ++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/python/src/spider/task_executor/task_executor.py b/python/src/spider/task_executor/task_executor.py index 67328db4d..e4afc0db1 100644 --- a/python/src/spider/task_executor/task_executor.py +++ b/python/src/spider/task_executor/task_executor.py @@ -10,7 +10,7 @@ import msgpack -from spider import client, core, storage +from spider import client, storage from spider.task_executor.task_executor_message import get_request_body, TaskExecutorResponseType from spider.utils import msgpack_decoder, msgpack_encoder @@ -139,9 +139,10 @@ def main() -> None: ) raise ValueError(msg) task_context = client.TaskContext(task_id, store) - arguments = [task_context] + parse_arguments( - store, list(signature.parameters.values()), arguments - ) + arguments = [ + task_context, + *parse_arguments(store, list(signature.parameters.values()), arguments), + ] results = function(*arguments) logger.debug("Function %s executed", function_name) diff --git a/python/src/spider/task_executor/task_executor_message.py b/python/src/spider/task_executor/task_executor_message.py index d0e2b8f20..7838e1b47 100644 --- a/python/src/spider/task_executor/task_executor_message.py +++ b/python/src/spider/task_executor/task_executor_message.py @@ -27,10 +27,14 @@ class TaskExecutorRequestType(IntEnum): class InvalidRequestTypeError(Exception): """Exception raised for invalid request types.""" - def __init__(self, message: str): + def __init__(self, message: str) -> None: + """Initializes the InvalidRequestTypeError with a message.""" super().__init__(message) +MinArgRequestLength = 2 + + def get_request_body(message: bytes) -> list[object]: """ Gets the request body from the request message. @@ -44,7 +48,7 @@ def get_request_body(message: bytes) -> list[object]: if not isinstance(data, list): msg = "Message is not a list." raise TypeError(msg) - if len(data) < 2: + if len(data) < MinArgRequestLength: msg = "Message is too short." raise TypeError(msg) message_header = int(data[0])