From cdea7df2b0d71abc143d1d070143f7eadef22985 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Fri, 7 Aug 2026 16:41:07 -0400 Subject: [PATCH 1/7] Done. --- tools/deployment/spider-helm/values.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/deployment/spider-helm/values.yaml b/tools/deployment/spider-helm/values.yaml index fe7d60f9..080e6367 100644 --- a/tools/deployment/spider-helm/values.yaml +++ b/tools/deployment/spider-helm/values.yaml @@ -41,8 +41,8 @@ spiderConfig: execution_manager: connection_pool_size: 4 liveness: - scheduler_heartbeat_interval_sec: 10 - storage_heartbeat_interval_sec: 10 + scheduler_heartbeat_interval_sec: 1 + storage_heartbeat_interval_sec: 1 log_level: "INFO" scheduler_poll_wait_ms: 1000 task_executor: @@ -59,12 +59,12 @@ spiderConfig: scheduler: policy: "round_robin" config: - active_job_queue_capacity: 64 + active_job_queue_capacity: 16 cleanup_ready_task_capacity: 256 commit_ready_task_capacity: 256 - dispatch_queue_capacity: 64 + dispatch_queue_capacity: 16 finalizing_job_expiration_timeout_sec: 300 - ready_task_capacity: 65536 + ready_task_capacity: 1048576 storage_poll_timeout_ms: 10 tick_interval_ms: 5 From 1c7977da4b1913cb3eb8bd1753358858bcfa6fa7 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Fri, 7 Aug 2026 17:05:18 -0400 Subject: [PATCH 2/7] Done. --- components/spider-storage/src/state/runtime.rs | 16 ++++++++-------- .../tests/runtime_recovery_test.rs | 8 ++++---- .../spider-helm/templates/configmap.yaml | 2 +- tools/deployment/spider-helm/values.yaml | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/components/spider-storage/src/state/runtime.rs b/components/spider-storage/src/state/runtime.rs index bada722d..53ad4c41 100644 --- a/components/spider-storage/src/state/runtime.rs +++ b/components/spider-storage/src/state/runtime.rs @@ -29,13 +29,13 @@ use crate::task_instance_pool::create_task_instance_pool; /// Runtime configuration for the storage service. #[derive(Clone, Debug, Deserialize)] pub struct RuntimeConfig { - pub db_config: DatabaseConfig, + pub db: DatabaseConfig, #[serde(default)] - pub ready_queue_config: ReadyQueueConfig, + pub ready_queue: ReadyQueueConfig, #[serde(default)] - pub task_instance_pool_config: TaskInstancePoolConfig, + pub task_instance_pool: TaskInstancePoolConfig, #[serde(default)] - pub job_cache_gc_config: JobCacheGcConfig, + pub job_cache_gc: JobCacheGcConfig, } /// Runtime state for the storage service. @@ -153,15 +153,15 @@ pub async fn create_runtime( StorageServerError, > { let cancellation_token = CancellationToken::new(); - let db = MariaDbStorageConnector::connect(&config.db_config).await?; + let db = MariaDbStorageConnector::connect(&config.db).await?; let session_id = db.session_id(); let (ready_queue_sender, ready_queue_receiver) = - create_ready_queue(&config.ready_queue_config).map_err(CacheError::from)?; + create_ready_queue(&config.ready_queue).map_err(CacheError::from)?; let (task_instance_pool_connector, task_instance_pool_join_handle) = create_task_instance_pool( ready_queue_sender.clone(), db.clone(), cancellation_token.clone(), - &config.task_instance_pool_config, + &config.task_instance_pool, ) .map_err(CacheError::from)?; @@ -174,7 +174,7 @@ pub async fn create_runtime( let (job_cache_gc_handle, job_cache_gc_join_handle) = create_job_cache_gc( job_cache.clone(), cancellation_token.clone(), - &config.job_cache_gc_config, + &config.job_cache_gc, ) .map_err(CacheError::from)?; let service_state = ServiceState::new(ServiceStateParams { diff --git a/components/spider-storage/tests/runtime_recovery_test.rs b/components/spider-storage/tests/runtime_recovery_test.rs index 012f0cee..870c53ee 100644 --- a/components/spider-storage/tests/runtime_recovery_test.rs +++ b/components/spider-storage/tests/runtime_recovery_test.rs @@ -201,10 +201,10 @@ async fn restarted_storage_cache_recovers_cleanup_ready_job() -> anyhow::Result< /// while other configurations set to default. fn create_runtime_config() -> RuntimeConfig { RuntimeConfig { - db_config: create_mariadb_config(), - ready_queue_config: ReadyQueueConfig::default(), - task_instance_pool_config: TaskInstancePoolConfig::default(), - job_cache_gc_config: JobCacheGcConfig::default(), + db: create_mariadb_config(), + ready_queue: ReadyQueueConfig::default(), + task_instance_pool: TaskInstancePoolConfig::default(), + job_cache_gc: JobCacheGcConfig::default(), } } diff --git a/tools/deployment/spider-helm/templates/configmap.yaml b/tools/deployment/spider-helm/templates/configmap.yaml index 4dfc7502..7b3a3894 100644 --- a/tools/deployment/spider-helm/templates/configmap.yaml +++ b/tools/deployment/spider-helm/templates/configmap.yaml @@ -37,7 +37,7 @@ data: host: "0.0.0.0" port: {{ .Values.spiderConfig.storage.port }} runtime: - db_config: + db: host: {{ include "spider.databaseHost" . | quote }} max_connections: {{ .Values.spiderConfig.database.max_connections }} name: {{ .Values.spiderConfig.database.name | quote }} diff --git a/tools/deployment/spider-helm/values.yaml b/tools/deployment/spider-helm/values.yaml index 080e6367..086df268 100644 --- a/tools/deployment/spider-helm/values.yaml +++ b/tools/deployment/spider-helm/values.yaml @@ -72,7 +72,7 @@ spiderConfig: log_level: "INFO" port: 50051 runtime: - ready_queue_config: + ready_queue: cleanup_capacity: 256 commit_capacity: 256 task_capacity: 1048576 From ab46c64d67ba4ce2f83b693301ca912e639af97d Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Fri, 7 Aug 2026 17:05:51 -0400 Subject: [PATCH 3/7] Update chart version. --- tools/deployment/spider-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/spider-helm/Chart.yaml b/tools/deployment/spider-helm/Chart.yaml index 8de00ec2..5b15c1c8 100644 --- a/tools/deployment/spider-helm/Chart.yaml +++ b/tools/deployment/spider-helm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: "v2" name: "spider" description: "A Helm chart for the Spider Huntsman deployment" type: "application" -version: "0.1.8" +version: "0.1.9" appVersion: "0.1.0-dev" home: "https://github.com/y-scope/spider" sources: ["https://github.com/y-scope/spider"] From d6ffc054a822f34b9434e1f0bbfdfbfdcc6cb085 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Fri, 7 Aug 2026 17:06:25 -0400 Subject: [PATCH 4/7] Update chart version. --- tools/deployment/spider-helm/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/deployment/spider-helm/Chart.yaml b/tools/deployment/spider-helm/Chart.yaml index 5b15c1c8..4fa15b4e 100644 --- a/tools/deployment/spider-helm/Chart.yaml +++ b/tools/deployment/spider-helm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: "v2" name: "spider" description: "A Helm chart for the Spider Huntsman deployment" type: "application" -version: "0.1.9" +version: "0.1.10" appVersion: "0.1.0-dev" home: "https://github.com/y-scope/spider" sources: ["https://github.com/y-scope/spider"] From ac0edca0a23745529689db612fa20ec61392d8e3 Mon Sep 17 00:00:00 2001 From: LinZhihao-723 Date: Fri, 7 Aug 2026 19:29:40 -0400 Subject: [PATCH 5/7] Done. --- tools/deployment/spider-helm/Chart.yaml | 2 +- tools/deployment/spider-helm/templates/configmap.yaml | 7 +++---- tools/deployment/spider-helm/values.yaml | 10 ++++++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tools/deployment/spider-helm/Chart.yaml b/tools/deployment/spider-helm/Chart.yaml index 4fa15b4e..46de908b 100644 --- a/tools/deployment/spider-helm/Chart.yaml +++ b/tools/deployment/spider-helm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: "v2" name: "spider" description: "A Helm chart for the Spider Huntsman deployment" type: "application" -version: "0.1.10" +version: "0.1.11" appVersion: "0.1.0-dev" home: "https://github.com/y-scope/spider" sources: ["https://github.com/y-scope/spider"] diff --git a/tools/deployment/spider-helm/templates/configmap.yaml b/tools/deployment/spider-helm/templates/configmap.yaml index 7b3a3894..e5707305 100644 --- a/tools/deployment/spider-helm/templates/configmap.yaml +++ b/tools/deployment/spider-helm/templates/configmap.yaml @@ -26,10 +26,9 @@ data: advertised_endpoint: host: {{ include "spider.componentFullname" (dict "root" . "component" "scheduler") | quote }} port: {{ .Values.spiderConfig.scheduler.port }} - scheduler: - policy: {{ .Values.spiderConfig.scheduler.runtime.scheduler.policy | quote }} - config: - {{- toYaml .Values.spiderConfig.scheduler.runtime.scheduler.config | nindent 10 }} + {{- with .Values.spiderConfig.scheduler.runtime }} + {{- toYaml . | nindent 6 }} + {{- end }} storage_endpoint: host: {{ include "spider.componentFullname" (dict "root" . "component" "storage") | quote }} port: {{ .Values.spiderConfig.storage.port }} diff --git a/tools/deployment/spider-helm/values.yaml b/tools/deployment/spider-helm/values.yaml index 086df268..41d52609 100644 --- a/tools/deployment/spider-helm/values.yaml +++ b/tools/deployment/spider-helm/values.yaml @@ -56,6 +56,9 @@ spiderConfig: log_level: "INFO" port: 50052 runtime: + em_registry: + dead_em_cutoff_sec: 30 + liveness_tracking_interval_ms: 1000 scheduler: policy: "round_robin" config: @@ -72,10 +75,17 @@ spiderConfig: log_level: "INFO" port: 50051 runtime: + job_cache_gc: + gc_interval_sec: 30 + terminated_job_retention_sec: 300 ready_queue: cleanup_capacity: 256 commit_capacity: 256 task_capacity: 1048576 + task_instance_pool: + execution_manager_stale_cutoff_sec: 60 + gc_interval_sec: 30 + message_channel_capacity: 128 worker: extra_envs: [] From 92de4c7f75a01e30341926953bb765b4fc2891f8 Mon Sep 17 00:00:00 2001 From: Lin Zhihao <59785146+LinZhihao-723@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:30:52 -0400 Subject: [PATCH 6/7] Apply suggestions from code review Co-authored-by: ChenXing Yang <60459812+20001020ycx@users.noreply.github.com> --- tools/deployment/spider-helm/templates/configmap.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/deployment/spider-helm/templates/configmap.yaml b/tools/deployment/spider-helm/templates/configmap.yaml index e5707305..b6de066f 100644 --- a/tools/deployment/spider-helm/templates/configmap.yaml +++ b/tools/deployment/spider-helm/templates/configmap.yaml @@ -27,8 +27,10 @@ data: host: {{ include "spider.componentFullname" (dict "root" . "component" "scheduler") | quote }} port: {{ .Values.spiderConfig.scheduler.port }} {{- with .Values.spiderConfig.scheduler.runtime }} + {{- with (omit . "advertised_endpoint") }} {{- toYaml . | nindent 6 }} {{- end }} + {{- end }} storage_endpoint: host: {{ include "spider.componentFullname" (dict "root" . "component" "storage") | quote }} port: {{ .Values.spiderConfig.storage.port }} From a1661d61763e6e52ac28019da3225e4bbaa22dc2 Mon Sep 17 00:00:00 2001 From: ChenXing Yang <20001020ycx@gmail.com> Date: Tue, 11 Aug 2026 13:43:21 -0400 Subject: [PATCH 7/7] refactor(helm): Collapse the nested `with` blocks in the scheduler runtime passthrough. --- tools/deployment/spider-helm/templates/configmap.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/deployment/spider-helm/templates/configmap.yaml b/tools/deployment/spider-helm/templates/configmap.yaml index b6de066f..73b426fe 100644 --- a/tools/deployment/spider-helm/templates/configmap.yaml +++ b/tools/deployment/spider-helm/templates/configmap.yaml @@ -27,9 +27,7 @@ data: host: {{ include "spider.componentFullname" (dict "root" . "component" "scheduler") | quote }} port: {{ .Values.spiderConfig.scheduler.port }} {{- with .Values.spiderConfig.scheduler.runtime }} - {{- with (omit . "advertised_endpoint") }} - {{- toYaml . | nindent 6 }} - {{- end }} + {{- toYaml (omit . "advertised_endpoint") | nindent 6 }} {{- end }} storage_endpoint: host: {{ include "spider.componentFullname" (dict "root" . "component" "storage") | quote }}