diff --git a/tools/deployment/spider-helm/Chart.yaml b/tools/deployment/spider-helm/Chart.yaml index 726a529b..4cfb59ab 100644 --- a/tools/deployment/spider-helm/Chart.yaml +++ b/tools/deployment/spider-helm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: "v2" name: "spider" description: "A Helm chart for the Spider Huntsman deployment" type: "application" -version: "0.1.2" +version: "0.1.3" appVersion: "0.1.0" home: "https://github.com/y-scope/spider" sources: ["https://github.com/y-scope/spider"] diff --git a/tools/deployment/spider-helm/templates/configmap.yaml b/tools/deployment/spider-helm/templates/configmap.yaml index ac1dd104..4dfc7502 100644 --- a/tools/deployment/spider-helm/templates/configmap.yaml +++ b/tools/deployment/spider-helm/templates/configmap.yaml @@ -5,6 +5,19 @@ metadata: labels: {{- include "spider.labels" . | nindent 4 }} data: + execution-manager.yaml: | + connection_pool_size: {{ .Values.spiderConfig.execution_manager.connection_pool_size }} + liveness: + {{- toYaml .Values.spiderConfig.execution_manager.liveness | nindent 6 }} + scheduler: + host: {{ include "spider.componentFullname" (dict "root" . "component" "scheduler") | quote }} + port: {{ .Values.spiderConfig.scheduler.port }} + scheduler_poll_wait_ms: {{ .Values.spiderConfig.execution_manager.scheduler_poll_wait_ms }} + storage: + host: {{ include "spider.componentFullname" (dict "root" . "component" "storage") | quote }} + port: {{ .Values.spiderConfig.storage.port }} + task_executor: + {{- toYaml .Values.spiderConfig.execution_manager.task_executor | nindent 6 }} scheduler.yaml: | connection_pool_size: {{ .Values.spiderConfig.scheduler.connection_pool_size }} host: "0.0.0.0" diff --git a/tools/deployment/spider-helm/templates/worker-deployment.yaml b/tools/deployment/spider-helm/templates/worker-deployment.yaml new file mode 100644 index 00000000..fede13a2 --- /dev/null +++ b/tools/deployment/spider-helm/templates/worker-deployment.yaml @@ -0,0 +1,40 @@ +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "spider.componentFullname" (dict "root" . "component" "worker") }} + labels: + {{- include "spider.labels" . | nindent 4 }} + app.kubernetes.io/component: "worker" +spec: + replicas: {{ .Values.spiderConfig.worker.replicas }} + selector: + matchLabels: + {{- include "spider.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "worker" + template: + metadata: + labels: + {{- include "spider.labels" . | nindent 8 }} + app.kubernetes.io/component: "worker" + spec: + containers: + - name: "execution-manager" + image: {{ include "spider.imageRef" (dict "root" . "component" "worker") | quote }} + imagePullPolicy: {{ .Values.image.worker.pullPolicy | quote }} + command: ["spider-execution-manager", "--config", "/etc/spider/execution-manager.yaml"] + {{- with .Values.spiderConfig.worker.extra_envs }} + env: + {{- range $name, $value := . }} + - name: {{ $name | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + volumeMounts: + - name: "config" + mountPath: "/etc/spider/execution-manager.yaml" + subPath: "execution-manager.yaml" + readOnly: true + volumes: + - name: "config" + configMap: + name: {{ include "spider.componentFullname" (dict "root" . "component" "config") }} diff --git a/tools/deployment/spider-helm/values.yaml b/tools/deployment/spider-helm/values.yaml index 80160284..7f2ba452 100644 --- a/tools/deployment/spider-helm/values.yaml +++ b/tools/deployment/spider-helm/values.yaml @@ -14,6 +14,10 @@ image: pullPolicy: "Always" repository: "ghcr.io/y-scope/spider/storage" tag: "main" + worker: + pullPolicy: "Always" + repository: "ghcr.io/y-scope/spider/worker" + tag: "main" spiderConfig: # List of third-party services bundled (deployed) as part of the chart. @@ -34,6 +38,18 @@ spiderConfig: root_password: "spider-root-password" username: "spider-user" + execution_manager: + connection_pool_size: 4 + liveness: + scheduler_heartbeat_interval_sec: 10 + storage_heartbeat_interval_sec: 10 + scheduler_poll_wait_ms: 1000 + task_executor: + bin_path: "/usr/local/bin/spider-task-executor" + inherited_env: [] + log_dir: "/tmp/spider/task-executor" + package_dir: "/opt/spider/packages" + scheduler: connection_pool_size: 4 port: 50052 @@ -57,3 +73,7 @@ spiderConfig: cleanup_capacity: 256 commit_capacity: 256 task_capacity: 1048576 + + worker: + extra_envs: {} + replicas: 4