diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index 192d61a98f..8385e19fae 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.16" +version: "0.1.2-dev.17" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index 5cb9aa1dc1..9e78032242 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -115,6 +115,15 @@ data: {{- end }}{{/* with .aws_authentication */}} {{- end }}{{/* if eq .type "fs" */}} {{- end }}{{/* with .Values.clpConfig.logs_input */}} + {{- with .Values.clpConfig.log_ingestor }} + log_ingestor: + buffer_flush_threshold: {{ .buffer_flush_threshold | int }} + buffer_flush_timeout: {{ .buffer_flush_timeout | int }} + channel_capacity: {{ .channel_capacity | int }} + host: "localhost" + logging_level: {{ .logging_level | quote }} + port: 3002 + {{- end }} package: query_engine: {{ .Values.clpConfig.package.query_engine | quote }} storage_engine: {{ .Values.clpConfig.package.storage_engine | quote }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml new file mode 100644 index 0000000000..11e0bc2fb9 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-deployment.yaml @@ -0,0 +1,88 @@ +{{- if .Values.clpConfig.log_ingestor }} +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp.fullname" . }}-log-ingestor + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "log-ingestor" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "log-ingestor" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "log-ingestor" + spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher + terminationGracePeriodSeconds: 60 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + containers: + - name: "log-ingestor" + image: "{{ include "clp.image.ref" . }}" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" + env: + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "username" + - name: "CLP_LOGS_DIR" + value: "/var/log/log_ingestor" + - name: "RUST_LOG" + value: {{ .Values.clpConfig.log_ingestor.logging_level | quote }} + ports: + - name: "log-ingestor" + containerPort: 3002 + volumeMounts: + - name: {{ include "clp.volumeName" (dict + "component_category" "log-ingestor" + "name" "logs" + ) | quote }} + mountPath: "/var/log/log_ingestor" + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true + command: [ + "/opt/clp/bin/log-ingestor", + "--config", "/etc/clp-config.yaml", + "--host", "0.0.0.0", + "--port", "3002" + ] + readinessProbe: + {{- include "clp.readinessProbeTimings" . | nindent 12 }} + httpGet: &log-ingestor-health-check + path: "/health" + port: "log-ingestor" + livenessProbe: + {{- include "clp.livenessProbeTimings" . | nindent 12 }} + httpGet: *log-ingestor-health-check + volumes: + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "log-ingestor" + "name" "logs" + ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config +{{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml new file mode 100644 index 0000000000..6ed940bc0c --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-logs-pv.yaml @@ -0,0 +1,11 @@ +{{- if .Values.clpConfig.log_ingestor }} +{{- include "clp.createLocalPv" (dict + "root" . + "component_category" "log-ingestor" + "name" "logs" + "nodeRole" "control-plane" + "capacity" "10Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/log_ingestor" .Values.clpConfig.logs_directory) +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml b/tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml new file mode 100644 index 0000000000..95eca00f60 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-logs-pvc.yaml @@ -0,0 +1,9 @@ +{{- if .Values.clpConfig.log_ingestor }} +{{- include "clp.createPvc" (dict + "root" . + "component_category" "log-ingestor" + "name" "logs" + "capacity" "10Gi" + "accessModes" (list "ReadWriteOnce") +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/log-ingestor-service.yaml b/tools/deployment/package-helm/templates/log-ingestor-service.yaml new file mode 100644 index 0000000000..628ffb7139 --- /dev/null +++ b/tools/deployment/package-helm/templates/log-ingestor-service.yaml @@ -0,0 +1,18 @@ +{{- if .Values.clpConfig.log_ingestor }} +apiVersion: "v1" +kind: "Service" +metadata: + name: {{ include "clp.fullname" . }}-log-ingestor + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "log-ingestor" +spec: + type: "NodePort" + selector: + {{- include "clp.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: "log-ingestor" + ports: + - port: 3002 + targetPort: "log-ingestor" + nodePort: {{ .Values.clpConfig.log_ingestor.port }} +{{- end }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index 474e1f1ab3..f88d5e04b4 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -59,7 +59,7 @@ mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ "$CLP_HOME/var/data/"{archives,streams,staged-archives,staged-streams} \ "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ - "$CLP_HOME/var/log/"{api_server,garbage_collector,mcp_server} \ + "$CLP_HOME/var/log/"{api_server,garbage_collector,log_ingestor,mcp_server} \ "$CLP_HOME/var/tmp" \ "$CLP_HOME/samples" @@ -85,18 +85,21 @@ cat <