diff --git a/tools/deployment/package-helm/Chart.yaml b/tools/deployment/package-helm/Chart.yaml index feeffca787..443a3c4a16 100644 --- a/tools/deployment/package-helm/Chart.yaml +++ b/tools/deployment/package-helm/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: "v2" name: "clp" -version: "0.1.2-dev.10" +version: "0.1.2-dev.11" description: "A Helm chart for CLP's (Compressed Log Processor) package deployment" type: "application" appVersion: "0.7.1-dev" diff --git a/tools/deployment/package-helm/templates/configmap.yaml b/tools/deployment/package-helm/templates/configmap.yaml index dd9299b734..78cf6b6677 100644 --- a/tools/deployment/package-helm/templates/configmap.yaml +++ b/tools/deployment/package-helm/templates/configmap.yaml @@ -11,6 +11,11 @@ data: storage: directory: "/var/data/archives" type: "fs" + {{- if .Values.clpConfig.archive_output.retention_period }} + retention_period: {{ .Values.clpConfig.archive_output.retention_period | int }} + {{- else }} + retention_period: null + {{- end }} target_archive_size: {{ .Values.clpConfig.archive_output.target_archive_size | int }} target_dictionaries_size: {{ .Values.clpConfig.archive_output.target_dictionaries_size | int }} @@ -26,6 +31,11 @@ data: compression_worker: logging_level: {{ .Values.clpConfig.compression_worker.logging_level | quote }} data_directory: "/var/data" + garbage_collector: + logging_level: {{ .Values.clpConfig.garbage_collector.logging_level | quote }} + sweep_interval: + archive: {{ .Values.clpConfig.garbage_collector.sweep_interval.archive | int }} + search_result: {{ .Values.clpConfig.garbage_collector.sweep_interval.search_result | int }} database: auto_commit: false compress: true @@ -68,6 +78,11 @@ data: db_name: {{ .Values.clpConfig.results_cache.db_name | quote }} host: {{ include "clp.fullname" . }}-results-cache port: 27017 + {{- if .Values.clpConfig.results_cache.retention_period }} + retention_period: {{ .Values.clpConfig.results_cache.retention_period | int }} + {{- else }} + retention_period: null + {{- end }} stream_collection_name: {{ .Values.clpConfig.results_cache.stream_collection_name | quote }} stream_output: storage: diff --git a/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml new file mode 100644 index 0000000000..a4f810cb73 --- /dev/null +++ b/tools/deployment/package-helm/templates/garbage-collector-deployment.yaml @@ -0,0 +1,109 @@ +{{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }} +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: {{ include "clp.fullname" . }}-garbage-collector + labels: + {{- include "clp.labels" . | nindent 4 }} + app.kubernetes.io/component: "garbage-collector" +spec: + replicas: 1 + selector: + matchLabels: + {{- include "clp.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: "garbage-collector" + template: + metadata: + labels: + {{- include "clp.labels" . | nindent 8 }} + app.kubernetes.io/component: "garbage-collector" + spec: + serviceAccountName: {{ include "clp.fullname" . }}-job-watcher + terminationGracePeriodSeconds: 10 + securityContext: + runAsUser: {{ .Values.securityContext.firstParty.uid }} + runAsGroup: {{ .Values.securityContext.firstParty.gid }} + fsGroup: {{ .Values.securityContext.firstParty.gid }} + initContainers: + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "db-table-creator" + ) | nindent 10 }} + - {{- include "clp.waitFor" (dict + "root" . + "type" "job" + "name" "results-cache-indices-creator" + ) | nindent 10 }} + containers: + - name: "garbage-collector" + image: "{{ include "clp.image.ref" . }}" + imagePullPolicy: "{{ .Values.image.clpPackage.pullPolicy }}" + env: + - name: "CLP_DB_PASS" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "password" + - name: "CLP_DB_USER" + valueFrom: + secretKeyRef: + name: {{ include "clp.fullname" . }}-database + key: "username" + - name: "CLP_HOME" + value: "/opt/clp" + - name: "CLP_LOGGING_LEVEL" + value: {{ .Values.clpConfig.garbage_collector.logging_level | quote }} + - name: "CLP_LOGS_DIR" + value: "/var/log/garbage_collector" + - name: "PYTHONPATH" + value: "/opt/clp/lib/python3/site-packages" + volumeMounts: + - name: "config" + mountPath: "/etc/clp-config.yaml" + subPath: "clp-config.yaml" + readOnly: true + - name: "tmp" + mountPath: "/var/log" + - name: {{ include "clp.volumeName" (dict + "component_category" "garbage-collector" + "name" "logs" + ) | quote }} + mountPath: "/var/log/garbage_collector" + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "archives" + ) | quote }} + mountPath: "/var/data/archives" + - name: {{ include "clp.volumeName" (dict + "component_category" "shared-data" + "name" "streams" + ) | quote }} + mountPath: "/var/data/streams" + command: [ + "python3", "-u", + "-m", "job_orchestration.garbage_collector.garbage_collector", + "--config", "/etc/clp-config.yaml" + ] + volumes: + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "garbage-collector" + "name" "logs" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "archives" + ) | nindent 10 }} + - {{- include "clp.pvcVolume" (dict + "root" . + "component_category" "shared-data" + "name" "streams" + ) | nindent 10 }} + - name: "config" + configMap: + name: {{ include "clp.fullname" . }}-config + - name: "tmp" + emptyDir: {} +{{- end }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml new file mode 100644 index 0000000000..608e03739b --- /dev/null +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pv.yaml @@ -0,0 +1,11 @@ +{{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }} +{{- include "clp.createLocalPv" (dict + "root" . + "component_category" "garbage-collector" + "name" "logs" + "nodeRole" "control-plane" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") + "hostPath" (printf "%s/garbage_collector" .Values.clpConfig.logs_directory) +) }} +{{- end }} diff --git a/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml new file mode 100644 index 0000000000..9f40e096e8 --- /dev/null +++ b/tools/deployment/package-helm/templates/garbage-collector-logs-pvc.yaml @@ -0,0 +1,9 @@ +{{- if or .Values.clpConfig.archive_output.retention_period .Values.clpConfig.results_cache.retention_period }} +{{- include "clp.createPvc" (dict + "root" . + "component_category" "garbage-collector" + "name" "logs" + "capacity" "5Gi" + "accessModes" (list "ReadWriteOnce") +) }} +{{- end }} diff --git a/tools/deployment/package-helm/test.sh b/tools/deployment/package-helm/test.sh index abdd857512..937125c7a2 100755 --- a/tools/deployment/package-helm/test.sh +++ b/tools/deployment/package-helm/test.sh @@ -59,6 +59,7 @@ mkdir -p "$CLP_HOME/var/"{data,log}/{database,queue,redis,results_cache} \ "$CLP_HOME/var/data/"{archives,streams} \ "$CLP_HOME/var/log/"{compression_scheduler,compression_worker,user} \ "$CLP_HOME/var/log/"{query_scheduler,query_worker,reducer} \ + "$CLP_HOME/var/log/garbage_collector" \ "$CLP_HOME/var/tmp" \ "$CLP_HOME/samples" @@ -67,6 +68,12 @@ wget -O - https://zenodo.org/records/10516402/files/postgresql.tar.gz?download=1 | tar xz -C "$CLP_HOME/samples" & SAMPLE_DOWNLOAD_PID=$! +# Generate sample log file for garbage collector testing. +cat < /tmp/clp/samples/test-gc.jsonl +{"timestamp": $(date +%s%3N), "level": "INFO", "message": "User login successful"} +{"timestamp": $(date +%s%3N), "level": "ERROR", "message": "Database connection failed"} +EOF + cat <