Save MLS migration statistics in S3 bucket

cabal.project

@@ -50,6 +50,7 @@ packages:
   , tools/fedcalls/
   , tools/rex/
   , tools/stern/
+  , tools/mlsstats/
 
 tests: True
 benchmarks: True
@@ -114,6 +115,8 @@ package polysemy-wire-zoo
     ghc-options: -Werror
 package proxy
     ghc-options: -Werror
+package mlsstats
+    ghc-options: -Werror
 package repair-handles
     ghc-options: -Werror
 package rex

changelog.d/5-internal/mls-save-migration-statistics

@@ -0,0 +1,3 @@
+New cron job to save data usable to watch the progress of the Proteus to MLS migration in S3 bucket.
+
+**IMPORTANT:** This cron job is _not_ meant for general use! It can leak data about one team to other teams.

charts/mlsstats/.helmignore

@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj

charts/mlsstats/Chart.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+description: mlsstats - Push Proteus to MLS statistics to S3
+name: mlsstats
+version: 0.0.1

charts/mlsstats/README.md

@@ -0,0 +1,7 @@
+# mlsstats
+
+The kubernetes cronjob resource will spawn a new `mlsstats-XXXXXX` pod every day. Logs for the pod can be gathered with `kubectl log`.
+
+## Important note
+
+This cron job is _not_ meant for general use! It can leak data about one team to other teams.

charts/mlsstats/templates/_helpers.tpl

@@ -0,0 +1,13 @@
+{{/* Allow KubeVersion to be overridden. */}}
+{{- define "kubeVersion" -}}
+  {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}}
+{{- end -}}
+
+{{/* Get Batch API Version */}}
+{{- define "batch.apiVersion" -}}
+  {{- if and (.Capabilities.APIVersions.Has "batch/v1") (semverCompare ">= 1.21-0" (include "kubeVersion" .)) -}}
+      {{- print "batch/v1" -}}
+  {{- else -}}
+    {{- print "batch/v1beta1" -}}
+  {{- end -}}
+{{- end -}}

charts/mlsstats/templates/cronjob.yaml

@@ -0,0 +1,60 @@
+apiVersion: {{ include "batch.apiVersion" . }}
+kind: CronJob
+metadata:
+  name: {{ .Release.Name }}
+  labels:
+    app: mlsstats
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  concurrencyPolicy: Forbid
+  schedule: {{ .Values.schedule | quote }}
+  jobTemplate:
+    metadata:
+      labels:
+        app: mlsstats
+        release: {{ .Release.Name }}
+      annotations:
+        # An annotation of the configmap checksum ensures changes to the configmap cause a redeployment upon `helm upgrade`
+        checksum/secret: {{ include (print .Template.BasePath "/secret.yaml") . | sha256sum }}
+    spec:
+      backoffLimit: 0
+      template:
+        spec:
+          restartPolicy: Never
+          containers:
+            - name: mlsstats
+              image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+              imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}
+              args: [ "mlsstats"
+                      , "--brig-cassandra-host", {{ .Values.config.cassandra.brig.host | quote }}
+                      , "--brig-cassandra-port", {{ .Values.config.cassandra.brig.port | quote }}
+                      , "--brig-cassandra-keyspace", {{ .Values.config.cassandra.brig.keyspace | quote }}
+                      , "--galley-cassandra-host", {{ .Values.config.cassandra.galley.host | quote }}
+                      , "--galley-cassandra-port", {{ .Values.config.cassandra.galley.port | quote }}
+                      , "--galley-cassandra-keyspace", {{ .Values.config.cassandra.galley.keyspace | quote }}
+                      , "--cassandra-pagesize", {{ .Values.config.cassandra.pagesize | quote }}
+                      , "--s3-endpoint", {{ .Values.config.s3.endpoint | quote -}}
+                      , "--s3-region", {{ .Values.config.s3.region | quote -}}
+                      , "--s3-addressing-style", {{ .Values.config.s3.addressingStyle | quote }}
+                      , "--s3-bucket-name", {{ .Values.config.s3.bucket.name | quote }}
+                      , "--s3-bucket-dir", {{ .Values.config.s3.bucket.directory | quote }}
+                    ]
+              resources:
+              env:
+              {{- if hasKey .Values.secrets "awsKeyId" }}
+              - name: AWS_ACCESS_KEY_ID
+                valueFrom:
+                  secretKeyRef:
+                    name: mlsstats
+                    key: awsKeyId
+              - name: AWS_SECRET_ACCESS_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: mlsstats
+                    key: awsSecretKey
+              {{- end }}
+              - name: AWS_REGION
+                value: "{{ .Values.config.s3.region }}"
+{{ toYaml .Values.resources | indent 16 }}

charts/mlsstats/templates/secret.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}
+  labels:
+    app: mlsstats
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+type: Opaque
+data:
+  {{/* for_helm_linting is necessary only since the 'with' block below does not throw an error upon an empty .Values.secrets */}}
+  for_helm_linting: {{ required "No .secrets found in configuration. Did you forget to helm <command> -f path/to/secrets.yaml ?" .Values.secrets | quote | b64enc | quote }}
+
+  {{- with .Values.secrets }}
+  {{- if .awsKeyId }}
+  awsKeyId: {{ .awsKeyId | b64enc | quote }}
+  awsSecretKey: {{ .awsSecretKey | b64enc | quote }}
+  {{- end }}
+  {{- end }}

charts/mlsstats/values.yaml

@@ -0,0 +1,30 @@
+image:
+  repository: quay.io/wire/mlsstats
+  tag: 0.1
+resources:
+  requests:
+    memory: "256Mi"
+    cpu: "100m"
+  limits:
+    memory: "256Mi"
+    cpu: "100m"
+schedule: "23 3 * * *"
+config:
+  cassandra:
+    brig:
+      host: cassandra
+      port: 9042
+      keyspace: brig
+    galley:
+      host: cassandra
+      port: 9042
+      keyspace: galley
+    pagesize: 1024
+  s3:
+    endpoint: https://s3.eu-west-1.amazonaws.com/
+    region: eu-west-1
+    addressingStyle: auto
+    bucket:
+      name: mlsstats
+      directory: "/"
+secrets: {}

charts/wire-server/requirements.yaml

@@ -129,3 +129,8 @@ dependencies:
   repository: "file://../integration"
   tags:
     - integration
+- name: mlsstats
+  version: "0.0.42"
+  repository: "file://../mlsstats"
+  tags:
+    - mlsstats

charts/wire-server/values.yaml

@@ -12,3 +12,4 @@ tags:
   federation: false # see also galley.config.enableFederation and brig.config.enableFederation
   sftd: false
   backoffice: false
+  mlsstats: false

nix/local-haskell-packages.nix

@@ -52,6 +52,7 @@
   repair-handles = hself.callPackage ../tools/db/repair-handles/default.nix { inherit gitignoreSource; };
   service-backfill = hself.callPackage ../tools/db/service-backfill/default.nix { inherit gitignoreSource; };
   fedcalls = hself.callPackage ../tools/fedcalls/default.nix { inherit gitignoreSource; };
+  mlsstats = hself.callPackage ../tools/mlsstats/default.nix { inherit gitignoreSource; };
   rex = hself.callPackage ../tools/rex/default.nix { inherit gitignoreSource; };
   stern = hself.callPackage ../tools/stern/default.nix { inherit gitignoreSource; };
 }

tools/mlsstats/.ormolu

@@ -0,0 +1 @@
+../../.ormolu