diff --git a/Makefile b/Makefile index 03b2615509..dbe1f612e5 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ DOCKER_TAG ?= $(USER) # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything) HELM_SEMVER ?= 0.0.42 # The list of helm charts needed on internal kubernetes testing environments -CHARTS_INTEGRATION := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn +CHARTS_INTEGRATION := wire-server databases-ephemeral redis-cluster rabbitmq fake-aws ingress-nginx-controller nginx-ingress-controller nginx-ingress-services fluent-bit kibana sftd restund coturn k8ssandra-test-cluster # The list of helm charts to publish on S3 # FUTUREWORK: after we "inline local subcharts", # (e.g. move charts/brig to charts/wire-server/brig) diff --git a/changelog.d/2-features/cassandra-tls b/changelog.d/2-features/cassandra-tls new file mode 100644 index 0000000000..e8baaaf2ed --- /dev/null +++ b/changelog.d/2-features/cassandra-tls @@ -0,0 +1,6 @@ +Allow the configuration of TLS-secured connections to Cassandra. TLS is used +when a certificate is provided. This is either done with +`--tls-ca-certificate-file` for cli commands or the configuration attribute +`cassandra.tlsCa` for services. In Helm charts, the certificate is provided as +literal PEM string; either as attribute `cassandra.tlsCa` (analog to service +configuration) or by a reference to a secret (`cassandra.tlsCaSecretRef`.) diff --git a/charts/brig/templates/_helpers.tpl b/charts/brig/templates/_helpers.tpl index 762fb52c2f..857c0203de 100644 --- a/charts/brig/templates/_helpers.tpl +++ b/charts/brig/templates/_helpers.tpl @@ -7,3 +7,19 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "brig-cassandra" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} diff --git a/charts/brig/templates/cassandra-secret.yaml b/charts/brig/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..fa84800147 --- /dev/null +++ b/charts/brig/templates/cassandra-secret.yaml @@ -0,0 +1,15 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} +{{- if not (empty .Values.config.cassandra.tlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: brig-cassandra + labels: + app: brig + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.config.cassandra.tlsCa | b64enc | quote }} +{{- end }} diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml index 2ed6eb6833..f2a43d2ed8 100644 --- a/charts/brig/templates/configmap.yaml +++ b/charts/brig/templates/configmap.yaml @@ -28,6 +28,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} + {{- if eq (include "useCassandraTLS" .) "true" }} + tlsCa: /etc/wire/brig/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} + {{- end }} elasticsearch: url: http://{{ .elasticsearch.host }}:{{ .elasticsearch.port }} diff --git a/charts/brig/templates/deployment.yaml b/charts/brig/templates/deployment.yaml index 29f8ebc003..bc1261391b 100644 --- a/charts/brig/templates/deployment.yaml +++ b/charts/brig/templates/deployment.yaml @@ -46,6 +46,11 @@ spec: - name: "geoip" emptyDir: {} {{- end }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "brig-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end}} {{- if .Values.config.geoip.enabled }} # Brig needs GeoIP database to be downloaded before it can start. initContainers: @@ -102,6 +107,10 @@ spec: - name: "geoip" mountPath: "/usr/share/GeoIP" {{- end }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "brig-cassandra" + mountPath: "/etc/wire/brig/cassandra" + {{- end }} env: - name: LOG_LEVEL value: {{ .Values.config.logLevel }} diff --git a/charts/brig/templates/tests/brig-integration.yaml b/charts/brig/templates/tests/brig-integration.yaml index 1599c3860b..aff0f6d525 100644 --- a/charts/brig/templates/tests/brig-integration.yaml +++ b/charts/brig/templates/tests/brig-integration.yaml @@ -44,6 +44,11 @@ spec: - name: "brig-integration-secrets" secret: secretName: "brig-integration" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "brig-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end}} containers: - name: integration image: "{{ .Values.image.repository }}-integration:{{ .Values.image.tag }}" @@ -101,6 +106,10 @@ spec: # non-default locations # (see corresp. TODO in galley.) mountPath: "/etc/wire/integration-secrets" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "brig-cassandra" + mountPath: "/etc/wire/brig/cassandra" + {{- end }} env: # these dummy values are necessary for Amazonka's "Discover" diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml index b1109ba493..305502e30e 100644 --- a/charts/brig/values.yaml +++ b/charts/brig/values.yaml @@ -20,6 +20,14 @@ config: logNetStrings: false cassandra: host: aws-cassandra +# To enable TLS provide a CA: +# tlsCa: +# +# Or refer to an existing secret (containing the CA): +# tlsCaSecretRef: +# name: +# key: + elasticsearch: host: elasticsearch-client port: 9200 diff --git a/charts/cassandra-migrations/templates/_helpers.tpl b/charts/cassandra-migrations/templates/_helpers.tpl index 551b901999..0d805051ff 100644 --- a/charts/cassandra-migrations/templates/_helpers.tpl +++ b/charts/cassandra-migrations/templates/_helpers.tpl @@ -107,6 +107,125 @@ Thus the order of priority is: {{- end -}} {{- end -}} +{{/* NOTE: Cassandra TLS helpers + +Cassandra connections can be configured per service or with a general configuration. +Thus, there are three functions per service that fallback to the general +configuration if the specific one does not exist: + +- useTls -> Bool: Do we use Cassandra TLS connections for this + service? + +- tlsCa -> String: TLS CA PEM string (if configured) + +- tlsSecretRef -> YAML: Dict with keys `name` (name of the + secret to use) and `key` (name of the entry in the secret) +*/}} + +{{- define "useTlsGalley" -}} +{{ $cassandraGalley := default .Values.cassandra .Values.cassandraGalley }} +{{- if or $cassandraGalley.tlsCa $cassandraGalley.tlsCaSecretRef -}} +true +{{- else}} +false +{{- end }} +{{- end -}} + +{{- define "tlsCaGalley" -}} +{{ $cassandraGalley := default .Values.cassandra .Values.cassandraGalley }} +{{- if hasKey $cassandraGalley "tlsCa" -}} +{{- $cassandraGalley.tlsCa }} +{{ else }} +{{- end -}} +{{- end -}} + +{{- define "tlsSecretRefGalley" -}} +{{ $cassandraGalley := default .Values.cassandra .Values.cassandraGalley }} +{{- if $cassandraGalley.tlsCaSecretRef -}} +{{ $cassandraGalley.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "galley-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} + +{{- define "useTlsBrig" -}} +{{ $cassandraBrig := default .Values.cassandra .Values.cassandraBrig }} +{{- if or $cassandraBrig.tlsCa $cassandraBrig.tlsCaSecretRef -}} +true +{{- else}} +false +{{- end }} +{{- end -}} + +{{- define "tlsCaBrig" -}} +{{ $cassandraBrig := default .Values.cassandra .Values.cassandraBrig }} +{{- if hasKey $cassandraBrig "tlsCa" -}} +{{- $cassandraBrig.tlsCa }} +{{ else }} +{{- end -}} +{{- end -}} + +{{- define "tlsSecretRefBrig" -}} +{{ $cassandraBrig := default .Values.cassandra .Values.cassandraBrig }} +{{- if $cassandraBrig.tlsCaSecretRef -}} +{{ $cassandraBrig.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "brig-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} + +{{- define "useTlsSpar" -}} +{{ $cassandraSpar := default .Values.cassandra .Values.cassandraSpar }} +{{- if or $cassandraSpar.tlsCa $cassandraSpar.tlsCaSecretRef -}} +true +{{- else}} +false +{{- end }} +{{- end -}} + +{{- define "tlsCaSpar" -}} +{{ $cassandraSpar := default .Values.cassandra .Values.cassandraSpar }} +{{- if hasKey $cassandraSpar "tlsCa" -}} +{{- $cassandraSpar.tlsCa }} +{{ else }} +{{- end -}} +{{- end -}} + +{{- define "tlsSecretRefSpar" -}} +{{ $cassandraSpar := default .Values.cassandra .Values.cassandraSpar }} +{{- if $cassandraSpar.tlsCaSecretRef -}} +{{ $cassandraSpar.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "spar-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} + +{{- define "useTlsGundeck" -}} +{{ $cassandraGundeck := default .Values.cassandra .Values.cassandraGundeck }} +{{- if or $cassandraGundeck.tlsCa $cassandraGundeck.tlsCaSecretRef -}} +true +{{- else}} +false +{{- end }} +{{- end -}} + +{{- define "tlsCaGundeck" -}} +{{ $cassandraGundeck := default .Values.cassandra .Values.cassandraGundeck }} +{{- if hasKey $cassandraGundeck "tlsCa" -}} +{{- $cassandraGundeck.tlsCa }} +{{ else }} +{{- end -}} +{{- end -}} + +{{- define "tlsSecretRefGundeck" -}} +{{ $cassandraGundeck := default .Values.cassandra .Values.cassandraGundeck }} +{{- if $cassandraGundeck.tlsCaSecretRef -}} +{{ $cassandraGundeck.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "gundeck-cassandra-cert" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} + {{/* Allow KubeVersion to be overridden. */}} {{- define "kubeVersion" -}} {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} diff --git a/charts/cassandra-migrations/templates/cassandra-certs.yaml b/charts/cassandra-migrations/templates/cassandra-certs.yaml new file mode 100644 index 0000000000..3bea0c6f5d --- /dev/null +++ b/charts/cassandra-migrations/templates/cassandra-certs.yaml @@ -0,0 +1,75 @@ +{{- if ne (trim (include "tlsCaBrig" .)) "" }} +apiVersion: v1 +kind: Secret +metadata: + name: brig-cassandra-cert + labels: + app: cassandra-migrations + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +type: Opaque +data: + ca.pem: {{ include "tlsCaBrig" . | b64enc | quote }} +{{- end}} +{{- if ne (trim (include "tlsCaGalley" .)) "" }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: galley-cassandra-cert + labels: + app: cassandra-migrations + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +type: Opaque +data: + ca.pem: {{ include "tlsCaGalley" . | b64enc | quote }} +{{- end}} +{{- if ne (trim (include "tlsCaGundeck" .)) "" }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: gundeck-cassandra-cert + labels: + app: cassandra-migrations + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +type: Opaque +data: + ca.pem: {{ include "tlsCaGundeck" . | b64enc | quote }} +{{- end}} +{{- if ne (trim (include "tlsCaSpar" .)) "" }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: spar-cassandra-cert + labels: + app: cassandra-migrations + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" + annotations: + "helm.sh/hook": pre-install,pre-upgrade,post-install,post-upgrade + "helm.sh/hook-weight": "0" + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +type: Opaque +data: + ca.pem: {{ include "tlsCaSpar" . | b64enc | quote }} +{{- end}} diff --git a/charts/cassandra-migrations/templates/galley-migrate-data.yaml b/charts/cassandra-migrations/templates/galley-migrate-data.yaml index dee40d0b24..127a6ab0b5 100644 --- a/charts/cassandra-migrations/templates/galley-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/galley-migrate-data.yaml @@ -42,4 +42,19 @@ spec: - "9042" - --cassandra-keyspace - galley + {{- if eq (include "useTlsGalley" .) "true" }} + - --tls-ca-certificate-file + - /certs/galley/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} + {{- end }} + {{- if eq (include "useTlsGalley" .) "true" }} + volumeMounts: + - name: galley-cassandra-cert + mountPath: "/certs/galley" + {{- end }} + {{- if eq (include "useTlsGalley" .) "true" }} + volumes: + - name: galley-cassandra-cert + secret: + secretName: {{ (include "tlsSecretRefGalley" . | fromYaml).name }} + {{- end }} {{- end }} diff --git a/charts/cassandra-migrations/templates/migrate-schema.yaml b/charts/cassandra-migrations/templates/migrate-schema.yaml index 5129fc4baf..e06aa2288a 100644 --- a/charts/cassandra-migrations/templates/migrate-schema.yaml +++ b/charts/cassandra-migrations/templates/migrate-schema.yaml @@ -9,7 +9,7 @@ metadata: heritage: {{ .Release.Service }} annotations: "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "0" + "helm.sh/hook-weight": "1" "helm.sh/hook-delete-policy": "before-hook-creation" spec: template: @@ -22,6 +22,27 @@ spec: # specifying cassandra-migrations as initContainers executes them sequentially, rather than in parallel # to avoid 'Column family ID mismatch' / schema disagreements # see https://stackoverflow.com/questions/29030661/creating-new-table-with-cqlsh-on-existing-keyspace-column-family-id-mismatch#40325651 for details. + volumes: + {{- if eq (include "useTlsGundeck" .) "true" }} + - name: gundeck-cassandra-cert + secret: + secretName: {{ (include "tlsSecretRefGundeck" . | fromYaml).name }} + {{- end }} + {{- if eq (include "useTlsBrig" .) "true" }} + - name: brig-cassandra-cert + secret: + secretName: {{ (include "tlsSecretRefBrig" . | fromYaml).name }} + {{- end }} + {{- if eq (include "useTlsGalley" .) "true" }} + - name: galley-cassandra-cert + secret: + secretName: {{ (include "tlsSecretRefGalley" . | fromYaml).name }} + {{- end }} + {{- if eq (include "useTlsSpar" .) "true" }} + - name: spar-cassandra-cert + secret: + secretName: {{ (include "tlsSecretRefSpar" . | fromYaml).name }} + {{- end }} initContainers: {{- if .Values.enableGundeckMigrations }} - name: gundeck-schema @@ -41,6 +62,16 @@ spec: - gundeck - {{ template "cassandraGundeckReplicationType" . }} - "{{ template "cassandraGundeckReplicationArg" . }}" + {{- if eq (include "useTlsGundeck" .) "true" }} + - --tls-ca-certificate-file + - /certs/gundeck/{{- (include "tlsSecretRefGundeck" . | fromYaml).key }} + {{- end }} + + {{- if eq (include "useTlsGundeck" .) "true" }} + volumeMounts: + - name: gundeck-cassandra-cert + mountPath: "/certs/gundeck" + {{- end }} {{- end }} {{- if .Values.enableBrigMigrations }} @@ -61,6 +92,16 @@ spec: - brig - {{ template "cassandraBrigReplicationType" . }} - "{{ template "cassandraBrigReplicationArg" . }}" + {{- if eq (include "useTlsBrig" .) "true" }} + - --tls-ca-certificate-file + - /certs/brig/{{- (include "tlsSecretRefBrig" . | fromYaml).key }} + {{- end }} + + {{- if eq (include "useTlsBrig" .) "true" }} + volumeMounts: + - name: brig-cassandra-cert + mountPath: "/certs/brig" + {{- end }} {{- end }} {{- if .Values.enableGalleyMigrations }} @@ -81,6 +122,16 @@ spec: - galley - {{ template "cassandraGalleyReplicationType" . }} - "{{ template "cassandraGalleyReplicationArg" . }}" + {{- if eq (include "useTlsGalley" .) "true" }} + - --tls-ca-certificate-file + - /certs/galley/{{- (include "tlsSecretRefGalley" . | fromYaml).key }} + {{- end }} + + {{- if eq (include "useTlsGalley" .) "true" }} + volumeMounts: + - name: galley-cassandra-cert + mountPath: "/certs/galley" + {{- end }} {{- end }} {{- if .Values.enableSparMigrations }} @@ -101,7 +152,17 @@ spec: - spar - {{ template "cassandraSparReplicationType" . }} - "{{ template "cassandraSparReplicationArg" . }}" - {{- end }} + {{- if eq (include "useTlsSpar" .) "true" }} + - --tls-ca-certificate-file + - /certs/spar/{{- (include "tlsSecretRefSpar" . | fromYaml).key }} + {{- end }} + + {{- if eq (include "useTlsSpar" .) "true" }} + volumeMounts: + - name: spar-cassandra-cert + mountPath: "/certs/spar" + {{- end }} + {{- end }} containers: - name: job-done diff --git a/charts/cassandra-migrations/templates/spar-migrate-data.yaml b/charts/cassandra-migrations/templates/spar-migrate-data.yaml index 1b9c48e066..051946ac2b 100644 --- a/charts/cassandra-migrations/templates/spar-migrate-data.yaml +++ b/charts/cassandra-migrations/templates/spar-migrate-data.yaml @@ -43,4 +43,32 @@ spec: - "9042" - --cassandra-keyspace-brig - brig + {{- if eq (include "useTlsBrig" .) "true" }} + - --tls-ca-certificate-file-brig + - /certs/brig/{{- (include "tlsSecretRefBrig" . | fromYaml).key }} + {{- end }} + {{- if eq (include "useTlsSpar" .) "true" }} + - --tls-ca-certificate-file-spar + - /certs/spar/{{- (include "tlsSecretRefSpar" . | fromYaml).key }} + {{- end }} + volumeMounts: + {{- if eq (include "useTlsBrig" .) "true" }} + - name: brig-cassandra-cert + mountPath: "/certs/brig" + {{- end }} + {{- if eq (include "useTlsSpar" .) "true" }} + - name: spar-cassandra-cert + mountPath: "/certs/spar" + {{- end }} + volumes: + {{- if eq (include "useTlsBrig" .) "true" }} + - name: brig-cassandra-cert + secret: + secretName: {{ (include "tlsSecretRefBrig" . | fromYaml).name }} + {{- end }} + {{- if eq (include "useTlsSpar" .) "true" }} + - name: spar-cassandra-cert + secret: + secretName: {{ (include "tlsSecretRefSpar" . | fromYaml).name }} + {{- end }} {{- end }} diff --git a/charts/cassandra-migrations/values.yaml b/charts/cassandra-migrations/values.yaml index bf2a31d1b6..283a010884 100644 --- a/charts/cassandra-migrations/values.yaml +++ b/charts/cassandra-migrations/values.yaml @@ -47,7 +47,29 @@ images: # cassandraGundeck: # host: cassandra-ephemeral-galley # replicationMap: eu-west-1:3 - +# +# To enable TLS/SSL connections provide the certificate as PEM string: +# +# cassandra: +# host: cassandra-external +# replicationFactor: 3 +# tlsCa: +# +# This also works for dedicated service setups. E.g. +# +# cassandraGalley: +# host: cassandra-ephemeral-galley +# replicationMap: eu-west-1:3 +# tlsCa: +# +# You may also directly refer to a Secret resource: +# +# cassandra: +# host: cassandra-external +# replicationFactor: 3 +# tlsCaSecretRef: +# name: +# key: # Overriding the following is only useful during datacenter migration time periods, # where some other job already migrates schemas. diff --git a/charts/elasticsearch-index/templates/_helpers.tpl b/charts/elasticsearch-index/templates/_helpers.tpl index 762fb52c2f..47bf703112 100644 --- a/charts/elasticsearch-index/templates/_helpers.tpl +++ b/charts/elasticsearch-index/templates/_helpers.tpl @@ -7,3 +7,19 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "elasticsearch-index-migrate-cassandra-client-ca" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} diff --git a/charts/elasticsearch-index/templates/cassandra-secret.yaml b/charts/elasticsearch-index/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..93486dd962 --- /dev/null +++ b/charts/elasticsearch-index/templates/cassandra-secret.yaml @@ -0,0 +1,14 @@ +{{- if not (empty .Values.cassandra.tlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-index-migrate-cassandra-client-ca + labels: + app: elasticsearch-index-migrate-data + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.cassandra.tlsCa | b64enc | quote }} +{{- end }} diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml index 3ef47bcf5e..3d54e1f51b 100644 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ b/charts/elasticsearch-index/templates/migrate-data.yaml @@ -43,3 +43,18 @@ spec: - "{{ required "missing elasticsearch-index.galley.host!" .Values.galley.host }}" - --galley-port - "{{ required "missing elasticsearch-index.galley.port!" .Values.galley.port }}" + {{- if eq (include "useCassandraTLS" .Values) "true" }} + - --tls-ca-certificate-file + - /certs/{{- (include "tlsSecretRef" .Values | fromYaml).key }} + {{- end }} + {{- if eq (include "useCassandraTLS" .Values) "true" }} + volumeMounts: + - name: elasticsearch-index-migrate-cassandra-client-ca + mountPath: "/certs" + {{- end }} + {{- if eq (include "useCassandraTLS" .Values) "true" }} + volumes: + - name: elasticsearch-index-migrate-cassandra-client-ca + secret: + secretName: {{ (include "tlsSecretRef" .Values | fromYaml).name }} + {{- end}} diff --git a/charts/elasticsearch-index/values.yaml b/charts/elasticsearch-index/values.yaml index 4cbd2e5110..93e8a97ef6 100644 --- a/charts/elasticsearch-index/values.yaml +++ b/charts/elasticsearch-index/values.yaml @@ -8,6 +8,13 @@ cassandra: # host: port: 9042 keyspace: brig +# To enable TLS provide a CA: +# tlsCa: +# +# Or refer to an existing secret (containing the CA): +# tlsCaSecretRef: +# name: +# key: galley: host: galley port: 8080 diff --git a/charts/galley/templates/_helpers.tpl b/charts/galley/templates/_helpers.tpl index 762fb52c2f..a9de4a20a9 100644 --- a/charts/galley/templates/_helpers.tpl +++ b/charts/galley/templates/_helpers.tpl @@ -7,3 +7,19 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "galley-cassandra" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} diff --git a/charts/galley/templates/cassandra-secret.yaml b/charts/galley/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..eb34aeb30b --- /dev/null +++ b/charts/galley/templates/cassandra-secret.yaml @@ -0,0 +1,15 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} +{{- if not (empty .Values.config.cassandra.tlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: galley-cassandra + labels: + app: galley + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.config.cassandra.tlsCa | b64enc | quote }} +{{- end }} diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml index 690bfd993c..3ac139136d 100644 --- a/charts/galley/templates/configmap.yaml +++ b/charts/galley/templates/configmap.yaml @@ -21,6 +21,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} + {{- if eq (include "useCassandraTLS" .) "true" }} + tlsCa: /etc/wire/galley/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} + {{- end }} brig: host: brig diff --git a/charts/galley/templates/deployment.yaml b/charts/galley/templates/deployment.yaml index a9f2f50fb9..df9eee0c20 100644 --- a/charts/galley/templates/deployment.yaml +++ b/charts/galley/templates/deployment.yaml @@ -36,6 +36,11 @@ spec: - name: "galley-secrets" secret: secretName: "galley" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "galley-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end }} containers: - name: galley image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -49,6 +54,10 @@ spec: mountPath: "/etc/wire/galley/conf" - name: "galley-secrets" mountPath: "/etc/wire/galley/secrets" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "galley-cassandra" + mountPath: "/etc/wire/galley/cassandra" + {{- end }} env: {{- if hasKey .Values.secrets "awsKeyId" }} - name: AWS_ACCESS_KEY_ID diff --git a/charts/galley/templates/tests/galley-integration.yaml b/charts/galley/templates/tests/galley-integration.yaml index e187022837..1fdd9e206a 100644 --- a/charts/galley/templates/tests/galley-integration.yaml +++ b/charts/galley/templates/tests/galley-integration.yaml @@ -40,6 +40,11 @@ spec: - name: "galley-secrets" secret: secretName: "galley" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "galley-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end }} containers: - name: integration image: "{{ .Values.image.repository }}-integration:{{ .Values.image.tag }}" @@ -84,6 +89,10 @@ spec: mountPath: "/etc/wire/integration-secrets" - name: "galley-secrets" mountPath: "/etc/wire/galley/secrets" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "galley-cassandra" + mountPath: "/etc/wire/galley/cassandra" + {{- end }} env: # these dummy values are necessary for Amazonka's "Discover" - name: AWS_ACCESS_KEY_ID diff --git a/charts/galley/values.yaml b/charts/galley/values.yaml index 0de07d8b4c..d96f07b6e7 100644 --- a/charts/galley/values.yaml +++ b/charts/galley/values.yaml @@ -22,6 +22,13 @@ config: cassandra: host: aws-cassandra replicaCount: 3 +# To enable TLS provide a CA: +# tlsCa: +# +# Or refer to an existing secret (containing the CA): +# tlsCaSecretRef: +# name: +# key: enableFederation: false # keep enableFederation default in sync with brig and cargohold chart's config.enableFederation as well as wire-server chart's tags.federation # Not used if enableFederation is false rabbitmq: diff --git a/charts/gundeck/templates/_helpers.tpl b/charts/gundeck/templates/_helpers.tpl index 762fb52c2f..ed317e0b21 100644 --- a/charts/gundeck/templates/_helpers.tpl +++ b/charts/gundeck/templates/_helpers.tpl @@ -7,3 +7,19 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "gundeck-cassandra" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} diff --git a/charts/gundeck/templates/cassandra-secret.yaml b/charts/gundeck/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..68dd7c9d34 --- /dev/null +++ b/charts/gundeck/templates/cassandra-secret.yaml @@ -0,0 +1,15 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} +{{- if not (empty .Values.config.cassandra.tlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: gundeck-cassandra + labels: + app: brig + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.config.cassandra.tlsCa | b64enc | quote }} +{{- end }} diff --git a/charts/gundeck/templates/configmap.yaml b/charts/gundeck/templates/configmap.yaml index 527f521c26..b01a63d844 100644 --- a/charts/gundeck/templates/configmap.yaml +++ b/charts/gundeck/templates/configmap.yaml @@ -25,6 +25,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} + {{- if eq (include "useCassandraTLS" .) "true" }} + tlsCa: /etc/wire/gundeck/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} + {{- end }} redis: host: {{ .redis.host }} diff --git a/charts/gundeck/templates/deployment.yaml b/charts/gundeck/templates/deployment.yaml index 27255185da..20ca798824 100644 --- a/charts/gundeck/templates/deployment.yaml +++ b/charts/gundeck/templates/deployment.yaml @@ -32,6 +32,11 @@ spec: - name: "gundeck-config" configMap: name: "gundeck" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "gundeck-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end}} containers: - name: gundeck image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -43,6 +48,10 @@ spec: volumeMounts: - name: "gundeck-config" mountPath: "/etc/wire/gundeck/conf" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "gundeck-cassandra" + mountPath: "/etc/wire/gundeck/cassandra" + {{- end }} env: {{- if hasKey .Values.secrets "awsKeyId" }} - name: AWS_ACCESS_KEY_ID diff --git a/charts/gundeck/templates/tests/gundeck-integration.yaml b/charts/gundeck/templates/tests/gundeck-integration.yaml index 7f92351be5..8b00f2c986 100644 --- a/charts/gundeck/templates/tests/gundeck-integration.yaml +++ b/charts/gundeck/templates/tests/gundeck-integration.yaml @@ -13,6 +13,11 @@ spec: - name: "gundeck-config" configMap: name: "gundeck" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "gundeck-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end}} containers: - name: integration # TODO: When deployed to staging (or real AWS env), _all_ tests should be run @@ -54,6 +59,10 @@ spec: mountPath: "/etc/wire/integration" - name: "gundeck-config" mountPath: "/etc/wire/gundeck/conf" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "gundeck-cassandra" + mountPath: "/etc/wire/gundeck/cassandra" + {{- end }} env: # these dummy values are necessary for Amazonka's "Discover" - name: AWS_ACCESS_KEY_ID diff --git a/charts/gundeck/values.yaml b/charts/gundeck/values.yaml index 2841636144..87c338a69d 100644 --- a/charts/gundeck/values.yaml +++ b/charts/gundeck/values.yaml @@ -20,6 +20,13 @@ config: logNetStrings: false cassandra: host: aws-cassandra +# To enable TLS provide a CA: +# tlsCa: +# +# Or refer to an existing secret (containing the CA): +# tlsCaSecretRef: +# name: +# key: redis: host: redis-ephemeral-master port: 6379 diff --git a/charts/integration/templates/_helpers.tpl b/charts/integration/templates/_helpers.tpl index e138d2f1bb..e278f287d1 100644 --- a/charts/integration/templates/_helpers.tpl +++ b/charts/integration/templates/_helpers.tpl @@ -36,4 +36,20 @@ {{- define "integrationTestHelperNewLabels" -}} {{- (semverCompare ">= 1.23-0" (include "kubeVersion" .)) -}} -{{- end -}} \ No newline at end of file +{{- end -}} + +{{- define "useCassandraTLS" -}} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "integration-cassandra" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} diff --git a/charts/integration/templates/cassandra-secret.yaml b/charts/integration/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..dd76b65067 --- /dev/null +++ b/charts/integration/templates/cassandra-secret.yaml @@ -0,0 +1,15 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} +{{- if not (empty .Values.config.cassandra.tlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: integration-cassandra + labels: + app: integration + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.config.cassandra.tlsCa | b64enc | quote }} +{{- end }} diff --git a/charts/integration/templates/configmap.yaml b/charts/integration/templates/configmap.yaml index 99a247203a..e18128cbf5 100644 --- a/charts/integration/templates/configmap.yaml +++ b/charts/integration/templates/configmap.yaml @@ -120,4 +120,8 @@ data: federatorExternalPort: {{ $dynamicBackend.federatorExternalPort }} {{- end }} cassandra: -{{ toYaml .Values.config.cassandra | indent 6}} + host: {{ .Values.config.cassandra.host }} + port: {{ .Values.config.cassandra.port }} + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + tlsCa: /etc/wire/galley/cassandra/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} + {{- end }} diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 2fe7718fa5..b3abde74cc 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -75,7 +75,11 @@ spec: - name: "nginz-secrets" secret: secretName: "nginz" - + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: integration-cassandra + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end }} restartPolicy: Never initContainers: @@ -86,6 +90,11 @@ spec: securityContext: {{- toYaml .Values.podSecurityContext | nindent 6 }} {{- end }} + volumeMounts: + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "integration-cassandra" + mountPath: "/certs" + {{- end }} env: - name: INTEGRATION_DYNAMIC_BACKENDS_POOLSIZE value: "{{ .Values.config.dynamicBackendsPoolsize }}" @@ -111,7 +120,14 @@ spec: - | set -euo pipefail # FUTUREWORK: Do all of this in the integration test binary - integration-dynamic-backends-db-schemas.sh --host {{ .Values.config.cassandra.host }} --port {{ .Values.config.cassandra.port }} --replication-factor {{ .Values.config.cassandra.replicationFactor }} + integration-dynamic-backends-db-schemas.sh \ + --host {{ .Values.config.cassandra.host }} \ + --port {{ .Values.config.cassandra.port }} \ + --replication-factor {{ .Values.config.cassandra.replicationFactor }} \ + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + --tls-ca-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }} + {{ end }} + integration-dynamic-backends-brig-index.sh --elasticsearch-server http://{{ .Values.config.elasticsearch.host }}:9200 integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }} integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }} @@ -212,6 +228,23 @@ spec: - name: nginz-secrets mountPath: /etc/wire/nginz/secrets + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "integration-cassandra" + mountPath: "/certs" + + - name: "integration-cassandra" + mountPath: "/etc/wire/brig/cassandra" + + - name: "integration-cassandra" + mountPath: "/etc/wire/galley/cassandra" + + - name: "integration-cassandra" + mountPath: "/etc/wire/gundeck/cassandra" + + - name: "integration-cassandra" + mountPath: "/etc/wire/spar/cassandra" + {{- end }} + env: # these dummy values are necessary for Amazonka's "Discover" - name: AWS_ACCESS_KEY_ID diff --git a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml index b91292e276..6fd8de25b9 100644 --- a/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml +++ b/charts/k8ssandra-test-cluster/templates/check-cluster-job.yaml @@ -12,7 +12,21 @@ spec: containers: - name: cassandra image: cassandra:3.11 + {{- if not .Values.client_encryption_options.enabled }} command: ["cqlsh", "k8ssandra-cluster-datacenter-1-service"] + {{- else }} + command: ["cqlsh", "--ssl", "k8ssandra-cluster-datacenter-1-service"] + env: + - name: SSL_CERTFILE + value: "/certs/ca.crt" + volumeMounts: + - name: cassandra-jks-keystore + mountPath: "/certs" + volumes: + - name: cassandra-jks-keystore + secret: + secretName: cassandra-jks-keystore + {{- end }} restartPolicy: OnFailure # Default is 6 retries. 8 is a bit arbitrary, but should be sufficient for # low resource environments (e.g. Wire-in-a-box.) diff --git a/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml b/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml new file mode 100644 index 0000000000..52e6f2d0eb --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/jks-store-pass.yaml @@ -0,0 +1,9 @@ +{{- if .Values.client_encryption_options.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: jks-password + namespace: {{ .Release.Namespace }} +data: + keystore-pass: {{ .Values.client_encryption_options.keystorePassword | b64enc }} +{{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml b/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml index 50560a52d5..35197d8b8f 100644 --- a/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml +++ b/charts/k8ssandra-test-cluster/templates/k8ssandra-cluster.yaml @@ -26,6 +26,10 @@ spec: gc_g1_max_gc_pause_ms: 300 gc_g1_initiating_heap_occupancy_percent: 55 gc_g1_parallel_threads: 16 + cassandraYaml: + client_encryption_options: + enabled: {{ .Values.client_encryption_options.enabled }} + optional: {{ .Values.client_encryption_options.optional }} datacenters: - metadata: name: datacenter-1 @@ -38,6 +42,21 @@ spec: resources: requests: storage: {{ .Values.storageSize }} + {{- if .Values.client_encryption_options.enabled }} + clientEncryptionStores: + keystoreSecretRef: + name: cassandra-jks-keystore + key: keystore.jks + keystorePasswordSecretRef: + key: keystore-pass + name: jks-password + truststoreSecretRef: + name: cassandra-jks-keystore + key: truststore.jks + truststorePasswordSecretRef: + key: keystore-pass + name: jks-password + {{- end }} reaper: autoScheduling: enabled: true diff --git a/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml b/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml new file mode 100644 index 0000000000..4b06b31110 --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/tls-certificate-bundle.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.client_encryption_options.enabled .Values.syncCACertToSecret }} +# Let trust-manager sync the CA PEM (and only that!) into secrets named +# `k8ssandra-tls-ca-certificate-` in all configured namespaces or only +# one if syncCACertNamespace is defined. This way we can hide the private key +# from public. +apiVersion: trust.cert-manager.io/v1alpha1 +kind: Bundle +metadata: + name: k8ssandra-tls-ca-certificate + namespace: {{ .Release.Namespace }} +spec: + sources: + - secret: + name: "cassandra-jks-keystore" + key: "ca.crt" + target: + secret: + key: "ca.crt" + {{- if hasKey .Values "syncCACertNamespace" }} + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.syncCACertNamespace }} + {{- end }} +{{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml b/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml new file mode 100644 index 0000000000..c7efd99c8a --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/tls-certificate.yaml @@ -0,0 +1,44 @@ +{{- if .Values.client_encryption_options.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cassandra-certificate + namespace: {{ .Release.Namespace }} +spec: + # Secret names are always required. + secretName: cassandra-jks-keystore + duration: 2160h # 90d + renewBefore: 360h # 15d + subject: + organizations: + - PIT squad + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + # commonName: example.com + isCA: false + privateKey: + algorithm: RSA + encoding: PKCS1 + size: 2048 + usages: + - server auth + - client auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - k8ssandra-cluster-datacenter-1-service.{{ .Release.Namespace }}.svc.cluster.local + - k8ssandra-cluster-datacenter-1-service + issuerRef: + name: ca-issuer + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: Issuer + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io + keystores: + jks: + create: true + passwordSecretRef: # Password used to encrypt the keystore + key: keystore-pass + name: jks-password +{{- end }} diff --git a/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml b/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml new file mode 100644 index 0000000000..65bc3dbad3 --- /dev/null +++ b/charts/k8ssandra-test-cluster/templates/tls-issuer.yaml @@ -0,0 +1,9 @@ +{{- if .Values.client_encryption_options.enabled }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ca-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +{{- end }} diff --git a/charts/k8ssandra-test-cluster/values.yaml b/charts/k8ssandra-test-cluster/values.yaml index 3aabc8db1a..a34ca0da5f 100644 --- a/charts/k8ssandra-test-cluster/values.yaml +++ b/charts/k8ssandra-test-cluster/values.yaml @@ -11,3 +11,22 @@ storageClassName: hcloud-volumes-encrypted # storage, it's fine to request 10GB. The memory units are described here: # https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory storageSize: 10G + +# These options relate to the client_encryption_options described in: +# https://cassandra.apache.org/doc/stable/cassandra/configuration/cass_yaml_file.html#client_encryption_options +client_encryption_options: + enabled: false + optional: true + # The password could be secured better. However, this chart is meant to be + # used as test setup. And, protecting a self-signed certificate isn't very + # useful. + keystorePassword: password + +# Guard the private key by syncing only the CA certificate to +# `k8ssandra-test-cluster-tls-ca-certificate` secrets. Requires `trust-manager` +# Helm chart to be installed (including CRDs.) +syncCACertToSecret: false + +# Limit syncing to this namespace. Otherwise, the secret is synced to all +# namespaces. +# syncCACertNamespace: diff --git a/charts/spar/templates/_helpers.tpl b/charts/spar/templates/_helpers.tpl index 762fb52c2f..958a0acc36 100644 --- a/charts/spar/templates/_helpers.tpl +++ b/charts/spar/templates/_helpers.tpl @@ -1,4 +1,3 @@ - {{/* Allow KubeVersion to be overridden. */}} {{- define "kubeVersion" -}} {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} @@ -7,3 +6,19 @@ {{- define "includeSecurityContext" -}} {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} {{- end -}} + +{{- define "useCassandraTLS" -}} +{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} +{{- end -}} + +{{/* Return a Dict of TLS CA secret name and key +This is used to switch between provided secret (e.g. by cert-manager) and +created one (in case the CA is provided as PEM string.) +*/}} +{{- define "tlsSecretRef" -}} +{{- if .cassandra.tlsCaSecretRef -}} +{{ .cassandra.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "spar-cassandra" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} diff --git a/charts/spar/templates/cassandra-secret.yaml b/charts/spar/templates/cassandra-secret.yaml new file mode 100644 index 0000000000..0a480e01bb --- /dev/null +++ b/charts/spar/templates/cassandra-secret.yaml @@ -0,0 +1,15 @@ +{{/* Secret for the provided Cassandra TLS CA. */}} +{{- if not (empty .Values.config.cassandra.tlsCa) }} +apiVersion: v1 +kind: Secret +metadata: + name: spar-cassandra + labels: + app: spar + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: "{{ .Release.Name }}" + heritage: "{{ .Release.Service }}" +type: Opaque +data: + ca.pem: {{ .Values.config.cassandra.tlsCa | b64enc | quote }} +{{- end }} diff --git a/charts/spar/templates/configmap.yaml b/charts/spar/templates/configmap.yaml index 98711a4679..8ae7b5c371 100644 --- a/charts/spar/templates/configmap.yaml +++ b/charts/spar/templates/configmap.yaml @@ -25,6 +25,9 @@ data: {{- if hasKey .cassandra "filterNodesByDatacentre" }} filterNodesByDatacentre: {{ .cassandra.filterNodesByDatacentre }} {{- end }} + {{- if eq (include "useCassandraTLS" .) "true" }} + tlsCa: /etc/wire/spar/cassandra/{{- (include "tlsSecretRef" . | fromYaml).key }} + {{- end }} maxttlAuthreq: {{ .maxttlAuthreq }} maxttlAuthresp: {{ .maxttlAuthresp }} diff --git a/charts/spar/templates/deployment.yaml b/charts/spar/templates/deployment.yaml index 6d65b5d151..c09fc2beac 100644 --- a/charts/spar/templates/deployment.yaml +++ b/charts/spar/templates/deployment.yaml @@ -30,6 +30,11 @@ spec: - name: "spar-config" configMap: name: "spar" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "spar-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end}} containers: - name: spar image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" @@ -41,6 +46,10 @@ spec: volumeMounts: - name: "spar-config" mountPath: "/etc/wire/spar/conf" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "spar-cassandra" + mountPath: "/etc/wire/spar/cassandra" + {{- end }} env: {{- with .Values.config.proxy }} {{- if .httpProxy }} diff --git a/charts/spar/templates/tests/spar-integration.yaml b/charts/spar/templates/tests/spar-integration.yaml index ff937f3d18..9cae732bfb 100644 --- a/charts/spar/templates/tests/spar-integration.yaml +++ b/charts/spar/templates/tests/spar-integration.yaml @@ -16,6 +16,11 @@ spec: - name: "spar-config" configMap: name: "spar" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "spar-cassandra" + secret: + secretName: {{ (include "tlsSecretRef" .Values.config | fromYaml).name }} + {{- end}} containers: - name: integration image: "{{ .Values.image.repository }}-integration:{{ .Values.image.tag }}" @@ -56,6 +61,10 @@ spec: mountPath: "/etc/wire/integration" - name: "spar-config" mountPath: "/etc/wire/spar/conf" + {{- if eq (include "useCassandraTLS" .Values.config) "true" }} + - name: "spar-cassandra" + mountPath: "/etc/wire/spar/cassandra" + {{- end }} resources: requests: memory: "512Mi" diff --git a/charts/spar/values.yaml b/charts/spar/values.yaml index 073fd5b0ee..9cb6c2c969 100644 --- a/charts/spar/values.yaml +++ b/charts/spar/values.yaml @@ -17,6 +17,13 @@ service: config: cassandra: host: aws-cassandra +# To enable TLS provide a CA: +# tlsCa: +# +# Or refer to an existing secret (containing the CA): +# tlsCaSecretRef: +# name: +# key: richInfoLimit: 5000 maxScimTokens: 0 logLevel: Info diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index fd8e6034ad..00b2ce6d56 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -807,3 +807,37 @@ CSP_EXTRA_SCRIPT_SRC: https://*.[[hostname]] CSP_EXTRA_STYLE_SRC: https://*.[[hostname]] CSP_EXTRA_WORKER_SRC: https://*.[[hostname]] ``` + +## TLS-encrypted Cassandra connections + +By default, all connections to Cassandra by the Wire backend are unencrypted. To +configure client-side TLS-encrypted connections (where the Wire backend is the +client), a **C**ertificate **A**uthority in PEM format needs to be configured. + +The ways differ regarding the kind of program: +- *Services* expect a `cassandra.tlsCa: ` attribute in their config file. +- *CLI commands* (e.g. migrations) accept a `--tls-ca-certificate-file ` parameter. + +When a CA PEM file is configured, all Cassandra connections are opened with TLS +encryption i.e. there is no fallback to unencrypted connections. This ensures +that connections that are expected to be secure, would not silently and +unnoticed be insecure. + +In Helm charts, the CA PEM is either provided as multiline string in the +`cassandra.tlsCa` attribute or as a reference to a `Secret` in +`cassandra.tlsCaSecretRef.name` and `cassandra.tlsCaSecretRef.key`. The `name` +is the name of the `Secret`, the `key` is the entry in it. Such a `Secret` can +e.g. be created by `cert-manager`. + +The CA may be self-signed. It is used to validate the certificate of the +Cassandra server. + +How to configure Cassandra to accept TLS-encrypted connections in general is +beyond the scope of this document. The `k8ssandra-test-cluster` Helm chart +provides an example how to do this for the Kubernetes solution *K8ssandra*. In +the example `cert-manager` generates a `Certificate` including Java KeyStores, +then `trust-manager` creates synchronized `Secret`s to make only the CA PEM +accessible to services (and not the private key.) + +The corresponding Cassandra options are described in Cassandra's documentation: +[client_encryption_options](https://cassandra.apache.org/doc/stable/cassandra/configuration/cass_yaml_file.html#client_encryption_options) diff --git a/hack/bin/integration-setup-federation.sh b/hack/bin/integration-setup-federation.sh index 294c719cea..d7e19e66ae 100755 --- a/hack/bin/integration-setup-federation.sh +++ b/hack/bin/integration-setup-federation.sh @@ -5,6 +5,7 @@ set -euo pipefail DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TOP_LEVEL="$DIR/../.." export NAMESPACE=${NAMESPACE:-test-integration} +# Available $HELMFILE_ENV profiles: default, default-ssl, kind, kind-ssl HELMFILE_ENV=${HELMFILE_ENV:-default} CHARTS_DIR="${TOP_LEVEL}/.local/charts" HELM_PARALLELISM=${HELM_PARALLELISM:-1} diff --git a/hack/bin/integration-teardown-federation.sh b/hack/bin/integration-teardown-federation.sh index fba9685490..01791d223c 100755 --- a/hack/bin/integration-teardown-federation.sh +++ b/hack/bin/integration-teardown-federation.sh @@ -5,6 +5,7 @@ TOP_LEVEL="$DIR/../.." set -ex +HELMFILE_ENV=${HELMFILE_ENV:-default} NAMESPACE=${NAMESPACE:-test-integration} export NAMESPACE_1="$NAMESPACE" export NAMESPACE_2="$NAMESPACE-fed2" @@ -22,6 +23,6 @@ else fi . "$DIR/helm_overrides.sh" -helmfile --file "${TOP_LEVEL}/hack/helmfile.yaml" destroy --skip-deps --skip-charts --concurrency 0 || echo "Failed to delete helm deployments, ignoring this failure as next steps will the destroy namespaces anyway." +helmfile --environment "$HELMFILE_ENV" --file "${TOP_LEVEL}/hack/helmfile.yaml" destroy --skip-deps --skip-charts --concurrency 0 || echo "Failed to delete helm deployments, ignoring this failure as next steps will the destroy namespaces anyway." kubectl delete namespace "$NAMESPACE_1" "$NAMESPACE_2" diff --git a/hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl b/hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl new file mode 100644 index 0000000000..b550775fbd --- /dev/null +++ b/hack/helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl @@ -0,0 +1,9 @@ +storageClassName: {{ .Values.storageClass }} + +client_encryption_options: + enabled: true + optional: false + # This password is used to decrypt the internal Java Keystore. No need to be + # careful about it: It's worthless without cluster access and even with it, + # you could impersonate as a test cassandra db... + keystorePassword: p4ssw0rd diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 49d6812bb4..02c8c47499 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -18,15 +18,26 @@ tags: cassandra-migrations: imagePullPolicy: {{ .Values.imagePullPolicy }} cassandra: - host: cassandra-ephemeral + host: {{ .Values.cassandraHost }} replicationFactor: 1 + {{- if .Values.useK8ssandraSSL.enabled }} + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" + {{- end }} + elasticsearch-index: imagePullPolicy: {{ .Values.imagePullPolicy }} elasticsearch: host: elasticsearch-ephemeral index: directory_test cassandra: - host: cassandra-ephemeral + host: {{ .Values.cassandraHost }} + {{- if .Values.useK8ssandraSSL.enabled }} + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" + {{- end }} brig: replicaCount: 1 @@ -41,8 +52,13 @@ brig: teamCreatorWelcome: https://teams.wire.com/login teamMemberWelcome: https://wire.com/download cassandra: - host: cassandra-ephemeral + host: {{ .Values.cassandraHost }} replicaCount: 1 + {{- if .Values.useK8ssandraSSL.enabled }} + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" + {{- end }} elasticsearch: host: elasticsearch-ephemeral index: directory_test @@ -186,8 +202,13 @@ galley: imagePullPolicy: {{ .Values.imagePullPolicy }} config: cassandra: - host: cassandra-ephemeral + host: {{ .Values.cassandraHost }} replicaCount: 1 + {{- if .Values.useK8ssandraSSL.enabled }} + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" + {{- end }} enableFederation: true # keep in sync with brig.config.enableFederation, cargohold.config.enableFederation and tags.federator! settings: maxConvAndTeamSize: 16 @@ -248,8 +269,13 @@ gundeck: memory: 1024Mi config: cassandra: - host: cassandra-ephemeral + host: {{ .Values.cassandraHost }} replicaCount: 1 + {{- if .Values.useK8ssandraSSL.enabled }} + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" + {{- end }} redis: host: redis-ephemeral-master connectionMode: master @@ -322,7 +348,12 @@ spar: config: tlsDisableCertValidation: true cassandra: - host: cassandra-ephemeral + host: {{ .Values.cassandraHost }} + {{- if .Values.useK8ssandraSSL.enabled }} + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" + {{- end }} logLevel: Debug domain: zinfra.io appUri: http://spar:8080/ @@ -380,8 +411,17 @@ background-worker: integration: ingress: class: "nginx-{{ .Release.Namespace }}" - {{- if .Values.uploadXml }} config: + cassandra: + host: {{ .Values.cassandraHost }} + port: 9042 + replicationFactor: 1 + {{- if .Values.useK8ssandraSSL.enabled }} + tlsCaSecretRef: + name: cassandra-jks-keystore + key: ca.crt + {{- end }} + {{- if .Values.uploadXml }} uploadXml: baseUrl: {{ .Values.uploadXml.baseUrl }} secrets: diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index 878cb016f5..e82a1373a3 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -11,18 +11,39 @@ helmDefaults: timeout: 600 devel: true createNamespace: true - environments: default: values: - ./helm_vars/common.yaml.gotmpl - imagePullPolicy: Always - storageClass: hcloud-volumes + - cassandraHost: cassandra-ephemeral + - useK8ssandraSSL: + enabled: false + default-ssl: + values: + - ./helm_vars/common.yaml.gotmpl + - imagePullPolicy: Always + - storageClass: hcloud-volumes + - cassandraHost: k8ssandra-cluster-datacenter-1-service + - useK8ssandraSSL: + enabled: true kind: values: - ./helm_vars/common.yaml.gotmpl - imagePullPolicy: Never - storageClass: standard + - cassandraHost: cassandra-ephemeral + - useK8ssandraSSL: + enabled: false + kind-ssl: + values: + - ./helm_vars/common.yaml.gotmpl + - imagePullPolicy: Never + - storageClass: standard + - cassandraHost: k8ssandra-cluster-datacenter-1-service + - useK8ssandraSSL: + enabled: true --- repositories: - name: stable @@ -46,7 +67,6 @@ releases: chart: '../.local/charts/fake-aws' values: - './helm_vars/fake-aws/values.yaml' - - name: 'databases-ephemeral' namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/databases-ephemeral' @@ -55,6 +75,20 @@ releases: namespace: '{{ .Values.namespace2 }}' chart: '../.local/charts/databases-ephemeral' + - name: k8ssandra-test-cluster + chart: '../.local/charts/k8ssandra-test-cluster' + namespace: '{{ .Values.namespace1 }}' + values: + - './helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl' + condition: useK8ssandraSSL.enabled + + - name: k8ssandra-test-cluster + chart: '../.local/charts/k8ssandra-test-cluster' + namespace: '{{ .Values.namespace2 }}' + values: + - './helm_vars/k8ssandra-test-cluster/values.yaml.gotmpl' + condition: useK8ssandraSSL.enabled + - name: 'rabbitmq' namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/rabbitmq' diff --git a/integration/default.nix b/integration/default.nix index efdcfb7e04..84d88787e4 100644 --- a/integration/default.nix +++ b/integration/default.nix @@ -29,6 +29,7 @@ , filepath , gitignoreSource , hex +, HsOpenSSL , http-client , http-types , kan-extensions @@ -105,6 +106,7 @@ mkDerivation { extra filepath hex + HsOpenSSL http-client http-types kan-extensions diff --git a/integration/integration.cabal b/integration/integration.cabal index 6bf252ef0d..6ed62b2c61 100644 --- a/integration/integration.cabal +++ b/integration/integration.cabal @@ -171,6 +171,7 @@ library , extra , filepath , hex + , HsOpenSSL , http-client , http-types , kan-extensions diff --git a/integration/test/Testlib/Env.hs b/integration/test/Testlib/Env.hs index 2d98764ecf..63b99d9633 100644 --- a/integration/test/Testlib/Env.hs +++ b/integration/test/Testlib/Env.hs @@ -12,9 +12,12 @@ import Data.Map qualified as Map import Data.Maybe (fromMaybe) import Data.Set (Set) import Data.Set qualified as Set +import Data.Traversable (for) import Data.Yaml qualified as Yaml import Database.CQL.IO qualified as Cassandra import Network.HTTP.Client qualified as HTTP +import OpenSSL.Session qualified as OpenSSL +import System.Directory import System.Environment (lookupEnv) import System.Exit import System.FilePath @@ -53,12 +56,25 @@ mkGlobalEnv cfgFile = do if last ps == "services" then Just (joinPath (init ps)) else Nothing + getCassCertFilePath :: IO (Maybe FilePath) = + maybe + (pure Nothing) + ( \certFilePath -> + if isAbsolute certFilePath + then pure $ Just certFilePath + else for devEnvProjectRoot $ \projectRoot -> makeAbsolute $ combine projectRoot certFilePath + ) + intConfig.cassandra.cassTlsCa manager <- liftIO $ HTTP.newManager HTTP.defaultManagerSettings - let cassSettings = + + mbCassCertFilePath <- liftIO $ getCassCertFilePath + mbSSLContext <- liftIO $ createSSLContext mbCassCertFilePath + let basicCassSettings = Cassandra.defSettings - & Cassandra.setContacts intConfig.cassandra.host [] - & Cassandra.setPortNumber (fromIntegral intConfig.cassandra.port) + & Cassandra.setContacts intConfig.cassandra.cassHost [] + & Cassandra.setPortNumber (fromIntegral intConfig.cassandra.cassPort) + cassSettings = maybe basicCassSettings (\sslCtx -> Cassandra.setSSLContext sslCtx basicCassSettings) mbSSLContext cassClient <- Cassandra.init cassSettings let resources = backendResources (Map.elems intConfig.dynamicBackends) resourcePool <- @@ -92,6 +108,21 @@ mkGlobalEnv cfgFile = do gTempDir = tempDir, gTimeOutSeconds = timeOutSeconds } + where + createSSLContext :: Maybe FilePath -> IO (Maybe OpenSSL.SSLContext) + createSSLContext (Just certFilePath) = do + print ("TLS: Connecting to Cassandra with TLS. Provided CA path:" ++ certFilePath) + sslContext <- OpenSSL.context + OpenSSL.contextSetCAFile sslContext certFilePath + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = True, + vpClientOnce = True, + vpCallback = Nothing + } + pure $ Just sslContext + createSSLContext Nothing = pure Nothing mkEnv :: GlobalEnv -> Codensity IO Env mkEnv ge = do diff --git a/integration/test/Testlib/Types.hs b/integration/test/Testlib/Types.hs index b4f0711753..d08773d000 100644 --- a/integration/test/Testlib/Types.hs +++ b/integration/test/Testlib/Types.hs @@ -17,6 +17,7 @@ import Data.ByteString qualified as BS import Data.ByteString.Char8 qualified as C8 import Data.ByteString.Lazy qualified as L import Data.CaseInsensitive qualified as CI +import Data.Char (toLower) import Data.Default import Data.Functor import Data.IORef @@ -117,7 +118,7 @@ data IntegrationConfig = IntegrationConfig backendTwo :: BackendConfig, dynamicBackends :: Map String DynamicBackendConfig, rabbitmq :: RabbitMQConfig, - cassandra :: HostPort + cassandra :: CassandraConfig } deriving (Show, Generic) @@ -169,6 +170,23 @@ data HostPort = HostPort instance FromJSON HostPort +data CassandraConfig = CassandraConfig + { cassHost :: String, + cassPort :: Word16, + cassTlsCa :: Maybe FilePath + } + deriving (Show, Generic) + +instance FromJSON CassandraConfig where + parseJSON = genericParseJSON defaultOptions {fieldLabelModifier = lowerFirst . dropPrefix} + where + lowerFirst :: String -> String + lowerFirst (x : xs) = toLower x : xs + lowerFirst [] = "" + + dropPrefix :: String -> String + dropPrefix = Prelude.drop (length "cass") + -- | Initialised once per test. data Env = Env { serviceMap :: Map String ServiceMap, diff --git a/libs/cassandra-util/cassandra-util.cabal b/libs/cassandra-util/cassandra-util.cabal index 2612b0ee9b..af2e009420 100644 --- a/libs/cassandra-util/cassandra-util.cabal +++ b/libs/cassandra-util/cassandra-util.cabal @@ -15,6 +15,9 @@ library Cassandra Cassandra.CQL Cassandra.Exec + Cassandra.Helpers + Cassandra.MigrateSchema + Cassandra.Options Cassandra.Schema Cassandra.Settings Cassandra.Util @@ -77,6 +80,7 @@ library , cql-io >=0.14 , cql-io-tinylog , exceptions >=0.6 + , HsOpenSSL , imports , lens >=4.4 , lens-aeson >=1.0 diff --git a/libs/cassandra-util/default.nix b/libs/cassandra-util/default.nix index 5e634fad61..c7b1451a36 100644 --- a/libs/cassandra-util/default.nix +++ b/libs/cassandra-util/default.nix @@ -11,6 +11,7 @@ , cql-io-tinylog , exceptions , gitignoreSource +, HsOpenSSL , imports , lens , lens-aeson @@ -36,6 +37,7 @@ mkDerivation { cql-io cql-io-tinylog exceptions + HsOpenSSL imports lens lens-aeson diff --git a/libs/cassandra-util/src/Cassandra/Helpers.hs b/libs/cassandra-util/src/Cassandra/Helpers.hs new file mode 100644 index 0000000000..8a260d530b --- /dev/null +++ b/libs/cassandra-util/src/Cassandra/Helpers.hs @@ -0,0 +1,25 @@ +module Cassandra.Helpers where + +import Data.Aeson.TH +import Imports + +-- | Convenient helper to convert record field names to use as YAML fields. +-- NOTE: We typically use this for options in the configuration files! +-- If you are looking into converting record field name to JSON to be used +-- over the API, look for toJSONFieldName in the Data.Json.Util module. +-- It converts field names into snake_case +-- +-- Example: +-- newtype TeamName = TeamName { teamName :: Text } +-- deriveJSON toJSONFieldName ''teamName +-- +-- would generate {To/From}JSON instances where +-- the field name is "teamName" +toOptionFieldName :: Options +toOptionFieldName = defaultOptions {fieldLabelModifier = lowerFirst . dropPrefix} + where + lowerFirst :: String -> String + lowerFirst (x : xs) = toLower x : xs + lowerFirst [] = "" + dropPrefix :: String -> String + dropPrefix = dropWhile ('_' ==) diff --git a/libs/cassandra-util/src/Cassandra/MigrateSchema.hs b/libs/cassandra-util/src/Cassandra/MigrateSchema.hs new file mode 100644 index 0000000000..33b0825834 --- /dev/null +++ b/libs/cassandra-util/src/Cassandra/MigrateSchema.hs @@ -0,0 +1,133 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} + +module Cassandra.MigrateSchema (migrateSchema) where + +import Cassandra (Client, Consistency (All, One), Keyspace (Keyspace), PrepQuery, QueryString (QueryString), R, S, Version (V4), W, params, query, query1, retry, runClient, write, x1) +import Cassandra.Schema +import Cassandra.Settings (Policy, defSettings, initialContactsPlain, setConnectTimeout, setContacts, setLogger, setMaxConnections, setPolicy, setPoolStripes, setPortNumber, setProtocolVersion, setResponseTimeout, setSendTimeout) +import Cassandra.Util (initCassandra) +import Control.Retry +import Data.List.NonEmpty qualified as NonEmpty +import Data.Text (pack) +import Data.Text.Lazy (fromStrict) +import Data.Time.Clock +import Data.UUID (UUID) +import Database.CQL.IO (Policy (Policy, acceptable, current, display, hostCount, onEvent, select, setup), schema) +import Database.CQL.IO.Tinylog qualified as CT +import Imports hiding (All, fromString, init, intercalate, log) +import System.Logger qualified as Log + +-- FUTUREWORK: We could use the System.Logger.Class here in the future, but we don't have a ReaderT IO here (yet) +migrateSchema :: Log.Logger -> MigrationOpts -> [Migration] -> IO () +migrateSchema l o ms = do + hosts <- initialContactsPlain $ pack (migHost o) + let cqlSettings = + setLogger (CT.mkLogger l) + . setContacts (NonEmpty.head hosts) (NonEmpty.tail hosts) + . setPortNumber (fromIntegral $ migPort o) + . setMaxConnections 1 + . setPoolStripes 1 + -- 'migrationPolicy' ensures we only talk to one host for all queries + -- required for correct functioning of 'waitForSchemaConsistency' + . setPolicy migrationPolicy + -- use higher timeouts on schema migrations to reduce the probability + -- of a timeout happening during 'migAction' or 'metaInsert', + -- as that can lead to a state where schema migrations cannot be re-run + -- without manual action. + -- (due to e.g. "cannot create table X, already exists" errors) + . setConnectTimeout 20 + . setSendTimeout 20 + . setResponseTimeout 50 + . setProtocolVersion V4 + $ defSettings + cas <- initCassandra cqlSettings o.migTlsCa l + runClient cas $ do + let keyspace = Keyspace . migKeyspace $ o + when (migReset o) $ do + info "Dropping keyspace." + void $ schema (dropKeyspace keyspace) (params All ()) + createKeyspace keyspace (migRepl o) + useKeyspace keyspace + void $ schema metaCreate (params All ()) + migrations <- newer <$> schemaVersion + if null migrations + then info "No new migrations." + else info "New migrations found." + forM_ migrations $ \Migration {..} -> do + info $ "[" <> pack (show migVersion) <> "] " <> migText + migAction + now <- liftIO getCurrentTime + write metaInsert (params All (migVersion, migText, now)) + info "Waiting for schema version consistency across peers..." + waitForSchemaConsistency + info "... done waiting." + where + newer v = + dropWhile (maybe (const False) (>=) v . migVersion) + . sortBy (\x y -> migVersion x `compare` migVersion y) + $ ms + info = liftIO . Log.log l Log.Info . Log.msg + dropKeyspace :: Keyspace -> QueryString S () () + dropKeyspace (Keyspace k) = QueryString $ "drop keyspace if exists \"" <> fromStrict k <> "\"" + metaCreate :: QueryString S () () + metaCreate = "create columnfamily if not exists meta (id int, version int, descr text, date timestamp, primary key (id, version))" + metaInsert :: QueryString W (Int32, Text, UTCTime) () + metaInsert = "insert into meta (id, version, descr, date) values (1,?,?,?)" + +-- | Retrieve and compare local and peer system schema versions. +-- if they don't match, retry once per second for 30 seconds +waitForSchemaConsistency :: Client () +waitForSchemaConsistency = do + void $ retryWhileN 30 inDisagreement getSystemVersions + where + getSystemVersions :: Client (UUID, [UUID]) + getSystemVersions = do + -- These two sub-queries must be made to the same node. + -- (comparing local from node A and peers from node B wouldn't be correct) + -- using the custom 'migrationPolicy' when connecting to cassandra ensures this. + mbLocalVersion <- systemLocalVersion + peers <- systemPeerVersions + case mbLocalVersion of + Just localVersion -> pure $ (localVersion, peers) + Nothing -> error "No system_version in system.local (should never happen)" + inDisagreement :: (UUID, [UUID]) -> Bool + inDisagreement (localVersion, peers) = not $ all (== localVersion) peers + systemLocalVersion :: Client (Maybe UUID) + systemLocalVersion = fmap runIdentity <$> qry + where + qry = retry x1 (query1 cql (params One ())) + cql :: PrepQuery R () (Identity UUID) + cql = "select schema_version from system.local" + systemPeerVersions :: Client [UUID] + systemPeerVersions = fmap runIdentity <$> qry + where + qry = retry x1 (query cql (params One ())) + cql :: PrepQuery R () (Identity UUID) + cql = "select schema_version from system.peers" + +retryWhileN :: (MonadIO m) => Int -> (a -> Bool) -> m a -> m a +retryWhileN n f m = + retrying + (constantDelay 1000000 <> limitRetries n) + (const (pure . f)) + (const m) + +-- | The migrationPolicy selects only one and always the same host +migrationPolicy :: IO Policy +migrationPolicy = do + h <- newIORef Nothing + pure $ + Policy + { setup = setHost h, + onEvent = const $ pure (), + select = readIORef h, + acceptable = const $ pure True, + hostCount = fromIntegral . length . maybeToList <$> readIORef h, + display = ("migrationPolicy: " ++) . show <$> readIORef h, + current = maybeToList <$> readIORef h + } + where + setHost h (a : _) _ = writeIORef h (Just a) + setHost _ _ _ = pure () diff --git a/libs/cassandra-util/src/Cassandra/Options.hs b/libs/cassandra-util/src/Cassandra/Options.hs new file mode 100644 index 0000000000..f1f62056ee --- /dev/null +++ b/libs/cassandra-util/src/Cassandra/Options.hs @@ -0,0 +1,38 @@ +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TemplateHaskell #-} + +module Cassandra.Options where + +import Cassandra.Helpers +import Control.Lens +import Data.Aeson.TH +import Imports + +data Endpoint = Endpoint + { _host :: !Text, + _port :: !Word16 + } + deriving (Show, Generic) + +deriveFromJSON toOptionFieldName ''Endpoint + +makeLenses ''Endpoint + +data CassandraOpts = CassandraOpts + { _endpoint :: !Endpoint, + _keyspace :: !Text, + -- | If this option is unset, use all available nodes. + -- If this option is set, use only cassandra nodes in the given datacentre + -- + -- This option is most likely only necessary during a cassandra DC migration + -- FUTUREWORK: remove this option again, or support a datacentre migration feature + _filterNodesByDatacentre :: !(Maybe Text), + _tlsCa :: Maybe FilePath + } + deriving (Show, Generic) + +deriveFromJSON toOptionFieldName ''CassandraOpts + +makeLenses ''CassandraOpts diff --git a/libs/cassandra-util/src/Cassandra/Schema.hs b/libs/cassandra-util/src/Cassandra/Schema.hs index 79bbf51b9a..72676858a4 100644 --- a/libs/cassandra-util/src/Cassandra/Schema.hs +++ b/libs/cassandra-util/src/Cassandra/Schema.hs @@ -1,6 +1,5 @@ {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} -- for ReplicationStrategy {-# OPTIONS_GHC -Wno-partial-fields #-} @@ -34,33 +33,23 @@ module Cassandra.Schema versionCheck, createKeyspace, useKeyspace, - migrateSchema, migrationOptsParser, schema', ) where -import Cassandra (Client, Consistency (All, One), Keyspace (Keyspace), PrepQuery, QueryParams (QueryParams), QueryString (QueryString), R, S, Version (V4), W, params, query, query1, retry, runClient, write, x1, x5) -import Cassandra qualified as CQL (init) -import Cassandra.Settings (Policy, defSettings, initialContactsPlain, setConnectTimeout, setContacts, setLogger, setMaxConnections, setPolicy, setPoolStripes, setPortNumber, setProtocolVersion, setResponseTimeout, setSendTimeout) +import Cassandra (Client, Consistency (All, One), Keyspace (Keyspace), QueryParams (QueryParams), QueryString (QueryString), params, query1, retry, x5) import Control.Monad.Catch -import Control.Retry import Data.Aeson -import Data.List.NonEmpty qualified as NonEmpty import Data.List.Split (splitOn) import Data.Text (intercalate, pack) import Data.Text.Lazy (fromStrict) import Data.Text.Lazy qualified as LT import Data.Text.Lazy.Builder (fromString, fromText, toLazyText) -import Data.Time.Clock -import Data.UUID (UUID) -import Database.CQL.IO (HostResponse, Policy (Policy, acceptable, current, display, hostCount, onEvent, select, setup), getResult, request, schema) -import Database.CQL.IO.Tinylog qualified as CT +import Database.CQL.IO (HostResponse, getResult, request, schema) import Database.CQL.Protocol (Query (Query), Request (RqQuery)) import Imports hiding (All, fromString, init, intercalate, log) import Options.Applicative hiding (info) --- FUTUREWORK: We could use the System.Logger.Class here in the future, but we don't have a ReaderT IO here (yet) -import System.Logger qualified as Log data Migration = Migration { migVersion :: Int32, @@ -73,7 +62,8 @@ data MigrationOpts = MigrationOpts migPort :: Word16, migKeyspace :: Text, migRepl :: ReplicationStrategy, - migReset :: Bool + migReset :: Bool, + migTlsCa :: Maybe FilePath } deriving (Eq, Show, Generic) @@ -163,118 +153,6 @@ useKeyspace (Keyspace k) = void . getResult =<< qry prms = QueryParams One False () Nothing Nothing Nothing Nothing cql = QueryString $ "use \"" <> fromStrict k <> "\"" -migrateSchema :: Log.Logger -> MigrationOpts -> [Migration] -> IO () -migrateSchema l o ms = do - hosts <- initialContactsPlain $ pack (migHost o) - p <- - CQL.init - $ setLogger (CT.mkLogger l) - . setContacts (NonEmpty.head hosts) (NonEmpty.tail hosts) - . setPortNumber (fromIntegral $ migPort o) - . setMaxConnections 1 - . setPoolStripes 1 - -- 'migrationPolicy' ensures we only talk to one host for all queries - -- required for correct functioning of 'waitForSchemaConsistency' - . setPolicy migrationPolicy - -- use higher timeouts on schema migrations to reduce the probability - -- of a timeout happening during 'migAction' or 'metaInsert', - -- as that can lead to a state where schema migrations cannot be re-run - -- without manual action. - -- (due to e.g. "cannot create table X, already exists" errors) - . setConnectTimeout 20 - . setSendTimeout 20 - . setResponseTimeout 50 - . setProtocolVersion V4 - $ defSettings - runClient p $ do - let keyspace = Keyspace . migKeyspace $ o - when (migReset o) $ do - info "Dropping keyspace." - void $ schema (dropKeyspace keyspace) (params All ()) - createKeyspace keyspace (migRepl o) - useKeyspace keyspace - void $ schema metaCreate (params All ()) - migrations <- newer <$> schemaVersion - if null migrations - then info "No new migrations." - else info "New migrations found." - forM_ migrations $ \Migration {..} -> do - info $ "[" <> pack (show migVersion) <> "] " <> migText - migAction - now <- liftIO getCurrentTime - write metaInsert (params All (migVersion, migText, now)) - info "Waiting for schema version consistency across peers..." - waitForSchemaConsistency - info "... done waiting." - where - newer v = - dropWhile (maybe (const False) (>=) v . migVersion) - . sortBy (\x y -> migVersion x `compare` migVersion y) - $ ms - info = liftIO . Log.log l Log.Info . Log.msg - dropKeyspace :: Keyspace -> QueryString S () () - dropKeyspace (Keyspace k) = QueryString $ "drop keyspace if exists \"" <> fromStrict k <> "\"" - metaCreate :: QueryString S () () - metaCreate = "create columnfamily if not exists meta (id int, version int, descr text, date timestamp, primary key (id, version))" - metaInsert :: QueryString W (Int32, Text, UTCTime) () - metaInsert = "insert into meta (id, version, descr, date) values (1,?,?,?)" - --- | Retrieve and compare local and peer system schema versions. --- if they don't match, retry once per second for 30 seconds -waitForSchemaConsistency :: Client () -waitForSchemaConsistency = do - void $ retryWhileN 30 inDisagreement getSystemVersions - where - getSystemVersions :: Client (UUID, [UUID]) - getSystemVersions = do - -- These two sub-queries must be made to the same node. - -- (comparing local from node A and peers from node B wouldn't be correct) - -- using the custom 'migrationPolicy' when connecting to cassandra ensures this. - mbLocalVersion <- systemLocalVersion - peers <- systemPeerVersions - case mbLocalVersion of - Just localVersion -> pure $ (localVersion, peers) - Nothing -> error "No system_version in system.local (should never happen)" - inDisagreement :: (UUID, [UUID]) -> Bool - inDisagreement (localVersion, peers) = not $ all (== localVersion) peers - systemLocalVersion :: Client (Maybe UUID) - systemLocalVersion = fmap runIdentity <$> qry - where - qry = retry x1 (query1 cql (params One ())) - cql :: PrepQuery R () (Identity UUID) - cql = "select schema_version from system.local" - systemPeerVersions :: Client [UUID] - systemPeerVersions = fmap runIdentity <$> qry - where - qry = retry x1 (query cql (params One ())) - cql :: PrepQuery R () (Identity UUID) - cql = "select schema_version from system.peers" - -retryWhileN :: (MonadIO m) => Int -> (a -> Bool) -> m a -> m a -retryWhileN n f m = - retrying - (constantDelay 1000000 <> limitRetries n) - (const (pure . f)) - (const m) - --- | The migrationPolicy selects only one and always the same host -migrationPolicy :: IO Policy -migrationPolicy = do - h <- newIORef Nothing - pure $ - Policy - { setup = setHost h, - onEvent = const $ pure (), - select = readIORef h, - acceptable = const $ pure True, - hostCount = fromIntegral . length . maybeToList <$> readIORef h, - display = ("migrationPolicy: " ++) . show <$> readIORef h, - current = maybeToList <$> readIORef h - } - where - setHost h (a : _) _ = writeIORef h (Just a) - setHost _ _ _ = pure () - migrationOptsParser :: Parser MigrationOpts migrationOptsParser = MigrationOpts @@ -311,3 +189,8 @@ migrationOptsParser = ( long "reset" <> help "Reset the keyspace before running migrations" ) + <*> ( (optional . strOption) + ( long "tls-ca-certificate-file" + <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + ) + ) diff --git a/libs/cassandra-util/src/Cassandra/Util.hs b/libs/cassandra-util/src/Cassandra/Util.hs index 942684cb6e..f8b793f77d 100644 --- a/libs/cassandra-util/src/Cassandra/Util.hs +++ b/libs/cassandra-util/src/Cassandra/Util.hs @@ -17,32 +17,94 @@ module Cassandra.Util ( defInitCassandra, + initCassandraForService, + initCassandra, Writetime (..), writetimeToInt64, ) where -import Cassandra (ClientState, init) import Cassandra.CQL -import Cassandra.Settings (defSettings, setContacts, setKeyspace, setLogger, setPortNumber) +import Cassandra.Options +import Cassandra.Schema +import Cassandra.Settings (dcFilterPolicyIfConfigured, initialContactsDisco, initialContactsPlain, mkLogger) +import Control.Lens import Data.Aeson import Data.Fixed -import Data.Text (unpack) +import Data.List.NonEmpty qualified as NE +import Data.Text (pack, unpack) import Data.Time (UTCTime, nominalDiffTimeToSeconds) import Data.Time.Clock (secondsToNominalDiffTime) import Data.Time.Clock.POSIX +import Database.CQL.IO import Database.CQL.IO.Tinylog qualified as CT import Imports hiding (init) +import OpenSSL.Session qualified as OpenSSL import System.Logger qualified as Log -defInitCassandra :: Text -> Text -> Word16 -> Log.Logger -> IO ClientState -defInitCassandra ks h p lg = - init - $ setLogger (CT.mkLogger lg) - . setPortNumber (fromIntegral p) - . setContacts (unpack h) [] - . setKeyspace (Keyspace ks) - $ defSettings +defInitCassandra :: CassandraOpts -> Log.Logger -> IO ClientState +defInitCassandra opts logger = do + let basicCasSettings = + setLogger (CT.mkLogger logger) + . setPortNumber (fromIntegral (opts ^. endpoint . port)) + . setContacts (unpack (opts ^. endpoint . host)) [] + . setKeyspace (Keyspace (opts ^. keyspace)) + . setProtocolVersion V4 + $ defSettings + initCassandra basicCasSettings (opts ^. tlsCa) logger + +-- | Create Cassandra `ClientState` ("connection") for a service +initCassandraForService :: + CassandraOpts -> + String -> + Maybe Text -> + Maybe Int32 -> + Log.Logger -> + IO ClientState +initCassandraForService opts serviceName discoUrl mbSchemaVersion logger = do + c <- + maybe + (initialContactsPlain (opts ^. endpoint . host)) + (initialContactsDisco ("cassandra_" ++ serviceName) . unpack) + discoUrl + let basicCasSettings = + setLogger (mkLogger (Log.clone (Just (pack ("cassandra." ++ serviceName))) logger)) + . setContacts (NE.head c) (NE.tail c) + . setPortNumber (fromIntegral (opts ^. endpoint . port)) + . setKeyspace (Keyspace (opts ^. keyspace)) + . setMaxConnections 4 + . setPoolStripes 4 + . setSendTimeout 3 + . setResponseTimeout 10 + . setProtocolVersion V4 + . setPolicy (dcFilterPolicyIfConfigured logger (opts ^. filterNodesByDatacentre)) + $ defSettings + p <- initCassandra basicCasSettings (opts ^. tlsCa) logger + maybe (pure ()) (\v -> runClient p $ (versionCheck v)) mbSchemaVersion + pure p + +initCassandra :: Settings -> Maybe FilePath -> Log.Logger -> IO ClientState +initCassandra settings (Just tlsCaPath) logger = do + sslContext <- createSSLContext tlsCaPath + let settings' = setSSLContext sslContext settings + init settings' + where + createSSLContext :: FilePath -> IO OpenSSL.SSLContext + createSSLContext certFile = do + void . liftIO $ Log.debug logger (Log.msg ("TLS cert file path: " <> show certFile)) + sslContext <- OpenSSL.context + OpenSSL.contextSetCAFile sslContext certFile + OpenSSL.contextSetVerificationMode + sslContext + OpenSSL.VerifyPeer + { vpFailIfNoPeerCert = True, + vpClientOnce = True, + vpCallback = Nothing + } + pure sslContext +initCassandra settings Nothing logger = do + void . liftIO $ Log.debug logger (Log.msg ("No TLS cert file path configured." :: Text)) + init settings -- | Read cassandra's writetimes https://docs.datastax.com/en/dse/5.1/cql/cql/cql_using/useWritetime.html -- as UTCTime values without any loss of precision diff --git a/libs/types-common/src/Util/Options.hs b/libs/types-common/src/Util/Options.hs index d579c7fbb4..74437b78a2 100644 --- a/libs/types-common/src/Util/Options.hs +++ b/libs/types-common/src/Util/Options.hs @@ -20,10 +20,15 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module Util.Options where - +module Util.Options + ( module Util.Options, + -- TODO: Switch denpendees to the original module? + module Cassandra.Options, + ) +where + +import Cassandra.Options import Control.Lens -import Data.Aeson.TH import Data.ByteString.Char8 qualified as BS import Data.ByteString.Conversion import Data.Text.Encoding (encodeUtf8) @@ -48,17 +53,17 @@ instance FromByteString AWSEndpoint where "https" -> pure True "http" -> pure False x -> fail ("Unsupported scheme: " ++ show x) - host <- case url ^. authorityL <&> view (authorityHostL . hostBSL) of + awsHost <- case url ^. authorityL <&> view (authorityHostL . hostBSL) of Just h -> pure h Nothing -> fail ("No host in: " ++ show url) - port <- case urlPort url of + awsPort <- case urlPort url of Just p -> pure p Nothing -> pure $ if secure then 443 else 80 - pure $ AWSEndpoint host secure port + pure $ AWSEndpoint awsHost secure awsPort instance FromJSON AWSEndpoint where parseJSON = @@ -73,32 +78,6 @@ urlPort u = do makeLenses ''AWSEndpoint -data Endpoint = Endpoint - { _host :: !Text, - _port :: !Word16 - } - deriving (Show, Generic) - -deriveFromJSON toOptionFieldName ''Endpoint - -makeLenses ''Endpoint - -data CassandraOpts = CassandraOpts - { _endpoint :: !Endpoint, - _keyspace :: !Text, - -- | If this option is unset, use all available nodes. - -- If this option is set, use only cassandra nodes in the given datacentre - -- - -- This option is most likely only necessary during a cassandra DC migration - -- FUTUREWORK: remove this option again, or support a datacentre migration feature - _filterNodesByDatacentre :: !(Maybe Text) - } - deriving (Show, Generic) - -deriveFromJSON toOptionFieldName ''CassandraOpts - -makeLenses ''CassandraOpts - newtype FilePathSecrets = FilePathSecrets FilePath deriving (Eq, Show, FromJSON) diff --git a/libs/types-common/src/Util/Options/Common.hs b/libs/types-common/src/Util/Options/Common.hs index c052a53c33..14b997bee7 100644 --- a/libs/types-common/src/Util/Options/Common.hs +++ b/libs/types-common/src/Util/Options/Common.hs @@ -15,36 +15,19 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module Util.Options.Common where +module Util.Options.Common + ( module Cassandra.Helpers, + module Util.Options.Common, + ) +where -import Data.Aeson.TH +import Cassandra.Helpers (toOptionFieldName) import Data.ByteString.Char8 qualified as C import Data.Text qualified as T import Imports hiding (reader) import Options.Applicative import System.Posix.Env qualified as Posix --- | Convenient helper to convert record field names to use as YAML fields. --- NOTE: We typically use this for options in the configuration files! --- If you are looking into converting record field name to JSON to be used --- over the API, look for toJSONFieldName in the Data.Json.Util module. --- It converts field names into snake_case --- --- Example: --- newtype TeamName = TeamName { teamName :: Text } --- deriveJSON toJSONFieldName ''teamName --- --- would generate {To/From}JSON instances where --- the field name is "teamName" -toOptionFieldName :: Options -toOptionFieldName = defaultOptions {fieldLabelModifier = lowerFirst . dropPrefix} - where - lowerFirst :: String -> String - lowerFirst (x : xs) = toLower x : xs - lowerFirst [] = "" - dropPrefix :: String -> String - dropPrefix = dropWhile ('_' ==) - optOrEnv :: (a -> b) -> Maybe a -> (String -> b) -> String -> IO b optOrEnv getter conf reader var = case conf of Nothing -> reader <$> getEnv var diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal index 3df1d72450..29007477ef 100644 --- a/services/brig/brig.cabal +++ b/services/brig/brig.cabal @@ -534,7 +534,7 @@ executable brig-schema import: common-all main-is: Main.hs hs-source-dirs: schema - ghc-options: -funbox-strict-fields -Wredundant-constraints + ghc-options: -funbox-strict-fields -Wredundant-constraints -threaded default-extensions: TemplateHaskell build-depends: , base diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index 6f0f9e85e2..616bc4a79d 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -106,10 +106,9 @@ import Brig.User.Search.Index (IndexEnv (..), MonadIndexIO (..), runIndexIO) import Brig.User.Template import Brig.ZAuth (MonadZAuth (..), runZAuth) import Brig.ZAuth qualified as ZAuth -import Cassandra (Keyspace (Keyspace), runClient) +import Cassandra (runClient) import Cassandra qualified as Cas -import Cassandra.Schema (versionCheck) -import Cassandra.Settings qualified as Cas +import Cassandra.Util (initCassandraForService) import Control.AutoUpdate import Control.Error import Control.Exception.Enclosed (handleAny) @@ -120,12 +119,10 @@ import Data.ByteString.Conversion import Data.Domain import Data.GeoIP2 qualified as GeoIp import Data.IP -import Data.List.NonEmpty qualified as NE import Data.Metrics (Metrics) import Data.Metrics.Middleware qualified as Metrics import Data.Misc import Data.Qualified -import Data.Text (unpack) import Data.Text qualified as Text import Data.Text.Encoding (encodeUtf8) import Data.Text.Encoding qualified as Text @@ -424,27 +421,13 @@ initExtGetManager = do in verifyRsaFingerprint sha pinset initCassandra :: Opts -> Logger -> IO Cas.ClientState -initCassandra o g = do - c <- - maybe - (Cas.initialContactsPlain (Opt.cassandra o ^. endpoint . host)) - (Cas.initialContactsDisco "cassandra_brig" . unpack) - (Opt.discoUrl o) - p <- - Cas.init - $ Cas.setLogger (Cas.mkLogger (Log.clone (Just "cassandra.brig") g)) - . Cas.setContacts (NE.head c) (NE.tail c) - . Cas.setPortNumber (fromIntegral (Opt.cassandra o ^. endpoint . port)) - . Cas.setKeyspace (Keyspace (Opt.cassandra o ^. keyspace)) - . Cas.setMaxConnections 4 - . Cas.setPoolStripes 4 - . Cas.setSendTimeout 3 - . Cas.setResponseTimeout 10 - . Cas.setProtocolVersion Cas.V4 - . Cas.setPolicy (Cas.dcFilterPolicyIfConfigured g (Opt.cassandra o ^. filterNodesByDatacentre)) - $ Cas.defSettings - runClient p $ versionCheck schemaVersion - pure p +initCassandra o g = + initCassandraForService + (Opt.cassandra o) + "brig" + (Opt.discoUrl o) + (Just schemaVersion) + g initCredentials :: (FromJSON a) => FilePathSecrets -> IO a initCredentials secretFile = do diff --git a/services/brig/src/Brig/Index/Eval.hs b/services/brig/src/Brig/Index/Eval.hs index 3b6e220043..ed412d8d0d 100644 --- a/services/brig/src/Brig/Index/Eval.hs +++ b/services/brig/src/Brig/Index/Eval.hs @@ -26,7 +26,7 @@ import Brig.Index.Migrations import Brig.Index.Options import Brig.User.Search.Index import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra.Util (defInitCassandra) import Control.Lens import Control.Monad.Catch import Control.Retry @@ -101,14 +101,7 @@ runCommand l = \case <*> pure mgr initES esURI mgr = ES.mkBHEnv (toESServer esURI) mgr - initDb cas = - C.init - $ C.setLogger (C.mkLogger l) - . C.setContacts (view cHost cas) [] - . C.setPortNumber (fromIntegral (view cPort cas)) - . C.setKeyspace (view cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings + initDb cas = defInitCassandra (toCassandraOpts cas) l waitForTaskToComplete :: forall a m. (ES.MonadBH m, MonadThrow m, FromJSON a) => Int -> ES.TaskNodeId -> m () waitForTaskToComplete timeoutSeconds taskNodeId = do diff --git a/services/brig/src/Brig/Index/Migrations.hs b/services/brig/src/Brig/Index/Migrations.hs index 27d7559fed..da7e78cc1a 100644 --- a/services/brig/src/Brig/Index/Migrations.hs +++ b/services/brig/src/Brig/Index/Migrations.hs @@ -23,8 +23,7 @@ where import Brig.Index.Migrations.Types import Brig.Index.Options qualified as Opts import Brig.User.Search.Index qualified as Search -import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra.Util (defInitCassandra) import Control.Lens (view, (^.)) import Control.Monad.Catch (MonadThrow, catchAll, finally, throwM) import Data.Aeson (Value, object, (.=)) @@ -86,14 +85,8 @@ mkEnv l es cas galleyEndpoint = do <*> pure mgr <*> pure galleyEndpoint where - initCassandra = - C.init - $ C.setLogger (C.mkLogger l) - . C.setContacts (view Opts.cHost cas) [] - . C.setPortNumber (fromIntegral (view Opts.cPort cas)) - . C.setKeyspace (view Opts.cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings + initCassandra = defInitCassandra (Opts.toCassandraOpts cas) l + initLogger = pure l createMigrationsIndexIfNotPresent :: (MonadThrow m, ES.MonadBH m, Log.MonadLogger m) => m () diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs index 6a08700e09..89da5997cb 100644 --- a/services/brig/src/Brig/Index/Options.hs +++ b/services/brig/src/Brig/Index/Options.hs @@ -29,8 +29,10 @@ module Brig.Index.Options esIndexRefreshInterval, esDeleteTemplate, CassandraSettings, + toCassandraOpts, cHost, cPort, + cTlsCa, cKeyspace, localElasticSettings, localCassandraSettings, @@ -49,6 +51,7 @@ import Brig.Index.Types (CreateIndexSettings (..)) import Cassandra qualified as C import Control.Lens import Data.ByteString.Lens +import Data.Text qualified as Text import Data.Text.Strict.Lens import Data.Time.Clock (NominalDiffTime) import Database.Bloodhound qualified as ES @@ -56,7 +59,7 @@ import Imports import Options.Applicative import URI.ByteString import URI.ByteString.QQ -import Util.Options (Endpoint (..)) +import Util.Options (CassandraOpts (..), Endpoint (..)) data Command = Create ElasticSettings Endpoint @@ -82,7 +85,8 @@ data ElasticSettings = ElasticSettings data CassandraSettings = CassandraSettings { _cHost :: String, _cPort :: Word16, - _cKeyspace :: C.Keyspace + _cKeyspace :: C.Keyspace, + _cTlsCa :: Maybe FilePath } deriving (Show) @@ -100,6 +104,15 @@ makeLenses ''CassandraSettings makeLenses ''ReindexFromAnotherIndexSettings +toCassandraOpts :: CassandraSettings -> CassandraOpts +toCassandraOpts cas = + CassandraOpts + { _endpoint = Endpoint (Text.pack (cas ^. cHost)) (cas ^. cPort), + _keyspace = C.unKeyspace (cas ^. cKeyspace), + _filterNodesByDatacentre = Nothing, + _tlsCa = cas ^. cTlsCa + } + mkCreateIndexSettings :: ElasticSettings -> CreateIndexSettings mkCreateIndexSettings es = CreateIndexSettings @@ -125,7 +138,8 @@ localCassandraSettings = CassandraSettings { _cHost = "localhost", _cPort = 9042, - _cKeyspace = C.Keyspace "brig_test" + _cKeyspace = C.Keyspace "brig_test", + _cTlsCa = Nothing } elasticServerParser :: Parser (URIRef Absolute) @@ -247,6 +261,11 @@ cassandraSettingsParser = <> showDefault ) ) + <*> ( (optional . strOption) + ( long "tls-ca-certificate-file" + <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + ) + ) reindexToAnotherIndexSettingsParser :: Parser ReindexFromAnotherIndexSettings reindexToAnotherIndexSettingsParser = diff --git a/services/brig/src/Brig/Schema/Run.hs b/services/brig/src/Brig/Schema/Run.hs index 15f996f73b..049a51e5f5 100644 --- a/services/brig/src/Brig/Schema/Run.hs +++ b/services/brig/src/Brig/Schema/Run.hs @@ -56,6 +56,7 @@ import Brig.Schema.V78_ClientLastActive qualified as V78_ClientLastActive import Brig.Schema.V79_ConnectionRemoteIndex qualified as V79_ConnectionRemoteIndex import Brig.Schema.V80_KeyPackageCiphersuite qualified as V80_KeyPackageCiphersuite import Brig.Schema.V81_AddFederationRemoteTeams qualified as V81_AddFederationRemoteTeams +import Cassandra.MigrateSchema (migrateSchema) import Cassandra.Schema import Control.Exception (finally) import Imports diff --git a/services/brig/test/integration/Run.hs b/services/brig/test/integration/Run.hs index 324dc92a6d..babbbdce48 100644 --- a/services/brig/test/integration/Run.hs +++ b/services/brig/test/integration/Run.hs @@ -133,12 +133,9 @@ runTests iConf brigOpts otherArgs = do Opts.TurnSourceFiles files -> files Opts.TurnSourceDNS _ -> error "The integration tests can only be run when TurnServers are sourced from files" localDomain = brigOpts ^. Opts.optionSettings . Opts.federationDomain - casHost = (\v -> Opts.cassandra v ^. endpoint . host) brigOpts - casPort = (\v -> Opts.cassandra v ^. endpoint . port) brigOpts - casKey = (\v -> Opts.cassandra v ^. keyspace) brigOpts awsOpts = Opts.aws brigOpts lg <- Logger.new Logger.defSettings -- TODO: use mkLogger'? - db <- defInitCassandra casKey casHost casPort lg + db <- defInitCassandra (brigOpts.cassandra) lg mg <- newManager tlsManagerSettings let fedBrigClient = FedClient @'Brig mg (brig iConf) emailAWSOpts <- parseEmailAWSOpts diff --git a/services/galley/galley.cabal b/services/galley/galley.cabal index 1c9908ea67..6d63451aed 100644 --- a/services/galley/galley.cabal +++ b/services/galley/galley.cabal @@ -556,6 +556,7 @@ executable galley-integration executable galley-migrate-data import: common-all main-is: ../main.hs + ghc-options: -threaded -- cabal-fmt: expand migrate-data/src other-modules: @@ -591,6 +592,7 @@ executable galley-schema import: common-all main-is: Main.hs hs-source-dirs: schema + ghc-options: -threaded default-extensions: TemplateHaskell build-depends: , galley diff --git a/services/galley/migrate-data/src/Galley/DataMigration.hs b/services/galley/migrate-data/src/Galley/DataMigration.hs index 1ab4bc54bb..ac79bcc0fc 100644 --- a/services/galley/migrate-data/src/Galley/DataMigration.hs +++ b/services/galley/migrate-data/src/Galley/DataMigration.hs @@ -18,7 +18,8 @@ module Galley.DataMigration (cassandraSettingsParser, migrate) where import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra.Options +import Cassandra.Util (defInitCassandra) import Control.Monad.Catch (finally) import Data.Text qualified as Text import Data.Time (UTCTime, getCurrentTime) @@ -32,9 +33,19 @@ import System.Logger.Class qualified as Log data CassandraSettings = CassandraSettings { cHost :: String, cPort :: Word16, - cKeyspace :: C.Keyspace + cKeyspace :: C.Keyspace, + cTlsCa :: Maybe FilePath } +toCassandraOpts :: CassandraSettings -> CassandraOpts +toCassandraOpts cas = + CassandraOpts + { _endpoint = Endpoint (Text.pack (cas.cHost)) (cas.cPort), + _keyspace = C.unKeyspace (cas.cKeyspace), + _filterNodesByDatacentre = Nothing, + _tlsCa = cas.cTlsCa + } + cassandraSettingsParser :: Parser CassandraSettings cassandraSettingsParser = CassandraSettings @@ -53,6 +64,11 @@ cassandraSettingsParser = <> Opts.value "galley_test" ) ) + <*> ( (Opts.optional . Opts.strOption) + ( Opts.long "tls-ca-certificate-file" + <> Opts.help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" + ) + ) migrate :: Logger -> CassandraSettings -> [Migration] -> IO () migrate l cas ms = do @@ -69,14 +85,7 @@ mkEnv l cas = <$> initCassandra <*> initLogger where - initCassandra = - C.init - $ C.setLogger (C.mkLogger l) - . C.setContacts (cHost cas) [] - . C.setPortNumber (fromIntegral (cPort cas)) - . C.setKeyspace (cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings + initCassandra = defInitCassandra (toCassandraOpts cas) l initLogger = pure l -- | Runs only the migrations which need to run diff --git a/services/galley/src/Galley/App.hs b/services/galley/src/Galley/App.hs index 8eded00734..14873001de 100644 --- a/services/galley/src/Galley/App.hs +++ b/services/galley/src/Galley/App.hs @@ -46,16 +46,13 @@ where import Bilge hiding (Request, header, host, options, port, statusCode, statusMessage) import Cassandra hiding (Set) -import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra.Util (initCassandraForService) import Control.Error hiding (err) import Control.Lens hiding ((.=)) -import Data.List.NonEmpty qualified as NE import Data.Metrics.Middleware import Data.Misc import Data.Qualified import Data.Range -import Data.Text (unpack) import Data.Time.Clock import Galley.API.Error import Galley.Aws qualified as Aws @@ -106,7 +103,6 @@ import System.Logger qualified as Log import System.Logger.Class (Logger) import System.Logger.Extended qualified as Logger import UnliftIO.Exception qualified as UnliftIO -import Util.Options import Wire.API.Conversation.Protocol import Wire.API.Error import Wire.API.Federation.Error @@ -173,25 +169,13 @@ createEnv m o l = do <*> pure codeURIcfg initCassandra :: Opts -> Logger -> IO ClientState -initCassandra o l = do - c <- - maybe - (C.initialContactsPlain (o ^. cassandra . endpoint . host)) - (C.initialContactsDisco "cassandra_galley" . unpack) - (o ^. discoUrl) - C.init - . C.setLogger (C.mkLogger (Logger.clone (Just "cassandra.galley") l)) - . C.setContacts (NE.head c) (NE.tail c) - . C.setPortNumber (fromIntegral $ o ^. cassandra . endpoint . port) - . C.setKeyspace (Keyspace $ o ^. cassandra . keyspace) - . C.setMaxConnections 4 - . C.setMaxStreams 128 - . C.setPoolStripes 4 - . C.setSendTimeout 3 - . C.setResponseTimeout 10 - . C.setProtocolVersion C.V4 - . C.setPolicy (C.dcFilterPolicyIfConfigured l (o ^. cassandra . filterNodesByDatacentre)) - $ C.defSettings +initCassandra o l = + initCassandraForService + (o ^. cassandra) + "galley" + (o ^. discoUrl) + Nothing + l initHttpManager :: Opts -> IO Manager initHttpManager o = do diff --git a/services/galley/src/Galley/Schema/Run.hs b/services/galley/src/Galley/Schema/Run.hs index 5291020674..91ae2c3185 100644 --- a/services/galley/src/Galley/Schema/Run.hs +++ b/services/galley/src/Galley/Schema/Run.hs @@ -17,6 +17,7 @@ module Galley.Schema.Run where +import Cassandra.MigrateSchema (migrateSchema) import Cassandra.Schema import Control.Exception (finally) import Galley.Schema.V20 qualified as V20 diff --git a/services/galley/test/integration/Run.hs b/services/galley/test/integration/Run.hs index 7c33dadd62..149c935262 100644 --- a/services/galley/test/integration/Run.hs +++ b/services/galley/test/integration/Run.hs @@ -124,11 +124,8 @@ main = withOpenSSL $ runTests go convMaxSize <- optOrEnv maxSize gConf read "CONV_MAX_SIZE" awsEnv <- initAwsEnv e q -- Initialize cassandra - let ch = fromJust gConf ^. cassandra . endpoint . host - let cp = fromJust gConf ^. cassandra . endpoint . port - let ck = fromJust gConf ^. cassandra . keyspace lg <- Logger.new Logger.defSettings - db <- defInitCassandra ck ch cp lg + db <- defInitCassandra (fromJust gConf ^. cassandra) lg teamEventWatcher <- sequence $ SQS.watchSQSQueue <$> ((^. Aws.awsEnv) <$> awsEnv) <*> q pure $ TestSetup (fromJust gConf) (fromJust iConf) m g b c awsEnv convMaxSize db (FedClient m galleyEndpoint) teamEventWatcher queueName' = fmap (view queueName) . view journal diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index df8850991d..fdc67f2f22 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -20,14 +20,12 @@ module Gundeck.Env where import Bilge hiding (host, port) -import Cassandra (ClientState, Keyspace (..)) -import Cassandra qualified as C -import Cassandra.Settings qualified as C +import Cassandra (ClientState) +import Cassandra.Util (initCassandraForService) import Control.AutoUpdate import Control.Concurrent.Async (Async) import Control.Lens (makeLenses, (^.)) import Control.Retry (capDelay, exponentialBackoff) -import Data.List.NonEmpty qualified as NE import Data.Metrics.Middleware (Metrics) import Data.Misc (Milliseconds (..)) import Data.Text (unpack) @@ -45,7 +43,6 @@ import Network.HTTP.Client (responseTimeoutMicro) import Network.HTTP.Client.TLS (tlsManagerSettings) import System.Logger qualified as Log import System.Logger.Extended qualified as Logger -import Util.Options data Env = Env { _reqId :: !RequestId, @@ -69,11 +66,6 @@ schemaVersion = 7 createEnv :: Metrics -> Opts -> IO ([Async ()], Env) createEnv m o = do l <- Logger.mkLogger (o ^. logLevel) (o ^. logNetStrings) (o ^. logFormat) - c <- - maybe - (C.initialContactsPlain (o ^. cassandra . endpoint . host)) - (C.initialContactsDisco "cassandra_gundeck" . unpack) - (o ^. discoUrl) n <- newManager tlsManagerSettings @@ -91,19 +83,13 @@ createEnv m o = do pure ([rAddThread], Just rAdd) p <- - C.init - $ C.setLogger (C.mkLogger (Logger.clone (Just "cassandra.gundeck") l)) - . C.setContacts (NE.head c) (NE.tail c) - . C.setPortNumber (fromIntegral $ o ^. cassandra . endpoint . port) - . C.setKeyspace (Keyspace (o ^. cassandra . keyspace)) - . C.setMaxConnections 4 - . C.setMaxStreams 128 - . C.setPoolStripes 4 - . C.setSendTimeout 3 - . C.setResponseTimeout 10 - . C.setProtocolVersion C.V4 - . C.setPolicy (C.dcFilterPolicyIfConfigured l (o ^. cassandra . filterNodesByDatacentre)) - $ C.defSettings + initCassandraForService + (o ^. cassandra) + "gundeck" + (o ^. discoUrl) + Nothing + l + a <- Aws.mkEnv l o n io <- mkAutoUpdate diff --git a/services/gundeck/src/Gundeck/Schema/Run.hs b/services/gundeck/src/Gundeck/Schema/Run.hs index 056363c244..ccec5141e4 100644 --- a/services/gundeck/src/Gundeck/Schema/Run.hs +++ b/services/gundeck/src/Gundeck/Schema/Run.hs @@ -17,6 +17,7 @@ module Gundeck.Schema.Run where +import Cassandra.MigrateSchema (migrateSchema) import Cassandra.Schema import Control.Exception (finally) import Gundeck.Schema.V1 qualified as V1 diff --git a/services/gundeck/test/integration/Main.hs b/services/gundeck/test/integration/Main.hs index 9ab372ede3..767f28a4ae 100644 --- a/services/gundeck/test/integration/Main.hs +++ b/services/gundeck/test/integration/Main.hs @@ -112,11 +112,8 @@ main = withOpenSSL $ runTests go c = CannonR . mkRequest $ cannon iConf c2 = CannonR . mkRequest $ cannon2 iConf b = BrigR $ mkRequest iConf.brig - ch = gConf ^. cassandra . endpoint . host - cp = gConf ^. cassandra . endpoint . port - ck = gConf ^. cassandra . keyspace lg <- Logger.new Logger.defSettings - db <- defInitCassandra ck ch cp lg + db <- defInitCassandra (gConf ^. cassandra) lg pure $ TestSetup m g c c2 b db lg gConf (redis2 iConf) releaseOpts _ = pure () mkRequest (Endpoint h p) = Bilge.host (encodeUtf8 h) . Bilge.port p diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Options.hs b/services/spar/migrate-data/src/Spar/DataMigration/Options.hs index fdb70667eb..6850ca92ec 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Options.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Options.hs @@ -71,3 +71,9 @@ cassandraSettingsParser ks = <> showDefault ) ) + <*> ( (optional . strOption) + ( long ("tls-ca-certificate-file-" ++ ks) + <> help ("Location of a PEM encoded list of CA certificates to be used when verifying" ++ ks ++ "'s Cassandra server's certificate") + <> showDefault + ) + ) diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs index 4b13b42578..c41bd2d2cc 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Run.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Run.hs @@ -19,8 +19,9 @@ module Spar.DataMigration.Run where +import Cassandra (ClientState) import qualified Cassandra as C -import qualified Cassandra.Settings as C +import Cassandra.Util (defInitCassandra) import Control.Lens import Control.Monad.Catch (finally) import qualified Data.Text as Text @@ -64,14 +65,9 @@ mkEnv settings = do . Log.setLogLevel (if s ^. setDebug == Debug then Log.Debug else Log.Info) $ Log.defSettings - initCassandra cas l = - C.init - . C.setLogger (C.mkLogger l) - . C.setContacts (cas ^. cHosts) [] - . C.setPortNumber (fromIntegral $ cas ^. cPort) - . C.setKeyspace (cas ^. cKeyspace) - . C.setProtocolVersion C.V4 - $ C.defSettings + + initCassandra :: CassandraSettings -> Log.Logger -> IO ClientState + initCassandra cas l = defInitCassandra (toCassandraOpts cas) l cleanup :: (MonadIO m) => Env -> m () cleanup env = do diff --git a/services/spar/migrate-data/src/Spar/DataMigration/Types.hs b/services/spar/migrate-data/src/Spar/DataMigration/Types.hs index 751b3d20df..64d7a13c0e 100644 --- a/services/spar/migrate-data/src/Spar/DataMigration/Types.hs +++ b/services/spar/migrate-data/src/Spar/DataMigration/Types.hs @@ -21,7 +21,9 @@ module Spar.DataMigration.Types where import qualified Cassandra as C +import Cassandra.Options import Control.Lens +import qualified Data.Text as Text import Imports import Numeric.Natural (Natural) import qualified System.Logger as Logger @@ -62,10 +64,20 @@ data MigratorSettings = MigratorSettings data CassandraSettings = CassandraSettings { _cHosts :: !String, _cPort :: !Word16, - _cKeyspace :: !C.Keyspace + _cKeyspace :: !C.Keyspace, + _cTlsCa :: Maybe FilePath } deriving (Show) makeLenses ''MigratorSettings makeLenses ''CassandraSettings + +toCassandraOpts :: CassandraSettings -> CassandraOpts +toCassandraOpts cas = + CassandraOpts + { _endpoint = Endpoint (Text.pack (cas ^. cHosts)) (cas ^. cPort), + _keyspace = C.unKeyspace (cas ^. cKeyspace), + _filterNodesByDatacentre = Nothing, + _tlsCa = cas ^. cTlsCa + } diff --git a/services/spar/src/Spar/Run.hs b/services/spar/src/Spar/Run.hs index e8bd47f0a4..2a9a427f64 100644 --- a/services/spar/src/Spar/Run.hs +++ b/services/spar/src/Spar/Run.hs @@ -30,11 +30,9 @@ where import qualified Bilge import Cassandra as Cas -import qualified Cassandra.Schema as Cas -import qualified Cassandra.Settings as Cas +import Cassandra.Util (initCassandraForService) import Control.Lens (to, (^.)) import Data.Id -import Data.List.NonEmpty as NE import Data.Metrics.Servant (servantPrometheusMiddleware) import Data.Proxy (Proxy (Proxy)) import qualified Data.UUID as UUID @@ -50,12 +48,12 @@ import Spar.API (SparAPI, app) import Spar.App import qualified Spar.Data as Data import Spar.Data.Instances () -import Spar.Options +import Spar.Options as Opt import Spar.Orphans () import System.Logger (Logger, msg, val, (.=), (~~)) import qualified System.Logger as Log import qualified System.Logger.Extended as Log -import Util.Options (endpoint, filterNodesByDatacentre, host, keyspace, port) +import Util.Options import Wire.API.Routes.Version.Wai import Wire.Sem.Logger.TinyLog @@ -63,29 +61,13 @@ import Wire.Sem.Logger.TinyLog -- cassandra initCassandra :: Opts -> Logger -> IO ClientState -initCassandra opts lgr = do - let cassOpts = cassandra opts - connectString <- - maybe - (Cas.initialContactsPlain (cassOpts ^. endpoint . host)) - (Cas.initialContactsDisco "cassandra_spar" . cs) - (discoUrl opts) - cas <- - Cas.init $ - Cas.defSettings - & Cas.setLogger (Cas.mkLogger (Log.clone (Just "cassandra.spar") lgr)) - & Cas.setContacts (NE.head connectString) (NE.tail connectString) - & Cas.setPortNumber (fromIntegral $ cassOpts ^. endpoint . port) - & Cas.setKeyspace (Keyspace $ cassOpts ^. keyspace) - & Cas.setMaxConnections 4 - & Cas.setMaxStreams 128 - & Cas.setPoolStripes 4 - & Cas.setSendTimeout 3 - & Cas.setResponseTimeout 10 - & Cas.setProtocolVersion V4 - & Cas.setPolicy (Cas.dcFilterPolicyIfConfigured lgr (cassOpts ^. filterNodesByDatacentre)) - runClient cas $ Cas.versionCheck Data.schemaVersion - pure cas +initCassandra opts lgr = + initCassandraForService + (Opt.cassandra opts) + "spar" + (Opt.discoUrl opts) + (Just Data.schemaVersion) + lgr ---------------------------------------------------------------------- -- servant / wai / warp diff --git a/services/spar/src/Spar/Schema/Run.hs b/services/spar/src/Spar/Schema/Run.hs index 4fef2264a3..a853c1c13c 100644 --- a/services/spar/src/Spar/Schema/Run.hs +++ b/services/spar/src/Spar/Schema/Run.hs @@ -17,6 +17,7 @@ module Spar.Schema.Run where +import Cassandra.MigrateSchema (migrateSchema) import Cassandra.Schema import Control.Exception (finally) import Imports