WPB-6717 ES Credentials

Makefile

@@ -294,9 +294,9 @@ db-reset: c
 	./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 --reset
 	./dist/spar-schema --keyspace spar_test2 --replication-factor 1 --reset
 	./integration/scripts/integration-dynamic-backends-db-schemas.sh --replication-factor 1 --reset
-	./dist/brig-index reset --elasticsearch-index-prefix directory --elasticsearch-server http://localhost:9200 > /dev/null
-	./dist/brig-index reset --elasticsearch-index-prefix directory2 --elasticsearch-server http://localhost:9200 > /dev/null
-	./integration/scripts/integration-dynamic-backends-brig-index.sh --elasticsearch-server http://localhost:9200 > /dev/null
+	./dist/brig-index reset --elasticsearch-index-prefix directory --elasticsearch-server http://localhost:9200 --elasticsearch-credentials ./services/brig/test/resources/elasticsearch-credentials.yaml > /dev/null
+	./dist/brig-index reset --elasticsearch-index-prefix directory2 --elasticsearch-server http://localhost:9200 --elasticsearch-credentials ./services/brig/test/resources/elasticsearch-credentials.yaml > /dev/null
+	./integration/scripts/integration-dynamic-backends-brig-index.sh --elasticsearch-server http://localhost:9200 --elasticsearch-credentials ./services/brig/test/resources/elasticsearch-credentials.yaml > /dev/null
 
 
 
@@ -312,9 +312,9 @@ db-migrate: c
 	./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 > /dev/null
 	./dist/spar-schema --keyspace spar_test2 --replication-factor 1 > /dev/null
 	./integration/scripts/integration-dynamic-backends-db-schemas.sh --replication-factor 1 > /dev/null
-	./dist/brig-index reset --elasticsearch-index-prefix directory --elasticsearch-server http://localhost:9200 > /dev/null
-	./dist/brig-index reset --elasticsearch-index-prefix directory2 --elasticsearch-server http://localhost:9200 > /dev/null
-	./integration/scripts/integration-dynamic-backends-brig-index.sh --elasticsearch-server http://localhost:9200 > /dev/null
+	./dist/brig-index reset --elasticsearch-index-prefix directory --elasticsearch-server http://localhost:9200 --elasticsearch-credentials ./services/brig/test/resources/elasticsearch-credentials.yaml > /dev/null
+	./dist/brig-index reset --elasticsearch-index-prefix directory2 --elasticsearch-server http://localhost:9200 --elasticsearch-credentials ./services/brig/test/resources/elasticsearch-credentials.yaml > /dev/null
+	./integration/scripts/integration-dynamic-backends-brig-index.sh --elasticsearch-server http://localhost:9200 --elasticsearch-credentials ./services/brig/test/resources/elasticsearch-credentials.yaml > /dev/null
 
 #################################
 ## dependencies

changelog.d/2-features/WPB-6717

@@ -0,0 +1 @@
+Support for Elasticsearch password authentication

charts/brig/templates/configmap.yaml

@@ -38,6 +38,12 @@ data:
       {{- if .elasticsearch.additionalWriteIndex }}
       additionalWriteIndex: {{ .elasticsearch.additionalWriteIndex }}
       {{- end }}
+      {{- if $.Values.secrets.elasticsearch }}
+      credentials: /etc/wire/brig/secrets/elasticsearch-credentials.yaml
+      {{- end }}
+      {{- if $.Values.secrets.elasticsearchAdditional }}
+      additionalCredentials: /etc/wire/brig/secrets/elasticsearch-additional-credentials.yaml
+      {{- end }}
 
     cargohold:
       host: cargohold

charts/brig/templates/secret.yaml

@@ -35,4 +35,10 @@ data:
   rabbitmqUsername: {{ .rabbitmq.username | b64enc | quote }}
   rabbitmqPassword: {{ .rabbitmq.password | b64enc | quote }}
   {{- end }}
+  {{- if .elasticsearch }}
+  elasticsearch-credentials.yaml: {{ .elasticsearch | toYaml | b64enc }}
+  {{- end }}
+  {{- if .elasticsearchAdditional }}
+  elasticsearch-additional-credentials.yaml: {{ .elasticsearchAdditional | toYaml | b64enc }}
+  {{- end }}
   {{- end }}

charts/elasticsearch-ephemeral/templates/es.yaml

@@ -32,6 +32,11 @@ spec:
           value: "single-node"
         - name: "action.auto_create_index"
           value: ".watches,.triggered_watches,.watcher-history-*,pod-*,node-*"
+        - name: "xpack.security.enabled"
+          value: "true"
+        # setting the password here is ok, as this chart is only used for integration tests on CI
+        - name: "ELASTIC_PASSWORD"
+          value: "changeme"
         ports:
         - containerPort: 9200
           name: http

charts/elasticsearch-index/templates/create-index.yaml

@@ -21,20 +21,34 @@ spec:
         chart: "{{.Chart.Name}}-{{.Chart.Version}}"
     spec:
       restartPolicy: OnFailure
+      {{- if hasKey .Values.secrets "elasticsearch" }}
+      volumes:
+        - name: elasticsearch-index-secrets
+          secret:
+            secretName: elasticsearch-index
+      {{- end }}
       initContainers:
         # Creates index in elasticsearch only when it doesn't exist.
         # Does nothing if the index exists.
         - name: brig-index-create
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-          imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}
-        {{- if eq (include "includeSecurityContext" .) "true" }}
+          {{- if hasKey .Values.secrets "elasticsearch" }}
+          volumeMounts:
+            - name: "elasticsearch-index-secrets"
+              mountPath: "/etc/wire/elasticsearch-index/secrets"
+          {{- end }}
+          {{- if eq (include "includeSecurityContext" .) "true" }}
           securityContext:
             {{- toYaml .Values.podSecurityContext | nindent 12 }}
-        {{- end }}
+          {{- end }}
           args:
             - create
             - --elasticsearch-server
             - "http://{{ required "missing elasticsearch-index.elasticsearch.host!" .Values.elasticsearch.host }}:{{ .Values.elasticsearch.port }}"
+            {{- if hasKey .Values.secrets "elasticsearch" }}
+            - --elasticsearch-credentials
+            - "/etc/wire/elasticsearch-index/secrets/elasticsearch-credentials.yaml"
+            {{- end }}
             - --elasticsearch-index
             - "{{ or (.Values.elasticsearch.additionalWriteIndex) (.Values.elasticsearch.index) }}"
             - --elasticsearch-shards=5
@@ -48,13 +62,22 @@ spec:
         - name: brig-index-update-mapping
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}
-        {{- if eq (include "includeSecurityContext" .) "true" }}
+          {{- if hasKey .Values.secrets "elasticsearch" }}
+          volumeMounts:
+            - name: "elasticsearch-index-secrets"
+              mountPath: "/etc/wire/elasticsearch-index/secrets"
+          {{- end }}
+          {{- if eq (include "includeSecurityContext" .) "true" }}
           securityContext:
             {{- toYaml .Values.podSecurityContext | nindent 12 }}
-        {{- end }}
+          {{- end }}
           args:
             - update-mapping
             - --elasticsearch-server
             - "http://{{ required "missing elasticsearch-index.elasticsearch.host!" .Values.elasticsearch.host }}:{{ .Values.elasticsearch.port }}"
+            {{- if hasKey .Values.secrets "elasticsearch" }}
+            - --elasticsearch-credentials
+            - "/etc/wire/elasticsearch-index/secrets/elasticsearch-credentials.yaml"
+            {{- end }}
             - --elasticsearch-index
             - "{{ or (.Values.elasticsearch.additionalWriteIndex) (.Values.elasticsearch.index) }}"

charts/elasticsearch-index/templates/migrate-data.yaml

@@ -31,6 +31,10 @@ spec:
             - migrate-data
             - --elasticsearch-server
             - "http://{{ required "missing elasticsearch-index.elasticsearch.host!" .Values.elasticsearch.host }}:{{ .Values.elasticsearch.port }}"
+            {{- if hasKey .Values.secrets "elasticsearch" }}
+            - --elasticsearch-credentials
+            - "/etc/wire/elasticsearch-index/secrets/elasticsearch-credentials.yaml"
+            {{- end }}
             - --elasticsearch-index
             - "{{ or (.Values.elasticsearch.additionalWriteIndex) (.Values.elasticsearch.index) }}"
             - --cassandra-host
@@ -47,14 +51,23 @@ spec:
             - --tls-ca-certificate-file
             - /certs/{{- (include "tlsSecretRef" .Values | fromYaml).key }}
             {{- end }}
-          {{- if eq (include "useCassandraTLS" .Values) "true" }}
           volumeMounts:
+          {{- if hasKey .Values.secrets "elasticsearch" }}
+            - name: "elasticsearch-index-secrets"
+              mountPath: "/etc/wire/elasticsearch-index/secrets"
+          {{- end }}
+          {{- if eq (include "useCassandraTLS" .Values) "true" }}
             - name: elasticsearch-index-migrate-cassandra-client-ca
               mountPath: "/certs"
           {{- end }}
-      {{- if eq (include "useCassandraTLS" .Values) "true" }}
       volumes:
+        {{- if hasKey .Values.secrets "elasticsearch" }}
+        - name: elasticsearch-index-secrets
+          secret:
+            secretName: elasticsearch-index
+        {{- end }}
+        {{- if eq (include "useCassandraTLS" .Values) "true" }}
         - name: elasticsearch-index-migrate-cassandra-client-ca
           secret:
             secretName: {{ (include "tlsSecretRef" .Values | fromYaml).name }}
-      {{- end}}
+        {{- end}}

charts/elasticsearch-index/templates/secret.yaml

@@ -0,0 +1,20 @@
+{{- if hasKey .Values.secrets "elasticsearch" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: elasticsearch-index
+  labels:
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-delete-policy": "before-hook-creation"
+type: Opaque
+data:
+  {{- with .Values.secrets }}
+  {{- if .elasticsearch }}
+  elasticsearch-credentials.yaml: {{ .elasticsearch | toYaml | b64enc }}
+  {{- end }}
+  {{- end }}
+{{- end }}

charts/elasticsearch-index/values.yaml

@@ -30,3 +30,5 @@ podSecurityContext:
   runAsNonRoot: true
   seccompProfile:
     type: RuntimeDefault
+
+secrets: {}

charts/integration/templates/integration-integration.yaml

@@ -127,7 +127,7 @@ spec:
           --tls-ca-certificate-file /certs/{{- (include "tlsSecretRef" .Values.config | fromYaml).key }}
           {{ end }}
 
-        integration-dynamic-backends-brig-index.sh --elasticsearch-server http://{{ .Values.config.elasticsearch.host }}:9200
+        integration-dynamic-backends-brig-index.sh --elasticsearch-server http://elastic:changeme@{{ .Values.config.elasticsearch.host }}:9200
         integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }}
         integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }}
         {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }}

deploy/dockerephemeral/docker-compose.yaml

@@ -159,10 +159,13 @@ services:
 
   elasticsearch:
     container_name: demo_wire_elasticsearch
-    #image: elasticsearch:5.6
-    image: julialongtin/elasticsearch:0.0.9-amd64
-    # https://hub.docker.com/_/elastic is deprecated, but 6.2.4 did not work without further changes.
-    # image: docker.elastic.co/elasticsearch/elasticsearch:6.2.4
+    build:
+      context: .
+      dockerfile_inline: |
+        FROM julialongtin/elasticsearch:0.0.9-amd64
+        RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install x-pack -b
+        # this seems to be necessary to run X-Pack on Alpine (https://discuss.elastic.co/t/elasticsearch-failing-to-start-due-to-x-pack/85125/7)
+        RUN rm -rf /usr/share/elasticsearch/plugins/x-pack/platform/linux-x86_64
     ulimits:
       nofile:
         soft: 65536
@@ -171,10 +174,9 @@ services:
       - "127.0.0.1:9200:9200"
       - "127.0.0.1:9300:9300"
     environment:
+      - "xpack.ml.enabled=false"
+      - "xpack.security.enabled=true"
       - "bootstrap.system_call_filter=false"
-# ES_JVM_OPTIONS is reserved, so...
-# what's present in the jvm.options file by default.
-#      - "JVM_OPTIONS_ES=-Xmx2g -Xms2g"
       - "JVM_OPTIONS_ES=-Xmx512m -Xms512m"
       - "discovery.type=single-node"
     networks:

deploy/dockerephemeral/federation-v0/brig.yaml

@@ -10,7 +10,8 @@ cassandra:
   # filterNodesByDatacentre: datacenter1
 
 elasticsearch:
-  url: http://demo_wire_elasticsearch:9200
+  # FUTUREWORK: use separate ES v0 instance
+  url: http://elastic:changeme@demo_wire_elasticsearch:9200
   index: directory_test
 
 rabbitmq:

docs/src/developer/reference/config-options.md

@@ -853,3 +853,31 @@ accessible to services (and not the private key.)
 
 The corresponding Cassandra options are described in Cassandra's documentation:
 [client_encryption_options](https://cassandra.apache.org/doc/stable/cassandra/configuration/cass_yaml_file.html#client_encryption_options)
+
+## Configure Elasticsearch basic authentication
+
+When the Wire backend is configured to work against a custom Elasticsearch instance, it may be desired to enable basic authentication for the internal communication between the Wire backend and the ES instance. To do so the Elasticsearch credentials can be set in wire-server's secrets for `brig` and `elasticsearch-index` as follows:
+
+```yaml
+brig:
+  secrets:
+    elasticsearch:
+      username: elastic
+      password: changeme
+
+elasticsearch-index:
+  secrets:
+    elasticsearch:
+      username: elastic
+      password: changeme
+```
+
+In some cases an additional Elasticsearch instance is needed (e.g. for index migrations). To configure credentials for the additional ES instance add the secret as follows:
+
+```yaml
+brig:
+  secrets:
+    elasticsearchAdditional:
+      username: elastic
+      password: changeme
+```

hack/helm_vars/wire-server/values.yaml.gotmpl

@@ -38,6 +38,10 @@ elasticsearch-index:
         name: "cassandra-jks-keystore"
         key: "ca.crt"
     {{- end }}
+  secrets:
+    elasticsearch:
+      username: "elastic"
+      password: "changeme"
 
 brig:
   replicaCount: 1
@@ -151,6 +155,12 @@ brig:
     rabbitmq:
       username: {{ .Values.rabbitmqUsername }}
       password: {{ .Values.rabbitmqPassword }}
+    elasticsearch:
+      username: "elastic"
+      password: "changeme"
+    elasticsearchAdditional:
+      username: "elastic"
+      password: "changeme"
   tests:
     enableFederationTests: true
     {{- if .Values.uploadXml }}

libs/types-common/src/Data/Credentials.hs

@@ -0,0 +1,31 @@
+-- This file is part of the Wire Server implementation.
+--
+-- Copyright (C) 2024 Wire Swiss GmbH <opensource@wire.com>
+--
+-- This program is free software: you can redistribute it and/or modify it under
+-- the terms of the GNU Affero General Public License as published by the Free
+-- Software Foundation, either version 3 of the License, or (at your option) any
+-- later version.
+--
+-- This program is distributed in the hope that it will be useful, but WITHOUT
+-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+-- details.
+--
+-- You should have received a copy of the GNU Affero General Public License along
+-- with this program. If not, see <https://www.gnu.org/licenses/>.
+
+module Data.Credentials where
+
+import Data.Aeson (FromJSON)
+import Data.Text
+import Imports
+
+-- | Generic credentials for authenticating a user. Usually used for deserializing from a secret yaml file.
+data Credentials = Credentials
+  { username :: Text,
+    password :: Text
+  }
+  deriving stock (Generic)
+
+instance FromJSON Credentials

libs/types-common/src/Util/Options.hs

@@ -79,14 +79,19 @@ urlPort u = do
 makeLenses ''AWSEndpoint
 
 newtype FilePathSecrets = FilePathSecrets FilePath
-  deriving (Eq, Show, FromJSON)
+  deriving (Eq, Show, FromJSON, IsString)
 
-loadSecret :: FromJSON a => FilePathSecrets -> IO (Either String a)
+initCredentials :: (MonadIO m, FromJSON a) => FilePathSecrets -> m a
+initCredentials secretFile = do
+  dat <- loadSecret secretFile
+  pure $ either (\e -> error $ "Could not load secrets from " ++ show secretFile ++ ": " ++ e) id dat
+
+loadSecret :: (MonadIO m, FromJSON a) => FilePathSecrets -> m (Either String a)
 loadSecret (FilePathSecrets p) = do
   path <- canonicalizePath p
   exists <- doesFileExist path
   if exists
-    then over _Left show . decodeEither' <$> BS.readFile path
+    then liftIO $ over _Left show . decodeEither' <$> BS.readFile path
     else pure (Left "File doesn't exist")
 
 -- | Get configuration options from the command line or configuration file.

libs/types-common/types-common.cabal

@@ -15,6 +15,7 @@ library
   exposed-modules:
     Data.Code
     Data.CommaSeparatedList
+    Data.Credentials
     Data.Domain
     Data.ETag
     Data.Handle

services/brig/brig.integration.yaml

@@ -12,6 +12,8 @@ cassandra:
 elasticsearch:
   url: http://127.0.0.1:9200
   index: directory_test
+  credentials: test/resources/elasticsearch-credentials.yaml
+  additionalCredentials: test/resources/elasticsearch-credentials.yaml
 
 rabbitmq:
   host: 127.0.0.1