From 44bd8b79e01bbbc81db5d9da5d33fed33904e99f Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 25 Mar 2024 16:30:40 +0100 Subject: [PATCH 01/14] dockerephemeral: Start redis in master mode for testing redis migrations locally This seems to have been deleted by mistake in https://github.com/wireapp/wire-server/pull/3719 --- deploy/dockerephemeral/docker-compose.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/dockerephemeral/docker-compose.yaml b/deploy/dockerephemeral/docker-compose.yaml index 2ad299a6d1..614c0c8fbc 100644 --- a/deploy/dockerephemeral/docker-compose.yaml +++ b/deploy/dockerephemeral/docker-compose.yaml @@ -77,6 +77,14 @@ services: networks: - demo_wire + redis: + container_name: demo_wire_redis + image: redis:6.2.14-alpine + ports: + - "127.0.0.1:6379:6379" + networks: + - demo_wire + redis-cluster: image: 'redis:6.0-alpine' command: redis-cli --cluster create 172.20.0.31:6373 172.20.0.32:6374 172.20.0.33:6375 172.20.0.34:6376 172.20.0.35:6377 172.20.0.36:6378 --cluster-replicas 1 --cluster-yes From 8d127bd7e27dcb371505464bf5d1a5096f628d11 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 25 Mar 2024 17:01:19 +0100 Subject: [PATCH 02/14] dockerephemeral: Start redis with creds --- deploy/dockerephemeral/docker-compose.yaml | 9 ++++++--- deploy/dockerephemeral/docker/redis-master-mode.conf | 1 + deploy/dockerephemeral/docker/redis-node-1.conf | 2 ++ deploy/dockerephemeral/docker/redis-node-2.conf | 2 ++ deploy/dockerephemeral/docker/redis-node-3.conf | 2 ++ deploy/dockerephemeral/docker/redis-node-4.conf | 2 ++ deploy/dockerephemeral/docker/redis-node-5.conf | 2 ++ deploy/dockerephemeral/docker/redis-node-6.conf | 2 ++ 8 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 deploy/dockerephemeral/docker/redis-master-mode.conf diff --git a/deploy/dockerephemeral/docker-compose.yaml b/deploy/dockerephemeral/docker-compose.yaml index 614c0c8fbc..fe0a3f3b49 100644 --- a/deploy/dockerephemeral/docker-compose.yaml +++ b/deploy/dockerephemeral/docker-compose.yaml @@ -77,17 +77,20 @@ services: networks: - demo_wire - redis: + redis-master: container_name: demo_wire_redis - image: redis:6.2.14-alpine + image: redis:6.0-alpine + command: redis-server /usr/local/etc/redis/redis.conf ports: - "127.0.0.1:6379:6379" + volumes: + - ./docker/redis-master-mode.conf:/usr/local/etc/redis/redis.conf networks: - demo_wire redis-cluster: image: 'redis:6.0-alpine' - command: redis-cli --cluster create 172.20.0.31:6373 172.20.0.32:6374 172.20.0.33:6375 172.20.0.34:6376 172.20.0.35:6377 172.20.0.36:6378 --cluster-replicas 1 --cluster-yes + command: redis-cli --cluster create 172.20.0.31:6373 172.20.0.32:6374 172.20.0.33:6375 172.20.0.34:6376 172.20.0.35:6377 172.20.0.36:6378 --cluster-replicas 1 --cluster-yes -a very-secure-redis-password networks: redis: ipv4_address: 172.20.0.30 diff --git a/deploy/dockerephemeral/docker/redis-master-mode.conf b/deploy/dockerephemeral/docker/redis-master-mode.conf new file mode 100644 index 0000000000..d71dbc51c9 --- /dev/null +++ b/deploy/dockerephemeral/docker/redis-master-mode.conf @@ -0,0 +1 @@ +requirepass very-secure-redis-master-password \ No newline at end of file diff --git a/deploy/dockerephemeral/docker/redis-node-1.conf b/deploy/dockerephemeral/docker/redis-node-1.conf index 3f7f7d69ff..833d14343b 100644 --- a/deploy/dockerephemeral/docker/redis-node-1.conf +++ b/deploy/dockerephemeral/docker/redis-node-1.conf @@ -3,3 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes +requirepass very-secure-redis-password +masterauth very-secure-redis-password \ No newline at end of file diff --git a/deploy/dockerephemeral/docker/redis-node-2.conf b/deploy/dockerephemeral/docker/redis-node-2.conf index c81ccd43ff..bed6fd8918 100644 --- a/deploy/dockerephemeral/docker/redis-node-2.conf +++ b/deploy/dockerephemeral/docker/redis-node-2.conf @@ -3,3 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes +requirepass very-secure-redis-password +masterauth very-secure-redis-password \ No newline at end of file diff --git a/deploy/dockerephemeral/docker/redis-node-3.conf b/deploy/dockerephemeral/docker/redis-node-3.conf index 6ae5804185..91ab093304 100644 --- a/deploy/dockerephemeral/docker/redis-node-3.conf +++ b/deploy/dockerephemeral/docker/redis-node-3.conf @@ -3,3 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes +requirepass very-secure-redis-password +masterauth very-secure-redis-password diff --git a/deploy/dockerephemeral/docker/redis-node-4.conf b/deploy/dockerephemeral/docker/redis-node-4.conf index 1c3464629e..9ade3d1b14 100644 --- a/deploy/dockerephemeral/docker/redis-node-4.conf +++ b/deploy/dockerephemeral/docker/redis-node-4.conf @@ -3,3 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes +requirepass very-secure-redis-password +masterauth very-secure-redis-password diff --git a/deploy/dockerephemeral/docker/redis-node-5.conf b/deploy/dockerephemeral/docker/redis-node-5.conf index e28f7909b5..47d283706c 100644 --- a/deploy/dockerephemeral/docker/redis-node-5.conf +++ b/deploy/dockerephemeral/docker/redis-node-5.conf @@ -3,3 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes +requirepass very-secure-redis-password +masterauth very-secure-redis-password diff --git a/deploy/dockerephemeral/docker/redis-node-6.conf b/deploy/dockerephemeral/docker/redis-node-6.conf index b77c8e19c9..43f708edfa 100644 --- a/deploy/dockerephemeral/docker/redis-node-6.conf +++ b/deploy/dockerephemeral/docker/redis-node-6.conf @@ -3,3 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes +requirepass very-secure-redis-password +masterauth very-secure-redis-password From ac307dd91a3f3ec930a3357ba31e81dd0ad856c3 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 25 Mar 2024 17:02:33 +0100 Subject: [PATCH 03/14] nix: Pin hedis to our fork which does auth correctly --- nix/haskell-pins.nix | 9 +++++++++ nix/manual-overrides.nix | 3 +++ services/gundeck/src/Gundeck/Presence/Data.hs | 7 ++++--- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/nix/haskell-pins.nix b/nix/haskell-pins.nix index 1edd473ead..831fe39eff 100644 --- a/nix/haskell-pins.nix +++ b/nix/haskell-pins.nix @@ -116,6 +116,15 @@ let }; }; + # PR: https://github.com/informatikr/hedis/pull/224 + hedis = { + src = fetchgit { + url = "https://github.com/wireapp/hedis"; + rev = "81cdd8a2350b96168a06662c2601a41141a19f2d"; + sha256 = "sha256-0g6x9UOUq7s5ClnxMXvjYR2AsWNA6ymv1tYlQC44hGs="; + }; + }; + # Our fork because we need to a few special things http-client = { src = fetchgit { diff --git a/nix/manual-overrides.nix b/nix/manual-overrides.nix index 51d0f437ae..2a70e83728 100644 --- a/nix/manual-overrides.nix +++ b/nix/manual-overrides.nix @@ -23,6 +23,9 @@ hself: hsuper: { transitive-anns = hlib.dontCheck hsuper.transitive-anns; warp = hlib.dontCheck hsuper.warp; + # Tests require a running redis + hedis = hlib.dontCheck hsuper.hedis; + # --------------------- # need to be jailbroken # (these need to be fixed upstream eventually) diff --git a/services/gundeck/src/Gundeck/Presence/Data.hs b/services/gundeck/src/Gundeck/Presence/Data.hs index 1536dab9ea..158e898221 100644 --- a/services/gundeck/src/Gundeck/Presence/Data.hs +++ b/services/gundeck/src/Gundeck/Presence/Data.hs @@ -25,13 +25,14 @@ where import Control.Monad.Catch import Control.Monad.Except -import Data.Aeson +import Data.Aeson as Aeson import Data.ByteString qualified as Strict import Data.ByteString.Builder (byteString) import Data.ByteString.Char8 qualified as StrictChars import Data.ByteString.Conversion hiding (fromList) import Data.ByteString.Lazy qualified as Lazy import Data.Id +import Data.List.NonEmpty qualified as NonEmpty import Data.Misc (Milliseconds) import Database.Redis import Gundeck.Monad (Gundeck, posixTime, runWithAdditionalRedis) @@ -61,10 +62,10 @@ add p = do now <- posixTime let k = toKey (userId p) let v = toField (connId p) - let d = Lazy.toStrict $ encode $ PresenceData (resource p) (clientId p) now + let d = Lazy.toStrict $ Aeson.encode $ PresenceData p.resource p.clientId now runWithAdditionalRedis . retry x3 $ do void . fromTxResult <=< (liftRedis . multiExec) $ do - void $ hset k v d + void $ hset k (NonEmpty.singleton (v, d)) -- nb. All presences of a user are expired 'maxIdleTime' after the -- last presence was registered. A client who keeps a presence -- (i.e. websocket) connected for longer than 'maxIdleTime' will be From 79c6e69e054d313bd8a4e6def381dfccb5c56ff6 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 25 Mar 2024 17:03:54 +0100 Subject: [PATCH 04/14] gundeck: Log uncaught errors in redis connection --- services/gundeck/src/Gundeck/Redis.hs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/gundeck/src/Gundeck/Redis.hs b/services/gundeck/src/Gundeck/Redis.hs index 721106bdbd..a4784349db 100644 --- a/services/gundeck/src/Gundeck/Redis.hs +++ b/services/gundeck/src/Gundeck/Redis.hs @@ -61,7 +61,7 @@ connectRobust :: connectRobust l retryStrategy connectLowLevel = do robustConnection <- newEmptyMVar @IO @Connection thread <- - async $ safeForever $ do + async $ safeForever l $ do Log.info l $ Log.msg (Log.val "connecting to Redis") conn <- retry connectLowLevel Log.info l $ Log.msg (Log.val "successfully connected to Redis") @@ -117,9 +117,11 @@ instance Exception PingException safeForever :: forall m. (MonadUnliftIO m) => + Logger -> m () -> m () -safeForever action = +safeForever l action = forever $ - action `catchAny` \_ -> do + action `catchAny` \e -> do + Log.err l $ Log.msg (Log.val "Uncaught exception while connecting to redis") . Log.field "error" (displayException e) threadDelay 1e6 -- pause to keep worst-case noise in logs manageable From 289cbf9153bb6b4ddb310ff466ec700e023442d5 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 25 Mar 2024 17:04:42 +0100 Subject: [PATCH 05/14] gundeck: Use redis creds from environment when provided --- .envrc | 7 ++++++- services/gundeck/src/Gundeck/Env.hs | 20 +++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.envrc b/.envrc index b7b3f2c35f..6d0a2d866b 100644 --- a/.envrc +++ b/.envrc @@ -49,6 +49,11 @@ export LANG=en_US.UTF-8 export RABBITMQ_USERNAME=guest export RABBITMQ_PASSWORD=alpaca-grapefruit +# Redis + +export REDIS_PASSWORD=very-secure-redis-password +export REDIS_ADDITIONAL_WRITE_PASSWORD=very-secure-redis-master-password + # Integration tests export INTEGRATION_DYNAMIC_BACKENDS_POOLSIZE=3 @@ -58,7 +63,7 @@ export AWS_REGION="eu-west-1" export AWS_ACCESS_KEY_ID="dummykey" export AWS_SECRET_ACCESS_KEY="dummysecret" -# integration test suite timeout +# integration test suite timeout export TEST_TIMEOUT_SECONDS=2 # allow local .envrc overrides diff --git a/services/gundeck/src/Gundeck/Env.hs b/services/gundeck/src/Gundeck/Env.hs index fdc67f2f22..c9e8a4d286 100644 --- a/services/gundeck/src/Gundeck/Env.hs +++ b/services/gundeck/src/Gundeck/Env.hs @@ -26,6 +26,7 @@ import Control.AutoUpdate import Control.Concurrent.Async (Async) import Control.Lens (makeLenses, (^.)) import Control.Retry (capDelay, exponentialBackoff) +import Data.ByteString.Char8 qualified as BSChar8 import Data.Metrics.Middleware (Metrics) import Data.Misc (Milliseconds (..)) import Data.Text (unpack) @@ -74,12 +75,16 @@ createEnv m o = do managerResponseTimeout = responseTimeoutMicro 5000000 } - (rThread, r) <- createRedisPool l (o ^. redis) "main-redis" + redisUsername <- BSChar8.pack <$$> lookupEnv "REDIS_USERNAME" + redisPassword <- BSChar8.pack <$$> lookupEnv "REDIS_PASSWORD" + (rThread, r) <- createRedisPool l (o ^. redis) redisUsername redisPassword "main-redis" (rAdditionalThreads, rAdditional) <- case o ^. redisAdditionalWrite of Nothing -> pure ([], Nothing) Just additionalRedis -> do - (rAddThread, rAdd) <- createRedisPool l additionalRedis "additional-write-redis" + additionalRedisUsername <- BSChar8.pack <$$> lookupEnv "REDIS_ADDITIONAL_WRITE_USERNAME" + addtionalRedisPassword <- BSChar8.pack <$$> lookupEnv "REDIS_ADDITIONAL_WRITE_PASSWORD" + (rAddThread, rAdd) <- createRedisPool l additionalRedis additionalRedisUsername addtionalRedisPassword "additional-write-redis" pure ([rAddThread], Just rAdd) p <- @@ -103,12 +108,14 @@ reqIdMsg :: RequestId -> Logger.Msg -> Logger.Msg reqIdMsg = ("request" Logger..=) . unRequestId {-# INLINE reqIdMsg #-} -createRedisPool :: Logger.Logger -> RedisEndpoint -> ByteString -> IO (Async (), Redis.RobustConnection) -createRedisPool l ep identifier = do +createRedisPool :: Logger.Logger -> RedisEndpoint -> Maybe ByteString -> Maybe ByteString -> ByteString -> IO (Async (), Redis.RobustConnection) +createRedisPool l ep username password identifier = do let redisConnInfo = Redis.defaultConnectInfo { Redis.connectHost = unpack $ ep ^. O.host, Redis.connectPort = Redis.PortNumber (fromIntegral $ ep ^. O.port), + Redis.connectUsername = username, + Redis.connectAuth = password, Redis.connectTimeout = Just (secondsToNominalDiffTime 5), Redis.connectMaxConnections = 100 } @@ -116,10 +123,13 @@ createRedisPool l ep identifier = do Log.info l $ Log.msg (Log.val $ "starting connection to " <> identifier <> "...") . Log.field "connectionMode" (show $ ep ^. O.connectionMode) - . Log.field "connInfo" (show redisConnInfo) + . Log.field "connInfo" (safeShowConnInfo redisConnInfo) let connectWithRetry = Redis.connectRobust l (capDelay 1000000 (exponentialBackoff 50000)) r <- case ep ^. O.connectionMode of Master -> connectWithRetry $ Redis.checkedConnect redisConnInfo Cluster -> connectWithRetry $ Redis.checkedConnectCluster redisConnInfo Log.info l $ Log.msg (Log.val $ "Established connection to " <> identifier <> ".") pure r + +safeShowConnInfo :: Redis.ConnectInfo -> String +safeShowConnInfo connInfo = show $ connInfo {Redis.connectAuth = "[REDACTED]" <$ Redis.connectAuth connInfo} From 77868acfde6911976ffb624de5a2732400ce5208 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Tue, 26 Mar 2024 09:39:34 +0100 Subject: [PATCH 06/14] totally unnecessary commit --- .envrc | 2 +- deploy/dockerephemeral/docker-compose.yaml | 2 +- deploy/dockerephemeral/docker/redis-node-1.conf | 4 ++-- deploy/dockerephemeral/docker/redis-node-2.conf | 4 ++-- deploy/dockerephemeral/docker/redis-node-3.conf | 4 ++-- deploy/dockerephemeral/docker/redis-node-4.conf | 4 ++-- deploy/dockerephemeral/docker/redis-node-5.conf | 4 ++-- deploy/dockerephemeral/docker/redis-node-6.conf | 4 ++-- 8 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.envrc b/.envrc index 6d0a2d866b..6fd34b7098 100644 --- a/.envrc +++ b/.envrc @@ -51,7 +51,7 @@ export RABBITMQ_PASSWORD=alpaca-grapefruit # Redis -export REDIS_PASSWORD=very-secure-redis-password +export REDIS_PASSWORD=very-secure-redis-cluster-password export REDIS_ADDITIONAL_WRITE_PASSWORD=very-secure-redis-master-password # Integration tests diff --git a/deploy/dockerephemeral/docker-compose.yaml b/deploy/dockerephemeral/docker-compose.yaml index fe0a3f3b49..d49e141dfb 100644 --- a/deploy/dockerephemeral/docker-compose.yaml +++ b/deploy/dockerephemeral/docker-compose.yaml @@ -90,7 +90,7 @@ services: redis-cluster: image: 'redis:6.0-alpine' - command: redis-cli --cluster create 172.20.0.31:6373 172.20.0.32:6374 172.20.0.33:6375 172.20.0.34:6376 172.20.0.35:6377 172.20.0.36:6378 --cluster-replicas 1 --cluster-yes -a very-secure-redis-password + command: redis-cli --cluster create 172.20.0.31:6373 172.20.0.32:6374 172.20.0.33:6375 172.20.0.34:6376 172.20.0.35:6377 172.20.0.36:6378 --cluster-replicas 1 --cluster-yes -a very-secure-redis-cluster-password networks: redis: ipv4_address: 172.20.0.30 diff --git a/deploy/dockerephemeral/docker/redis-node-1.conf b/deploy/dockerephemeral/docker/redis-node-1.conf index 833d14343b..011df166cd 100644 --- a/deploy/dockerephemeral/docker/redis-node-1.conf +++ b/deploy/dockerephemeral/docker/redis-node-1.conf @@ -3,5 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes -requirepass very-secure-redis-password -masterauth very-secure-redis-password \ No newline at end of file +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password \ No newline at end of file diff --git a/deploy/dockerephemeral/docker/redis-node-2.conf b/deploy/dockerephemeral/docker/redis-node-2.conf index bed6fd8918..fa2850e923 100644 --- a/deploy/dockerephemeral/docker/redis-node-2.conf +++ b/deploy/dockerephemeral/docker/redis-node-2.conf @@ -3,5 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes -requirepass very-secure-redis-password -masterauth very-secure-redis-password \ No newline at end of file +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password \ No newline at end of file diff --git a/deploy/dockerephemeral/docker/redis-node-3.conf b/deploy/dockerephemeral/docker/redis-node-3.conf index 91ab093304..81d01b5421 100644 --- a/deploy/dockerephemeral/docker/redis-node-3.conf +++ b/deploy/dockerephemeral/docker/redis-node-3.conf @@ -3,5 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes -requirepass very-secure-redis-password -masterauth very-secure-redis-password +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-4.conf b/deploy/dockerephemeral/docker/redis-node-4.conf index 9ade3d1b14..50361d2281 100644 --- a/deploy/dockerephemeral/docker/redis-node-4.conf +++ b/deploy/dockerephemeral/docker/redis-node-4.conf @@ -3,5 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes -requirepass very-secure-redis-password -masterauth very-secure-redis-password +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-5.conf b/deploy/dockerephemeral/docker/redis-node-5.conf index 47d283706c..68885b25b4 100644 --- a/deploy/dockerephemeral/docker/redis-node-5.conf +++ b/deploy/dockerephemeral/docker/redis-node-5.conf @@ -3,5 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes -requirepass very-secure-redis-password -masterauth very-secure-redis-password +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password diff --git a/deploy/dockerephemeral/docker/redis-node-6.conf b/deploy/dockerephemeral/docker/redis-node-6.conf index 43f708edfa..07da632579 100644 --- a/deploy/dockerephemeral/docker/redis-node-6.conf +++ b/deploy/dockerephemeral/docker/redis-node-6.conf @@ -3,5 +3,5 @@ cluster-enabled yes cluster-config-file nodes.conf cluster-node-timeout 5000 appendonly yes -requirepass very-secure-redis-password -masterauth very-secure-redis-password +requirepass very-secure-redis-cluster-password +masterauth very-secure-redis-cluster-password From 9e658411d6a4e3f5e639457d6531fa46fa3aeadd Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Tue, 26 Mar 2024 15:55:02 +0100 Subject: [PATCH 07/14] hack/helmfile: Spin up extra redis for testing redis migration --- charts/gundeck/templates/tests/configmap.yaml | 2 +- hack/helmfile.yaml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/charts/gundeck/templates/tests/configmap.yaml b/charts/gundeck/templates/tests/configmap.yaml index e2051925c1..b3e1423acf 100644 --- a/charts/gundeck/templates/tests/configmap.yaml +++ b/charts/gundeck/templates/tests/configmap.yaml @@ -41,6 +41,6 @@ data: # a "redis migration" test in gundeck makes use of a second (distinct) redis redis2: - host: redis-ephemeral-master + host: redis-ephemeral-2-master port: 6379 connectionMode: master diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index 78634f17b2..9c4814580e 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -71,6 +71,14 @@ releases: namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/databases-ephemeral' + # Required for testing redis migration + - name: 'redis-ephemeral-2' + namespace: '{{ .Values.namespace1 }}' + chart: '../.local/charts/redis-ephemeral' + values: + - redis-ephemeral: + nameOverride: redis-ephemeral-2 + - name: 'databases-ephemeral' namespace: '{{ .Values.namespace2 }}' chart: '../.local/charts/databases-ephemeral' From 9899f500f367567b3a19d94350ca8b956e860591 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Tue, 26 Mar 2024 15:55:55 +0100 Subject: [PATCH 08/14] hack: Run redis with auth --- charts/gundeck/templates/deployment.yaml | 28 +++++++++++++++++++ charts/gundeck/templates/secret.yaml | 14 +++++++++- .../templates/tests/gundeck-integration.yaml | 28 +++++++++++++++++++ charts/gundeck/templates/tests/secret.yaml | 13 ++++++++- .../templates/integration-integration.yaml | 14 ++++++++++ charts/integration/templates/secret.yaml | 10 +++++++ .../redis-cluster/values.yaml.gotmpl | 1 + hack/helm_vars/wire-server/values.yaml.gotmpl | 17 +++++++---- hack/helmfile.yaml | 13 +++++++++ 9 files changed, 131 insertions(+), 7 deletions(-) diff --git a/charts/gundeck/templates/deployment.yaml b/charts/gundeck/templates/deployment.yaml index 20ca798824..ec1e064ccc 100644 --- a/charts/gundeck/templates/deployment.yaml +++ b/charts/gundeck/templates/deployment.yaml @@ -65,6 +65,34 @@ spec: name: gundeck key: awsSecretKey {{- end }} + {{- if hasKey .Values.secrets "redisUsername" }} + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: gundeck + key: redisUsername + {{- end }} + {{- if hasKey .Values.secrets "redisPassword" }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck + key: redisPassword + {{- end }} + {{- if hasKey .Values.secrets "redisAdditionalWriteUsername" }} + - name: REDIS_ADDITIONAL_WRITE_USERNAME + valueFrom: + secretKeyRef: + name: gundeck + key: redisAdditionalWriteUsername + {{- end }} + {{- if hasKey .Values.secrets "redisAdditionalWritePassword" }} + - name: REDIS_ADDITIONAL_WRITE_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck + key: redisAdditionalWritePassword + {{- end }} - name: AWS_REGION value: "{{ .Values.config.aws.region }}" {{- with .Values.config.proxy }} diff --git a/charts/gundeck/templates/secret.yaml b/charts/gundeck/templates/secret.yaml index 459ab0f24f..f20aeaea5a 100644 --- a/charts/gundeck/templates/secret.yaml +++ b/charts/gundeck/templates/secret.yaml @@ -1,4 +1,4 @@ -{{- if hasKey .Values.secrets "awsKeyId" }} +{{- if not (empty .Values.secrets) }} apiVersion: v1 kind: Secret metadata: @@ -13,5 +13,17 @@ data: {{- with .Values.secrets }} awsKeyId: {{ .awsKeyId | b64enc | quote }} awsSecretKey: {{ .awsSecretKey | b64enc | quote }} + {{- if hasKey . "redisUsername" }} + redisUsername: {{ .redisUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisPassword" }} + redisPassword: {{ .redisPassword | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisAdditionalWriteUsername" }} + redisAdditionalWriteUsername: {{ .redisAdditionalWriteUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisAdditionalWritePassword" }} + redisAdditionalWritePassword: {{ .redisAdditionalWritePassword | b64enc | quote }} + {{- end }} {{- end }} {{- end }} diff --git a/charts/gundeck/templates/tests/gundeck-integration.yaml b/charts/gundeck/templates/tests/gundeck-integration.yaml index 8b00f2c986..088ed679bd 100644 --- a/charts/gundeck/templates/tests/gundeck-integration.yaml +++ b/charts/gundeck/templates/tests/gundeck-integration.yaml @@ -73,6 +73,34 @@ spec: value: "eu-west-1" - name: TEST_XML value: /tmp/result.xml + {{- if hasKey .Values.secrets "redisUsername" }} + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: gundeck + key: redisUsername + {{- end }} + {{- if hasKey .Values.secrets "redisPassword" }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck + key: redisPassword + {{- end }} + {{- if and (hasKey .Values.tests "secrets") (hasKey .Values.tests.secrets "redisAdditionalWriteUsername") }} + - name: REDIS_ADDITIONAL_WRITE_USERNAME + valueFrom: + secretKeyRef: + name: gundeck-integration + key: redisAdditionalWriteUsername + {{- end }} + {{- if and (hasKey .Values.tests "secrets") (hasKey .Values.tests.secrets "redisAdditionalWritePassword") }} + - name: REDIS_ADDITIONAL_WRITE_PASSWORD + valueFrom: + secretKeyRef: + name: gundeck-integration + key: redisAdditionalWritePassword + {{- end }} {{- if .Values.tests.config.uploadXml }} - name: UPLOAD_XML_S3_BASE_URL value: {{ .Values.tests.config.uploadXml.baseUrl }} diff --git a/charts/gundeck/templates/tests/secret.yaml b/charts/gundeck/templates/tests/secret.yaml index 1af8959e4c..ff5712545c 100644 --- a/charts/gundeck/templates/tests/secret.yaml +++ b/charts/gundeck/templates/tests/secret.yaml @@ -1,3 +1,4 @@ +{{- if not (empty .Values.tests.secrets) }} apiVersion: v1 kind: Secret metadata: @@ -10,7 +11,17 @@ metadata: type: Opaque data: {{- with .Values.tests.secrets }} + {{- if hasKey . "uploadXmlAwsAccessKeyId" }} uploadXmlAwsAccessKeyId: {{ .uploadXmlAwsAccessKeyId | b64enc | quote }} + {{- end }} + {{- if hasKey . "uploadXmlAwsSecretAccessKey" }} uploadXmlAwsSecretAccessKey: {{ .uploadXmlAwsSecretAccessKey | b64enc | quote }} {{- end }} - + {{- if hasKey . "redisAdditionalWriteUsername" }} + redisAdditionalWriteUsername: {{ .redisAdditionalWriteUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisAdditionalWritePassword" }} + redisAdditionalWritePassword: {{ .redisAdditionalWritePassword | b64enc | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 5f6a98f759..e0d72e62a8 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -262,6 +262,20 @@ spec: secretKeyRef: name: brig key: rabbitmqPassword + {{- if hasKey .Values.secrets "redisUsername" }} + - name: REDIS_USERNAME + valueFrom: + secretKeyRef: + name: integration + key: redisUsername + {{- end }} + {{- if hasKey .Values.secrets "redisPassword" }} + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: integration + key: redisPassword + {{- end }} - name: TEST_XML value: /tmp/result.xml {{- if .Values.config.uploadXml }} diff --git a/charts/integration/templates/secret.yaml b/charts/integration/templates/secret.yaml index 52c3199b5f..32f6085176 100644 --- a/charts/integration/templates/secret.yaml +++ b/charts/integration/templates/secret.yaml @@ -10,6 +10,16 @@ metadata: type: Opaque data: {{- with .Values.secrets }} + {{- if hasKey . "uploadXmlAwsAccessKeyId" }} uploadXmlAwsAccessKeyId: {{ .uploadXmlAwsAccessKeyId | b64enc | quote }} + {{- end }} + {{- if hasKey . "uploadXmlAwsSecretAccessKey" }} uploadXmlAwsSecretAccessKey: {{ .uploadXmlAwsSecretAccessKey | b64enc | quote }} {{- end }} + {{- if hasKey . "redisUsername" }} + redisUsername: {{ .redisUsername | b64enc | quote }} + {{- end }} + {{- if hasKey . "redisPassword" }} + redisPassword: {{ .redisPassword | b64enc | quote }} + {{- end }} + {{- end }} diff --git a/hack/helm_vars/redis-cluster/values.yaml.gotmpl b/hack/helm_vars/redis-cluster/values.yaml.gotmpl index 658cb79556..9d81712a59 100644 --- a/hack/helm_vars/redis-cluster/values.yaml.gotmpl +++ b/hack/helm_vars/redis-cluster/values.yaml.gotmpl @@ -6,3 +6,4 @@ redis-cluster: size: 100Mi volumePermissions: enabled: true + password: very-secure-redis-cluster-password diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index d215f8efd6..1547d6f846 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -313,15 +313,19 @@ gundeck: secrets: awsKeyId: dummykey awsSecretKey: dummysecret + redisPassword: very-secure-redis-master-password tests: {{- if .Values.uploadXml }} config: uploadXml: baseUrl: {{ .Values.uploadXml.baseUrl }} + {{- end }} secrets: + {{- if .Values.uploadXml }} uploadXmlAwsAccessKeyId: {{ .Values.uploadXml.awsAccessKeyId }} uploadXmlAwsSecretAccessKey: {{ .Values.uploadXml.awsSecretAccessKey }} - {{- end }} + {{- end }} + redisAdditionalWritePassword: very-secure-redis-master-password-2 nginz: replicaCount: 1 @@ -444,18 +448,21 @@ integration: host: {{ .Values.cassandraHost }} port: 9042 replicationFactor: 1 - {{- if .Values.useK8ssandraSSL.enabled }} + {{- if .Values.useK8ssandraSSL.enabled }} tlsCaSecretRef: name: cassandra-jks-keystore key: ca.crt - {{- end }} - {{- if .Values.uploadXml }} + {{- end }} + {{- if .Values.uploadXml }} uploadXml: baseUrl: {{ .Values.uploadXml.baseUrl }} + {{- end }} secrets: + {{- if .Values.uploadXml }} uploadXmlAwsAccessKeyId: {{ .Values.uploadXml.awsAccessKeyId }} uploadXmlAwsSecretAccessKey: {{ .Values.uploadXml.awsSecretAccessKey }} - {{- end }} + {{- end }} + redisPassword: very-secure-redis-master-password tls: caNamespace: wire-federation-v0 diff --git a/hack/helmfile.yaml b/hack/helmfile.yaml index 9c4814580e..0d34c25295 100644 --- a/hack/helmfile.yaml +++ b/hack/helmfile.yaml @@ -67,9 +67,15 @@ releases: chart: '../.local/charts/fake-aws' values: - './helm_vars/fake-aws/values.yaml' + - name: 'databases-ephemeral' namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/databases-ephemeral' + values: + - redis-ephemeral: + redis-ephemeral: + usePassword: true + password: very-secure-redis-master-password # Required for testing redis migration - name: 'redis-ephemeral-2' @@ -78,10 +84,17 @@ releases: values: - redis-ephemeral: nameOverride: redis-ephemeral-2 + usePassword: true + password: very-secure-redis-master-password-2 - name: 'databases-ephemeral' namespace: '{{ .Values.namespace2 }}' chart: '../.local/charts/databases-ephemeral' + values: + - redis-ephemeral: + redis-ephemeral: + usePassword: true + password: very-secure-redis-master-password - name: k8ssandra-test-cluster chart: '../.local/charts/k8ssandra-test-cluster' From 55fea7f9fdbf61fecf2754e93cf4f8e53a9fa02e Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Wed, 27 Mar 2024 11:37:45 +0100 Subject: [PATCH 09/14] changelog --- changelog.d/2-features/redis-creds | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/2-features/redis-creds diff --git a/changelog.d/2-features/redis-creds b/changelog.d/2-features/redis-creds new file mode 100644 index 0000000000..0ced29e8b4 --- /dev/null +++ b/changelog.d/2-features/redis-creds @@ -0,0 +1 @@ +Support authenticating to redis \ No newline at end of file From 896d7ee5951081d14f2538d1e33462a756722c2f Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Wed, 27 Mar 2024 14:20:36 +0100 Subject: [PATCH 10/14] gundeck/integration: Set creds correctly for redis after migration --- services/gundeck/test/integration/API.hs | 9 +++++++-- services/gundeck/test/integration/Util.hs | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/services/gundeck/test/integration/API.hs b/services/gundeck/test/integration/API.hs index 4ef4bd327b..0d9f128b4a 100644 --- a/services/gundeck/test/integration/API.hs +++ b/services/gundeck/test/integration/API.hs @@ -64,7 +64,7 @@ import System.Timeout (timeout) import Test.Tasty import Test.Tasty.HUnit import TestSetup -import Util (runRedisProxy, withSettingsOverrides) +import Util (runRedisProxy, withEnvOverrides, withSettingsOverrides) import Wire.API.Internal.Notification import Prelude qualified @@ -921,7 +921,12 @@ testRedisMigration = do map resource . decodePresence <$> (getPresence g (toByteString' uid) lookupEnv "REDIS_ADDITIONAL_WRITE_USERNAME" + password <- ("REDIS_PASSWORD",) <$$> lookupEnv "REDIS_ADDITIONAL_WRITE_PASSWORD" + pure $ catMaybes [username, password] + + withEnvOverrides redis2CredsAsRedis1Creds $ withSettingsOverrides (redis .~ redis2) $ do g <- view tsGundeck retrievedPresence <- map resource . decodePresence <$> (getPresence g (toByteString' uid) [(String, String)] -> m a -> m a +withEnvOverrides envOverrides action = do + bracket (setEnvVars envOverrides) (resetEnvVars) $ const action + where + setEnvVars :: [(String, String)] -> m [(String, Maybe String)] + setEnvVars newVars = liftIO $ do + oldVars <- mapM (\(k, _) -> (k,) <$> lookupEnv k) newVars + mapM_ (uncurry setEnv) newVars + pure oldVars + + resetEnvVars :: [(String, Maybe String)] -> m () + resetEnvVars = + mapM_ (\(k, mV) -> maybe (unsetEnv k) (setEnv k) mV) + runRedisProxy :: Text -> Word16 -> Word16 -> IO () runRedisProxy redisHost redisPort proxyPort = do (servAddr : _) <- getAddrInfo Nothing (Just $ Text.unpack redisHost) (Just $ show redisPort) From bc810955b217af022dc3f689f6daf41386d1fd34 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Wed, 27 Mar 2024 15:20:30 +0100 Subject: [PATCH 11/14] docs/config-options: Wrap text for ES basic auth section --- docs/src/developer/reference/config-options.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index 857dee0080..a4c9672d68 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -856,7 +856,11 @@ The corresponding Cassandra options are described in Cassandra's documentation: ## Configure Elasticsearch basic authentication -When the Wire backend is configured to work against a custom Elasticsearch instance, it may be desired to enable basic authentication for the internal communication between the Wire backend and the ES instance. To do so the Elasticsearch credentials can be set in wire-server's secrets for `brig` and `elasticsearch-index` as follows: +When the Wire backend is configured to work against a custom Elasticsearch +instance, it may be desired to enable basic authentication for the internal +communication between the Wire backend and the ES instance. To do so the +Elasticsearch credentials can be set in wire-server's secrets for `brig` and +`elasticsearch-index` as follows: ```yaml brig: @@ -872,7 +876,9 @@ elasticsearch-index: password: changeme ``` -In some cases an additional Elasticsearch instance is needed (e.g. for index migrations). To configure credentials for the additional ES instance add the secret as follows: +In some cases an additional Elasticsearch instance is needed (e.g. for index +migrations). To configure credentials for the additional ES instance add the +secret as follows: ```yaml brig: From 5655765b0e6fc90fff426d28fa2f227523e57ec3 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Wed, 27 Mar 2024 15:29:40 +0100 Subject: [PATCH 12/14] docs/config-options: Document settings creds for redis --- .../src/developer/reference/config-options.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index a4c9672d68..1b55a856b2 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -887,3 +887,33 @@ brig: username: elastic password: changeme ``` + +## Configure Redis authentication + +If the redis used needs authentication with either username and password or just +password (legacy auth), it can be configured like this: + +```yaml +gundeck: + secrets: + redisUsername: + redisPassword: +``` + +**NOTE**: When using redis < 6, the `redisUsername` must not be set at all (not +even set to `null` or empty string, they key must be absent from the config). +When using redis >= 6 and using legacy auth, the `redisUsername` must either be +not set at all or set to `"default"`. + +While doing migrations to another redis instance, the credentials for the +addtional redis can be set as follows: + +```yaml +gundeck: + secrets: + redisAdditionalWriteUsername: # Do not set this at all when using legacy auth + redisAdditionalWritePassword: +``` + +**NOTE**: `redisAddtiionalWriteUsername` follows same restrictions as +`redisUsername` when using legacy auth. From 70a3544c523938d0b09d70bc7b6f8daed091cf64 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Wed, 27 Mar 2024 16:09:47 +0100 Subject: [PATCH 13/14] charts/redis: Allow not having AWS creds --- charts/gundeck/templates/secret.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/gundeck/templates/secret.yaml b/charts/gundeck/templates/secret.yaml index f20aeaea5a..eae9c4ab33 100644 --- a/charts/gundeck/templates/secret.yaml +++ b/charts/gundeck/templates/secret.yaml @@ -11,8 +11,12 @@ metadata: type: Opaque data: {{- with .Values.secrets }} + {{- if hasKey . "awsKeyId" }} awsKeyId: {{ .awsKeyId | b64enc | quote }} + {{- end }} + {{- if hasKey . "awsSecretKey" }} awsSecretKey: {{ .awsSecretKey | b64enc | quote }} + {{- end }} {{- if hasKey . "redisUsername" }} redisUsername: {{ .redisUsername | b64enc | quote }} {{- end }} From 249a745f2f62edb592cf6dd4a47408beca4c983b Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Wed, 27 Mar 2024 16:13:08 +0100 Subject: [PATCH 14/14] docs: Fix typo --- docs/src/developer/reference/config-options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index 1b55a856b2..d45c805dc6 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -901,7 +901,7 @@ gundeck: ``` **NOTE**: When using redis < 6, the `redisUsername` must not be set at all (not -even set to `null` or empty string, they key must be absent from the config). +even set to `null` or empty string, the key must be absent from the config). When using redis >= 6 and using legacy auth, the `redisUsername` must either be not set at all or set to `"default"`.